diff options
Diffstat (limited to 'tools')
1282 files changed, 177664 insertions, 0 deletions
diff --git a/tools/bloatview/bloatdiff.pl b/tools/bloatview/bloatdiff.pl new file mode 100755 index 0000000000..a9bfa97226 --- /dev/null +++ b/tools/bloatview/bloatdiff.pl @@ -0,0 +1,372 @@ +#!/usr/bin/perl -w +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +################################################################################ + +sub usage() { + print <<EOUSAGE; +# bloatdiff.pl - munges the output from +# XPCOM_MEM_BLOAT_LOG=1 +# firefox -P default resource:///res/bloatcycle.html +# so that it does some summary and stats stuff. +# +# To show leak test results for a set of changes, do something like this: +# +# XPCOM_MEM_BLOAT_LOG=1 +# firefox -P default resource:///res/bloatcycle.html > a.out +# **make change** +# firefox -P default resource:///res/bloatcycle.html > b.out +# bloatdiff.pl a.out b.out + +EOUSAGE +} + +$OLDFILE = $ARGV[0]; +$NEWFILE = $ARGV[1]; +#$LABEL = $ARGV[2]; + +if (!$OLDFILE or + ! -e $OLDFILE or + -z $OLDFILE) { + print "\nError: Previous log file not specified, does not exist, or is empty.\n\n"; + &usage(); + exit 1; +} + +if (!$NEWFILE or + ! -e $NEWFILE or + -z $NEWFILE) { + print "\nError: Current log file not specified, does not exist, or is empty.\n\n"; + &usage(); + exit 1; +} + +sub processFile { + my ($filename, $map, $prevMap) = @_; + open(FH, $filename); + while (<FH>) { + if (m{ + ^\s*(\d+)\s # Line number + ([\w:]+)\s+ # Name + (-?\d+)\s+ # Size + (-?\d+)\s+ # Leaked + (-?\d+)\s+ # Objects Total + (-?\d+)\s+ # Objects Rem + \(\s*(-?[\d.]+)\s+ # Objects Mean + \+/-\s+ + ([\w.]+)\)\s+ # Objects StdDev + (-?\d+)\s+ # Reference Total + (-?\d+)\s+ # Reference Rem + \(\s*(-?[\d.]+)\s+ # Reference Mean + \+/-\s+ + ([\w\.]+)\) # Reference StdDev + }x) { + $$map{$2} = { name => $2, + size => $3, + leaked => $4, + objTotal => $5, + objRem => $6, + objMean => $7, + objStdDev => $8, + refTotal => $9, + refRem => $10, + refMean => $11, + refStdDev => $12, + bloat => $3 * $5 # size * objTotal + }; + } else { +# print "failed to parse: $_\n"; + } + } + close(FH); +} + +%oldMap = (); +processFile($OLDFILE, \%oldMap); + +%newMap = (); +processFile($NEWFILE, \%newMap); + +################################################################################ + +$inf = 9999999.99; + +sub getLeaksDelta { + my ($key) = @_; + my $oldLeaks = $oldMap{$key}{leaked} || 0; + my $newLeaks = $newMap{$key}{leaked}; + my $percentLeaks = 0; + if ($oldLeaks == 0) { + if ($newLeaks != 0) { + # there weren't any leaks before, but now there are! + $percentLeaks = $inf; + } + } + else { + $percentLeaks = ($newLeaks - $oldLeaks) / $oldLeaks * 100; + } + # else we had no record of this class before + return ($newLeaks - $oldLeaks, $percentLeaks); +} + +################################################################################ + +sub getBloatDelta { + my ($key) = @_; + my $newBloat = $newMap{$key}{bloat}; + my $percentBloat = 0; + my $oldSize = $oldMap{$key}{size} || 0; + my $oldTotal = $oldMap{$key}{objTotal} || 0; + my $oldBloat = $oldTotal * $oldSize; + if ($oldBloat == 0) { + if ($newBloat != 0) { + # this class wasn't used before, but now it is + $percentBloat = $inf; + } + } + else { + $percentBloat = ($newBloat - $oldBloat) / $oldBloat * 100; + } + # else we had no record of this class before + return ($newBloat - $oldBloat, $percentBloat); +} + +################################################################################ + +foreach $key (keys %newMap) { + my ($newLeaks, $percentLeaks) = getLeaksDelta($key); + my ($newBloat, $percentBloat) = getBloatDelta($key); + $newMap{$key}{leakDelta} = $newLeaks; + $newMap{$key}{leakPercent} = $percentLeaks; + $newMap{$key}{bloatDelta} = $newBloat; + $newMap{$key}{bloatPercent} = $percentBloat; +} + +################################################################################ + +# Print a value of bytes out in a reasonable +# KB, MB, or GB form. Copied from build-seamonkey-util.pl, sorry. -mcafee +sub PrintSize($) { + + # print a number with 3 significant figures + sub PrintNum($) { + my ($num) = @_; + my $rv; + if ($num < 1) { + $rv = sprintf "%.3f", ($num); + } elsif ($num < 10) { + $rv = sprintf "%.2f", ($num); + } elsif ($num < 100) { + $rv = sprintf "%.1f", ($num); + } else { + $rv = sprintf "%d", ($num); + } + } + + my ($size) = @_; + my $rv; + if ($size > 1000000000) { + $rv = PrintNum($size / 1000000000.0) . "G"; + } elsif ($size > 1000000) { + $rv = PrintNum($size / 1000000.0) . "M"; + } elsif ($size > 1000) { + $rv = PrintNum($size / 1000.0) . "K"; + } else { + $rv = PrintNum($size); + } +} + + +print "Bloat/Leak Delta Report\n"; +print "--------------------------------------------------------------------------------------\n"; +print "Current file: $NEWFILE\n"; +print "Previous file: $OLDFILE\n"; +print "----------------------------------------------leaks------leaks%------bloat------bloat%\n"; + + if (! $newMap{"TOTAL"} or + ! $newMap{"TOTAL"}{bloat}) { + # It's OK if leaked or leakPercent are 0 (in fact, that would be good). + # If bloatPercent is zero, it is also OK, because we may have just had + # two runs exactly the same or with no new bloat. + print "\nError: unable to calculate bloat/leak data.\n"; + print "There is no data present.\n\n"; + print "HINT - Did your test run complete successfully?\n"; + print "HINT - Are you pointing at the right log files?\n\n"; + &usage(); + exit 1; + } + +printf "%-40s %10s %10.2f%% %10s %10.2f%%\n", + ("TOTAL", + $newMap{"TOTAL"}{leaked}, $newMap{"TOTAL"}{leakPercent}, + $newMap{"TOTAL"}{bloat}, $newMap{"TOTAL"}{bloatPercent}); + +################################################################################ + +sub percentStr { + my ($p) = @_; + if ($p == $inf) { + return "-"; + } + else { + return sprintf "%10.2f%%", $p; + } +} + +# NEW LEAKS +@keys = sort { $newMap{$b}{leakPercent} <=> $newMap{$a}{leakPercent} } keys %newMap; +my $needsHeading = 1; +my $total = 0; +foreach $key (@keys) { + my $percentLeaks = $newMap{$key}{leakPercent}; + my $leaks = $newMap{$key}{leaked}; + if ($percentLeaks > 0 && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--NEW-LEAKS-----------------------------------leaks------leaks%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, percentStr($percentLeaks)); + $total += $leaks; + } +} +if (!$needsHeading) { + printf "%-40s %10s\n", ("TOTAL", $total); +} + +# FIXED LEAKS +@keys = sort { $newMap{$b}{leakPercent} <=> $newMap{$a}{leakPercent} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $percentLeaks = $newMap{$key}{leakPercent}; + my $leaks = $newMap{$key}{leaked}; + if ($percentLeaks < 0 && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--FIXED-LEAKS---------------------------------leaks------leaks%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, percentStr($percentLeaks)); + $total += $leaks; + } +} +if (!$needsHeading) { + printf "%-40s %10s\n", ("TOTAL", $total); +} + +# NEW BLOAT +@keys = sort { $newMap{$b}{bloatPercent} <=> $newMap{$a}{bloatPercent} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $percentBloat = $newMap{$key}{bloatPercent}; + my $bloat = $newMap{$key}{bloat}; + if ($percentBloat > 0 && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--NEW-BLOAT-----------------------------------bloat------bloat%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $bloat, percentStr($percentBloat)); + $total += $bloat; + } +} +if (!$needsHeading) { + printf "%-40s %10s\n", ("TOTAL", $total); +} + +# ALL LEAKS +@keys = sort { $newMap{$b}{leaked} <=> $newMap{$a}{leaked} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $leaks = $newMap{$key}{leaked}; + my $percentLeaks = $newMap{$key}{leakPercent}; + if ($leaks > 0) { + if ($needsHeading) { + printf "--ALL-LEAKS-----------------------------------leaks------leaks%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, percentStr($percentLeaks)); + if ($key !~ /TOTAL/) { + $total += $leaks; + } + } +} +if (!$needsHeading) { +# printf "%-40s %10s\n", ("TOTAL", $total); +} + +# ALL BLOAT +@keys = sort { $newMap{$b}{bloat} <=> $newMap{$a}{bloat} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $bloat = $newMap{$key}{bloat}; + my $percentBloat = $newMap{$key}{bloatPercent}; + if ($bloat > 0) { + if ($needsHeading) { + printf "--ALL-BLOAT-----------------------------------bloat------bloat%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $bloat, percentStr($percentBloat)); + if ($key !~ /TOTAL/) { + $total += $bloat; + } + } +} +if (!$needsHeading) { +# printf "%-40s %10s\n", ("TOTAL", $total); +} + +# NEW CLASSES +@keys = sort { $newMap{$b}{bloatDelta} <=> $newMap{$a}{bloatDelta} } keys %newMap; +$needsHeading = 1; +my $ltotal = 0; +my $btotal = 0; +foreach $key (@keys) { + my $leaks = $newMap{$key}{leaked}; + my $bloat = $newMap{$key}{bloat}; + my $percentBloat = $newMap{$key}{bloatPercent}; + if ($percentBloat == $inf && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--CLASSES-NOT-REPORTED-LAST-TIME--------------leaks------bloat------------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, $bloat); + if ($key !~ /TOTAL/) { + $ltotal += $leaks; + $btotal += $bloat; + } + } +} +if (!$needsHeading) { + printf "%-40s %10s %10s\n", ("TOTAL", $ltotal, $btotal); +} + +# OLD CLASSES +@keys = sort { ($oldMap{$b}{bloat} || 0) <=> ($oldMap{$a}{bloat} || 0) } keys %oldMap; +$needsHeading = 1; +$ltotal = 0; +$btotal = 0; +foreach $key (@keys) { + if (!defined($newMap{$key})) { + my $leaks = $oldMap{$key}{leaked}; + my $bloat = $oldMap{$key}{bloat}; + if ($needsHeading) { + printf "--CLASSES-THAT-WENT-AWAY----------------------leaks------bloat------------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, $bloat); + if ($key !~ /TOTAL/) { + $ltotal += $leaks; + $btotal += $bloat; + } + } +} +if (!$needsHeading) { + printf "%-40s %10s %10s\n", ("TOTAL", $ltotal, $btotal); +} + +print "--------------------------------------------------------------------------------------\n"; diff --git a/tools/bloatview/bloattable.pl b/tools/bloatview/bloattable.pl new file mode 100755 index 0000000000..e8acfabed3 --- /dev/null +++ b/tools/bloatview/bloattable.pl @@ -0,0 +1,590 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# bloattable [-debug] [-source] [-byte n|-obj n|-ref n] <file1> <file2> ... <filen> > <html-file> +# +# file1, file2, ... filen should be successive BloatView files generated from the same run. +# Summarize them in an HTML table. Output the HTML to the standard output. +# +# If -debug is set, create a slightly larger html file which is more suitable for debugging this script. +# If -source is set, create an html file that prints the html source as the output +# If -byte n, -obj n, or -ref n is given, make the page default to showing byte, object, or reference statistics, +# respectively, and sort by the nth column (n is zero-based, so the first column has n==0). +# +# See http://lxr.mozilla.org/mozilla/source/xpcom/doc/MemoryTools.html + +use 5.004; +use strict; +use diagnostics; +use File::Basename; +use Getopt::Long; + +# The generated HTML is almost entirely generated by a script. Only the <HTML>, <HEAD>, and <BODY> elements are explicit +# because a <SCRIPT> element cannot officially be a direct descendant of an <HTML> element. +# The script itself is almost all generated by an eval of a large string. This allows the script to reproduce itself +# when making a new page using document.write's. Re-sorting the page causes it to regenerate itself in this way. + + + +# Return the file's modification date. +sub fileModDate($) { + my ($pathName) = @_; + my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = + stat $pathName or die "Can't stat '$pathName'"; + return $mtime; +} + + +sub fileCoreName($) { + my ($pathName) = @_; + my $fileName = basename($pathName, ""); + $fileName =~ s/\..*//; + return $fileName; +} + + +# Convert a raw string into a single-quoted JavaScript string. +sub singleQuoteString($) { + local ($_) = @_; + s/\\/\\\\/g; + s/'/\\'/g; + s/\n/\\n/g; + s/<\//<\\\//g; + return "'$_'"; +} + + +# Convert a raw string into a double-quoted JavaScript string. +sub doubleQuoteString($) { + local ($_) = @_; + s/\\/\\\\/g; + s/"/\\"/g; + s/\n/\\n/g; + s/<\//<\\\//g; + return "\"$_\""; +} + + +# Quote special HTML characters in the string. +sub quoteHTML($) { + local ($_) = @_; + s/\&/&/g; + s/</</g; + s/>/>/g; + s/ / /g; + s/\n/<BR>\n/g; + return $_; +} + + +# Write the generated page to the standard output. +# The script source code is read from this file past the __END__ marker +# @$scriptData is the JavaScript source for the tables passed to JavaScript. Each entry is one line of JavaScript. +# @$persistentScriptData is the same as @scriptData, but persists when the page reloads itself. +# If $debug is true, generate the script directly instead of having it eval itself. +# If $source is true, generate a script that displays the page's source instead of the page itself. +sub generate(\@\@$$$$) { + my ($scriptData, $persistentScriptData, $debug, $source, $showMode, $sortColumn) = @_; + + my @scriptSource = <DATA>; + chomp @scriptSource; + print <<'EOS'; +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd"> +<HTML> +<HEAD> +<SCRIPT type="text/javascript"> +EOS + + foreach (@$scriptData) {print "$_\n";} + print "\n"; + + print "var srcArray = [\n"; + my @quotedScriptSource = map { + my $line = $_; + $line =~ s/^\s+//g; + # $line =~ s/^\/\/SOURCE\s+//g if $source; + $line =~ s/^\/\/.*//g; + $line =~ s/\s+$//g; + $line eq "" ? () : $line + } @$persistentScriptData, @scriptSource; + my $lastQuotedLine = pop @quotedScriptSource; + foreach (@quotedScriptSource) {print doubleQuoteString($_), ",\n";} + print doubleQuoteString($lastQuotedLine), "];\n\n"; + + if ($debug) { + push @quotedScriptSource, $lastQuotedLine; + foreach (@quotedScriptSource) { + s/<\//<\\\//g; # This fails if a regexp ends with a '<'. Oh well.... + print "$_\n"; + } + print "\n"; + } else { + print "eval(srcArray.join(\"\\n\"));\n\n"; + } + print "showMode = $showMode;\n"; + print "sortColumn = $sortColumn;\n"; + if ($source) { + print <<'EOS'; +function writeQuotedHTML(s) { + document.write(quoteHTML(s.toString()).replace(/\n/g, '<BR>\n')); +} + +var quotingDocument = { + write: function () { + for (var i = 0; i < arguments.length; i++) + writeQuotedHTML(arguments[i]); + }, + writeln: function () { + for (var i = 0; i < arguments.length; i++) + writeQuotedHTML(arguments[i]); + document.writeln('<BR>'); + } +}; +EOS + } else { + print "showHead(document);\n"; + } + print "</SCRIPT>\n"; + print "</HEAD>\n\n"; + print "<BODY>\n"; + if ($source) { + print "<P><TT>"; + print quoteHTML "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 Transitional//EN\" \"http://www.w3.org/TR/REC-html40/loose.dtd\">\n"; + print quoteHTML "<HTML>\n"; + print quoteHTML "<HEAD>\n"; + print "<SCRIPT type=\"text/javascript\">showHead(quotingDocument);</SCRIPT>\n"; + print quoteHTML "</HEAD>\n\n"; + print quoteHTML "<BODY>\n"; + print "<SCRIPT type=\"text/javascript\">showBody(quotingDocument);</SCRIPT>\n"; + print quoteHTML "</BODY>\n"; + print quoteHTML "</HTML>\n"; + print "</TT></P>\n"; + } else { + print "<SCRIPT type=\"text/javascript\">showBody(document);</SCRIPT>\n"; + } + print "</BODY>\n"; + print "</HTML>\n"; +} + + + +# Read the bloat file into hash table $h. The hash table is indexed by class names; +# each entry is a list with the following elements: +# bytesAlloc Total number of bytes allocated +# bytesNet Total number of bytes allocated but not deallocated +# objectsAlloc Total number of objects allocated +# objectsNet Total number of objects allocated but not deallocated +# refsAlloc Total number of references AddRef'd +# refsNet Total number of references AddRef'd but not Released +# Except for TOTAL, all hash table entries refer to mutually exclusive data. +# $sizes is a hash table indexed by class names. Each entry of that table contains the class's instance size. +sub readBloatFile($\%\%) { + my ($file, $h, $sizes) = @_; + local $_; # Needed for 'while (<FILE>)' below. + + my $readSomething = 0; + open FILE, $file; + while (<FILE>) { + if (my ($name, $size, $bytesNet, $objectsAlloc, $objectsNet, $refsAlloc, $refsNet) = + /^\s*(?:\d+)\s+([\w:]+)\s+(\d+)\s+(-?\d+)\s+(\d+)\s+(-?\d+)\s*\([^()]*\)\s*(\d+)\s+(-?\d+)\s*\([^()]*\)\s*$/) { + my $bytesAlloc; + if ($name eq "TOTAL") { + $size = "undefined"; + $bytesAlloc = "undefined"; + } else { + $bytesAlloc = $objectsAlloc * $size; + if ($bytesNet != $objectsNet * $size) { + print STDERR "In '$file', class $name bytesNet != objectsNet * size: $bytesNet != $objectsNet * $size\n"; + } + } + print STDERR "Duplicate entry $name in '$file'\n" if $$h{$name}; + $$h{$name} = [$bytesAlloc, $bytesNet, $objectsAlloc, $objectsNet, $refsAlloc, $refsNet]; + + my $oldSize = $$sizes{$name}; + print STDERR "Mismatch of sizes of class $name: $oldSize and $size\n" if defined($oldSize) && $size ne $oldSize; + $$sizes{$name} = $size; + $readSomething = 1; + } elsif (/^\s*(?:\d+)\s+([\w:]+)\s/) { + print STDERR "Unable to parse '$file' line: $_"; + } + } + close FILE; + print STDERR "No data in '$file'\n" unless $readSomething; + return $h; +} + + +my %sizes; # <class-name> => <instance-size> +my %tables; # <file-name> => <bloat-table>; see readBloatFile for format of <bloat-table> + +# Generate the JavaScript source code for the row named $c. $l can contain the initial entries of the row. +sub genTableRowSource($$) { + my ($l, $c) = @_; + my $lastE; + foreach (@ARGV) { + my $e = $tables{$_}{$c}; + if (defined($lastE) && !defined($e)) { + $e = [0,0,0,0,0,0]; + print STDERR "Class $c is defined in an earlier file but not in '$_'\n"; + } + if (defined $e) { + if (defined $lastE) { + for (my $i = 0; $i <= $#$e; $i++) { + my $n = $$e[$i]; + $l .= ($n eq "undefined" ? "undefined" : $n - $$lastE[$i]) . ","; + } + $l .= " "; + } else { + $l .= join(",", @$e) . ", "; + } + $lastE = $e; + } else { + $l .= "0,0,0,0,0,0, "; + } + } + $l .= join(",", @$lastE); + return "[$l]"; +} + + + +my $debug; +my $source; +my $showMode; +my $sortColumn; +my @modeOptions; + +GetOptions("debug" => \$debug, "source" => \$source, "byte=i" => \$modeOptions[0], "obj=i" => \$modeOptions[1], "ref=i" => \$modeOptions[2]); +for (my $i = 0; $i != 3; $i++) { + my $modeOption = $modeOptions[$i]; + if ($modeOption) { + die "Only one of -byte, -obj, or -ref may be given" if defined $showMode; + my $nFileColumns = scalar(@ARGV) + 1; + die "-byte, -obj, or -ref column number out of range" if $modeOption < 0 || $modeOption >= 2 + 2*$nFileColumns; + $showMode = $i; + if ($modeOption >= 2) { + $modeOption -= 2; + $sortColumn = 2 + $showMode*2; + if ($modeOption >= $nFileColumns) { + $sortColumn++; + $modeOption -= $nFileColumns; + } + $sortColumn += $modeOption*6; + } else { + $sortColumn = $modeOption; + } + } +} +unless (defined $showMode) { + $showMode = 0; + $sortColumn = 0; +} + +# Read all of the bloat files. +foreach (@ARGV) { + unless ($tables{$_}) { + my $f = $_; + my %table; + + readBloatFile $_, %table, %sizes; + $tables{$_} = \%table; + } +} +die "No input" unless %sizes; + +my @scriptData; # JavaScript source for the tables passed to JavaScript. Each entry is one line of JavaScript. +my @persistentScriptData; # Same as @scriptData, but persists the page reloads itself. + +# Print a list of bloat file names. +push @persistentScriptData, "var nFiles = " . scalar(@ARGV) . ";"; +push @persistentScriptData, "var fileTags = [" . join(", ", map {singleQuoteString substr(fileCoreName($_), -10)} @ARGV) . "];"; +push @persistentScriptData, "var fileNames = [" . join(", ", map {singleQuoteString $_} @ARGV) . "];"; +push @persistentScriptData, "var fileDates = [" . join(", ", map {singleQuoteString localtime fileModDate $_} @ARGV) . "];"; + +# Print the bloat tables. +push @persistentScriptData, "var totals = " . genTableRowSource('"TOTAL", undefined, ', "TOTAL") . ";"; +push @scriptData, "var classTables = ["; +delete $sizes{"TOTAL"}; +my @classes = sort(keys %sizes); +for (my $i = 0; $i <= $#classes; $i++) { + my $c = $classes[$i]; + push @scriptData, genTableRowSource(doubleQuoteString($c).", ".$sizes{$c}.", ", $c) . ($i == $#classes ? "];" : ","); +} + +generate(@scriptData, @persistentScriptData, $debug, $source, $showMode, $sortColumn); +1; + + +# The source of the eval'd JavaScript follows. +# Comments starting with // that are alone on a line are stripped by the Perl script. +__END__ + +// showMode: 0=bytes, 1=objects, 2=references +var showMode; +var modeName; +var modeNameUpper; + +var sortColumn; + +// Sort according to the sortColumn. Column 0 is sorted alphabetically in ascending order. +// All other columns are sorted numerically in descending order, with column 0 used for a secondary sort. +// Undefined is always listed last. +function sortCompare(x, y) { + if (sortColumn) { + var xc = x[sortColumn]; + var yc = y[sortColumn]; + if (xc < yc || xc === undefined && yc !== undefined) return 1; + if (yc < xc || yc === undefined && xc !== undefined) return -1; + } + + var x0 = x[0]; + var y0 = y[0]; + if (x0 > y0 || x0 === undefined && y0 !== undefined) return 1; + if (y0 > x0 || y0 === undefined && x0 !== undefined) return -1; + return 0; +} + + +// Quote special HTML characters in the string. +function quoteHTML(s) { + s = s.replace(/&/g, '&'); + // Can't use /</g because HTML interprets '</g' as ending the script! + s = s.replace(/\x3C/g, '<'); + s = s.replace(/>/g, '>'); + s = s.replace(/ /g, ' '); + return s; +} + + +function writeFileTable(d) { + d.writeln('<TABLE border=1 cellspacing=1 cellpadding=0>'); + d.writeln('<TR>\n<TH>Name</TH>\n<TH>File</TH>\n<TH>Date</TH>\n</TR>'); + for (var i = 0; i < nFiles; i++) + d.writeln('<TR>\n<TD>'+quoteHTML(fileTags[i])+'</TD>\n<TD><TT>'+quoteHTML(fileNames[i])+'</TT></TD>\n<TD>'+quoteHTML(fileDates[i])+'</TD>\n</TR>'); + d.writeln('</TABLE>'); +} + + +function writeReloadLink(d, column, s, rowspan) { + d.write(rowspan == 1 ? '<TH>' : '<TH rowspan='+rowspan+'>'); + if (column != sortColumn) + d.write('<A href="javascript:reloadSelf('+column+','+showMode+')">'); + d.write(s); + if (column != sortColumn) + d.write('</A>'); + d.writeln('</TH>'); +} + +function writeClassTableRow(d, row, base, modeName) { + if (modeName) { + d.writeln('<TR>\n<TH>'+modeName+'</TH>'); + } else { + d.writeln('<TR>\n<TD><A href="javascript:showRowDetail(\''+row[0]+'\')">'+quoteHTML(row[0])+'</A></TD>'); + var v = row[1]; + d.writeln('<TD class=num>'+(v === undefined ? '' : v)+'</TD>'); + } + for (var i = 0; i != 2; i++) { + var c = base + i; + for (var j = 0; j <= nFiles; j++) { + v = row[c]; + var style = 'num'; + if (j != nFiles) + if (v > 0) { + style = 'pos'; + v = '+'+v; + } else + style = 'neg'; + d.writeln('<TD class='+style+'>'+(v === undefined ? '' : v)+'</TD>'); + c += 6; + } + } + d.writeln('</TR>'); +} + +function writeClassTable(d) { + var base = 2 + showMode*2; + + // Make a copy because a sort is destructive. + var table = classTables.concat(); + table.sort(sortCompare); + + d.writeln('<TABLE border=1 cellspacing=1 cellpadding=0>'); + + d.writeln('<TR>'); + writeReloadLink(d, 0, 'Class Name', 2); + writeReloadLink(d, 1, 'Instance<BR>Size', 2); + d.writeln('<TH colspan='+(nFiles+1)+'>'+modeNameUpper+'s allocated</TH>'); + d.writeln('<TH colspan='+(nFiles+1)+'>'+modeNameUpper+'s allocated but not freed</TH>\n</TR>'); + d.writeln('<TR>'); + for (var i = 0; i != 2; i++) { + var c = base + i; + for (var j = 0; j <= nFiles; j++) { + writeReloadLink(d, c, j == nFiles ? 'Total' : quoteHTML(fileTags[j]), 1); + c += 6; + } + } + d.writeln('</TR>'); + + writeClassTableRow(d, totals, base, 0); + for (var r = 0; r < table.length; r++) + writeClassTableRow(d, table[r], base, 0); + + d.writeln('</TABLE>'); +} + + +var modeNames = ["byte", "object", "reference"]; +var modeNamesUpper = ["Byte", "Object", "Reference"]; +var styleSheet = '<STYLE type="TEXT/CSS">\n'+ + 'BODY {background-color: #FFFFFF; color: #000000}\n'+ + '.num {text-align: right}\n'+ + '.pos {text-align: right; color: #CC0000}\n'+ + '.neg {text-align: right; color: #009900}\n'+ + '</STYLE>'; + + +function showHead(d) { + modeName = modeNames[showMode]; + modeNameUpper = modeNamesUpper[showMode]; + d.writeln('<TITLE>'+modeNameUpper+' Bloats</TITLE>'); + d.writeln(styleSheet); +} + +function showBody(d) { + d.writeln('<H1>'+modeNameUpper+' Bloats</H1>'); + writeFileTable(d); + d.write('<FORM>'); + for (var i = 0; i != 3; i++) + if (i != showMode) { + var newSortColumn = sortColumn; + if (sortColumn >= 2) + newSortColumn = sortColumn + (i-showMode)*2; + d.write('<INPUT type="button" value="Show '+modeNamesUpper[i]+'s" onClick="reloadSelf('+newSortColumn+','+i+')">'); + } + d.writeln('</FORM>'); + d.writeln('<P>The numbers do not include <CODE>malloc</CODE>\'d data such as string contents.</P>'); + d.writeln('<P>Click on a column heading to sort by that column. Click on a class name to see details for that class.</P>'); + writeClassTable(d); +} + + +function showRowDetail(rowName) { + var row; + var i; + + if (rowName == "TOTAL") + row = totals; + else { + for (i = 0; i < classTables.length; i++) + if (rowName == classTables[i][0]) { + row = classTables[i]; + break; + } + } + if (row) { + var w = window.open("", "ClassTableRowDetails"); + var d = w.document; + d.open(); + d.writeln('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">'); + d.writeln('<HTML>\n<HEAD>\n<TITLE>'+quoteHTML(rowName)+' bloat details</TITLE>'); + d.writeln(styleSheet); + d.writeln('</HEAD>\n\n<BODY>'); + d.writeln('<H2>'+quoteHTML(rowName)+'</H2>'); + if (row[1] !== undefined) + d.writeln('<P>Each instance has '+row[1]+' bytes.</P>'); + + d.writeln('<TABLE border=1 cellspacing=1 cellpadding=0>'); + d.writeln('<TR>\n<TH></TH>\n<TH colspan='+(nFiles+1)+'>Allocated</TH>'); + d.writeln('<TH colspan='+(nFiles+1)+'>Allocated but not freed</TH>\n</TR>'); + d.writeln('<TR>\n<TH></TH>'); + for (i = 0; i != 2; i++) + for (var j = 0; j <= nFiles; j++) + d.writeln('<TH>'+(j == nFiles ? 'Total' : quoteHTML(fileTags[j]))+'</TH>'); + d.writeln('</TR>'); + + for (i = 0; i != 3; i++) + writeClassTableRow(d, row, 2+i*2, modeNamesUpper[i]+'s'); + + d.writeln('</TABLE>\n</BODY>\n</HTML>'); + d.close(); + } + return undefined; +} + + +function stringSource(s) { + s = s.replace(/\\/g, '\\\\'); + s = s.replace(/"/g, '\\"'); + s = s.replace(/<\//g, '<\\/'); + return '"'+s+'"'; +} + +function reloadSelf(n,m) { + // Need to cache these because globals go away on document.open(). + var sa = srcArray; + var ss = stringSource; + var ct = classTables; + var i; + + document.open(); + // Uncomment this and comment the document.open() line above to see the reloaded page's source. + //var w = window.open("", "NewDoc"); + //var d = w.document; + //var document = new Object; + //document.write = function () { + // for (var i = 0; i < arguments.length; i++) { + // var s = arguments[i].toString(); + // s = s.replace(/&/g, '&'); + // s = s.replace(/\x3C/g, '<'); + // s = s.replace(/>/g, '>'); + // s = s.replace(/ /g, ' '); + // d.write(s); + // } + //}; + //document.writeln = function () { + // for (var i = 0; i < arguments.length; i++) { + // var s = arguments[i].toString(); + // s = s.replace(/&/g, '&'); + // s = s.replace(/\x3C/g, '<'); + // s = s.replace(/>/g, '>'); + // s = s.replace(/ /g, ' '); + // d.write(s); + // } + // d.writeln('<BR>'); + //}; + + document.writeln('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">'); + document.writeln('<HTML>\n<HEAD>\n<SCRIPT type="text/javascript">'); + + // Manually copy non-persistent script data + if (!ct.length) + document.writeln('var classTables = [];'); + else { + document.writeln('var classTables = ['); + for (i = 0; i < ct.length; i++) { + var row = ct[i]; + document.write('[' + ss(row[0])); + for (var j = 1; j < row.length; j++) + document.write(',' + row[j]); + document.writeln(']' + (i == ct.length-1 ? '];' : ',')); + } + } + + document.writeln('var srcArray = ['); + for (i = 0; i < sa.length; i++) { + document.write(ss(sa[i])); + if (i != sa.length-1) + document.writeln(','); + } + document.writeln('];'); + document.writeln('eval(srcArray.join("\\n"));'); + document.writeln('showMode = '+m+';'); + document.writeln('sortColumn = '+n+';'); + document.writeln('showHead(document);'); + document.writeln('</SCRIPT>\n</HEAD>\n\n<BODY>\n<SCRIPT type="text/javascript">showBody(document);</SCRIPT>\n</BODY>\n</HTML>'); + document.close(); + return undefined; +} diff --git a/tools/browsertime/README.md b/tools/browsertime/README.md new file mode 100644 index 0000000000..bd76cf65be --- /dev/null +++ b/tools/browsertime/README.md @@ -0,0 +1,15 @@ +=========== +Browsertime +=========== + +This code is obtained from `https://github.com/sitespeedio/browsertime` and all +changes to the code must be made there. It's installed through `./mach browsertime --setup` +and it's used when running `./mach browsertime` or through raptor with +`./mach raptor --browsertime`. + +You can update this code by running `./mach browsertime --update-upstream-url <URL>`, where +URL is a link to a github tarball of a particular commit in the aforementioned repository. + +For more information, you can consult the wiki `https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Browsertime`. +If you have any questions about this code, or browsertime, you can find us in +`#browsertime`, or `#perftest` in Riot. diff --git a/tools/browsertime/mach_commands.py b/tools/browsertime/mach_commands.py new file mode 100644 index 0000000000..8386dd2c54 --- /dev/null +++ b/tools/browsertime/mach_commands.py @@ -0,0 +1,690 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +r"""Make it easy to install and run [browsertime](https://github.com/sitespeedio/browsertime). + +Browsertime is a harness for running performance tests, similar to +Mozilla's Raptor testing framework. Browsertime is written in Node.js +and uses Selenium WebDriver to drive multiple browsers including +Chrome, Chrome for Android, Firefox, and (pending the resolution of +[Bug 1525126](https://bugzilla.mozilla.org/show_bug.cgi?id=1525126) +and similar tickets) Firefox for Android and GeckoView-based vehicles. + +Right now a custom version of browsertime and the underlying +geckodriver binary are needed to support GeckoView-based vehicles; +this module accommodates those in-progress custom versions. + +To get started, run +``` +./mach browsertime --setup [--clobber] +``` +This will populate `tools/browsertime/node_modules`. + +To invoke browsertime, run +``` +./mach browsertime [ARGS] +``` +All arguments are passed through to browsertime. +""" + +import argparse +import collections +import contextlib +import json +import logging +import os +import platform +import re +import stat +import subprocess +import sys +import time + +import mozpack.path as mozpath +from mach.decorators import Command, CommandArgument +from mozbuild.base import BinaryNotFoundException, MachCommandBase +from mozbuild.util import mkdir +from six import StringIO + +AUTOMATION = "MOZ_AUTOMATION" in os.environ +BROWSERTIME_ROOT = os.path.dirname(__file__) + +PILLOW_VERSION = "8.4.0" # version 8.4.0 currently supports python 3.6 to 3.10 +PYSSIM_VERSION = "0.4" +SCIPY_VERSION = "1.2.3" +NUMPY_VERSION = "1.16.1" +OPENCV_VERSION = "4.5.4.60" + +py3_minor = sys.version_info.minor +if py3_minor > 7: + SCIPY_VERSION = "1.9.3" + NUMPY_VERSION = "1.23.5" + PILLOW_VERSION = "9.2.0" + OPENCV_VERSION = "4.6.0.66" + +MIN_NODE_VERSION = "16.0.0" + +IS_APPLE_SILICON = sys.platform.startswith( + "darwin" +) and platform.processor().startswith("arm") + + +@contextlib.contextmanager +def silence(): + oldout, olderr = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = StringIO(), StringIO() + yield + finally: + sys.stdout, sys.stderr = oldout, olderr + + +def node_path(command_context): + import platform + + from mozbuild.nodeutil import find_node_executable + from packaging.version import Version + + state_dir = command_context._mach_context.state_dir + cache_path = os.path.join(state_dir, "browsertime", "node-16") + + NODE_FAILURE_MSG = ( + "Could not locate a node binary that is at least version {}. ".format( + MIN_NODE_VERSION + ) + + "Please run `./mach raptor --browsertime -t amazon` to install it " + + "from the Taskcluster Toolchain artifacts." + ) + + # Check standard locations first + node_exe = find_node_executable(min_version=Version(MIN_NODE_VERSION)) + if node_exe and (node_exe[0] is not None): + return os.path.abspath(node_exe[0]) + if not os.path.exists(cache_path): + raise Exception(NODE_FAILURE_MSG) + + # Check the browsertime-specific node location next + node_name = "node" + if platform.system() == "Windows": + node_name = "node.exe" + node_exe_path = os.path.join( + state_dir, + "browsertime", + "node-16", + "node", + ) + else: + node_exe_path = os.path.join( + state_dir, + "browsertime", + "node-16", + "node", + "bin", + ) + + node_exe = os.path.join(node_exe_path, node_name) + if not os.path.exists(node_exe): + raise Exception(NODE_FAILURE_MSG) + + return os.path.abspath(node_exe) + + +def package_path(): + """The path to the `browsertime` directory. + + Override the default with the `BROWSERTIME` environment variable.""" + override = os.environ.get("BROWSERTIME", None) + if override: + return override + + return mozpath.join(BROWSERTIME_ROOT, "node_modules", "browsertime") + + +def browsertime_path(): + """The path to the `browsertime.js` script.""" + # On Windows, invoking `node_modules/.bin/browsertime{.cmd}` + # doesn't work when invoked as an argument to our specific + # binary. Since we want our version of node, invoke the + # actual script directly. + return mozpath.join(package_path(), "bin", "browsertime.js") + + +def visualmetrics_path(): + """The path to the `visualmetrics.py` script.""" + return mozpath.join(package_path(), "browsertime", "visualmetrics-portable.py") + + +def host_platform(): + is_64bits = sys.maxsize > 2**32 + + if sys.platform.startswith("win"): + if is_64bits: + return "win64" + elif sys.platform.startswith("linux"): + if is_64bits: + return "linux64" + elif sys.platform.startswith("darwin"): + return "darwin" + + raise ValueError("sys.platform is not yet supported: {}".format(sys.platform)) + + +# Map from `host_platform()` to a `fetch`-like syntax. +host_fetches = { + "darwin": { + "ffmpeg": { + "type": "static-url", + "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-macos.zip", # noqa + # An extension to `fetch` syntax. + "path": "ffmpeg-macos", + }, + }, + "linux64": { + "ffmpeg": { + "type": "static-url", + "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-i686-static.tar.xz", # noqa + # An extension to `fetch` syntax. + "path": "ffmpeg-4.4.1-i686-static", + }, + }, + "win64": { + "ffmpeg": { + "type": "static-url", + "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-full_build.zip", # noqa + # An extension to `fetch` syntax. + "path": "ffmpeg-4.4.1-full_build", + }, + }, +} + + +def artifact_cache_path(command_context): + r"""Downloaded artifacts will be kept here.""" + # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE. + return mozpath.join(command_context._mach_context.state_dir, "cache", "browsertime") + + +def state_path(command_context): + r"""Unpacked artifacts will be kept here.""" + # The convention is $MOZBUILD_STATE_PATH/$FEATURE. + return mozpath.join(command_context._mach_context.state_dir, "browsertime") + + +def setup_prerequisites(command_context): + r"""Install browsertime and visualmetrics.py prerequisites.""" + + from mozbuild.action.tooltool import unpack_file + from mozbuild.artifact_cache import ArtifactCache + + # Download the visualmetrics-portable.py requirements. + artifact_cache = ArtifactCache( + artifact_cache_path(command_context), + log=command_context.log, + skip_cache=False, + ) + + fetches = host_fetches[host_platform()] + for tool, fetch in sorted(fetches.items()): + archive = artifact_cache.fetch(fetch["url"]) + # TODO: assert type, verify sha256 (and size?). + + if fetch.get("unpack", True): + cwd = os.getcwd() + try: + mkdir(state_path(command_context)) + os.chdir(state_path(command_context)) + command_context.log( + logging.INFO, + "browsertime", + {"path": archive}, + "Unpacking temporary location {path}", + ) + + unpack_file(archive) + + # Make sure the expected path exists after extraction + path = os.path.join(state_path(command_context), fetch.get("path")) + if not os.path.exists(path): + raise Exception("Cannot find an extracted directory: %s" % path) + + try: + # Some archives provide binaries that don't have the + # executable bit set so we need to set it here + for root, dirs, files in os.walk(path): + for edir in dirs: + loc_to_change = os.path.join(root, edir) + st = os.stat(loc_to_change) + os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC) + for efile in files: + loc_to_change = os.path.join(root, efile) + st = os.stat(loc_to_change) + os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC) + except Exception as e: + raise Exception( + "Could not set executable bit in %s, error: %s" % (path, str(e)) + ) + finally: + os.chdir(cwd) + + +def setup_browsertime( + command_context, + should_clobber=False, + new_upstream_url="", + install_vismet_reqs=False, +): + r"""Install browsertime and visualmetrics.py prerequisites and the Node.js package.""" + + sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint")) + import setup_helper + + if not new_upstream_url: + setup_prerequisites(command_context) + + if new_upstream_url: + package_json_path = os.path.join(BROWSERTIME_ROOT, "package.json") + + command_context.log( + logging.INFO, + "browsertime", + { + "new_upstream_url": new_upstream_url, + "package_json_path": package_json_path, + }, + "Updating browsertime node module version in {package_json_path} " + "to {new_upstream_url}", + ) + + if not re.search("/tarball/[a-f0-9]{40}$", new_upstream_url): + raise ValueError( + "New upstream URL does not end with /tarball/[a-f0-9]{40}: '%s'" + % new_upstream_url + ) + + with open(package_json_path) as f: + existing_body = json.loads( + f.read(), object_pairs_hook=collections.OrderedDict + ) + + existing_body["devDependencies"]["browsertime"] = new_upstream_url + + updated_body = json.dumps(existing_body, indent=2) + + with open(package_json_path, "w") as f: + f.write(updated_body) + + # Install the browsertime Node.js requirements. + if not setup_helper.check_node_executables_valid(): + return 1 + + # To use a custom `geckodriver`, set + # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url) + # to an endpoint with binaries named like + # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31. + if AUTOMATION: + os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true" + os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true" + + if install_vismet_reqs: + # Hide this behind a flag so we don't install them by default in Raptor + command_context.log( + logging.INFO, "browsertime", {}, "Installing python requirements" + ) + activate_browsertime_virtualenv(command_context) + + command_context.log( + logging.INFO, + "browsertime", + {"package_json": mozpath.join(BROWSERTIME_ROOT, "package.json")}, + "Installing browsertime node module from {package_json}", + ) + + # Add the mozbuild Node binary path to the OS environment in Apple Silicon. + # During the browesertime installation, it seems installation of sitespeed.io + # sub dependencies look for a global Node rather than the mozbuild Node binary. + # Normally `--scripts-prepend-node-path` should prevent this but it seems to still + # look for a system Node in the environment. This logic ensures the same Node is used. + node_dir = os.path.dirname(node_path(command_context)) + if IS_APPLE_SILICON and node_dir not in os.environ["PATH"]: + os.environ["PATH"] += os.pathsep + node_dir + + status = setup_helper.package_setup( + BROWSERTIME_ROOT, + "browsertime", + should_update=new_upstream_url != "", + should_clobber=should_clobber, + no_optional=new_upstream_url or AUTOMATION, + ) + + if status: + return status + if new_upstream_url or AUTOMATION: + return 0 + if install_vismet_reqs: + return check(command_context) + + return 0 + + +def node(command_context, args): + r"""Invoke node (interactively) with the given arguments.""" + return command_context.run_process( + [node_path(command_context)] + args, + append_env=append_env(command_context), + pass_thru=True, # Allow user to run Node interactively. + ensure_exit_code=False, # Don't throw on non-zero exit code. + cwd=mozpath.join(command_context.topsrcdir), + ) + + +def append_env(command_context, append_path=True): + fetches = host_fetches[host_platform()] + + # Ensure that `ffmpeg` is found and added to the environment + path = os.environ.get("PATH", "").split(os.pathsep) if append_path else [] + path_to_ffmpeg = mozpath.join( + state_path(command_context), fetches["ffmpeg"]["path"] + ) + + path.insert( + 0, + path_to_ffmpeg + if host_platform().startswith("linux") + else mozpath.join(path_to_ffmpeg, "bin"), + ) # noqa + + # Ensure that bare `node` and `npm` in scripts, including post-install + # scripts, finds the binary we're invoking with. Without this, it's + # easy for compiled extensions to get mismatched versions of the Node.js + # extension API. + node_dir = os.path.dirname(node_path(command_context)) + path = [node_dir] + path + + append_env = { + "PATH": os.pathsep.join(path), + # Bug 1560193: The JS library browsertime uses to execute commands + # (execa) will muck up the PATH variable and put the directory that + # node is in first in path. If this is globally-installed node, + # that means `/usr/bin` will be inserted first which means that we + # will get `/usr/bin/python` for `python`. + # + # Our fork of browsertime supports a `PYTHON` environment variable + # that points to the exact python executable to use. + "PYTHON": command_context.virtualenv_manager.python_path, + } + + return append_env + + +def _need_install(command_context, package): + from pip._internal.req.constructors import install_req_from_line + + req = install_req_from_line(package) + req.check_if_exists(use_user_site=False) + if req.satisfied_by is None: + return True + venv_site_lib = os.path.abspath( + os.path.join(command_context.virtualenv_manager.bin_path, "..", "lib") + ) + site_packages = os.path.abspath(req.satisfied_by.location) + return not site_packages.startswith(venv_site_lib) + + +def activate_browsertime_virtualenv(command_context, *args, **kwargs): + r"""Activates virtualenv. + + This function will also install Pillow and pyssim if needed. + It will raise an error in case the install failed. + """ + # TODO: Remove `./mach browsertime` completely, as a follow up to Bug 1758990 + MachCommandBase.activate_virtualenv(command_context, *args, **kwargs) + + # installing Python deps on the fly + for dep in ( + "Pillow==%s" % PILLOW_VERSION, + "pyssim==%s" % PYSSIM_VERSION, + "scipy==%s" % SCIPY_VERSION, + "numpy==%s" % NUMPY_VERSION, + "opencv-python==%s" % OPENCV_VERSION, + ): + if _need_install(command_context, dep): + subprocess.check_call( + [ + command_context.virtualenv_manager.python_path, + "-m", + "pip", + "install", + dep, + ] + ) + + +def check(command_context): + r"""Run `visualmetrics.py --check`.""" + command_context.activate_virtualenv() + + args = ["--check"] + status = command_context.run_process( + [command_context.virtualenv_manager.python_path, visualmetrics_path()] + args, + # For --check, don't allow user's path to interfere with path testing except on Linux + append_env=append_env( + command_context, append_path=host_platform().startswith("linux") + ), + pass_thru=True, + ensure_exit_code=False, # Don't throw on non-zero exit code. + cwd=mozpath.join(command_context.topsrcdir), + ) + + sys.stdout.flush() + sys.stderr.flush() + + if status: + return status + + # Avoid logging the command (and, on Windows, the environment). + command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) + print("browsertime version:", end=" ") + + sys.stdout.flush() + sys.stderr.flush() + + return node(command_context, [browsertime_path()] + ["--version"]) + + +def extra_default_args(command_context, args=[]): + # Add Mozilla-specific default arguments. This is tricky because browsertime is quite + # loose about arguments; repeat arguments are generally accepted but then produce + # difficult to interpret type errors. + + def extract_browser_name(args): + "Extracts the browser name if any" + # These are BT arguments, it's BT job to check them + # here we just want to extract the browser name + res = re.findall(r"(--browser|-b)[= ]([\w]+)", " ".join(args)) + if res == []: + return None + return res[0][-1] + + def matches(args, *flags): + "Return True if any argument matches any of the given flags (maybe with an argument)." + for flag in flags: + if flag in args or any(arg.startswith(flag + "=") for arg in args): + return True + return False + + extra_args = [] + + # Default to Firefox. Override with `-b ...` or `--browser=...`. + specifies_browser = matches(args, "-b", "--browser") + if not specifies_browser: + extra_args.extend(("-b", "firefox")) + + # Default to not collect HAR. Override with `--skipHar=false`. + specifies_har = matches(args, "--har", "--skipHar", "--gzipHar") + if not specifies_har: + extra_args.append("--skipHar") + + if not matches(args, "--android"): + # If --firefox.binaryPath is not specified, default to the objdir binary + # Note: --firefox.release is not a real browsertime option, but it will + # silently ignore it instead and default to a release installation. + specifies_binaryPath = matches( + args, + "--firefox.binaryPath", + "--firefox.release", + "--firefox.nightly", + "--firefox.beta", + "--firefox.developer", + ) + + if not specifies_binaryPath: + specifies_binaryPath = extract_browser_name(args) == "chrome" + + if not specifies_binaryPath: + try: + extra_args.extend( + ("--firefox.binaryPath", command_context.get_binary_path()) + ) + except BinaryNotFoundException as e: + command_context.log( + logging.ERROR, + "browsertime", + {"error": str(e)}, + "ERROR: {error}", + ) + command_context.log( + logging.INFO, + "browsertime", + {}, + "Please run |./mach build| " + "or specify a Firefox binary with --firefox.binaryPath.", + ) + return 1 + + if extra_args: + command_context.log( + logging.DEBUG, + "browsertime", + {"extra_args": extra_args}, + "Running browsertime with extra default arguments: {extra_args}", + ) + + return extra_args + + +def _verify_node_install(command_context): + # check if Node is installed + sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint")) + import setup_helper + + with silence(): + node_valid = setup_helper.check_node_executables_valid() + if not node_valid: + print("Can't find Node. did you run ./mach bootstrap ?") + return False + + # check if the browsertime package has been deployed correctly + # for this we just check for the browsertime directory presence + if not os.path.exists(browsertime_path()): + print("Could not find browsertime.js, try ./mach browsertime --setup") + print("If that still fails, try ./mach browsertime --setup --clobber") + return False + + return True + + +@Command( + "browsertime", + category="testing", + description="Run [browsertime](https://github.com/sitespeedio/browsertime) " + "performance tests.", +) +@CommandArgument( + "--verbose", + action="store_true", + help="Verbose output for what commands the build is running.", +) +@CommandArgument("--update-upstream-url", default="") +@CommandArgument("--setup", default=False, action="store_true") +@CommandArgument("--clobber", default=False, action="store_true") +@CommandArgument( + "--skip-cache", + default=False, + action="store_true", + help="Skip all local caches to force re-fetching remote artifacts.", +) +@CommandArgument("--check-browsertime", default=False, action="store_true") +@CommandArgument( + "--install-vismet-reqs", + default=False, + action="store_true", + help="Add this flag to get the visual metrics requirements installed.", +) +@CommandArgument( + "--browsertime-help", + default=False, + action="store_true", + help="Show the browsertime help message.", +) +@CommandArgument("args", nargs=argparse.REMAINDER) +def browsertime( + command_context, + args, + verbose=False, + update_upstream_url="", + setup=False, + clobber=False, + skip_cache=False, + check_browsertime=False, + browsertime_help=False, + install_vismet_reqs=False, +): + command_context._set_log_level(verbose) + + # Output a message before going further to make sure the + # user knows that this tool is unsupported by the perftest + # team and point them to our supported tools. Pause a bit to + # make sure the user sees this message. + command_context.log( + logging.INFO, + "browsertime", + {}, + "[INFO] This command should be used for browsertime setup only.\n" + "If you are looking to run performance tests on your patch, use " + "`./mach raptor --browsertime` instead.\n\nYou can get visual-metrics " + "by using the --browsertime-video and --browsertime-visualmetrics. " + "Here is a sample command for raptor-browsertime: \n\t`./mach raptor " + "--browsertime -t amazon --browsertime-video --browsertime-visualmetrics`\n\n" + "See this wiki page for more information if needed: " + "https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Browsertime\n\n", + ) + time.sleep(5) + + if update_upstream_url: + return setup_browsertime( + command_context, + new_upstream_url=update_upstream_url, + install_vismet_reqs=install_vismet_reqs, + ) + elif setup: + return setup_browsertime( + command_context, + should_clobber=clobber, + install_vismet_reqs=install_vismet_reqs, + ) + else: + if not _verify_node_install(command_context): + return 1 + + if check_browsertime: + return check(command_context) + + if browsertime_help: + args.append("--help") + + activate_browsertime_virtualenv(command_context) + default_args = extra_default_args(command_context, args) + if default_args == 1: + return 1 + return node(command_context, [browsertime_path()] + default_args + args) diff --git a/tools/browsertime/moz.build b/tools/browsertime/moz.build new file mode 100644 index 0000000000..d590ad00cf --- /dev/null +++ b/tools/browsertime/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Testing", "Raptor") diff --git a/tools/browsertime/package-lock.json b/tools/browsertime/package-lock.json new file mode 100644 index 0000000000..b50278c6dc --- /dev/null +++ b/tools/browsertime/package-lock.json @@ -0,0 +1,5723 @@ +{ + "name": "mozilla-central-tools-browsertime", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "mozilla-central-tools-browsertime", + "license": "MPL-2.0", + "dependencies": { + "package.json": "^2.0.1" + }, + "devDependencies": { + "browsertime": "https://github.com/sitespeedio/browsertime/tarball/62de4fc9abc8067fb58378999b1bc4a4c42f9eb5" + } + }, + "node_modules/@cypress/xvfb": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@cypress/xvfb/-/xvfb-1.2.4.tgz", + "integrity": "sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q==", + "dev": true, + "dependencies": { + "debug": "^3.1.0", + "lodash.once": "^4.1.1" + } + }, + "node_modules/@devicefarmer/adbkit": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit/-/adbkit-3.2.5.tgz", + "integrity": "sha512-+J479WWZW3GU3t40flicDfiDrFz6vpiy2RcBQPEhFcs/3La9pOtr4Bgz2Q02E4luUG2RAL068rqIkKNUTy3tZw==", + "dev": true, + "dependencies": { + "@devicefarmer/adbkit-logcat": "^2.1.2", + "@devicefarmer/adbkit-monkey": "~1.2.1", + "bluebird": "~3.7", + "commander": "^9.1.0", + "debug": "~4.3.1", + "node-forge": "^1.3.1", + "split": "~1.0.1" + }, + "bin": { + "adbkit": "bin/adbkit" + }, + "engines": { + "node": ">= 0.10.4" + } + }, + "node_modules/@devicefarmer/adbkit-logcat": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit-logcat/-/adbkit-logcat-2.1.3.tgz", + "integrity": "sha512-yeaGFjNBc/6+svbDeul1tNHtNChw6h8pSHAt5D+JsedUrMTN7tla7B15WLDyekxsuS2XlZHRxpuC6m92wiwCNw==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/@devicefarmer/adbkit-monkey": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit-monkey/-/adbkit-monkey-1.2.1.tgz", + "integrity": "sha512-ZzZY/b66W2Jd6NHbAhLyDWOEIBWC11VizGFk7Wx7M61JZRz7HR9Cq5P+65RKWUU7u6wgsE8Lmh9nE4Mz+U2eTg==", + "dev": true, + "engines": { + "node": ">= 0.10.4" + } + }, + "node_modules/@devicefarmer/adbkit/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@devicefarmer/adbkit/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/@jimp/bmp": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.22.10.tgz", + "integrity": "sha512-1UXRl1Nw1KptZ1r0ANqtXOst9vGH51dq7keVKQzyyTO2lz4dOaezS9StuSTNh+RmiHg/SVPaFRpPfB0S/ln4Kg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10", + "bmp-js": "^0.1.0" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/core": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.22.10.tgz", + "integrity": "sha512-ZKyrehVy6wu1PnBXIUpn/fXmyMRQiVSbvHDubgXz4bfTOao3GiOurKHjByutQIgozuAN6ZHWiSge1dKA+dex3w==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10", + "any-base": "^1.1.0", + "buffer": "^5.2.0", + "exif-parser": "^0.1.12", + "file-type": "^16.5.4", + "isomorphic-fetch": "^3.0.0", + "pixelmatch": "^4.0.2", + "tinycolor2": "^1.6.0" + } + }, + "node_modules/@jimp/custom": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.22.10.tgz", + "integrity": "sha512-sPZkUYe1hu0iIgNisjizxPJqq2vaaKvkCkPoXq2U6UV3ZA1si/WVdrg25da3IcGIEV+83AoHgM8TvqlLgrCJsg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/core": "^0.22.10" + } + }, + "node_modules/@jimp/gif": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.22.10.tgz", + "integrity": "sha512-yEX2dSpamvkSx1PPDWGnKeWDrBz0vrCKjVG/cn4Zr68MRRT75tbZIeOrBa+RiUpY3ho5ix7d36LkYvt3qfUIhQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10", + "gifwrap": "^0.10.1", + "omggif": "^1.0.9" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/jpeg": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.22.10.tgz", + "integrity": "sha512-6bu98pAcVN4DY2oiDLC4TOgieX/lZrLd1tombWZOFCN5PBmqaHQxm7IUmT+Wj4faEvh8QSHgVLSA+2JQQRJWVA==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10", + "jpeg-js": "^0.4.4" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-blit": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.22.10.tgz", + "integrity": "sha512-6EI8Sl+mxYHEIy6Yteh6eknD+EZguKpNdr3sCKxNezmLR0+vK99vHcllo6uGSjXXiwtwS67Xqxn8SsoatL+UJQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-blur": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.22.10.tgz", + "integrity": "sha512-4XRTWuPVdMXJeclJMisXPGizeHtTryVaVV5HnuQXpKqIZtzXReCCpNGH8q/i0kBQOQMXhGWS3mpqOEwtpPePKw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-circle": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.22.10.tgz", + "integrity": "sha512-mhcwTO1ywRxiCgtLGge6tDDIDPlX6qkI3CY+BjgGG/XhVHccCddXgOGLdlf+5OuKIEF2Nqs0V01LQEQIJFTmEw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-color": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.22.10.tgz", + "integrity": "sha512-e4t3L7Kedd96E0x1XjsTM6NcgulKUU66HdFTao7Tc9FYJRFSlttARZ/C6LEryGDm/i69R6bJEpo7BkNz0YL55Q==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10", + "tinycolor2": "^1.6.0" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-contain": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.22.10.tgz", + "integrity": "sha512-eP8KrzctuEoqibQAxi9WhbnoRosydhiwg+IYya3dKuKDBTrD9UHt+ERlPQ/lTNWHzV/l4S1ntV3r9s9saJgsXA==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blit": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5", + "@jimp/plugin-scale": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-cover": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.22.10.tgz", + "integrity": "sha512-kJCwL5T1igfa0InCfkE7bBeqg26m46aoRt10ug+rvm11P6RrvRMGrgINFyIKB+mnB7CiyBN/MOula1CvLhSInQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-crop": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5", + "@jimp/plugin-scale": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-crop": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.22.10.tgz", + "integrity": "sha512-BOZ+YGaZlhU7c5ye65RxikicXH0Ki0It6/XHISvipR5WZrfjLjL2Ke20G+AGnwBQc76gKenVcMXVUCnEjtZV+Q==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-displace": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.22.10.tgz", + "integrity": "sha512-llNiWWMTKISDXt5+cXI0GaFmZWAjlT+4fFLYf4eXquuL/9wZoQsEBhv2GdGd48mkiS8jZq1Nnb2Q4ehEPTvrzw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-dither": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.22.10.tgz", + "integrity": "sha512-05WLmeV5M+P/0FS+bWf13hMew2X0oa8w9AtmevL2UyA/5GqiyvP2Xm5WfGQ8oFiiMvpnL6RFomJQOZtWca0C2w==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-fisheye": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.22.10.tgz", + "integrity": "sha512-InjiXvc7Gkzrx8VWtU97kDqV7ENnhHGPULymJWeZaF2aicud9Fpk4iCtd/DcZIrk7Cbe60A8RwNXN00HXIbSCg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-flip": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.22.10.tgz", + "integrity": "sha512-42GkGtTHWnhnwTMPVK/kXObZbkYIpQWfuIfy5EMEMk6zRj05zpv4vsjkKWfuemweZINwfvD7wDJF7FVFNNcZZg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-rotate": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-gaussian": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.22.10.tgz", + "integrity": "sha512-ykrG/6lTp9Q5YA8jS5XzwMHtRxb9HOFMgtmnrUZ8kU+BK8REecfy9Ic5BUEOjCYvS1a/xLsnrZQU07iiYxBxFg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-invert": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.22.10.tgz", + "integrity": "sha512-d8j9BlUJYs/c994t4azUWSWmQq4LLPG4ecm8m6SSNqap+S/HlVQGqjYhJEBbY9EXkOTYB9vBL9bqwSM1Rr6paA==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-mask": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.22.10.tgz", + "integrity": "sha512-yRBs1230XZkz24uFTdTcSlZ0HXZpIWzM3iFQN56MzZ7USgdVZjPPDCQ8I9RpqfZ36nDflQkUO0wV7ucsi4ogow==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-normalize": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.22.10.tgz", + "integrity": "sha512-Wk9GX6eJMchX/ZAazVa70Fagu+OXMvHiPY+HrcEwcclL+p1wo8xAHEsf9iKno7Ja4EU9lLhbBRY5hYJyiKMEkg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-print": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.22.10.tgz", + "integrity": "sha512-1U3VloIR+beE1kWPdGEJMiE2h1Do29iv3w8sBbvPyRP4qXxRFcDpmCGtctsrKmb1krlBFlj8ubyAY90xL+5n9w==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10", + "load-bmfont": "^1.4.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blit": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-resize": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.22.10.tgz", + "integrity": "sha512-ixomxVcnAONXDgaq0opvAx4UAOiEhOA/tipuhFFOvPKFd4yf1BAnEviB5maB0SBHHkJXPUSzDp/73xVTMGSe7g==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-rotate": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.22.10.tgz", + "integrity": "sha512-eeFX8dnRyf3LAdsdXWKWuN18hLRg8zy1cP0cP9rHzQVWRK7ck/QsLxK1vHq7MADGwQalNaNTJ9SQxH6c8mz6jw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blit": ">=0.3.5", + "@jimp/plugin-crop": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-scale": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.22.10.tgz", + "integrity": "sha512-TG/H0oUN69C9ArBCZg4PmuoixFVKIiru8282KzSB/Tp1I0xwX0XLTv3dJ5pobPlIgPcB+TmD4xAIdkCT4rtWxg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-shadow": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.22.10.tgz", + "integrity": "sha512-TN9xm6fI7XfxbMUQqFPZjv59Xdpf0tSiAQdINB4g6pJMWiVANR/74OtDONoy3KKpenu5Y38s+FkrtID/KcQAhw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blur": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-threshold": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.22.10.tgz", + "integrity": "sha512-DA2lSnU0TgIRbAgmXaxroYw3Ad6J2DOFEoJp0NleSm2h3GWbZEE5yW9U2B6hD3iqn4AenG4E2b2WzHXZyzSutw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-color": ">=0.8.0", + "@jimp/plugin-resize": ">=0.8.0" + } + }, + "node_modules/@jimp/plugins": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.22.10.tgz", + "integrity": "sha512-KDMZyM6pmvS8freB+UBLko1TO/k4D7URS/nphCozuH+P7i3UMe7NdckXKJ8u+WD6sqN0YFYvBehpkpnUiw/91w==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/plugin-blit": "^0.22.10", + "@jimp/plugin-blur": "^0.22.10", + "@jimp/plugin-circle": "^0.22.10", + "@jimp/plugin-color": "^0.22.10", + "@jimp/plugin-contain": "^0.22.10", + "@jimp/plugin-cover": "^0.22.10", + "@jimp/plugin-crop": "^0.22.10", + "@jimp/plugin-displace": "^0.22.10", + "@jimp/plugin-dither": "^0.22.10", + "@jimp/plugin-fisheye": "^0.22.10", + "@jimp/plugin-flip": "^0.22.10", + "@jimp/plugin-gaussian": "^0.22.10", + "@jimp/plugin-invert": "^0.22.10", + "@jimp/plugin-mask": "^0.22.10", + "@jimp/plugin-normalize": "^0.22.10", + "@jimp/plugin-print": "^0.22.10", + "@jimp/plugin-resize": "^0.22.10", + "@jimp/plugin-rotate": "^0.22.10", + "@jimp/plugin-scale": "^0.22.10", + "@jimp/plugin-shadow": "^0.22.10", + "@jimp/plugin-threshold": "^0.22.10", + "timm": "^1.6.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/png": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.22.10.tgz", + "integrity": "sha512-RYinU7tZToeeR2g2qAMn42AU+8OUHjXPKZZ9RkmoL4bguA1xyZWaSdr22/FBkmnHhOERRlr02KPDN1OTOYHLDQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.10", + "pngjs": "^6.0.0" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/tiff": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.22.10.tgz", + "integrity": "sha512-OaivlSYzpNTHyH/h7pEtl3A7F7TbsgytZs52GLX/xITW92ffgDgT6PkldIrMrET6ERh/hdijNQiew7IoEEr2og==", + "dev": true, + "optional": true, + "dependencies": { + "utif2": "^4.0.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/types": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.22.10.tgz", + "integrity": "sha512-u/r+XYzbCx4zZukDmxx8S0er3Yq3iDPI6+31WKX0N18i2qPPJYcn8qwIFurfupRumGvJ8SlGLCgt/T+Y8zzUIw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/bmp": "^0.22.10", + "@jimp/gif": "^0.22.10", + "@jimp/jpeg": "^0.22.10", + "@jimp/png": "^0.22.10", + "@jimp/tiff": "^0.22.10", + "timm": "^1.6.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/utils": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.22.10.tgz", + "integrity": "sha512-ztlOK9Mm2iLG2AMoabzM4i3WZ/FtshcgsJCbZCRUs/DKoeS2tySRJTnQZ1b7Roq0M4Ce+FUAxnCAcBV0q7PH9w==", + "dev": true, + "optional": true, + "dependencies": { + "regenerator-runtime": "^0.13.3" + } + }, + "node_modules/@sitespeed.io/chromedriver": { + "version": "119.0.6045-105", + "resolved": "https://registry.npmjs.org/@sitespeed.io/chromedriver/-/chromedriver-119.0.6045-105.tgz", + "integrity": "sha512-DfQQaqTB28e05kG3CWjC9OWKeNTWiqgu5cl6CvYQsd2MTDDDRUQ0a+VZ8KTSrRx6xZCsTBgzZK2kNBNiMvNH8w==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "node-downloader-helper": "2.1.9", + "node-stream-zip": "1.15.0" + } + }, + "node_modules/@sitespeed.io/edgedriver": { + "version": "119.0.2151-42", + "resolved": "https://registry.npmjs.org/@sitespeed.io/edgedriver/-/edgedriver-119.0.2151-42.tgz", + "integrity": "sha512-+jGP9BmWgh/yoNcJKyiYP0anF0m2H6+cjk1MaHvzgkIdrFMVfJQIN9+tmwCBiN4Ave52IHjDdHhEjK7B+SWvrA==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "node-downloader-helper": "2.1.7", + "node-stream-zip": "1.15.0" + } + }, + "node_modules/@sitespeed.io/edgedriver/node_modules/node-downloader-helper": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.7.tgz", + "integrity": "sha512-3dBuMF/XPy5WFi3XiiXaglafzoycRH5GjmRz1nAt2uI9D+TcBrc+n/AzH8bzLHR85Wsf6vZSZblzw+MiUS/WNQ==", + "dev": true, + "bin": { + "ndh": "bin/ndh" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sitespeed.io/geckodriver": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/@sitespeed.io/geckodriver/-/geckodriver-0.33.0.tgz", + "integrity": "sha512-w6w+x9/Q44JekTPi8NlRsfh5Uz4TfquJcUEs0tA/oEcxLVxRS7VtaiaJEE0GzzN6cUmFS6Twas7E4bCA4k/Yxg==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "node-downloader-helper": "2.1.5", + "node-stream-zip": "1.15.0", + "tar": "6.1.13" + } + }, + "node_modules/@sitespeed.io/geckodriver/node_modules/node-downloader-helper": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.5.tgz", + "integrity": "sha512-sLedzfv8C4VMAvTdDQcjLFAl3gydNeBXh2bLcCzvZRmd4EK0rkoTxJ8tkxnriUSJO/n13skJzH7l6CzCdBwYGg==", + "dev": true, + "bin": { + "ndh": "bin/ndh" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sitespeed.io/throttle": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@sitespeed.io/throttle/-/throttle-5.0.0.tgz", + "integrity": "sha512-eul4I7IllA6l3+GGX1aW/D75XYux0ODuZDzstKD0kAuvIkpQ4BVLkFBoLXQN50gLMFGqZ3QWMobhQ5L2/6sFgg==", + "dev": true, + "dependencies": { + "minimist": "1.2.6" + }, + "bin": { + "throttle": "bin/index.js" + }, + "engines": { + "node": ">=14.16" + } + }, + "node_modules/@sitespeed.io/tracium": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@sitespeed.io/tracium/-/tracium-0.3.3.tgz", + "integrity": "sha512-dNZafjM93Y+F+sfwTO5gTpsGXlnc/0Q+c2+62ViqP3gkMWvHEMSKkaEHgVJLcLg3i/g19GSIPziiKpgyne07Bw==", + "dev": true, + "dependencies": { + "debug": "^4.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@sitespeed.io/tracium/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@sitespeed.io/tracium/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "dev": true, + "optional": true + }, + "node_modules/@types/node": { + "version": "16.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", + "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==", + "dev": true, + "optional": true + }, + "node_modules/abs": { + "version": "1.3.14", + "resolved": "https://registry.npmjs.org/abs/-/abs-1.3.14.tgz", + "integrity": "sha512-PrS26IzwKLWwuURpiKl8wRmJ2KdR/azaVrLEBWG/TALwT20Y7qjtYp1qcMLHA4206hBHY5phv3w4pjf9NPv4Vw==", + "dependencies": { + "ul": "^5.0.0" + } + }, + "node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/any-base": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz", + "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==", + "dev": true, + "optional": true + }, + "node_modules/async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha512-nSVgobk4rv61R9PUSDtYt7mPVB2olxNR5RWJcAsH676/ef11bUZwvu7+RGYrYauVdDPcO519v68wRhXQtxsV9w==", + "dev": true + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "dev": true + }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "dev": true, + "optional": true + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/browsertime": { + "version": "18.0.0", + "resolved": "https://github.com/sitespeedio/browsertime/tarball/62de4fc9abc8067fb58378999b1bc4a4c42f9eb5", + "integrity": "sha512-dtX8pNd4HLQIBBphbTs4Ok0FTt/+zgikbjxI0B2YEjzOEtbSI//ofn4woOYdIC7JOiTtKhYB79eqXaIbVXORqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cypress/xvfb": "1.2.4", + "@devicefarmer/adbkit": "3.2.5", + "@sitespeed.io/chromedriver": "119.0.6045-105", + "@sitespeed.io/edgedriver": "119.0.2151-42", + "@sitespeed.io/geckodriver": "0.33.0", + "@sitespeed.io/throttle": "5.0.0", + "@sitespeed.io/tracium": "0.3.3", + "btoa": "1.2.1", + "chrome-har": "0.13.2", + "chrome-remote-interface": "0.33.0", + "dayjs": "1.11.10", + "execa": "8.0.1", + "fast-stats": "0.0.6", + "ff-test-bidi-har-export": "0.0.12", + "find-up": "6.3.0", + "get-port": "7.0.0", + "hasbin": "1.2.3", + "intel": "1.2.0", + "lodash.get": "4.4.2", + "lodash.groupby": "4.6.0", + "lodash.isempty": "4.4.0", + "lodash.merge": "4.6.2", + "lodash.pick": "4.4.0", + "lodash.set": "4.3.2", + "selenium-webdriver": "4.15.0", + "yargs": "17.7.2" + }, + "bin": { + "browsertime": "bin/browsertime.js" + }, + "engines": { + "node": ">=14.19.1" + }, + "optionalDependencies": { + "jimp": "0.22.10" + } + }, + "node_modules/btoa": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", + "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==", + "dev": true, + "bin": { + "btoa": "bin/btoa.js" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true, + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-equal": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz", + "integrity": "sha512-RgSV6InVQ9ODPdLWJ5UAqBqJBOg370Nz6ZQtRzpt6nUjc8v0St97uJ4PYC6NztqIScrAXafKM3mZPMygSe1ggA==", + "dev": true, + "optional": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/capture-stack-trace": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.2.tgz", + "integrity": "sha512-X/WM2UQs6VMHUtjUDnZTRI+i1crWteJySFzr9UpGoQa4WQffXVTTXuekjl7TjZRlcF2XfjgITT0HxZ9RnxeT0w==", + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dev": true, + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/chrome-har": { + "version": "0.13.2", + "resolved": "https://registry.npmjs.org/chrome-har/-/chrome-har-0.13.2.tgz", + "integrity": "sha512-QiwyoilXiGVLG9Y0UMzWOyuao/PctTU9AAOTMqH7BuuulY1e0foDZ/O9qmLfdBAe6MbwIl9aDYvrlbyna3uRZw==", + "dev": true, + "dependencies": { + "dayjs": "1.11.7", + "debug": "4.3.4", + "tough-cookie": "4.1.3", + "uuid": "9.0.0" + }, + "engines": { + "node": ">=14.19.1" + } + }, + "node_modules/chrome-har/node_modules/dayjs": { + "version": "1.11.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz", + "integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==", + "dev": true + }, + "node_modules/chrome-har/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/chrome-har/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/chrome-remote-interface": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/chrome-remote-interface/-/chrome-remote-interface-0.33.0.tgz", + "integrity": "sha512-tv/SgeBfShXk43fwFpQ9wnS7mOCPzETnzDXTNxCb6TqKOiOeIfbrJz+2NAp8GmzwizpKa058wnU1Te7apONaYg==", + "dev": true, + "dependencies": { + "commander": "2.11.x", + "ws": "^7.2.0" + }, + "bin": { + "chrome-remote-interface": "bin/client.js" + } + }, + "node_modules/chrome-remote-interface/node_modules/commander": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", + "dev": true + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "dev": true, + "engines": { + "node": "^12.20.0 || >=14" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "node_modules/create-error-class": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", + "integrity": "sha512-gYTKKexFO3kh200H1Nit76sRwRtOY32vQd3jpAQKpLtZqyNsSQNfI4N7o3eP2wUjV35pTWKRYqFUDBvUha/Pkw==", + "dependencies": { + "capture-stack-trace": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/dayjs": { + "version": "1.11.10", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", + "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==", + "dev": true + }, + "node_modules/dbug": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/dbug/-/dbug-0.4.2.tgz", + "integrity": "sha512-nrmsMK1msY0WXwfA2czrKVDgpIYJR2JJaq5cX4DwW7Rxm11nXHqouh9wmubEs44bHYxk8CqeP/Jx4URqSB961w==", + "dev": true + }, + "node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deffy": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/deffy/-/deffy-2.2.4.tgz", + "integrity": "sha512-pLc9lsbsWjr6RxmJ2OLyvm+9l4j1yK69h+TML/gUit/t3vTijpkNGh8LioaJYTGO7F25m6HZndADcUOo2PsiUg==", + "dependencies": { + "typpy": "^2.0.0" + } + }, + "node_modules/dom-walk": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.2.tgz", + "integrity": "sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w==", + "dev": true, + "optional": true + }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/err": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/err/-/err-1.1.1.tgz", + "integrity": "sha512-N97Ybd2jJHVQ+Ft3Q5+C2gM3kgygkdeQmEqbN2z15UTVyyEsIwLA1VK39O1DHEJhXbwIFcJLqm6iARNhFANcQA==", + "dependencies": { + "typpy": "^2.2.0" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/exec-limiter": { + "version": "3.2.13", + "resolved": "https://registry.npmjs.org/exec-limiter/-/exec-limiter-3.2.13.tgz", + "integrity": "sha512-86Ri699bwiHZVBzTzNj8gspqAhCPchg70zPVWIh3qzUOA1pUMcb272Em3LPk8AE0mS95B9yMJhtqF8vFJAn0dA==", + "dependencies": { + "limit-it": "^3.0.0", + "typpy": "^2.1.0" + } + }, + "node_modules/execa": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", + "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^8.0.1", + "human-signals": "^5.0.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^3.0.0" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exif-parser": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz", + "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==", + "dev": true, + "optional": true + }, + "node_modules/fast-stats": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/fast-stats/-/fast-stats-0.0.6.tgz", + "integrity": "sha512-m0zkwa7Z07Wc4xm1YtcrCHmhzNxiYRrrfUyhkdhSZPzaAH/Ewbocdaq7EPVBFz19GWfIyyPcLfRHjHJYe83jlg==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/ff-test-bidi-har-export": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/ff-test-bidi-har-export/-/ff-test-bidi-har-export-0.0.12.tgz", + "integrity": "sha512-ccJZc14x/1ymgcLpUBz52Rci/UsbboqJ5wgiPrcHQMyh8YOwNJLGt3yGygIHNhiShZ8aA8H4jOmQU980Ngot9Q==", + "dev": true + }, + "node_modules/file-type": { + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "dev": true, + "optional": true, + "dependencies": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "node_modules/find-up": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", + "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==", + "dev": true, + "dependencies": { + "locate-path": "^7.1.0", + "path-exists": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/function.name": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/function.name/-/function.name-1.0.13.tgz", + "integrity": "sha512-mVrqdoy5npWZyoXl4DxCeuVF6delDcQjVS9aPdvLYlBxtMTZDR2B5GVEQEoM1jJyspCqg3C0v4ABkLE7tp9xFA==", + "dependencies": { + "noop6": "^1.0.1" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-port": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-7.0.0.tgz", + "integrity": "sha512-mDHFgApoQd+azgMdwylJrv2DX47ywGq1i5VFJE7fZ0dttNq3iQMfsU4IvEgBHojA3KqEudyu7Vq+oN8kNaNkWw==", + "dev": true, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-stream": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", + "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", + "dev": true, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gifwrap": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/gifwrap/-/gifwrap-0.10.1.tgz", + "integrity": "sha512-2760b1vpJHNmLzZ/ubTtNnEx5WApN/PYWJvXvgS+tL1egTTthayFYIQQNi136FLEDcN/IyEY2EcGpIITD6eYUw==", + "dev": true, + "optional": true, + "dependencies": { + "image-q": "^4.0.0", + "omggif": "^1.0.10" + } + }, + "node_modules/git-package-json": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/git-package-json/-/git-package-json-1.4.10.tgz", + "integrity": "sha512-DRAcvbzd2SxGK7w8OgYfvKqhFliT5keX0lmSmVdgScgf1kkl5tbbo7Pam6uYoCa1liOiipKxQZG8quCtGWl/fA==", + "dependencies": { + "deffy": "^2.2.1", + "err": "^1.1.1", + "gry": "^5.0.0", + "normalize-package-data": "^2.3.5", + "oargv": "^3.4.1", + "one-by-one": "^3.1.0", + "r-json": "^1.2.1", + "r-package-json": "^1.0.0", + "tmp": "0.0.28" + } + }, + "node_modules/git-source": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/git-source/-/git-source-1.1.10.tgz", + "integrity": "sha512-XZZ7ZgnLL35oLgM/xjnLYgtlKlxJG0FohC1kWDvGkU7s1VKGXK0pFF/g1itQEwQ3D+uTQzBnzPi8XbqOv7Wc1Q==", + "dependencies": { + "git-url-parse": "^5.0.1" + } + }, + "node_modules/git-up": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/git-up/-/git-up-1.2.1.tgz", + "integrity": "sha512-SRVN3rOLACva8imc7BFrB6ts5iISWKH1/h/1Z+JZYoUI7UVQM7gQqk4M2yxUENbq2jUUT09NEND5xwP1i7Ktlw==", + "dependencies": { + "is-ssh": "^1.0.0", + "parse-url": "^1.0.0" + } + }, + "node_modules/git-url-parse": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/git-url-parse/-/git-url-parse-5.0.1.tgz", + "integrity": "sha512-4uSiOgrryNEMBX+gTWogenYRUh2j1D+95STTSEF2RCTgLkfJikl8c7BGr0Bn274hwuxTsbS2/FQ5pVS9FoXegQ==", + "dependencies": { + "git-up": "^1.0.0" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/global": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", + "integrity": "sha512-wv/LAoHdRE3BeTGz53FAamhGlPLhlssK45usmGFThIi4XqnBmjKQ16u+RNbP7WvigRZDxUsM0J3gcQ5yicaL0w==", + "dev": true, + "optional": true, + "dependencies": { + "min-document": "^2.19.0", + "process": "^0.11.10" + } + }, + "node_modules/got": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-5.6.0.tgz", + "integrity": "sha512-MnypzkaW8dldA8AbJFjMs7y14+ykd2V8JCLKSvX1Gmzx1alH3Y+3LArywHDoAF2wS3pnZp4gacoYtvqBeF6drQ==", + "dependencies": { + "create-error-class": "^3.0.1", + "duplexer2": "^0.1.4", + "is-plain-obj": "^1.0.0", + "is-redirect": "^1.0.0", + "is-retry-allowed": "^1.0.0", + "is-stream": "^1.0.0", + "lowercase-keys": "^1.0.0", + "node-status-codes": "^1.0.0", + "object-assign": "^4.0.1", + "parse-json": "^2.1.0", + "pinkie-promise": "^2.0.0", + "read-all-stream": "^3.0.0", + "readable-stream": "^2.0.5", + "timed-out": "^2.0.0", + "unzip-response": "^1.0.0", + "url-parse-lax": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/got/node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gry": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/gry/-/gry-5.0.8.tgz", + "integrity": "sha512-meq9ZjYVpLzZh3ojhTg7IMad9grGsx6rUUKHLqPnhLXzJkRQvEL2U3tQpS5/WentYTtHtxkT3Ew/mb10D6F6/g==", + "dependencies": { + "abs": "^1.2.1", + "exec-limiter": "^3.0.0", + "one-by-one": "^3.0.0", + "ul": "^5.0.0" + } + }, + "node_modules/has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/hasbin": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/hasbin/-/hasbin-1.2.3.tgz", + "integrity": "sha512-CCd8e/w2w28G8DyZvKgiHnQJ/5XXDz6qiUHnthvtag/6T5acUeN5lqq+HMoBqcmgWueWDhiCplrw0Kb1zDACRg==", + "dev": true, + "dependencies": { + "async": "~1.5" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==" + }, + "node_modules/human-signals": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", + "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "dev": true, + "engines": { + "node": ">=16.17.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, + "node_modules/image-q": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/image-q/-/image-q-4.0.0.tgz", + "integrity": "sha512-PfJGVgIfKQJuq3s0tTDOKtztksibuUEbJQIYT3by6wctQo+Rdlh7ef4evJ5NCdxY4CfMbvFkocEwbl4BF8RlJw==", + "dev": true, + "optional": true, + "dependencies": { + "@types/node": "16.9.1" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "dev": true + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "node_modules/intel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/intel/-/intel-1.2.0.tgz", + "integrity": "sha512-CUDyAtEeEeDo5YtwANOuDhxuFEOgInHvbMrBbhXCD4tAaHuzHM2llevtTeq2bmP8Jf7NkpN305pwDncRmhc1Wg==", + "dev": true, + "dependencies": { + "chalk": "^1.1.0", + "dbug": "~0.4.2", + "stack-trace": "~0.0.9", + "strftime": "~0.10.0", + "symbol": "~0.3.1", + "utcstring": "~0.1.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" + }, + "node_modules/is-core-module": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "dependencies": { + "hasown": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-function": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.2.tgz", + "integrity": "sha512-lw7DUp0aWXYg+CBCN+JKkcE0Q2RayZnSvnZBlwgxHBQhqt5pZNVy4Ri7H9GmmXkdu7LUthszM+Tor1u/2iBcpQ==", + "dev": true, + "optional": true + }, + "node_modules/is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha512-cr/SlUEe5zOGmzvj9bUyC4LVvkNVAXu4GytXLNMr1pny+a65MpQ9IJzFHD5vi7FyJgb4qt27+eS3TuQnqB+RQw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-retry-allowed": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", + "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-ssh": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/is-ssh/-/is-ssh-1.4.0.tgz", + "integrity": "sha512-x7+VxdxOdlV3CYpjvRLBv5Lo9OJerlYanjwFrPR9fuGPjCiNiCzFgAWpiLAohSbsnH4ZAys3SBh+hq5rJosxUQ==", + "dependencies": { + "protocols": "^2.0.1" + } + }, + "node_modules/is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/isomorphic-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", + "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", + "dev": true, + "optional": true, + "dependencies": { + "node-fetch": "^2.6.1", + "whatwg-fetch": "^3.4.1" + } + }, + "node_modules/iterate-object": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/iterate-object/-/iterate-object-1.3.4.tgz", + "integrity": "sha512-4dG1D1x/7g8PwHS9aK6QV5V94+ZvyP4+d19qDv43EzImmrndysIl4prmJ1hWWIGCqrZHyaHBm6BSEWHOLnpoNw==" + }, + "node_modules/jimp": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.22.10.tgz", + "integrity": "sha512-lCaHIJAgTOsplyJzC1w/laxSxrbSsEBw4byKwXgUdMmh+ayPsnidTblenQm+IvhIs44Gcuvlb6pd2LQ0wcKaKg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/custom": "^0.22.10", + "@jimp/plugins": "^0.22.10", + "@jimp/types": "^0.22.10", + "regenerator-runtime": "^0.13.3" + } + }, + "node_modules/jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "dev": true, + "optional": true + }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "dev": true, + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/limit-it": { + "version": "3.2.10", + "resolved": "https://registry.npmjs.org/limit-it/-/limit-it-3.2.10.tgz", + "integrity": "sha512-T0NK99pHnkimldr1WUqvbGV1oWDku/xC9J/OqzJFsV1jeOS6Bwl8W7vkeQIBqwiON9dTALws+rX/XPMQqWerDQ==", + "dependencies": { + "typpy": "^2.0.0" + } + }, + "node_modules/load-bmfont": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", + "integrity": "sha512-8UyQoYmdRDy81Brz6aLAUhfZLwr5zV0L3taTQ4hju7m6biuwiWiJXjPhBJxbUQJA8PrkvJ/7Enqmwk2sM14soA==", + "dev": true, + "optional": true, + "dependencies": { + "buffer-equal": "0.0.1", + "mime": "^1.3.4", + "parse-bmfont-ascii": "^1.0.3", + "parse-bmfont-binary": "^1.0.5", + "parse-bmfont-xml": "^1.1.4", + "phin": "^2.9.1", + "xhr": "^2.0.1", + "xtend": "^4.0.0" + } + }, + "node_modules/locate-path": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", + "dev": true, + "dependencies": { + "p-locate": "^6.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "dev": true + }, + "node_modules/lodash.isempty": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz", + "integrity": "sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg==", + "dev": true + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "dev": true + }, + "node_modules/lodash.pick": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", + "integrity": "sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==", + "dev": true + }, + "node_modules/lodash.set": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", + "integrity": "sha512-4hNPN5jlm/N/HLMCO43v8BXKq9Z7QdAGc/VGrRD61w8gN9g/6jF9A4L1pbUgBLCffi0w9VsXfTOij5x8iTyFvg==", + "dev": true + }, + "node_modules/lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "optional": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/min-document": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", + "integrity": "sha512-9Wy1B3m3f66bPPmU5hdA4DR4PB2OfDU/+GS3yAB7IQozE3tqXaVv2zOjgla7MEGSRv95+ILmOuvhLkOK6wJtCQ==", + "dev": true, + "optional": true, + "dependencies": { + "dom-walk": "^0.1.0" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + }, + "node_modules/minipass": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.8.tgz", + "integrity": "sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/node-downloader-helper": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.9.tgz", + "integrity": "sha512-FSvAol2Z8UP191sZtsUZwHIN0eGoGue3uEXGdWIH5228e9KH1YHXT7fN8Oa33UGf+FbqGTQg3sJfrRGzmVCaJA==", + "dev": true, + "bin": { + "ndh": "bin/ndh" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "optional": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "dev": true, + "engines": { + "node": ">= 6.13.0" + } + }, + "node_modules/node-status-codes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz", + "integrity": "sha512-1cBMgRxdMWE8KeWCqk2RIOrvUb0XCwYfEsY5/y2NlXyq4Y/RumnOZvTj4Nbr77+Vb2C+kyBoRTdkNOS8L3d/aQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/node-stream-zip": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.15.0.tgz", + "integrity": "sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw==", + "dev": true, + "engines": { + "node": ">=0.12.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/antelle" + } + }, + "node_modules/noop6": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/noop6/-/noop6-1.0.9.tgz", + "integrity": "sha512-DB3Hwyd89dPr5HqEPg3YHjzvwh/mCqizC1zZ8vyofqc+TQRyPDnT4wgXXbLGF4z9YAzwwTLi8pNLhGqcbSjgkA==" + }, + "node_modules/normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dependencies": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "node_modules/npm-run-path": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.1.0.tgz", + "integrity": "sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==", + "dev": true, + "dependencies": { + "path-key": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/oargv": { + "version": "3.4.10", + "resolved": "https://registry.npmjs.org/oargv/-/oargv-3.4.10.tgz", + "integrity": "sha512-SXaMANv9sr7S/dP0vj0+Ybipa47UE1ntTWQ2rpPRhC6Bsvfl+Jg03Xif7jfL0sWKOYWK8oPjcZ5eJ82t8AP/8g==", + "dependencies": { + "iterate-object": "^1.1.0", + "ul": "^5.0.0" + } + }, + "node_modules/obj-def": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/obj-def/-/obj-def-1.0.9.tgz", + "integrity": "sha512-bQ4ya3VYD6FAA1+s6mEhaURRHSmw4+sKaXE6UyXZ1XDYc5D+c7look25dFdydmLd18epUegh398gdDkMUZI9xg==", + "dependencies": { + "deffy": "^2.2.2" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/omggif": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", + "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==", + "dev": true, + "optional": true + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/one-by-one": { + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/one-by-one/-/one-by-one-3.2.8.tgz", + "integrity": "sha512-HR/pSzZdm46Xqj58K+Bu64kMbSTw8/u77AwWvV+rprO/OsuR++pPlkUJn+SmwqBGRgHKwSKQ974V3uls7crIeQ==", + "dependencies": { + "obj-def": "^1.0.0", + "sliced": "^1.0.1" + } + }, + "node_modules/onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "dev": true, + "dependencies": { + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/p-limit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^1.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz", + "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==", + "dev": true, + "dependencies": { + "p-limit": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-2.4.0.tgz", + "integrity": "sha512-PRg65iXMTt/uK8Rfh5zvzkUbfAPitF17YaCY+IbHsYgksiLvtzWWTUildHth3mVaZ7871OJ7gtP4LBRBlmAdXg==", + "dependencies": { + "got": "^5.0.0", + "registry-auth-token": "^3.0.1", + "registry-url": "^3.0.3", + "semver": "^5.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/package-json-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/package-json-path/-/package-json-path-1.0.9.tgz", + "integrity": "sha512-uNu7f6Ef7tQHZRnkyVnCtzdSYVN9uBtge/sG7wzcUaawFWkPYUq67iXxRGrQSg/q0tzxIB8jSyIYUKjG2Jn//A==", + "dependencies": { + "abs": "^1.2.1" + } + }, + "node_modules/package.json": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/package.json/-/package.json-2.0.1.tgz", + "integrity": "sha512-pSxZ6XR5yEawRN2ekxx9IKgPN5uNAYco7MCPxtBEWMKO3UKWa1X2CtQMzMgloeGj2g2o6cue3Sb5iPkByIJqlw==", + "deprecated": "Use pkg.json instead.", + "dependencies": { + "git-package-json": "^1.4.0", + "git-source": "^1.1.0", + "package-json": "^2.3.1" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true + }, + "node_modules/parse-bmfont-ascii": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz", + "integrity": "sha512-U4RrVsUFCleIOBsIGYOMKjn9PavsGOXxbvYGtMOEfnId0SVNsgehXh1DxUdVPLoxd5mvcEtvmKs2Mmf0Mpa1ZA==", + "dev": true, + "optional": true + }, + "node_modules/parse-bmfont-binary": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-binary/-/parse-bmfont-binary-1.0.6.tgz", + "integrity": "sha512-GxmsRea0wdGdYthjuUeWTMWPqm2+FAd4GI8vCvhgJsFnoGhTrLhXDDupwTo7rXVAgaLIGoVHDZS9p/5XbSqeWA==", + "dev": true, + "optional": true + }, + "node_modules/parse-bmfont-xml": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/parse-bmfont-xml/-/parse-bmfont-xml-1.1.4.tgz", + "integrity": "sha512-bjnliEOmGv3y1aMEfREMBJ9tfL3WR0i0CKPj61DnSLaoxWR3nLrsQrEbCId/8rF4NyRF0cCqisSVXyQYWM+mCQ==", + "dev": true, + "optional": true, + "dependencies": { + "xml-parse-from-string": "^1.0.0", + "xml2js": "^0.4.5" + } + }, + "node_modules/parse-headers": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.5.tgz", + "integrity": "sha512-ft3iAoLOB/MlwbNXgzy43SWGP6sQki2jQvAyBg/zDFAgr9bfNWZIUj42Kw2eJIl8kEi4PbgE6U1Zau/HwI75HA==", + "dev": true, + "optional": true + }, + "node_modules/parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==", + "dependencies": { + "error-ex": "^1.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parse-url": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/parse-url/-/parse-url-1.3.11.tgz", + "integrity": "sha512-1wj9nkgH/5EboDxLwaTMGJh3oH3f+Gue+aGdh631oCqoSBpokzmMmOldvOeBPtB8GJBYJbaF93KPzlkU+Y1ksg==", + "dependencies": { + "is-ssh": "^1.3.0", + "protocols": "^1.4.0" + } + }, + "node_modules/parse-url/node_modules/protocols": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/protocols/-/protocols-1.4.8.tgz", + "integrity": "sha512-IgjKyaUSjsROSO8/D49Ab7hP8mJgTYcqApOqdPhLoPxAplXmkp+zRvsrSQjFn5by0rhm4VH0GAUELIPpx7B1yg==" + }, + "node_modules/path-exists": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", + "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, + "node_modules/peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/phin": { + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/phin/-/phin-2.9.3.tgz", + "integrity": "sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==", + "dev": true, + "optional": true + }, + "node_modules/pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "dependencies": { + "pinkie": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pixelmatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-4.0.2.tgz", + "integrity": "sha512-J8B6xqiO37sU/gkcMglv6h5Jbd9xNER7aHzpfRdNmV4IbQBzBpe4l9XmbG+xPF/znacgu2jfEw+wHffaq/YkXA==", + "dev": true, + "optional": true, + "dependencies": { + "pngjs": "^3.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, + "node_modules/pixelmatch/node_modules/pngjs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==", + "dev": true, + "optional": true, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=12.13.0" + } + }, + "node_modules/prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha512-PhmXi5XmoyKw1Un4E+opM2KcsJInDvKyuOumcjjw3waw86ZNjHwVUOOWLc4bCzLdcKNaWBH9e99sbWzDQsVaYg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "optional": true, + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "node_modules/protocols": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.1.tgz", + "integrity": "sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==" + }, + "node_modules/psl": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", + "dev": true + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true + }, + "node_modules/r-json": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/r-json/-/r-json-1.2.10.tgz", + "integrity": "sha512-hu9vyLjSlHXT62NAS7DjI9WazDlvjN0lgp3n431dCVnirVcLkZIpzSwA3orhZEKzdDD2jqNYI+w0yG0aFf4kpA==" + }, + "node_modules/r-package-json": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/r-package-json/-/r-package-json-1.0.9.tgz", + "integrity": "sha512-G4Vpf1KImWmmPFGdtWQTU0L9zk0SjqEC4qs/jE7AQ+Ylmr5kizMzGeC4wnHp5+ijPqNN+2ZPpvyjVNdN1CDVcg==", + "dependencies": { + "package-json-path": "^1.0.0", + "r-json": "^1.2.1" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/read-all-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/read-all-stream/-/read-all-stream-3.1.0.tgz", + "integrity": "sha512-DI1drPHbmBcUDWrJ7ull/F2Qb8HkwBncVx8/RpKYFSIACYaVRQReISYPdZz/mt1y1+qMCOrfReTopERmaxtP6w==", + "dependencies": { + "pinkie-promise": "^2.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "dev": true, + "optional": true, + "dependencies": { + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/readable-web-to-node-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "optional": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "dev": true, + "optional": true + }, + "node_modules/registry-auth-token": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", + "integrity": "sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==", + "dependencies": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha512-ZbgR5aZEdf4UKZVBPYIgaglBmSF2Hi94s2PcIHhRGFjKYu+chjJdYfHn4rt3hB6eCKLJ8giVIIfgMa1ehDfZKA==", + "dependencies": { + "rc": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, + "node_modules/resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/sax": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz", + "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==", + "dev": true, + "optional": true + }, + "node_modules/selenium-webdriver": { + "version": "4.15.0", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.15.0.tgz", + "integrity": "sha512-BNG1bq+KWiBGHcJ/wULi0eKY0yaDqFIbEmtbsYJmfaEghdCkXBsx1akgOorhNwjBipOr0uwpvNXqT6/nzl+zjg==", + "dev": true, + "dependencies": { + "jszip": "^3.10.1", + "tmp": "^0.2.1", + "ws": ">=8.14.2" + }, + "engines": { + "node": ">= 14.20.0" + } + }, + "node_modules/selenium-webdriver/node_modules/tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "dependencies": { + "rimraf": "^3.0.0" + }, + "engines": { + "node": ">=8.17.0" + } + }, + "node_modules/selenium-webdriver/node_modules/ws": { + "version": "8.14.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz", + "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==", + "dev": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/semver": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "dev": true + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/sliced": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", + "integrity": "sha512-VZBmZP8WU3sMOZm1bdgTadsQbcscK0UM8oKxKVBs4XAhUo2Xxzm/OFMGBkPusxw9xL3Uy8LrzEqGqJhclsr0yA==" + }, + "node_modules/spdx-correct": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", + "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", + "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==" + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.16", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.16.tgz", + "integrity": "sha512-eWN+LnM3GR6gPu35WxNgbGl8rmY1AEmoMDvL/QD6zYmPWgywxWqJWNdLGT+ke8dKNWrcYgYjPpG5gbTfghP8rw==" + }, + "node_modules/split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "dev": true, + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/strftime": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.2.tgz", + "integrity": "sha512-Y6IZaTVM80chcMe7j65Gl/0nmlNdtt+KWPle5YeCAjmsBfw+id2qdaJ5MDrxUq+OmHKab+jHe7mUjU/aNMSZZg==", + "dev": true, + "engines": { + "node": ">=0.2.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "dev": true, + "optional": true, + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/symbol": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/symbol/-/symbol-0.3.1.tgz", + "integrity": "sha512-SxMrE6uv9zhnBmTCpZna1u0TcZix1k2QASZ/DpF13rAo+0Ts40faFYsMTuAirgvbbjHw1byhJ949/fP20XzVZA==", + "dev": true + }, + "node_modules/tar": { + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==", + "dev": true, + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^4.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, + "node_modules/timed-out": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-2.0.0.tgz", + "integrity": "sha512-pqqJOi1rF5zNs/ps4vmbE4SFCrM4iR7LW+GHAsHqO/EumqbIWceioevYLM5xZRgQSH6gFgL9J/uB7EcJhQ9niQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/timm": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/timm/-/timm-1.7.1.tgz", + "integrity": "sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw==", + "dev": true, + "optional": true + }, + "node_modules/tinycolor2": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", + "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", + "dev": true, + "optional": true + }, + "node_modules/tmp": { + "version": "0.0.28", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz", + "integrity": "sha512-c2mmfiBmND6SOVxzogm1oda0OJ1HZVIk/5n26N59dDTh80MUeavpiCls4PGAdkX1PFkKokLpcf7prSjCeXLsJg==", + "dependencies": { + "os-tmpdir": "~1.0.1" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "dev": true, + "optional": true, + "dependencies": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/tough-cookie": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz", + "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==", + "dev": true, + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "optional": true + }, + "node_modules/typpy": { + "version": "2.3.13", + "resolved": "https://registry.npmjs.org/typpy/-/typpy-2.3.13.tgz", + "integrity": "sha512-vOxIcQz9sxHi+rT09SJ5aDgVgrPppQjwnnayTrMye1ODaU8gIZTDM19t9TxmEElbMihx2Nq/0/b/MtyKfayRqA==", + "dependencies": { + "function.name": "^1.0.3" + } + }, + "node_modules/ul": { + "version": "5.2.15", + "resolved": "https://registry.npmjs.org/ul/-/ul-5.2.15.tgz", + "integrity": "sha512-svLEUy8xSCip5IWnsRa0UOg+2zP0Wsj4qlbjTmX6GJSmvKMHADBuHOm1dpNkWqWPIGuVSqzUkV3Cris5JrlTRQ==", + "dependencies": { + "deffy": "^2.2.2", + "typpy": "^2.3.4" + } + }, + "node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/unzip-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-1.0.2.tgz", + "integrity": "sha512-pwCcjjhEcpW45JZIySExBHYv5Y9EeL2OIGEfrSKp2dMUFGFv4CpvZkwJbVge8OvGH2BNNtJBx67DuKuJhf+N5Q==", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/url-parse-lax": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", + "integrity": "sha512-BVA4lR5PIviy2PMseNd2jbFQ+jwSwQGdJejf5ctd1rEXt0Ypd7yanUK9+lYechVlN5VaTJGsu2U/3MDDu6KgBA==", + "dependencies": { + "prepend-http": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/utcstring": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/utcstring/-/utcstring-0.1.0.tgz", + "integrity": "sha512-1EpWQ6CECkoys7aX3LImrFo4nYIigY2RQHJTvgzZQCB4/oA6jJvTLTcgilTxX57GrSHDIVMtGwYd+SujGJvvyw==", + "dev": true + }, + "node_modules/utif2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/utif2/-/utif2-4.1.0.tgz", + "integrity": "sha512-+oknB9FHrJ7oW7A2WZYajOcv4FcDR4CfoGB0dPNfxbi4GO05RRnFmt5oa23+9w32EanrYcSJWspUiJkLMs+37w==", + "dev": true, + "optional": true, + "dependencies": { + "pako": "^1.0.11" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "optional": true + }, + "node_modules/whatwg-fetch": { + "version": "3.6.19", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.19.tgz", + "integrity": "sha512-d67JP4dHSbm2TrpFj8AbO8DnL1JXL5J9u0Kq2xW6d0TFDbCA3Muhdt8orXC22utleTVj7Prqt82baN6RBvnEgw==", + "dev": true, + "optional": true + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "optional": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "dev": true, + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xhr": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/xhr/-/xhr-2.6.0.tgz", + "integrity": "sha512-/eCGLb5rxjx5e3mF1A7s+pLlR6CGyqWN91fv1JgER5mVWg1MZmlhBvy9kjcsOdRk8RrIujotWyJamfyrp+WIcA==", + "dev": true, + "optional": true, + "dependencies": { + "global": "~4.4.0", + "is-function": "^1.0.1", + "parse-headers": "^2.0.0", + "xtend": "^4.0.0" + } + }, + "node_modules/xml-parse-from-string": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz", + "integrity": "sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==", + "dev": true, + "optional": true + }, + "node_modules/xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "dev": true, + "optional": true, + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "dev": true, + "optional": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, + "optional": true, + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "dev": true, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + }, + "dependencies": { + "@cypress/xvfb": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@cypress/xvfb/-/xvfb-1.2.4.tgz", + "integrity": "sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q==", + "dev": true, + "requires": { + "debug": "^3.1.0", + "lodash.once": "^4.1.1" + } + }, + "@devicefarmer/adbkit": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit/-/adbkit-3.2.5.tgz", + "integrity": "sha512-+J479WWZW3GU3t40flicDfiDrFz6vpiy2RcBQPEhFcs/3La9pOtr4Bgz2Q02E4luUG2RAL068rqIkKNUTy3tZw==", + "dev": true, + "requires": { + "@devicefarmer/adbkit-logcat": "^2.1.2", + "@devicefarmer/adbkit-monkey": "~1.2.1", + "bluebird": "~3.7", + "commander": "^9.1.0", + "debug": "~4.3.1", + "node-forge": "^1.3.1", + "split": "~1.0.1" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "@devicefarmer/adbkit-logcat": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit-logcat/-/adbkit-logcat-2.1.3.tgz", + "integrity": "sha512-yeaGFjNBc/6+svbDeul1tNHtNChw6h8pSHAt5D+JsedUrMTN7tla7B15WLDyekxsuS2XlZHRxpuC6m92wiwCNw==", + "dev": true + }, + "@devicefarmer/adbkit-monkey": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit-monkey/-/adbkit-monkey-1.2.1.tgz", + "integrity": "sha512-ZzZY/b66W2Jd6NHbAhLyDWOEIBWC11VizGFk7Wx7M61JZRz7HR9Cq5P+65RKWUU7u6wgsE8Lmh9nE4Mz+U2eTg==", + "dev": true + }, + "@jimp/bmp": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.22.10.tgz", + "integrity": "sha512-1UXRl1Nw1KptZ1r0ANqtXOst9vGH51dq7keVKQzyyTO2lz4dOaezS9StuSTNh+RmiHg/SVPaFRpPfB0S/ln4Kg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10", + "bmp-js": "^0.1.0" + } + }, + "@jimp/core": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.22.10.tgz", + "integrity": "sha512-ZKyrehVy6wu1PnBXIUpn/fXmyMRQiVSbvHDubgXz4bfTOao3GiOurKHjByutQIgozuAN6ZHWiSge1dKA+dex3w==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10", + "any-base": "^1.1.0", + "buffer": "^5.2.0", + "exif-parser": "^0.1.12", + "file-type": "^16.5.4", + "isomorphic-fetch": "^3.0.0", + "pixelmatch": "^4.0.2", + "tinycolor2": "^1.6.0" + } + }, + "@jimp/custom": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.22.10.tgz", + "integrity": "sha512-sPZkUYe1hu0iIgNisjizxPJqq2vaaKvkCkPoXq2U6UV3ZA1si/WVdrg25da3IcGIEV+83AoHgM8TvqlLgrCJsg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/core": "^0.22.10" + } + }, + "@jimp/gif": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.22.10.tgz", + "integrity": "sha512-yEX2dSpamvkSx1PPDWGnKeWDrBz0vrCKjVG/cn4Zr68MRRT75tbZIeOrBa+RiUpY3ho5ix7d36LkYvt3qfUIhQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10", + "gifwrap": "^0.10.1", + "omggif": "^1.0.9" + } + }, + "@jimp/jpeg": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.22.10.tgz", + "integrity": "sha512-6bu98pAcVN4DY2oiDLC4TOgieX/lZrLd1tombWZOFCN5PBmqaHQxm7IUmT+Wj4faEvh8QSHgVLSA+2JQQRJWVA==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10", + "jpeg-js": "^0.4.4" + } + }, + "@jimp/plugin-blit": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.22.10.tgz", + "integrity": "sha512-6EI8Sl+mxYHEIy6Yteh6eknD+EZguKpNdr3sCKxNezmLR0+vK99vHcllo6uGSjXXiwtwS67Xqxn8SsoatL+UJQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-blur": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.22.10.tgz", + "integrity": "sha512-4XRTWuPVdMXJeclJMisXPGizeHtTryVaVV5HnuQXpKqIZtzXReCCpNGH8q/i0kBQOQMXhGWS3mpqOEwtpPePKw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-circle": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.22.10.tgz", + "integrity": "sha512-mhcwTO1ywRxiCgtLGge6tDDIDPlX6qkI3CY+BjgGG/XhVHccCddXgOGLdlf+5OuKIEF2Nqs0V01LQEQIJFTmEw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-color": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.22.10.tgz", + "integrity": "sha512-e4t3L7Kedd96E0x1XjsTM6NcgulKUU66HdFTao7Tc9FYJRFSlttARZ/C6LEryGDm/i69R6bJEpo7BkNz0YL55Q==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10", + "tinycolor2": "^1.6.0" + } + }, + "@jimp/plugin-contain": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.22.10.tgz", + "integrity": "sha512-eP8KrzctuEoqibQAxi9WhbnoRosydhiwg+IYya3dKuKDBTrD9UHt+ERlPQ/lTNWHzV/l4S1ntV3r9s9saJgsXA==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-cover": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.22.10.tgz", + "integrity": "sha512-kJCwL5T1igfa0InCfkE7bBeqg26m46aoRt10ug+rvm11P6RrvRMGrgINFyIKB+mnB7CiyBN/MOula1CvLhSInQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-crop": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.22.10.tgz", + "integrity": "sha512-BOZ+YGaZlhU7c5ye65RxikicXH0Ki0It6/XHISvipR5WZrfjLjL2Ke20G+AGnwBQc76gKenVcMXVUCnEjtZV+Q==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-displace": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.22.10.tgz", + "integrity": "sha512-llNiWWMTKISDXt5+cXI0GaFmZWAjlT+4fFLYf4eXquuL/9wZoQsEBhv2GdGd48mkiS8jZq1Nnb2Q4ehEPTvrzw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-dither": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.22.10.tgz", + "integrity": "sha512-05WLmeV5M+P/0FS+bWf13hMew2X0oa8w9AtmevL2UyA/5GqiyvP2Xm5WfGQ8oFiiMvpnL6RFomJQOZtWca0C2w==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-fisheye": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.22.10.tgz", + "integrity": "sha512-InjiXvc7Gkzrx8VWtU97kDqV7ENnhHGPULymJWeZaF2aicud9Fpk4iCtd/DcZIrk7Cbe60A8RwNXN00HXIbSCg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-flip": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.22.10.tgz", + "integrity": "sha512-42GkGtTHWnhnwTMPVK/kXObZbkYIpQWfuIfy5EMEMk6zRj05zpv4vsjkKWfuemweZINwfvD7wDJF7FVFNNcZZg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-gaussian": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.22.10.tgz", + "integrity": "sha512-ykrG/6lTp9Q5YA8jS5XzwMHtRxb9HOFMgtmnrUZ8kU+BK8REecfy9Ic5BUEOjCYvS1a/xLsnrZQU07iiYxBxFg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-invert": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.22.10.tgz", + "integrity": "sha512-d8j9BlUJYs/c994t4azUWSWmQq4LLPG4ecm8m6SSNqap+S/HlVQGqjYhJEBbY9EXkOTYB9vBL9bqwSM1Rr6paA==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-mask": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.22.10.tgz", + "integrity": "sha512-yRBs1230XZkz24uFTdTcSlZ0HXZpIWzM3iFQN56MzZ7USgdVZjPPDCQ8I9RpqfZ36nDflQkUO0wV7ucsi4ogow==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-normalize": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.22.10.tgz", + "integrity": "sha512-Wk9GX6eJMchX/ZAazVa70Fagu+OXMvHiPY+HrcEwcclL+p1wo8xAHEsf9iKno7Ja4EU9lLhbBRY5hYJyiKMEkg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-print": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.22.10.tgz", + "integrity": "sha512-1U3VloIR+beE1kWPdGEJMiE2h1Do29iv3w8sBbvPyRP4qXxRFcDpmCGtctsrKmb1krlBFlj8ubyAY90xL+5n9w==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10", + "load-bmfont": "^1.4.1" + } + }, + "@jimp/plugin-resize": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.22.10.tgz", + "integrity": "sha512-ixomxVcnAONXDgaq0opvAx4UAOiEhOA/tipuhFFOvPKFd4yf1BAnEviB5maB0SBHHkJXPUSzDp/73xVTMGSe7g==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-rotate": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.22.10.tgz", + "integrity": "sha512-eeFX8dnRyf3LAdsdXWKWuN18hLRg8zy1cP0cP9rHzQVWRK7ck/QsLxK1vHq7MADGwQalNaNTJ9SQxH6c8mz6jw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-scale": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.22.10.tgz", + "integrity": "sha512-TG/H0oUN69C9ArBCZg4PmuoixFVKIiru8282KzSB/Tp1I0xwX0XLTv3dJ5pobPlIgPcB+TmD4xAIdkCT4rtWxg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-shadow": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.22.10.tgz", + "integrity": "sha512-TN9xm6fI7XfxbMUQqFPZjv59Xdpf0tSiAQdINB4g6pJMWiVANR/74OtDONoy3KKpenu5Y38s+FkrtID/KcQAhw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugin-threshold": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.22.10.tgz", + "integrity": "sha512-DA2lSnU0TgIRbAgmXaxroYw3Ad6J2DOFEoJp0NleSm2h3GWbZEE5yW9U2B6hD3iqn4AenG4E2b2WzHXZyzSutw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10" + } + }, + "@jimp/plugins": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.22.10.tgz", + "integrity": "sha512-KDMZyM6pmvS8freB+UBLko1TO/k4D7URS/nphCozuH+P7i3UMe7NdckXKJ8u+WD6sqN0YFYvBehpkpnUiw/91w==", + "dev": true, + "optional": true, + "requires": { + "@jimp/plugin-blit": "^0.22.10", + "@jimp/plugin-blur": "^0.22.10", + "@jimp/plugin-circle": "^0.22.10", + "@jimp/plugin-color": "^0.22.10", + "@jimp/plugin-contain": "^0.22.10", + "@jimp/plugin-cover": "^0.22.10", + "@jimp/plugin-crop": "^0.22.10", + "@jimp/plugin-displace": "^0.22.10", + "@jimp/plugin-dither": "^0.22.10", + "@jimp/plugin-fisheye": "^0.22.10", + "@jimp/plugin-flip": "^0.22.10", + "@jimp/plugin-gaussian": "^0.22.10", + "@jimp/plugin-invert": "^0.22.10", + "@jimp/plugin-mask": "^0.22.10", + "@jimp/plugin-normalize": "^0.22.10", + "@jimp/plugin-print": "^0.22.10", + "@jimp/plugin-resize": "^0.22.10", + "@jimp/plugin-rotate": "^0.22.10", + "@jimp/plugin-scale": "^0.22.10", + "@jimp/plugin-shadow": "^0.22.10", + "@jimp/plugin-threshold": "^0.22.10", + "timm": "^1.6.1" + } + }, + "@jimp/png": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.22.10.tgz", + "integrity": "sha512-RYinU7tZToeeR2g2qAMn42AU+8OUHjXPKZZ9RkmoL4bguA1xyZWaSdr22/FBkmnHhOERRlr02KPDN1OTOYHLDQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.10", + "pngjs": "^6.0.0" + } + }, + "@jimp/tiff": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.22.10.tgz", + "integrity": "sha512-OaivlSYzpNTHyH/h7pEtl3A7F7TbsgytZs52GLX/xITW92ffgDgT6PkldIrMrET6ERh/hdijNQiew7IoEEr2og==", + "dev": true, + "optional": true, + "requires": { + "utif2": "^4.0.1" + } + }, + "@jimp/types": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.22.10.tgz", + "integrity": "sha512-u/r+XYzbCx4zZukDmxx8S0er3Yq3iDPI6+31WKX0N18i2qPPJYcn8qwIFurfupRumGvJ8SlGLCgt/T+Y8zzUIw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/bmp": "^0.22.10", + "@jimp/gif": "^0.22.10", + "@jimp/jpeg": "^0.22.10", + "@jimp/png": "^0.22.10", + "@jimp/tiff": "^0.22.10", + "timm": "^1.6.1" + } + }, + "@jimp/utils": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.22.10.tgz", + "integrity": "sha512-ztlOK9Mm2iLG2AMoabzM4i3WZ/FtshcgsJCbZCRUs/DKoeS2tySRJTnQZ1b7Roq0M4Ce+FUAxnCAcBV0q7PH9w==", + "dev": true, + "optional": true, + "requires": { + "regenerator-runtime": "^0.13.3" + } + }, + "@sitespeed.io/chromedriver": { + "version": "119.0.6045-105", + "resolved": "https://registry.npmjs.org/@sitespeed.io/chromedriver/-/chromedriver-119.0.6045-105.tgz", + "integrity": "sha512-DfQQaqTB28e05kG3CWjC9OWKeNTWiqgu5cl6CvYQsd2MTDDDRUQ0a+VZ8KTSrRx6xZCsTBgzZK2kNBNiMvNH8w==", + "dev": true, + "requires": { + "node-downloader-helper": "2.1.9", + "node-stream-zip": "1.15.0" + } + }, + "@sitespeed.io/edgedriver": { + "version": "119.0.2151-42", + "resolved": "https://registry.npmjs.org/@sitespeed.io/edgedriver/-/edgedriver-119.0.2151-42.tgz", + "integrity": "sha512-+jGP9BmWgh/yoNcJKyiYP0anF0m2H6+cjk1MaHvzgkIdrFMVfJQIN9+tmwCBiN4Ave52IHjDdHhEjK7B+SWvrA==", + "dev": true, + "requires": { + "node-downloader-helper": "2.1.7", + "node-stream-zip": "1.15.0" + }, + "dependencies": { + "node-downloader-helper": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.7.tgz", + "integrity": "sha512-3dBuMF/XPy5WFi3XiiXaglafzoycRH5GjmRz1nAt2uI9D+TcBrc+n/AzH8bzLHR85Wsf6vZSZblzw+MiUS/WNQ==", + "dev": true + } + } + }, + "@sitespeed.io/geckodriver": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/@sitespeed.io/geckodriver/-/geckodriver-0.33.0.tgz", + "integrity": "sha512-w6w+x9/Q44JekTPi8NlRsfh5Uz4TfquJcUEs0tA/oEcxLVxRS7VtaiaJEE0GzzN6cUmFS6Twas7E4bCA4k/Yxg==", + "dev": true, + "requires": { + "node-downloader-helper": "2.1.5", + "node-stream-zip": "1.15.0", + "tar": "6.1.13" + }, + "dependencies": { + "node-downloader-helper": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.5.tgz", + "integrity": "sha512-sLedzfv8C4VMAvTdDQcjLFAl3gydNeBXh2bLcCzvZRmd4EK0rkoTxJ8tkxnriUSJO/n13skJzH7l6CzCdBwYGg==", + "dev": true + } + } + }, + "@sitespeed.io/throttle": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@sitespeed.io/throttle/-/throttle-5.0.0.tgz", + "integrity": "sha512-eul4I7IllA6l3+GGX1aW/D75XYux0ODuZDzstKD0kAuvIkpQ4BVLkFBoLXQN50gLMFGqZ3QWMobhQ5L2/6sFgg==", + "dev": true, + "requires": { + "minimist": "1.2.6" + } + }, + "@sitespeed.io/tracium": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@sitespeed.io/tracium/-/tracium-0.3.3.tgz", + "integrity": "sha512-dNZafjM93Y+F+sfwTO5gTpsGXlnc/0Q+c2+62ViqP3gkMWvHEMSKkaEHgVJLcLg3i/g19GSIPziiKpgyne07Bw==", + "dev": true, + "requires": { + "debug": "^4.1.1" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "dev": true, + "optional": true + }, + "@types/node": { + "version": "16.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", + "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==", + "dev": true, + "optional": true + }, + "abs": { + "version": "1.3.14", + "resolved": "https://registry.npmjs.org/abs/-/abs-1.3.14.tgz", + "integrity": "sha512-PrS26IzwKLWwuURpiKl8wRmJ2KdR/azaVrLEBWG/TALwT20Y7qjtYp1qcMLHA4206hBHY5phv3w4pjf9NPv4Vw==", + "requires": { + "ul": "^5.0.0" + } + }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "dev": true + }, + "any-base": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz", + "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==", + "dev": true, + "optional": true + }, + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha512-nSVgobk4rv61R9PUSDtYt7mPVB2olxNR5RWJcAsH676/ef11bUZwvu7+RGYrYauVdDPcO519v68wRhXQtxsV9w==", + "dev": true + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "optional": true + }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "dev": true + }, + "bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "dev": true, + "optional": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "browsertime": { + "version": "https://github.com/sitespeedio/browsertime/tarball/62de4fc9abc8067fb58378999b1bc4a4c42f9eb5", + "integrity": "sha512-dtX8pNd4HLQIBBphbTs4Ok0FTt/+zgikbjxI0B2YEjzOEtbSI//ofn4woOYdIC7JOiTtKhYB79eqXaIbVXORqw==", + "dev": true, + "requires": { + "@cypress/xvfb": "1.2.4", + "@devicefarmer/adbkit": "3.2.5", + "@sitespeed.io/chromedriver": "119.0.6045-105", + "@sitespeed.io/edgedriver": "119.0.2151-42", + "@sitespeed.io/geckodriver": "0.33.0", + "@sitespeed.io/throttle": "5.0.0", + "@sitespeed.io/tracium": "0.3.3", + "btoa": "1.2.1", + "chrome-har": "0.13.2", + "chrome-remote-interface": "0.33.0", + "dayjs": "1.11.10", + "execa": "8.0.1", + "fast-stats": "0.0.6", + "ff-test-bidi-har-export": "0.0.12", + "find-up": "6.3.0", + "get-port": "7.0.0", + "hasbin": "1.2.3", + "intel": "1.2.0", + "jimp": "0.22.10", + "lodash.get": "4.4.2", + "lodash.groupby": "4.6.0", + "lodash.isempty": "4.4.0", + "lodash.merge": "4.6.2", + "lodash.pick": "4.4.0", + "lodash.set": "4.3.2", + "selenium-webdriver": "4.15.0", + "yargs": "17.7.2" + } + }, + "btoa": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", + "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==", + "dev": true + }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "optional": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "buffer-equal": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz", + "integrity": "sha512-RgSV6InVQ9ODPdLWJ5UAqBqJBOg370Nz6ZQtRzpt6nUjc8v0St97uJ4PYC6NztqIScrAXafKM3mZPMygSe1ggA==", + "dev": true, + "optional": true + }, + "capture-stack-trace": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.2.tgz", + "integrity": "sha512-X/WM2UQs6VMHUtjUDnZTRI+i1crWteJySFzr9UpGoQa4WQffXVTTXuekjl7TjZRlcF2XfjgITT0HxZ9RnxeT0w==" + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "chrome-har": { + "version": "0.13.2", + "resolved": "https://registry.npmjs.org/chrome-har/-/chrome-har-0.13.2.tgz", + "integrity": "sha512-QiwyoilXiGVLG9Y0UMzWOyuao/PctTU9AAOTMqH7BuuulY1e0foDZ/O9qmLfdBAe6MbwIl9aDYvrlbyna3uRZw==", + "dev": true, + "requires": { + "dayjs": "1.11.7", + "debug": "4.3.4", + "tough-cookie": "4.1.3", + "uuid": "9.0.0" + }, + "dependencies": { + "dayjs": { + "version": "1.11.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz", + "integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==", + "dev": true + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "chrome-remote-interface": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/chrome-remote-interface/-/chrome-remote-interface-0.33.0.tgz", + "integrity": "sha512-tv/SgeBfShXk43fwFpQ9wnS7mOCPzETnzDXTNxCb6TqKOiOeIfbrJz+2NAp8GmzwizpKa058wnU1Te7apONaYg==", + "dev": true, + "requires": { + "commander": "2.11.x", + "ws": "^7.2.0" + }, + "dependencies": { + "commander": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", + "dev": true + } + } + }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "create-error-class": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", + "integrity": "sha512-gYTKKexFO3kh200H1Nit76sRwRtOY32vQd3jpAQKpLtZqyNsSQNfI4N7o3eP2wUjV35pTWKRYqFUDBvUha/Pkw==", + "requires": { + "capture-stack-trace": "^1.0.0" + } + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "dayjs": { + "version": "1.11.10", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", + "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==", + "dev": true + }, + "dbug": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/dbug/-/dbug-0.4.2.tgz", + "integrity": "sha512-nrmsMK1msY0WXwfA2czrKVDgpIYJR2JJaq5cX4DwW7Rxm11nXHqouh9wmubEs44bHYxk8CqeP/Jx4URqSB961w==", + "dev": true + }, + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, + "deffy": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/deffy/-/deffy-2.2.4.tgz", + "integrity": "sha512-pLc9lsbsWjr6RxmJ2OLyvm+9l4j1yK69h+TML/gUit/t3vTijpkNGh8LioaJYTGO7F25m6HZndADcUOo2PsiUg==", + "requires": { + "typpy": "^2.0.0" + } + }, + "dom-walk": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.2.tgz", + "integrity": "sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w==", + "dev": true, + "optional": true + }, + "duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "requires": { + "readable-stream": "^2.0.2" + } + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "err": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/err/-/err-1.1.1.tgz", + "integrity": "sha512-N97Ybd2jJHVQ+Ft3Q5+C2gM3kgygkdeQmEqbN2z15UTVyyEsIwLA1VK39O1DHEJhXbwIFcJLqm6iARNhFANcQA==", + "requires": { + "typpy": "^2.2.0" + } + }, + "error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true + }, + "exec-limiter": { + "version": "3.2.13", + "resolved": "https://registry.npmjs.org/exec-limiter/-/exec-limiter-3.2.13.tgz", + "integrity": "sha512-86Ri699bwiHZVBzTzNj8gspqAhCPchg70zPVWIh3qzUOA1pUMcb272Em3LPk8AE0mS95B9yMJhtqF8vFJAn0dA==", + "requires": { + "limit-it": "^3.0.0", + "typpy": "^2.1.0" + } + }, + "execa": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", + "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "get-stream": "^8.0.1", + "human-signals": "^5.0.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^3.0.0" + } + }, + "exif-parser": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz", + "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==", + "dev": true, + "optional": true + }, + "fast-stats": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/fast-stats/-/fast-stats-0.0.6.tgz", + "integrity": "sha512-m0zkwa7Z07Wc4xm1YtcrCHmhzNxiYRrrfUyhkdhSZPzaAH/Ewbocdaq7EPVBFz19GWfIyyPcLfRHjHJYe83jlg==", + "dev": true + }, + "ff-test-bidi-har-export": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/ff-test-bidi-har-export/-/ff-test-bidi-har-export-0.0.12.tgz", + "integrity": "sha512-ccJZc14x/1ymgcLpUBz52Rci/UsbboqJ5wgiPrcHQMyh8YOwNJLGt3yGygIHNhiShZ8aA8H4jOmQU980Ngot9Q==", + "dev": true + }, + "file-type": { + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "dev": true, + "optional": true, + "requires": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + } + }, + "find-up": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", + "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==", + "dev": true, + "requires": { + "locate-path": "^7.1.0", + "path-exists": "^5.0.0" + } + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + } + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" + }, + "function.name": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/function.name/-/function.name-1.0.13.tgz", + "integrity": "sha512-mVrqdoy5npWZyoXl4DxCeuVF6delDcQjVS9aPdvLYlBxtMTZDR2B5GVEQEoM1jJyspCqg3C0v4ABkLE7tp9xFA==", + "requires": { + "noop6": "^1.0.1" + } + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "get-port": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-7.0.0.tgz", + "integrity": "sha512-mDHFgApoQd+azgMdwylJrv2DX47ywGq1i5VFJE7fZ0dttNq3iQMfsU4IvEgBHojA3KqEudyu7Vq+oN8kNaNkWw==", + "dev": true + }, + "get-stream": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", + "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", + "dev": true + }, + "gifwrap": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/gifwrap/-/gifwrap-0.10.1.tgz", + "integrity": "sha512-2760b1vpJHNmLzZ/ubTtNnEx5WApN/PYWJvXvgS+tL1egTTthayFYIQQNi136FLEDcN/IyEY2EcGpIITD6eYUw==", + "dev": true, + "optional": true, + "requires": { + "image-q": "^4.0.0", + "omggif": "^1.0.10" + } + }, + "git-package-json": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/git-package-json/-/git-package-json-1.4.10.tgz", + "integrity": "sha512-DRAcvbzd2SxGK7w8OgYfvKqhFliT5keX0lmSmVdgScgf1kkl5tbbo7Pam6uYoCa1liOiipKxQZG8quCtGWl/fA==", + "requires": { + "deffy": "^2.2.1", + "err": "^1.1.1", + "gry": "^5.0.0", + "normalize-package-data": "^2.3.5", + "oargv": "^3.4.1", + "one-by-one": "^3.1.0", + "r-json": "^1.2.1", + "r-package-json": "^1.0.0", + "tmp": "0.0.28" + } + }, + "git-source": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/git-source/-/git-source-1.1.10.tgz", + "integrity": "sha512-XZZ7ZgnLL35oLgM/xjnLYgtlKlxJG0FohC1kWDvGkU7s1VKGXK0pFF/g1itQEwQ3D+uTQzBnzPi8XbqOv7Wc1Q==", + "requires": { + "git-url-parse": "^5.0.1" + } + }, + "git-up": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/git-up/-/git-up-1.2.1.tgz", + "integrity": "sha512-SRVN3rOLACva8imc7BFrB6ts5iISWKH1/h/1Z+JZYoUI7UVQM7gQqk4M2yxUENbq2jUUT09NEND5xwP1i7Ktlw==", + "requires": { + "is-ssh": "^1.0.0", + "parse-url": "^1.0.0" + } + }, + "git-url-parse": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/git-url-parse/-/git-url-parse-5.0.1.tgz", + "integrity": "sha512-4uSiOgrryNEMBX+gTWogenYRUh2j1D+95STTSEF2RCTgLkfJikl8c7BGr0Bn274hwuxTsbS2/FQ5pVS9FoXegQ==", + "requires": { + "git-up": "^1.0.0" + } + }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "global": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", + "integrity": "sha512-wv/LAoHdRE3BeTGz53FAamhGlPLhlssK45usmGFThIi4XqnBmjKQ16u+RNbP7WvigRZDxUsM0J3gcQ5yicaL0w==", + "dev": true, + "optional": true, + "requires": { + "min-document": "^2.19.0", + "process": "^0.11.10" + } + }, + "got": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-5.6.0.tgz", + "integrity": "sha512-MnypzkaW8dldA8AbJFjMs7y14+ykd2V8JCLKSvX1Gmzx1alH3Y+3LArywHDoAF2wS3pnZp4gacoYtvqBeF6drQ==", + "requires": { + "create-error-class": "^3.0.1", + "duplexer2": "^0.1.4", + "is-plain-obj": "^1.0.0", + "is-redirect": "^1.0.0", + "is-retry-allowed": "^1.0.0", + "is-stream": "^1.0.0", + "lowercase-keys": "^1.0.0", + "node-status-codes": "^1.0.0", + "object-assign": "^4.0.1", + "parse-json": "^2.1.0", + "pinkie-promise": "^2.0.0", + "read-all-stream": "^3.0.0", + "readable-stream": "^2.0.5", + "timed-out": "^2.0.0", + "unzip-response": "^1.0.0", + "url-parse-lax": "^1.0.0" + }, + "dependencies": { + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==" + } + } + }, + "gry": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/gry/-/gry-5.0.8.tgz", + "integrity": "sha512-meq9ZjYVpLzZh3ojhTg7IMad9grGsx6rUUKHLqPnhLXzJkRQvEL2U3tQpS5/WentYTtHtxkT3Ew/mb10D6F6/g==", + "requires": { + "abs": "^1.2.1", + "exec-limiter": "^3.0.0", + "one-by-one": "^3.0.0", + "ul": "^5.0.0" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "hasbin": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/hasbin/-/hasbin-1.2.3.tgz", + "integrity": "sha512-CCd8e/w2w28G8DyZvKgiHnQJ/5XXDz6qiUHnthvtag/6T5acUeN5lqq+HMoBqcmgWueWDhiCplrw0Kb1zDACRg==", + "dev": true, + "requires": { + "async": "~1.5" + } + }, + "hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "requires": { + "function-bind": "^1.1.2" + } + }, + "hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==" + }, + "human-signals": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", + "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "dev": true + }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "optional": true + }, + "image-q": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/image-q/-/image-q-4.0.0.tgz", + "integrity": "sha512-PfJGVgIfKQJuq3s0tTDOKtztksibuUEbJQIYT3by6wctQo+Rdlh7ef4evJ5NCdxY4CfMbvFkocEwbl4BF8RlJw==", + "dev": true, + "optional": true, + "requires": { + "@types/node": "16.9.1" + } + }, + "immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "intel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/intel/-/intel-1.2.0.tgz", + "integrity": "sha512-CUDyAtEeEeDo5YtwANOuDhxuFEOgInHvbMrBbhXCD4tAaHuzHM2llevtTeq2bmP8Jf7NkpN305pwDncRmhc1Wg==", + "dev": true, + "requires": { + "chalk": "^1.1.0", + "dbug": "~0.4.2", + "stack-trace": "~0.0.9", + "strftime": "~0.10.0", + "symbol": "~0.3.1", + "utcstring": "~0.1.0" + } + }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" + }, + "is-core-module": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "requires": { + "hasown": "^2.0.0" + } + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "is-function": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.2.tgz", + "integrity": "sha512-lw7DUp0aWXYg+CBCN+JKkcE0Q2RayZnSvnZBlwgxHBQhqt5pZNVy4Ri7H9GmmXkdu7LUthszM+Tor1u/2iBcpQ==", + "dev": true, + "optional": true + }, + "is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==" + }, + "is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha512-cr/SlUEe5zOGmzvj9bUyC4LVvkNVAXu4GytXLNMr1pny+a65MpQ9IJzFHD5vi7FyJgb4qt27+eS3TuQnqB+RQw==" + }, + "is-retry-allowed": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", + "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==" + }, + "is-ssh": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/is-ssh/-/is-ssh-1.4.0.tgz", + "integrity": "sha512-x7+VxdxOdlV3CYpjvRLBv5Lo9OJerlYanjwFrPR9fuGPjCiNiCzFgAWpiLAohSbsnH4ZAys3SBh+hq5rJosxUQ==", + "requires": { + "protocols": "^2.0.1" + } + }, + "is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "isomorphic-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", + "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", + "dev": true, + "optional": true, + "requires": { + "node-fetch": "^2.6.1", + "whatwg-fetch": "^3.4.1" + } + }, + "iterate-object": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/iterate-object/-/iterate-object-1.3.4.tgz", + "integrity": "sha512-4dG1D1x/7g8PwHS9aK6QV5V94+ZvyP4+d19qDv43EzImmrndysIl4prmJ1hWWIGCqrZHyaHBm6BSEWHOLnpoNw==" + }, + "jimp": { + "version": "0.22.10", + "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.22.10.tgz", + "integrity": "sha512-lCaHIJAgTOsplyJzC1w/laxSxrbSsEBw4byKwXgUdMmh+ayPsnidTblenQm+IvhIs44Gcuvlb6pd2LQ0wcKaKg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/custom": "^0.22.10", + "@jimp/plugins": "^0.22.10", + "@jimp/types": "^0.22.10", + "regenerator-runtime": "^0.13.3" + } + }, + "jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "dev": true, + "optional": true + }, + "jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "dev": true, + "requires": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "requires": { + "immediate": "~3.0.5" + } + }, + "limit-it": { + "version": "3.2.10", + "resolved": "https://registry.npmjs.org/limit-it/-/limit-it-3.2.10.tgz", + "integrity": "sha512-T0NK99pHnkimldr1WUqvbGV1oWDku/xC9J/OqzJFsV1jeOS6Bwl8W7vkeQIBqwiON9dTALws+rX/XPMQqWerDQ==", + "requires": { + "typpy": "^2.0.0" + } + }, + "load-bmfont": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", + "integrity": "sha512-8UyQoYmdRDy81Brz6aLAUhfZLwr5zV0L3taTQ4hju7m6biuwiWiJXjPhBJxbUQJA8PrkvJ/7Enqmwk2sM14soA==", + "dev": true, + "optional": true, + "requires": { + "buffer-equal": "0.0.1", + "mime": "^1.3.4", + "parse-bmfont-ascii": "^1.0.3", + "parse-bmfont-binary": "^1.0.5", + "parse-bmfont-xml": "^1.1.4", + "phin": "^2.9.1", + "xhr": "^2.0.1", + "xtend": "^4.0.0" + } + }, + "locate-path": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", + "dev": true, + "requires": { + "p-locate": "^6.0.0" + } + }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, + "lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "dev": true + }, + "lodash.isempty": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz", + "integrity": "sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg==", + "dev": true + }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, + "lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "dev": true + }, + "lodash.pick": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", + "integrity": "sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==", + "dev": true + }, + "lodash.set": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", + "integrity": "sha512-4hNPN5jlm/N/HLMCO43v8BXKq9Z7QdAGc/VGrRD61w8gN9g/6jF9A4L1pbUgBLCffi0w9VsXfTOij5x8iTyFvg==", + "dev": true + }, + "lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" + }, + "merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "optional": true + }, + "mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true + }, + "min-document": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", + "integrity": "sha512-9Wy1B3m3f66bPPmU5hdA4DR4PB2OfDU/+GS3yAB7IQozE3tqXaVv2zOjgla7MEGSRv95+ILmOuvhLkOK6wJtCQ==", + "dev": true, + "optional": true, + "requires": { + "dom-walk": "^0.1.0" + } + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + }, + "minipass": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.8.tgz", + "integrity": "sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==", + "dev": true + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + } + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node-downloader-helper": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.9.tgz", + "integrity": "sha512-FSvAol2Z8UP191sZtsUZwHIN0eGoGue3uEXGdWIH5228e9KH1YHXT7fN8Oa33UGf+FbqGTQg3sJfrRGzmVCaJA==", + "dev": true + }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "optional": true, + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "dev": true + }, + "node-status-codes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz", + "integrity": "sha512-1cBMgRxdMWE8KeWCqk2RIOrvUb0XCwYfEsY5/y2NlXyq4Y/RumnOZvTj4Nbr77+Vb2C+kyBoRTdkNOS8L3d/aQ==" + }, + "node-stream-zip": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.15.0.tgz", + "integrity": "sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw==", + "dev": true + }, + "noop6": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/noop6/-/noop6-1.0.9.tgz", + "integrity": "sha512-DB3Hwyd89dPr5HqEPg3YHjzvwh/mCqizC1zZ8vyofqc+TQRyPDnT4wgXXbLGF4z9YAzwwTLi8pNLhGqcbSjgkA==" + }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "requires": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "npm-run-path": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.1.0.tgz", + "integrity": "sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==", + "dev": true, + "requires": { + "path-key": "^4.0.0" + }, + "dependencies": { + "path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true + } + } + }, + "oargv": { + "version": "3.4.10", + "resolved": "https://registry.npmjs.org/oargv/-/oargv-3.4.10.tgz", + "integrity": "sha512-SXaMANv9sr7S/dP0vj0+Ybipa47UE1ntTWQ2rpPRhC6Bsvfl+Jg03Xif7jfL0sWKOYWK8oPjcZ5eJ82t8AP/8g==", + "requires": { + "iterate-object": "^1.1.0", + "ul": "^5.0.0" + } + }, + "obj-def": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/obj-def/-/obj-def-1.0.9.tgz", + "integrity": "sha512-bQ4ya3VYD6FAA1+s6mEhaURRHSmw4+sKaXE6UyXZ1XDYc5D+c7look25dFdydmLd18epUegh398gdDkMUZI9xg==", + "requires": { + "deffy": "^2.2.2" + } + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" + }, + "omggif": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", + "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==", + "dev": true, + "optional": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "one-by-one": { + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/one-by-one/-/one-by-one-3.2.8.tgz", + "integrity": "sha512-HR/pSzZdm46Xqj58K+Bu64kMbSTw8/u77AwWvV+rprO/OsuR++pPlkUJn+SmwqBGRgHKwSKQ974V3uls7crIeQ==", + "requires": { + "obj-def": "^1.0.0", + "sliced": "^1.0.1" + } + }, + "onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "dev": true, + "requires": { + "mimic-fn": "^4.0.0" + } + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==" + }, + "p-limit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", + "dev": true, + "requires": { + "yocto-queue": "^1.0.0" + } + }, + "p-locate": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz", + "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==", + "dev": true, + "requires": { + "p-limit": "^4.0.0" + } + }, + "package-json": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-2.4.0.tgz", + "integrity": "sha512-PRg65iXMTt/uK8Rfh5zvzkUbfAPitF17YaCY+IbHsYgksiLvtzWWTUildHth3mVaZ7871OJ7gtP4LBRBlmAdXg==", + "requires": { + "got": "^5.0.0", + "registry-auth-token": "^3.0.1", + "registry-url": "^3.0.3", + "semver": "^5.1.0" + } + }, + "package-json-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/package-json-path/-/package-json-path-1.0.9.tgz", + "integrity": "sha512-uNu7f6Ef7tQHZRnkyVnCtzdSYVN9uBtge/sG7wzcUaawFWkPYUq67iXxRGrQSg/q0tzxIB8jSyIYUKjG2Jn//A==", + "requires": { + "abs": "^1.2.1" + } + }, + "package.json": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/package.json/-/package.json-2.0.1.tgz", + "integrity": "sha512-pSxZ6XR5yEawRN2ekxx9IKgPN5uNAYco7MCPxtBEWMKO3UKWa1X2CtQMzMgloeGj2g2o6cue3Sb5iPkByIJqlw==", + "requires": { + "git-package-json": "^1.4.0", + "git-source": "^1.1.0", + "package-json": "^2.3.1" + } + }, + "pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true + }, + "parse-bmfont-ascii": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz", + "integrity": "sha512-U4RrVsUFCleIOBsIGYOMKjn9PavsGOXxbvYGtMOEfnId0SVNsgehXh1DxUdVPLoxd5mvcEtvmKs2Mmf0Mpa1ZA==", + "dev": true, + "optional": true + }, + "parse-bmfont-binary": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-binary/-/parse-bmfont-binary-1.0.6.tgz", + "integrity": "sha512-GxmsRea0wdGdYthjuUeWTMWPqm2+FAd4GI8vCvhgJsFnoGhTrLhXDDupwTo7rXVAgaLIGoVHDZS9p/5XbSqeWA==", + "dev": true, + "optional": true + }, + "parse-bmfont-xml": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/parse-bmfont-xml/-/parse-bmfont-xml-1.1.4.tgz", + "integrity": "sha512-bjnliEOmGv3y1aMEfREMBJ9tfL3WR0i0CKPj61DnSLaoxWR3nLrsQrEbCId/8rF4NyRF0cCqisSVXyQYWM+mCQ==", + "dev": true, + "optional": true, + "requires": { + "xml-parse-from-string": "^1.0.0", + "xml2js": "^0.4.5" + } + }, + "parse-headers": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.5.tgz", + "integrity": "sha512-ft3iAoLOB/MlwbNXgzy43SWGP6sQki2jQvAyBg/zDFAgr9bfNWZIUj42Kw2eJIl8kEi4PbgE6U1Zau/HwI75HA==", + "dev": true, + "optional": true + }, + "parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==", + "requires": { + "error-ex": "^1.2.0" + } + }, + "parse-url": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/parse-url/-/parse-url-1.3.11.tgz", + "integrity": "sha512-1wj9nkgH/5EboDxLwaTMGJh3oH3f+Gue+aGdh631oCqoSBpokzmMmOldvOeBPtB8GJBYJbaF93KPzlkU+Y1ksg==", + "requires": { + "is-ssh": "^1.3.0", + "protocols": "^1.4.0" + }, + "dependencies": { + "protocols": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/protocols/-/protocols-1.4.8.tgz", + "integrity": "sha512-IgjKyaUSjsROSO8/D49Ab7hP8mJgTYcqApOqdPhLoPxAplXmkp+zRvsrSQjFn5by0rhm4VH0GAUELIPpx7B1yg==" + } + } + }, + "path-exists": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", + "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==", + "dev": true + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, + "peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", + "dev": true, + "optional": true + }, + "phin": { + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/phin/-/phin-2.9.3.tgz", + "integrity": "sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==", + "dev": true, + "optional": true + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==" + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "requires": { + "pinkie": "^2.0.0" + } + }, + "pixelmatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-4.0.2.tgz", + "integrity": "sha512-J8B6xqiO37sU/gkcMglv6h5Jbd9xNER7aHzpfRdNmV4IbQBzBpe4l9XmbG+xPF/znacgu2jfEw+wHffaq/YkXA==", + "dev": true, + "optional": true, + "requires": { + "pngjs": "^3.0.0" + }, + "dependencies": { + "pngjs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==", + "dev": true, + "optional": true + } + } + }, + "pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "dev": true, + "optional": true + }, + "prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha512-PhmXi5XmoyKw1Un4E+opM2KcsJInDvKyuOumcjjw3waw86ZNjHwVUOOWLc4bCzLdcKNaWBH9e99sbWzDQsVaYg==" + }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "optional": true + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "protocols": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.1.tgz", + "integrity": "sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==" + }, + "psl": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", + "dev": true + }, + "punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true + }, + "querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true + }, + "r-json": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/r-json/-/r-json-1.2.10.tgz", + "integrity": "sha512-hu9vyLjSlHXT62NAS7DjI9WazDlvjN0lgp3n431dCVnirVcLkZIpzSwA3orhZEKzdDD2jqNYI+w0yG0aFf4kpA==" + }, + "r-package-json": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/r-package-json/-/r-package-json-1.0.9.tgz", + "integrity": "sha512-G4Vpf1KImWmmPFGdtWQTU0L9zk0SjqEC4qs/jE7AQ+Ylmr5kizMzGeC4wnHp5+ijPqNN+2ZPpvyjVNdN1CDVcg==", + "requires": { + "package-json-path": "^1.0.0", + "r-json": "^1.2.1" + } + }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + } + }, + "read-all-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/read-all-stream/-/read-all-stream-3.1.0.tgz", + "integrity": "sha512-DI1drPHbmBcUDWrJ7ull/F2Qb8HkwBncVx8/RpKYFSIACYaVRQReISYPdZz/mt1y1+qMCOrfReTopERmaxtP6w==", + "requires": { + "pinkie-promise": "^2.0.0", + "readable-stream": "^2.0.0" + } + }, + "readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "dev": true, + "optional": true, + "requires": { + "readable-stream": "^3.6.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "optional": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "dev": true, + "optional": true + }, + "registry-auth-token": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", + "integrity": "sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==", + "requires": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha512-ZbgR5aZEdf4UKZVBPYIgaglBmSF2Hi94s2PcIHhRGFjKYu+chjJdYfHn4rt3hB6eCKLJ8giVIIfgMa1ehDfZKA==", + "requires": { + "rc": "^1.0.1" + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true + }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, + "resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "requires": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "sax": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.3.0.tgz", + "integrity": "sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==", + "dev": true, + "optional": true + }, + "selenium-webdriver": { + "version": "4.15.0", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.15.0.tgz", + "integrity": "sha512-BNG1bq+KWiBGHcJ/wULi0eKY0yaDqFIbEmtbsYJmfaEghdCkXBsx1akgOorhNwjBipOr0uwpvNXqT6/nzl+zjg==", + "dev": true, + "requires": { + "jszip": "^3.10.1", + "tmp": "^0.2.1", + "ws": ">=8.14.2" + }, + "dependencies": { + "tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "requires": { + "rimraf": "^3.0.0" + } + }, + "ws": { + "version": "8.14.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz", + "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==", + "dev": true, + "requires": {} + } + } + }, + "semver": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==" + }, + "setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "dev": true + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true + }, + "sliced": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", + "integrity": "sha512-VZBmZP8WU3sMOZm1bdgTadsQbcscK0UM8oKxKVBs4XAhUo2Xxzm/OFMGBkPusxw9xL3Uy8LrzEqGqJhclsr0yA==" + }, + "spdx-correct": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", + "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", + "requires": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-exceptions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", + "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==" + }, + "spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "requires": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-license-ids": { + "version": "3.0.16", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.16.tgz", + "integrity": "sha512-eWN+LnM3GR6gPu35WxNgbGl8rmY1AEmoMDvL/QD6zYmPWgywxWqJWNdLGT+ke8dKNWrcYgYjPpG5gbTfghP8rw==" + }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "dev": true, + "requires": { + "through": "2" + } + }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "dev": true + }, + "strftime": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.2.tgz", + "integrity": "sha512-Y6IZaTVM80chcMe7j65Gl/0nmlNdtt+KWPle5YeCAjmsBfw+id2qdaJ5MDrxUq+OmHKab+jHe7mUjU/aNMSZZg==", + "dev": true + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "dev": true + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==" + }, + "strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "dev": true, + "optional": true, + "requires": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "dev": true + }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" + }, + "symbol": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/symbol/-/symbol-0.3.1.tgz", + "integrity": "sha512-SxMrE6uv9zhnBmTCpZna1u0TcZix1k2QASZ/DpF13rAo+0Ts40faFYsMTuAirgvbbjHw1byhJ949/fP20XzVZA==", + "dev": true + }, + "tar": { + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^4.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + } + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, + "timed-out": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-2.0.0.tgz", + "integrity": "sha512-pqqJOi1rF5zNs/ps4vmbE4SFCrM4iR7LW+GHAsHqO/EumqbIWceioevYLM5xZRgQSH6gFgL9J/uB7EcJhQ9niQ==" + }, + "timm": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/timm/-/timm-1.7.1.tgz", + "integrity": "sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw==", + "dev": true, + "optional": true + }, + "tinycolor2": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", + "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", + "dev": true, + "optional": true + }, + "tmp": { + "version": "0.0.28", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz", + "integrity": "sha512-c2mmfiBmND6SOVxzogm1oda0OJ1HZVIk/5n26N59dDTh80MUeavpiCls4PGAdkX1PFkKokLpcf7prSjCeXLsJg==", + "requires": { + "os-tmpdir": "~1.0.1" + } + }, + "token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "dev": true, + "optional": true, + "requires": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + } + }, + "tough-cookie": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz", + "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==", + "dev": true, + "requires": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + } + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "optional": true + }, + "typpy": { + "version": "2.3.13", + "resolved": "https://registry.npmjs.org/typpy/-/typpy-2.3.13.tgz", + "integrity": "sha512-vOxIcQz9sxHi+rT09SJ5aDgVgrPppQjwnnayTrMye1ODaU8gIZTDM19t9TxmEElbMihx2Nq/0/b/MtyKfayRqA==", + "requires": { + "function.name": "^1.0.3" + } + }, + "ul": { + "version": "5.2.15", + "resolved": "https://registry.npmjs.org/ul/-/ul-5.2.15.tgz", + "integrity": "sha512-svLEUy8xSCip5IWnsRa0UOg+2zP0Wsj4qlbjTmX6GJSmvKMHADBuHOm1dpNkWqWPIGuVSqzUkV3Cris5JrlTRQ==", + "requires": { + "deffy": "^2.2.2", + "typpy": "^2.3.4" + } + }, + "universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true + }, + "unzip-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-1.0.2.tgz", + "integrity": "sha512-pwCcjjhEcpW45JZIySExBHYv5Y9EeL2OIGEfrSKp2dMUFGFv4CpvZkwJbVge8OvGH2BNNtJBx67DuKuJhf+N5Q==" + }, + "url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "url-parse-lax": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", + "integrity": "sha512-BVA4lR5PIviy2PMseNd2jbFQ+jwSwQGdJejf5ctd1rEXt0Ypd7yanUK9+lYechVlN5VaTJGsu2U/3MDDu6KgBA==", + "requires": { + "prepend-http": "^1.0.1" + } + }, + "utcstring": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/utcstring/-/utcstring-0.1.0.tgz", + "integrity": "sha512-1EpWQ6CECkoys7aX3LImrFo4nYIigY2RQHJTvgzZQCB4/oA6jJvTLTcgilTxX57GrSHDIVMtGwYd+SujGJvvyw==", + "dev": true + }, + "utif2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/utif2/-/utif2-4.1.0.tgz", + "integrity": "sha512-+oknB9FHrJ7oW7A2WZYajOcv4FcDR4CfoGB0dPNfxbi4GO05RRnFmt5oa23+9w32EanrYcSJWspUiJkLMs+37w==", + "dev": true, + "optional": true, + "requires": { + "pako": "^1.0.11" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "dev": true + }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "requires": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "optional": true + }, + "whatwg-fetch": { + "version": "3.6.19", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.19.tgz", + "integrity": "sha512-d67JP4dHSbm2TrpFj8AbO8DnL1JXL5J9u0Kq2xW6d0TFDbCA3Muhdt8orXC22utleTVj7Prqt82baN6RBvnEgw==", + "dev": true, + "optional": true + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "optional": true, + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "dev": true, + "requires": {} + }, + "xhr": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/xhr/-/xhr-2.6.0.tgz", + "integrity": "sha512-/eCGLb5rxjx5e3mF1A7s+pLlR6CGyqWN91fv1JgER5mVWg1MZmlhBvy9kjcsOdRk8RrIujotWyJamfyrp+WIcA==", + "dev": true, + "optional": true, + "requires": { + "global": "~4.4.0", + "is-function": "^1.0.1", + "parse-headers": "^2.0.0", + "xtend": "^4.0.0" + } + }, + "xml-parse-from-string": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz", + "integrity": "sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==", + "dev": true, + "optional": true + }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "dev": true, + "optional": true, + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "dev": true, + "optional": true + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, + "optional": true + }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true + }, + "yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "dev": true + } + } +} diff --git a/tools/browsertime/package.json b/tools/browsertime/package.json new file mode 100644 index 0000000000..48a87e2390 --- /dev/null +++ b/tools/browsertime/package.json @@ -0,0 +1,14 @@ +{ + "name": "mozilla-central-tools-browsertime", + "description": "This package file is for node modules used in mozilla-central/tools/browsertime", + "repository": {}, + "license": "MPL-2.0", + "dependencies": { + "package.json": "^2.0.1" + }, + "devDependencies": { + "browsertime": "https://github.com/sitespeedio/browsertime/tarball/62de4fc9abc8067fb58378999b1bc4a4c42f9eb5" + }, + "notes(private)": "We don't want to publish to npm, so this is marked as private", + "private": true +}
\ No newline at end of file diff --git a/tools/clang-tidy/config.yaml b/tools/clang-tidy/config.yaml new file mode 100644 index 0000000000..773e92914f --- /dev/null +++ b/tools/clang-tidy/config.yaml @@ -0,0 +1,345 @@ +--- +target: obj-x86_64-pc-linux-gnu +# It is used by 'mach static-analysis' and 'phabricator static-analysis bot' +# in order to have consistency across the used checkers. +# All the clang checks used by the static-analysis tools. +# +# To add a new checker: +# 1. Add it in this file +# 2. Create a C/C++ test case in tools/clang-tidy/test/ reproducing the +# warning/error that the checker will detect +# 3. Run './mach static-analysis autotest -d' to create the reference +# 4. Check the json file in tools/clang-tidy/test/ +# 5. Commit this file + the .cpp test case + the json result +platforms: + - linux64 + - macosx64 + - win32 + - win64 +# Minimum clang-tidy version that is required for all the following checkers +# to work properly. +# This is also used by 'mach clang-format' +package_version: "17.0.6" +clang_checkers: + - name: -* + publish: !!bool no + - name: bugprone-argument-comment + reliability: high + - name: bugprone-assert-side-effect + reliability: high + - name: bugprone-bool-pointer-implicit-conversion + reliability: low + - name: bugprone-forward-declaration-namespace + reliability: high + - name: bugprone-incorrect-roundings + reliability: high + - name: bugprone-integer-division + reliability: high + - name: bugprone-macro-parentheses + reliability: medium + - name: bugprone-macro-repeated-side-effects + reliability: high + - name: bugprone-misplaced-widening-cast + reliability: high + - name: bugprone-move-forwarding-reference + reliability: high + - name: bugprone-multiple-statement-macro + # Incompatible with our code base, see bug 1496379. + publish: !!bool no + reliability: high + - name: bugprone-sizeof-expression + reliability: high + - name: bugprone-string-constructor + reliability: high + - name: bugprone-string-integer-assignment + reliability: high + - name: bugprone-suspicious-memset-usage + reliability: high + - name: bugprone-suspicious-missing-comma + reliability: high + - name: bugprone-suspicious-semicolon + reliability: high + - name: bugprone-suspicious-string-compare + reliability: high + - name: bugprone-swapped-arguments + reliability: high + - name: bugprone-switch-missing-default-case + reliability: high + - name: bugprone-too-small-loop-variable + reliability: high + - name: bugprone-unused-raii + reliability: high + - name: bugprone-use-after-move + reliability: high + - name: clang-analyzer-core.CallAndMessage + reliability: medium + - name: clang-analyzer-core.DivideZero + reliability: high + - name: clang-analyzer-core.NonNullParamChecker + reliability: high + - name: clang-analyzer-core.NullDereference + reliability: medium + - name: clang-analyzer-core.UndefinedBinaryOperatorResult + reliability: medium + - name: clang-analyzer-core.uninitialized.Assign + reliability: medium + - name: clang-analyzer-core.uninitialized.Branch + reliability: medium + - name: clang-analyzer-cplusplus.Move + reliability: high + - name: clang-analyzer-cplusplus.NewDelete + reliability: medium + - name: clang-analyzer-cplusplus.NewDeleteLeaks + reliability: medium + - name: clang-analyzer-deadcode.DeadStores + reliability: high + - name: clang-analyzer-optin.performance.Padding + reliability: high + config: + - key: AllowedPad + value: 2 + - name: clang-analyzer-security.FloatLoopCounter + reliability: high + - name: clang-analyzer-security.insecureAPI.bcmp + reliability: high + - name: clang-analyzer-security.insecureAPI.bcopy + reliability: high + - name: clang-analyzer-security.insecureAPI.bzero + reliability: high + - name: clang-analyzer-security.insecureAPI.getpw + reliability: high + # We don't add clang-analyzer-security.insecureAPI.gets here; it's deprecated. + - name: clang-analyzer-security.insecureAPI.mkstemp + reliability: high + - name: clang-analyzer-security.insecureAPI.mktemp + reliability: high + - name: clang-analyzer-security.insecureAPI.rand + reliability: low + # C checker, that is outdated and doesn't check for the new std::rand calls. + publish: !!bool no + - name: clang-analyzer-security.insecureAPI.strcpy + reliability: low + # The functions that should be used differ on POSIX and Windows, and there + # isn't a consensus on how we should approach this. + publish: !!bool no + - name: clang-analyzer-security.insecureAPI.UncheckedReturn + reliability: low + - name: clang-analyzer-security.insecureAPI.vfork + reliability: medium + - name: clang-analyzer-unix.Malloc + reliability: high + - name: clang-analyzer-unix.cstring.BadSizeArg + reliability: high + - name: clang-analyzer-unix.cstring.NullArg + reliability: high + - name: cppcoreguidelines-narrowing-conversions + reliability: high + - name: cppcoreguidelines-pro-type-member-init + reliability: medium + - name: misc-include-cleaner + # Disable this checker until we move to before/after + reliability: high + publish: !!bool no + - name: misc-non-copyable-objects + reliability: high + - name: misc-redundant-expression + reliability: medium + - name: misc-unused-alias-decls + reliability: high + - name: misc-unused-using-decls + reliability: high + - name: modernize-avoid-bind + restricted-platforms: + - win32 + - win64 + reliability: medium + - name: modernize-concat-nested-namespaces + reliability: high + - name: modernize-deprecated-ios-base-aliases + reliability: high + - name: modernize-loop-convert + reliability: high + - name: modernize-raw-string-literal + reliability: high + - name: modernize-redundant-void-arg + reliability: high + # We still have some old C code that is built with a C compiler, so this + # might break the build. + publish: !!bool no + - name: modernize-shrink-to-fit + reliability: high + - name: modernize-use-auto + reliability: high + # Controversial, see bug 1371052. + publish: !!bool no + - name: modernize-use-bool-literals + reliability: high + - name: modernize-use-equals-default + reliability: high + - name: modernize-use-equals-delete + reliability: high + - name: modernize-use-nullptr + reliability: high + - name: modernize-use-override + reliability: low + # Too noisy because of the way how we implement NS_IMETHOD. See Bug 1420366. + publish: !!bool no + - name: modernize-use-using + reliability: high + - name: mozilla-* + reliability: high + - name: performance-avoid-endl + reliability: high + # enable from clang 18 + # - name: performance-enum-size + # reliability: high + - name: performance-faster-string-find + reliability: high + - name: performance-for-range-copy + reliability: high + - name: performance-implicit-conversion-in-loop + reliability: high + - name: performance-inefficient-algorithm + restricted-platforms: + - linux64 + - macosx64 + reliability: high + # Disable as the test does not support C++17 yet + publish: !!bool no + - name: performance-inefficient-string-concatenation + reliability: high + - name: performance-inefficient-vector-operation + reliability: high + - name: performance-move-const-arg + reliability: high + config: + - key: CheckTriviallyCopyableMove + # As per Bug 1558359 - disable detection of trivially copyable types + # that do not have a move constructor. + value: 0 + - name: performance-move-constructor-init + reliability: high + - name: performance-noexcept-move-constructor + reliability: high + - name: performance-type-promotion-in-math-fn + reliability: high + - name: performance-unnecessary-copy-initialization + reliability: high + - name: performance-unnecessary-value-param + reliability: high + config: + - key: AllowedTypes + # Allow EnumSet because it only has a non-trivial copy constructor + # in debug builds. + value: ::mozilla::EnumSet + - name: readability-braces-around-statements + reliability: high + config: + - key: ShortStatementLines + # Allow `if (foo) return;` without braces + # Still warns on `if (foo)\n return;` + value: 1 + - name: readability-const-return-type + reliability: high + # Note: this can be loosened up by using the ShortStatementLines option + - name: readability-container-size-empty + reliability: high + - name: readability-delete-null-pointer + reliability: high + - name: readability-else-after-return + reliability: high + config: + - key: WarnOnConditionVariables + # Disable as we don't mind this kind of behavior + value: 0 + - name: readability-implicit-bool-conversion + reliability: low + # On automation the config flags act strange. Please see Bug 1500241. + publish: !!bool no + config: + - key: AllowIntegerConditions + # The check will allow conditional integer conversions. + value: 1 + - key: AllowPointerConditions + # The check will allow conditional pointer conversions. + value: 1 + - name: readability-inconsistent-declaration-parameter-name + reliability: high + - name: readability-isolate-declaration + # As per bug 1558987 - we don't want to have this enabled + publish: !!bool no + reliability: high + - name: readability-magic-numbers + # Bug 1553495 - we must see first its impact on our code. + publish: !!bool no + reliability: high + - name: readability-misleading-indentation + reliability: high + - name: readability-non-const-parameter + reliability: high + - name: readability-qualified-auto + reliability: high + - name: readability-redundant-control-flow + reliability: high + - name: readability-redundant-member-init + reliability: high + - name: readability-redundant-preprocessor + reliability: high + - name: readability-redundant-smartptr-get + reliability: high + - name: readability-redundant-string-cstr + reliability: high + - name: readability-redundant-string-init + reliability: high + - name: readability-static-accessed-through-instance + reliability: high + - name: readability-simplify-boolean-expr + reliability: high + config: + - key: SimplifyDeMorgan + # Don't want to enable DeMorgan expressions because of MOZ_ASSERT() + # See Bug 1804160 + value: 0 + - name: readability-uniqueptr-delete-release + reliability: high + # We don't publish the google checkers since we are interested in only having + # a general idea how our code complies with the rules added by these checkers. + - name: google-build-explicit-make-pair + reliability: low + publish: !!bool no + - name: google-build-namespaces + reliability: low + publish: !!bool no + - name: google-build-using-namespace + reliability: low + publish: !!bool no + - name: google-default-arguments + reliability: low + publish: !!bool no + - name: google-explicit-constructor + reliability: low + publish: !!bool no + - name: google-global-names-in-headers + reliability: low + publish: !!bool no + - name: google-readability-casting + reliability: low + publish: !!bool no + - name: google-readability-function-size + reliability: low + publish: !!bool no + - name: google-readability-namespace-comments + reliability: low + publish: !!bool no + - name: google-readability-todo + reliability: low + publish: !!bool no + - name: google-runtime-int + reliability: low + publish: !!bool no + - name: google-runtime-operator + reliability: low + publish: !!bool no + - name: google-runtime-references + reliability: low + publish: !!bool no diff --git a/tools/clang-tidy/test/bugprone-argument-comment.cpp b/tools/clang-tidy/test/bugprone-argument-comment.cpp new file mode 100644 index 0000000000..3099575a35 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-argument-comment.cpp @@ -0,0 +1,6 @@ +// bugprone-argument-comment + +void f(int x, int y); +void g() { + f(/*y=*/0, /*z=*/0); +} diff --git a/tools/clang-tidy/test/bugprone-argument-comment.json b/tools/clang-tidy/test/bugprone-argument-comment.json new file mode 100644 index 0000000000..034f6dbf87 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-argument-comment.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "argument name 'y' in comment does not match parameter name 'x'", + "bugprone-argument-comment" + ], + [ + "warning", + "argument name 'z' in comment does not match parameter name 'y'", + "bugprone-argument-comment" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-assert-side-effect.cpp b/tools/clang-tidy/test/bugprone-assert-side-effect.cpp new file mode 100644 index 0000000000..7cc0a79cf1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-assert-side-effect.cpp @@ -0,0 +1,8 @@ +#include "structures.h" + +// bugprone-assert-side-effect +void misc_assert_side_effect() { + int X = 0; + assert(X == 1); + assert(X = 1); +} diff --git a/tools/clang-tidy/test/bugprone-assert-side-effect.json b/tools/clang-tidy/test/bugprone-assert-side-effect.json new file mode 100644 index 0000000000..b17a456064 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-assert-side-effect.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "side effect in assert() condition discarded in release builds", + "bugprone-assert-side-effect" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.cpp b/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.cpp new file mode 100644 index 0000000000..602fa9a578 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.cpp @@ -0,0 +1,20 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-bool-pointer-implicit-conversion.html + +bool test(bool* pointer_to_bool, int* pointer_to_int) +{ + if (pointer_to_bool) { // warning for pointer to bool + } + + if (pointer_to_int) { // no warning for pointer to int + } + + if (!pointer_to_bool) { // no warning, but why not?? + } + + if (pointer_to_bool != nullptr) { // no warning for nullptr comparison + } + + // no warning on return, but why not?? + // clang-tidy bug: https://bugs.llvm.org/show_bug.cgi?id=38060 + return pointer_to_bool; +} diff --git a/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.json b/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.json new file mode 100644 index 0000000000..e4d64ee683 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "dubious check of 'bool *' against 'nullptr', did you mean to dereference it?", + "bugprone-bool-pointer-implicit-conversion" + ], + { "reliability": "low" } +] diff --git a/tools/clang-tidy/test/bugprone-forward-declaration-namespace.cpp b/tools/clang-tidy/test/bugprone-forward-declaration-namespace.cpp new file mode 100644 index 0000000000..f93d54b0a1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-forward-declaration-namespace.cpp @@ -0,0 +1,3 @@ +namespace na { struct A; } +namespace nb { struct A {}; } +nb::A a; diff --git a/tools/clang-tidy/test/bugprone-forward-declaration-namespace.json b/tools/clang-tidy/test/bugprone-forward-declaration-namespace.json new file mode 100644 index 0000000000..aee1ec88e5 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-forward-declaration-namespace.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "no definition found for 'A', but a definition with the same name 'A' found in another namespace 'nb'", + "bugprone-forward-declaration-namespace" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-incorrect-roundings.cpp b/tools/clang-tidy/test/bugprone-incorrect-roundings.cpp new file mode 100644 index 0000000000..ee37b56ae0 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-incorrect-roundings.cpp @@ -0,0 +1,7 @@ +void f1() +{ + double d; + int x; + + x = (d + 0.5); +} diff --git a/tools/clang-tidy/test/bugprone-incorrect-roundings.json b/tools/clang-tidy/test/bugprone-incorrect-roundings.json new file mode 100644 index 0000000000..f8da228b16 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-incorrect-roundings.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "casting (double + 0.5) to integer leads to incorrect rounding; consider using lround (#include <cmath>) instead", + "bugprone-incorrect-roundings" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-integer-division.cpp b/tools/clang-tidy/test/bugprone-integer-division.cpp new file mode 100644 index 0000000000..058edffe39 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-integer-division.cpp @@ -0,0 +1,5 @@ +float f() { + int a = 2; + int b = 10; + return a/b; +} diff --git a/tools/clang-tidy/test/bugprone-integer-division.json b/tools/clang-tidy/test/bugprone-integer-division.json new file mode 100644 index 0000000000..95c43fc8e0 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-integer-division.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "result of integer division used in a floating point context; possible loss of precision", + "bugprone-integer-division" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-macro-parentheses.cpp b/tools/clang-tidy/test/bugprone-macro-parentheses.cpp new file mode 100644 index 0000000000..53c22090a1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-macro-parentheses.cpp @@ -0,0 +1,5 @@ +#define BAD1 -1 +#define BAD2 1+2 +#define BAD3(A) (A+1) +#define BAD4(x) ((unsigned char)(x & 0xff)) +#define BAD5(X) A*B=(C*)X+2 diff --git a/tools/clang-tidy/test/bugprone-macro-parentheses.json b/tools/clang-tidy/test/bugprone-macro-parentheses.json new file mode 100644 index 0000000000..c50bf76cbd --- /dev/null +++ b/tools/clang-tidy/test/bugprone-macro-parentheses.json @@ -0,0 +1,28 @@ +[ + [ + "warning", + "macro replacement list should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + [ + "warning", + "macro replacement list should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + [ + "warning", + "macro argument should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + [ + "warning", + "macro argument should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + [ + "warning", + "macro argument should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.cpp b/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.cpp new file mode 100644 index 0000000000..2dedb3aca9 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.cpp @@ -0,0 +1,28 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-macro-repeated-side-effects.html + +#define MACRO_WITHOUT_REPEATED_ARG(x) (x) +#define MACRO_WITH_REPEATED_ARG(x) ((x) + (x)) + +static int g; + +int function_with_side_effects(int i) +{ + g += i; + return g; +} + +void test() +{ + int i; + i = MACRO_WITHOUT_REPEATED_ARG(1); // OK + i = MACRO_WITH_REPEATED_ARG(1); // OK + + i = MACRO_WITHOUT_REPEATED_ARG(i); // OK + i = MACRO_WITH_REPEATED_ARG(i); // OK + + i = MACRO_WITHOUT_REPEATED_ARG(function_with_side_effects(i)); // OK + i = MACRO_WITH_REPEATED_ARG(function_with_side_effects(i)); // NO WARNING + + i = MACRO_WITHOUT_REPEATED_ARG(i++); // OK + i = MACRO_WITH_REPEATED_ARG(i++); // WARNING +} diff --git a/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.json b/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.json new file mode 100644 index 0000000000..236120ae05 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "side effects in the 1st macro argument 'x' are repeated in macro expansion", + "bugprone-macro-repeated-side-effects" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-misplaced-widening-cast.cpp b/tools/clang-tidy/test/bugprone-misplaced-widening-cast.cpp new file mode 100644 index 0000000000..e75541bf6e --- /dev/null +++ b/tools/clang-tidy/test/bugprone-misplaced-widening-cast.cpp @@ -0,0 +1,3 @@ +long f(int x) { + return (long)(x * 1000); +} diff --git a/tools/clang-tidy/test/bugprone-misplaced-widening-cast.json b/tools/clang-tidy/test/bugprone-misplaced-widening-cast.json new file mode 100644 index 0000000000..bd0d9f2c00 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-misplaced-widening-cast.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "either cast from 'int' to 'long' is ineffective, or there is loss of precision before the conversion", + "bugprone-misplaced-widening-cast" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-move-forwarding-reference.cpp b/tools/clang-tidy/test/bugprone-move-forwarding-reference.cpp new file mode 100644 index 0000000000..45be15ee65 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-move-forwarding-reference.cpp @@ -0,0 +1,25 @@ + +namespace std { +template <typename> struct remove_reference; + +template <typename _Tp> struct remove_reference { typedef _Tp type; }; + +template <typename _Tp> struct remove_reference<_Tp &> { typedef _Tp type; }; + +template <typename _Tp> struct remove_reference<_Tp &&> { typedef _Tp type; }; + +template <typename _Tp> +constexpr typename std::remove_reference<_Tp>::type &&move(_Tp &&__t); + +} // namespace std + +// Standard case. +template <typename T, typename U> void f1(U &&SomeU) { + T SomeT(std::move(SomeU)); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: forwarding reference passed to + // CHECK-FIXES: T SomeT(std::forward<U>(SomeU)); +} + +void foo() { + f1<int, int>(2); +} diff --git a/tools/clang-tidy/test/bugprone-move-forwarding-reference.json b/tools/clang-tidy/test/bugprone-move-forwarding-reference.json new file mode 100644 index 0000000000..43166e4af1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-move-forwarding-reference.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "forwarding reference passed to std::move(), which may unexpectedly cause lvalues to be moved; use std::forward() instead", + "bugprone-move-forwarding-reference" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-multiple-statement-macro.cpp b/tools/clang-tidy/test/bugprone-multiple-statement-macro.cpp new file mode 100644 index 0000000000..7ad2a2df64 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-multiple-statement-macro.cpp @@ -0,0 +1,10 @@ +void F(); + +#define BAD_MACRO(x) \ + F(); \ + F() + +void positives() { + if (1) + BAD_MACRO(1); +} diff --git a/tools/clang-tidy/test/bugprone-multiple-statement-macro.json b/tools/clang-tidy/test/bugprone-multiple-statement-macro.json new file mode 100644 index 0000000000..81c3dc89e5 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-multiple-statement-macro.json @@ -0,0 +1,7 @@ +[ + [ + "warning", + "multiple statement macro used without braces; some statements will be unconditionally executed", + "bugprone-multiple-statement-macro" + ] +] diff --git a/tools/clang-tidy/test/bugprone-sizeof-expression.cpp b/tools/clang-tidy/test/bugprone-sizeof-expression.cpp new file mode 100644 index 0000000000..f5f1f469ce --- /dev/null +++ b/tools/clang-tidy/test/bugprone-sizeof-expression.cpp @@ -0,0 +1,3 @@ +class C { + int size() { return sizeof(this); } +}; diff --git a/tools/clang-tidy/test/bugprone-sizeof-expression.json b/tools/clang-tidy/test/bugprone-sizeof-expression.json new file mode 100644 index 0000000000..438194296b --- /dev/null +++ b/tools/clang-tidy/test/bugprone-sizeof-expression.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "suspicious usage of 'sizeof(this)'; did you mean 'sizeof(*this)'", + "bugprone-sizeof-expression" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-string-constructor.cpp b/tools/clang-tidy/test/bugprone-string-constructor.cpp new file mode 100644 index 0000000000..8b6a4980a3 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-string-constructor.cpp @@ -0,0 +1,17 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-string-constructor.html + +#include "structures.h" + +void test() +{ + // A common mistake is to swap parameters to the ‘fill’ string-constructor. + std::string str('x', 50); // should be str(50, 'x') + + // Calling the string-literal constructor with a length bigger than the + // literal is suspicious and adds extra random characters to the string. + std::string("test", 200); // Will include random characters after "test". + + // Creating an empty string from constructors with parameters is considered + // suspicious. The programmer should use the empty constructor instead. + std::string("test", 0); // Creation of an empty string. +} diff --git a/tools/clang-tidy/test/bugprone-string-constructor.json b/tools/clang-tidy/test/bugprone-string-constructor.json new file mode 100644 index 0000000000..7e2ba1764f --- /dev/null +++ b/tools/clang-tidy/test/bugprone-string-constructor.json @@ -0,0 +1,18 @@ +[ + [ + "warning", + "string constructor parameters are probably swapped; expecting string(count, character)", + "bugprone-string-constructor" + ], + [ + "warning", + "length is bigger than string literal size", + "bugprone-string-constructor" + ], + [ + "warning", + "constructor creating an empty string", + "bugprone-string-constructor" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-string-integer-assignment.cpp b/tools/clang-tidy/test/bugprone-string-integer-assignment.cpp new file mode 100644 index 0000000000..30ef46b922 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-string-integer-assignment.cpp @@ -0,0 +1,28 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-string-integer-assignment.html + +#include "structures.h" + +void test_int() +{ + // Numeric types can be implicitly casted to character types. + std::string s; + int x = 5965; + s = 6; // warning + s = x; // warning +} + +void test_conversion() +{ + // Use the appropriate conversion functions or character literals. + std::string s; + int x = 5965; + s = '6'; // OK + s = std::to_string(x); // OK +} + +void test_cast() +{ + // In order to suppress false positives, use an explicit cast. + std::string s; + s = static_cast<char>(6); // OK +} diff --git a/tools/clang-tidy/test/bugprone-string-integer-assignment.json b/tools/clang-tidy/test/bugprone-string-integer-assignment.json new file mode 100644 index 0000000000..37912fd950 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-string-integer-assignment.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "an integer is interpreted as a character code when assigning it to a string; if this is intended, cast the integer to the appropriate character type; if you want a string representation, use the appropriate conversion facility", + "bugprone-string-integer-assignment" + ], + [ + "warning", + "an integer is interpreted as a character code when assigning it to a string; if this is intended, cast the integer to the appropriate character type; if you want a string representation, use the appropriate conversion facility", + "bugprone-string-integer-assignment" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-suspicious-memset-usage.cpp b/tools/clang-tidy/test/bugprone-suspicious-memset-usage.cpp new file mode 100644 index 0000000000..71fe7239a1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-memset-usage.cpp @@ -0,0 +1,22 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-suspicious-memset-usage.html + +#include "structures.h" + +void test(int* ip, char* cp) +{ + // Case 1: Fill value is a character '0' instead of NUL '\0'. + memset(ip, '0', 1); // WARNING: suspicious for non-char pointers + memset(cp, '0', 1); // OK for char pointers + + // Case 2: Fill value is truncated. + memset(ip, 0xabcd, 1); // WARNING: fill value gets truncated + memset(ip, 0x00cd, 1); // OK because value 0xcd is not truncated. + memset(ip, 0x00, 1); // OK because value is not truncated. + + // Case 3: Byte count is zero. + memset(ip, sizeof(int), 0); // WARNING: zero length, potentially swapped + memset(ip, sizeof(int), 1); // OK with non-zero length + + // See clang bug https://bugs.llvm.org/show_bug.cgi?id=38098 + memset(ip, 8, 0); // OK with zero length without sizeof +} diff --git a/tools/clang-tidy/test/bugprone-suspicious-memset-usage.json b/tools/clang-tidy/test/bugprone-suspicious-memset-usage.json new file mode 100644 index 0000000000..73f26cf7c1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-memset-usage.json @@ -0,0 +1,18 @@ +[ + [ + "warning", + "memset fill value is char '0', potentially mistaken for int 0", + "bugprone-suspicious-memset-usage" + ], + [ + "warning", + "memset fill value is out of unsigned character range, gets truncated", + "bugprone-suspicious-memset-usage" + ], + [ + "warning", + "memset of size zero, potentially swapped arguments", + "bugprone-suspicious-memset-usage" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-suspicious-missing-comma.cpp b/tools/clang-tidy/test/bugprone-suspicious-missing-comma.cpp new file mode 100644 index 0000000000..de1634e11f --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-missing-comma.cpp @@ -0,0 +1,9 @@ +const char* Cartoons[] = { + "Bugs Bunny", + "Homer Simpson", + "Mickey Mouse", + "Bart Simpson", + "Charlie Brown" // There is a missing comma here. + "Fred Flintstone", + "Popeye", +}; diff --git a/tools/clang-tidy/test/bugprone-suspicious-missing-comma.json b/tools/clang-tidy/test/bugprone-suspicious-missing-comma.json new file mode 100644 index 0000000000..f4adb6b33a --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-missing-comma.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "suspicious string literal, probably missing a comma", + "bugprone-suspicious-missing-comma" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-suspicious-semicolon.cpp b/tools/clang-tidy/test/bugprone-suspicious-semicolon.cpp new file mode 100644 index 0000000000..7a90a87cd3 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-semicolon.cpp @@ -0,0 +1,8 @@ + +// bugprone-suspicious-semicolon +void nop(); +void fail1() +{ + int x = 0; + if(x > 5); nop(); +} diff --git a/tools/clang-tidy/test/bugprone-suspicious-semicolon.json b/tools/clang-tidy/test/bugprone-suspicious-semicolon.json new file mode 100644 index 0000000000..c94a1d5abb --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-semicolon.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "potentially unintended semicolon", + "bugprone-suspicious-semicolon" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-suspicious-string-compare.cpp b/tools/clang-tidy/test/bugprone-suspicious-string-compare.cpp new file mode 100644 index 0000000000..505a9d282c --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-string-compare.cpp @@ -0,0 +1,8 @@ +static const char A[] = "abc"; + +int strcmp(const char *, const char *); + +int test_warning_patterns() { + if (strcmp(A, "a")) + return 0; +} diff --git a/tools/clang-tidy/test/bugprone-suspicious-string-compare.json b/tools/clang-tidy/test/bugprone-suspicious-string-compare.json new file mode 100644 index 0000000000..eda430ef01 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-string-compare.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "function 'strcmp' is called without explicitly comparing result", + "bugprone-suspicious-string-compare" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-swapped-arguments.cpp b/tools/clang-tidy/test/bugprone-swapped-arguments.cpp new file mode 100644 index 0000000000..074df52050 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-swapped-arguments.cpp @@ -0,0 +1,26 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-swapped-arguments.html + +void test_d_i(double d, int i); +void test_d_i_i(double d, int i, int ii); +void test_i_d(int i, double d); +void test_i_i_d(int i, int ii, double d); + +void test() +{ + double d = 1; + int i = 1; + + test_d_i(d, i); // OK + test_d_i(i, d); // WARNING + + test_i_d(i, d); // OK + test_i_d(d, i); // WARNING + + test_i_i_d(i, i, d); // OK + test_i_i_d(i, d, i); // WARNING + test_i_i_d(d, i, i); // NO WARNING after second parameter + + test_d_i_i(d, i, i); // OK + test_d_i_i(i, d, i); // WARNING + test_d_i_i(i, i, d); // NO WARNING after second parameter +} diff --git a/tools/clang-tidy/test/bugprone-swapped-arguments.json b/tools/clang-tidy/test/bugprone-swapped-arguments.json new file mode 100644 index 0000000000..1ec3750688 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-swapped-arguments.json @@ -0,0 +1,23 @@ +[ + [ + "warning", + "argument with implicit conversion from 'int' to 'double' followed by argument converted from 'double' to 'int', potentially swapped arguments.", + "bugprone-swapped-arguments" + ], + [ + "warning", + "argument with implicit conversion from 'double' to 'int' followed by argument converted from 'int' to 'double', potentially swapped arguments.", + "bugprone-swapped-arguments" + ], + [ + "warning", + "argument with implicit conversion from 'double' to 'int' followed by argument converted from 'int' to 'double', potentially swapped arguments.", + "bugprone-swapped-arguments" + ], + [ + "warning", + "argument with implicit conversion from 'int' to 'double' followed by argument converted from 'double' to 'int', potentially swapped arguments.", + "bugprone-swapped-arguments" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-switch-missing-default-case.cpp b/tools/clang-tidy/test/bugprone-switch-missing-default-case.cpp new file mode 100644 index 0000000000..fac442d90d --- /dev/null +++ b/tools/clang-tidy/test/bugprone-switch-missing-default-case.cpp @@ -0,0 +1,7 @@ +void func() { + int radius; + switch (radius) { + case 0: + break; + } +} diff --git a/tools/clang-tidy/test/bugprone-switch-missing-default-case.json b/tools/clang-tidy/test/bugprone-switch-missing-default-case.json new file mode 100644 index 0000000000..f6c52b808a --- /dev/null +++ b/tools/clang-tidy/test/bugprone-switch-missing-default-case.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "switching on non-enum value without default case may not cover all cases", + "bugprone-switch-missing-default-case" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-too-small-loop-variable.cpp b/tools/clang-tidy/test/bugprone-too-small-loop-variable.cpp new file mode 100644 index 0000000000..4dc8ed7a22 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-too-small-loop-variable.cpp @@ -0,0 +1,4 @@ +int main() { + long size = 294967296l; + for (short i = 0; i < size; ++i) {} +} diff --git a/tools/clang-tidy/test/bugprone-too-small-loop-variable.json b/tools/clang-tidy/test/bugprone-too-small-loop-variable.json new file mode 100644 index 0000000000..23f781936c --- /dev/null +++ b/tools/clang-tidy/test/bugprone-too-small-loop-variable.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "loop variable has narrower type 'short' than iteration's upper bound 'long'", + "bugprone-too-small-loop-variable" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-unused-raii.cpp b/tools/clang-tidy/test/bugprone-unused-raii.cpp new file mode 100644 index 0000000000..6a4a0a15ab --- /dev/null +++ b/tools/clang-tidy/test/bugprone-unused-raii.cpp @@ -0,0 +1,25 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-unused-raii.html + +struct scoped_lock +{ + scoped_lock() {} + ~scoped_lock() {} +}; + +#define SCOPED_LOCK_MACRO(m) scoped_lock() + +struct trivial_scoped_lock +{ + trivial_scoped_lock() {} +}; + +scoped_lock test() +{ + scoped_lock(); // misc-unused-raii warning! + + SCOPED_LOCK_MACRO(); // no warning for macros + + trivial_scoped_lock(); // no warning for trivial objects without destructors + + return scoped_lock(); // no warning for return values +} diff --git a/tools/clang-tidy/test/bugprone-unused-raii.json b/tools/clang-tidy/test/bugprone-unused-raii.json new file mode 100644 index 0000000000..99a9936485 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-unused-raii.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "object destroyed immediately after creation; did you mean to name the object?", + "bugprone-unused-raii" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-use-after-move.cpp b/tools/clang-tidy/test/bugprone-use-after-move.cpp new file mode 100644 index 0000000000..f90a8700d6 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-use-after-move.cpp @@ -0,0 +1,62 @@ +namespace std { +typedef unsigned size_t; + +template <typename T> +struct unique_ptr { + unique_ptr(); + T *get() const; + explicit operator bool() const; + void reset(T *ptr); + T &operator*() const; + T *operator->() const; + T& operator[](size_t i) const; +}; + +template <typename> +struct remove_reference; + +template <typename _Tp> +struct remove_reference { + typedef _Tp type; +}; + +template <typename _Tp> +struct remove_reference<_Tp &> { + typedef _Tp type; +}; + +template <typename _Tp> +struct remove_reference<_Tp &&> { + typedef _Tp type; +}; + +template <typename _Tp> +constexpr typename std::remove_reference<_Tp>::type &&move(_Tp &&__t) noexcept { + return static_cast<typename remove_reference<_Tp>::type &&>(__t); +} +} + +class A { +public: + A(); + A(const A &); + A(A &&); + + A &operator=(const A &); + A &operator=(A &&); + + void foo() const; + int getInt() const; + + operator bool() const; + + int i; +}; + +void func() { + std::unique_ptr<A> ptr; + std::move(ptr); + ptr.get(); + static_cast<bool>(ptr); + *ptr; +} diff --git a/tools/clang-tidy/test/bugprone-use-after-move.json b/tools/clang-tidy/test/bugprone-use-after-move.json new file mode 100644 index 0000000000..50d7210781 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-use-after-move.json @@ -0,0 +1,4 @@ +[ + ["warning", "'ptr' used after it was moved", "bugprone-use-after-move"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.cpp b/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.cpp new file mode 100644 index 0000000000..286189e25b --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.cpp @@ -0,0 +1,10 @@ +struct S { + int x; +}; + +void f(struct S s); + +void test() { + struct S s; + f(s); +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.json b/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.json new file mode 100644 index 0000000000..ecf097e287 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Passed-by-value struct argument contains uninitialized data (e.g., field: 'x')", + "clang-analyzer-core.CallAndMessage" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.DivideZero.cpp b/tools/clang-tidy/test/clang-analyzer-core.DivideZero.cpp new file mode 100644 index 0000000000..aac9f7c9f6 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.DivideZero.cpp @@ -0,0 +1,4 @@ +void test(int z) { + if (z == 0) + int x = 1 / z; +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.DivideZero.json b/tools/clang-tidy/test/clang-analyzer-core.DivideZero.json new file mode 100644 index 0000000000..78a64ad35b --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.DivideZero.json @@ -0,0 +1,4 @@ +[ + ["warning", "Division by zero", "clang-analyzer-core.DivideZero"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.cpp b/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.cpp new file mode 100644 index 0000000000..bc071f5453 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.cpp @@ -0,0 +1,6 @@ +int f(int *p) __attribute__((nonnull)); + +void test(int *p) { + if (!p) + f(p); // warn +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.json b/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.json new file mode 100644 index 0000000000..deaae128ff --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Null pointer passed to 1st parameter expecting 'nonnull'", + "clang-analyzer-core.NonNullParamChecker" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.NullDereference.cpp b/tools/clang-tidy/test/clang-analyzer-core.NullDereference.cpp new file mode 100644 index 0000000000..6c9c555532 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.NullDereference.cpp @@ -0,0 +1,9 @@ +class C { +public: + int x; +}; + +void test() { + C *pc = 0; + int k = pc->x; +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.NullDereference.json b/tools/clang-tidy/test/clang-analyzer-core.NullDereference.json new file mode 100644 index 0000000000..4c9258b2b7 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.NullDereference.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Access to field 'x' results in a dereference of a null pointer (loaded from variable 'pc')", + "clang-analyzer-core.NullDereference" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.cpp b/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.cpp new file mode 100644 index 0000000000..1351c35f44 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.cpp @@ -0,0 +1,4 @@ +void test() { + int x; + int y = x + 1; +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.json b/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.json new file mode 100644 index 0000000000..9ec6c8a809 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The left operand of '+' is a garbage value", + "clang-analyzer-core.UndefinedBinaryOperatorResult" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.cpp b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.cpp new file mode 100644 index 0000000000..e9685d48a4 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.cpp @@ -0,0 +1,4 @@ +void test() { + int x; + x |= 1; +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.json b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.json new file mode 100644 index 0000000000..3691af741c --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The left expression of the compound assignment is an uninitialized value. The computed value will also be garbage", + "clang-analyzer-core.uninitialized.Assign" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.cpp b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.cpp new file mode 100644 index 0000000000..7a985eb080 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.cpp @@ -0,0 +1,6 @@ +void test() { + int x; + if (x) { + return; + } +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.json b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.json new file mode 100644 index 0000000000..00c6b3ad1e --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Branch condition evaluates to a garbage value", + "clang-analyzer-core.uninitialized.Branch" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.cpp b/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.cpp new file mode 100644 index 0000000000..65f2e36561 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.cpp @@ -0,0 +1,10 @@ +class P {}; + +void bar(P) {} + +void foo(int n) { + P x; + for (int i = n; i >= 0; --i) { + bar(static_cast<P&&>(x)); + } +} diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.json b/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.json new file mode 100644 index 0000000000..e60f052e86 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Moved-from object 'x' is moved", + "clang-analyzer-cplusplus.Move" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.cpp b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.cpp new file mode 100644 index 0000000000..d886d74989 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.cpp @@ -0,0 +1,50 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +void use(int *p); + +void test_use_parameter_after_delete(int *p) +{ + delete p; + use(p); // warning: use after free +} + +class SomeClass { +public: + void f(); +}; + +void test_use_local_after_delete() +{ + SomeClass *c = new SomeClass; + delete c; + c->f(); // warning: use after free +} + +// XXX clang documentation says this should cause a warning but it doesn't! +void test_delete_alloca() +{ + int *p = (int *)__builtin_alloca(sizeof(int)); + delete p; // NO warning: deleting memory allocated by alloca +} + +void test_double_free() +{ + int *p = new int; + delete p; + delete p; // warning: attempt to free released +} + +void test_delete_local() +{ + int i; + delete &i; // warning: delete address of local +} + +// XXX clang documentation says this should cause a warning but it doesn't! +void test_delete_offset() +{ + int *p = new int[1]; + delete[] (++p); + // NO warning: argument to 'delete[]' is offset by 4 bytes + // from the start of memory allocated by 'new[]' +} diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.json b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.json new file mode 100644 index 0000000000..99a9cd0dd2 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.json @@ -0,0 +1,23 @@ +[ + [ + "warning", + "Use of memory after it is freed", + "clang-analyzer-cplusplus.NewDelete" + ], + [ + "warning", + "Use of memory after it is freed", + "clang-analyzer-cplusplus.NewDelete" + ], + [ + "warning", + "Attempt to free released memory", + "clang-analyzer-cplusplus.NewDelete" + ], + [ + "warning", + "Argument to 'delete' is the address of the local variable 'i', which is not memory allocated by 'new'", + "clang-analyzer-cplusplus.NewDelete" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.cpp b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.cpp new file mode 100644 index 0000000000..60772a30db --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.cpp @@ -0,0 +1,6 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +void test() +{ + int *p = new int; +} // warning diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.json b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.json new file mode 100644 index 0000000000..37490a67f8 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Potential leak of memory pointed to by 'p'", + "clang-analyzer-cplusplus.NewDeleteLeaks" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.cpp b/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.cpp new file mode 100644 index 0000000000..4e1c2c851d --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.cpp @@ -0,0 +1,6 @@ + +// clang-analyzer-deadcode.DeadStores +void test() { + int x; + x = 1; // warn +} diff --git a/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.json b/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.json new file mode 100644 index 0000000000..c46d5f2c2a --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Value stored to 'x' is never read", + "clang-analyzer-deadcode.DeadStores" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.cpp b/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.cpp new file mode 100644 index 0000000000..0b851a68d3 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.cpp @@ -0,0 +1,6 @@ +struct OverlyAlignedChar { + char c1; + int x; + char c2; + char c __attribute__((aligned(4096))); +}; diff --git a/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.json b/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.json new file mode 100644 index 0000000000..a7ce9df9a5 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Excessive padding in 'struct OverlyAlignedChar' (8185 padding bytes, where 4089 is optimal). Optimal fields order: c, c1, c2, x, consider reordering the fields or adding explicit padding members", + "clang-analyzer-optin.performance.Padding" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.cpp b/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.cpp new file mode 100644 index 0000000000..60dcdad746 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.cpp @@ -0,0 +1,3 @@ +void test() { + for (float x = 0.1f; x <= 1.0f; x += 0.1f) {} +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.json b/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.json new file mode 100644 index 0000000000..792f5b13dc --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Variable 'x' with floating point type 'float' should not be used as a loop counter", + "clang-analyzer-security.FloatLoopCounter" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.cpp new file mode 100644 index 0000000000..f1dede2d51 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test() { + setuid(1); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.json new file mode 100644 index 0000000000..a4f89bc0d9 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The return value from the call to 'setuid' is not checked. If an error occurs in 'setuid', the following code may execute with unexpected privileges", + "clang-analyzer-security.insecureAPI.UncheckedReturn" + ], + { "reliability": "low" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.cpp new file mode 100644 index 0000000000..8821807518 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +int test_bcmp(void *a, void *b, size_t n) { + return bcmp(a, b, n); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.json new file mode 100644 index 0000000000..ea7d6267ee --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The bcmp() function is obsoleted by memcmp()", + "clang-analyzer-security.insecureAPI.bcmp" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.cpp new file mode 100644 index 0000000000..67df7d1638 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test_bcopy(void *a, void *b, size_t n) { + bcopy(a, b, n); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.json new file mode 100644 index 0000000000..d752f6c7a8 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The bcopy() function is obsoleted by memcpy() or memmove()", + "clang-analyzer-security.insecureAPI.bcopy" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.cpp new file mode 100644 index 0000000000..d3b5aa685f --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test_bzero(void *a, size_t n) { + bzero(a, n); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.json new file mode 100644 index 0000000000..cdc654a176 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The bzero() function is obsoleted by memset()", + "clang-analyzer-security.insecureAPI.bzero" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.cpp new file mode 100644 index 0000000000..c3da0b1970 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.cpp @@ -0,0 +1,6 @@ +#include "structures.h" + +void test() { + char buff[1024]; + getpw(2, buff); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.json new file mode 100644 index 0000000000..2f80393d54 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The getpw() function is dangerous as it may overflow the provided buffer. It is obsoleted by getpwuid()", + "clang-analyzer-security.insecureAPI.getpw" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.cpp new file mode 100644 index 0000000000..f096c29de3 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.cpp @@ -0,0 +1,6 @@ +#include <stdio.h> + +void test() { + char buff[1024]; + gets(buff); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.json new file mode 100644 index 0000000000..1d2212ee27 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.json @@ -0,0 +1 @@ +"[[\"error\", \"use of undeclared identifier 'gets'\", \"clang-diagnostic-error\"]]" diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.cpp new file mode 100644 index 0000000000..904fc92ce6 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test() { + mkstemp("XX"); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.json new file mode 100644 index 0000000000..cca843ce93 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Call to 'mkstemp' should have at least 6 'X's in the format string to be secure (2 'X's seen)", + "clang-analyzer-security.insecureAPI.mkstemp" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.cpp new file mode 100644 index 0000000000..8bb511a7d7 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test() { + char *x = mktemp("/tmp/zxcv"); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.json new file mode 100644 index 0000000000..ce58bfdddf --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Call to function 'mktemp' is insecure as it always creates or uses insecure temporary file. Use 'mkstemp' instead", + "clang-analyzer-security.insecureAPI.mktemp" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.cpp new file mode 100644 index 0000000000..2274127c50 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.cpp @@ -0,0 +1,4 @@ +#include <stdlib.h> +void test() { + random(); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.json new file mode 100644 index 0000000000..7669f38d3e --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.json @@ -0,0 +1 @@ +"[[\"warning\", \"The 'random' function produces a sequence of values that an adversary may be able to predict. Use 'arc4random' instead\", \"clang-analyzer-security.insecureAPI.rand\"]]" diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.cpp new file mode 100644 index 0000000000..41713adb4e --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.cpp @@ -0,0 +1,7 @@ +#include <string.h> +void test() { + char x[4]; + char *y = "abcd"; + + strcpy(x, y); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.json new file mode 100644 index 0000000000..874de88ded --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.json @@ -0,0 +1 @@ +"[[\"warning\", \"Call to function 'strcpy' is insecure as it does not provide bounding of the memory buffer. Replace unbounded copy functions with analogous functions that support length arguments such as 'strlcpy'. CWE-119\", \"clang-analyzer-security.insecureAPI.strcpy\"], [\"note\", \"Call to function 'strcpy' is insecure as it does not provide bounding of the memory buffer. Replace unbounded copy functions with analogous functions that support length arguments such as 'strlcpy'. CWE-119\", null]]" diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.cpp new file mode 100644 index 0000000000..619d986cf7 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test() { + vfork(); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.json new file mode 100644 index 0000000000..dd681fd537 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Call to function 'vfork' is insecure as it can lead to denial of service situations in the parent process. Replace calls to vfork with calls to the safer 'posix_spawn' function", + "clang-analyzer-security.insecureAPI.vfork" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-unix.Malloc.cpp b/tools/clang-tidy/test/clang-analyzer-unix.Malloc.cpp new file mode 100644 index 0000000000..a08422b336 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.Malloc.cpp @@ -0,0 +1,37 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +#include "structures.h" + +void test_malloc() +{ + int *p = (int*) malloc(1); + free(p); + free(p); // warning: attempt to free released memory +} + +void test_use_after_free() +{ + int *p = (int*) malloc(sizeof(int)); + free(p); + *p = 1; // warning: use after free +} + +void test_leak() +{ + int *p = (int*) malloc(1); + if (p) + return; // warning: memory is never released +} + +void test_free_local() +{ + int a[] = { 1 }; + free(a); // warning: argument is not allocated by malloc +} + +void test_free_offset() +{ + int *p = (int*) malloc(sizeof(char)); + p = p - 1; + free(p); // warning: argument to free() is offset by -4 bytes +} diff --git a/tools/clang-tidy/test/clang-analyzer-unix.Malloc.json b/tools/clang-tidy/test/clang-analyzer-unix.Malloc.json new file mode 100644 index 0000000000..701cbba680 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.Malloc.json @@ -0,0 +1,20 @@ +[ + ["warning", "Attempt to free released memory", "clang-analyzer-unix.Malloc"], + ["warning", "Use of memory after it is freed", "clang-analyzer-unix.Malloc"], + [ + "warning", + "Potential leak of memory pointed to by 'p'", + "clang-analyzer-unix.Malloc" + ], + [ + "warning", + "Argument to free() is the address of the local variable 'a', which is not memory allocated by malloc()", + "clang-analyzer-unix.Malloc" + ], + [ + "warning", + "Argument to free() is offset by -4 bytes from the start of memory allocated by malloc()", + "clang-analyzer-unix.Malloc" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.cpp b/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.cpp new file mode 100644 index 0000000000..124737c3f4 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.cpp @@ -0,0 +1,9 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +#include "structures.h" + +void test() +{ + char dest[3]; + strncat(dest, "***", sizeof(dest)); // warning : potential buffer overflow +} diff --git a/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.json b/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.json new file mode 100644 index 0000000000..dc6b9facf1 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Potential buffer overflow. Replace with 'sizeof(dest) - strlen(dest) - 1' or use a safer 'strlcat' API", + "clang-analyzer-unix.cstring.BadSizeArg" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.cpp b/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.cpp new file mode 100644 index 0000000000..30cdaf6ce9 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.cpp @@ -0,0 +1,14 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +#include "structures.h" + +int my_strlen(const char* s) +{ + return strlen(s); // warning +} + +int bad_caller() +{ + const char* s = nullptr; + return my_strlen(s); +} diff --git a/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.json b/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.json new file mode 100644 index 0000000000..a1270cafd8 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Null pointer passed as 1st argument to string length function", + "clang-analyzer-unix.cstring.NullArg" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.cpp b/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.cpp new file mode 100644 index 0000000000..b10b0fa1c3 --- /dev/null +++ b/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.cpp @@ -0,0 +1,4 @@ +class Foo { + int f; + void a_f(double val) { f = val;} +}; diff --git a/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.json b/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.json new file mode 100644 index 0000000000..9dca259044 --- /dev/null +++ b/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "narrowing conversion from 'double' to 'int'", + "cppcoreguidelines-narrowing-conversions" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.cpp b/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.cpp new file mode 100644 index 0000000000..be331829f3 --- /dev/null +++ b/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.cpp @@ -0,0 +1,7 @@ +struct Foo final { + int x; +}; + +void foo() { + Foo y; +} diff --git a/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.json b/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.json new file mode 100644 index 0000000000..11e79f1579 --- /dev/null +++ b/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "uninitialized record type: 'y'", + "cppcoreguidelines-pro-type-member-init" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/misc-include-cleaner.cpp b/tools/clang-tidy/test/misc-include-cleaner.cpp new file mode 100644 index 0000000000..09e2e0e512 --- /dev/null +++ b/tools/clang-tidy/test/misc-include-cleaner.cpp @@ -0,0 +1 @@ +#include "structures.h" diff --git a/tools/clang-tidy/test/misc-include-cleaner.json b/tools/clang-tidy/test/misc-include-cleaner.json new file mode 100644 index 0000000000..abeee22b6f --- /dev/null +++ b/tools/clang-tidy/test/misc-include-cleaner.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "included header \"structures.h\" is not used directly", + "misc-include-cleaner" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/misc-non-copyable-objects.cpp b/tools/clang-tidy/test/misc-non-copyable-objects.cpp new file mode 100644 index 0000000000..2f9060a818 --- /dev/null +++ b/tools/clang-tidy/test/misc-non-copyable-objects.cpp @@ -0,0 +1,5 @@ +namespace std { +typedef struct FILE {} FILE; +} + +void g(std::FILE f); diff --git a/tools/clang-tidy/test/misc-non-copyable-objects.json b/tools/clang-tidy/test/misc-non-copyable-objects.json new file mode 100644 index 0000000000..30826f3700 --- /dev/null +++ b/tools/clang-tidy/test/misc-non-copyable-objects.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'f' declared as type 'FILE', which is unsafe to copy; did you mean 'FILE *'?", + "misc-non-copyable-objects" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/misc-redundant-expression.cpp b/tools/clang-tidy/test/misc-redundant-expression.cpp new file mode 100644 index 0000000000..0ccc9c55f7 --- /dev/null +++ b/tools/clang-tidy/test/misc-redundant-expression.cpp @@ -0,0 +1,3 @@ +int TestSimpleEquivalent(int X, int Y) { + if (X - X) return 1; +} diff --git a/tools/clang-tidy/test/misc-redundant-expression.json b/tools/clang-tidy/test/misc-redundant-expression.json new file mode 100644 index 0000000000..e4bffef44c --- /dev/null +++ b/tools/clang-tidy/test/misc-redundant-expression.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "both sides of operator are equivalent", + "misc-redundant-expression" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/misc-unused-alias-decls.cpp b/tools/clang-tidy/test/misc-unused-alias-decls.cpp new file mode 100644 index 0000000000..1bbaa2db0a --- /dev/null +++ b/tools/clang-tidy/test/misc-unused-alias-decls.cpp @@ -0,0 +1,18 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/misc-unused-alias-decls.html + +namespace n1 { + namespace n2 { + namespace n3 { + int qux = 42; + } + } +} + +namespace n1_unused = ::n1; // WARNING +namespace n12_unused = n1::n2; // WARNING +namespace n123 = n1::n2::n3; // OK + +int test() +{ + return n123::qux; +} diff --git a/tools/clang-tidy/test/misc-unused-alias-decls.json b/tools/clang-tidy/test/misc-unused-alias-decls.json new file mode 100644 index 0000000000..1144566e4e --- /dev/null +++ b/tools/clang-tidy/test/misc-unused-alias-decls.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "namespace alias decl 'n1_unused' is unused", + "misc-unused-alias-decls" + ], + [ + "warning", + "namespace alias decl 'n12_unused' is unused", + "misc-unused-alias-decls" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/misc-unused-using-decls.cpp b/tools/clang-tidy/test/misc-unused-using-decls.cpp new file mode 100644 index 0000000000..6564711aba --- /dev/null +++ b/tools/clang-tidy/test/misc-unused-using-decls.cpp @@ -0,0 +1,4 @@ + +// misc-unused-using-decls +namespace n { class C; } +using n::C; diff --git a/tools/clang-tidy/test/misc-unused-using-decls.json b/tools/clang-tidy/test/misc-unused-using-decls.json new file mode 100644 index 0000000000..fc4156adca --- /dev/null +++ b/tools/clang-tidy/test/misc-unused-using-decls.json @@ -0,0 +1,4 @@ +[ + ["warning", "using decl 'C' is unused", "misc-unused-using-decls"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-avoid-bind.cpp b/tools/clang-tidy/test/modernize-avoid-bind.cpp new file mode 100644 index 0000000000..c46fb31cd5 --- /dev/null +++ b/tools/clang-tidy/test/modernize-avoid-bind.cpp @@ -0,0 +1,6 @@ +#include "structures.h" + +int add(int x, int y) { return x + y; } +void f_bind() { + auto clj = std::bind(add, 2, 2); +} diff --git a/tools/clang-tidy/test/modernize-avoid-bind.json b/tools/clang-tidy/test/modernize-avoid-bind.json new file mode 100644 index 0000000000..915f62f042 --- /dev/null +++ b/tools/clang-tidy/test/modernize-avoid-bind.json @@ -0,0 +1,4 @@ +[ + ["warning", "prefer a lambda to std::bind", "modernize-avoid-bind"], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/modernize-concat-nested-namespaces.cpp b/tools/clang-tidy/test/modernize-concat-nested-namespaces.cpp new file mode 100644 index 0000000000..0ff35d0e07 --- /dev/null +++ b/tools/clang-tidy/test/modernize-concat-nested-namespaces.cpp @@ -0,0 +1,5 @@ +namespace mozilla { +namespace dom { +void foo(); +} +} diff --git a/tools/clang-tidy/test/modernize-concat-nested-namespaces.json b/tools/clang-tidy/test/modernize-concat-nested-namespaces.json new file mode 100644 index 0000000000..2c7bc6f52b --- /dev/null +++ b/tools/clang-tidy/test/modernize-concat-nested-namespaces.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "nested namespaces can be concatenated", + "modernize-concat-nested-namespaces" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.cpp b/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.cpp new file mode 100644 index 0000000000..288fac8483 --- /dev/null +++ b/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.cpp @@ -0,0 +1,18 @@ +namespace std { +class ios_base { +public: + typedef int io_state; + typedef int open_mode; + typedef int seek_dir; + + typedef int streampos; + typedef int streamoff; +}; + +template <class CharT> +class basic_ios : public ios_base { +}; +} // namespace std + +// Test function return values (declaration) +std::ios_base::io_state f_5(); diff --git a/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.json b/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.json new file mode 100644 index 0000000000..7d6d0d055f --- /dev/null +++ b/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'std::ios_base::io_state' is deprecated; use 'std::ios_base::iostate' instead", + "modernize-deprecated-ios-base-aliases" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-loop-convert.cpp b/tools/clang-tidy/test/modernize-loop-convert.cpp new file mode 100644 index 0000000000..2205846c7e --- /dev/null +++ b/tools/clang-tidy/test/modernize-loop-convert.cpp @@ -0,0 +1,7 @@ +int arr[6] = {1, 2, 3, 4, 5, 6}; + +void bar(void) { + for (int i = 0; i < 6; ++i) { + (void)arr[i]; + } +} diff --git a/tools/clang-tidy/test/modernize-loop-convert.json b/tools/clang-tidy/test/modernize-loop-convert.json new file mode 100644 index 0000000000..105e44bc2b --- /dev/null +++ b/tools/clang-tidy/test/modernize-loop-convert.json @@ -0,0 +1,4 @@ +[ + ["warning", "use range-based for loop instead", "modernize-loop-convert"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-raw-string-literal.cpp b/tools/clang-tidy/test/modernize-raw-string-literal.cpp new file mode 100644 index 0000000000..d120aad9bf --- /dev/null +++ b/tools/clang-tidy/test/modernize-raw-string-literal.cpp @@ -0,0 +1 @@ +char const *const ManyQuotes("quotes:\'\'\'\'"); diff --git a/tools/clang-tidy/test/modernize-raw-string-literal.json b/tools/clang-tidy/test/modernize-raw-string-literal.json new file mode 100644 index 0000000000..595deaa7db --- /dev/null +++ b/tools/clang-tidy/test/modernize-raw-string-literal.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "escaped string literal can be written as a raw string literal", + "modernize-raw-string-literal" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-redundant-void-arg.cpp b/tools/clang-tidy/test/modernize-redundant-void-arg.cpp new file mode 100644 index 0000000000..078e0e13ca --- /dev/null +++ b/tools/clang-tidy/test/modernize-redundant-void-arg.cpp @@ -0,0 +1,5 @@ +// modernize-redundant-void-arg + +int foo(void) { + return 0; +} diff --git a/tools/clang-tidy/test/modernize-redundant-void-arg.json b/tools/clang-tidy/test/modernize-redundant-void-arg.json new file mode 100644 index 0000000000..a34b4069b6 --- /dev/null +++ b/tools/clang-tidy/test/modernize-redundant-void-arg.json @@ -0,0 +1 @@ +"[[\"warning\", \"redundant void argument list in function definition\", \"modernize-redundant-void-arg\"]]" diff --git a/tools/clang-tidy/test/modernize-shrink-to-fit.cpp b/tools/clang-tidy/test/modernize-shrink-to-fit.cpp new file mode 100644 index 0000000000..c545a517a3 --- /dev/null +++ b/tools/clang-tidy/test/modernize-shrink-to-fit.cpp @@ -0,0 +1,10 @@ +#include "structures.h" + +void f() { + std::vector<int> v; + + std::vector<int>(v).swap(v); + + std::vector<int> &vref = v; + std::vector<int>(vref).swap(vref); +} diff --git a/tools/clang-tidy/test/modernize-shrink-to-fit.json b/tools/clang-tidy/test/modernize-shrink-to-fit.json new file mode 100644 index 0000000000..f8e4eda19f --- /dev/null +++ b/tools/clang-tidy/test/modernize-shrink-to-fit.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "the shrink_to_fit method should be used to reduce the capacity of a shrinkable container", + "modernize-shrink-to-fit" + ], + [ + "warning", + "the shrink_to_fit method should be used to reduce the capacity of a shrinkable container", + "modernize-shrink-to-fit" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-use-auto.cpp b/tools/clang-tidy/test/modernize-use-auto.cpp new file mode 100644 index 0000000000..ba54d0cb8a --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-auto.cpp @@ -0,0 +1,11 @@ +#include <vector> + +void func() { + int val = 42; + std::vector<int> my_container; + for (std::vector<int>::iterator I = my_container.begin(), + E = my_container.end(); + I != E; + ++I) { + } +} diff --git a/tools/clang-tidy/test/modernize-use-auto.json b/tools/clang-tidy/test/modernize-use-auto.json new file mode 100644 index 0000000000..5885e8b6d5 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-auto.json @@ -0,0 +1 @@ +"[[\"warning\", \"use auto when declaring iterators\", \"modernize-use-auto\"]]" diff --git a/tools/clang-tidy/test/modernize-use-bool-literals.cpp b/tools/clang-tidy/test/modernize-use-bool-literals.cpp new file mode 100644 index 0000000000..58b7ec8f29 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-bool-literals.cpp @@ -0,0 +1,5 @@ +void foo() { + bool p = 1; + bool f = static_cast<bool>(1); + bool x = p ? 1 : 0; +} diff --git a/tools/clang-tidy/test/modernize-use-bool-literals.json b/tools/clang-tidy/test/modernize-use-bool-literals.json new file mode 100644 index 0000000000..4d5effc1b9 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-bool-literals.json @@ -0,0 +1,23 @@ +[ + [ + "warning", + "converting integer literal to bool, use bool literal instead", + "modernize-use-bool-literals" + ], + [ + "warning", + "converting integer literal to bool, use bool literal instead", + "modernize-use-bool-literals" + ], + [ + "warning", + "converting integer literal to bool, use bool literal instead", + "modernize-use-bool-literals" + ], + [ + "warning", + "converting integer literal to bool, use bool literal instead", + "modernize-use-bool-literals" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-use-equals-default.cpp b/tools/clang-tidy/test/modernize-use-equals-default.cpp new file mode 100644 index 0000000000..ea5f71d93a --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-equals-default.cpp @@ -0,0 +1,6 @@ + +class IL { +public: + IL() {} + ~IL() {} +}; diff --git a/tools/clang-tidy/test/modernize-use-equals-default.json b/tools/clang-tidy/test/modernize-use-equals-default.json new file mode 100644 index 0000000000..5119e361b3 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-equals-default.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "use '= default' to define a trivial default constructor", + "modernize-use-equals-default" + ], + [ + "warning", + "use '= default' to define a trivial destructor", + "modernize-use-equals-default" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-use-equals-delete.cpp b/tools/clang-tidy/test/modernize-use-equals-delete.cpp new file mode 100644 index 0000000000..f08848c0a6 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-equals-delete.cpp @@ -0,0 +1,7 @@ +struct PositivePrivate { +private: + PositivePrivate(); + PositivePrivate(const PositivePrivate &); + PositivePrivate &operator=(PositivePrivate &&); + ~PositivePrivate(); +}; diff --git a/tools/clang-tidy/test/modernize-use-equals-delete.json b/tools/clang-tidy/test/modernize-use-equals-delete.json new file mode 100644 index 0000000000..bcb9354149 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-equals-delete.json @@ -0,0 +1,23 @@ +[ + [ + "warning", + "use '= delete' to prohibit calling of a special member function", + "modernize-use-equals-delete" + ], + [ + "warning", + "use '= delete' to prohibit calling of a special member function", + "modernize-use-equals-delete" + ], + [ + "warning", + "use '= delete' to prohibit calling of a special member function", + "modernize-use-equals-delete" + ], + [ + "warning", + "use '= delete' to prohibit calling of a special member function", + "modernize-use-equals-delete" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-use-nullptr.cpp b/tools/clang-tidy/test/modernize-use-nullptr.cpp new file mode 100644 index 0000000000..4b8b3ee3c0 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-nullptr.cpp @@ -0,0 +1,5 @@ +#define NULL 0 +void f(void) { + char *str = NULL; // ok + (void)str; +} diff --git a/tools/clang-tidy/test/modernize-use-nullptr.json b/tools/clang-tidy/test/modernize-use-nullptr.json new file mode 100644 index 0000000000..44bd18a10c --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-nullptr.json @@ -0,0 +1 @@ +[["warning", "use nullptr", "modernize-use-nullptr"], { "reliability": "high" }] diff --git a/tools/clang-tidy/test/modernize-use-override.cpp b/tools/clang-tidy/test/modernize-use-override.cpp new file mode 100644 index 0000000000..1cbec3868c --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-override.cpp @@ -0,0 +1,8 @@ +class Base { +public: + virtual void foo() = 0; +}; + +class Deriv : public Base { + void foo(); +}; diff --git a/tools/clang-tidy/test/modernize-use-override.json b/tools/clang-tidy/test/modernize-use-override.json new file mode 100644 index 0000000000..64f8b6e870 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-override.json @@ -0,0 +1 @@ +"[[\"warning\", \"annotate this function with 'override' or (rarely) 'final'\", \"modernize-use-override\"]]" diff --git a/tools/clang-tidy/test/modernize-use-using.cpp b/tools/clang-tidy/test/modernize-use-using.cpp new file mode 100644 index 0000000000..0e7b73222c --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-using.cpp @@ -0,0 +1,5 @@ +template <typename T> +class Test { + typedef typename T::iterator Iter; +}; +typedef int Type; diff --git a/tools/clang-tidy/test/modernize-use-using.json b/tools/clang-tidy/test/modernize-use-using.json new file mode 100644 index 0000000000..50ee7060a9 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-using.json @@ -0,0 +1,5 @@ +[ + ["warning", "use 'using' instead of 'typedef'", "modernize-use-using"], + ["warning", "use 'using' instead of 'typedef'", "modernize-use-using"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-avoid-endl.cpp b/tools/clang-tidy/test/performance-avoid-endl.cpp new file mode 100644 index 0000000000..1d7b395b57 --- /dev/null +++ b/tools/clang-tidy/test/performance-avoid-endl.cpp @@ -0,0 +1,34 @@ +namespace std { + template <typename CharT> + class basic_ostream { + public: + template <typename T> + basic_ostream& operator<<(T); + basic_ostream& operator<<(basic_ostream<CharT>& (*)(basic_ostream<CharT>&)); + }; + + template <typename CharT> + class basic_iostream : public basic_ostream<CharT> {}; + + using ostream = basic_ostream<char>; + using wostream = basic_ostream<wchar_t>; + + using iostream = basic_iostream<char>; + using wiostream = basic_iostream<wchar_t>; + + ostream cout; + wostream wcout; + + ostream cerr; + wostream wcerr; + + ostream clog; + wostream wclog; + + template<typename CharT> + basic_ostream<CharT>& endl(basic_ostream<CharT>&); +} // namespace std + +int main() { + std::cout << "Hello" << std::endl; +} diff --git a/tools/clang-tidy/test/performance-avoid-endl.json b/tools/clang-tidy/test/performance-avoid-endl.json new file mode 100644 index 0000000000..b5e7695f64 --- /dev/null +++ b/tools/clang-tidy/test/performance-avoid-endl.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "do not use 'std::endl' with streams; use '\\n' instead", + "performance-avoid-endl" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-faster-string-find.cpp b/tools/clang-tidy/test/performance-faster-string-find.cpp new file mode 100644 index 0000000000..d7ac3d0c3c --- /dev/null +++ b/tools/clang-tidy/test/performance-faster-string-find.cpp @@ -0,0 +1,6 @@ +#include "structures.h" + +void foo() { + std::string str; + str.find("A"); +} diff --git a/tools/clang-tidy/test/performance-faster-string-find.json b/tools/clang-tidy/test/performance-faster-string-find.json new file mode 100644 index 0000000000..1ab2d7ba08 --- /dev/null +++ b/tools/clang-tidy/test/performance-faster-string-find.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'find' called with a string literal consisting of a single character; consider using the more effective overload accepting a character", + "performance-faster-string-find" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-for-range-copy.cpp b/tools/clang-tidy/test/performance-for-range-copy.cpp new file mode 100644 index 0000000000..264dd42896 --- /dev/null +++ b/tools/clang-tidy/test/performance-for-range-copy.cpp @@ -0,0 +1,30 @@ +template <typename T> +struct Iterator { + void operator++() {} + const T& operator*() { + static T* TT = new T(); + return *TT; + } + bool operator!=(const Iterator &) { return false; } + typedef const T& const_reference; +}; +template <typename T> +struct View { + T begin() { return T(); } + T begin() const { return T(); } + T end() { return T(); } + T end() const { return T(); } + typedef typename T::const_reference const_reference; +}; + +struct S { + S(); + S(const S &); + ~S(); + S &operator=(const S &); +}; + +void negativeConstReference() { + for (const S S1 : View<Iterator<S>>()) { + } +} diff --git a/tools/clang-tidy/test/performance-for-range-copy.json b/tools/clang-tidy/test/performance-for-range-copy.json new file mode 100644 index 0000000000..2082041aad --- /dev/null +++ b/tools/clang-tidy/test/performance-for-range-copy.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the loop variable's type is not a reference type; this creates a copy in each iteration; consider making this a reference", + "performance-for-range-copy" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-implicit-conversion-in-loop.cpp b/tools/clang-tidy/test/performance-implicit-conversion-in-loop.cpp new file mode 100644 index 0000000000..75ed510f3f --- /dev/null +++ b/tools/clang-tidy/test/performance-implicit-conversion-in-loop.cpp @@ -0,0 +1,43 @@ +// Iterator returning by value. +template <typename T> +struct Iterator { + void operator++(); + T operator*(); + bool operator!=(const Iterator& other); +}; + +// The template argument is an iterator type, and a view is an object you can +// run a for loop on. +template <typename T> +struct View { + T begin(); + T end(); +}; + +// With this class, the implicit conversion is a call to the (implicit) +// constructor of the class. +template <typename T> +class ImplicitWrapper { + public: + // Implicit! + ImplicitWrapper(const T& t); +}; + +template <typename T> +class OperatorWrapper { + public: + OperatorWrapper() = delete; +}; + +struct SimpleClass { + int foo; + operator OperatorWrapper<SimpleClass>(); +}; + +typedef View<Iterator<SimpleClass>> SimpleView; + +void ImplicitSimpleClassIterator() { + for (const ImplicitWrapper<SimpleClass>& foo : SimpleView()) {} + for (const ImplicitWrapper<SimpleClass> foo : SimpleView()) {} + for (ImplicitWrapper<SimpleClass> foo : SimpleView()) {} +} diff --git a/tools/clang-tidy/test/performance-implicit-conversion-in-loop.json b/tools/clang-tidy/test/performance-implicit-conversion-in-loop.json new file mode 100644 index 0000000000..d9bcb1e00a --- /dev/null +++ b/tools/clang-tidy/test/performance-implicit-conversion-in-loop.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the type of the loop variable 'foo' is different from the one returned by the iterator and generates an implicit conversion; you can either change the type to the matching one ('const SimpleClass &' but 'const auto&' is always a valid option) or remove the reference to make it explicit that you are creating a new value", + "performance-implicit-conversion-in-loop" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-inefficient-algorithm.cpp b/tools/clang-tidy/test/performance-inefficient-algorithm.cpp new file mode 100644 index 0000000000..b508260dbd --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-algorithm.cpp @@ -0,0 +1,30 @@ +namespace std { +template <typename T> struct less { + bool operator()(const T &lhs, const T &rhs) { return lhs < rhs; } +}; + +template <typename T> struct greater { + bool operator()(const T &lhs, const T &rhs) { return lhs > rhs; } +}; + +struct iterator_type {}; + +template <typename K, typename Cmp = less<K>> struct set { + typedef iterator_type iterator; + iterator find(const K &k); + unsigned count(const K &k); + + iterator begin(); + iterator end(); + iterator begin() const; + iterator end() const; +}; + +template <typename FwIt, typename K> +FwIt find(FwIt, FwIt end, const K &) { return end; } +} + +template <typename T> void f(const T &t) { + std::set<int> s; + find(s.begin(), s.end(), 46); +} diff --git a/tools/clang-tidy/test/performance-inefficient-algorithm.json b/tools/clang-tidy/test/performance-inefficient-algorithm.json new file mode 100644 index 0000000000..e3c575bd85 --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-algorithm.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "this STL algorithm call should be replaced with a container method", + "performance-inefficient-algorithm" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-inefficient-string-concatenation.cpp b/tools/clang-tidy/test/performance-inefficient-string-concatenation.cpp new file mode 100644 index 0000000000..5a5860f215 --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-string-concatenation.cpp @@ -0,0 +1,13 @@ +#include "structures.h" + +extern void fstring(std::string); + +void foo() { + std::string mystr1, mystr2; + auto myautostr1 = mystr1; + auto myautostr2 = mystr2; + + for (int i = 0; i < 10; ++i) { + fstring(mystr1 + mystr2 + mystr1); + } +} diff --git a/tools/clang-tidy/test/performance-inefficient-string-concatenation.json b/tools/clang-tidy/test/performance-inefficient-string-concatenation.json new file mode 100644 index 0000000000..8c7223db6d --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-string-concatenation.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "string concatenation results in allocation of unnecessary temporary strings; consider using 'operator+=' or 'string::append()' instead", + "performance-inefficient-string-concatenation" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-inefficient-vector-operation.cpp b/tools/clang-tidy/test/performance-inefficient-vector-operation.cpp new file mode 100644 index 0000000000..4f0e143aa2 --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-vector-operation.cpp @@ -0,0 +1,10 @@ +#include "structures.h" + +void foo() +{ + std::vector<int> v; + int n = 100; + for (int i = 0; i < n; ++i) { + v.push_back(n); + } +} diff --git a/tools/clang-tidy/test/performance-inefficient-vector-operation.json b/tools/clang-tidy/test/performance-inefficient-vector-operation.json new file mode 100644 index 0000000000..d3e9e769d2 --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-vector-operation.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'push_back' is called inside a loop; consider pre-allocating the container capacity before the loop", + "performance-inefficient-vector-operation" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-move-const-arg.cpp b/tools/clang-tidy/test/performance-move-const-arg.cpp new file mode 100644 index 0000000000..39f1ce06ec --- /dev/null +++ b/tools/clang-tidy/test/performance-move-const-arg.cpp @@ -0,0 +1,33 @@ +namespace std { +template <typename _Tp> +struct remove_reference { + typedef _Tp type; +}; + +template <typename _Tp> +constexpr typename std::remove_reference<_Tp>::type &&move(_Tp &&__t) { + return static_cast<typename std::remove_reference<_Tp>::type &&>(__t); +} +} // namespace std + +struct TriviallyCopyable { + int i; +}; + +class A { +public: + A() {} + A(const A &rhs) {} + A(A &&rhs) {} +}; + +void f(TriviallyCopyable) {} + +void g() { + TriviallyCopyable obj; + f(std::move(obj)); +} + +A f5(const A x5) { + return std::move(x5); +} diff --git a/tools/clang-tidy/test/performance-move-const-arg.json b/tools/clang-tidy/test/performance-move-const-arg.json new file mode 100644 index 0000000000..d56fe6bf37 --- /dev/null +++ b/tools/clang-tidy/test/performance-move-const-arg.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "std::move of the const variable 'x5' has no effect; remove std::move() or make the variable non-const", + "performance-move-const-arg" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-move-constructor-init.cpp b/tools/clang-tidy/test/performance-move-constructor-init.cpp new file mode 100644 index 0000000000..243b399e95 --- /dev/null +++ b/tools/clang-tidy/test/performance-move-constructor-init.cpp @@ -0,0 +1,11 @@ +struct B { + B() {} + B(const B&) {} + B(B &&) {} +}; + +struct D : B { + D() : B() {} + D(const D &RHS) : B(RHS) {} + D(D &&RHS) : B(RHS) {} +}; diff --git a/tools/clang-tidy/test/performance-move-constructor-init.json b/tools/clang-tidy/test/performance-move-constructor-init.json new file mode 100644 index 0000000000..17582a86e7 --- /dev/null +++ b/tools/clang-tidy/test/performance-move-constructor-init.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "move constructor initializes base class by calling a copy constructor", + "performance-move-constructor-init" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-noexcept-move-constructor.cpp b/tools/clang-tidy/test/performance-noexcept-move-constructor.cpp new file mode 100644 index 0000000000..8b4900b00d --- /dev/null +++ b/tools/clang-tidy/test/performance-noexcept-move-constructor.cpp @@ -0,0 +1,4 @@ +class A { + A(A &&); + A &operator=(A &&); +}; diff --git a/tools/clang-tidy/test/performance-noexcept-move-constructor.json b/tools/clang-tidy/test/performance-noexcept-move-constructor.json new file mode 100644 index 0000000000..94823b9ed5 --- /dev/null +++ b/tools/clang-tidy/test/performance-noexcept-move-constructor.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "move constructors should be marked noexcept", + "performance-noexcept-move-constructor" + ], + [ + "warning", + "move assignment operators should be marked noexcept", + "performance-noexcept-move-constructor" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-type-promotion-in-math-fn.cpp b/tools/clang-tidy/test/performance-type-promotion-in-math-fn.cpp new file mode 100644 index 0000000000..9a6fcf9848 --- /dev/null +++ b/tools/clang-tidy/test/performance-type-promotion-in-math-fn.cpp @@ -0,0 +1,7 @@ +double acos(double); + +void check_all_fns() +{ + float a; + acos(a); +} diff --git a/tools/clang-tidy/test/performance-type-promotion-in-math-fn.json b/tools/clang-tidy/test/performance-type-promotion-in-math-fn.json new file mode 100644 index 0000000000..577d2ddc91 --- /dev/null +++ b/tools/clang-tidy/test/performance-type-promotion-in-math-fn.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "call to 'acos' promotes float to double", + "performance-type-promotion-in-math-fn" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-unnecessary-copy-initialization.cpp b/tools/clang-tidy/test/performance-unnecessary-copy-initialization.cpp new file mode 100644 index 0000000000..ca0f591a3e --- /dev/null +++ b/tools/clang-tidy/test/performance-unnecessary-copy-initialization.cpp @@ -0,0 +1,7 @@ +#include "structures.h" + +extern const std::string& constReference(); + +void foo() { + const std::string UnnecessaryCopy = constReference(); +} diff --git a/tools/clang-tidy/test/performance-unnecessary-copy-initialization.json b/tools/clang-tidy/test/performance-unnecessary-copy-initialization.json new file mode 100644 index 0000000000..fcb16746ed --- /dev/null +++ b/tools/clang-tidy/test/performance-unnecessary-copy-initialization.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the const qualified variable 'UnnecessaryCopy' is copy-constructed from a const reference but is never used; consider removing the statement", + "performance-unnecessary-copy-initialization" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-unnecessary-value-param.cpp b/tools/clang-tidy/test/performance-unnecessary-value-param.cpp new file mode 100644 index 0000000000..ed5f36f7fa --- /dev/null +++ b/tools/clang-tidy/test/performance-unnecessary-value-param.cpp @@ -0,0 +1,4 @@ +#include "structures.h" + +void f(const std::string Value) { +} diff --git a/tools/clang-tidy/test/performance-unnecessary-value-param.json b/tools/clang-tidy/test/performance-unnecessary-value-param.json new file mode 100644 index 0000000000..35ed09e4be --- /dev/null +++ b/tools/clang-tidy/test/performance-unnecessary-value-param.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the const qualified parameter 'Value' is copied for each invocation; consider making it a reference", + "performance-unnecessary-value-param" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-braces-around-statements.cpp b/tools/clang-tidy/test/readability-braces-around-statements.cpp new file mode 100644 index 0000000000..ce456ec2e3 --- /dev/null +++ b/tools/clang-tidy/test/readability-braces-around-statements.cpp @@ -0,0 +1,38 @@ + +void do_something(const char *) {} + +bool cond(const char *) { + return false; +} + +void test() { +if (cond("if0") /*comment*/) do_something("same-line"); + +if (cond("if1") /*comment*/) + do_something("next-line"); + + if (!1) return; + if (!2) { return; } + + if (!3) + return; + + if (!4) { + return; + } +} + +void foo() { +if (1) while (2) if (3) for (;;) do ; while(false) /**/;/**/ +} + +void f() {} + +void foo2() { + constexpr bool a = true; + if constexpr (a) { + f(); + } else { + f(); + } +} diff --git a/tools/clang-tidy/test/readability-braces-around-statements.json b/tools/clang-tidy/test/readability-braces-around-statements.json new file mode 100644 index 0000000000..7cc8ae280a --- /dev/null +++ b/tools/clang-tidy/test/readability-braces-around-statements.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "statement should be inside braces", + "readability-braces-around-statements" + ], + [ + "warning", + "statement should be inside braces", + "readability-braces-around-statements" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-const-return-type.cpp b/tools/clang-tidy/test/readability-const-return-type.cpp new file mode 100644 index 0000000000..695ed1d83f --- /dev/null +++ b/tools/clang-tidy/test/readability-const-return-type.cpp @@ -0,0 +1,5 @@ +const int p1() { +// CHECK-MESSAGES: [[@LINE-1]]:1: warning: return type 'const int' is 'const'-qualified at the top level, which may reduce code readability without improving const correctness +// CHECK-FIXES: int p1() { + return 1; +} diff --git a/tools/clang-tidy/test/readability-const-return-type.json b/tools/clang-tidy/test/readability-const-return-type.json new file mode 100644 index 0000000000..f6c634c0b1 --- /dev/null +++ b/tools/clang-tidy/test/readability-const-return-type.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "return type 'const int' is 'const'-qualified at the top level, which may reduce code readability without improving const correctness", + "readability-const-return-type" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-container-size-empty.cpp b/tools/clang-tidy/test/readability-container-size-empty.cpp new file mode 100644 index 0000000000..1dcfe68bd7 --- /dev/null +++ b/tools/clang-tidy/test/readability-container-size-empty.cpp @@ -0,0 +1,7 @@ +#include "structures.h" + +void foo() { + std::string a; + if (a.size()) + return; +} diff --git a/tools/clang-tidy/test/readability-container-size-empty.json b/tools/clang-tidy/test/readability-container-size-empty.json new file mode 100644 index 0000000000..8c51b1c039 --- /dev/null +++ b/tools/clang-tidy/test/readability-container-size-empty.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the 'empty' method should be used to check for emptiness instead of 'size'", + "readability-container-size-empty" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-delete-null-pointer.cpp b/tools/clang-tidy/test/readability-delete-null-pointer.cpp new file mode 100644 index 0000000000..e083404043 --- /dev/null +++ b/tools/clang-tidy/test/readability-delete-null-pointer.cpp @@ -0,0 +1,6 @@ +void func() { + int* f = 0; + if (f) { + delete f; + } +} diff --git a/tools/clang-tidy/test/readability-delete-null-pointer.json b/tools/clang-tidy/test/readability-delete-null-pointer.json new file mode 100644 index 0000000000..2a7184cdb1 --- /dev/null +++ b/tools/clang-tidy/test/readability-delete-null-pointer.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'if' statement is unnecessary; deleting null pointer has no effect", + "readability-delete-null-pointer" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-else-after-return.cpp b/tools/clang-tidy/test/readability-else-after-return.cpp new file mode 100644 index 0000000000..2c72b68a92 --- /dev/null +++ b/tools/clang-tidy/test/readability-else-after-return.cpp @@ -0,0 +1,10 @@ +void f() { + +} + +void foo() { + if (true) + return; + else + f(); +} diff --git a/tools/clang-tidy/test/readability-else-after-return.json b/tools/clang-tidy/test/readability-else-after-return.json new file mode 100644 index 0000000000..7d2f8b0adf --- /dev/null +++ b/tools/clang-tidy/test/readability-else-after-return.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "do not use 'else' after 'return'", + "readability-else-after-return" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-implicit-bool-conversion.cpp b/tools/clang-tidy/test/readability-implicit-bool-conversion.cpp new file mode 100644 index 0000000000..c30089a126 --- /dev/null +++ b/tools/clang-tidy/test/readability-implicit-bool-conversion.cpp @@ -0,0 +1,55 @@ + +#define MOZ_IMPLICIT __attribute__((annotate("moz_implicit"))) + +void takesChar(char); +void takesShort(short); +void takesInt(int); +void takesLong(long); + +void takesUChar(unsigned char); +void takesUShort(unsigned short); +void takesUInt(unsigned int); +void takesULong(unsigned long); + +struct InitializedWithInt { + MOZ_IMPLICIT InitializedWithInt(int); +}; + +void f() { + bool b = true; + char s0 = b; + short s1 = b; + int s2 = b; + long s3 = b; + + unsigned char u0 = b; + unsigned short u1 = b; + unsigned u2 = b; + unsigned long u3 = b; + + takesChar(b); + takesShort(b); + takesInt(b); + takesLong(b); + takesUChar(b); + takesUShort(b); + takesUInt(b); + takesULong(b); + + InitializedWithInt i = b; + (InitializedWithInt(b)); + + bool x = b; + + int exp = (int)true; + + if (x == b) {} + if (x != b) {} + + if (b == exp) {} + if (exp == b) {} + + char* ptr; + // Shouldn't trigger a checker warning since we are using AllowPointerConditions + if (ptr) {} +} diff --git a/tools/clang-tidy/test/readability-implicit-bool-conversion.json b/tools/clang-tidy/test/readability-implicit-bool-conversion.json new file mode 100644 index 0000000000..b10044f03f --- /dev/null +++ b/tools/clang-tidy/test/readability-implicit-bool-conversion.json @@ -0,0 +1,102 @@ +[ + [ + "warning", + "implicit conversion bool -> 'char'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'short'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'long'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned char'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned short'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned long'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'char'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'short'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'long'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned char'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned short'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned long'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ] +] diff --git a/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.cpp b/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.cpp new file mode 100644 index 0000000000..acf13b240b --- /dev/null +++ b/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.cpp @@ -0,0 +1,6 @@ +struct S { + void f(int x); +}; + +void S::f(int y) { +} diff --git a/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.json b/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.json new file mode 100644 index 0000000000..f5e04a1041 --- /dev/null +++ b/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "function 'S::f' has a definition with different parameter names", + "readability-inconsistent-declaration-parameter-name" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-isolate-declaration.cpp b/tools/clang-tidy/test/readability-isolate-declaration.cpp new file mode 100644 index 0000000000..5d64e0171f --- /dev/null +++ b/tools/clang-tidy/test/readability-isolate-declaration.cpp @@ -0,0 +1,7 @@ +void f() { + int * pointer = nullptr, value = 42, * const const_ptr = &value; + // This declaration will be diagnosed and transformed into: + // int * pointer = nullptr; + // int value = 42; + // int * const const_ptr = &value; +} diff --git a/tools/clang-tidy/test/readability-isolate-declaration.json b/tools/clang-tidy/test/readability-isolate-declaration.json new file mode 100644 index 0000000000..3f8ccf52fa --- /dev/null +++ b/tools/clang-tidy/test/readability-isolate-declaration.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "multiple declarations in a single statement reduces readability", + "readability-isolate-declaration" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-magic-numbers.cpp b/tools/clang-tidy/test/readability-magic-numbers.cpp new file mode 100644 index 0000000000..430fa17056 --- /dev/null +++ b/tools/clang-tidy/test/readability-magic-numbers.cpp @@ -0,0 +1,4 @@ +void func() { + int radius = 2; + double circleArea = 3.1415926535 * radius * radius; +} diff --git a/tools/clang-tidy/test/readability-magic-numbers.json b/tools/clang-tidy/test/readability-magic-numbers.json new file mode 100644 index 0000000000..ee5d0f204c --- /dev/null +++ b/tools/clang-tidy/test/readability-magic-numbers.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "3.1415926535 is a magic number; consider replacing it with a named constant", + "readability-magic-numbers" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-misleading-indentation.cpp b/tools/clang-tidy/test/readability-misleading-indentation.cpp new file mode 100644 index 0000000000..8b23010170 --- /dev/null +++ b/tools/clang-tidy/test/readability-misleading-indentation.cpp @@ -0,0 +1,20 @@ +void f() +{ +} + +void foo() { + if (1) + if (0) + f(); + else + f(); +} + +void foo2() { + constexpr bool a = true; + if constexpr (a) { + f(); + } else { + f(); + } +} diff --git a/tools/clang-tidy/test/readability-misleading-indentation.json b/tools/clang-tidy/test/readability-misleading-indentation.json new file mode 100644 index 0000000000..9ef6d30c45 --- /dev/null +++ b/tools/clang-tidy/test/readability-misleading-indentation.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "different indentation for 'if' and corresponding 'else'", + "readability-misleading-indentation" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-non-const-parameter.cpp b/tools/clang-tidy/test/readability-non-const-parameter.cpp new file mode 100644 index 0000000000..5a0ffd0e30 --- /dev/null +++ b/tools/clang-tidy/test/readability-non-const-parameter.cpp @@ -0,0 +1,5 @@ +void warn1(int *first, int *last) { + *first = 0; + if (first < last) { + } +} diff --git a/tools/clang-tidy/test/readability-non-const-parameter.json b/tools/clang-tidy/test/readability-non-const-parameter.json new file mode 100644 index 0000000000..fb83ec8572 --- /dev/null +++ b/tools/clang-tidy/test/readability-non-const-parameter.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "pointer parameter 'last' can be pointer to const", + "readability-non-const-parameter" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-qualified-auto.cpp b/tools/clang-tidy/test/readability-qualified-auto.cpp new file mode 100644 index 0000000000..03c5164e2c --- /dev/null +++ b/tools/clang-tidy/test/readability-qualified-auto.cpp @@ -0,0 +1,6 @@ +typedef int *MyPtr; +MyPtr getPtr(); + +void foo() { + auto TdNakedPtr = getPtr(); +} diff --git a/tools/clang-tidy/test/readability-qualified-auto.json b/tools/clang-tidy/test/readability-qualified-auto.json new file mode 100644 index 0000000000..eaf0a176cf --- /dev/null +++ b/tools/clang-tidy/test/readability-qualified-auto.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'auto TdNakedPtr' can be declared as 'auto *TdNakedPtr'", + "readability-qualified-auto" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-control-flow.cpp b/tools/clang-tidy/test/readability-redundant-control-flow.cpp new file mode 100644 index 0000000000..d221715542 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-control-flow.cpp @@ -0,0 +1,5 @@ +extern void g(); +void f() { + g(); + return; +} diff --git a/tools/clang-tidy/test/readability-redundant-control-flow.json b/tools/clang-tidy/test/readability-redundant-control-flow.json new file mode 100644 index 0000000000..efedaf1009 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-control-flow.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "redundant return statement at the end of a function with a void return type", + "readability-redundant-control-flow" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-member-init.cpp b/tools/clang-tidy/test/readability-redundant-member-init.cpp new file mode 100644 index 0000000000..d8ebe08ec4 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-member-init.cpp @@ -0,0 +1,11 @@ + +struct S { + S() = default; + S(int i) : i(i) {} + int i = 1; +}; + +struct F1 { + F1() : f() {} + S f; +}; diff --git a/tools/clang-tidy/test/readability-redundant-member-init.json b/tools/clang-tidy/test/readability-redundant-member-init.json new file mode 100644 index 0000000000..65b24ba2dd --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-member-init.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "initializer for member 'f' is redundant", + "readability-redundant-member-init" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-preprocessor.cpp b/tools/clang-tidy/test/readability-redundant-preprocessor.cpp new file mode 100644 index 0000000000..317905dab8 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-preprocessor.cpp @@ -0,0 +1,5 @@ +#ifndef FOO +#ifdef FOO // inner ifdef is considered redundant +void f(); +#endif +#endif diff --git a/tools/clang-tidy/test/readability-redundant-preprocessor.json b/tools/clang-tidy/test/readability-redundant-preprocessor.json new file mode 100644 index 0000000000..e3b7fb4ce1 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-preprocessor.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "nested redundant #ifdef; consider removing it", + "readability-redundant-preprocessor" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-smartptr-get.cpp b/tools/clang-tidy/test/readability-redundant-smartptr-get.cpp new file mode 100644 index 0000000000..66eb8d6c37 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-smartptr-get.cpp @@ -0,0 +1,19 @@ +#define NULL __null + +namespace std { + +template <typename T> +struct unique_ptr { + T& operator*() const; + T* operator->() const; + T* get() const; + explicit operator bool() const noexcept; +}; +} + +struct A { +}; + +void foo() { + A& b2 = *std::unique_ptr<A>().get(); +} diff --git a/tools/clang-tidy/test/readability-redundant-smartptr-get.json b/tools/clang-tidy/test/readability-redundant-smartptr-get.json new file mode 100644 index 0000000000..c5f06cbc21 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-smartptr-get.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "redundant get() call on smart pointer", + "readability-redundant-smartptr-get" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-string-cstr.cpp b/tools/clang-tidy/test/readability-redundant-string-cstr.cpp new file mode 100644 index 0000000000..d35a2998e5 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-string-cstr.cpp @@ -0,0 +1,7 @@ +#include "structures.h" + +void foo() { + std::string a = "Mozilla"; + std::string tmp; + tmp.assign(a.c_str()); +} diff --git a/tools/clang-tidy/test/readability-redundant-string-cstr.json b/tools/clang-tidy/test/readability-redundant-string-cstr.json new file mode 100644 index 0000000000..cce79a38a2 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-string-cstr.json @@ -0,0 +1,4 @@ +[ + ["warning", "redundant call to 'c_str'", "readability-redundant-string-cstr"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-string-init.cpp b/tools/clang-tidy/test/readability-redundant-string-init.cpp new file mode 100644 index 0000000000..39bae4bd56 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-string-init.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +int foo() { + std::string a = ""; +} diff --git a/tools/clang-tidy/test/readability-redundant-string-init.json b/tools/clang-tidy/test/readability-redundant-string-init.json new file mode 100644 index 0000000000..3092808205 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-string-init.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "redundant string initialization", + "readability-redundant-string-init" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-simplify-boolean-expr.cpp b/tools/clang-tidy/test/readability-simplify-boolean-expr.cpp new file mode 100644 index 0000000000..277a469880 --- /dev/null +++ b/tools/clang-tidy/test/readability-simplify-boolean-expr.cpp @@ -0,0 +1,2 @@ +bool a1 = false; +bool aa = false == a1; diff --git a/tools/clang-tidy/test/readability-simplify-boolean-expr.json b/tools/clang-tidy/test/readability-simplify-boolean-expr.json new file mode 100644 index 0000000000..45eb9e1a66 --- /dev/null +++ b/tools/clang-tidy/test/readability-simplify-boolean-expr.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "redundant boolean literal supplied to boolean operator", + "readability-simplify-boolean-expr" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-static-accessed-through-instance.cpp b/tools/clang-tidy/test/readability-static-accessed-through-instance.cpp new file mode 100644 index 0000000000..95f13b994c --- /dev/null +++ b/tools/clang-tidy/test/readability-static-accessed-through-instance.cpp @@ -0,0 +1,11 @@ +struct C { + static int x; +}; + +int C::x = 0; + +// Expressions with side effects +C &f(int, int, int, int); +void g() { + f(1, 2, 3, 4).x; +} diff --git a/tools/clang-tidy/test/readability-static-accessed-through-instance.json b/tools/clang-tidy/test/readability-static-accessed-through-instance.json new file mode 100644 index 0000000000..67c84c188b --- /dev/null +++ b/tools/clang-tidy/test/readability-static-accessed-through-instance.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "static member accessed through instance", + "readability-static-accessed-through-instance" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-uniqueptr-delete-release.cpp b/tools/clang-tidy/test/readability-uniqueptr-delete-release.cpp new file mode 100644 index 0000000000..4fce9cde4e --- /dev/null +++ b/tools/clang-tidy/test/readability-uniqueptr-delete-release.cpp @@ -0,0 +1,6 @@ +#include "structures.h" + +int foo() { + std::unique_ptr<int> P; + delete P.release(); +} diff --git a/tools/clang-tidy/test/readability-uniqueptr-delete-release.json b/tools/clang-tidy/test/readability-uniqueptr-delete-release.json new file mode 100644 index 0000000000..79330346e1 --- /dev/null +++ b/tools/clang-tidy/test/readability-uniqueptr-delete-release.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "prefer '= nullptr' to reset 'unique_ptr<>' objects", + "readability-uniqueptr-delete-release" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/structures.h b/tools/clang-tidy/test/structures.h new file mode 100644 index 0000000000..0966109635 --- /dev/null +++ b/tools/clang-tidy/test/structures.h @@ -0,0 +1,108 @@ +// Proxy file in order to define generic data types, to avoid binding with system headers + +typedef __SIZE_TYPE__ size_t; + +namespace std { + +typedef size_t size_t; + +template <class T> +class vector { + public: + typedef T* iterator; + typedef const T* const_iterator; + typedef T& reference; + typedef const T& const_reference; + typedef size_t size_type; + + explicit vector(); + explicit vector(size_type n); + + void swap(vector &other); + void push_back(const T& val); + + template <class... Args> void emplace_back(Args &&... args); + + void reserve(size_t n); + void resize(size_t n); + + size_t size(); + const_reference operator[] (size_type) const; + reference operator[] (size_type); + + const_iterator begin() const; + const_iterator end() const; +}; + +template <typename T> +class basic_string { +public: + typedef basic_string<T> _Type; + basic_string() {} + basic_string(const T *p); + basic_string(const T *p, size_t count); + basic_string(size_t count, char ch); + ~basic_string() {} + size_t size() const; + bool empty() const; + size_t find (const char* s, size_t pos = 0) const; + const T *c_str() const; + _Type& assign(const T *s); + basic_string<T> &operator=(T ch); + basic_string<T> *operator+=(const basic_string<T> &) {} + friend basic_string<T> operator+(const basic_string<T> &, const basic_string<T> &) {} +}; +typedef basic_string<char> string; +typedef basic_string<wchar_t> wstring; + +string to_string(int value); + +template <typename T> +struct default_delete {}; + +template <typename T, typename D = default_delete<T>> +class unique_ptr { + public: + unique_ptr(); + ~unique_ptr(); + explicit unique_ptr(T*); + template <typename U, typename E> + unique_ptr(unique_ptr<U, E>&&); + T* release(); +}; + +template <class Fp, class... Arguments> +class bind_rt {}; + +template <class Fp, class... Arguments> +bind_rt<Fp, Arguments...> bind(Fp &&, Arguments &&...); +} + +typedef unsigned int uid_t; +typedef unsigned int pid_t; + +int bcmp(void *, void *, size_t); +void bcopy(void *, void *, size_t); +void bzero(void *, size_t); + +int getpw(uid_t uid, char *buf); +int setuid(uid_t uid); + +int mkstemp(char *tmpl); +char *mktemp(char *tmpl); + +pid_t vfork(void); + +int abort() { return 0; } + +#define assert(x) \ + if (!(x)) \ + (void)abort() + +size_t strlen(const char *s); +char *strncat(char *s1, const char *s2, size_t); + +void free(void *ptr); +void *malloc(size_t size); + +void *memset(void *b, int c, size_t len); diff --git a/tools/code-coverage/CodeCoverageHandler.cpp b/tools/code-coverage/CodeCoverageHandler.cpp new file mode 100644 index 0000000000..fbe7494c6b --- /dev/null +++ b/tools/code-coverage/CodeCoverageHandler.cpp @@ -0,0 +1,158 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <stdio.h> +#ifdef XP_WIN +# include <process.h> +# define getpid _getpid +#else +# include <signal.h> +# include <unistd.h> +#endif +#include "js/experimental/CodeCoverage.h" +#include "mozilla/Atomics.h" +#include "mozilla/dom/ScriptSettings.h" // for AutoJSAPI +#include "mozilla/CodeCoverageHandler.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/DebugOnly.h" +#include "nsAppRunner.h" +#include "nsIFile.h" +#include "nsIOutputStream.h" +#include "nsNetUtil.h" +#include "nsPrintfCString.h" +#include "prtime.h" + +using namespace mozilla; + +// The __gcov_flush function writes the coverage counters to gcda files and then +// resets them to zero. It is defined at +// https://github.com/gcc-mirror/gcc/blob/aad93da1a579b9ae23ede6b9cf8523360f0a08b4/libgcc/libgcov-interface.c. +// __gcov_flush is protected by a mutex in GCC, but not in LLVM, so we are using +// a CrossProcessMutex to protect it. + +extern "C" void __gcov_flush(); +extern "C" void __gcov_dump(); +extern "C" void __gcov_reset(); + +StaticAutoPtr<CodeCoverageHandler> CodeCoverageHandler::instance; + +void CodeCoverageHandler::FlushCounters(const bool initialized) { + static Atomic<bool> hasBeenInitialized(false); + if (!hasBeenInitialized) { + hasBeenInitialized = initialized; + return; + } + + printf_stderr("[CodeCoverage] Requested flush for %d.\n", getpid()); + + CrossProcessMutexAutoLock lock(*CodeCoverageHandler::Get()->GetMutex()); + +#if defined(__clang__) && __clang_major__ >= 12 + __gcov_dump(); + __gcov_reset(); +#else + __gcov_flush(); +#endif + + printf_stderr("[CodeCoverage] flush completed.\n"); + + const char* outDir = getenv("JS_CODE_COVERAGE_OUTPUT_DIR"); + if (!outDir || *outDir == 0) { + return; + } + + dom::AutoJSAPI jsapi; + jsapi.Init(); + size_t length; + JS::UniqueChars result = js::GetCodeCoverageSummaryAll(jsapi.cx(), &length); + if (!result) { + return; + } + + nsCOMPtr<nsIFile> file; + + nsresult rv = NS_NewNativeLocalFile(nsDependentCString(outDir), false, + getter_AddRefs(file)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + rv = file->AppendNative( + nsPrintfCString("%lu-%d.info", PR_Now() / PR_USEC_PER_MSEC, getpid())); + + rv = file->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 0666); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + nsCOMPtr<nsIOutputStream> outputStream; + rv = NS_NewLocalFileOutputStream(getter_AddRefs(outputStream), file); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + char* data = result.get(); + while (length) { + uint32_t n = 0; + rv = outputStream->Write(data, length, &n); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + data += n; + length -= n; + } + + rv = outputStream->Close(); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + printf_stderr("[CodeCoverage] JS flush completed.\n"); +} + +void CodeCoverageHandler::FlushCountersSignalHandler(int) { FlushCounters(); } + +void CodeCoverageHandler::SetSignalHandlers() { +#ifndef XP_WIN + printf_stderr("[CodeCoverage] Setting handlers for process %d.\n", getpid()); + + struct sigaction dump_sa; + dump_sa.sa_handler = CodeCoverageHandler::FlushCountersSignalHandler; + dump_sa.sa_flags = SA_RESTART; + sigemptyset(&dump_sa.sa_mask); + DebugOnly<int> r1 = sigaction(SIGUSR1, &dump_sa, nullptr); + MOZ_ASSERT(r1 == 0, "Failed to install GCOV SIGUSR1 handler"); +#endif +} + +CodeCoverageHandler::CodeCoverageHandler() : mGcovLock("GcovLock") { + SetSignalHandlers(); +} + +CodeCoverageHandler::CodeCoverageHandler(CrossProcessMutexHandle aHandle) + : mGcovLock(std::move(aHandle)) { + SetSignalHandlers(); +} + +void CodeCoverageHandler::Init() { + MOZ_ASSERT(!instance); + MOZ_ASSERT(XRE_IsParentProcess()); + instance = new CodeCoverageHandler(); + ClearOnShutdown(&instance); + + // Don't really flush but just make FlushCounters usable. + FlushCounters(true); +} + +void CodeCoverageHandler::Init(CrossProcessMutexHandle aHandle) { + MOZ_ASSERT(!instance); + MOZ_ASSERT(!XRE_IsParentProcess()); + instance = new CodeCoverageHandler(std::move(aHandle)); + ClearOnShutdown(&instance); + + // Don't really flush but just make FlushCounters usable. + FlushCounters(true); +} + +CodeCoverageHandler* CodeCoverageHandler::Get() { + MOZ_ASSERT(instance); + return instance; +} + +CrossProcessMutex* CodeCoverageHandler::GetMutex() { return &mGcovLock; } + +CrossProcessMutexHandle CodeCoverageHandler::GetMutexHandle() { + return mGcovLock.CloneHandle(); +} diff --git a/tools/code-coverage/CodeCoverageHandler.h b/tools/code-coverage/CodeCoverageHandler.h new file mode 100644 index 0000000000..7c1b33ec8a --- /dev/null +++ b/tools/code-coverage/CodeCoverageHandler.h @@ -0,0 +1,38 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_codecoveragehandler_h +#define mozilla_codecoveragehandler_h + +#include "mozilla/StaticPtr.h" +#include "mozilla/ipc/CrossProcessMutex.h" + +namespace mozilla { + +class CodeCoverageHandler { + public: + static void Init(); + static void Init(CrossProcessMutexHandle aHandle); + static CodeCoverageHandler* Get(); + CrossProcessMutex* GetMutex(); + CrossProcessMutexHandle GetMutexHandle(); + static void FlushCounters(const bool initialized = false); + static void FlushCountersSignalHandler(int); + + private: + CodeCoverageHandler(); + explicit CodeCoverageHandler(CrossProcessMutexHandle aHandle); + + static StaticAutoPtr<CodeCoverageHandler> instance; + CrossProcessMutex mGcovLock; + + DISALLOW_COPY_AND_ASSIGN(CodeCoverageHandler); + + void SetSignalHandlers(); +}; + +} // namespace mozilla + +#endif // mozilla_codecoveragehandler_h diff --git a/tools/code-coverage/PerTestCoverageUtils.sys.mjs b/tools/code-coverage/PerTestCoverageUtils.sys.mjs new file mode 100644 index 0000000000..179dd905a5 --- /dev/null +++ b/tools/code-coverage/PerTestCoverageUtils.sys.mjs @@ -0,0 +1,88 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This is the directory where gcov is emitting the gcda files. +const gcovPrefixPath = Services.env.get("GCOV_PREFIX"); +// This is the directory where codecoverage.py is expecting to see the gcda files. +const gcovResultsPath = Services.env.get("GCOV_RESULTS_DIR"); +// This is the directory where the JS engine is emitting the lcov files. +const jsvmPrefixPath = Services.env.get("JS_CODE_COVERAGE_OUTPUT_DIR"); +// This is the directory where codecoverage.py is expecting to see the lcov files. +const jsvmResultsPath = Services.env.get("JSVM_RESULTS_DIR"); + +function awaitPromise(promise) { + let ret; + let complete = false; + let error = null; + promise + .catch(e => (error = e)) + .then(v => { + ret = v; + complete = true; + }); + Services.tm.spinEventLoopUntil( + "PerTestCoverageUtils.sys.mjs:awaitPromise", + () => complete + ); + if (error) { + throw new Error(error); + } + return ret; +} + +export var PerTestCoverageUtils = class PerTestCoverageUtilsClass { + // Resets the counters to 0. + static async beforeTest() { + if (!PerTestCoverageUtils.enabled) { + return; + } + + // Flush the counters. + let codeCoverageService = Cc[ + "@mozilla.org/tools/code-coverage;1" + ].getService(Ci.nsICodeCoverage); + await codeCoverageService.flushCounters(); + + // Remove coverage files created by the flush, and those that might have been created between the end of a previous test and the beginning of the next one (e.g. some tests can create a new content process for every sub-test). + await IOUtils.remove(gcovPrefixPath, { + recursive: true, + ignoreAbsent: true, + }); + await IOUtils.remove(jsvmPrefixPath, { + recursive: true, + ignoreAbsent: true, + }); + + // Move coverage files from the GCOV_RESULTS_DIR and JSVM_RESULTS_DIR directories, so we can accumulate the counters. + await IOUtils.move(gcovResultsPath, gcovPrefixPath); + await IOUtils.move(jsvmResultsPath, jsvmPrefixPath); + } + + static beforeTestSync() { + awaitPromise(this.beforeTest()); + } + + // Dumps counters and moves the gcda files in the directory expected by codecoverage.py. + static async afterTest() { + if (!PerTestCoverageUtils.enabled) { + return; + } + + // Flush the counters. + let codeCoverageService = Cc[ + "@mozilla.org/tools/code-coverage;1" + ].getService(Ci.nsICodeCoverage); + await codeCoverageService.flushCounters(); + + // Move the coverage files in GCOV_RESULTS_DIR and JSVM_RESULTS_DIR, so that the execution from now to shutdown (or next test) is not counted. + await IOUtils.move(gcovPrefixPath, gcovResultsPath); + await IOUtils.move(jsvmPrefixPath, jsvmResultsPath); + } + + static afterTestSync() { + awaitPromise(this.afterTest()); + } +}; + +PerTestCoverageUtils.enabled = !!gcovResultsPath; diff --git a/tools/code-coverage/components.conf b/tools/code-coverage/components.conf new file mode 100644 index 0000000000..a30600525c --- /dev/null +++ b/tools/code-coverage/components.conf @@ -0,0 +1,14 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + 'cid': '{93576af0-a62f-4c88-bc12-f1855d4e0173}', + 'contract_ids': ['@mozilla.org/tools/code-coverage;1'], + 'type': 'nsCodeCoverage', + 'headers': ['/tools/code-coverage/nsCodeCoverage.h'] + }, +] diff --git a/tools/code-coverage/docs/index.rst b/tools/code-coverage/docs/index.rst new file mode 100644 index 0000000000..36333b4829 --- /dev/null +++ b/tools/code-coverage/docs/index.rst @@ -0,0 +1,209 @@ +Code coverage +============= + +What is Code Coverage? +---------------------- + +**Code coverage** essentially measures how often certain lines are hit, +branches taken or conditions met in a program, given some test that you +run on it. + +There are two very important things to keep in mind when talking about +code coverage: + +- If a certain branch of code is not hit at all while running tests, + then those tests will never be able to find a bug in this particular + piece of the code. +- If a certain branch of code is executed (even very often), this still + is not a clear indication of the *quality of a test*. It could be + that a test exercises the code but does not actually check that the + code performs *correctly*. + +As a conclusion, we can use code coverage to find areas that need (more) +tests, but we cannot use it to confirm that certain areas are well +tested. + + +Firefox Code Coverage reports +----------------------------- + +We automatically run code coverage builds and tests on all +mozilla-central runs, for Linux and Windows. C/C++, Rust and JavaScript +are supported. + +The generated reports can be found at https://coverage.moz.tools/. The +reports can be filtered by platform and/or test suite. + +We also generate a report of all totally uncovered files, which can be +found at https://coverage.moz.tools/#view=zero. You can use this to find +areas of code that should be tested, or code that is no longer used +(dead code, which could be removed). + + +C/C++ Code Coverage on Firefox +------------------------------ + +There are several ways to get C/C++ coverage information for +mozilla-central, including creating your own coverage builds. The next +sections describe the available options. + + +Generate Code Coverage report from a try build (or any other CI build) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To spin a code coverage build, you need to select the linux64-ccov +platform (use --full when using the fuzzy selector to get the ccov +builds to show up). + +E.g. for a try build: + +.. code:: shell + + ./mach try fuzzy -q 'linux64-ccov' + +There are two options now, you can either generate the report locally or +use a one-click loaner. + + +Generate report using a one-click loaner +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Select the B job on Treeherder and get a one-click loaner. + +In the loaner, download and execute the script +https://github.com/mozilla/code-coverage/blob/master/report/firefox_code_coverage/codecoverage.py: + +.. code:: shell + + wget https://raw.githubusercontent.com/mozilla/code-coverage/master/report/firefox_code_coverage/codecoverage.py + python codecoverage.py + +This command will automatically generate a HTML report of the code +coverage information in the **report** subdirectory in your current +working directory. + + +Generate report locally +^^^^^^^^^^^^^^^^^^^^^^^ + +Prerequisites: + +- Create and activate a new `virtualenv`_, then run: + +.. code:: shell + + pip install firefox-code-coverage + +Given a treeherder linux64-ccov build (with its branch, e.g. +\`mozilla-central\` or \`try`, and revision, the tip commit hash of your +push), run the following command: + +.. code:: shell + + firefox-code-coverage PATH/TO/MOZILLA/SRC/DIR/ BRANCH REVISION + +This command will automatically download code coverage artifacts from +the treeherder build and generate an HTML report of the code coverage +information. The report will be stored in the **report** subdirectory in +your current working directory. + +.. _virtualenv: https://docs.python.org/3/tutorial/venv.html + +Creating your own Coverage Build +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Linux, Windows and Mac OS X it is straightforward to generate an +instrumented build using GCC or Clang. Adding the following lines to +your ``.mozconfig`` file should be sufficient: + +.. code:: shell + + # Enable code coverage + ac_add_options --enable-coverage + + # Needed for e10s: + # With the sandbox, content processes can't write updated coverage counters in the gcda files. + ac_add_options --disable-sandbox + +Some additional options might be needed, check the code-coverage +mozconfigs used on CI to be sure: +browser/config/mozconfigs/linux64/code-coverage, +browser/config/mozconfigs/win64/code-coverage, +browser/config/mozconfigs/macosx64/code-coverage. + +Make sure you are not running with :ref:`artifact build <Understanding Artifact Builds>` +enabled, as it can prevent coverage artifacts from being created. + +You can then create your build as usual. Once the build is complete, you +can run any tests/tools you would like to run and the coverage data gets +automatically written to special files. In order to view/process this +data, we recommend using the +`grcov <https://github.com/mozilla/grcov>`__ tool, a tool to manage and +visualize gcov results. You can also use the same process explained +earlier for CI builds. + + +Debugging Failing Tests on the Try Server +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When code coverage is run through a push to try, all the data that is +created is ingested by ActiveData and processed into a different data +format for analysis. Anytime a code coverage run generates \*.gcda and +\*.gcno files, ActiveData starts working. Now, sometimes, a test will +permanently fail when it is running on a build that is instrumented with +GCOV. To debug these issues without overloading ActiveData with garbage +coverage data, open the file +`taskcluster/gecko_taskgraph/transforms/test/__init__.py <https://searchfox.org/mozilla-central/source/taskcluster/gecko_taskgraph/transforms/test/__init__.py#516>`__ +and add the following line, + +.. code:: python + + test['mozharness'].setdefault('extra-options', []).append('--disable-ccov-upload') + +right after this line of code: + +.. code:: python + + test['mozharness'].setdefault('extra-options', []).append('--code-coverage') + +Now when you push to try to debug some failing tests, or anything else, +there will not be any code coverage artifacts uploaded from the build +machines or from the test machines. + + +JS Debugger Per Test Code Coverage on Firefox +--------------------------------------------- + +There are two ways to get javascript per test code coverage information +for mozilla-central. The next sections describe these options. + + +Generate Per Test Code Coverage from a try build (or any other treeherder build) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To spin a code coverage build, you need to select the linux64-jsdcov +platform. E.g. for a try build: + +.. code:: shell + + ./mach try fuzzy -q 'linux64-jsdcov' + +This produces JavaScript Object Notation (JSON) files that can be +downloaded from the treeherder testing machines and processed or +analyzed locally. + + +Generate Per Test Code Coverage Locally +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To generate the JSON files containing coverage information locally, simply +add an extra argument called ``--jscov-dir-prefix`` which accepts a +directory as it's input and stores the resulting data in that directory. +For example, to collect code coverage for the entire Mochitest suite: + +.. code:: shell + + ./mach mochitest --jscov-dir-prefix /PATH/TO/COVERAGE/DIR/ + +Currently, only the Mochitest and Xpcshell test suites have this +capability. diff --git a/tools/code-coverage/moz.build b/tools/code-coverage/moz.build new file mode 100644 index 0000000000..a4df736a72 --- /dev/null +++ b/tools/code-coverage/moz.build @@ -0,0 +1,39 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +TESTING_JS_MODULES += ["PerTestCoverageUtils.sys.mjs"] + +if CONFIG["MOZ_CODE_COVERAGE"]: + XPIDL_MODULE = "code-coverage" + + XPIDL_SOURCES += [ + "nsICodeCoverage.idl", + ] + + SOURCES += [ + "CodeCoverageHandler.cpp", + "nsCodeCoverage.cpp", + ] + + XPCOM_MANIFESTS += [ + "components.conf", + ] + + EXPORTS.mozilla += [ + "CodeCoverageHandler.h", + ] + + LOCAL_INCLUDES += [ + "/ipc/chromium/src", + "/xpcom/base", + ] + + include("/ipc/chromium/chromium-config.mozbuild") + + XPCSHELL_TESTS_MANIFESTS += ["tests/xpcshell/xpcshell.toml"] + MOCHITEST_MANIFESTS += ["tests/mochitest/mochitest.toml"] + + FINAL_LIBRARY = "xul" diff --git a/tools/code-coverage/nsCodeCoverage.cpp b/tools/code-coverage/nsCodeCoverage.cpp new file mode 100644 index 0000000000..5d7ed1927c --- /dev/null +++ b/tools/code-coverage/nsCodeCoverage.cpp @@ -0,0 +1,96 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCodeCoverage.h" +#include "mozilla/CodeCoverageHandler.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/dom/Promise.h" + +using mozilla::dom::ContentParent; +using mozilla::dom::Promise; + +NS_IMPL_ISUPPORTS(nsCodeCoverage, nsICodeCoverage) + +nsCodeCoverage::nsCodeCoverage() {} + +nsCodeCoverage::~nsCodeCoverage() {} + +enum RequestType { Flush }; + +class ProcessCount final { + NS_INLINE_DECL_REFCOUNTING(ProcessCount); + + public: + explicit ProcessCount(uint32_t c) : mCount(c) {} + operator uint32_t() const { return mCount; } + ProcessCount& operator--() { + mCount--; + return *this; + } + + private: + ~ProcessCount() {} + uint32_t mCount; +}; + +namespace { + +nsresult Request(JSContext* cx, Promise** aPromise, RequestType requestType) { + MOZ_ASSERT(XRE_IsParentProcess()); + MOZ_ASSERT(NS_IsMainThread()); + + nsIGlobalObject* global = xpc::CurrentNativeGlobal(cx); + if (NS_WARN_IF(!global)) { + return NS_ERROR_FAILURE; + } + + mozilla::ErrorResult result; + RefPtr<Promise> promise = Promise::Create(global, result); + if (NS_WARN_IF(result.Failed())) { + return result.StealNSResult(); + } + + uint32_t processCount = 0; + for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) { + mozilla::Unused << cp; + ++processCount; + } + + if (requestType == RequestType::Flush) { + mozilla::CodeCoverageHandler::FlushCounters(); + } + + if (processCount == 0) { + promise->MaybeResolveWithUndefined(); + } else { + RefPtr<ProcessCount> processCountHolder(new ProcessCount(processCount)); + + auto resolve = [processCountHolder, promise](bool unused) { + if (--(*processCountHolder) == 0) { + promise->MaybeResolveWithUndefined(); + } + }; + + auto reject = [promise](mozilla::ipc::ResponseRejectReason&& aReason) { + promise->MaybeReject(NS_ERROR_FAILURE); + }; + + for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) { + if (requestType == RequestType::Flush) { + cp->SendFlushCodeCoverageCounters(resolve, reject); + } + } + } + + promise.forget(aPromise); + return NS_OK; +} + +} // anonymous namespace + +NS_IMETHODIMP nsCodeCoverage::FlushCounters(JSContext* cx, Promise** aPromise) { + return Request(cx, aPromise, RequestType::Flush); +} diff --git a/tools/code-coverage/nsCodeCoverage.h b/tools/code-coverage/nsCodeCoverage.h new file mode 100644 index 0000000000..936566ac02 --- /dev/null +++ b/tools/code-coverage/nsCodeCoverage.h @@ -0,0 +1,22 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef tools_codecoverage_nscodecoverage_h +#define tools_codecoverage_nscodecoverage_h + +#include "nsICodeCoverage.h" + +class nsCodeCoverage final : nsICodeCoverage { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICODECOVERAGE + + nsCodeCoverage(); + + private: + ~nsCodeCoverage(); +}; + +#endif // tools_codecoverage_nscodecoverage_h diff --git a/tools/code-coverage/nsICodeCoverage.idl b/tools/code-coverage/nsICodeCoverage.idl new file mode 100644 index 0000000000..ec4ad40ae1 --- /dev/null +++ b/tools/code-coverage/nsICodeCoverage.idl @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +/** + * The nsICodeCoverage component allows controlling the code coverage counters + * collected by Firefox during execution. + * By flushing the counters, one can analyze the coverage information + * for a subset of the program execution (e.g. startup code coverage). + * + */ + +[scriptable, uuid(57d92056-37b4-4d0a-a52f-deb8f6dac8bc)] +interface nsICodeCoverage : nsISupports +{ + /** + * Write the coverage counters to disk, and reset them in memory to 0. + */ + [implicit_jscontext] + Promise flushCounters(); +}; diff --git a/tools/code-coverage/tests/mochitest/mochitest.toml b/tools/code-coverage/tests/mochitest/mochitest.toml new file mode 100644 index 0000000000..66f6fd02d1 --- /dev/null +++ b/tools/code-coverage/tests/mochitest/mochitest.toml @@ -0,0 +1,3 @@ +[DEFAULT] + +["test_coverage_specialpowers.html"] diff --git a/tools/code-coverage/tests/mochitest/test_coverage_specialpowers.html b/tools/code-coverage/tests/mochitest/test_coverage_specialpowers.html new file mode 100644 index 0000000000..301206ac48 --- /dev/null +++ b/tools/code-coverage/tests/mochitest/test_coverage_specialpowers.html @@ -0,0 +1,38 @@ +<!DOCTYPE HTML> +<html> +<!-- +https://bugzilla.mozilla.org/show_bug.cgi?id=1380659 +--> +<head> + <meta charset="utf-8"> + <title>Test for Bug 123456</title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/> + <script type="application/javascript"> + + /** Test for Bug 1380659 **/ + + SimpleTest.waitForExplicitFinish(); + + (async function() { + await SpecialPowers.requestDumpCoverageCounters(); + SimpleTest.ok(true, "Counters dumped."); + + await SpecialPowers.requestResetCoverageCounters(); + SimpleTest.ok(true, "Counters reset."); + + SimpleTest.finish(); + })(); + + </script> +</head> +<body> +<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1380659">Mozilla Bug 1380659</a> +<p id="display"></p> +<div id="content" style="display: none"> + +</div> +<pre id="test"> +</pre> +</body> +</html> diff --git a/tools/code-coverage/tests/xpcshell/head.js b/tools/code-coverage/tests/xpcshell/head.js new file mode 100644 index 0000000000..3642c5794c --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/head.js @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +function getFiles() { + // This is the directory where gcov is emitting the gcda files. + const jsCoveragePath = Services.env.get("JS_CODE_COVERAGE_OUTPUT_DIR"); + + const jsCoverageDir = Cc["@mozilla.org/file/local;1"].createInstance( + Ci.nsIFile + ); + jsCoverageDir.initWithPath(jsCoveragePath); + + let files = []; + + let entries = jsCoverageDir.directoryEntries; + while (entries.hasMoreElements()) { + files.push(entries.nextFile); + } + + return files; +} + +function diffFiles(files_after, files_before) { + let files_before_set = new Set(files_before.map(file => file.leafName)); + return files_after.filter(file => !files_before_set.has(file.leafName)); +} + +const BASENAME_RE = new RegExp("([^/\\\\]+)$"); + +function parseRecords(files) { + let records = new Map(); + + for (let file of files) { + const lines = Cu.readUTF8File(file).split("\n"); + let currentSF = null; + + for (let line of lines) { + let [recordType, ...recordContent] = line.split(":"); + recordContent = recordContent.join(":"); + + switch (recordType) { + case "FNDA": { + if (currentSF == null) { + throw new Error("SF missing"); + } + + let [hits, name] = recordContent.split(","); + currentSF.push({ + type: "FNDA", + hits, + name, + }); + break; + } + + case "FN": { + if (currentSF == null) { + throw new Error("SF missing"); + } + + let name = recordContent.split(",")[1]; + currentSF.push({ + type: "FN", + name, + }); + break; + } + + case "SF": { + if ( + recordContent.startsWith("resource:") || + recordContent.startsWith("chrome:") + ) { + recordContent = recordContent.split("/").at(-1); + } else { + if (AppConstants.platform == "win") { + recordContent = recordContent.replace(/\//g, "\\"); + } + const match = BASENAME_RE.exec(recordContent); + if (match.length) { + recordContent = match[0]; + } + } + + currentSF = []; + + records.set(recordContent, currentSF); + break; + } + } + } + } + + return records; +} diff --git a/tools/code-coverage/tests/xpcshell/support.js b/tools/code-coverage/tests/xpcshell/support.js new file mode 100644 index 0000000000..9189427111 --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/support.js @@ -0,0 +1,5 @@ +function test_code_coverage_func2() { + return 22; +} + +test_code_coverage_func2(); diff --git a/tools/code-coverage/tests/xpcshell/test_basic.js b/tools/code-coverage/tests/xpcshell/test_basic.js new file mode 100644 index 0000000000..9523a37ca2 --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/test_basic.js @@ -0,0 +1,115 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function test_code_coverage_func1() { + return 22; +} + +function test_code_coverage_func2() { + return 22; +} + +async function run_test() { + do_test_pending(); + + Assert.ok("@mozilla.org/tools/code-coverage;1" in Cc); + + const codeCoverageCc = Cc["@mozilla.org/tools/code-coverage;1"]; + Assert.ok(!!codeCoverageCc); + + const codeCoverage = codeCoverageCc.getService(Ci.nsICodeCoverage); + Assert.ok(!!codeCoverage); + + const files_orig = getFiles(); + + test_code_coverage_func1(); + + // Flush counters for the first time, we should see this function executed, but test_code_coverage_func not executed. + await codeCoverage.flushCounters(); + + const first_flush_files = getFiles(); + const first_flush_records = parseRecords( + diffFiles(first_flush_files, files_orig) + ); + + Assert.ok(first_flush_records.has("test_basic.js")); + let fnRecords = first_flush_records + .get("test_basic.js") + .filter(record => record.type == "FN"); + let fndaRecords = first_flush_records + .get("test_basic.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok(fnRecords.some(record => record.name == "run_test")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func1") + ); + Assert.ok( + fndaRecords.some(record => record.name == "run_test" && record.hits == 1) + ); + Assert.ok( + !fndaRecords.some(record => record.name == "run_test" && record.hits != 1) + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits == 1 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits != 1 + ) + ); + Assert.ok( + !fndaRecords.some(record => record.name == "test_code_coverage_func2") + ); + + test_code_coverage_func2(); + + // Flush counters for the second time, we should see this function not executed, but test_code_coverage_func executed. + await codeCoverage.flushCounters(); + + const second_flush_files = getFiles(); + const second_flush_records = parseRecords( + diffFiles(second_flush_files, first_flush_files) + ); + + Assert.ok(second_flush_records.has("test_basic.js")); + fnRecords = second_flush_records + .get("test_basic.js") + .filter(record => record.type == "FN"); + fndaRecords = second_flush_records + .get("test_basic.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok(fnRecords.some(record => record.name == "run_test")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func1") + ); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func2") + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits == 0 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits != 0 + ) + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func2" && record.hits == 1 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func2" && record.hits != 1 + ) + ); + + do_test_finished(); +} diff --git a/tools/code-coverage/tests/xpcshell/test_basic_child_and_parent.js b/tools/code-coverage/tests/xpcshell/test_basic_child_and_parent.js new file mode 100644 index 0000000000..f074c20776 --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/test_basic_child_and_parent.js @@ -0,0 +1,120 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function test_code_coverage_func1() { + return 22; +} + +async function run_test() { + do_load_child_test_harness(); + do_test_pending(); + + const codeCoverage = Cc["@mozilla.org/tools/code-coverage;1"].getService( + Ci.nsICodeCoverage + ); + + const files_orig = getFiles(); + + test_code_coverage_func1(); + + await codeCoverage.flushCounters(); + + const first_flush_files = getFiles(); + const first_flush_records = parseRecords( + diffFiles(first_flush_files, files_orig) + ); + + Assert.ok(first_flush_records.has("test_basic_child_and_parent.js")); + Assert.ok(!first_flush_records.has("support.js")); + let fnRecords = first_flush_records + .get("test_basic_child_and_parent.js") + .filter(record => record.type == "FN"); + let fndaRecords = first_flush_records + .get("test_basic_child_and_parent.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok(fnRecords.some(record => record.name == "run_test")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func1") + ); + Assert.ok( + fndaRecords.some(record => record.name == "run_test" && record.hits == 1) + ); + Assert.ok( + !fndaRecords.some(record => record.name == "run_test" && record.hits != 1) + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits == 1 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits != 1 + ) + ); + + sendCommand("load('support.js');", async function () { + await codeCoverage.flushCounters(); + + const second_flush_files = getFiles(); + const second_flush_records = parseRecords( + diffFiles(second_flush_files, first_flush_files) + ); + + Assert.ok(second_flush_records.has("test_basic_child_and_parent.js")); + fnRecords = second_flush_records + .get("test_basic_child_and_parent.js") + .filter(record => record.type == "FN"); + fndaRecords = second_flush_records + .get("test_basic_child_and_parent.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok(fnRecords.some(record => record.name == "run_test")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func1") + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits == 0 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits != 0 + ) + ); + Assert.ok(second_flush_records.has("support.js")); + fnRecords = second_flush_records + .get("support.js") + .filter(record => record.type == "FN"); + fndaRecords = second_flush_records + .get("support.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func2") + ); + Assert.ok( + fndaRecords.some(record => record.name == "top-level" && record.hits == 1) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "top-level" && record.hits != 1 + ) + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func2" && record.hits == 1 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func2" && record.hits != 1 + ) + ); + + do_test_finished(); + }); +} diff --git a/tools/code-coverage/tests/xpcshell/xpcshell.toml b/tools/code-coverage/tests/xpcshell/xpcshell.toml new file mode 100644 index 0000000000..e1aacfeada --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/xpcshell.toml @@ -0,0 +1,7 @@ +[DEFAULT] +head = "head.js" +support-files = ["support.js"] + +["test_basic.js"] + +["test_basic_child_and_parent.js"] diff --git a/tools/compare-locales/mach_commands.py b/tools/compare-locales/mach_commands.py new file mode 100644 index 0000000000..56d101467b --- /dev/null +++ b/tools/compare-locales/mach_commands.py @@ -0,0 +1,409 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import logging +import os +import tempfile +from pathlib import Path + +from appdirs import user_config_dir +from hglib.error import CommandError +from mach.base import FailedCommandError +from mach.decorators import Command, CommandArgument +from mozrelease.scriptworker_canary import get_secret +from redo import retry + + +@Command( + "compare-locales", + category="build", + description="Run source checks on a localization.", +) +@CommandArgument( + "config_paths", + metavar="l10n.toml", + nargs="+", + help="TOML or INI file for the project", +) +@CommandArgument( + "l10n_base_dir", + metavar="l10n-base-dir", + help="Parent directory of localizations", +) +@CommandArgument( + "locales", + nargs="*", + metavar="locale-code", + help="Locale code and top-level directory of each localization", +) +@CommandArgument( + "-q", + "--quiet", + action="count", + default=0, + help="""Show less data. +Specified once, don't show obsolete entities. Specified twice, also hide +missing entities. Specify thrice to exclude warnings and four times to +just show stats""", +) +@CommandArgument("-m", "--merge", help="""Use this directory to stage merged files""") +@CommandArgument( + "--validate", action="store_true", help="Run compare-locales against reference" +) +@CommandArgument( + "--json", + help="""Serialize to JSON. Value is the name of +the output file, pass "-" to serialize to stdout and hide the default output. +""", +) +@CommandArgument( + "-D", + action="append", + metavar="var=value", + default=[], + dest="defines", + help="Overwrite variables in TOML files", +) +@CommandArgument( + "--full", action="store_true", help="Compare projects that are disabled" +) +@CommandArgument( + "--return-zero", action="store_true", help="Return 0 regardless of l10n status" +) +def compare(command_context, **kwargs): + """Run compare-locales.""" + from compare_locales.commands import CompareLocales + + class ErrorHelper(object): + """Dummy ArgumentParser to marshall compare-locales + commandline errors to mach exceptions. + """ + + def error(self, msg): + raise FailedCommandError(msg) + + def exit(self, message=None, status=0): + raise FailedCommandError(message, exit_code=status) + + cmd = CompareLocales() + cmd.parser = ErrorHelper() + return cmd.handle(**kwargs) + + +# https://stackoverflow.com/a/14117511 +def _positive_int(value): + value = int(value) + if value <= 0: + raise argparse.ArgumentTypeError(f"{value} must be a positive integer.") + return value + + +class RetryError(Exception): + ... + + +VCT_PATH = Path(".").resolve() / "vct" +VCT_URL = "https://hg.mozilla.org/hgcustom/version-control-tools/" +FXTREE_PATH = VCT_PATH / "hgext" / "firefoxtree" +HGRC_PATH = Path(user_config_dir("hg")).joinpath("hgrc") + + +@Command( + "l10n-cross-channel", + category="misc", + description="Create cross-channel content.", +) +@CommandArgument( + "--strings-path", + "-s", + metavar="en-US", + type=Path, + default=Path("en-US"), + help="Path to mercurial repository for gecko-strings-quarantine", +) +@CommandArgument( + "--outgoing-path", + "-o", + type=Path, + help="create an outgoing() patch if there are changes", +) +@CommandArgument( + "--attempts", + type=_positive_int, + default=1, + help="Number of times to try (for automation)", +) +@CommandArgument( + "--ssh-secret", + action="store", + help="Taskcluster secret to use to push (for automation)", +) +@CommandArgument( + "actions", + choices=("prep", "create", "push", "clean"), + nargs="+", + # This help block will be poorly formatted until we fix bug 1714239 + help=""" + "prep": clone repos and pull heads. + "create": create the en-US strings commit an optionally create an + outgoing() patch. + "push": push the en-US strings to the quarantine repo. + "clean": clean up any sub-repos. + """, +) +def cross_channel( + command_context, + strings_path, + outgoing_path, + actions, + attempts, + ssh_secret, + **kwargs, +): + """Run l10n cross-channel content generation.""" + # This can be any path, as long as the name of the directory is en-US. + # Not entirely sure where this is a requirement; perhaps in l10n + # string manipulation logic? + if strings_path.name != "en-US": + raise FailedCommandError("strings_path needs to be named `en-US`") + command_context.activate_virtualenv() + # XXX pin python requirements + command_context.virtualenv_manager.install_pip_requirements( + Path(os.path.dirname(__file__)) / "requirements.in" + ) + strings_path = strings_path.resolve() # abspath + if outgoing_path: + outgoing_path = outgoing_path.resolve() # abspath + get_config = kwargs.get("get_config", None) + try: + with tempfile.TemporaryDirectory() as ssh_key_dir: + retry( + _do_create_content, + attempts=attempts, + retry_exceptions=(RetryError,), + args=( + command_context, + strings_path, + outgoing_path, + ssh_secret, + Path(ssh_key_dir), + actions, + get_config, + ), + ) + except RetryError as exc: + raise FailedCommandError(exc) from exc + + +def _do_create_content( + command_context, + strings_path, + outgoing_path, + ssh_secret, + ssh_key_dir, + actions, + get_config, +): + from mozxchannel import CrossChannelCreator, get_default_config + + get_config = get_config or get_default_config + + config = get_config(Path(command_context.topsrcdir), strings_path) + ccc = CrossChannelCreator(config) + status = 0 + changes = False + ssh_key_secret = None + ssh_key_file = None + + if "prep" in actions: + if ssh_secret: + if not os.environ.get("MOZ_AUTOMATION"): + raise CommandError( + "I don't know how to fetch the ssh secret outside of automation!" + ) + ssh_key_secret = get_secret(ssh_secret) + ssh_key_file = ssh_key_dir.joinpath("id_rsa") + ssh_key_file.write_text(ssh_key_secret["ssh_privkey"]) + ssh_key_file.chmod(0o600) + # Set up firefoxtree for comm per bug 1659691 comment 22 + if os.environ.get("MOZ_AUTOMATION") and not HGRC_PATH.exists(): + _clone_hg_repo(command_context, VCT_URL, VCT_PATH) + hgrc_content = [ + "[extensions]", + f"firefoxtree = {FXTREE_PATH}", + "", + "[ui]", + "username = trybld", + ] + if ssh_key_file: + hgrc_content.extend( + [ + f"ssh = ssh -i {ssh_key_file} -l {ssh_key_secret['user']}", + ] + ) + HGRC_PATH.write_text("\n".join(hgrc_content)) + if strings_path.exists() and _check_outgoing(command_context, strings_path): + _strip_outgoing(command_context, strings_path) + # Clone strings + source repos, pull heads + for repo_config in (config["strings"], *config["source"].values()): + if not repo_config["path"].exists(): + _clone_hg_repo( + command_context, repo_config["url"], str(repo_config["path"]) + ) + for head in repo_config["heads"].keys(): + command = ["hg", "--cwd", str(repo_config["path"]), "pull"] + command.append(head) + status = _retry_run_process( + command_context, command, ensure_exit_code=False + ) + if status not in (0, 255): # 255 on pull with no changes + raise RetryError(f"Failure on pull: status {status}!") + if repo_config.get("update_on_pull"): + command = [ + "hg", + "--cwd", + str(repo_config["path"]), + "up", + "-C", + "-r", + head, + ] + status = _retry_run_process( + command_context, command, ensure_exit_code=False + ) + if status not in (0, 255): # 255 on pull with no changes + raise RetryError(f"Failure on update: status {status}!") + _check_hg_repo( + command_context, + repo_config["path"], + heads=repo_config.get("heads", {}).keys(), + ) + else: + _check_hg_repo(command_context, strings_path) + for repo_config in config.get("source", {}).values(): + _check_hg_repo( + command_context, + repo_config["path"], + heads=repo_config.get("heads", {}).keys(), + ) + if _check_outgoing(command_context, strings_path): + raise RetryError(f"check: Outgoing changes in {strings_path}!") + + if "create" in actions: + try: + status = ccc.create_content() + changes = True + _create_outgoing_patch(command_context, outgoing_path, strings_path) + except CommandError as exc: + if exc.ret != 1: + raise RetryError(exc) from exc + command_context.log(logging.INFO, "create", {}, "No new strings.") + + if "push" in actions: + if changes: + _retry_run_process( + command_context, + [ + "hg", + "--cwd", + str(strings_path), + "push", + "-r", + ".", + config["strings"]["push_url"], + ], + line_handler=print, + ) + else: + command_context.log(logging.INFO, "push", {}, "Skipping empty push.") + + if "clean" in actions: + for repo_config in config.get("source", {}).values(): + if repo_config.get("post-clobber", False): + _nuke_hg_repo(command_context, str(repo_config["path"])) + + return status + + +def _check_outgoing(command_context, strings_path): + status = _retry_run_process( + command_context, + ["hg", "--cwd", str(strings_path), "out", "-r", "."], + ensure_exit_code=False, + ) + if status == 0: + return True + if status == 1: + return False + raise RetryError(f"Outgoing check in {strings_path} returned unexpected {status}!") + + +def _strip_outgoing(command_context, strings_path): + _retry_run_process( + command_context, + [ + "hg", + "--config", + "extensions.strip=", + "--cwd", + str(strings_path), + "strip", + "--no-backup", + "outgoing()", + ], + ) + + +def _create_outgoing_patch(command_context, path, strings_path): + if not path: + return + if not path.parent.exists(): + os.makedirs(path.parent) + with open(path, "w") as fh: + + def writeln(line): + fh.write(f"{line}\n") + + _retry_run_process( + command_context, + [ + "hg", + "--cwd", + str(strings_path), + "log", + "--patch", + "--verbose", + "-r", + "outgoing()", + ], + line_handler=writeln, + ) + + +def _retry_run_process(command_context, *args, error_msg=None, **kwargs): + try: + return command_context.run_process(*args, **kwargs) + except Exception as exc: + raise RetryError(error_msg or str(exc)) from exc + + +def _check_hg_repo(command_context, path, heads=None): + if not (path.is_dir() and (path / ".hg").is_dir()): + raise RetryError(f"{path} is not a Mercurial repository") + if heads: + for head in heads: + _retry_run_process( + command_context, + ["hg", "--cwd", str(path), "log", "-r", head], + error_msg=f"check: {path} has no head {head}!", + ) + + +def _clone_hg_repo(command_context, url, path): + _retry_run_process(command_context, ["hg", "clone", url, str(path)]) + + +def _nuke_hg_repo(command_context, path): + _retry_run_process(command_context, ["rm", "-rf", str(path)]) diff --git a/tools/compare-locales/requirements.in b/tools/compare-locales/requirements.in new file mode 100644 index 0000000000..d6fbb9d25b --- /dev/null +++ b/tools/compare-locales/requirements.in @@ -0,0 +1,2 @@ +redo +dataclasses; python_version < '3.7' diff --git a/tools/crashreporter/injector/app.mozbuild b/tools/crashreporter/injector/app.mozbuild new file mode 100644 index 0000000000..2a514db94d --- /dev/null +++ b/tools/crashreporter/injector/app.mozbuild @@ -0,0 +1,14 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + "/toolkit/crashreporter/google-breakpad/src/common", + "/toolkit/crashreporter/google-breakpad/src/processor", + "/tools/crashreporter/injector", +] + +if CONFIG["OS_TARGET"] in ("Linux", "Android"): + DIRS += [ + "/toolkit/crashreporter/google-breakpad/src/common/linux", + ] diff --git a/tools/crashreporter/injector/injector.cc b/tools/crashreporter/injector/injector.cc new file mode 100644 index 0000000000..eeb74038c8 --- /dev/null +++ b/tools/crashreporter/injector/injector.cc @@ -0,0 +1,40 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "linux/handler/exception_handler.h" + +using google_breakpad::ExceptionHandler; + +static ExceptionHandler* gExceptionHandler = nullptr; + +// Flag saying whether to generate a minidump. Can be (probably temporarily) set +// to false when a crash is expected. +bool __attribute__((visibility("default"))) gBreakpadInjectorEnabled = true; + +bool TestEnabled(void* /* context */) { return gBreakpadInjectorEnabled; } + +bool SetGlobalExceptionHandler( + ExceptionHandler::FilterCallback filterCallback, + ExceptionHandler::MinidumpCallback minidumpCallback) { + const char* tempPath = getenv("TMPDIR"); + if (!tempPath) tempPath = "/tmp"; + + google_breakpad::MinidumpDescriptor descriptor(tempPath); + + gExceptionHandler = new ExceptionHandler(descriptor, filterCallback, + minidumpCallback, nullptr, true, -1); + if (!gExceptionHandler) return false; + + return true; +} + +// Called when loading the DLL (eg via LD_PRELOAD, or the JS shell --dll +// option). +void __attribute__((constructor)) SetBreakpadExceptionHandler() { + if (gExceptionHandler) abort(); + + if (!SetGlobalExceptionHandler(TestEnabled, nullptr)) abort(); + + if (!gExceptionHandler) abort(); +} diff --git a/tools/crashreporter/injector/moz.build b/tools/crashreporter/injector/moz.build new file mode 100644 index 0000000000..d16300fbfb --- /dev/null +++ b/tools/crashreporter/injector/moz.build @@ -0,0 +1,36 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SharedLibrary("breakpadinjector") + +UNIFIED_SOURCES += [ + "/toolkit/crashreporter/breakpad-client/linux/crash_generation/crash_generation_client.cc", + "/toolkit/crashreporter/breakpad-client/linux/dump_writer_common/thread_info.cc", + "/toolkit/crashreporter/breakpad-client/linux/dump_writer_common/ucontext_reader.cc", + "/toolkit/crashreporter/breakpad-client/linux/handler/exception_handler.cc", + "/toolkit/crashreporter/breakpad-client/linux/handler/guid_generator.cc", + "/toolkit/crashreporter/breakpad-client/linux/handler/minidump_descriptor.cc", + "/toolkit/crashreporter/breakpad-client/linux/log/log.cc", + "/toolkit/crashreporter/breakpad-client/linux/microdump_writer/microdump_writer.cc", + "/toolkit/crashreporter/breakpad-client/linux/minidump_writer/linux_dumper.cc", + "/toolkit/crashreporter/breakpad-client/linux/minidump_writer/linux_ptrace_dumper.cc", + "/toolkit/crashreporter/breakpad-client/linux/minidump_writer/minidump_writer.cc", + "/toolkit/crashreporter/breakpad-client/minidump_file_writer.cc", + "injector.cc", +] + +USE_LIBS += [ + "breakpad_common_s", + "breakpad_linux_common_s", +] + +DisableStlWrapping() + +# On Linux we override the guid_creator.h header and use our own instead +if CONFIG["OS_TARGET"] in ("Linux", "Android"): + DEFINES["COMMON_LINUX_GUID_CREATOR_H__"] = 1 + +include("/toolkit/crashreporter/crashreporter.mozbuild") diff --git a/tools/crashreporter/injector/moz.configure b/tools/crashreporter/injector/moz.configure new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/tools/crashreporter/injector/moz.configure @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py b/tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py new file mode 100755 index 0000000000..64b55b5007 --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python + +# Copyright 2015 Michael R. Miller. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +PackageSymbolDumper.py + +Dumps Breakpad symbols for the contents of an Apple update installer. Given a +path to an Apple update installer as a .dmg or a path to a specific package +within the disk image, PackageSymbolDumper mounts, traverses, and dumps symbols +for all applicable frameworks and dylibs found within. + +Required tools for Linux: + pax + gzip + tar + xpwn's dmg (https://github.com/planetbeing/xpwn) + +Created on Apr 11, 2012 + +@author: mrmiller +""" +import argparse +import concurrent.futures +import errno +import logging +import os +import shutil +import stat +import subprocess +import tempfile +import traceback + +from mozpack.macpkg import Pbzx, uncpio, unxar +from scrapesymbols.gathersymbols import process_paths + + +def expand_pkg(pkg_path, out_path): + """ + Expands the contents of an installer package to some directory. + + @param pkg_path: a path to an installer package (.pkg) + @param out_path: a path to hold the package contents + """ + for name, content in unxar(open(pkg_path, "rb")): + with open(os.path.join(out_path, name), "wb") as fh: + shutil.copyfileobj(content, fh) + + +def expand_dmg(dmg_path, out_path): + """ + Expands the contents of a DMG file to some directory. + + @param dmg_path: a path to a disk image file (.dmg) + @param out_path: a path to hold the image contents + """ + + with tempfile.NamedTemporaryFile() as f: + subprocess.check_call( + ["dmg", "extract", dmg_path, f.name], stdout=subprocess.DEVNULL + ) + subprocess.check_call( + ["hfsplus", f.name, "extractall"], stdout=subprocess.DEVNULL, cwd=out_path + ) + + +def expand_zip(zip_path, out_path): + """ + Expands the contents of a ZIP archive to some directory. + + @param dmg_path: a path to a ZIP archive (.zip) + @param out_path: a path to hold the archive contents + """ + subprocess.check_call( + ["unzip", "-d", out_path, zip_path], stdout=subprocess.DEVNULL + ) + + +def filter_files(function, path): + """ + Yield file paths matching a filter function by walking the + hierarchy rooted at path. + + @param function: a function taking in a filename that returns true to + include the path + @param path: the root path of the hierarchy to traverse + """ + for root, _dirs, files in os.walk(path): + for filename in files: + if function(filename): + yield os.path.join(root, filename) + + +def find_packages(path): + """ + Returns a list of installer packages (as determined by the .pkg extension), + disk images (as determined by the .dmg extension) or ZIP archives found + within path. + + @param path: root path to search for .pkg, .dmg and .zip files + """ + return filter_files( + lambda filename: os.path.splitext(filename)[1] in (".pkg", ".dmg", ".zip") + and not filename.startswith("._"), + path, + ) + + +def find_all_packages(paths): + """ + Yield installer package files, disk images and ZIP archives found in all + of `paths`. + + @param path: list of root paths to search for .pkg & .dmg files + """ + for path in paths: + logging.info("find_all_packages: {}".format(path)) + for pkg in find_packages(path): + yield pkg + + +def find_payloads(path): + """ + Returns a list of possible installer package payload paths. + + @param path: root path for an installer package + """ + return filter_files( + lambda filename: "Payload" in filename or ".pax.gz" in filename, path + ) + + +def extract_payload(payload_path, output_path): + """ + Extracts the contents of an installer package payload to a given directory. + + @param payload_path: path to an installer package's payload + @param output_path: output path for the payload's contents + @return True for success, False for failure. + """ + header = open(payload_path, "rb").read(2) + try: + if header == b"BZ": + logging.info("Extracting bzip2 payload") + extract = "bzip2" + subprocess.check_call( + 'cd {dest} && {extract} -dc {payload} | pax -r -k -s ":^/::"'.format( + extract=extract, payload=payload_path, dest=output_path + ), + shell=True, + ) + return True + elif header == b"\x1f\x8b": + logging.info("Extracting gzip payload") + extract = "gzip" + subprocess.check_call( + 'cd {dest} && {extract} -dc {payload} | pax -r -k -s ":^/::"'.format( + extract=extract, payload=payload_path, dest=output_path + ), + shell=True, + ) + return True + elif header == b"pb": + logging.info("Extracting pbzx payload") + + for path, st, content in uncpio(Pbzx(open(payload_path, "rb"))): + if not path or not stat.S_ISREG(st.mode): + continue + out = os.path.join(output_path, path.decode()) + os.makedirs(os.path.dirname(out), exist_ok=True) + with open(out, "wb") as fh: + shutil.copyfileobj(content, fh) + + return True + else: + # Unsupported format + logging.error( + "Unknown payload format: 0x{0:x}{1:x}".format(header[0], header[1]) + ) + return False + + except Exception: + return False + + +def shutil_error_handler(caller, path, excinfo): + logging.error('Could not remove "{path}": {info}'.format(path=path, info=excinfo)) + + +def write_symbol_file(dest, filename, contents): + full_path = os.path.join(dest, filename) + try: + os.makedirs(os.path.dirname(full_path)) + with open(full_path, "wb") as sym_file: + sym_file.write(contents) + except os.error as e: + if e.errno != errno.EEXIST: + raise + + +def dump_symbols(executor, dump_syms, path, dest): + system_library = os.path.join("System", "Library") + subdirectories = [ + os.path.join(system_library, "Frameworks"), + os.path.join(system_library, "PrivateFrameworks"), + os.path.join(system_library, "Extensions"), + os.path.join("usr", "lib"), + ] + + paths_to_dump = [os.path.join(path, d) for d in subdirectories] + existing_paths = [path for path in paths_to_dump if os.path.exists(path)] + + for filename, contents in process_paths( + paths=existing_paths, + executor=executor, + dump_syms=dump_syms, + verbose=True, + write_all=True, + platform="darwin", + ): + if filename and contents: + logging.info("Added symbol file " + str(filename, "utf-8")) + write_symbol_file(dest, str(filename, "utf-8"), contents) + + +def dump_symbols_from_payload(executor, dump_syms, payload_path, dest): + """ + Dumps all the symbols found inside the payload of an installer package. + + @param dump_syms: path to the dump_syms executable + @param payload_path: path to an installer package's payload + @param dest: output path for symbols + """ + temp_dir = None + logging.info("Dumping symbols from payload: " + payload_path) + try: + temp_dir = tempfile.mkdtemp() + logging.info("Extracting payload to {path}.".format(path=temp_dir)) + if not extract_payload(payload_path, temp_dir): + logging.error("Could not extract payload: " + payload_path) + return False + + dump_symbols(executor, dump_syms, temp_dir, dest) + + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir, onerror=shutil_error_handler) + + return True + + +def dump_symbols_from_package(executor, dump_syms, pkg, dest): + """ + Dumps all the symbols found inside an installer package. + + @param dump_syms: path to the dump_syms executable + @param pkg: path to an installer package + @param dest: output path for symbols + """ + successful = True + temp_dir = None + logging.info("Dumping symbols from package: " + pkg) + try: + temp_dir = tempfile.mkdtemp() + if os.path.splitext(pkg)[1] == ".pkg": + expand_pkg(pkg, temp_dir) + elif os.path.splitext(pkg)[1] == ".zip": + expand_zip(pkg, temp_dir) + else: + expand_dmg(pkg, temp_dir) + + # check for any subpackages + for subpackage in find_packages(temp_dir): + logging.info("Found subpackage at: " + subpackage) + res = dump_symbols_from_package(executor, dump_syms, subpackage, dest) + if not res: + logging.error("Error while dumping subpackage: " + subpackage) + + # dump symbols from any payloads (only expecting one) in the package + for payload in find_payloads(temp_dir): + res = dump_symbols_from_payload(executor, dump_syms, payload, dest) + if not res: + successful = False + + # dump symbols directly extracted from the package + dump_symbols(executor, dump_syms, temp_dir, dest) + + except Exception as e: + traceback.print_exc() + logging.error("Exception while dumping symbols from package: {}".format(e)) + successful = False + + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir, onerror=shutil_error_handler) + + return successful + + +def read_processed_packages(tracking_file): + if tracking_file is None or not os.path.exists(tracking_file): + return set() + logging.info("Reading processed packages from {}".format(tracking_file)) + return set(open(tracking_file, "r").read().splitlines()) + + +def write_processed_packages(tracking_file, processed_packages): + if tracking_file is None: + return + logging.info( + "Writing {} processed packages to {}".format( + len(processed_packages), tracking_file + ) + ) + open(tracking_file, "w").write("\n".join(processed_packages)) + + +def process_packages(package_finder, to, tracking_file, dump_syms): + processed_packages = read_processed_packages(tracking_file) + with concurrent.futures.ProcessPoolExecutor() as executor: + for pkg in package_finder(): + if pkg in processed_packages: + logging.info("Skipping already-processed package: {}".format(pkg)) + else: + dump_symbols_from_package(executor, dump_syms, pkg, to) + processed_packages.add(pkg) + write_processed_packages(tracking_file, processed_packages) + + +def main(): + parser = argparse.ArgumentParser( + description="Extracts Breakpad symbols from a Mac OS X support update." + ) + parser.add_argument( + "--dump_syms", + default="dump_syms", + type=str, + help="path to the Breakpad dump_syms executable", + ) + parser.add_argument( + "--tracking-file", + type=str, + help="Path to a file in which to store information " + + "about already-processed packages", + ) + parser.add_argument( + "search", nargs="+", help="Paths to search recursively for packages" + ) + parser.add_argument("to", type=str, help="destination path for the symbols") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + for p in ("requests.packages.urllib3.connectionpool", "urllib3"): + urllib3_logger = logging.getLogger(p) + urllib3_logger.setLevel(logging.ERROR) + + if not args.search or not all(os.path.exists(p) for p in args.search): + logging.error("Invalid search path") + return + if not os.path.exists(args.to): + logging.error("Invalid path to destination") + return + + def finder(): + return find_all_packages(args.search) + + process_packages(finder, args.to, args.tracking_file, args.dump_syms) + + +if __name__ == "__main__": + main() diff --git a/tools/crashreporter/system-symbols/mac/get_update_packages.py b/tools/crashreporter/system-symbols/mac/get_update_packages.py new file mode 100644 index 0000000000..3192fa3ef0 --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/get_update_packages.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 Ted Mielczarek <ted@mielczarek.org> +# and Michael R. Miller <michaelrmmiller@gmail.com> +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import argparse +import concurrent.futures +import logging +import os +import re +import shutil +import subprocess +import tempfile + +import requests +import urlparse +from PackageSymbolDumper import find_packages, process_packages + +OSX_RE = re.compile(r"10\.[0-9]+\.[0-9]+") + + +def extract_dmg(dmg_path, dest): + logging.info("extract_dmg({}, {})".format(dmg_path, dest)) + with tempfile.NamedTemporaryFile() as f: + subprocess.check_call( + ["dmg", "extract", dmg_path, f.name], stdout=subprocess.DEVNULL + ) + subprocess.check_call(["hfsplus", f.name, "extractall"], cwd=dest) + + +def get_update_packages(): + for i in range(16): + logging.info("get_update_packages: page " + str(i)) + url = ( + "https://km.support.apple.com/kb/index?page=downloads_browse&sort=recency" + "&facet=all&category=PF6&locale=en_US&offset=%d" % i + ) + res = requests.get(url) + if res.status_code != 200: + break + data = res.json() + downloads = data.get("downloads", []) + if not downloads: + break + for d in downloads: + title = d.get("title", "") + if OSX_RE.search(title) and "Combo" not in title: + logging.info("Title: " + title) + if "fileurl" in d: + yield d["fileurl"] + else: + logging.warn("No fileurl in download!") + + +def fetch_url_to_file(url, download_dir): + filename = os.path.basename(urlparse.urlsplit(url).path) + local_filename = os.path.join(download_dir, filename) + if os.path.isfile(local_filename): + logging.info("{} already exists, skipping".format(local_filename)) + return None + r = requests.get(url, stream=True) + res_len = int(r.headers.get("content-length", "0")) + logging.info("Downloading {} -> {} ({} bytes)".format(url, local_filename, res_len)) + with open(local_filename, "wb") as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + return local_filename + + +def fetch_and_extract_dmg(url, tmpdir): + logging.info("fetch_and_extract_dmg: " + url) + filename = fetch_url_to_file(url, tmpdir) + if not filename: + return [] + # Extract dmg contents to a subdir + subdir = tempfile.mkdtemp(dir=tmpdir) + extract_dmg(filename, subdir) + packages = list(find_packages(subdir)) + logging.info( + "fetch_and_extract_dmg({}): found packages: {}".format(url, str(packages)) + ) + return packages + + +def find_update_packages(tmpdir): + logging.info("find_update_packages") + # Only download 2 packages at a time. + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + jobs = dict( + (executor.submit(fetch_and_extract_dmg, url, tmpdir), url) + for url in get_update_packages() + ) + for future in concurrent.futures.as_completed(jobs): + url = jobs[future] + if future.exception() is not None: + logging.error( + "exception downloading {}: {}".format(url, future.exception()) + ) + else: + for pkg in future.result(): + yield pkg + + +def main(): + parser = argparse.ArgumentParser( + description="Download OS X update packages and dump symbols from them" + ) + parser.add_argument( + "--dump_syms", + default="dump_syms", + type=str, + help="path to the Breakpad dump_syms executable", + ) + parser.add_argument("to", type=str, help="destination path for the symbols") + args = parser.parse_args() + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + for p in ("requests.packages.urllib3.connectionpool", "urllib3"): + urllib3_logger = logging.getLogger(p) + urllib3_logger.setLevel(logging.ERROR) + try: + tmpdir = tempfile.mkdtemp(suffix=".osxupdates") + + def finder(): + return find_update_packages(tmpdir) + + process_packages(finder, args.to, None, args.dump_syms) + finally: + shutil.rmtree(tmpdir) + + +if __name__ == "__main__": + main() diff --git a/tools/crashreporter/system-symbols/mac/list-packages.py b/tools/crashreporter/system-symbols/mac/list-packages.py new file mode 100755 index 0000000000..444c27be9d --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/list-packages.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# Copyright 2015 Ted Mielczarek. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import sys + +from reposadolib import reposadocommon + +reposadocommon.get_main_dir = lambda: "/usr/local/bin/" + +products = reposadocommon.get_product_info() +args = [] +for product_id, product in products.items(): + try: + title = product["title"] + except KeyError: + print("Missing title in {}, skipping".format(product), file=sys.stderr) + continue + + try: + major_version = int(product["version"].split(".")[0]) + except Exception: + print( + "Cannot extract the major version number in {}, skipping".format(product), + file=sys.stderr, + ) + continue + + if ( + title.startswith("OS X") + or title.startswith("Mac OS X") + or title.startswith("macOS") + ) and major_version <= 10: + args.append(product_id) + else: + print("Skipping %r for repo_sync" % title, file=sys.stderr) +if "JUST_ONE_PACKAGE" in os.environ: + args = args[:1] + +print("\n".join(args)) diff --git a/tools/crashreporter/system-symbols/mac/run.sh b/tools/crashreporter/system-symbols/mac/run.sh new file mode 100755 index 0000000000..8dec95dffe --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/run.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +set -v -e -x + +base="$(realpath "$(dirname "$0")")" +export PATH="$PATH:/builds/worker/bin:$base:${MOZ_FETCHES_DIR}/dmg" + +cd /builds/worker + +if test "$PROCESSED_PACKAGES_INDEX" && test "$PROCESSED_PACKAGES_PATH" && test "$TASKCLUSTER_ROOT_URL"; then + PROCESSED_PACKAGES="$TASKCLUSTER_ROOT_URL/api/index/v1/task/$PROCESSED_PACKAGES_INDEX/artifacts/$PROCESSED_PACKAGES_PATH" +fi + +if test "$PROCESSED_PACKAGES"; then + rm -f processed-packages + if test `curl --output /dev/null --silent --head --location "$PROCESSED_PACKAGES" -w "%{http_code}"` = 200; then + curl -L "$PROCESSED_PACKAGES" | gzip -dc > processed-packages + elif test -f "$PROCESSED_PACKAGES"; then + gzip -dc "$PROCESSED_PACKAGES" > processed-packages + fi + if test -f processed-packages; then + # Prevent reposado from downloading packages that have previously been + # dumped. + for f in $(cat processed-packages); do + mkdir -p "$(dirname "$f")" + touch "$f" + done + fi +fi + +mkdir -p /opt/data-reposado/html /opt/data-reposado/metadata artifacts + +# First, just fetch all the update info. +python3 /usr/local/bin/repo_sync --no-download + +# Next, fetch just the update packages we're interested in. +packages=$(python3 "${base}/list-packages.py") + +for package in ${packages}; do + # repo_sync is super-chatty, let's pipe stderr to separate files + python3 /usr/local/bin/repo_sync "--product-id=${package}" 2> "artifacts/repo_sync-product-id-${package}.stderr" + # Stop downloading packages if we have more than 10 GiB of them to process + download_size=$(du -B1073741824 -s /opt/data-reposado | cut -f1) + if [ ${download_size} -gt 10 ]; then + break + fi +done + +du -sh /opt/data-reposado + +# Now scrape symbols out of anything that was downloaded. +mkdir -p symbols tmp +env TMP=tmp python3 "${base}/PackageSymbolDumper.py" --tracking-file=/builds/worker/processed-packages --dump_syms=$MOZ_FETCHES_DIR/dump_syms/dump_syms /opt/data-reposado/html/content/downloads /builds/worker/symbols + +# Hand out artifacts +gzip -c processed-packages > artifacts/processed-packages.gz + +cd symbols +zip -r9 /builds/worker/artifacts/target.crashreporter-symbols.zip ./* || echo "No symbols dumped" diff --git a/tools/crashreporter/system-symbols/mac/scrapesymbols/__init__.py b/tools/crashreporter/system-symbols/mac/scrapesymbols/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/scrapesymbols/__init__.py diff --git a/tools/crashreporter/system-symbols/mac/scrapesymbols/gathersymbols.py b/tools/crashreporter/system-symbols/mac/scrapesymbols/gathersymbols.py new file mode 100644 index 0000000000..9998dcac27 --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/scrapesymbols/gathersymbols.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import argparse +import concurrent.futures +import datetime +import os +import subprocess +import sys +import traceback +import urllib +import zipfile + +import requests + +if sys.platform == "darwin": + SYSTEM_DIRS = [ + "/usr/lib", + "/System/Library/Frameworks", + "/System/Library/PrivateFrameworks", + "/System/Library/Extensions", + ] +else: + SYSTEM_DIRS = ["/lib", "/usr/lib"] +SYMBOL_SERVER_URL = "https://symbols.mozilla.org/" + + +def should_process(f, platform=sys.platform): + """Determine if a file is a platform binary""" + if platform == "darwin": + """ + The 'file' command can error out. One example is "illegal byte + sequence" on a Japanese language UTF8 text file. So we must wrap the + command in a try/except block to prevent the script from terminating + prematurely when this happens. + """ + try: + filetype = subprocess.check_output(["file", "-Lb", f], text=True) + except subprocess.CalledProcessError: + return False + """Skip kernel extensions""" + if "kext bundle" in filetype: + return False + return filetype.startswith("Mach-O") + else: + return subprocess.check_output(["file", "-Lb", f], text=True).startswith("ELF") + return False + + +def get_archs(filename, platform=sys.platform): + """ + Find the list of architectures present in a Mach-O file, or a single-element + list on non-OS X. + """ + architectures = [] + output = subprocess.check_output(["file", "-Lb", filename], text=True) + for string in output.split(" "): + if string == "arm64e": + architectures.append("arm64e") + elif string == "x86_64_haswell": + architectures.append("x86_64h") + elif string == "x86_64": + architectures.append("x86_64") + elif string == "i386": + architectures.append("i386") + + return architectures + + +def server_has_file(filename): + """ + Send the symbol server a HEAD request to see if it has this symbol file. + """ + try: + r = requests.head( + urllib.parse.urljoin(SYMBOL_SERVER_URL, urllib.parse.quote(filename)) + ) + return r.status_code == 200 + except requests.exceptions.RequestException: + return False + + +def process_file(dump_syms, path, arch, verbose, write_all): + arch_arg = ["-a", arch] + try: + stderr = None if verbose else subprocess.DEVNULL + stdout = subprocess.check_output([dump_syms] + arch_arg + [path], stderr=stderr) + except subprocess.CalledProcessError: + if verbose: + print("Processing %s%s...failed." % (path, " [%s]" % arch if arch else "")) + return None, None + module = stdout.splitlines()[0] + bits = module.split(b" ", 4) + if len(bits) != 5: + return None, None + _, platform, cpu_arch, debug_id, debug_file = bits + if verbose: + sys.stdout.write("Processing %s [%s]..." % (path, arch)) + filename = os.path.join(debug_file, debug_id, debug_file + b".sym") + # see if the server already has this symbol file + if not write_all: + if server_has_file(filename): + if verbose: + print("already on server.") + return None, None + # Collect for uploading + if verbose: + print("done.") + return filename, stdout + + +def get_files(paths, platform=sys.platform): + """ + For each entry passed in paths if the path is a file that can + be processed, yield it, otherwise if it is a directory yield files + under it that can be processed. + """ + for path in paths: + if os.path.isdir(path): + for root, subdirs, files in os.walk(path): + for f in files: + fullpath = os.path.join(root, f) + if should_process(fullpath, platform=platform): + yield fullpath + elif should_process(path, platform=platform): + yield path + + +def process_paths( + paths, executor, dump_syms, verbose, write_all=False, platform=sys.platform +): + jobs = set() + for fullpath in get_files(paths, platform=platform): + while os.path.islink(fullpath): + fullpath = os.path.join(os.path.dirname(fullpath), os.readlink(fullpath)) + if platform == "linux": + # See if there's a -dbg package installed and dump that instead. + dbgpath = "/usr/lib/debug" + fullpath + if os.path.isfile(dbgpath): + fullpath = dbgpath + for arch in get_archs(fullpath, platform=platform): + jobs.add( + executor.submit( + process_file, dump_syms, fullpath, arch, verbose, write_all + ) + ) + for job in concurrent.futures.as_completed(jobs): + try: + yield job.result() + except Exception as e: + traceback.print_exc(file=sys.stderr) + print("Error: %s" % str(e), file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-v", "--verbose", action="store_true", help="Produce verbose output" + ) + parser.add_argument( + "--all", + action="store_true", + help="Gather all system symbols, not just missing ones.", + ) + parser.add_argument("dump_syms", help="Path to dump_syms binary") + parser.add_argument( + "files", nargs="*", help="Specific files from which to gather symbols." + ) + args = parser.parse_args() + args.dump_syms = os.path.abspath(args.dump_syms) + # check for the dump_syms binary + if ( + not os.path.isabs(args.dump_syms) + or not os.path.exists(args.dump_syms) + or not os.access(args.dump_syms, os.X_OK) + ): + print( + "Error: can't find dump_syms binary at %s!" % args.dump_syms, + file=sys.stderr, + ) + return 1 + file_list = set() + executor = concurrent.futures.ProcessPoolExecutor() + zip_path = os.path.abspath("symbols.zip") + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + for filename, contents in process_paths( + args.files if args.files else SYSTEM_DIRS, + executor, + args.dump_syms, + args.verbose, + args.all, + ): + if filename and contents and filename not in file_list: + file_list.add(filename) + zf.writestr(filename, contents) + zf.writestr( + "ossyms-1.0-{platform}-{date}-symbols.txt".format( + platform=sys.platform.title(), + date=datetime.datetime.now().strftime("%Y%m%d%H%M%S"), + ), + "\n".join(file_list), + ) + if file_list: + if args.verbose: + print("Generated %s with %d symbols" % (zip_path, len(file_list))) + else: + os.unlink("symbols.zip") + + +if __name__ == "__main__": + main() diff --git a/tools/crashreporter/system-symbols/win/LICENSE b/tools/crashreporter/system-symbols/win/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/tools/crashreporter/system-symbols/win/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/crashreporter/system-symbols/win/known-microsoft-symbols.txt b/tools/crashreporter/system-symbols/win/known-microsoft-symbols.txt new file mode 100644 index 0000000000..d63dc716e9 --- /dev/null +++ b/tools/crashreporter/system-symbols/win/known-microsoft-symbols.txt @@ -0,0 +1,17 @@ +d2d1.pdb
+d3d10level9.pdb
+d3d10warp.pdb
+d3d11.pdb
+d3d9.pdb
+d3dcompiler_47.pdb
+d3dim700.pdb
+kernel32.pdb
+kernelbase.pdb
+ntdll.pdb
+user32.pdb
+wkernel32.pdb
+wkernelbase.pdb
+wntdll.pdb
+ws2_32.pdb
+wuser32.pdb
+zipwriter.pdb
diff --git a/tools/crashreporter/system-symbols/win/run.sh b/tools/crashreporter/system-symbols/win/run.sh new file mode 100755 index 0000000000..f95b2b160a --- /dev/null +++ b/tools/crashreporter/system-symbols/win/run.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -v -e -x + +base="$(realpath "$(dirname "$0")")" + +export DUMP_SYMS_PATH="${MOZ_FETCHES_DIR}/dump_syms/dump_syms" + +mkdir -p artifacts && \ +ulimit -n 16384 && \ +python3 "${base}/symsrv-fetch.py" artifacts/target.crashreporter-symbols.zip diff --git a/tools/crashreporter/system-symbols/win/scrape-report.py b/tools/crashreporter/system-symbols/win/scrape-report.py new file mode 100644 index 0000000000..9bc21801c3 --- /dev/null +++ b/tools/crashreporter/system-symbols/win/scrape-report.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# +# Copyright 2016 Mozilla +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import json +import logging +import os +import sys + +import requests +import urlparse + +log = logging.getLogger() + + +def fetch_missing_symbols_from_crash(file_or_crash): + if os.path.isfile(file_or_crash): + log.info("Fetching missing symbols from JSON file: %s" % file_or_crash) + j = {"json_dump": json.load(open(file_or_crash, "rb"))} + else: + if "report/index/" in file_or_crash: + crash_id = urlparse.urlparse(file_or_crash).path.split("/")[-1] + else: + crash_id = file_or_crash + url = ( + "https://crash-stats.mozilla.org/api/ProcessedCrash/" + "?crash_id={crash_id}&datatype=processed".format(crash_id=crash_id) + ) + log.info("Fetching missing symbols from crash: %s" % url) + r = requests.get(url) + if r.status_code != 200: + log.error("Failed to fetch crash %s" % url) + return set() + j = r.json() + return set( + [ + (m["debug_file"], m["debug_id"], m["filename"], m["code_id"]) + for m in j["json_dump"]["modules"] + if "missing_symbols" in m + ] + ) + + +def main(): + logging.basicConfig() + log.setLevel(logging.DEBUG) + urllib3_logger = logging.getLogger("urllib3") + urllib3_logger.setLevel(logging.ERROR) + + if len(sys.argv) < 2: + log.error("Specify a crash URL or ID") + sys.exit(1) + symbols = fetch_missing_symbols_from_crash(sys.argv[1]) + log.info("Found %d missing symbols" % len(symbols)) + c = csv.writer(sys.stdout) + c.writerow(["debug_file", "debug_id", "code_file", "code_id"]) + for row in symbols: + c.writerow(row) + + +if __name__ == "__main__": + main() diff --git a/tools/crashreporter/system-symbols/win/skiplist.txt b/tools/crashreporter/system-symbols/win/skiplist.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/crashreporter/system-symbols/win/skiplist.txt diff --git a/tools/crashreporter/system-symbols/win/symsrv-fetch.py b/tools/crashreporter/system-symbols/win/symsrv-fetch.py new file mode 100644 index 0000000000..be4bd101d5 --- /dev/null +++ b/tools/crashreporter/system-symbols/win/symsrv-fetch.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python +# +# Copyright 2016 Mozilla +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script will fetch a thousand recent crashes from Socorro, and try to +# retrieve missing symbols from Microsoft's symbol server. It honors a list +# (ignorelist.txt) of symbols that are known to be from our applications, +# and it maintains its own list of symbols that the MS symbol server +# doesn't have (skiplist.txt). +# +# The script also depends on having write access to the directory it is +# installed in, to write the skiplist text file. + +import argparse +import asyncio +import collections +import json +import logging +import os +import shutil +import zipfile +from collections import defaultdict +from tempfile import mkdtemp +from urllib.parse import quote, urljoin + +from aiofile import AIOFile, LineReader +from aiohttp import ClientSession, ClientTimeout +from aiohttp.connector import TCPConnector +from aiohttp_retry import JitterRetry, RetryClient + +# Just hardcoded here +MICROSOFT_SYMBOL_SERVER = "https://msdl.microsoft.com/download/symbols/" +USER_AGENT = "Microsoft-Symbol-Server/6.3.0.0" +MOZILLA_SYMBOL_SERVER = "https://symbols.mozilla.org/" +CRASHSTATS_API_URL = "https://crash-stats.mozilla.org/api/" +SUPERSEARCH_PARAM = "SuperSearch/?proto_signature=~.DLL&proto_signature=~.dll&platform=Windows&_results_number=1000" +PROCESSED_CRASHES_PARAM = "ProcessedCrash/?crash_id=" +HEADERS = {"User-Agent": USER_AGENT} +SYM_SRV = "SRV*{0}*https://msdl.microsoft.com/download/symbols;SRV*{0}*https://software.intel.com/sites/downloads/symbols;SRV*{0}*https://download.amd.com/dir/bin;SRV*{0}*https://driver-symbols.nvidia.com" # noqa +TIMEOUT = 7200 +RETRIES = 5 + + +MissingSymbol = collections.namedtuple( + "MissingSymbol", ["debug_file", "debug_id", "filename", "code_id"] +) +log = logging.getLogger() + + +def get_type(data): + # PDB v7 + if data.startswith(b"Microsoft C/C++ MSF 7.00"): + return "pdb-v7" + # PDB v2 + if data.startswith(b"Microsoft C/C++ program database 2.00"): + return "pdb-v2" + # DLL + if data.startswith(b"MZ"): + return "dll" + # CAB + if data.startswith(b"MSCF"): + return "cab" + + return "unknown" + + +async def exp_backoff(retry_num): + await asyncio.sleep(2**retry_num) + + +async def server_has_file(client, server, filename): + """ + Send the symbol server a HEAD request to see if it has this symbol file. + """ + url = urljoin(server, quote(filename)) + for i in range(RETRIES): + try: + async with client.head(url, headers=HEADERS, allow_redirects=True) as resp: + if resp.status == 200 and ( + ( + "microsoft" in server + and resp.headers["Content-Type"] == "application/octet-stream" + ) + or "mozilla" in server + ): + log.debug(f"File exists: {url}") + return True + else: + return False + except Exception as e: + # Sometimes we've SSL errors or disconnections... so in such a situation just retry + log.warning(f"Error with {url}: retry") + log.exception(e) + await exp_backoff(i) + + log.debug(f"Too many retries (HEAD) for {url}: give up.") + return False + + +async def fetch_file(client, server, filename): + """ + Fetch the file from the server + """ + url = urljoin(server, quote(filename)) + log.debug(f"Fetch url: {url}") + for i in range(RETRIES): + try: + async with client.get(url, headers=HEADERS, allow_redirects=True) as resp: + if resp.status == 200: + data = await resp.read() + typ = get_type(data) + if typ == "unknown": + # try again + await exp_backoff(i) + elif typ == "pdb-v2": + # too old: skip it + log.debug(f"PDB v2 (skipped because too old): {url}") + return None + else: + return data + else: + log.error(f"Cannot get data (status {resp.status}) for {url}: ") + except Exception as e: + log.warning(f"Error with {url}") + log.exception(e) + await asyncio.sleep(0.5) + + log.debug(f"Too many retries (GET) for {url}: give up.") + return None + + +def write_skiplist(skiplist): + with open("skiplist.txt", "w") as sf: + sf.writelines( + f"{debug_id} {debug_file}\n" for debug_id, debug_file in skiplist.items() + ) + + +async def fetch_crash(session, url): + async with session.get(url) as resp: + if resp.status == 200: + return json.loads(await resp.text()) + + raise RuntimeError("Network request returned status = " + str(resp.status)) + + +async def fetch_crashes(session, urls): + tasks = [] + for url in urls: + task = asyncio.create_task(fetch_crash(session, url)) + tasks.append(task) + results = await asyncio.gather(*tasks, return_exceptions=True) + return results + + +async def fetch_latest_crashes(client, url): + async with client.get(url + SUPERSEARCH_PARAM) as resp: + if resp.status != 200: + resp.raise_for_status() + data = await resp.text() + reply = json.loads(data) + crashes = [] + for crash in reply.get("hits"): + if "uuid" in crash: + crashes.append(crash.get("uuid")) + return crashes + + +async def fetch_missing_symbols(url): + log.info("Looking for missing symbols on %s" % url) + connector = TCPConnector(limit=4, limit_per_host=0) + missing_symbols = set() + crash_count = 0 + + client_session = ClientSession( + headers=HEADERS, connector=connector, timeout=ClientTimeout(total=TIMEOUT) + ) + while crash_count < 1000: + async with RetryClient( + client_session=client_session, + retry_options=JitterRetry(attempts=30, statuses=[429]), + ) as client: + crash_uuids = await fetch_latest_crashes(client, url) + urls = [url + PROCESSED_CRASHES_PARAM + uuid for uuid in crash_uuids] + crashes = await fetch_crashes(client, urls) + for crash in crashes: + if type(crash) is not dict: + continue + + crash_count += 1 + modules = crash.get("json_dump").get("modules") + for module in modules: + if module.get("missing_symbols"): + missing_symbols.add( + MissingSymbol( + module.get("debug_file"), + module.get("debug_id"), + module.get("filename"), + module.get("code_id"), + ) + ) + + return missing_symbols + + +async def get_list(filename): + alist = set() + try: + async with AIOFile(filename, "r") as In: + async for line in LineReader(In): + line = line.rstrip() + alist.add(line) + except FileNotFoundError: + pass + + log.debug(f"{filename} contains {len(alist)} items") + + return alist + + +async def get_skiplist(): + skiplist = {} + path = "skiplist.txt" + try: + async with AIOFile(path, "r") as In: + async for line in LineReader(In): + line = line.strip() + if line == "": + continue + s = line.split(" ", maxsplit=1) + if len(s) != 2: + continue + debug_id, debug_file = s + skiplist[debug_id] = debug_file.lower() + except FileNotFoundError: + pass + + log.debug(f"{path} contains {len(skiplist)} items") + + return skiplist + + +def get_missing_symbols(missing_symbols, skiplist, ignorelist): + modules = defaultdict(set) + stats = {"ignorelist": 0, "skiplist": 0} + for symbol in missing_symbols: + pdb = symbol.debug_file + debug_id = symbol.debug_id + code_file = symbol.filename + code_id = symbol.code_id + if pdb and debug_id and pdb.endswith(".pdb"): + if pdb.lower() in ignorelist: + stats["ignorelist"] += 1 + continue + + if skiplist.get(debug_id) != pdb.lower(): + modules[pdb].add((debug_id, code_file, code_id)) + else: + stats["skiplist"] += 1 + # We've asked the symbol server previously about this, + # so skip it. + log.debug("%s/%s already in skiplist", pdb, debug_id) + + return modules, stats + + +async def collect_info(client, filename, debug_id, code_file, code_id): + pdb_path = os.path.join(filename, debug_id, filename) + sym_path = os.path.join(filename, debug_id, filename.replace(".pdb", "") + ".sym") + + has_pdb = await server_has_file(client, MICROSOFT_SYMBOL_SERVER, pdb_path) + has_code = is_there = False + if has_pdb: + if not await server_has_file(client, MOZILLA_SYMBOL_SERVER, sym_path): + has_code = ( + code_file + and code_id + and await server_has_file( + client, + MICROSOFT_SYMBOL_SERVER, + f"{code_file}/{code_id}/{code_file}", + ) + ) + else: + # if the file is on moz sym server no need to do anything + is_there = True + has_pdb = False + + return (filename, debug_id, code_file, code_id, has_pdb, has_code, is_there) + + +async def check_x86_file(path): + async with AIOFile(path, "rb") as In: + head = b"MODULE windows x86 " + chunk = await In.read(len(head)) + if chunk == head: + return True + return False + + +async def run_command(cmd): + proc = await asyncio.create_subprocess_shell( + cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + _, err = await proc.communicate() + err = err.decode().strip() + + return err + + +async def dump_module( + output, symcache, filename, debug_id, code_file, code_id, has_code, dump_syms +): + sym_path = os.path.join(filename, debug_id, filename.replace(".pdb", ".sym")) + output_path = os.path.join(output, sym_path) + sym_srv = SYM_SRV.format(symcache) + res = {"path": sym_path, "error": "ok"} + + if has_code: + cmd = ( + f"{dump_syms} {code_file} --code-id {code_id} --check-cfi --inlines " + f"--store {output} --symbol-server '{sym_srv}' --verbose error" + ) + else: + cmd = ( + f"{dump_syms} {filename} --debug-id {debug_id} --check-cfi --inlines " + f"--store {output} --symbol-server '{sym_srv}' --verbose error" + ) + + err = await run_command(cmd) + + if err: + log.error(f"Error with {cmd}") + log.error(err) + res["error"] = "dump error" + return res + + if not os.path.exists(output_path): + log.error(f"Could not find file {output_path} after running {cmd}") + res["error"] = "dump error" + return res + + if not has_code and not await check_x86_file(output_path): + # PDB for 32 bits contains everything we need (symbols + stack unwind info) + # But PDB for 64 bits don't contain stack unwind info + # (they're in the binary (.dll/.exe) itself). + # So here we're logging because we've got a PDB (64 bits) without its DLL/EXE. + if code_file and code_id: + log.debug(f"x86_64 binary {code_file}/{code_id} required") + else: + log.debug(f"x86_64 binary for {filename}/{debug_id} required") + res["error"] = "no binary" + return res + + log.info(f"Successfully dumped: {filename}/{debug_id}") + return res + + +async def dump(output, symcache, modules, dump_syms): + tasks = [] + for filename, debug_id, code_file, code_id, has_code in modules: + tasks.append( + dump_module( + output, + symcache, + filename, + debug_id, + code_file, + code_id, + has_code, + dump_syms, + ) + ) + + res = await asyncio.gather(*tasks) + + # Even if we haven't CFI the generated file is useful to get symbols + # from addresses so keep error == 2. + file_index = {x["path"] for x in res if x["error"] in ["ok", "no binary"]} + stats = { + "dump_error": sum(1 for x in res if x["error"] == "dump error"), + "no_bin": sum(1 for x in res if x["error"] == "no binary"), + } + + return file_index, stats + + +async def collect(modules): + loop = asyncio.get_event_loop() + tasks = [] + + # In case of errors (Too many open files), just change limit_per_host + connector = TCPConnector(limit=100, limit_per_host=4) + + async with ClientSession( + loop=loop, timeout=ClientTimeout(total=TIMEOUT), connector=connector + ) as client: + for filename, ids in modules.items(): + for debug_id, code_file, code_id in ids: + tasks.append( + collect_info(client, filename, debug_id, code_file, code_id) + ) + + res = await asyncio.gather(*tasks) + to_dump = [] + stats = {"no_pdb": 0, "is_there": 0} + for filename, debug_id, code_file, code_id, has_pdb, has_code, is_there in res: + if not has_pdb: + if is_there: + stats["is_there"] += 1 + else: + stats["no_pdb"] += 1 + log.info(f"No pdb for {filename}/{debug_id}") + continue + + log.info( + f"To dump: {filename}/{debug_id}, {code_file}/{code_id} and has_code = {has_code}" + ) + to_dump.append((filename, debug_id, code_file, code_id, has_code)) + + log.info(f"Collected {len(to_dump)} files to dump") + + return to_dump, stats + + +async def make_dirs(path): + loop = asyncio.get_event_loop() + + def helper(path): + os.makedirs(path, exist_ok=True) + + await loop.run_in_executor(None, helper, path) + + +async def fetch_and_write(output, client, filename, file_id): + path = os.path.join(filename, file_id, filename) + data = await fetch_file(client, MICROSOFT_SYMBOL_SERVER, path) + + if not data: + return False + + output_dir = os.path.join(output, filename, file_id) + await make_dirs(output_dir) + + output_path = os.path.join(output_dir, filename) + async with AIOFile(output_path, "wb") as Out: + await Out.write(data) + + return True + + +async def fetch_all_modules(output, modules): + loop = asyncio.get_event_loop() + tasks = [] + fetched_modules = [] + + # In case of errors (Too many open files), just change limit_per_host + connector = TCPConnector(limit=100, limit_per_host=0) + + async with ClientSession( + loop=loop, timeout=ClientTimeout(total=TIMEOUT), connector=connector + ) as client: + for filename, debug_id, code_file, code_id, has_code in modules: + tasks.append(fetch_and_write(output, client, filename, debug_id)) + if has_code: + tasks.append(fetch_and_write(output, client, code_file, code_id)) + + res = await asyncio.gather(*tasks) + res = iter(res) + for filename, debug_id, code_file, code_id, has_code in modules: + fetched_pdb = next(res) + if has_code: + has_code = next(res) + if fetched_pdb: + fetched_modules.append( + (filename, debug_id, code_file, code_id, has_code) + ) + + return fetched_modules + + +def get_base_data(url): + async def helper(url): + return await asyncio.gather( + fetch_missing_symbols(url), + # Symbols that we know belong to us, so don't ask Microsoft for them. + get_list("ignorelist.txt"), + # Symbols that we know belong to Microsoft, so don't skiplist them. + get_list("known-microsoft-symbols.txt"), + # Symbols that we've asked for in the past unsuccessfully + get_skiplist(), + ) + + return asyncio.run(helper(url)) + + +def gen_zip(output, output_dir, file_index): + if not file_index: + return + + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as z: + for f in file_index: + z.write(os.path.join(output_dir, f), f) + log.info(f"Wrote zip as {output}") + + +def main(): + parser = argparse.ArgumentParser( + description="Fetch missing symbols from Microsoft symbol server" + ) + parser.add_argument( + "--crashstats-api", + type=str, + help="crash-stats API URL", + default=CRASHSTATS_API_URL, + ) + parser.add_argument("zip", type=str, help="output zip file") + parser.add_argument( + "--dump-syms", + type=str, + help="dump_syms path", + default=os.environ.get("DUMP_SYMS_PATH"), + ) + + args = parser.parse_args() + + assert args.dump_syms, "dump_syms path is empty" + + logging.basicConfig(level=logging.DEBUG) + aiohttp_logger = logging.getLogger("aiohttp.client") + aiohttp_logger.setLevel(logging.INFO) + log.info("Started") + + missing_symbols, ignorelist, known_ms_symbols, skiplist = get_base_data( + args.crashstats_api + ) + + modules, stats_skipped = get_missing_symbols(missing_symbols, skiplist, ignorelist) + + symbol_path = mkdtemp("symsrvfetch") + temp_path = mkdtemp(prefix="symcache") + + modules, stats_collect = asyncio.run(collect(modules)) + modules = asyncio.run(fetch_all_modules(temp_path, modules)) + + file_index, stats_dump = asyncio.run( + dump(symbol_path, temp_path, modules, args.dump_syms) + ) + + gen_zip(args.zip, symbol_path, file_index) + + shutil.rmtree(symbol_path, True) + shutil.rmtree(temp_path, True) + + write_skiplist(skiplist) + + if not file_index: + log.info(f"No symbols downloaded: {len(missing_symbols)} considered") + else: + log.info( + f"Total files: {len(missing_symbols)}, Stored {len(file_index)} symbol files" + ) + + log.info( + f"{stats_collect['is_there']} already present, {stats_skipped['ignorelist']} in ignored list, " # noqa + f"{stats_skipped['skiplist']} skipped, {stats_collect['no_pdb']} not found, " + f"{stats_dump['dump_error']} processed with errors, " + f"{stats_dump['no_bin']} processed but with no binaries (x86_64)" + ) + log.info("Finished, exiting") + + +if __name__ == "__main__": + main() diff --git a/tools/esmify/import-to-import_esmodule.js b/tools/esmify/import-to-import_esmodule.js new file mode 100644 index 0000000000..d8e0aee5bb --- /dev/null +++ b/tools/esmify/import-to-import_esmodule.js @@ -0,0 +1,472 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// jscodeshift rule to replace import calls for JSM with import calls for ESM +// or static import for ESM. + +/* eslint-env node */ + +const _path = require("path"); +const { isESMified } = require(_path.resolve(__dirname, "./is-esmified.js")); +const { + jsmExtPattern, + esmifyExtension, + isIdentifier, + isString, + warnForPath, + getPrevStatement, + getNextStatement, + isMemberExpressionWithIdentifiers, + rewriteMemberExpressionWithIdentifiers, + createMemberExpressionWithIdentifiers, +} = require(_path.resolve(__dirname, "./utils.js")); +const { + isImportESModuleCall, + replaceImportESModuleCall, + tryReplacingWithStaticImport, +} = require(_path.resolve(__dirname, "./static-import.js")); + +module.exports = function (fileInfo, api) { + const { jscodeshift } = api; + const root = jscodeshift(fileInfo.source); + doTranslate(fileInfo.path, jscodeshift, root); + return root.toSource({ lineTerminator: "\n" }); +}; + +module.exports.doTranslate = doTranslate; + +function isESMifiedAndTarget(resourceURI) { + const files = []; + if (!isESMified(resourceURI, files)) { + return false; + } + + if ("ESMIFY_TARGET_PREFIX" in process.env) { + const targetPrefix = process.env.ESMIFY_TARGET_PREFIX; + for (const esm of files) { + if (esm.startsWith(targetPrefix)) { + return true; + } + } + + return false; + } + + return true; +} + +const importCalls = [ + { + from: ["Cu", "import"], + to: ["ChromeUtils", "importESModule"], + }, + { + from: ["ChromeUtils", "import"], + to: ["ChromeUtils", "importESModule"], + }, + { + from: ["SpecialPowers", "ChromeUtils", "import"], + to: ["SpecialPowers", "ChromeUtils", "importESModule"], + }, +]; + +const singleLazyGetterCalls = [ + { + from: ["ChromeUtils", "defineModuleGetter"], + to: ["ChromeUtils", "defineESModuleGetters"], + }, + { + from: ["SpecialPowers", "ChromeUtils", "defineModuleGetter"], + to: ["SpecialPowers", "ChromeUtils", "defineESModuleGetters"], + }, +]; + +const multiLazyGettersCalls = [ + { + from: ["XPCOMUtils", "defineLazyModuleGetters"], + to: ["ChromeUtils", "defineESModuleGetters"], + }, +]; + +function isMemberExpressionMatchingPatterns(node, patterns) { + for (const item of patterns) { + if (isMemberExpressionWithIdentifiers(node, item.from)) { + return item; + } + } + + return null; +} + +function replaceImportCall(inputFile, jscodeshift, path, rewriteItem) { + if (path.node.arguments.length !== 1) { + warnForPath(inputFile, path, `import call should have only one argument`); + return; + } + + const resourceURINode = path.node.arguments[0]; + if (!isString(resourceURINode)) { + warnForPath(inputFile, path, `resource URI should be a string`); + return; + } + + const resourceURI = resourceURINode.value; + if (!resourceURI.match(jsmExtPattern)) { + warnForPath(inputFile, path, `Non-jsm: ${resourceURI}`); + return; + } + + if (!isESMifiedAndTarget(resourceURI)) { + return; + } + + if ( + !tryReplacingWithStaticImport( + jscodeshift, + inputFile, + path, + resourceURINode, + false + ) + ) { + rewriteMemberExpressionWithIdentifiers(path.node.callee, rewriteItem.to); + resourceURINode.value = esmifyExtension(resourceURI); + } +} + +// Find `ChromeUtils.defineESModuleGetters` or variant statement specified by +// expectedIDs, adjacent to `path` which uses the same target object. +function findDefineESModuleGettersStmt(path, expectedIDs) { + // `path` must be top-level. + if (path.parent.node.type !== "ExpressionStatement") { + return null; + } + + if (path.parent.parent.node.type !== "Program") { + return null; + } + + // Get previous or next statement with ChromeUtils.defineESModuleGetters. + let callStmt; + const prev = getPrevStatement(path.parent); + if ( + prev && + prev.type === "ExpressionStatement" && + prev.expression.type === "CallExpression" && + isMemberExpressionWithIdentifiers(prev.expression.callee, expectedIDs) + ) { + callStmt = prev; + } else { + const next = getNextStatement(path.parent); + if ( + next && + next.type === "ExpressionStatement" && + next.expression.type === "CallExpression" && + isMemberExpressionWithIdentifiers(next.expression.callee, expectedIDs) + ) { + callStmt = next; + } else { + return null; + } + } + + const call = callStmt.expression; + + if (call.arguments.length !== 2) { + return null; + } + + const modulesNode = call.arguments[1]; + if (modulesNode.type !== "ObjectExpression") { + return null; + } + + // Check if the target object is same. + if ( + path.node.arguments[0].type === "ThisExpression" && + call.arguments[0].type === "ThisExpression" + ) { + return callStmt; + } + + if ( + path.node.arguments[0].type === "Identifier" && + call.arguments[0].type === "Identifier" && + path.node.arguments[0].name === call.arguments[0].name + ) { + return callStmt; + } + + return null; +} + +function getPropKeyString(prop) { + if (prop.key.type === "Identifier") { + return prop.key.name; + } + + if (prop.key.type === "Literal") { + return prop.key.value.toString(); + } + + return ""; +} + +function sortProps(obj) { + obj.properties.sort((a, b) => { + return getPropKeyString(a) < getPropKeyString(b) ? -1 : 1; + }); +} + +// Move comments above `nodeFrom` before `nodeTo`. +function moveComments(nodeTo, nodeFrom) { + if (!nodeFrom.comments) { + return; + } + if (nodeTo.comments) { + nodeTo.comments = [...nodeTo.comments, ...nodeFrom.comments]; + } else { + nodeTo.comments = nodeFrom.comments; + } + nodeFrom.comments = []; +} + +function replaceLazyGetterCall(inputFile, jscodeshift, path, rewriteItem) { + if (path.node.arguments.length !== 3) { + warnForPath(inputFile, path, `lazy getter call should have 3 arguments`); + return; + } + + const nameNode = path.node.arguments[1]; + if (!isString(nameNode)) { + warnForPath(inputFile, path, `name should be a string`); + return; + } + + const resourceURINode = path.node.arguments[2]; + if (!isString(resourceURINode)) { + warnForPath(inputFile, path, `resource URI should be a string`); + return; + } + + const resourceURI = resourceURINode.value; + if (!resourceURI.match(jsmExtPattern)) { + warnForPath(inputFile, path, `Non-js/jsm: ${resourceURI}`); + return; + } + + if (!isESMifiedAndTarget(resourceURI)) { + return; + } + + resourceURINode.value = esmifyExtension(resourceURI); + const prop = jscodeshift.property( + "init", + jscodeshift.identifier(nameNode.value), + resourceURINode + ); + + const callStmt = findDefineESModuleGettersStmt(path, rewriteItem.to); + if (callStmt) { + // Move a property to existing ChromeUtils.defineESModuleGetters call. + + moveComments(callStmt, path.parent.node); + path.parent.prune(); + + callStmt.expression.arguments[1].properties.push(prop); + sortProps(callStmt.expression.arguments[1]); + } else { + // Convert this call into ChromeUtils.defineESModuleGetters. + + rewriteMemberExpressionWithIdentifiers(path.node.callee, rewriteItem.to); + path.node.arguments = [ + path.node.arguments[0], + jscodeshift.objectExpression([prop]), + ]; + } +} + +function replaceLazyGettersCall(inputFile, jscodeshift, path, rewriteItem) { + if (path.node.arguments.length !== 2) { + warnForPath(inputFile, path, `lazy getters call should have 2 arguments`); + return; + } + + const modulesNode = path.node.arguments[1]; + if (modulesNode.type !== "ObjectExpression") { + warnForPath(inputFile, path, `modules parameter should be an object`); + return; + } + + const esmProps = []; + const jsmProps = []; + + for (const prop of modulesNode.properties) { + const resourceURINode = prop.value; + if (!isString(resourceURINode)) { + warnForPath(inputFile, path, `resource URI should be a string`); + jsmProps.push(prop); + continue; + } + + const resourceURI = resourceURINode.value; + if (!resourceURI.match(jsmExtPattern)) { + warnForPath(inputFile, path, `Non-js/jsm: ${resourceURI}`); + jsmProps.push(prop); + continue; + } + + if (!isESMifiedAndTarget(resourceURI)) { + jsmProps.push(prop); + continue; + } + + esmProps.push(prop); + } + + if (esmProps.length === 0) { + return; + } + + let callStmt = findDefineESModuleGettersStmt(path, rewriteItem.to); + if (jsmProps.length === 0) { + if (callStmt) { + // Move all properties to existing ChromeUtils.defineESModuleGetters call. + + moveComments(callStmt, path.parent.node); + path.parent.prune(); + + for (const prop of esmProps) { + const resourceURINode = prop.value; + resourceURINode.value = esmifyExtension(resourceURINode.value); + callStmt.expression.arguments[1].properties.push(prop); + } + sortProps(callStmt.expression.arguments[1]); + } else { + // Convert this call into ChromeUtils.defineESModuleGetters. + + rewriteMemberExpressionWithIdentifiers(path.node.callee, rewriteItem.to); + for (const prop of esmProps) { + const resourceURINode = prop.value; + resourceURINode.value = esmifyExtension(resourceURINode.value); + } + } + } else { + // Move some properties to ChromeUtils.defineESModuleGetters. + + if (path.parent.node.type !== "ExpressionStatement") { + warnForPath(inputFile, path, `lazy getters call in unexpected context`); + return; + } + + if (!callStmt) { + callStmt = jscodeshift.expressionStatement( + jscodeshift.callExpression( + createMemberExpressionWithIdentifiers(jscodeshift, rewriteItem.to), + [path.node.arguments[0], jscodeshift.objectExpression([])] + ) + ); + path.parent.insertBefore(callStmt); + } + + moveComments(callStmt, path.parent.node); + + for (const prop of esmProps) { + const resourceURINode = prop.value; + resourceURINode.value = esmifyExtension(resourceURINode.value); + callStmt.expression.arguments[1].properties.push(prop); + } + sortProps(callStmt.expression.arguments[1]); + + path.node.arguments[1].properties = jsmProps; + } +} + +function getProp(obj, key) { + if (obj.type !== "ObjectExpression") { + return null; + } + + for (const prop of obj.properties) { + if (prop.computed) { + continue; + } + + if (!prop.key) { + continue; + } + + if (isIdentifier(prop.key, key)) { + return prop; + } + } + + return null; +} + +function tryReplaceActorDefinition(inputFile, path, name) { + const obj = path.node; + + const prop = getProp(obj, name); + if (!prop) { + return; + } + + const moduleURIProp = getProp(prop.value, "moduleURI"); + if (!moduleURIProp) { + return; + } + + if (!isString(moduleURIProp.value)) { + warnForPath(inputFile, path, `${name} moduleURI should be a string`); + return; + } + + const moduleURI = moduleURIProp.value.value; + if (!moduleURI.match(jsmExtPattern)) { + warnForPath(inputFile, path, `${name} Non-js/jsm: ${moduleURI}`); + return; + } + + if (!isESMifiedAndTarget(moduleURI)) { + return; + } + + moduleURIProp.key.name = "esModuleURI"; + moduleURIProp.value.value = esmifyExtension(moduleURI); +} + +function doTranslate(inputFile, jscodeshift, root) { + root.find(jscodeshift.CallExpression).forEach(path => { + if (isImportESModuleCall(path.node)) { + replaceImportESModuleCall(inputFile, jscodeshift, path, false); + return; + } + + const callee = path.node.callee; + + let item; + item = isMemberExpressionMatchingPatterns(callee, importCalls); + if (item) { + replaceImportCall(inputFile, jscodeshift, path, item); + return; + } + + item = isMemberExpressionMatchingPatterns(callee, singleLazyGetterCalls); + if (item) { + replaceLazyGetterCall(inputFile, jscodeshift, path, item); + return; + } + + item = isMemberExpressionMatchingPatterns(callee, multiLazyGettersCalls); + if (item) { + replaceLazyGettersCall(inputFile, jscodeshift, path, item); + } + }); + + root.find(jscodeshift.ObjectExpression).forEach(path => { + tryReplaceActorDefinition(inputFile, path, "parent"); + tryReplaceActorDefinition(inputFile, path, "child"); + }); +} diff --git a/tools/esmify/is-esmified.js b/tools/esmify/is-esmified.js new file mode 100644 index 0000000000..872b4cd877 --- /dev/null +++ b/tools/esmify/is-esmified.js @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// A utility to check if given JSM is already ESM-ified. + +/* eslint-env node */ + +const fs = require("fs"); +const _path = require("path"); +const { esmifyExtension } = require(_path.resolve(__dirname, "./utils.js")); + +let json_map; +if (process.env.ESMIFY_MAP_JSON) { + json_map = _path.resolve(process.env.ESMIFY_MAP_JSON); +} else { + json_map = _path.resolve(__dirname, "./map.json"); +} +const uri_map = JSON.parse(fs.readFileSync(json_map)); +const esm_uri_map = generateESMURIMap(uri_map); + +function generateESMURIMap(jsm_map) { + const esm_map = {}; + + for (let [uri, jsms] of Object.entries(jsm_map)) { + if (typeof jsms === "string") { + jsms = [jsms]; + } + esm_map[esmifyExtension(uri)] = jsms.map(esmifyExtension); + } + + return esm_map; +} + +function isESMifiedSlow(resourceURI) { + if (!(resourceURI in uri_map)) { + console.log(`WARNING: Unknown module: ${resourceURI}`); + return { result: false, jsms: [] }; + } + + let jsms = uri_map[resourceURI]; + if (typeof jsms === "string") { + jsms = [jsms]; + } + + const prefix = "../../"; + for (const jsm of jsms) { + if (fs.existsSync(prefix + jsm)) { + return { result: false, jsms }; + } + const esm = esmifyExtension(jsm); + if (!fs.existsSync(prefix + esm)) { + return { result: false, jsms }; + } + } + + return { result: true, jsms }; +} + +const isESMified_memo = {}; +function isESMified(resourceURI, files) { + if (!(resourceURI in isESMified_memo)) { + isESMified_memo[resourceURI] = isESMifiedSlow(resourceURI); + } + + for (const jsm of isESMified_memo[resourceURI].jsms) { + files.push(esmifyExtension(jsm)); + } + + return isESMified_memo[resourceURI].result; +} + +function getESMFiles(resourceURI) { + if (resourceURI in esm_uri_map) { + return esm_uri_map[resourceURI]; + } + return []; +} + +exports.isESMified = isESMified; +exports.getESMFiles = getESMFiles; diff --git a/tools/esmify/mach_commands.py b/tools/esmify/mach_commands.py new file mode 100644 index 0000000000..7b72c7b0e3 --- /dev/null +++ b/tools/esmify/mach_commands.py @@ -0,0 +1,908 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os +import pathlib +import re +import subprocess +import sys + +from mach.decorators import Command, CommandArgument + + +def path_sep_to_native(path_str): + """Make separators in the path OS native.""" + return pathlib.os.sep.join(path_str.split("/")) + + +def path_sep_from_native(path): + """Make separators in the path OS native.""" + return "/".join(str(path).split(pathlib.os.sep)) + + +excluded_from_convert_prefix = list( + map( + path_sep_to_native, + [ + # Testcases for actors. + "toolkit/actors/TestProcessActorChild.jsm", + "toolkit/actors/TestProcessActorParent.jsm", + "toolkit/actors/TestWindowChild.jsm", + "toolkit/actors/TestWindowParent.jsm", + "js/xpconnect/tests/unit/", + # Testcase for build system. + "python/mozbuild/mozbuild/test/", + ], + ) +) + + +def is_excluded_from_convert(path): + """Returns true if the JSM file shouldn't be converted to ESM.""" + path_str = str(path) + for prefix in excluded_from_convert_prefix: + if path_str.startswith(prefix): + return True + + return False + + +excluded_from_imports_prefix = list( + map( + path_sep_to_native, + [ + # Vendored or auto-generated files. + "browser/components/pocket/content/panels/js/vendor.bundle.js", + "devtools/client/debugger/dist/parser-worker.js", + "devtools/client/debugger/test/mochitest/examples/react/build/main.js", + "devtools/client/debugger/test/mochitest/examples/sourcemapped/polyfill-bundle.js", + "devtools/client/inspector/markup/test/shadowdom_open_debugger.min.js", + "devtools/client/shared/source-map-loader/test/browser/fixtures/bundle.js", + "layout/style/test/property_database.js", + "services/fxaccounts/FxAccountsPairingChannel.js", + "testing/web-platform/", + # Unrelated testcases that has edge case syntax. + "browser/components/sessionstore/test/unit/data/", + "devtools/client/debugger/src/workers/parser/tests/fixtures/", + "devtools/client/debugger/test/mochitest/examples/sourcemapped/fixtures/", + "devtools/client/webconsole/test/browser/test-syntaxerror-worklet.js", + "devtools/server/tests/xpcshell/test_framebindings-03.js", + "devtools/server/tests/xpcshell/test_framebindings-04.js", + "devtools/shared/tests/xpcshell/test_eventemitter_basic.js", + "devtools/shared/tests/xpcshell/test_eventemitter_static.js", + "dom/base/crashtests/module-with-syntax-error.js", + "dom/base/test/file_bug687859-16.js", + "dom/base/test/file_bug687859-16.js", + "dom/base/test/file_js_cache_syntax_error.js", + "dom/base/test/jsmodules/module_badSyntax.js", + "dom/canvas/test/reftest/webgl-utils.js", + "dom/encoding/test/file_utf16_be_bom.js", + "dom/encoding/test/file_utf16_le_bom.js", + "dom/html/test/bug649134/file_bug649134-1.sjs", + "dom/html/test/bug649134/file_bug649134-2.sjs", + "dom/media/webrtc/tests/mochitests/identity/idp-bad.js", + "dom/serviceworkers/test/file_js_cache_syntax_error.js", + "dom/serviceworkers/test/parse_error_worker.js", + "dom/workers/test/importScripts_worker_imported3.js", + "dom/workers/test/invalid.js", + "dom/workers/test/threadErrors_worker1.js", + "dom/xhr/tests/browser_blobFromFile.js", + "image/test/browser/browser_image.js", + "js/xpconnect/tests/chrome/test_bug732665_meta.js", + "js/xpconnect/tests/mochitest/class_static_worker.js", + "js/xpconnect/tests/unit/bug451678_subscript.js", + "js/xpconnect/tests/unit/error_other.sys.mjs", + "js/xpconnect/tests/unit/es6module_parse_error.js", + "js/xpconnect/tests/unit/recursive_importA.jsm", + "js/xpconnect/tests/unit/recursive_importB.jsm", + "js/xpconnect/tests/unit/syntax_error.jsm", + "js/xpconnect/tests/unit/test_defineModuleGetter.js", + "js/xpconnect/tests/unit/test_import.js", + "js/xpconnect/tests/unit/test_import_shim.js", + "js/xpconnect/tests/unit/test_recursive_import.js", + "js/xpconnect/tests/unit/test_unload.js", + "modules/libpref/test/unit/data/testParser.js", + "python/mozbuild/mozbuild/test/", + "remote/shared/messagehandler/test/browser/resources/modules/root/invalid.sys.mjs", + "testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js", + "testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js", + "toolkit/components/reader/Readerable.sys.mjs", + "toolkit/components/workerloader/tests/moduleF-syntax-error.js", + "tools/lint/test/", + "tools/update-packaging/test/", + # SpiderMonkey internals. + "js/examples/", + "js/src/", + # Files has macro. + "browser/app/profile/firefox.js", + "browser/branding/official/pref/firefox-branding.js", + "browser/components/enterprisepolicies/schemas/schema.sys.mjs", + "browser/locales/en-US/firefox-l10n.js", + "mobile/android/app/geckoview-prefs.js", + "mobile/android/locales/en-US/mobile-l10n.js", + "modules/libpref/greprefs.js", + "modules/libpref/init/all.js", + "testing/condprofile/condprof/tests/profile/user.js", + "testing/mozbase/mozprofile/tests/files/prefs_with_comments.js", + "toolkit/modules/AppConstants.sys.mjs", + "toolkit/mozapps/update/tests/data/xpcshellConstantsPP.js", + # Uniffi templates + "toolkit/components/uniffi-bindgen-gecko-js/src/templates/js/", + ], + ) +) + +EXCLUSION_FILES = [ + os.path.join("tools", "rewriting", "Generated.txt"), + os.path.join("tools", "rewriting", "ThirdPartyPaths.txt"), +] + + +def load_exclusion_files(): + for path in EXCLUSION_FILES: + with open(path, "r") as f: + for line in f: + p = path_sep_to_native(re.sub(r"\*$", "", line.strip())) + excluded_from_imports_prefix.append(p) + + +def is_excluded_from_imports(path): + """Returns true if the JS file content shouldn't be handled by + jscodeshift. + + This filter is necessary because jscodeshift cannot handle some + syntax edge cases and results in unexpected rewrite.""" + path_str = str(path) + for prefix in excluded_from_imports_prefix: + if path_str.startswith(prefix): + return True + + return False + + +# Wrapper for hg/git operations +class VCSUtils: + def run(self, cmd): + # Do not pass check=True because the pattern can match no file. + lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode() + return filter(lambda x: x != "", lines.split("\n")) + + +class HgUtils(VCSUtils): + def is_available(): + return pathlib.Path(".hg").exists() + + def rename(self, before, after): + cmd = ["hg", "rename", before, after] + subprocess.run(cmd, check=True) + + def find_jsms(self, path): + jsms = [] + + # NOTE: `set:glob:` syntax does not accept backslash on windows. + path = path_sep_from_native(path) + + cmd = ["hg", "files", f'set:glob:"{path}/**/*.jsm"'] + for line in self.run(cmd): + jsm = pathlib.Path(line) + if is_excluded_from_convert(jsm): + continue + jsms.append(jsm) + + cmd = [ + "hg", + "files", + rf"set:grep('EXPORTED_SYMBOLS = \[') and glob:\"{path}/**/*.js\"", + ] + for line in self.run(cmd): + jsm = pathlib.Path(line) + if is_excluded_from_convert(jsm): + continue + jsms.append(jsm) + + return jsms + + def find_all_jss(self, path): + jss = [] + + # NOTE: `set:glob:` syntax does not accept backslash on windows. + path = path_sep_from_native(path) + + cmd = [ + "hg", + "files", + f'set:glob:"{path}/**/*.jsm" or glob:"{path}/**/*.js" or ' + + f'glob:"{path}/**/*.mjs" or glob:"{path}/**/*.sjs"', + ] + for line in self.run(cmd): + js = pathlib.Path(line) + if is_excluded_from_imports(js): + continue + jss.append(js) + + return jss + + +class GitUtils(VCSUtils): + def is_available(): + return pathlib.Path(".git").exists() + + def rename(self, before, after): + cmd = ["git", "mv", before, after] + subprocess.run(cmd, check=True) + + def find_jsms(self, path): + jsms = [] + + cmd = ["git", "ls-files", f"{path}/*.jsm"] + for line in self.run(cmd): + jsm = pathlib.Path(line) + if is_excluded_from_convert(jsm): + continue + jsms.append(jsm) + + handled = {} + cmd = ["git", "grep", "EXPORTED_SYMBOLS = \\[", f"{path}/*.js"] + for line in self.run(cmd): + m = re.search("^([^:]+):", line) + if not m: + continue + filename = m.group(1) + if filename in handled: + continue + handled[filename] = True + jsm = pathlib.Path(filename) + if is_excluded_from_convert(jsm): + continue + jsms.append(jsm) + + return jsms + + def find_all_jss(self, path): + jss = [] + + cmd = [ + "git", + "ls-files", + f"{path}/*.jsm", + f"{path}/*.js", + f"{path}/*.mjs", + f"{path}/*.sjs", + ] + for line in self.run(cmd): + js = pathlib.Path(line) + if is_excluded_from_imports(js): + continue + jss.append(js) + + return jss + + +class Summary: + def __init__(self): + self.convert_errors = [] + self.import_errors = [] + self.rename_errors = [] + self.no_refs = [] + + +@Command( + "esmify", + category="misc", + description="ESMify JSM files.", +) +@CommandArgument( + "path", + nargs=1, + help="Path to the JSM file to ESMify, or the directory that contains " + "JSM files and/or JS files that imports ESM-ified JSM.", +) +@CommandArgument( + "--convert", + action="store_true", + help="Only perform the step 1 = convert part", +) +@CommandArgument( + "--imports", + action="store_true", + help="Only perform the step 2 = import calls part", +) +@CommandArgument( + "--prefix", + default="", + help="Restrict the target of import in the step 2 to ESM-ified JSM, by the " + "prefix match for the JSM file's path. e.g. 'browser/'.", +) +def esmify(command_context, path=None, convert=False, imports=False, prefix=""): + """ + This command does the following 2 steps: + 1. Convert the JSM file specified by `path` to ESM file, or the JSM files + inside the directory specified by `path` to ESM files, and also + fix references in build files and test definitions + 2. Convert import calls inside file(s) specified by `path` for ESM-ified + files to use new APIs + + Example 1: + # Convert all JSM files inside `browser/components/pagedata` directory, + # and replace all references for ESM-ified files in the entire tree to use + # new APIs + + $ ./mach esmify --convert browser/components/pagedata + $ ./mach esmify --imports . --prefix=browser/components/pagedata + + Example 2: + # Convert all JSM files inside `browser` directory, and replace all + # references for the JSM files inside `browser` directory to use + # new APIs + + $ ./mach esmify browser + """ + + def error(text): + command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}") + + def warn(text): + command_context.log(logging.WARN, "esmify", {}, f"[WARN] {text}") + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + # If no options is specified, perform both. + if not convert and not imports: + convert = True + imports = True + + path = pathlib.Path(path[0]) + + if not verify_path(command_context, path): + return 1 + + if HgUtils.is_available(): + vcs_utils = HgUtils() + elif GitUtils.is_available(): + vcs_utils = GitUtils() + else: + error( + "This script needs to be run inside mozilla-central " + "checkout of either mercurial or git." + ) + return 1 + + load_exclusion_files() + + info("Setting up jscodeshift...") + setup_jscodeshift() + + is_single_file = path.is_file() + + modified_files = [] + summary = Summary() + + if convert: + info("Searching files to convert to ESM...") + if is_single_file: + jsms = [path] + else: + jsms = vcs_utils.find_jsms(path) + + info(f"Found {len(jsms)} file(s) to convert to ESM.") + + info("Converting to ESM...") + jsms = convert_module(jsms, summary) + if jsms is None: + error("Failed to rewrite exports.") + return 1 + + info("Renaming...") + esms = rename_jsms(command_context, vcs_utils, jsms, summary) + + modified_files += esms + + if imports: + info("Searching files to rewrite imports...") + + if is_single_file: + if convert: + # Already converted above + jss = esms + else: + jss = [path] + else: + jss = vcs_utils.find_all_jss(path) + + info(f"Checking {len(jss)} JS file(s). Rewriting any matching imports...") + + result = rewrite_imports(jss, prefix, summary) + if result is None: + return 1 + + info(f"Rewritten {len(result)} file(s).") + + # Only modified files needs eslint fix + modified_files += result + + modified_files = list(set(modified_files)) + + info(f"Applying eslint --fix for {len(modified_files)} file(s)...") + eslint_fix(command_context, modified_files) + + def print_files(f, errors): + for [path, message] in errors: + f(f" * {path}") + if message: + f(f" {message}") + + if len(summary.convert_errors): + error("========") + error("Following files are not converted into ESM due to error:") + print_files(error, summary.convert_errors) + + if len(summary.import_errors): + warn("========") + warn("Following files are not rewritten to import ESMs due to error:") + warn( + "(NOTE: Errors related to 'private names' are mostly due to " + " preprocessor macros in the file):" + ) + print_files(warn, summary.import_errors) + + if len(summary.rename_errors): + error("========") + error("Following files are not renamed due to error:") + print_files(error, summary.rename_errors) + + if len(summary.no_refs): + warn("========") + warn("Following files are not found in any build files.") + warn("Please update references to those files manually:") + print_files(warn, summary.rename_errors) + + return 0 + + +def verify_path(command_context, path): + """Check if the path passed to the command is valid relative path.""" + + def error(text): + command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}") + + if not path.exists(): + error(f"{path} does not exist.") + return False + + if path.is_absolute(): + error("Path must be a relative path from mozilla-central checkout.") + return False + + return True + + +def find_file(path, target): + """Find `target` file in ancestor of path.""" + target_path = path.parent / target + if not target_path.exists(): + if path.parent == path: + return None + + return find_file(path.parent, target) + + return target_path + + +def try_rename_in(command_context, path, target, jsm_name, esm_name, jsm_path): + """Replace the occurrences of `jsm_name` with `esm_name` in `target` + file.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + if type(target) is str: + # Target is specified by filename, that may exist somewhere in + # the jsm's directory or ancestor directories. + target_path = find_file(path, target) + if not target_path: + return False + + # JSM should be specified with relative path in the file. + # + # Single moz.build or jar.mn can contain multiple files with same name. + # Search for relative path. + jsm_relative_path = jsm_path.relative_to(target_path.parent) + jsm_path_str = path_sep_from_native(str(jsm_relative_path)) + else: + # Target is specified by full path. + target_path = target + + # JSM should be specified with full path in the file. + jsm_path_str = path_sep_from_native(str(jsm_path)) + + jsm_path_re = re.compile(r"\b" + jsm_path_str.replace(".", r"\.") + r"\b") + jsm_name_re = re.compile(r"\b" + jsm_name.replace(".", r"\.") + r"\b") + + modified = False + content = "" + with open(target_path, "r") as f: + for line in f: + if jsm_path_re.search(line): + modified = True + line = jsm_name_re.sub(esm_name, line) + + content += line + + if modified: + info(f" {str(target_path)}") + info(f" {jsm_name} => {esm_name}") + with open(target_path, "w", newline="\n") as f: + f.write(content) + + return True + + +def try_rename_uri_in(command_context, target, jsm_name, esm_name, jsm_uri, esm_uri): + """Replace the occurrences of `jsm_uri` with `esm_uri` in `target` file.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + modified = False + content = "" + with open(target, "r") as f: + for line in f: + if jsm_uri in line: + modified = True + line = line.replace(jsm_uri, esm_uri) + + content += line + + if modified: + info(f" {str(target)}") + info(f" {jsm_name} => {esm_name}") + with open(target, "w", newline="\n") as f: + f.write(content) + + return True + + +def try_rename_components_conf(command_context, path, jsm_name, esm_name): + """Replace the occurrences of `jsm_name` with `esm_name` in components.conf + file.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + target_path = find_file(path, "components.conf") + if not target_path: + return False + + # Unlike try_rename_in, components.conf contains the URL instead of + # relative path, and also there are no known files with same name. + # Simply replace the filename. + + with open(target_path, "r") as f: + content = f.read() + + prop_re = re.compile( + "[\"']jsm[\"']:(.*)" + r"\b" + jsm_name.replace(".", r"\.") + r"\b" + ) + + if not prop_re.search(content): + return False + + info(f" {str(target_path)}") + info(f" {jsm_name} => {esm_name}") + + content = prop_re.sub(r"'esModule':\1" + esm_name, content) + with open(target_path, "w", newline="\n") as f: + f.write(content) + + return True + + +def esmify_name(name): + return re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", name) + + +def esmify_path(jsm_path): + jsm_name = jsm_path.name + esm_name = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_name) + esm_path = jsm_path.parent / esm_name + return esm_path + + +path_to_uri_map = None + + +def load_path_to_uri_map(): + global path_to_uri_map + + if path_to_uri_map: + return + + if "ESMIFY_MAP_JSON" in os.environ: + json_map = pathlib.Path(os.environ["ESMIFY_MAP_JSON"]) + else: + json_map = pathlib.Path(__file__).parent / "map.json" + + with open(json_map, "r") as f: + uri_to_path_map = json.loads(f.read()) + + path_to_uri_map = dict() + + for uri, paths in uri_to_path_map.items(): + if type(paths) is str: + paths = [paths] + + for path in paths: + path_to_uri_map[path] = uri + + +def find_jsm_uri(jsm_path): + load_path_to_uri_map() + + path = path_sep_from_native(jsm_path) + + if path in path_to_uri_map: + return path_to_uri_map[path] + + return None + + +def rename_single_file(command_context, vcs_utils, jsm_path, summary): + """Rename `jsm_path` to .sys.mjs, and fix references to the file in build + and test definitions.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + esm_path = esmify_path(jsm_path) + + jsm_name = jsm_path.name + esm_name = esm_path.name + + target_files = [ + ".eslintignore", + "moz.build", + "jar.mn", + "browser.toml", + "browser-common.toml", + "chrome.toml", + "mochitest.toml", + "xpcshell.toml", + "xpcshell-child-process.toml", + "xpcshell-common.toml", + "xpcshell-parent-process.toml", + pathlib.Path("tools", "lint", "eslint.yml"), + pathlib.Path("tools", "lint", "rejected-words.yml"), + ] + + info(f"{jsm_path} => {esm_path}") + + renamed = False + for target in target_files: + if try_rename_in( + command_context, jsm_path, target, jsm_name, esm_name, jsm_path + ): + renamed = True + + if try_rename_components_conf(command_context, jsm_path, jsm_name, esm_name): + renamed = True + + uri_target_files = [ + pathlib.Path( + "browser", "base", "content", "test", "performance", "browser_startup.js" + ), + pathlib.Path( + "browser", + "base", + "content", + "test", + "performance", + "browser_startup_content.js", + ), + pathlib.Path( + "browser", + "base", + "content", + "test", + "performance", + "browser_startup_content_subframe.js", + ), + pathlib.Path( + "toolkit", + "components", + "backgroundtasks", + "tests", + "browser", + "browser_xpcom_graph_wait.js", + ), + ] + + jsm_uri = find_jsm_uri(jsm_path) + if jsm_uri: + esm_uri = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_uri) + + for target in uri_target_files: + if try_rename_uri_in( + command_context, target, jsm_uri, esm_uri, jsm_name, esm_name + ): + renamed = True + + if not renamed: + summary.no_refs.append([jsm_path, None]) + + if not esm_path.exists(): + vcs_utils.rename(jsm_path, esm_path) + else: + summary.rename_errors.append([jsm_path, f"{esm_path} already exists"]) + + return esm_path + + +def rename_jsms(command_context, vcs_utils, jsms, summary): + esms = [] + for jsm in jsms: + esm = rename_single_file(command_context, vcs_utils, jsm, summary) + esms.append(esm) + + return esms + + +npm_prefix = pathlib.Path("tools") / "esmify" +path_from_npm_prefix = pathlib.Path("..") / ".." + + +def setup_jscodeshift(): + """Install jscodeshift.""" + cmd = [ + sys.executable, + "./mach", + "npm", + "install", + "jscodeshift", + "--save-dev", + "--prefix", + str(npm_prefix), + ] + subprocess.run(cmd, check=True) + + +def run_npm_command(args, env, stdin): + cmd = [ + sys.executable, + "./mach", + "npm", + "run", + ] + args + p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p.stdin.write(stdin) + p.stdin.close() + + ok_files = [] + errors = [] + while True: + line = p.stdout.readline() + if not line: + break + line = line.rstrip().decode() + + if line.startswith(" NOC "): + continue + + print(line) + + m = re.search(r"^ (OKK|ERR) ([^ ]+)(?: (.+))?", line) + if not m: + continue + + result = m.group(1) + # NOTE: path is written from `tools/esmify`. + path = pathlib.Path(m.group(2)).relative_to(path_from_npm_prefix) + error = m.group(3) + + if result == "OKK": + ok_files.append(path) + + if result == "ERR": + errors.append([path, error]) + + if p.wait() != 0: + return [None, None] + + return ok_files, errors + + +def convert_module(jsms, summary): + """Replace EXPORTED_SYMBOLS with export declarations, and replace + ChromeUtils.importESModule with static import as much as possible, + and return the list of successfully rewritten files.""" + + if len(jsms) == 0: + return [] + + env = os.environ.copy() + + stdin = "\n".join(map(str, paths_from_npm_prefix(jsms))).encode() + + ok_files, errors = run_npm_command( + [ + "convert_module", + "--prefix", + str(npm_prefix), + ], + env=env, + stdin=stdin, + ) + + if ok_files is None and errors is None: + return None + + summary.convert_errors.extend(errors) + + return ok_files + + +def rewrite_imports(jss, prefix, summary): + """Replace import calls for JSM with import calls for ESM or static import + for ESM.""" + + if len(jss) == 0: + return [] + + env = os.environ.copy() + env["ESMIFY_TARGET_PREFIX"] = prefix + + stdin = "\n".join(map(str, paths_from_npm_prefix(jss))).encode() + + ok_files, errors = run_npm_command( + [ + "rewrite_imports", + "--prefix", + str(npm_prefix), + ], + env=env, + stdin=stdin, + ) + + if ok_files is None and errors is None: + return None + + summary.import_errors.extend(errors) + + return ok_files + + +def paths_from_npm_prefix(paths): + """Convert relative path from mozilla-central to relative path from + tools/esmify.""" + return list(map(lambda path: path_from_npm_prefix / path, paths)) + + +def eslint_fix(command_context, files): + """Auto format files.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + if len(files) == 0: + return + + remaining = files[0:] + + # There can be too many files for single command line, perform by chunk. + max_files = 16 + while len(remaining) > max_files: + info(f"{len(remaining)} files remaining") + + chunk = remaining[0:max_files] + remaining = remaining[max_files:] + + cmd = [sys.executable, "./mach", "eslint", "--fix"] + chunk + subprocess.run(cmd, check=True) + + info(f"{len(remaining)} files remaining") + chunk = remaining + cmd = [sys.executable, "./mach", "eslint", "--fix"] + chunk + subprocess.run(cmd, check=True) diff --git a/tools/esmify/map.json b/tools/esmify/map.json new file mode 100644 index 0000000000..14491bdb07 --- /dev/null +++ b/tools/esmify/map.json @@ -0,0 +1,1134 @@ +{ + "chrome://devtools-startup/content/DevToolsShim.jsm": "devtools/startup/DevToolsShim.jsm", + "chrome://global/content/tabprompts.jsm": "toolkit/components/prompts/content/tabprompts.jsm", + "chrome://mochikit/content/ShutdownLeaksCollector.jsm": "testing/mochitest/ShutdownLeaksCollector.jsm", + "chrome://mochikit/content/tests/SimpleTest/StructuredLog.jsm": "testing/modules/StructuredLog.jsm", + "chrome://mochitests/content/browser/accessible/tests/browser/Common.jsm": "accessible/tests/browser/Common.jsm", + "chrome://mochitests/content/browser/accessible/tests/browser/Layout.jsm": "accessible/tests/browser/Layout.jsm", + "chrome://mochitests/content/browser/devtools/client/shared/sourceeditor/test/CodeMirrorTestActors.jsm": "devtools/client/shared/sourceeditor/test/CodeMirrorTestActors.jsm", + "chrome://mochitests/content/chrome/docshell/test/chrome/DocShellHelpers.jsm": "docshell/test/chrome/DocShellHelpers.jsm", + "chrome://mochitests/content/chrome/dom/console/tests/console.jsm": "dom/console/tests/console.jsm", + "chrome://mochitests/content/chrome/dom/network/tests/tcpsocket_test.jsm": "dom/network/tests/tcpsocket_test.jsm", + "chrome://mochitests/content/chrome/dom/url/tests/file_worker_url.jsm": "dom/url/tests/file_worker_url.jsm", + "chrome://mochitests/content/chrome/dom/url/tests/test_bug883784.jsm": "dom/url/tests/test_bug883784.jsm", + "chrome://mochitests/content/chrome/dom/workers/test/WorkerTest.jsm": "dom/workers/test/WorkerTest.jsm", + "chrome://pocket/content/Pocket.jsm": "browser/components/pocket/content/Pocket.jsm", + "chrome://pocket/content/SaveToPocket.jsm": "browser/components/pocket/content/SaveToPocket.jsm", + "chrome://pocket/content/pktApi.jsm": "browser/components/pocket/content/pktApi.jsm", + "chrome://pocket/content/pktTelemetry.jsm": "browser/components/pocket/content/pktTelemetry.jsm", + "chrome://remote/content/server/HTTPD.jsm": "netwerk/test/httpserver/httpd.js", + "resource:///actors/ASRouterChild.jsm": "browser/components/newtab/actors/ASRouterChild.jsm", + "resource:///actors/ASRouterParent.jsm": "browser/components/newtab/actors/ASRouterParent.jsm", + "resource:///actors/AboutLoginsChild.jsm": "browser/components/aboutlogins/AboutLoginsChild.jsm", + "resource:///actors/AboutLoginsParent.jsm": "browser/components/aboutlogins/AboutLoginsParent.jsm", + "resource:///actors/AboutNewTabChild.jsm": "browser/actors/AboutNewTabChild.jsm", + "resource:///actors/AboutNewTabParent.jsm": "browser/actors/AboutNewTabParent.jsm", + "resource:///actors/AboutPocketChild.jsm": "browser/actors/AboutPocketChild.jsm", + "resource:///actors/AboutPocketParent.jsm": "browser/actors/AboutPocketParent.jsm", + "resource:///actors/AboutPrivateBrowsingChild.jsm": "browser/actors/AboutPrivateBrowsingChild.jsm", + "resource:///actors/AboutPrivateBrowsingParent.jsm": "browser/actors/AboutPrivateBrowsingParent.jsm", + "resource:///actors/AboutProtectionsChild.jsm": "browser/actors/AboutProtectionsChild.jsm", + "resource:///actors/AboutProtectionsParent.jsm": "browser/actors/AboutProtectionsParent.jsm", + "resource:///actors/AboutReaderChild.jsm": "browser/actors/AboutReaderChild.jsm", + "resource:///actors/AboutReaderParent.jsm": "browser/actors/AboutReaderParent.jsm", + "resource:///actors/AboutTabCrashedChild.jsm": "browser/actors/AboutTabCrashedChild.jsm", + "resource:///actors/AboutTabCrashedParent.jsm": "browser/actors/AboutTabCrashedParent.jsm", + "resource:///actors/AboutWelcomeChild.jsm": "browser/components/newtab/aboutwelcome/AboutWelcomeChild.jsm", + "resource:///actors/AboutWelcomeParent.jsm": "browser/components/newtab/aboutwelcome/AboutWelcomeParent.jsm", + "resource:///actors/BlockedSiteChild.jsm": "browser/actors/BlockedSiteChild.jsm", + "resource:///actors/BlockedSiteParent.jsm": "browser/actors/BlockedSiteParent.jsm", + "resource:///actors/BrowserProcessChild.jsm": "browser/actors/BrowserProcessChild.jsm", + "resource:///actors/BrowserTabChild.jsm": "browser/actors/BrowserTabChild.jsm", + "resource:///actors/BrowserTabParent.jsm": "browser/actors/BrowserTabParent.jsm", + "resource:///actors/ClickHandlerChild.jsm": "browser/actors/ClickHandlerChild.jsm", + "resource:///actors/ClickHandlerParent.jsm": "browser/actors/ClickHandlerParent.jsm", + "resource:///actors/ContentDelegateChild.jsm": "mobile/android/actors/ContentDelegateChild.jsm", + "resource:///actors/ContentDelegateParent.jsm": "mobile/android/actors/ContentDelegateParent.jsm", + "resource:///actors/ContentSearchChild.jsm": "browser/actors/ContentSearchChild.jsm", + "resource:///actors/ContentSearchParent.jsm": "browser/actors/ContentSearchParent.jsm", + "resource:///actors/ContextMenuChild.jsm": "browser/actors/ContextMenuChild.jsm", + "resource:///actors/ContextMenuParent.jsm": "browser/actors/ContextMenuParent.jsm", + "resource:///actors/DOMFullscreenChild.jsm": "browser/actors/DOMFullscreenChild.jsm", + "resource:///actors/DOMFullscreenParent.jsm": "browser/actors/DOMFullscreenParent.jsm", + "resource:///actors/DecoderDoctorChild.jsm": "browser/actors/DecoderDoctorChild.jsm", + "resource:///actors/DecoderDoctorParent.jsm": "browser/actors/DecoderDoctorParent.jsm", + "resource:///actors/EncryptedMediaChild.jsm": "browser/actors/EncryptedMediaChild.jsm", + "resource:///actors/EncryptedMediaParent.jsm": "browser/actors/EncryptedMediaParent.jsm", + "resource:///actors/FormValidationChild.jsm": "browser/actors/FormValidationChild.jsm", + "resource:///actors/FormValidationParent.jsm": "browser/actors/FormValidationParent.jsm", + "resource:///actors/GeckoViewAutoFillChild.jsm": "mobile/android/actors/GeckoViewAutoFillChild.jsm", + "resource:///actors/GeckoViewAutoFillParent.jsm": "mobile/android/actors/GeckoViewAutoFillParent.jsm", + "resource:///actors/GeckoViewContentChild.jsm": "mobile/android/actors/GeckoViewContentChild.jsm", + "resource:///actors/GeckoViewContentParent.jsm": "mobile/android/actors/GeckoViewContentParent.jsm", + "resource:///actors/GeckoViewFormValidationChild.jsm": "mobile/android/actors/GeckoViewFormValidationChild.jsm", + "resource:///actors/GeckoViewPermissionChild.jsm": "mobile/android/actors/GeckoViewPermissionChild.jsm", + "resource:///actors/GeckoViewPermissionParent.jsm": "mobile/android/actors/GeckoViewPermissionParent.jsm", + "resource:///actors/GeckoViewPermissionProcessChild.jsm": "mobile/android/actors/GeckoViewPermissionProcessChild.jsm", + "resource:///actors/GeckoViewPermissionProcessParent.jsm": "mobile/android/actors/GeckoViewPermissionProcessParent.jsm", + "resource:///actors/GeckoViewPromptChild.jsm": "mobile/android/actors/GeckoViewPromptChild.jsm", + "resource:///actors/GeckoViewPrompterChild.jsm": "mobile/android/actors/GeckoViewPrompterChild.jsm", + "resource:///actors/GeckoViewPrompterParent.jsm": "mobile/android/actors/GeckoViewPrompterParent.jsm", + "resource:///actors/GeckoViewSettingsChild.jsm": "mobile/android/actors/GeckoViewSettingsChild.jsm", + "resource:///actors/InteractionsChild.jsm": "browser/components/places/InteractionsChild.jsm", + "resource:///actors/InteractionsParent.jsm": "browser/components/places/InteractionsParent.jsm", + "resource:///actors/LightweightThemeChild.jsm": "browser/actors/LightweightThemeChild.jsm", + "resource:///actors/LinkHandlerChild.jsm": "browser/actors/LinkHandlerChild.jsm", + "resource:///actors/LinkHandlerParent.jsm": "browser/actors/LinkHandlerParent.jsm", + "resource:///actors/LoadURIDelegateChild.jsm": "mobile/android/actors/LoadURIDelegateChild.jsm", + "resource:///actors/LoadURIDelegateParent.jsm": "mobile/android/actors/LoadURIDelegateParent.jsm", + "resource:///actors/MediaControlDelegateChild.jsm": "mobile/android/actors/MediaControlDelegateChild.jsm", + "resource:///actors/MediaControlDelegateParent.jsm": "mobile/android/actors/MediaControlDelegateParent.jsm", + "resource:///actors/PageDataChild.jsm": "browser/components/pagedata/PageDataChild.jsm", + "resource:///actors/PageDataParent.jsm": "browser/components/pagedata/PageDataParent.jsm", + "resource:///actors/PageInfoChild.jsm": "browser/actors/PageInfoChild.jsm", + "resource:///actors/PageStyleChild.jsm": "browser/actors/PageStyleChild.jsm", + "resource:///actors/PageStyleParent.jsm": "browser/actors/PageStyleParent.jsm", + "resource:///actors/PluginChild.jsm": "browser/actors/PluginChild.jsm", + "resource:///actors/PluginParent.jsm": "browser/actors/PluginParent.jsm", + "resource:///actors/PointerLockChild.jsm": "browser/actors/PointerLockChild.jsm", + "resource:///actors/PointerLockParent.jsm": "browser/actors/PointerLockParent.jsm", + "resource:///actors/ProgressDelegateChild.jsm": "mobile/android/actors/ProgressDelegateChild.jsm", + "resource:///actors/ProgressDelegateParent.jsm": "mobile/android/actors/ProgressDelegateParent.jsm", + "resource:///actors/PromptParent.jsm": "browser/actors/PromptParent.jsm", + "resource:///actors/RFPHelperChild.jsm": "browser/actors/RFPHelperChild.jsm", + "resource:///actors/RFPHelperParent.jsm": "browser/actors/RFPHelperParent.jsm", + "resource:///actors/RefreshBlockerChild.jsm": "browser/actors/RefreshBlockerChild.jsm", + "resource:///actors/RefreshBlockerParent.jsm": "browser/actors/RefreshBlockerParent.jsm", + "resource:///actors/ScreenshotsComponentChild.jsm": "browser/actors/ScreenshotsComponentChild.jsm", + "resource:///actors/ScrollDelegateChild.jsm": "mobile/android/actors/ScrollDelegateChild.jsm", + "resource:///actors/ScrollDelegateParent.jsm": "mobile/android/actors/ScrollDelegateParent.jsm", + "resource:///actors/SearchSERPTelemetryChild.jsm": "browser/actors/SearchSERPTelemetryChild.jsm", + "resource:///actors/SearchSERPTelemetryParent.jsm": "browser/actors/SearchSERPTelemetryParent.jsm", + "resource:///actors/SelectionActionDelegateChild.jsm": "mobile/android/actors/SelectionActionDelegateChild.jsm", + "resource:///actors/SelectionActionDelegateParent.jsm": "mobile/android/actors/SelectionActionDelegateParent.jsm", + "resource:///actors/SwitchDocumentDirectionChild.jsm": "browser/actors/SwitchDocumentDirectionChild.jsm", + "resource:///actors/WebRTCChild.jsm": "browser/actors/WebRTCChild.jsm", + "resource:///actors/WebRTCParent.jsm": "browser/actors/WebRTCParent.jsm", + "resource:///modules/360seMigrationUtils.jsm": "browser/components/migration/360seMigrationUtils.jsm", + "resource:///modules/AboutDebuggingRegistration.jsm": "devtools/startup/AboutDebuggingRegistration.jsm", + "resource:///modules/AboutDevToolsToolboxRegistration.jsm": "devtools/startup/AboutDevToolsToolboxRegistration.jsm", + "resource:///modules/AboutNewTab.jsm": "browser/modules/AboutNewTab.jsm", + "resource:///modules/AboutNewTabService.jsm": "browser/components/newtab/AboutNewTabService.jsm", + "resource://gre/modules/AppUpdater.jsm": "toolkit/mozapps/update/AppUpdater.jsm", + "resource:///modules/AsyncTabSwitcher.jsm": "browser/modules/AsyncTabSwitcher.jsm", + "resource:///modules/AttributionCode.jsm": "browser/components/attribution/AttributionCode.jsm", + "resource:///modules/BrowserContentHandler.jsm": "browser/components/BrowserContentHandler.jsm", + "resource:///modules/BrowserGlue.jsm": "browser/components/BrowserGlue.jsm", + "resource:///modules/BrowserSearchTelemetry.jsm": "browser/components/search/BrowserSearchTelemetry.jsm", + "resource:///modules/BrowserUIUtils.jsm": "browser/modules/BrowserUIUtils.jsm", + "resource:///modules/BrowserWindowTracker.jsm": "browser/modules/BrowserWindowTracker.jsm", + "resource:///modules/BuiltInThemeConfig.jsm": "browser/themes/BuiltInThemeConfig.jsm", + "resource:///modules/BuiltInThemes.jsm": "browser/themes/BuiltInThemes.jsm", + "resource:///modules/CaptiveDetect.jsm": "toolkit/components/captivedetect/CaptiveDetect.jsm", + "resource:///modules/ChromeMacOSLoginCrypto.jsm": "browser/components/migration/ChromeMacOSLoginCrypto.jsm", + "resource:///modules/ChromeMigrationUtils.jsm": "browser/components/migration/ChromeMigrationUtils.jsm", + "resource:///modules/ChromeProfileMigrator.jsm": "browser/components/migration/ChromeProfileMigrator.jsm", + "resource:///modules/ChromeWindowsLoginCrypto.jsm": "browser/components/migration/ChromeWindowsLoginCrypto.jsm", + "resource:///modules/CommonNames.jsm": "browser/components/places/CommonNames.jsm", + "resource:///modules/ContentCrashHandlers.jsm": "browser/modules/ContentCrashHandlers.jsm", + "resource:///modules/CustomizableUI.jsm": "browser/components/customizableui/CustomizableUI.jsm", + "resource:///modules/CustomizableWidgets.jsm": "browser/components/customizableui/CustomizableWidgets.jsm", + "resource:///modules/CustomizeMode.jsm": "browser/components/customizableui/CustomizeMode.jsm", + "resource:///modules/DevToolsStartup.jsm": "devtools/startup/DevToolsStartup.jsm", + "resource:///modules/Discovery.jsm": "browser/modules/Discovery.jsm", + "resource:///modules/DoHConfig.jsm": "browser/components/doh/DoHConfig.jsm", + "resource:///modules/DoHController.jsm": "browser/components/doh/DoHController.jsm", + "resource:///modules/DoHHeuristics.jsm": "browser/components/doh/DoHHeuristics.jsm", + "resource:///modules/DomainGroupBuilder.jsm": "browser/components/places/DomainGroupBuilder.jsm", + "resource:///modules/DownloadSpamProtection.jsm": "browser/components/downloads/DownloadSpamProtection.jsm", + "resource:///modules/DownloadsCommon.jsm": "browser/components/downloads/DownloadsCommon.jsm", + "resource:///modules/DownloadsMacFinderProgress.jsm": "browser/components/downloads/DownloadsMacFinderProgress.jsm", + "resource:///modules/DownloadsTaskbar.jsm": "browser/components/downloads/DownloadsTaskbar.jsm", + "resource:///modules/DownloadsViewUI.jsm": "browser/components/downloads/DownloadsViewUI.jsm", + "resource:///modules/DownloadsViewableInternally.jsm": "browser/components/downloads/DownloadsViewableInternally.jsm", + "resource:///modules/DragPositionManager.jsm": "browser/components/customizableui/DragPositionManager.jsm", + "resource:///modules/ESEDBReader.jsm": "browser/components/migration/ESEDBReader.jsm", + "resource:///modules/EdgeProfileMigrator.jsm": "browser/components/migration/EdgeProfileMigrator.jsm", + "resource:///modules/EveryWindow.jsm": "browser/modules/EveryWindow.jsm", + "resource:///modules/ExtensionControlledPopup.jsm": "browser/components/extensions/ExtensionControlledPopup.jsm", + "resource:///modules/ExtensionPopups.jsm": "browser/components/extensions/ExtensionPopups.jsm", + "resource:///modules/ExtensionsUI.jsm": "browser/modules/ExtensionsUI.jsm", + "resource:///modules/FaviconLoader.jsm": "browser/modules/FaviconLoader.jsm", + "resource:///modules/FirefoxProfileMigrator.jsm": "browser/components/migration/FirefoxProfileMigrator.jsm", + "resource:///modules/HeadlessShell.jsm": "browser/components/shell/HeadlessShell.jsm", + "resource:///modules/HomePage.jsm": "browser/modules/HomePage.jsm", + "resource:///modules/IEProfileMigrator.jsm": "browser/components/migration/IEProfileMigrator.jsm", + "resource:///modules/InstallerPrefs.jsm": "browser/components/installerprefs/InstallerPrefs.jsm", + "resource:///modules/Interactions.jsm": "browser/components/places/Interactions.jsm", + "resource:///modules/InteractionsBlocklist.jsm": "browser/components/places/InteractionsBlocklist.jsm", + "resource:///modules/LaterRun.jsm": "browser/modules/LaterRun.jsm", + "resource:///modules/LoginBreaches.jsm": "browser/components/aboutlogins/LoginBreaches.jsm", + "resource:///modules/MSMigrationUtils.jsm": "browser/components/migration/MSMigrationUtils.jsm", + "resource:///modules/MacAttribution.jsm": "browser/components/attribution/MacAttribution.jsm", + "resource:///modules/MacTouchBar.jsm": "browser/components/touchbar/MacTouchBar.jsm", + "resource:///modules/MigrationUtils.jsm": "browser/components/migration/MigrationUtils.jsm", + "resource:///modules/NewTabPagePreloading.jsm": "browser/modules/NewTabPagePreloading.jsm", + "resource:///modules/OpenInTabsUtils.jsm": "browser/modules/OpenInTabsUtils.jsm", + "resource:///modules/PageActions.jsm": "browser/modules/PageActions.jsm", + "resource:///modules/PanelMultiView.jsm": "browser/components/customizableui/PanelMultiView.jsm", + "resource:///modules/PartnerLinkAttribution.jsm": "browser/modules/PartnerLinkAttribution.jsm", + "resource:///modules/PinnedGroupBuilder.jsm": "browser/components/places/PinnedGroupBuilder.jsm", + "resource:///modules/PlacesUIUtils.jsm": "browser/components/places/PlacesUIUtils.jsm", + "resource:///modules/ProcessHangMonitor.jsm": "browser/modules/ProcessHangMonitor.jsm", + "resource:///modules/ProfileMigrator.jsm": "browser/components/migration/ProfileMigrator.jsm", + "resource:///modules/PromptCollection.jsm": "browser/components/prompts/PromptCollection.jsm", + "resource:///modules/SafariProfileMigrator.jsm": "browser/components/migration/SafariProfileMigrator.jsm", + "resource:///modules/Sanitizer.jsm": "browser/modules/Sanitizer.jsm", + "resource:///modules/ScreenshotChild.jsm": "browser/components/shell/ScreenshotChild.jsm", + "resource:///modules/ScreenshotsOverlayChild.jsm": "browser/components/screenshots/ScreenshotsOverlayChild.jsm", + "resource:///modules/ScreenshotsUtils.jsm": "browser/components/screenshots/ScreenshotsUtils.jsm", + "resource:///modules/SearchOneOffs.jsm": "browser/components/search/SearchOneOffs.jsm", + "resource:///modules/SearchSERPTelemetry.jsm": "browser/components/search/SearchSERPTelemetry.jsm", + "resource:///modules/SearchUIUtils.jsm": "browser/components/search/SearchUIUtils.jsm", + "resource:///modules/SearchWidgetTracker.jsm": "browser/components/customizableui/SearchWidgetTracker.jsm", + "resource:///modules/SelectionChangedMenulist.jsm": "browser/modules/SelectionChangedMenulist.jsm", + "resource:///modules/ShellService.jsm": "browser/components/shell/ShellService.jsm", + "resource:///modules/SiteDataManager.jsm": "browser/modules/SiteDataManager.jsm", + "resource:///modules/SitePermissions.jsm": "browser/modules/SitePermissions.jsm", + "resource:///modules/SnapshotGroups.jsm": "browser/components/places/SnapshotGroups.jsm", + "resource:///modules/SnapshotMonitor.jsm": "browser/components/places/SnapshotMonitor.jsm", + "resource:///modules/SnapshotScorer.jsm": "browser/components/places/SnapshotScorer.jsm", + "resource:///modules/SnapshotSelector.jsm": "browser/components/places/SnapshotSelector.jsm", + "resource:///modules/Snapshots.jsm": "browser/components/places/Snapshots.jsm", + "resource:///modules/StartupRecorder.jsm": "browser/components/StartupRecorder.jsm", + "resource:///modules/TRRPerformance.jsm": "browser/components/doh/TRRPerformance.jsm", + "resource:///modules/TabUnloader.jsm": "browser/modules/TabUnloader.jsm", + "resource:///modules/TabsList.jsm": "browser/modules/TabsList.jsm", + "resource:///modules/ThemeVariableMap.jsm": "browser/themes/ThemeVariableMap.jsm", + "resource:///modules/TransientPrefs.jsm": "browser/modules/TransientPrefs.jsm", + "resource:///modules/UITour.jsm": "browser/components/uitour/UITour.jsm", + "resource:///modules/UITourChild.jsm": "browser/components/uitour/UITourChild.jsm", + "resource:///modules/UITourParent.jsm": "browser/components/uitour/UITourParent.jsm", + "resource:///modules/UnitConverterSimple.jsm": "browser/components/urlbar/unitconverters/UnitConverterSimple.jsm", + "resource:///modules/UnitConverterTemperature.jsm": "browser/components/urlbar/unitconverters/UnitConverterTemperature.jsm", + "resource:///modules/UnitConverterTimezone.jsm": "browser/components/urlbar/unitconverters/UnitConverterTimezone.jsm", + "resource:///modules/UrlbarController.jsm": "browser/components/urlbar/UrlbarController.jsm", + "resource:///modules/UrlbarEventBufferer.jsm": "browser/components/urlbar/UrlbarEventBufferer.jsm", + "resource:///modules/UrlbarInput.jsm": "browser/components/urlbar/UrlbarInput.jsm", + "resource:///modules/UrlbarMuxerUnifiedComplete.jsm": "browser/components/urlbar/UrlbarMuxerUnifiedComplete.jsm", + "resource:///modules/UrlbarPrefs.jsm": "browser/components/urlbar/UrlbarPrefs.jsm", + "resource:///modules/UrlbarProviderAboutPages.jsm": "browser/components/urlbar/UrlbarProviderAboutPages.jsm", + "resource:///modules/UrlbarProviderAliasEngines.jsm": "browser/components/urlbar/UrlbarProviderAliasEngines.jsm", + "resource:///modules/UrlbarProviderAutofill.jsm": "browser/components/urlbar/UrlbarProviderAutofill.jsm", + "resource:///modules/UrlbarProviderBookmarkKeywords.jsm": "browser/components/urlbar/UrlbarProviderBookmarkKeywords.jsm", + "resource:///modules/UrlbarProviderCalculator.jsm": "browser/components/urlbar/UrlbarProviderCalculator.jsm", + "resource:///modules/UrlbarProviderExtension.jsm": "browser/components/urlbar/UrlbarProviderExtension.jsm", + "resource:///modules/UrlbarProviderHeuristicFallback.jsm": "browser/components/urlbar/UrlbarProviderHeuristicFallback.jsm", + "resource:///modules/UrlbarProviderInputHistory.jsm": "browser/components/urlbar/UrlbarProviderInputHistory.jsm", + "resource:///modules/UrlbarProviderInterventions.jsm": "browser/components/urlbar/UrlbarProviderInterventions.jsm", + "resource:///modules/UrlbarProviderOmnibox.jsm": "browser/components/urlbar/UrlbarProviderOmnibox.jsm", + "resource:///modules/UrlbarProviderOpenTabs.jsm": "browser/components/urlbar/UrlbarProviderOpenTabs.jsm", + "resource:///modules/UrlbarProviderPlaces.jsm": "browser/components/urlbar/UrlbarProviderPlaces.jsm", + "resource:///modules/UrlbarProviderPrivateSearch.jsm": "browser/components/urlbar/UrlbarProviderPrivateSearch.jsm", + "resource:///modules/UrlbarProviderQuickSuggest.jsm": "browser/components/urlbar/UrlbarProviderQuickSuggest.jsm", + "resource:///modules/UrlbarProviderRemoteTabs.jsm": "browser/components/urlbar/UrlbarProviderRemoteTabs.jsm", + "resource:///modules/UrlbarProviderSearchSuggestions.jsm": "browser/components/urlbar/UrlbarProviderSearchSuggestions.jsm", + "resource:///modules/UrlbarProviderSearchTips.jsm": "browser/components/urlbar/UrlbarProviderSearchTips.jsm", + "resource:///modules/UrlbarProviderTabToSearch.jsm": "browser/components/urlbar/UrlbarProviderTabToSearch.jsm", + "resource:///modules/UrlbarProviderTokenAliasEngines.jsm": "browser/components/urlbar/UrlbarProviderTokenAliasEngines.jsm", + "resource:///modules/UrlbarProviderTopSites.jsm": "browser/components/urlbar/UrlbarProviderTopSites.jsm", + "resource:///modules/UrlbarProviderUnitConversion.jsm": "browser/components/urlbar/UrlbarProviderUnitConversion.jsm", + "resource:///modules/UrlbarProvidersManager.jsm": "browser/components/urlbar/UrlbarProvidersManager.jsm", + "resource:///modules/UrlbarResult.jsm": "browser/components/urlbar/UrlbarResult.jsm", + "resource:///modules/UrlbarSearchOneOffs.jsm": "browser/components/urlbar/UrlbarSearchOneOffs.jsm", + "resource:///modules/UrlbarSearchUtils.jsm": "browser/components/urlbar/UrlbarSearchUtils.jsm", + "resource:///modules/UrlbarTokenizer.jsm": "browser/components/urlbar/UrlbarTokenizer.jsm", + "resource:///modules/UrlbarUtils.jsm": "browser/components/urlbar/UrlbarUtils.jsm", + "resource:///modules/UrlbarValueFormatter.jsm": "browser/components/urlbar/UrlbarValueFormatter.jsm", + "resource:///modules/UrlbarView.jsm": "browser/components/urlbar/UrlbarView.jsm", + "resource:///modules/WebProtocolHandlerRegistrar.jsm": "browser/components/protocolhandler/WebProtocolHandlerRegistrar.jsm", + "resource:///modules/WindowsJumpLists.jsm": "browser/modules/WindowsJumpLists.jsm", + "resource:///modules/WindowsPreviewPerTab.jsm": "browser/modules/WindowsPreviewPerTab.jsm", + "resource:///modules/ZoomUI.jsm": "browser/modules/ZoomUI.jsm", + "resource:///modules/asrouter/ASRouter.jsm": "browser/components/asrouter/modules/ASRouter.jsm", + "resource:///modules/asrouter/ASRouterDefaultConfig.jsm": "browser/components/asrouter/modules/ASRouterDefaultConfig.jsm", + "resource:///modules/asrouter/ASRouterParentProcessMessageHandler.jsm": "browser/components/asrouter/modules/ASRouterParentProcessMessageHandler.jsm", + "resource:///modules/asrouter/ASRouterPreferences.jsm": "browser/components/asrouter/modules/ASRouterPreferences.jsm", + "resource:///modules/asrouter/ASRouterTargeting.jsm": "browser/components/asrouter/modules/ASRouterTargeting.jsm", + "resource:///modules/asrouter/ASRouterTriggerListeners.jsm": "browser/components/asrouter/modules/ASRouterTriggerListeners.jsm", + "resource:///modules/asrouter/CFRPageActions.jsm": "browser/components/asrouter/modules/CFRPageActions.jsm", + "resource:///modules/asrouter/InfoBar.jsm": "browser/components/asrouter/modules/InfoBar.jsm", + "resource:///modules/asrouter/MomentsPageHub.jsm": "browser/components/asrouter/modules/MomentsPageHub.jsm", + "resource:///modules/asrouter/OnboardingMessageProvider.jsm": "browser/components/asrouter/modules/OnboardingMessageProvider.jsm", + "resource:///modules/asrouter/ToolbarBadgeHub.jsm": "browser/components/asrouter/modules/ToolbarBadgeHub.jsm", + "resource:///modules/asrouter/ToolbarPanelHub.jsm": "browser/components/asrouter/modules/ToolbarPanelHub.jsm", + "resource:///modules/distribution.js": "browser/components/distribution.js", + "resource:///modules/pagedata/OpenGraphPageData.jsm": "browser/components/pagedata/OpenGraphPageData.jsm", + "resource:///modules/pagedata/PageDataSchema.jsm": "browser/components/pagedata/PageDataSchema.jsm", + "resource:///modules/pagedata/PageDataService.jsm": "browser/components/pagedata/PageDataService.jsm", + "resource:///modules/pagedata/SchemaOrgPageData.jsm": "browser/components/pagedata/SchemaOrgPageData.jsm", + "resource:///modules/pagedata/TwitterPageData.jsm": "browser/components/pagedata/TwitterPageData.jsm", + "resource:///modules/policies/BookmarksPolicies.jsm": "browser/components/enterprisepolicies/helpers/BookmarksPolicies.jsm", + "resource:///modules/policies/Policies.jsm": "browser/components/enterprisepolicies/Policies.jsm", + "resource:///modules/policies/ProxyPolicies.jsm": "browser/components/enterprisepolicies/helpers/ProxyPolicies.jsm", + "resource:///modules/policies/WebsiteFilter.jsm": "browser/components/enterprisepolicies/helpers/WebsiteFilter.jsm", + "resource:///modules/policies/schema.jsm": "browser/components/enterprisepolicies/schemas/schema.jsm", + "resource:///modules/sessionstore/ContentRestore.jsm": "browser/components/sessionstore/ContentRestore.jsm", + "resource:///modules/sessionstore/ContentSessionStore.jsm": "browser/components/sessionstore/ContentSessionStore.jsm", + "resource:///modules/sessionstore/GlobalState.jsm": "browser/components/sessionstore/GlobalState.jsm", + "resource:///modules/sessionstore/RecentlyClosedTabsAndWindowsMenuUtils.jsm": "browser/components/sessionstore/RecentlyClosedTabsAndWindowsMenuUtils.jsm", + "resource:///modules/sessionstore/RunState.jsm": "browser/components/sessionstore/RunState.jsm", + "resource:///modules/sessionstore/SessionCookies.jsm": "browser/components/sessionstore/SessionCookies.jsm", + "resource:///modules/sessionstore/SessionFile.jsm": "browser/components/sessionstore/SessionFile.jsm", + "resource:///modules/sessionstore/SessionMigration.jsm": "browser/components/sessionstore/SessionMigration.jsm", + "resource:///modules/sessionstore/SessionSaver.jsm": "browser/components/sessionstore/SessionSaver.jsm", + "resource:///modules/sessionstore/SessionStartup.jsm": "browser/components/sessionstore/SessionStartup.jsm", + "resource:///modules/sessionstore/SessionStore.jsm": "browser/components/sessionstore/SessionStore.jsm", + "resource:///modules/sessionstore/SessionWriter.jsm": "browser/components/sessionstore/SessionWriter.jsm", + "resource:///modules/sessionstore/StartupPerformance.jsm": "browser/components/sessionstore/StartupPerformance.jsm", + "resource:///modules/sessionstore/TabAttributes.jsm": "browser/components/sessionstore/TabAttributes.jsm", + "resource:///modules/sessionstore/TabState.jsm": "browser/components/sessionstore/TabState.jsm", + "resource:///modules/sessionstore/TabStateCache.jsm": "browser/components/sessionstore/TabStateCache.jsm", + "resource:///modules/sessionstore/TabStateFlusher.jsm": "browser/components/sessionstore/TabStateFlusher.jsm", + "resource:///modules/syncedtabs/EventEmitter.jsm": "browser/components/syncedtabs/EventEmitter.jsm", + "resource:///modules/syncedtabs/SyncedTabsDeckComponent.js": "browser/components/syncedtabs/SyncedTabsDeckComponent.js", + "resource:///modules/syncedtabs/SyncedTabsDeckStore.js": "browser/components/syncedtabs/SyncedTabsDeckStore.js", + "resource:///modules/syncedtabs/SyncedTabsDeckView.js": "browser/components/syncedtabs/SyncedTabsDeckView.js", + "resource:///modules/syncedtabs/SyncedTabsListStore.js": "browser/components/syncedtabs/SyncedTabsListStore.js", + "resource:///modules/syncedtabs/TabListComponent.js": "browser/components/syncedtabs/TabListComponent.js", + "resource:///modules/syncedtabs/TabListView.js": "browser/components/syncedtabs/TabListView.js", + "resource:///modules/syncedtabs/util.js": "browser/components/syncedtabs/util.js", + "resource:///modules/webrtcUI.jsm": "browser/modules/webrtcUI.jsm", + "resource://activity-stream/aboutwelcome/lib/AboutWelcomeDefaults.jsm": "browser/components/newtab/aboutwelcome/lib/AboutWelcomeDefaults.jsm", + "resource://activity-stream/aboutwelcome/lib/AboutWelcomeTelemetry.jsm": "browser/components/newtab/aboutwelcome/lib/AboutWelcomeTelemetry.jsm", + "resource://activity-stream/common/Actions.jsm": "browser/components/newtab/common/Actions.jsm", + "resource://activity-stream/common/ActorConstants.jsm": "browser/components/newtab/common/ActorConstants.jsm", + "resource://activity-stream/common/Dedupe.jsm": "browser/components/newtab/common/Dedupe.jsm", + "resource://activity-stream/common/Reducers.jsm": "browser/components/newtab/common/Reducers.jsm", + "resource://activity-stream/lib/ASRouterNewTabHook.jsm": "browser/components/newtab/lib/ASRouterNewTabHook.jsm", + "resource://activity-stream/lib/AboutPreferences.jsm": "browser/components/newtab/lib/AboutPreferences.jsm", + "resource://activity-stream/lib/ActivityStream.jsm": "browser/components/newtab/lib/ActivityStream.jsm", + "resource://activity-stream/lib/ActivityStreamMessageChannel.jsm": "browser/components/newtab/lib/ActivityStreamMessageChannel.jsm", + "resource://activity-stream/lib/ActivityStreamStorage.jsm": "browser/components/newtab/lib/ActivityStreamStorage.jsm", + "resource://activity-stream/lib/CFRMessageProvider.jsm": "browser/components/newtab/lib/CFRMessageProvider.jsm", + "resource://activity-stream/lib/DefaultSites.jsm": "browser/components/newtab/lib/DefaultSites.jsm", + "resource://activity-stream/lib/DiscoveryStreamFeed.jsm": "browser/components/newtab/lib/DiscoveryStreamFeed.jsm", + "resource://activity-stream/lib/DownloadsManager.jsm": "browser/components/newtab/lib/DownloadsManager.jsm", + "resource://activity-stream/lib/FaviconFeed.jsm": "browser/components/newtab/lib/FaviconFeed.jsm", + "resource://activity-stream/lib/FeatureCalloutMessages.jsm": "browser/components/newtab/lib/FeatureCalloutMessages.jsm", + "resource://activity-stream/lib/FilterAdult.jsm": "browser/components/newtab/lib/FilterAdult.jsm", + "resource://activity-stream/lib/LinksCache.jsm": "browser/components/newtab/lib/LinksCache.jsm", + "resource://activity-stream/lib/NewTabInit.jsm": "browser/components/newtab/lib/NewTabInit.jsm", + "resource://activity-stream/lib/PanelTestProvider.jsm": "browser/components/newtab/lib/PanelTestProvider.jsm", + "resource://activity-stream/lib/PersistentCache.jsm": "browser/components/newtab/lib/PersistentCache.jsm", + "resource://activity-stream/lib/PersonalityProvider/NaiveBayesTextTagger.jsm": "browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm", + "resource://activity-stream/lib/PersonalityProvider/NmfTextTagger.jsm": "browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm", + "resource://activity-stream/lib/PersonalityProvider/PersonalityProvider.jsm": "browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm", + "resource://activity-stream/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm": "browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm", + "resource://activity-stream/lib/PersonalityProvider/RecipeExecutor.jsm": "browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm", + "resource://activity-stream/lib/PersonalityProvider/Tokenize.jsm": "browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm", + "resource://activity-stream/lib/PlacesFeed.jsm": "browser/components/newtab/lib/PlacesFeed.jsm", + "resource://activity-stream/lib/PrefsFeed.jsm": "browser/components/newtab/lib/PrefsFeed.jsm", + "resource://activity-stream/lib/RecommendationProvider.jsm": "browser/components/newtab/lib/RecommendationProvider.jsm", + "resource://activity-stream/lib/RemoteL10n.jsm": "browser/components/newtab/lib/RemoteL10n.jsm", + "resource://activity-stream/lib/Screenshots.jsm": "browser/components/newtab/lib/Screenshots.jsm", + "resource://activity-stream/lib/SearchShortcuts.jsm": "browser/components/newtab/lib/SearchShortcuts.jsm", + "resource://activity-stream/lib/SectionsManager.jsm": "browser/components/newtab/lib/SectionsManager.jsm", + "resource://activity-stream/lib/ShortURL.jsm": "browser/components/newtab/lib/ShortURL.jsm", + "resource://activity-stream/lib/SiteClassifier.jsm": "browser/components/newtab/lib/SiteClassifier.jsm", + "resource://activity-stream/lib/Spotlight.jsm": "browser/components/newtab/lib/Spotlight.jsm", + "resource://activity-stream/lib/Store.jsm": "browser/components/newtab/lib/Store.jsm", + "resource://activity-stream/lib/SystemTickFeed.jsm": "browser/components/newtab/lib/SystemTickFeed.jsm", + "resource://activity-stream/lib/TippyTopProvider.jsm": "browser/components/newtab/lib/TippyTopProvider.jsm", + "resource://activity-stream/lib/ToastNotification.jsm": "browser/components/newtab/lib/ToastNotification.jsm", + "resource://activity-stream/lib/TopSitesFeed.jsm": "browser/components/newtab/lib/TopSitesFeed.jsm", + "resource://activity-stream/lib/TopStoriesFeed.jsm": "browser/components/newtab/lib/TopStoriesFeed.jsm", + "resource://activity-stream/lib/UTEventReporting.jsm": "browser/components/newtab/lib/UTEventReporting.jsm", + "resource://android/assets/web_extensions/test-support/TestSupportChild.jsm": "mobile/android/geckoview/src/androidTest/assets/web_extensions/test-support/TestSupportChild.jsm", + "resource://android/assets/web_extensions/test-support/TestSupportProcessChild.jsm": "mobile/android/geckoview/src/androidTest/assets/web_extensions/test-support/TestSupportProcessChild.jsm", + "resource://app/modules/SnapshotSelector.jsm": "browser/components/places/SnapshotSelector.jsm", + "resource://autofill/AutofillTelemetry.jsm": "toolkit/components/formautofill/AutofillTelemetry.jsm", + "resource://autofill/CreditCardRuleset.jsm": "toolkit/components/formautofill/CreditCardRuleset.jsm", + "resource://autofill/FormAutofill.jsm": "toolkit/components/formautofill/FormAutofill.jsm", + "resource://autofill/FormAutofillChild.jsm": "toolkit/components/formautofill/FormAutofillChild.jsm", + "resource://autofill/FormAutofillContent.jsm": "toolkit/components/formautofill/FormAutofillContent.jsm", + "resource://autofill/FormAutofillHandler.jsm": "toolkit/components/formautofill/FormAutofillHandler.jsm", + "resource://autofill/FormAutofillHeuristics.jsm": "toolkit/components/formautofill/FormAutofillHeuristics.jsm", + "resource://autofill/FormAutofillNameUtils.jsm": "toolkit/components/formautofill/FormAutofillNameUtils.jsm", + "resource://autofill/FormAutofillParent.jsm": "toolkit/components/formautofill/FormAutofillParent.jsm", + "resource://autofill/FormAutofillPreferences.jsm": "toolkit/components/formautofill/FormAutofillPreferences.jsm", + "resource://autofill/FormAutofillStorageBase.jsm": "toolkit/components/formautofill/FormAutofillStorageBase.jsm", + "resource://autofill/FormAutofillSync.jsm": "toolkit/components/formautofill/FormAutofillSync.jsm", + "resource://autofill/Autofilltelemetry.jsm": "toolkit/components/formautofill/Autofilltelemetry.jsm", + "resource://autofill/FormAutofillUtils.jsm": "toolkit/components/formautofill/FormAutofillUtils.jsm", + "resource://autofill/ProfileAutoCompleteResult.jsm": "toolkit/components/formautofill/ProfileAutoCompleteResult.jsm", + "resource://autofill/phonenumberutils/PhoneNumber.jsm": "toolkit/components/formautofill/phonenumberutils/PhoneNumber.jsm", + "resource://autofill/phonenumberutils/PhoneNumberMetaData.jsm": "toolkit/components/formautofill/phonenumberutils/PhoneNumberMetaData.jsm", + "resource://autofill/phonenumberutils/PhoneNumberNormalizer.jsm": "toolkit/components/formautofill/phonenumberutils/PhoneNumberNormalizer.jsm", + "resource://damp-test/content/actors/DampLoadChild.jsm": "testing/talos/talos/tests/devtools/addon/content/actors/DampLoadChild.jsm", + "resource://damp-test/content/actors/DampLoadParent.jsm": "testing/talos/talos/tests/devtools/addon/content/actors/DampLoadParent.jsm", + "resource://devtools/client/framework/browser-toolbox/Launcher.jsm": "devtools/client/framework/browser-toolbox/Launcher.jsm", + "resource://devtools/client/jsonview/Converter.jsm": "devtools/client/jsonview/Converter.jsm", + "resource://devtools/client/jsonview/Sniffer.jsm": "devtools/client/jsonview/Sniffer.jsm", + "resource://devtools/client/performance-new/shared/background.jsm.js": "devtools/client/performance-new/shared/background.jsm.js", + "resource://devtools/client/performance-new/popup/menu-button.jsm.js": "devtools/client/performance-new/popup/menu-button.jsm.js", + "resource://devtools/client/performance-new/popup/logic.jsm.js": "devtools/client/performance-new/popup/logic.jsm.js", + "resource://devtools/client/performance-new/shared/symbolication.jsm.js": "devtools/client/performance-new/shared/symbolication.jsm.js", + "resource://devtools/client/performance-new/shared/typescript-lazy-load.jsm.js": "devtools/client/performance-new/shared/typescript-lazy-load.jsm.js", + "resource://devtools/client/storage/VariablesView.jsm": "devtools/client/storage/VariablesView.jsm", + "resource://devtools/client/styleeditor/StyleEditorUI.jsm": "devtools/client/styleeditor/StyleEditorUI.jsm", + "resource://devtools/client/styleeditor/StyleEditorUtil.jsm": "devtools/client/styleeditor/StyleEditorUtil.jsm", + "resource://devtools/client/styleeditor/StyleSheetEditor.jsm": "devtools/client/styleeditor/StyleSheetEditor.jsm", + "resource://devtools/server/actors/targets/target-actor-registry.jsm": "devtools/server/actors/targets/target-actor-registry.jsm", + "resource://devtools/server/actors/watcher/SessionDataHelpers.jsm": "devtools/server/actors/watcher/SessionDataHelpers.jsm", + "resource://devtools/server/actors/watcher/WatcherRegistry.jsm": "devtools/server/actors/watcher/WatcherRegistry.jsm", + "resource://devtools/server/actors/watcher/browsing-context-helpers.jsm": "devtools/server/actors/watcher/browsing-context-helpers.jsm", + "resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm": "devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm", + "resource://devtools/server/connectors/js-window-actor/DevToolsFrameParent.jsm": "devtools/server/connectors/js-window-actor/DevToolsFrameParent.jsm", + "resource://devtools/server/connectors/js-window-actor/DevToolsWorkerChild.jsm": "devtools/server/connectors/js-window-actor/DevToolsWorkerChild.jsm", + "resource://devtools/server/connectors/js-window-actor/DevToolsWorkerParent.jsm": "devtools/server/connectors/js-window-actor/DevToolsWorkerParent.jsm", + "resource://devtools/server/connectors/js-window-actor/WindowGlobalLogger.jsm": "devtools/server/connectors/js-window-actor/WindowGlobalLogger.jsm", + "resource://devtools/server/startup/content-process.jsm": "devtools/server/startup/content-process.jsm", + "resource://devtools/shared/loader/Loader.jsm": "devtools/shared/loader/Loader.jsm", + "resource://devtools/shared/loader/base-loader.js": "devtools/shared/loader/base-loader.js", + "resource://devtools/shared/loader/browser-loader.js": "devtools/shared/loader/browser-loader.js", + "resource://devtools/shared/loader/loader-plugin-raw.jsm": "devtools/shared/loader/loader-plugin-raw.jsm", + "resource://devtools/shared/loader/worker-loader.js": "devtools/shared/loader/worker-loader.js", + "resource://devtools/shared/security/DevToolsSocketStatus.jsm": "devtools/shared/security/DevToolsSocketStatus.jsm", + "resource://devtools/shared/test-helpers/tracked-objects.jsm": "devtools/shared/test-helpers/tracked-objects.jsm", + "resource://devtools/shared/validate-breakpoint.jsm": "devtools/shared/validate-breakpoint.jsm", + "resource://devtools/shared/worker/worker.js": "devtools/shared/worker/worker.js", + "resource://featuregates/FeatureGate.jsm": "toolkit/components/featuregates/FeatureGate.jsm", + "resource://featuregates/FeatureGateImplementation.jsm": "toolkit/components/featuregates/FeatureGateImplementation.jsm", + "resource://gre/actors/AboutHttpsOnlyErrorChild.jsm": "toolkit/actors/AboutHttpsOnlyErrorChild.jsm", + "resource://gre/actors/AboutHttpsOnlyErrorParent.jsm": "toolkit/actors/AboutHttpsOnlyErrorParent.jsm", + "resource://gre/actors/AudioPlaybackChild.jsm": "toolkit/actors/AudioPlaybackChild.jsm", + "resource://gre/actors/AudioPlaybackParent.jsm": "toolkit/actors/AudioPlaybackParent.jsm", + "resource://gre/actors/AutoCompleteChild.jsm": "toolkit/actors/AutoCompleteChild.jsm", + "resource://gre/actors/AutoCompleteParent.jsm": "toolkit/actors/AutoCompleteParent.jsm", + "resource://gre/actors/AutoScrollChild.jsm": "toolkit/actors/AutoScrollChild.jsm", + "resource://gre/actors/AutoScrollParent.jsm": "toolkit/actors/AutoScrollParent.jsm", + "resource://gre/actors/AutoplayChild.jsm": "toolkit/actors/AutoplayChild.jsm", + "resource://gre/actors/AutoplayParent.jsm": "toolkit/actors/AutoplayParent.jsm", + "resource://gre/actors/BackgroundThumbnailsChild.jsm": "toolkit/actors/BackgroundThumbnailsChild.jsm", + "resource://gre/actors/BrowserElementChild.jsm": "toolkit/actors/BrowserElementChild.jsm", + "resource://gre/actors/BrowserElementParent.jsm": "toolkit/actors/BrowserElementParent.jsm", + "resource://gre/actors/ClipboardReadPasteChild.jsm": "toolkit/actors/ClipboardReadPasteChild.jsm", + "resource://gre/actors/ClipboardReadPasteParent.jsm": "toolkit/actors/ClipboardReadPasteParent.jsm", + "resource://gre/actors/ContentMetaChild.jsm": "toolkit/actors/ContentMetaChild.jsm", + "resource://gre/actors/ContentMetaParent.jsm": "toolkit/actors/ContentMetaParent.jsm", + "resource://gre/actors/ControllersChild.jsm": "toolkit/actors/ControllersChild.jsm", + "resource://gre/actors/ControllersParent.jsm": "toolkit/actors/ControllersParent.jsm", + "resource://gre/actors/CookieBannerChild.jsm.jsm": "toolkit/components/cookiebanners/CookieBannerChild.jsm.jsm", + "resource://gre/actors/CookieBannerParent.jsm": "toolkit/components/cookiebanners/CookieBannerParent.jsm", + "resource://gre/actors/DateTimePickerChild.jsm": "toolkit/actors/DateTimePickerChild.jsm", + "resource://gre/actors/DateTimePickerParent.jsm": "toolkit/actors/DateTimePickerParent.jsm", + "resource://gre/actors/ExtFindChild.jsm": "toolkit/actors/ExtFindChild.jsm", + "resource://gre/actors/FindBarChild.jsm": "toolkit/actors/FindBarChild.jsm", + "resource://gre/actors/FindBarParent.jsm": "toolkit/actors/FindBarParent.jsm", + "resource://gre/actors/FinderChild.jsm": "toolkit/actors/FinderChild.jsm", + "resource://gre/actors/FormHistoryChild.jsm": "toolkit/components/satchel/FormHistoryChild.jsm", + "resource://gre/actors/FormHistoryParent.jsm": "toolkit/components/satchel/FormHistoryParent.jsm", + "resource://gre/actors/InlineSpellCheckerChild.jsm": "toolkit/actors/InlineSpellCheckerChild.jsm", + "resource://gre/actors/InlineSpellCheckerParent.jsm": "toolkit/actors/InlineSpellCheckerParent.jsm", + "resource://gre/actors/KeyPressEventModelCheckerChild.jsm": "toolkit/actors/KeyPressEventModelCheckerChild.jsm", + "resource://gre/actors/LayoutDebugChild.jsm": "layout/tools/layout-debug/LayoutDebugChild.jsm", + "resource://gre/actors/NetErrorChild.jsm": "toolkit/actors/NetErrorChild.jsm", + "resource://gre/actors/NetErrorParent.jsm": "toolkit/actors/NetErrorParent.jsm", + "resource://gre/actors/PictureInPictureChild.jsm": "toolkit/actors/PictureInPictureChild.jsm", + "resource://gre/actors/PopupBlockingChild.jsm": "toolkit/actors/PopupBlockingChild.jsm", + "resource://gre/actors/PopupBlockingParent.jsm": "toolkit/actors/PopupBlockingParent.jsm", + "resource://gre/actors/PrintingChild.jsm": "toolkit/actors/PrintingChild.jsm", + "resource://gre/actors/PrintingParent.jsm": "toolkit/actors/PrintingParent.jsm", + "resource://gre/actors/PrintingSelectionChild.jsm": "toolkit/actors/PrintingSelectionChild.jsm", + "resource://gre/actors/PurgeSessionHistoryChild.jsm": "toolkit/actors/PurgeSessionHistoryChild.jsm", + "resource://gre/actors/RemotePageChild.jsm": "toolkit/actors/RemotePageChild.jsm", + "resource://gre/actors/SelectChild.jsm": "toolkit/actors/SelectChild.jsm", + "resource://gre/actors/SelectParent.jsm": "toolkit/actors/SelectParent.jsm", + "resource://gre/actors/ThumbnailsChild.jsm": "toolkit/actors/ThumbnailsChild.jsm", + "resource://gre/actors/UAWidgetsChild.jsm": "toolkit/actors/UAWidgetsChild.jsm", + "resource://gre/actors/UnselectedTabHoverChild.jsm": "toolkit/actors/UnselectedTabHoverChild.jsm", + "resource://gre/actors/UnselectedTabHoverParent.jsm": "toolkit/actors/UnselectedTabHoverParent.jsm", + "resource://gre/actors/ViewSourceChild.jsm": "toolkit/actors/ViewSourceChild.jsm", + "resource://gre/actors/ViewSourcePageChild.jsm": "toolkit/actors/ViewSourcePageChild.jsm", + "resource://gre/actors/ViewSourcePageParent.jsm": "toolkit/actors/ViewSourcePageParent.jsm", + "resource://gre/actors/WebChannelChild.jsm": "toolkit/actors/WebChannelChild.jsm", + "resource://gre/actors/WebChannelParent.jsm": "toolkit/actors/WebChannelParent.jsm", + "resource://gre/modules/AboutCertViewerChild.jsm": "toolkit/components/certviewer/AboutCertViewerChild.jsm", + "resource://gre/modules/AboutCertViewerParent.jsm": "toolkit/components/certviewer/AboutCertViewerParent.jsm", + "resource://gre/modules/AboutPagesUtils.jsm": "toolkit/modules/AboutPagesUtils.jsm", + "resource://gre/modules/AboutReader.jsm": "toolkit/components/reader/AboutReader.jsm", + "resource://gre/modules/AbuseReporter.jsm": "toolkit/mozapps/extensions/AbuseReporter.jsm", + "resource://gre/modules/ActorManagerParent.jsm": "toolkit/modules/ActorManagerParent.jsm", + "resource://gre/modules/AddonManager.jsm": "toolkit/mozapps/extensions/AddonManager.jsm", + "resource://gre/modules/AddonSearchEngine.jsm": "toolkit/components/search/AddonSearchEngine.jsm", + "resource://gre/modules/AndroidLog.jsm": "mobile/android/modules/geckoview/AndroidLog.jsm", + "resource://gre/modules/AppConstants.jsm": "toolkit/modules/AppConstants.jsm", + "resource://gre/modules/AppMenuNotifications.jsm": "toolkit/modules/AppMenuNotifications.jsm", + "resource://gre/modules/AsanReporter.jsm": "toolkit/modules/AsanReporter.jsm", + "resource://gre/modules/AsyncPrefs.jsm": "toolkit/modules/AsyncPrefs.jsm", + "resource://gre/modules/AsyncShutdown.jsm": "toolkit/components/asyncshutdown/AsyncShutdown.jsm", + "resource://gre/modules/AutoCompleteSimpleSearch.jsm": "toolkit/components/autocomplete/AutoCompleteSimpleSearch.jsm", + "resource://gre/modules/BHRTelemetryService.jsm": "toolkit/components/backgroundhangmonitor/BHRTelemetryService.jsm", + "resource://gre/modules/BackgroundPageThumbs.jsm": "toolkit/components/thumbnails/BackgroundPageThumbs.jsm", + "resource://gre/modules/BackgroundTasksManager.jsm": "toolkit/components/backgroundtasks/BackgroundTasksManager.jsm", + "resource://gre/modules/BackgroundTasksUtils.jsm": "toolkit/components/backgroundtasks/BackgroundTasksUtils.jsm", + "resource://gre/modules/BackgroundUpdate.jsm": "toolkit/mozapps/update/BackgroundUpdate.jsm", + "resource://gre/modules/BinarySearch.jsm": "toolkit/modules/BinarySearch.jsm", + "resource://gre/modules/Bits.jsm": "toolkit/components/bitsdownload/Bits.jsm", + "resource://gre/modules/Blocklist.jsm": "toolkit/mozapps/extensions/Blocklist.jsm", + "resource://gre/modules/BookmarkHTMLUtils.jsm": "toolkit/components/places/BookmarkHTMLUtils.jsm", + "resource://gre/modules/BookmarkJSONUtils.jsm": "toolkit/components/places/BookmarkJSONUtils.jsm", + "resource://gre/modules/Bookmarks.jsm": "toolkit/components/places/Bookmarks.jsm", + "resource://gre/modules/BrowserElementParent.jsm": "dom/browser-element/BrowserElementParent.jsm", + "resource://gre/modules/BrowserElementPromptService.jsm": "dom/browser-element/BrowserElementPromptService.jsm", + "resource://gre/modules/BrowserTelemetryUtils.jsm": "toolkit/modules/BrowserTelemetryUtils.jsm", + "resource://gre/modules/BrowserUtils.jsm": "toolkit/modules/BrowserUtils.jsm", + "resource://gre/modules/CSV.js": "toolkit/components/passwordmgr/CSV.js", + "resource://gre/modules/CanonicalJSON.jsm": "toolkit/modules/CanonicalJSON.jsm", + "resource://gre/modules/CaptiveDetect.jsm": "toolkit/components/captivedetect/CaptiveDetect.jsm", + "resource://gre/modules/CertUtils.jsm": "toolkit/modules/CertUtils.jsm", + "resource://gre/modules/ChildCrashHandler.jsm": "mobile/android/modules/geckoview/ChildCrashHandler.jsm", + "resource://gre/modules/ClearDataService.jsm": "toolkit/components/cleardata/ClearDataService.jsm", + "resource://gre/modules/ClientID.jsm": "toolkit/components/telemetry/app/ClientID.jsm", + "resource://gre/modules/Color.jsm": "toolkit/modules/Color.jsm", + "resource://gre/modules/ColorPickerDelegate.jsm": "mobile/android/components/geckoview/ColorPickerDelegate.jsm", + "resource://gre/modules/CommonDialog.jsm": "toolkit/components/prompts/src/CommonDialog.jsm", + "resource://gre/modules/ComponentUtils.jsm": "js/xpconnect/loader/ComponentUtils.jsm", + "resource://gre/modules/ConduitsChild.jsm": "toolkit/components/extensions/ConduitsChild.jsm", + "resource://gre/modules/ConduitsParent.jsm": "toolkit/components/extensions/ConduitsParent.jsm", + "resource://gre/modules/Console.jsm": "toolkit/modules/Console.jsm", + "resource://gre/modules/ConsoleAPIStorage.jsm": "dom/console/ConsoleAPIStorage.jsm", + "resource://gre/modules/ContentAreaDropListener.jsm": "dom/base/ContentAreaDropListener.jsm", + "resource://gre/modules/ContentBlockingAllowList.jsm": "toolkit/components/antitracking/ContentBlockingAllowList.jsm", + "resource://gre/modules/ContentDOMReference.jsm": "toolkit/modules/ContentDOMReference.jsm", + "resource://gre/modules/ContentDispatchChooser.jsm": "toolkit/mozapps/handling/ContentDispatchChooser.jsm", + "resource://gre/modules/ContentPrefService2.jsm": "toolkit/components/contentprefs/ContentPrefService2.jsm", + "resource://gre/modules/ContentPrefServiceChild.jsm": "toolkit/components/contentprefs/ContentPrefServiceChild.jsm", + "resource://gre/modules/ContentPrefServiceParent.jsm": "toolkit/components/contentprefs/ContentPrefServiceParent.jsm", + "resource://gre/modules/ContentPrefStore.jsm": "toolkit/components/contentprefs/ContentPrefStore.jsm", + "resource://gre/modules/ContentPrefUtils.jsm": "toolkit/components/contentprefs/ContentPrefUtils.jsm", + "resource://gre/modules/ContextualIdentityService.jsm": "toolkit/components/contextualidentity/ContextualIdentityService.jsm", + "resource://gre/modules/Corroborate.jsm": "toolkit/components/corroborator/Corroborate.jsm", + "resource://gre/modules/CoveragePing.jsm": "toolkit/components/telemetry/pings/CoveragePing.jsm", + "resource://gre/modules/CrashManager.jsm": "toolkit/components/crashes/CrashManager.in.jsm", + "resource://gre/modules/CrashMonitor.jsm": "toolkit/components/crashmonitor/CrashMonitor.jsm", + "resource://gre/modules/CrashReports.jsm": "toolkit/crashreporter/CrashReports.jsm", + "resource://gre/modules/CrashService.jsm": "toolkit/components/crashes/CrashService.jsm", + "resource://gre/modules/CrashSubmit.jsm": "toolkit/crashreporter/CrashSubmit.jsm", + "resource://gre/modules/Credentials.jsm": "services/fxaccounts/Credentials.jsm", + "resource://gre/modules/CreditCard.jsm": "toolkit/modules/CreditCard.jsm", + "resource://gre/modules/CustomElementsListener.jsm": "toolkit/components/processsingleton/CustomElementsListener.jsm", + "resource://gre/modules/DOMRequestHelper.jsm": "dom/base/DOMRequestHelper.jsm", + "resource://gre/modules/DateTimePickerPanel.jsm": "toolkit/modules/DateTimePickerPanel.jsm", + "resource://gre/modules/DefaultCLH.jsm": "toolkit/components/DefaultCLH.jsm", + "resource://gre/modules/DeferredTask.jsm": "toolkit/modules/DeferredTask.jsm", + "resource://gre/modules/DelayedInit.jsm": "mobile/android/modules/geckoview/DelayedInit.jsm", + "resource://gre/modules/Deprecated.jsm": "toolkit/modules/Deprecated.jsm", + "resource://gre/modules/DownloadCore.jsm": "toolkit/components/downloads/DownloadCore.jsm", + "resource://gre/modules/DownloadHistory.jsm": "toolkit/components/downloads/DownloadHistory.jsm", + "resource://gre/modules/DownloadIntegration.jsm": "toolkit/components/downloads/DownloadIntegration.jsm", + "resource://gre/modules/DownloadLastDir.jsm": "toolkit/mozapps/downloads/DownloadLastDir.jsm", + "resource://gre/modules/DownloadLegacy.jsm": "toolkit/components/downloads/DownloadLegacy.jsm", + "resource://gre/modules/DownloadList.jsm": "toolkit/components/downloads/DownloadList.jsm", + "resource://gre/modules/DownloadPaths.jsm": "toolkit/components/downloads/DownloadPaths.jsm", + "resource://gre/modules/DownloadStore.jsm": "toolkit/components/downloads/DownloadStore.jsm", + "resource://gre/modules/DownloadUIHelper.jsm": "toolkit/components/downloads/DownloadUIHelper.jsm", + "resource://gre/modules/DownloadUtils.jsm": "toolkit/mozapps/downloads/DownloadUtils.jsm", + "resource://gre/modules/Downloads.jsm": "toolkit/components/downloads/Downloads.jsm", + "resource://gre/modules/E10SUtils.jsm": "toolkit/modules/E10SUtils.jsm", + "resource://gre/modules/EnterprisePolicies.jsm": "toolkit/components/enterprisepolicies/EnterprisePolicies.jsm", + "resource://gre/modules/EnterprisePoliciesContent.jsm": "toolkit/components/enterprisepolicies/EnterprisePoliciesContent.jsm", + "resource://gre/modules/EnterprisePoliciesParent.jsm": "toolkit/components/enterprisepolicies/EnterprisePoliciesParent.jsm", + "resource://gre/modules/EventEmitter.jsm": "toolkit/modules/EventEmitter.jsm", + "resource://gre/modules/EventPing.jsm": "toolkit/components/telemetry/pings/EventPing.jsm", + "resource://gre/modules/ExtHandlerService.jsm": "uriloader/exthandler/ExtHandlerService.jsm", + "resource://gre/modules/Extension.jsm": "toolkit/components/extensions/Extension.jsm", + "resource://gre/modules/ExtensionActions.jsm": "toolkit/components/extensions/ExtensionActions.jsm", + "resource://gre/modules/ExtensionActivityLog.jsm": "toolkit/components/extensions/ExtensionActivityLog.jsm", + "resource://gre/modules/ExtensionChild.jsm": "toolkit/components/extensions/ExtensionChild.jsm", + "resource://gre/modules/ExtensionChildDevToolsUtils.jsm": "toolkit/components/extensions/ExtensionChildDevToolsUtils.jsm", + "resource://gre/modules/ExtensionCommon.jsm": "toolkit/components/extensions/ExtensionCommon.jsm", + "resource://gre/modules/ExtensionContent.jsm": "toolkit/components/extensions/ExtensionContent.jsm", + "resource://gre/modules/ExtensionPageChild.jsm": "toolkit/components/extensions/ExtensionPageChild.jsm", + "resource://gre/modules/ExtensionParent.jsm": "toolkit/components/extensions/ExtensionParent.jsm", + "resource://gre/modules/ExtensionPermissions.jsm": "toolkit/components/extensions/ExtensionPermissions.jsm", + "resource://gre/modules/ExtensionPreferencesManager.jsm": "toolkit/components/extensions/ExtensionPreferencesManager.jsm", + "resource://gre/modules/ExtensionProcessScript.jsm": "toolkit/components/extensions/ExtensionProcessScript.jsm", + "resource://gre/modules/ExtensionScriptingStore.jsm": "toolkit/components/extensions/ExtensionScriptingStore.jsm", + "resource://gre/modules/ExtensionSearchHandler.jsm": "toolkit/components/places/ExtensionSearchHandler.jsm", + "resource://gre/modules/ExtensionSettingsStore.jsm": "toolkit/components/extensions/ExtensionSettingsStore.jsm", + "resource://gre/modules/ExtensionShortcuts.jsm": "toolkit/components/extensions/ExtensionShortcuts.jsm", + "resource://gre/modules/ExtensionStorage.jsm": "toolkit/components/extensions/ExtensionStorage.jsm", + "resource://gre/modules/ExtensionStorageComponents.jsm": "toolkit/components/extensions/storage/ExtensionStorageComponents.jsm", + "resource://gre/modules/ExtensionStorageIDB.jsm": "toolkit/components/extensions/ExtensionStorageIDB.jsm", + "resource://gre/modules/ExtensionStorageSync.jsm": "toolkit/components/extensions/ExtensionStorageSync.jsm", + "resource://gre/modules/ExtensionStorageSyncKinto.jsm": "toolkit/components/extensions/ExtensionStorageSyncKinto.jsm", + "resource://gre/modules/ExtensionTelemetry.jsm": "toolkit/components/extensions/ExtensionTelemetry.jsm", + "resource://gre/modules/ExtensionUtils.jsm": "toolkit/components/extensions/ExtensionUtils.jsm", + "resource://gre/modules/ExtensionWorkerChild.jsm": "toolkit/components/extensions/ExtensionWorkerChild.jsm", + "resource://gre/modules/FilePickerDelegate.jsm": "mobile/android/components/geckoview/FilePickerDelegate.jsm", + "resource://gre/modules/FileUtils.jsm": "toolkit/modules/FileUtils.jsm", + "resource://gre/modules/FindBarContent.jsm": "toolkit/modules/FindBarContent.jsm", + "resource://gre/modules/FindContent.jsm": "toolkit/components/extensions/FindContent.jsm", + "resource://gre/modules/Finder.jsm": "toolkit/modules/Finder.jsm", + "resource://gre/modules/FinderHighlighter.jsm": "toolkit/modules/FinderHighlighter.jsm", + "resource://gre/modules/FinderIterator.jsm": "toolkit/modules/FinderIterator.jsm", + "resource://gre/modules/FinderParent.jsm": "toolkit/modules/FinderParent.jsm", + "resource://gre/modules/FirefoxRelay.jsm": "toolkit/components/passwordmgr/FirefoxRelay.jsm", + "resource://gre/modules/FirstStartup.jsm": "toolkit/modules/FirstStartup.jsm", + "resource://gre/modules/ForgetAboutSite.jsm": "toolkit/components/forgetaboutsite/ForgetAboutSite.jsm", + "resource://gre/modules/FormAutoComplete.jsm": "toolkit/components/satchel/FormAutoComplete.jsm", + "resource://gre/modules/FormHistory.jsm": "toolkit/components/satchel/FormHistory.jsm", + "resource://gre/modules/FormHistoryStartup.jsm": "toolkit/components/satchel/FormHistoryStartup.jsm", + "resource://gre/modules/FormLikeFactory.jsm": "toolkit/modules/FormLikeFactory.jsm", + "resource://gre/modules/FxAccounts.jsm": "services/fxaccounts/FxAccounts.jsm", + "resource://gre/modules/FxAccountsClient.jsm": "services/fxaccounts/FxAccountsClient.jsm", + "resource://gre/modules/FxAccountsCommands.js": "services/fxaccounts/FxAccountsCommands.js", + "resource://gre/modules/FxAccountsCommon.js": "services/fxaccounts/FxAccountsCommon.js", + "resource://gre/modules/FxAccountsConfig.jsm": "services/fxaccounts/FxAccountsConfig.jsm", + "resource://gre/modules/FxAccountsDevice.jsm": "services/fxaccounts/FxAccountsDevice.jsm", + "resource://gre/modules/FxAccountsKeys.jsm": "services/fxaccounts/FxAccountsKeys.jsm", + "resource://gre/modules/FxAccountsPairing.jsm": "services/fxaccounts/FxAccountsPairing.jsm", + "resource://gre/modules/FxAccountsPairingChannel.js": "services/fxaccounts/FxAccountsPairingChannel.js", + "resource://gre/modules/FxAccountsProfile.jsm": "services/fxaccounts/FxAccountsProfile.jsm", + "resource://gre/modules/FxAccountsProfileClient.jsm": "services/fxaccounts/FxAccountsProfileClient.jsm", + "resource://gre/modules/FxAccountsPush.jsm": "services/fxaccounts/FxAccountsPush.jsm", + "resource://gre/modules/FxAccountsStorage.jsm": "services/fxaccounts/FxAccountsStorage.jsm", + "resource://gre/modules/FxAccountsTelemetry.jsm": "services/fxaccounts/FxAccountsTelemetry.jsm", + "resource://gre/modules/FxAccountsWebChannel.jsm": "services/fxaccounts/FxAccountsWebChannel.jsm", + "resource://gre/modules/GMPInstallManager.jsm": "toolkit/modules/GMPInstallManager.jsm", + "resource://gre/modules/GMPUtils.jsm": "toolkit/modules/GMPUtils.jsm", + "resource://gre/modules/GeckoViewAutocomplete.jsm": "mobile/android/modules/geckoview/GeckoViewAutocomplete.jsm", + "resource://gre/modules/GeckoViewAutofill.jsm": "mobile/android/modules/geckoview/GeckoViewAutofill.jsm", + "resource://gre/modules/GeckoViewChildModule.jsm": "mobile/android/modules/geckoview/GeckoViewChildModule.jsm", + "resource://gre/modules/GeckoViewConsole.jsm": "mobile/android/modules/geckoview/GeckoViewConsole.jsm", + "resource://gre/modules/GeckoViewContent.jsm": "mobile/android/modules/geckoview/GeckoViewContent.jsm", + "resource://gre/modules/GeckoViewContentBlocking.jsm": "mobile/android/modules/geckoview/GeckoViewContentBlocking.jsm", + "resource://gre/modules/GeckoViewMediaControl.jsm": "mobile/android/modules/geckoview/GeckoViewMediaControl.jsm", + "resource://gre/modules/GeckoViewModule.jsm": "mobile/android/modules/geckoview/GeckoViewModule.jsm", + "resource://gre/modules/GeckoViewNavigation.jsm": "mobile/android/modules/geckoview/GeckoViewNavigation.jsm", + "resource://gre/modules/GeckoViewPermission.jsm": "mobile/android/components/geckoview/GeckoViewPermission.jsm", + "resource://gre/modules/GeckoViewProcessHangMonitor.jsm": "mobile/android/modules/geckoview/GeckoViewProcessHangMonitor.jsm", + "resource://gre/modules/GeckoViewProgress.jsm": "mobile/android/modules/geckoview/GeckoViewProgress.jsm", + "resource://gre/modules/GeckoViewPrompt.jsm": "mobile/android/components/geckoview/GeckoViewPrompt.jsm", + "resource://gre/modules/GeckoViewPush.jsm": "mobile/android/components/geckoview/GeckoViewPush.jsm", + "resource://gre/modules/GeckoViewPushController.jsm": "mobile/android/modules/geckoview/GeckoViewPushController.jsm", + "resource://gre/modules/GeckoViewRemoteDebugger.jsm": "mobile/android/modules/geckoview/GeckoViewRemoteDebugger.jsm", + "resource://gre/modules/GeckoViewSelectionAction.jsm": "mobile/android/modules/geckoview/GeckoViewSelectionAction.jsm", + "resource://gre/modules/GeckoViewSettings.jsm": "mobile/android/modules/geckoview/GeckoViewSettings.jsm", + "resource://gre/modules/GeckoViewStartup.jsm": "mobile/android/components/geckoview/GeckoViewStartup.jsm", + "resource://gre/modules/GeckoViewStorageController.jsm": "mobile/android/modules/geckoview/GeckoViewStorageController.jsm", + "resource://gre/modules/GeckoViewTab.jsm": "mobile/android/modules/geckoview/GeckoViewTab.jsm", + "resource://gre/modules/GeckoViewTelemetry.jsm": "mobile/android/modules/geckoview/GeckoViewTelemetry.jsm", + "resource://gre/modules/GeckoViewTestUtils.jsm": "mobile/android/modules/geckoview/GeckoViewTestUtils.jsm", + "resource://gre/modules/GeckoViewWebExtension.jsm": "mobile/android/modules/geckoview/GeckoViewWebExtension.jsm", + "resource://gre/modules/Geometry.jsm": "toolkit/modules/Geometry.jsm", + "resource://gre/modules/HealthPing.jsm": "toolkit/components/telemetry/pings/HealthPing.jsm", + "resource://gre/modules/HelperAppDlg.jsm": "toolkit/mozapps/downloads/HelperAppDlg.jsm", + "resource://gre/modules/HiddenFrame.jsm": "toolkit/modules/HiddenFrame.jsm", + "resource://gre/modules/History.jsm": "toolkit/components/places/History.jsm", + "resource://gre/modules/Http.jsm": "toolkit/modules/Http.jsm", + "resource://gre/modules/IgnoreLists.jsm": "toolkit/modules/IgnoreLists.jsm", + "resource://gre/modules/ImageObjectProcessor.jsm": "dom/manifest/ImageObjectProcessor.jsm", + "resource://gre/modules/IndexedDB.jsm": "toolkit/modules/IndexedDB.jsm", + "resource://gre/modules/InlineSpellChecker.jsm": "toolkit/modules/InlineSpellChecker.jsm", + "resource://gre/modules/InlineSpellCheckerContent.jsm": "toolkit/modules/InlineSpellCheckerContent.jsm", + "resource://gre/modules/InputListAutoComplete.jsm": "toolkit/components/satchel/InputListAutoComplete.jsm", + "resource://gre/modules/InsecurePasswordUtils.jsm": "toolkit/components/passwordmgr/InsecurePasswordUtils.jsm", + "resource://gre/modules/Integration.jsm": "toolkit/modules/Integration.jsm", + "resource://gre/modules/JSONFile.jsm": "toolkit/modules/JSONFile.jsm", + "resource://gre/modules/JsonSchema.jsm": "toolkit/modules/JsonSchema.jsm", + "resource://gre/modules/KeywordUtils.jsm": "toolkit/modules/KeywordUtils.jsm", + "resource://gre/modules/LangPackMatcher.jsm": "intl/locale/LangPackMatcher.jsm", + "resource://gre/modules/LayoutUtils.jsm": "toolkit/modules/LayoutUtils.jsm", + "resource://gre/modules/LightweightThemeConsumer.jsm": "toolkit/modules/LightweightThemeConsumer.jsm", + "resource://gre/modules/LightweightThemeManager.jsm": "toolkit/mozapps/extensions/LightweightThemeManager.jsm", + "resource://gre/modules/LoadURIDelegate.jsm": "mobile/android/modules/geckoview/LoadURIDelegate.jsm", + "resource://gre/modules/LocationHelper.jsm": "dom/base/LocationHelper.jsm", + "resource://gre/modules/Log.jsm": "toolkit/modules/Log.jsm", + "resource://gre/modules/LoginAutoComplete.jsm": "toolkit/components/passwordmgr/LoginAutoComplete.jsm", + "resource://gre/modules/LoginCSVImport.jsm": "toolkit/components/passwordmgr/LoginCSVImport.jsm", + "resource://gre/modules/LoginExport.jsm": "toolkit/components/passwordmgr/LoginExport.jsm", + "resource://gre/modules/LoginFormFactory.jsm": "toolkit/components/passwordmgr/LoginFormFactory.jsm", + "resource://gre/modules/LoginHelper.jsm": "toolkit/components/passwordmgr/LoginHelper.jsm", + "resource://gre/modules/LoginInfo.jsm": "toolkit/components/passwordmgr/LoginInfo.jsm", + "resource://gre/modules/LoginManager.jsm": "toolkit/components/passwordmgr/LoginManager.jsm", + "resource://gre/modules/LoginManagerAuthPrompter.jsm": "toolkit/components/passwordmgr/LoginManagerAuthPrompter.jsm", + "resource://gre/modules/LoginManagerChild.jsm": "toolkit/components/passwordmgr/LoginManagerChild.jsm", + "resource://gre/modules/LoginManagerContextMenu.jsm": "toolkit/components/passwordmgr/LoginManagerContextMenu.jsm", + "resource://gre/modules/LoginManagerParent.jsm": "toolkit/components/passwordmgr/LoginManagerParent.jsm", + "resource://gre/modules/LoginManagerPrompter.jsm": "toolkit/components/passwordmgr/LoginManagerPrompter.jsm", + "resource://gre/modules/LoginRecipes.jsm": "toolkit/components/passwordmgr/LoginRecipes.jsm", + "resource://gre/modules/LoginRelatedRealms.jsm": "toolkit/components/passwordmgr/LoginRelatedRealms.jsm", + "resource://gre/modules/LoginStorageDelegate.jsm": "mobile/android/components/geckoview/LoginStorageDelegate.jsm", + "resource://gre/modules/LoginStore.jsm": "toolkit/components/passwordmgr/LoginStore.jsm", + "resource://gre/modules/MainProcessSingleton.jsm": "toolkit/components/processsingleton/MainProcessSingleton.jsm", + "resource://gre/modules/Manifest.jsm": "dom/manifest/Manifest.jsm", + "resource://gre/modules/ManifestFinder.jsm": "dom/manifest/ManifestFinder.jsm", + "resource://gre/modules/ManifestIcons.jsm": "dom/manifest/ManifestIcons.jsm", + "resource://gre/modules/ManifestMessagesChild.jsm": "dom/ipc/ManifestMessagesChild.jsm", + "resource://gre/modules/ManifestObtainer.jsm": "dom/manifest/ManifestObtainer.jsm", + "resource://gre/modules/ManifestProcessor.jsm": "dom/manifest/ManifestProcessor.jsm", + "resource://gre/modules/MatchURLFilters.jsm": "toolkit/components/extensions/MatchURLFilters.jsm", + "resource://gre/modules/MediaUtils.jsm": "mobile/android/modules/geckoview/MediaUtils.jsm", + "resource://gre/modules/MessageManagerProxy.jsm": "toolkit/components/extensions/MessageManagerProxy.jsm", + "resource://gre/modules/ModulesPing.jsm": "toolkit/components/telemetry/pings/ModulesPing.jsm", + "resource://gre/modules/MozProtocolHandler.jsm": "toolkit/components/mozprotocol/MozProtocolHandler.jsm", + "resource://gre/modules/NLP.jsm": "toolkit/modules/NLP.jsm", + "resource://gre/modules/NativeManifests.jsm": "toolkit/components/extensions/NativeManifests.jsm", + "resource://gre/modules/NativeMessaging.jsm": "toolkit/components/extensions/NativeMessaging.jsm", + "resource://gre/modules/NetUtil.jsm": "netwerk/base/NetUtil.jsm", + "resource://gre/modules/NetworkGeolocationProvider.jsm": "dom/system/NetworkGeolocationProvider.jsm", + "resource://gre/modules/NewPasswordModel.jsm": "toolkit/components/passwordmgr/NewPasswordModel.jsm", + "resource://gre/modules/NewTabUtils.jsm": "toolkit/modules/NewTabUtils.jsm", + "resource://gre/modules/NotificationStorage.jsm": "dom/notification/NotificationStorage.jsm", + "resource://gre/modules/OSCrypto.jsm": "toolkit/components/passwordmgr/OSCrypto.jsm", + "resource://gre/modules/OSCrypto_win.jsm": "toolkit/components/passwordmgr/OSCrypto_win.jsm", + "resource://gre/modules/OSKeyStore.jsm": "toolkit/modules/OSKeyStore.jsm", + "resource://gre/modules/ObjectUtils.jsm": "toolkit/modules/ObjectUtils.jsm", + "resource://gre/modules/OpenSearchEngine.jsm": "toolkit/components/search/OpenSearchEngine.jsm", + "resource://gre/modules/OsEnvironment.jsm": "toolkit/modules/OsEnvironment.jsm", + "resource://gre/modules/PageThumbUtils.jsm": "toolkit/components/thumbnails/PageThumbUtils.jsm", + "resource://gre/modules/PageThumbs.jsm": "toolkit/components/thumbnails/PageThumbs.jsm", + "resource://gre/modules/PageThumbsStorageService.jsm": "toolkit/components/thumbnails/PageThumbsStorageService.jsm", + "resource://gre/modules/PartitioningExceptionListService.jsm": "toolkit/components/antitracking/PartitioningExceptionListService.jsm", + "resource://gre/modules/PasswordGenerator.jsm": "toolkit/components/passwordmgr/PasswordGenerator.jsm", + "resource://gre/modules/PasswordRulesManager.jsm": "toolkit/components/passwordmgr/PasswordRulesManager.jsm", + "resource://gre/modules/PasswordRulesParser.jsm": "toolkit/components/passwordmgr/PasswordRulesParser.jsm", + "resource://gre/modules/PerformanceCounters.jsm": "toolkit/components/extensions/PerformanceCounters.jsm", + "resource://gre/modules/PermissionsUtils.jsm": "toolkit/modules/PermissionsUtils.jsm", + "resource://gre/modules/PictureInPicture.jsm": "toolkit/components/pictureinpicture/PictureInPicture.jsm", + "resource://gre/modules/PictureInPictureControls.jsm": "toolkit/components/pictureinpicture/PictureInPictureControls.jsm", + "resource://gre/modules/PlacesBackups.jsm": "toolkit/components/places/PlacesBackups.jsm", + "resource://gre/modules/PlacesDBUtils.jsm": "toolkit/components/places/PlacesDBUtils.jsm", + "resource://gre/modules/PlacesExpiration.jsm": "toolkit/components/places/PlacesExpiration.jsm", + "resource://gre/modules/PlacesPreviews.jsm": "toolkit/components/places/PlacesPreviews.jsm", + "resource://gre/modules/PlacesSyncUtils.jsm": "toolkit/components/places/PlacesSyncUtils.jsm", + "resource://gre/modules/PlacesTransactions.jsm": "toolkit/components/places/PlacesTransactions.jsm", + "resource://gre/modules/PlacesUtils.jsm": "toolkit/components/places/PlacesUtils.jsm", + "resource://gre/modules/PolicySearchEngine.jsm": "toolkit/components/search/PolicySearchEngine.jsm", + "resource://gre/modules/PopupNotifications.jsm": "toolkit/modules/PopupNotifications.jsm", + "resource://gre/modules/Preferences.jsm": "toolkit/modules/Preferences.jsm", + "resource://gre/modules/PrincipalsCollector.jsm": "toolkit/components/cleardata/PrincipalsCollector.jsm", + "resource://gre/modules/PrivateBrowsingUtils.jsm": "toolkit/modules/PrivateBrowsingUtils.jsm", + "resource://gre/modules/ProcessSelector.jsm": "dom/base/ProcessSelector.jsm", + "resource://gre/modules/ProcessType.jsm": "toolkit/modules/ProcessType.jsm", + "resource://gre/modules/ProfileAge.jsm": "toolkit/modules/ProfileAge.jsm", + "resource://gre/modules/PromiseUtils.jsm": "toolkit/modules/PromiseUtils.jsm", + "resource://gre/modules/PromiseWorker.jsm": "toolkit/components/promiseworker/PromiseWorker.jsm", + "resource://gre/modules/PromptCollection.jsm": "mobile/android/components/geckoview/PromptCollection.jsm", + "resource://gre/modules/Prompter.jsm": "toolkit/components/prompts/src/Prompter.jsm", + "resource://gre/modules/PropertyListUtils.jsm": "toolkit/modules/PropertyListUtils.jsm", + "resource://gre/modules/ProxyChannelFilter.jsm": "toolkit/components/extensions/ProxyChannelFilter.jsm", + "resource://gre/modules/PurgeTrackerService.jsm": "toolkit/components/antitracking/PurgeTrackerService.jsm", + "resource://gre/modules/RFPHelper.jsm": "toolkit/components/resistfingerprinting/RFPHelper.jsm", + "resource://gre/modules/ReaderMode.jsm": "toolkit/components/reader/ReaderMode.jsm", + "resource://gre/modules/Readerable.jsm": "toolkit/components/reader/Readerable.jsm", + "resource://gre/modules/Region.jsm": "toolkit/modules/Region.jsm", + "resource://gre/modules/RemotePageAccessManager.jsm": "toolkit/modules/RemotePageAccessManager.jsm", + "resource://gre/modules/RemoteWebNavigation.jsm": "toolkit/components/remotebrowserutils/RemoteWebNavigation.jsm", + "resource://gre/modules/ResetProfile.jsm": "toolkit/modules/ResetProfile.jsm", + "resource://gre/modules/ResponsivenessMonitor.jsm": "toolkit/modules/ResponsivenessMonitor.jsm", + "resource://gre/modules/SafeBrowsing.jsm": "toolkit/components/url-classifier/SafeBrowsing.jsm", + "resource://gre/modules/SanityTest.jsm": "toolkit/components/gfx/SanityTest.jsm", + "resource://gre/modules/Schemas.jsm": "toolkit/components/extensions/Schemas.jsm", + "resource://gre/modules/SearchEngine.jsm": "toolkit/components/search/SearchEngine.jsm", + "resource://gre/modules/SearchEngineSelector.jsm": "toolkit/components/search/SearchEngineSelector.jsm", + "resource://gre/modules/SearchService.jsm": "toolkit/components/search/SearchService.jsm", + "resource://gre/modules/SearchSettings.jsm": "toolkit/components/search/SearchSettings.jsm", + "resource://gre/modules/SearchStaticData.jsm": "toolkit/components/search/SearchStaticData.jsm", + "resource://gre/modules/SearchSuggestionController.jsm": "toolkit/components/search/SearchSuggestionController.jsm", + "resource://gre/modules/SearchSuggestions.jsm": "toolkit/components/search/SearchSuggestions.jsm", + "resource://gre/modules/SearchUtils.jsm": "toolkit/components/search/SearchUtils.jsm", + "resource://gre/modules/SecurityInfo.jsm": "toolkit/components/extensions/webrequest/SecurityInfo.jsm", + "resource://gre/modules/SelectionUtils.jsm": "toolkit/modules/SelectionUtils.jsm", + "resource://gre/modules/ServiceRequest.jsm": "toolkit/modules/ServiceRequest.jsm", + "resource://gre/modules/ServiceWorkerCleanUp.jsm": "toolkit/components/cleardata/ServiceWorkerCleanUp.jsm", + "resource://gre/modules/Services.jsm": "toolkit/modules/Services.jsm", + "resource://gre/modules/SessionStoreFunctions.jsm": "toolkit/components/sessionstore/SessionStoreFunctions.jsm", + "resource://gre/modules/ShareDelegate.jsm": "mobile/android/components/geckoview/ShareDelegate.jsm", + "resource://gre/modules/SharedPromptUtils.jsm": "toolkit/components/prompts/src/PromptUtils.jsm", + "resource://gre/modules/ShieldContentProcess.jsm": "toolkit/components/normandy/ShieldContentProcess.jsm", + "resource://gre/modules/ShortcutUtils.jsm": "toolkit/modules/ShortcutUtils.jsm", + "resource://gre/modules/SimpleServices.jsm": "toolkit/components/utils/SimpleServices.jsm", + "resource://gre/modules/SignUpFormRuleset.jsm": "toolkit/components/passwordmgr/SignUpFormRuleset.jsm", + "resource://gre/modules/SlowScriptDebug.jsm": "dom/base/SlowScriptDebug.jsm", + "resource://gre/modules/Sqlite.jsm": "toolkit/modules/Sqlite.jsm", + "resource://gre/modules/SubDialog.jsm": "toolkit/modules/SubDialog.jsm", + "resource://gre/modules/Subprocess.jsm": "toolkit/modules/subprocess/Subprocess.jsm", + "resource://gre/modules/SyncedBookmarksMirror.jsm": "toolkit/components/places/SyncedBookmarksMirror.jsm", + "resource://gre/modules/TaggingService.jsm": "toolkit/components/places/TaggingService.jsm", + "resource://gre/modules/TaskScheduler.jsm": "toolkit/components/taskscheduler/TaskScheduler.jsm", + "resource://gre/modules/TaskSchedulerMacOSImpl.jsm": "toolkit/components/taskscheduler/TaskSchedulerMacOSImpl.jsm", + "resource://gre/modules/TaskSchedulerWinImpl.jsm": "toolkit/components/taskscheduler/TaskSchedulerWinImpl.jsm", + "resource://gre/modules/TelemetryArchive.jsm": "toolkit/components/telemetry/app/TelemetryArchive.jsm", + "resource://gre/modules/TelemetryController.jsm": "toolkit/components/telemetry/app/TelemetryController.jsm", + "resource://gre/modules/TelemetryControllerBase.jsm": "toolkit/components/telemetry/app/TelemetryControllerBase.jsm", + "resource://gre/modules/TelemetryControllerContent.jsm": "toolkit/components/telemetry/app/TelemetryControllerContent.jsm", + "resource://gre/modules/TelemetryControllerParent.jsm": "toolkit/components/telemetry/app/TelemetryControllerParent.jsm", + "resource://gre/modules/TelemetryEnvironment.jsm": "toolkit/components/telemetry/app/TelemetryEnvironment.jsm", + "resource://gre/modules/TelemetryReportingPolicy.jsm": "toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm", + "resource://gre/modules/TelemetryScheduler.jsm": "toolkit/components/telemetry/app/TelemetryScheduler.jsm", + "resource://gre/modules/TelemetrySend.jsm": "toolkit/components/telemetry/app/TelemetrySend.jsm", + "resource://gre/modules/TelemetrySession.jsm": "toolkit/components/telemetry/pings/TelemetrySession.jsm", + "resource://gre/modules/TelemetryStartup.jsm": "toolkit/components/telemetry/TelemetryStartup.jsm", + "resource://gre/modules/TelemetryStorage.jsm": "toolkit/components/telemetry/app/TelemetryStorage.jsm", + "resource://gre/modules/TelemetryTimestamps.jsm": "toolkit/components/telemetry/app/TelemetryTimestamps.jsm", + "resource://gre/modules/TelemetryUtils.jsm": "toolkit/components/telemetry/app/TelemetryUtils.jsm", + "resource://gre/modules/TerminatorTelemetry.jsm": "toolkit/components/terminator/TerminatorTelemetry.jsm", + "resource://gre/modules/Timer.jsm": "toolkit/modules/Timer.jsm", + "resource://gre/modules/TooltipTextProvider.jsm": "toolkit/components/tooltiptext/TooltipTextProvider.jsm", + "resource://gre/modules/TrackingDBService.jsm": "toolkit/components/antitracking/TrackingDBService.jsm", + "resource://gre/modules/Troubleshoot.jsm": "toolkit/modules/Troubleshoot.jsm", + "resource://gre/modules/URIFixup.jsm": "docshell/base/URIFixup.jsm", + "resource://gre/modules/URLDecorationAnnotationsService.jsm": "toolkit/components/antitracking/URLDecorationAnnotationsService.jsm", + "resource://gre/modules/URLFormatter.jsm": "toolkit/components/urlformatter/URLFormatter.jsm", + "resource://gre/modules/URLQueryStrippingListService.jsm": "toolkit/components/antitracking/URLQueryStrippingListService.jsm", + "resource://gre/modules/UninstallPing.jsm": "toolkit/components/telemetry/pings/UninstallPing.jsm", + "resource://gre/modules/UntrustedModulesPing.jsm": "toolkit/components/telemetry/pings/UntrustedModulesPing.jsm", + "resource://gre/modules/UpdateListener.jsm": "toolkit/mozapps/update/UpdateListener.jsm", + "resource://gre/modules/UpdatePing.jsm": "toolkit/components/telemetry/pings/UpdatePing.jsm", + "resource://gre/modules/UpdateService.jsm": "toolkit/mozapps/update/UpdateService.jsm", + "resource://gre/modules/UpdateServiceStub.jsm": "toolkit/mozapps/update/UpdateServiceStub.jsm", + "resource://gre/modules/UpdateTelemetry.jsm": "toolkit/mozapps/update/UpdateTelemetry.jsm", + "resource://gre/modules/UpdateTimerManager.jsm": "toolkit/components/timermanager/UpdateTimerManager.jsm", + "resource://gre/modules/UpdateUtils.jsm": "toolkit/modules/UpdateUtils.jsm", + "resource://gre/modules/UrlClassifierExceptionListService.jsm": "netwerk/url-classifier/UrlClassifierExceptionListService.jsm", + "resource://gre/modules/UrlClassifierHashCompleter.jsm": "toolkit/components/url-classifier/UrlClassifierHashCompleter.jsm", + "resource://gre/modules/UrlClassifierLib.jsm": "toolkit/components/url-classifier/UrlClassifierLib.jsm", + "resource://gre/modules/UrlClassifierListManager.jsm": "toolkit/components/url-classifier/UrlClassifierListManager.jsm", + "resource://gre/modules/UserSearchEngine.jsm": "toolkit/components/search/UserSearchEngine.jsm", + "resource://gre/modules/ValueExtractor.jsm": "dom/manifest/ValueExtractor.jsm", + "resource://gre/modules/WebChannel.jsm": "toolkit/modules/WebChannel.jsm", + "resource://gre/modules/WebHandlerApp.jsm": "uriloader/exthandler/WebHandlerApp.jsm", + "resource://gre/modules/WebNavigation.jsm": "toolkit/components/extensions/WebNavigation.jsm", + "resource://gre/modules/WebNavigationFrames.jsm": "toolkit/components/extensions/WebNavigationFrames.jsm", + "resource://gre/modules/WebRequest.jsm": "toolkit/components/extensions/webrequest/WebRequest.jsm", + "resource://gre/modules/WebRequestUpload.jsm": "toolkit/components/extensions/webrequest/WebRequestUpload.jsm", + "resource://gre/modules/WebVTTParserWrapper.jsm": "dom/media/webvtt/WebVTTParserWrapper.jsm", + "resource://gre/modules/WellKnownOpportunisticUtils.jsm": "netwerk/protocol/http/WellKnownOpportunisticUtils.jsm", + "resource://gre/modules/WindowsRegistry.jsm": "toolkit/modules/WindowsRegistry.jsm", + "resource://gre/modules/XPCOMUtils.jsm": "js/xpconnect/loader/XPCOMUtils.jsm", + "resource://gre/modules/addonManager.js": "toolkit/mozapps/extensions/addonManager.js", + "resource://gre/modules/addons/AddonRepository.jsm": "toolkit/mozapps/extensions/internal/AddonRepository.jsm", + "resource://gre/modules/addons/AddonSettings.jsm": "toolkit/mozapps/extensions/internal/AddonSettings.jsm", + "resource://gre/modules/addons/AddonUpdateChecker.jsm": "toolkit/mozapps/extensions/internal/AddonUpdateChecker.jsm", + "resource://gre/modules/addons/GMPProvider.jsm": "toolkit/mozapps/extensions/internal/GMPProvider.jsm", + "resource://gre/modules/addons/ProductAddonChecker.jsm": "toolkit/mozapps/extensions/internal/ProductAddonChecker.jsm", + "resource://gre/modules/addons/XPIDatabase.jsm": "toolkit/mozapps/extensions/internal/XPIDatabase.jsm", + "resource://gre/modules/addons/XPIInstall.jsm": "toolkit/mozapps/extensions/internal/XPIInstall.jsm", + "resource://gre/modules/addons/XPIProvider.jsm": "toolkit/mozapps/extensions/internal/XPIProvider.jsm", + "resource://gre/modules/amContentHandler.jsm": "toolkit/mozapps/extensions/amContentHandler.jsm", + "resource://gre/modules/amInstallTrigger.jsm": "toolkit/mozapps/extensions/amInstallTrigger.jsm", + "resource://gre/modules/amWebAPI.jsm": "toolkit/mozapps/extensions/amWebAPI.jsm", + "resource://gre/modules/backgroundtasks/BackgroundTask_backgroundupdate.jsm": "toolkit/mozapps/update/BackgroundTask_backgroundupdate.jsm", + "resource://gre/modules/components-utils/ClientEnvironment.jsm": "toolkit/components/utils/ClientEnvironment.jsm", + "resource://gre/modules/components-utils/FilterExpressions.jsm": "toolkit/components/utils/FilterExpressions.jsm", + "resource://gre/modules/components-utils/JsonSchemaValidator.jsm": "toolkit/components/utils/JsonSchemaValidator.jsm", + "resource://gre/modules/components-utils/Sampling.jsm": "toolkit/components/utils/Sampling.jsm", + "resource://gre/modules/components-utils/WindowsInstallsInfo.jsm": "toolkit/components/utils/WindowsInstallsInfo.jsm", + "resource://gre/modules/components-utils/WindowsVersionInfo.jsm": "toolkit/components/utils/WindowsVersionInfo.jsm", + "resource://gre/modules/components-utils/mozjexl.js": "toolkit/components/utils/mozjexl.js", + "resource://gre/modules/crypto-SDR.js": "toolkit/components/passwordmgr/crypto-SDR.js", + "resource://gre/modules/ctypes.jsm": "toolkit/components/ctypes/ctypes.jsm", + "resource://gre/modules/handlers/HandlerList.jsm": "uriloader/exthandler/HandlerList.jsm", + "resource://gre/modules/jsdebugger.jsm": "devtools/platform/jsdebugger.jsm", + "resource://gre/modules/kvstore.jsm": "toolkit/components/kvstore/kvstore.jsm", + "resource://gre/modules/lz4.js": "toolkit/components/lz4/lz4.js", + "resource://gre/modules/lz4_internal.js": "toolkit/components/lz4/lz4_internal.js", + "resource://gre/modules/media/IdpSandbox.jsm": "dom/media/IdpSandbox.jsm", + "resource://gre/modules/media/PeerConnection.jsm": "dom/media/PeerConnection.jsm", + "resource://gre/modules/media/PeerConnectionIdp.jsm": "dom/media/PeerConnectionIdp.jsm", + "resource://gre/modules/mozIntl.jsm": "toolkit/components/mozintl/mozIntl.jsm", + "resource://gre/modules/narrate/NarrateControls.jsm": "toolkit/components/narrate/NarrateControls.jsm", + "resource://gre/modules/narrate/Narrator.jsm": "toolkit/components/narrate/Narrator.jsm", + "resource://gre/modules/narrate/VoiceSelect.jsm": "toolkit/components/narrate/VoiceSelect.jsm", + "resource://gre/modules/netwerk-dns/PublicSuffixList.jsm": "netwerk/dns/PublicSuffixList.jsm", + "resource://gre/modules/nsAsyncShutdown.jsm": "toolkit/components/asyncshutdown/nsAsyncShutdown.jsm", + "resource://gre/modules/nsCrashMonitor.jsm": "toolkit/components/crashmonitor/nsCrashMonitor.jsm", + "resource://gre/modules/nsFormAutoCompleteResult.jsm": "toolkit/components/satchel/nsFormAutoCompleteResult.jsm", + "resource://gre/modules/pdfjs.js": "toolkit/components/pdfjs/pdfjs.js", + "resource://gre/modules/policies/WindowsGPOParser.jsm": "toolkit/components/enterprisepolicies/WindowsGPOParser.jsm", + "resource://gre/modules/policies/macOSPoliciesParser.jsm": "toolkit/components/enterprisepolicies/macOSPoliciesParser.jsm", + "resource://gre/modules/psm/DER.jsm": "security/manager/ssl/DER.jsm", + "resource://gre/modules/psm/RemoteSecuritySettings.jsm": "security/manager/ssl/RemoteSecuritySettings.jsm", + "resource://gre/modules/psm/X509.jsm": "security/manager/ssl/X509.jsm", + "resource://gre/modules/reader/ReaderWorker.jsm": "toolkit/components/reader/ReaderWorker.jsm", + "resource://gre/modules/reflect.jsm": "toolkit/components/reflect/reflect.jsm", + "resource://gre/modules/remotepagemanager/MessagePort.jsm": "toolkit/components/remotepagemanager/MessagePort.jsm", + "resource://gre/modules/remotepagemanager/RemotePageManagerChild.jsm": "toolkit/components/remotepagemanager/RemotePageManagerChild.jsm", + "resource://gre/modules/remotepagemanager/RemotePageManagerParent.jsm": "toolkit/components/remotepagemanager/RemotePageManagerParent.jsm", + "resource://gre/modules/services-automation/ServicesAutomation.jsm": "services/automation/ServicesAutomation.jsm", + "resource://gre/modules/sessionstore/PrivacyFilter.jsm": "toolkit/modules/sessionstore/PrivacyFilter.jsm", + "resource://gre/modules/sessionstore/PrivacyLevel.jsm": "toolkit/modules/sessionstore/PrivacyLevel.jsm", + "resource://gre/modules/sessionstore/SessionHistory.jsm": "toolkit/modules/sessionstore/SessionHistory.jsm", + "resource://gre/modules/sessionstore/Utils.jsm": "toolkit/modules/sessionstore/Utils.jsm", + "resource://gre/modules/storage-geckoview.js": "toolkit/components/passwordmgr/storage-geckoview.js", + "resource://gre/modules/storage-json.js": "toolkit/components/passwordmgr/storage-json.js", + "resource://gre/modules/subprocess/subprocess_common.jsm": "toolkit/modules/subprocess/subprocess_common.jsm", + "resource://gre/modules/subprocess/subprocess_unix.jsm": "toolkit/modules/subprocess/subprocess_unix.jsm", + "resource://gre/modules/subprocess/subprocess_win.jsm": "toolkit/modules/subprocess/subprocess_win.jsm", + "resource://gre/modules/third_party/fathom/fathom.jsm": "toolkit/modules/third_party/fathom/fathom.jsm", + "resource://gre/modules/third_party/jsesc/jsesc.js": "toolkit/modules/third_party/jsesc/jsesc.js", + "resource://gre/modules/translation/LanguageDetector.jsm": "toolkit/components/translation/LanguageDetector.jsm", + "resource://gre/modules/txEXSLTRegExFunctions.jsm": "dom/xslt/xslt/txEXSLTRegExFunctions.jsm", + "resource://gre/modules/vtt.jsm": "dom/media/webvtt/vtt.jsm", + "resource://messaging-system/lib/Logger.jsm": "toolkit/components/messaging-system/lib/Logger.jsm", + "resource://messaging-system/lib/SpecialMessageActions.jsm": "toolkit/components/messaging-system/lib/SpecialMessageActions.jsm", + "resource://messaging-system/targeting/Targeting.jsm": "toolkit/components/messaging-system/targeting/Targeting.jsm", + "resource://mozscreenshots/Screenshot.jsm": "browser/tools/mozscreenshots/mozscreenshots/extension/Screenshot.jsm", + "resource://mozscreenshots/TestRunner.jsm": "browser/tools/mozscreenshots/mozscreenshots/extension/TestRunner.jsm", + "resource://nimbus/ExperimentAPI.jsm": "toolkit/components/nimbus/ExperimentAPI.jsm", + "resource://nimbus/lib/ExperimentManager.jsm": "toolkit/components/nimbus/lib/ExperimentManager.jsm", + "resource://nimbus/lib/ExperimentStore.jsm": "toolkit/components/nimbus/lib/ExperimentStore.jsm", + "resource://nimbus/lib/RemoteSettingsExperimentLoader.jsm": "toolkit/components/nimbus/lib/RemoteSettingsExperimentLoader.jsm", + "resource://nimbus/lib/SharedDataMap.jsm": "toolkit/components/nimbus/lib/SharedDataMap.jsm", + "resource://normandy-content/AboutPages.jsm": "toolkit/components/normandy/content/AboutPages.jsm", + "resource://normandy-content/ShieldFrameChild.jsm": "toolkit/components/normandy/content/ShieldFrameChild.jsm", + "resource://normandy-content/ShieldFrameParent.jsm": "toolkit/components/normandy/content/ShieldFrameParent.jsm", + "resource://normandy-vendor/PropTypes.js": "toolkit/components/normandy/vendor/PropTypes.js", + "resource://normandy-vendor/React.js": "toolkit/components/normandy/vendor/React.js", + "resource://normandy-vendor/ReactDOM.js": "toolkit/components/normandy/vendor/ReactDOM.js", + "resource://normandy-vendor/classnames.js": "toolkit/components/normandy/vendor/classnames.js", + "resource://normandy/Normandy.jsm": "toolkit/components/normandy/Normandy.jsm", + "resource://normandy/NormandyMigrations.jsm": "toolkit/components/normandy/NormandyMigrations.jsm", + "resource://normandy/actions/AddonRollbackAction.jsm": "toolkit/components/normandy/actions/AddonRollbackAction.jsm", + "resource://normandy/actions/AddonRolloutAction.jsm": "toolkit/components/normandy/actions/AddonRolloutAction.jsm", + "resource://normandy/actions/BaseAction.jsm": "toolkit/components/normandy/actions/BaseAction.jsm", + "resource://normandy/actions/BaseStudyAction.jsm": "toolkit/components/normandy/actions/BaseStudyAction.jsm", + "resource://normandy/actions/BranchedAddonStudyAction.jsm": "toolkit/components/normandy/actions/BranchedAddonStudyAction.jsm", + "resource://normandy/actions/ConsoleLogAction.jsm": "toolkit/components/normandy/actions/ConsoleLogAction.jsm", + "resource://normandy/actions/MessagingExperimentAction.jsm": "toolkit/components/normandy/actions/MessagingExperimentAction.jsm", + "resource://normandy/actions/PreferenceExperimentAction.jsm": "toolkit/components/normandy/actions/PreferenceExperimentAction.jsm", + "resource://normandy/actions/PreferenceRollbackAction.jsm": "toolkit/components/normandy/actions/PreferenceRollbackAction.jsm", + "resource://normandy/actions/PreferenceRolloutAction.jsm": "toolkit/components/normandy/actions/PreferenceRolloutAction.jsm", + "resource://normandy/actions/ShowHeartbeatAction.jsm": "toolkit/components/normandy/actions/ShowHeartbeatAction.jsm", + "resource://normandy/actions/schemas/index.js": "toolkit/components/normandy/actions/schemas/index.js", + "resource://normandy/lib/ActionsManager.jsm": "toolkit/components/normandy/lib/ActionsManager.jsm", + "resource://normandy/lib/AddonRollouts.jsm": "toolkit/components/normandy/lib/AddonRollouts.jsm", + "resource://normandy/lib/AddonStudies.jsm": "toolkit/components/normandy/lib/AddonStudies.jsm", + "resource://normandy/lib/CleanupManager.jsm": "toolkit/components/normandy/lib/CleanupManager.jsm", + "resource://normandy/lib/ClientEnvironment.jsm": "toolkit/components/normandy/lib/ClientEnvironment.jsm", + "resource://normandy/lib/EventEmitter.jsm": "toolkit/components/normandy/lib/EventEmitter.jsm", + "resource://normandy/lib/Heartbeat.jsm": "toolkit/components/normandy/lib/Heartbeat.jsm", + "resource://normandy/lib/LogManager.jsm": "toolkit/components/normandy/lib/LogManager.jsm", + "resource://normandy/lib/NormandyAddonManager.jsm": "toolkit/components/normandy/lib/NormandyAddonManager.jsm", + "resource://normandy/lib/NormandyApi.jsm": "toolkit/components/normandy/lib/NormandyApi.jsm", + "resource://normandy/lib/NormandyUtils.jsm": "toolkit/components/normandy/lib/NormandyUtils.jsm", + "resource://normandy/lib/PrefUtils.jsm": "toolkit/components/normandy/lib/PrefUtils.jsm", + "resource://normandy/lib/PreferenceExperiments.jsm": "toolkit/components/normandy/lib/PreferenceExperiments.jsm", + "resource://normandy/lib/PreferenceRollouts.jsm": "toolkit/components/normandy/lib/PreferenceRollouts.jsm", + "resource://normandy/lib/RecipeRunner.jsm": "toolkit/components/normandy/lib/RecipeRunner.jsm", + "resource://normandy/lib/ShieldPreferences.jsm": "toolkit/components/normandy/lib/ShieldPreferences.jsm", + "resource://normandy/lib/Storage.jsm": "toolkit/components/normandy/lib/Storage.jsm", + "resource://normandy/lib/TelemetryEvents.jsm": "toolkit/components/normandy/lib/TelemetryEvents.jsm", + "resource://normandy/lib/Uptake.jsm": "toolkit/components/normandy/lib/Uptake.jsm", + "resource://pdf.js/PdfJs.jsm": "toolkit/components/pdfjs/content/PdfJs.jsm", + "resource://pdf.js/PdfJsDefaultPreferences.jsm": "toolkit/components/pdfjs/content/PdfJsDefaultPreferences.jsm", + "resource://pdf.js/PdfJsNetwork.jsm": "toolkit/components/pdfjs/content/PdfJsNetwork.jsm", + "resource://pdf.js/PdfJsTelemetry.jsm": "toolkit/components/pdfjs/content/PdfJsTelemetry.jsm", + "resource://pdf.js/PdfSandbox.jsm": "toolkit/components/pdfjs/content/PdfSandbox.jsm", + "resource://pdf.js/PdfStreamConverter.jsm": "toolkit/components/pdfjs/content/PdfStreamConverter.jsm", + "resource://pdf.js/PdfjsChild.jsm": "toolkit/components/pdfjs/content/PdfjsChild.jsm", + "resource://pdf.js/PdfjsParent.jsm": "toolkit/components/pdfjs/content/PdfjsParent.jsm", + "resource://pdf.js/build/pdf.sandbox.external.js": "toolkit/components/pdfjs/content/build/pdf.sandbox.external.js", + "resource://reftest/AsyncSpellCheckTestHelper.jsm": "editor/AsyncSpellCheckTestHelper.jsm", + "resource://reftest/PerTestCoverageUtils.jsm": "tools/code-coverage/PerTestCoverageUtils.jsm", + "resource://reftest/ReftestFissionChild.jsm": "layout/tools/reftest/ReftestFissionChild.jsm", + "resource://reftest/ReftestFissionParent.jsm": "layout/tools/reftest/ReftestFissionParent.jsm", + "resource://reftest/StructuredLog.jsm": "testing/modules/StructuredLog.jsm", + "resource://reftest/globals.jsm": "layout/tools/reftest/globals.jsm", + "resource://reftest/manifest.jsm": "layout/tools/reftest/manifest.jsm", + "resource://reftest/reftest.jsm": "layout/tools/reftest/reftest.jsm", + "resource://report-site-issue/tabExtrasActor.jsm": "browser/extensions/report-site-issue/experimentalAPIs/actors/tabExtrasActor.jsm", + "resource://services-automation/ServicesAutomation.jsm": "services/automation/ServicesAutomation.jsm", + "resource://services-common/async.js": "services/common/async.js", + "resource://services-common/hawkclient.js": "services/common/hawkclient.js", + "resource://services-common/hawkrequest.js": "services/common/hawkrequest.js", + "resource://services-common/kinto-http-client.js": "services/common/kinto-http-client.js", + "resource://services-common/kinto-offline-client.js": "services/common/kinto-offline-client.js", + "resource://services-common/kinto-storage-adapter.js": "services/common/kinto-storage-adapter.js", + "resource://services-common/logmanager.js": "services/common/logmanager.js", + "resource://services-common/observers.js": "services/common/observers.js", + "resource://services-common/rest.js": "services/common/rest.js", + "resource://services-common/tokenserverclient.js": "services/common/tokenserverclient.js", + "resource://services-common/uptake-telemetry.js": "services/common/uptake-telemetry.js", + "resource://services-common/utils.js": "services/common/utils.js", + "resource://services-crypto/WeaveCrypto.js": "services/crypto/modules/WeaveCrypto.js", + "resource://services-crypto/jwcrypto.jsm": "services/crypto/modules/jwcrypto.jsm", + "resource://services-crypto/utils.js": "services/crypto/modules/utils.js", + "resource://services-settings/Attachments.jsm": "services/settings/Attachments.jsm", + "resource://services-settings/Database.jsm": "services/settings/Database.jsm", + "resource://services-settings/IDBHelpers.jsm": "services/settings/IDBHelpers.jsm", + "resource://services-settings/RemoteSettingsClient.jsm": "services/settings/RemoteSettingsClient.jsm", + "resource://services-settings/RemoteSettingsComponents.jsm": "services/settings/RemoteSettingsComponents.jsm", + "resource://services-settings/RemoteSettingsWorker.jsm": "services/settings/RemoteSettingsWorker.jsm", + "resource://services-settings/SharedUtils.jsm": "services/settings/SharedUtils.jsm", + "resource://services-settings/SyncHistory.jsm": "services/settings/SyncHistory.jsm", + "resource://services-settings/Utils.jsm": "services/settings/Utils.jsm", + "resource://services-settings/remote-settings.js": "services/settings/remote-settings.js", + "resource://services-sync/SyncDisconnect.jsm": "services/sync/modules/SyncDisconnect.jsm", + "resource://services-sync/SyncedTabs.jsm": "services/sync/modules/SyncedTabs.jsm", + "resource://services-sync/UIState.jsm": "services/sync/modules/UIState.jsm", + "resource://services-sync/Weave.jsm": "services/sync/Weave.jsm", + "resource://services-sync/addonsreconciler.js": "services/sync/modules/addonsreconciler.js", + "resource://services-sync/addonutils.js": "services/sync/modules/addonutils.js", + "resource://services-sync/bridged_engine.js": "services/sync/modules/bridged_engine.js", + "resource://services-sync/collection_validator.js": "services/sync/modules/collection_validator.js", + "resource://services-sync/constants.js": "services/sync/modules/constants.js", + "resource://services-sync/doctor.js": "services/sync/modules/doctor.js", + "resource://services-sync/engines.js": "services/sync/modules/engines.js", + "resource://services-sync/engines/addons.js": "services/sync/modules/engines/addons.js", + "resource://services-sync/engines/bookmarks.js": "services/sync/modules/engines/bookmarks.js", + "resource://services-sync/engines/clients.js": "services/sync/modules/engines/clients.js", + "resource://services-sync/engines/extension-storage.js": "services/sync/modules/engines/extension-storage.js", + "resource://services-sync/engines/forms.js": "services/sync/modules/engines/forms.js", + "resource://services-sync/engines/history.js": "services/sync/modules/engines/history.js", + "resource://services-sync/engines/passwords.js": "services/sync/modules/engines/passwords.js", + "resource://services-sync/engines/prefs.js": "services/sync/modules/engines/prefs.js", + "resource://services-sync/engines/tabs.js": "services/sync/modules/engines/tabs.js", + "resource://services-sync/keys.js": "services/sync/modules/keys.js", + "resource://services-sync/main.js": "services/sync/modules/main.js", + "resource://services-sync/policies.js": "services/sync/modules/policies.js", + "resource://services-sync/record.js": "services/sync/modules/record.js", + "resource://services-sync/resource.js": "services/sync/modules/resource.js", + "resource://services-sync/service.js": "services/sync/modules/service.js", + "resource://services-sync/stages/declined.js": "services/sync/modules/stages/declined.js", + "resource://services-sync/stages/enginesync.js": "services/sync/modules/stages/enginesync.js", + "resource://services-sync/status.js": "services/sync/modules/status.js", + "resource://services-sync/sync_auth.js": "services/sync/modules/sync_auth.js", + "resource://services-sync/telemetry.js": "services/sync/modules/telemetry.js", + "resource://services-sync/util.js": "services/sync/modules/util.js", + "resource://specialpowers/AppTestDelegate.jsm": "testing/specialpowers/content/AppTestDelegate.jsm", + "resource://specialpowers/AppTestDelegateChild.jsm": "testing/specialpowers/content/AppTestDelegateChild.jsm", + "resource://specialpowers/AppTestDelegateParent.jsm": "testing/specialpowers/content/AppTestDelegateParent.jsm", + "resource://specialpowers/MockColorPicker.jsm": "testing/specialpowers/content/MockColorPicker.jsm", + "resource://specialpowers/MockFilePicker.jsm": "testing/specialpowers/content/MockFilePicker.jsm", + "resource://specialpowers/MockPermissionPrompt.jsm": "testing/specialpowers/content/MockPermissionPrompt.jsm", + "resource://specialpowers/SpecialPowersChild.jsm": "testing/specialpowers/content/SpecialPowersChild.jsm", + "resource://specialpowers/SpecialPowersEventUtils.jsm": "testing/specialpowers/content/SpecialPowersEventUtils.jsm", + "resource://specialpowers/SpecialPowersParent.jsm": "testing/specialpowers/content/SpecialPowersParent.jsm", + "resource://specialpowers/SpecialPowersSandbox.jsm": "testing/specialpowers/content/SpecialPowersSandbox.jsm", + "resource://specialpowers/WrapPrivileged.jsm": "testing/specialpowers/content/WrapPrivileged.jsm", + "resource://talos-powers/TalosParentProfiler.jsm": "testing/talos/talos/talos-powers/content/TalosParentProfiler.jsm", + "resource://test/AllowJavascriptChild.jsm": "docshell/test/unit/AllowJavascriptChild.jsm", + "resource://test/AllowJavascriptParent.jsm": "docshell/test/unit/AllowJavascriptParent.jsm", + "resource://test/Census.jsm": "devtools/shared/heapsnapshot/tests/xpcshell/Census.jsm", + "resource://test/CrashTestUtils.jsm": "toolkit/crashreporter/test/CrashTestUtils.jsm", + "resource://test/GlobalObjectsModule.jsm": "dom/indexedDB/test/unit/GlobalObjectsModule.jsm", + "resource://test/Match.jsm": "devtools/shared/heapsnapshot/tests/xpcshell/Match.jsm", + "resource://test/TestRunner.jsm": "browser/tools/mozscreenshots/mozscreenshots/extension/TestRunner.jsm", + "resource://test/broadcast_handler.jsm": "dom/push/test/xpcshell/broadcast_handler.jsm", + "resource://testing-common/AddonTestUtils.jsm": "toolkit/mozapps/extensions/internal/AddonTestUtils.jsm", + "resource://testing-common/AppData.jsm": "testing/modules/AppData.jsm", + "resource://testing-common/AppInfo.jsm": "testing/modules/AppInfo.jsm", + "resource://testing-common/Assert.jsm": "testing/modules/Assert.jsm", + "resource://testing-common/AsyncSpellCheckTestHelper.jsm": "editor/AsyncSpellCheckTestHelper.jsm", + "resource://testing-common/BackgroundTasksTestUtils.jsm": "toolkit/components/backgroundtasks/BackgroundTasksTestUtils.jsm", + "resource://testing-common/BrowserTestUtils.jsm": "testing/mochitest/BrowserTestUtils/BrowserTestUtils.jsm", + "resource://testing-common/BrowserTestUtilsChild.jsm": "testing/mochitest/BrowserTestUtils/BrowserTestUtilsChild.jsm", + "resource://testing-common/BrowserTestUtilsParent.jsm": "testing/mochitest/BrowserTestUtils/BrowserTestUtilsParent.jsm", + "resource://testing-common/ContentEventListenerChild.jsm": "testing/mochitest/BrowserTestUtils/ContentEventListenerChild.jsm", + "resource://testing-common/ContentEventListenerParent.jsm": "testing/mochitest/BrowserTestUtils/ContentEventListenerParent.jsm", + "resource://testing-common/ContentTask.jsm": "testing/mochitest/BrowserTestUtils/ContentTask.jsm", + "resource://testing-common/ContentTaskUtils.jsm": "testing/mochitest/BrowserTestUtils/ContentTaskUtils.jsm", + "resource://testing-common/CookieXPCShellUtils.jsm": "netwerk/cookie/CookieXPCShellUtils.jsm", + "resource://testing-common/CoverageUtils.jsm": "testing/modules/CoverageUtils.jsm", + "resource://testing-common/CrashManagerTest.jsm": "toolkit/components/crashes/CrashManagerTest.jsm", + "resource://testing-common/CustomizableUITestUtils.jsm": "browser/components/customizableui/test/CustomizableUITestUtils.jsm", + "resource://testing-common/DoHTestUtils.jsm": "browser/components/doh/DoHTestUtils.jsm", + "resource://testing-common/EnterprisePolicyTesting.jsm": "toolkit/components/enterprisepolicies/tests/EnterprisePolicyTesting.jsm", + "resource://testing-common/ExtensionTestCommon.jsm": "toolkit/components/extensions/ExtensionTestCommon.jsm", + "resource://testing-common/ExtensionXPCShellUtils.jsm": "toolkit/components/extensions/ExtensionXPCShellUtils.jsm", + "resource://testing-common/FileTestUtils.jsm": "testing/modules/FileTestUtils.jsm", + "resource://testing-common/FluentSyntax.jsm": "intl/l10n/FluentSyntax.jsm", + "resource://testing-common/FormHistoryTestUtils.jsm": "toolkit/components/satchel/test/FormHistoryTestUtils.jsm", + "resource://testing-common/HandlerServiceTestUtils.jsm": "uriloader/exthandler/tests/HandlerServiceTestUtils.jsm", + "resource://testing-common/LangPackMatcherTestUtils.jsm": "intl/locale/tests/LangPackMatcherTestUtils.jsm", + "resource://testing-common/LoginTestUtils.jsm": "toolkit/components/passwordmgr/test/LoginTestUtils.jsm", + "resource://testing-common/MessageChannel.jsm": "toolkit/components/extensions/MessageChannel.jsm", + "resource://testing-common/MockDocument.jsm": "toolkit/modules/tests/modules/MockDocument.jsm", + "resource://testing-common/MockFilePicker.jsm": "testing/specialpowers/content/MockFilePicker.jsm", + "resource://testing-common/MockRegistrar.jsm": "testing/modules/MockRegistrar.jsm", + "resource://testing-common/MockRegistry.jsm": "testing/modules/MockRegistry.jsm", + "resource://testing-common/NimbusTestUtils.jsm": "toolkit/components/nimbus/test/NimbusTestUtils.jsm", + "resource://testing-common/NormandyTestUtils.jsm": "toolkit/components/normandy/test/NormandyTestUtils.jsm", + "resource://testing-common/OSKeyStoreTestUtils.jsm": "toolkit/modules/tests/modules/OSKeyStoreTestUtils.jsm", + "resource://testing-common/PerTestCoverageUtils.jsm": "tools/code-coverage/PerTestCoverageUtils.jsm", + "resource://testing-common/PermissionTestUtils.jsm": "extensions/permissions/test/PermissionTestUtils.jsm", + "resource://testing-common/PlacesTestUtils.jsm": "toolkit/components/places/tests/PlacesTestUtils.jsm", + "resource://testing-common/PromiseTestUtils.jsm": "toolkit/modules/tests/modules/PromiseTestUtils.jsm", + "resource://testing-common/PromptTestUtils.jsm": "toolkit/components/prompts/test/PromptTestUtils.jsm", + "resource://testing-common/QuickSuggestTestUtils.jsm": "browser/components/urlbar/tests/quicksuggest/QuickSuggestTestUtils.jsm", + "resource://testing-common/RegionTestUtils.jsm": "toolkit/modules/tests/xpcshell/RegionTestUtils.jsm", + "resource://testing-common/SearchTestUtils.jsm": "toolkit/components/search/tests/SearchTestUtils.jsm", + "resource://testing-common/Sinon.jsm": "testing/modules/Sinon.jsm", + "resource://testing-common/SiteDataTestUtils.jsm": "toolkit/components/cleardata/SiteDataTestUtils.jsm", + "resource://testing-common/StructuredLog.jsm": "testing/modules/StructuredLog.jsm", + "resource://testing-common/TelemetryArchiveTesting.jsm": "toolkit/components/telemetry/tests/unit/TelemetryArchiveTesting.jsm", + "resource://testing-common/TelemetryEnvironmentTesting.jsm": "toolkit/components/telemetry/tests/unit/TelemetryEnvironmentTesting.jsm", + "resource://testing-common/TelemetryTestUtils.jsm": "toolkit/components/telemetry/tests/utils/TelemetryTestUtils.jsm", + "resource://testing-common/TestIntegration.jsm": "toolkit/modules/tests/xpcshell/TestIntegration.jsm", + "resource://testing-common/TestInterfaceJS.jsm": "dom/bindings/test/TestInterfaceJS.jsm", + "resource://testing-common/TestProcessActorChild.jsm": "toolkit/actors/TestProcessActorChild.jsm", + "resource://testing-common/TestProcessActorParent.jsm": "toolkit/actors/TestProcessActorParent.jsm", + "resource://testing-common/TestUtils.jsm": "testing/modules/TestUtils.jsm", + "resource://testing-common/TestWindowChild.jsm": "toolkit/actors/TestWindowChild.jsm", + "resource://testing-common/TestWindowParent.jsm": "toolkit/actors/TestWindowParent.jsm", + "resource://testing-common/UrlClassifierTestUtils.jsm": "toolkit/components/url-classifier/tests/UrlClassifierTestUtils.jsm", + "resource://testing-common/UrlbarTestUtils.jsm": "browser/components/urlbar/tests/UrlbarTestUtils.jsm", + "resource://testing-common/XPCShellContentUtils.jsm": "testing/modules/XPCShellContentUtils.jsm", + "resource://testing-common/backgroundtasks/BackgroundTask_automaticrestart.jsm": "toolkit/components/backgroundtasks/tests/BackgroundTask_automaticrestart.jsm", + "resource://testing-common/backgroundtasks/BackgroundTask_shouldprocessupdates.jsm": "toolkit/components/backgroundtasks/tests/BackgroundTask_shouldprocessupdates.jsm", + "resource://testing-common/backgroundtasks/BackgroundTask_wait.jsm": "toolkit/components/backgroundtasks/tests/BackgroundTask_wait.jsm", + "resource://testing-common/cookie_filtering_helper.jsm": "netwerk/test/browser/cookie_filtering_helper.jsm", + "resource://testing-common/dom/quota/test/modules/ModuleLoader.jsm": "dom/quota/test/modules/system/ModuleLoader.jsm", + "resource://testing-common/dom/quota/test/modules/StorageUtils.jsm": "dom/quota/test/modules/system/StorageUtils.jsm", + "resource://testing-common/dom/quota/test/modules/WorkerDriver.jsm": "dom/quota/test/modules/system/WorkerDriver.jsm", + "resource://testing-common/early_hint_preload_test_helper.jsm": "netwerk/test/browser/early_hint_preload_test_helper.jsm", + "resource://testing-common/httpd.js": "netwerk/test/httpserver/httpd.js", + "resource://testing-common/services/common/logging.js": "services/common/modules-testing/logging.js", + "resource://testing-common/services/sync/fakeservices.js": "services/sync/modules-testing/fakeservices.js", + "resource://testing-common/services/sync/fxa_utils.js": "services/sync/modules-testing/fxa_utils.js", + "resource://testing-common/services/sync/rotaryengine.js": "services/sync/modules-testing/rotaryengine.js", + "resource://testing-common/services/sync/utils.js": "services/sync/modules-testing/utils.js", + "resource://tps/auth/fxaccounts.jsm": "services/sync/tps/extensions/tps/resource/auth/fxaccounts.jsm", + "resource://tps/logger.jsm": "services/sync/tps/extensions/tps/resource/logger.jsm", + "resource://tps/modules/addons.jsm": "services/sync/tps/extensions/tps/resource/modules/addons.jsm", + "resource://tps/modules/bookmarkValidator.jsm": "services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.jsm", + "resource://tps/modules/bookmarks.jsm": "services/sync/tps/extensions/tps/resource/modules/bookmarks.jsm", + "resource://tps/modules/formautofill.jsm": "services/sync/tps/extensions/tps/resource/modules/formautofill.jsm", + "resource://tps/modules/forms.jsm": "services/sync/tps/extensions/tps/resource/modules/forms.jsm", + "resource://tps/modules/history.jsm": "services/sync/tps/extensions/tps/resource/modules/history.jsm", + "resource://tps/modules/passwords.jsm": "services/sync/tps/extensions/tps/resource/modules/passwords.jsm", + "resource://tps/modules/prefs.jsm": "services/sync/tps/extensions/tps/resource/modules/prefs.jsm", + "resource://tps/modules/tabs.jsm": "services/sync/tps/extensions/tps/resource/modules/tabs.jsm", + "resource://tps/modules/windows.jsm": "services/sync/tps/extensions/tps/resource/modules/windows.jsm", + "resource://tps/quit.js": "services/sync/tps/extensions/tps/resource/quit.js", + "resource://tps/tps.jsm": "services/sync/tps/extensions/tps/resource/tps.jsm", + "resource://webcompat/AboutCompat.jsm": "browser/extensions/webcompat/about-compat/AboutCompat.jsm", + "resource://testing-common/PerfTestHelpers.jsm": "browser/base/content/test/performance/PerfTestHelpers.jsm", + "resource:///modules/QuickActionsLoaderDefault.jsm": "browser/components/urlbar/QuickActionsLoaderDefault.jsm", + "resource:///modules/UrlbarProviderQuickActions.jsm": "browser/components/urlbar/UrlbarProviderQuickActions.jsm", + "resource://gre/modules/UrlClassifierRemoteSettingsService.jsm": "toolkit/components/url-classifier/UrlClassifierRemoteSettingsService.jsm", + + "resource:///modules/BrowserUsageTelemetry.jsm": [ + "browser/modules/BrowserUsageTelemetry.jsm", + "mobile/android/modules/geckoview/BrowserUsageTelemetry.jsm" + ], + "resource:///modules/ExtensionBrowsingData.jsm": [ + "browser/components/extensions/ExtensionBrowsingData.jsm", + "mobile/android/components/extensions/ExtensionBrowsingData.jsm" + ], + "resource://autofill/FormAutofillPrompter.jsm": [ + "toolkit/components/formautofill/android/FormAutofillPrompter.jsm", + "toolkit/components/formautofill/default/FormAutofillPrompter.jsm" + ], + "resource://autofill/FormAutofillStorage.jsm": [ + "toolkit/components/formautofill/android/FormAutofillStorage.jsm", + "toolkit/components/formautofill/default/FormAutofillStorage.jsm" + ], + "resource://gre/modules/NotificationDB.jsm": [ + "dom/notification/new/NotificationDB.jsm", + "dom/notification/old/NotificationDB.jsm" + ], + "resource://gre/modules/XULStore.jsm": [ + "toolkit/components/xulstore/new/XULStore.jsm", + "toolkit/components/xulstore/old/XULStore.jsm" + ], + "resource://testing-common/AppUiTestDelegate.jsm": [ + "browser/components/extensions/test/AppUiTestDelegate.jsm", + "mobile/android/modules/test/AppUiTestDelegate.jsm" + ] +} diff --git a/tools/esmify/package-lock.json b/tools/esmify/package-lock.json new file mode 100644 index 0000000000..dea539947b --- /dev/null +++ b/tools/esmify/package-lock.json @@ -0,0 +1,7895 @@ +{ + "name": "esmify", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "devDependencies": { + "jscodeshift": "^0.13.1" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", + "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.1.0", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", + "integrity": "sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==", + "dev": true, + "dependencies": { + "@babel/highlight": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.18.6.tgz", + "integrity": "sha512-tzulrgDT0QD6U7BJ4TKVk2SDDg7wlP39P9yAx1RfLy7vP/7rsDRlWVfbWxElslu56+r7QOhB2NSDsabYYruoZQ==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.18.6.tgz", + "integrity": "sha512-cQbWBpxcbbs/IUredIPkHiAGULLV8iwgNRMFzvbhEXISp4f3rUUXE5+TIw6KwUWUR3DwyI6gmBRnmAtYaWehwQ==", + "dev": true, + "dependencies": { + "@ampproject/remapping": "^2.1.0", + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helpers": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.1", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.18.6.tgz", + "integrity": "sha512-AIwwoOS8axIC5MZbhNHRLKi3D+DMpvDf9XUcu3pIVAfOHFT45f4AoDAltRbHIQomCipkCZxrNkfpOEHhJz/VKw==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6", + "@jridgewell/gen-mapping": "^0.3.0", + "jsesc": "^2.5.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/generator/node_modules/@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/helper-annotate-as-pure": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz", + "integrity": "sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-builder-binary-assignment-operator-visitor": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.18.6.tgz", + "integrity": "sha512-KT10c1oWEpmrIRYnthbzHgoOf6B+Xd6a5yhdbNtdhtG7aO1or5HViuf1TQR36xY/QprXA5nvxO6nAjhJ4y38jw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-explode-assignable-expression": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.18.6.tgz", + "integrity": "sha512-vFjbfhNCzqdeAtZflUFrG5YIFqGTqsctrtkZ1D/NB0mDW9TwW3GmmUepYY4G9wCET5rY5ugz4OGTcLd614IzQg==", + "dev": true, + "dependencies": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "browserslist": "^4.20.2", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-class-features-plugin": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.18.6.tgz", + "integrity": "sha512-YfDzdnoxHGV8CzqHGyCbFvXg5QESPFkXlHtvdCkesLjjVMT2Adxe4FGUR5ChIb3DxSaXO12iIOCWoXdsUVwnqw==", + "dev": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-member-expression-to-functions": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-regexp-features-plugin": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.18.6.tgz", + "integrity": "sha512-7LcpH1wnQLGrI+4v+nPp+zUvIkF9x0ddv1Hkdue10tg3gmRnLy97DXh4STiOf1qeIInyD69Qv5kKSZzKD8B/7A==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "regexpu-core": "^5.1.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-define-polyfill-provider": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.1.tgz", + "integrity": "sha512-J9hGMpJQmtWmj46B3kBHmL38UhJGhYX7eqkcq+2gsstyYt341HmPeWspihX43yVRA0mS+8GGk2Gckc7bY/HCmA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-compilation-targets": "^7.13.0", + "@babel/helper-module-imports": "^7.12.13", + "@babel/helper-plugin-utils": "^7.13.0", + "@babel/traverse": "^7.13.0", + "debug": "^4.1.1", + "lodash.debounce": "^4.0.8", + "resolve": "^1.14.2", + "semver": "^6.1.2" + }, + "peerDependencies": { + "@babel/core": "^7.4.0-0" + } + }, + "node_modules/@babel/helper-environment-visitor": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.6.tgz", + "integrity": "sha512-8n6gSfn2baOY+qlp+VSzsosjCVGFqWKmDF0cCWOybh52Dw3SEyoWR1KrhMJASjLwIEkkAufZ0xvr+SxLHSpy2Q==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-explode-assignable-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz", + "integrity": "sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-function-name": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.18.6.tgz", + "integrity": "sha512-0mWMxV1aC97dhjCah5U5Ua7668r5ZmSC2DLfH2EZnf9c3/dHZKiFa5pRLMH5tjSl471tY6496ZWk/kjNONBxhw==", + "dev": true, + "dependencies": { + "@babel/template": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-hoist-variables": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz", + "integrity": "sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-member-expression-to-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.18.6.tgz", + "integrity": "sha512-CeHxqwwipekotzPDUuJOfIMtcIHBuc7WAzLmTYWctVigqS5RktNMQ5bEwQSuGewzYnCtTWa3BARXeiLxDTv+Ng==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz", + "integrity": "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.18.6.tgz", + "integrity": "sha512-L//phhB4al5uucwzlimruukHB3jRd5JGClwRMD/ROrVjXfLqovYnvQrK/JK36WYyVwGGO7OD3kMyVTjx+WVPhw==", + "dev": true, + "dependencies": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/helper-validator-identifier": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-optimise-call-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.18.6.tgz", + "integrity": "sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.18.6.tgz", + "integrity": "sha512-gvZnm1YAAxh13eJdkb9EWHBnF3eAub3XTLCZEehHT2kWxiKVRL64+ae5Y6Ivne0mVHmMYKT+xWgZO+gQhuLUBg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-remap-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.18.6.tgz", + "integrity": "sha512-z5wbmV55TveUPZlCLZvxWHtrjuJd+8inFhk7DG0WW87/oJuGDcjDiu7HIvGcpf5464L6xKCg3vNkmlVVz9hwyQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-wrap-function": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-replace-supers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.18.6.tgz", + "integrity": "sha512-fTf7zoXnUGl9gF25fXCWE26t7Tvtyn6H4hkLSYhATwJvw2uYxd3aoXplMSe0g9XbwK7bmxNes7+FGO0rB/xC0g==", + "dev": true, + "dependencies": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-member-expression-to-functions": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-simple-access": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.18.6.tgz", + "integrity": "sha512-iNpIgTgyAvDQpDj76POqg+YEt8fPxx3yaNBg3S30dxNKm2SWfYhD0TGrK/Eu9wHpUW63VQU894TsTg+GLbUa1g==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.18.6.tgz", + "integrity": "sha512-4KoLhwGS9vGethZpAhYnMejWkX64wsnHPDwvOsKWU6Fg4+AlK2Jz3TyjQLMEPvz+1zemi/WBdkYxCD0bAfIkiw==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-split-export-declaration": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz", + "integrity": "sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.18.6.tgz", + "integrity": "sha512-MmetCkz9ej86nJQV+sFCxoGGrUbU3q02kgLciwkrt9QqEB7cP39oKEY0PakknEO0Gu20SskMRi+AYZ3b1TpN9g==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz", + "integrity": "sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-wrap-function": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.18.6.tgz", + "integrity": "sha512-I5/LZfozwMNbwr/b1vhhuYD+J/mU+gfGAj5td7l5Rv9WYmH6i3Om69WGKNmlIpsVW/mF6O5bvTKbvDQZVgjqOw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-function-name": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.18.6.tgz", + "integrity": "sha512-vzSiiqbQOghPngUYt/zWGvK3LAsPhz55vc9XNN0xAl2gV4ieShI2OQli5duxWHD+72PZPTKAcfcZDE1Cwc5zsQ==", + "dev": true, + "dependencies": { + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz", + "integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.18.6", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.18.6.tgz", + "integrity": "sha512-uQVSa9jJUe/G/304lXspfWVpKpK4euFLgGiMQFOCpM/bgcAdeoHwi/OQz23O9GK2osz26ZiXRRV9aV+Yl1O8tw==", + "dev": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz", + "integrity": "sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.18.6.tgz", + "integrity": "sha512-Udgu8ZRgrBrttVz6A0EVL0SJ1z+RLbIeqsu632SA1hf0awEppD6TvdznoH+orIF8wtFFAV/Enmw9Y+9oV8TQcw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.13.0" + } + }, + "node_modules/@babel/plugin-proposal-async-generator-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.18.6.tgz", + "integrity": "sha512-WAz4R9bvozx4qwf74M+sfqPMKfSqwM0phxPTR6iJIi8robgzXwkEgmeJG1gEKhm6sDqT/U9aV3lfcqybIpev8w==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-class-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz", + "integrity": "sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==", + "dev": true, + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-class-static-block": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.18.6.tgz", + "integrity": "sha512-+I3oIiNxrCpup3Gi8n5IGMwj0gOCAjcJUSQEcotNnCCPMEnixawOQ+KeJPlgfjzx+FKQ1QSyZOWe7wmoJp7vhw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-class-static-block": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.12.0" + } + }, + "node_modules/@babel/plugin-proposal-dynamic-import": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.18.6.tgz", + "integrity": "sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-dynamic-import": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-export-namespace-from": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.6.tgz", + "integrity": "sha512-zr/QcUlUo7GPo6+X1wC98NJADqmy5QTFWWhqeQWiki4XHafJtLl/YMGkmRB2szDD2IYJCCdBTd4ElwhId9T7Xw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-json-strings": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.18.6.tgz", + "integrity": "sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-logical-assignment-operators": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.18.6.tgz", + "integrity": "sha512-zMo66azZth/0tVd7gmkxOkOjs2rpHyhpcFo565PUP37hSp6hSd9uUKIfTDFMz58BwqgQKhJ9YxtM5XddjXVn+Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz", + "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-numeric-separator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz", + "integrity": "sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-numeric-separator": "^7.10.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-object-rest-spread": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.18.6.tgz", + "integrity": "sha512-9yuM6wr4rIsKa1wlUAbZEazkCrgw2sMPEXCr4Rnwetu7cEW1NydkCWytLuYletbf8vFxdJxFhwEZqMpOx2eZyw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-transform-parameters": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-catch-binding": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.18.6.tgz", + "integrity": "sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-chaining": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.18.6.tgz", + "integrity": "sha512-PatI6elL5eMzoypFAiYDpYQyMtXTn+iMhuxxQt5mAXD4fEmKorpSI3PHd+i3JXBJN3xyA6MvJv7at23HffFHwA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6", + "@babel/plugin-syntax-optional-chaining": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-private-methods": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz", + "integrity": "sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-private-property-in-object": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.18.6.tgz", + "integrity": "sha512-9Rysx7FOctvT5ouj5JODjAFAkgGoudQuLPamZb0v1TGLpapdNaftzifU8NTWQm0IRjqoYypdrSmyWgkocDQ8Dw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-unicode-property-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.18.6.tgz", + "integrity": "sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-export-namespace-from": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz", + "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-flow": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz", + "integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-assertions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.18.6.tgz", + "integrity": "sha512-/DU3RXad9+bZwrgWJQKbr39gYbJpLJHezqEzRzi/BHRlJ9zsQb4CK2CA/5apllXNomwA1qHwzvHl+AdEmC5krQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.18.6.tgz", + "integrity": "sha512-mAWAuq4rvOepWCBid55JuRNvpTNf2UGVgoz4JV0fXEKolsVZDzsa4NqCef758WZJj/GDu0gVGItjKFiClTAmZA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-arrow-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.18.6.tgz", + "integrity": "sha512-9S9X9RUefzrsHZmKMbDXxweEH+YlE8JJEuat9FdvW9Qh1cw7W64jELCtWNkPBPX5En45uy28KGvA/AySqUh8CQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.18.6.tgz", + "integrity": "sha512-ARE5wZLKnTgPW7/1ftQmSi1CmkqqHo2DNmtztFhvgtOWSDfq0Cq9/9L+KnZNYSNrydBekhW3rwShduf59RoXag==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoped-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.18.6.tgz", + "integrity": "sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoping": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.18.6.tgz", + "integrity": "sha512-pRqwb91C42vs1ahSAWJkxOxU1RHWDn16XAa6ggQ72wjLlWyYeAcLvTtE0aM8ph3KNydy9CQF2nLYcjq1WysgxQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-classes": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.18.6.tgz", + "integrity": "sha512-XTg8XW/mKpzAF3actL554Jl/dOYoJtv3l8fxaEczpgz84IeeVf+T1u2CSvPHuZbt0w3JkIx4rdn/MRQI7mo0HQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-computed-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.18.6.tgz", + "integrity": "sha512-9repI4BhNrR0KenoR9vm3/cIc1tSBIo+u1WVjKCAynahj25O8zfbiE6JtAtHPGQSs4yZ+bA8mRasRP+qc+2R5A==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-destructuring": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.18.6.tgz", + "integrity": "sha512-tgy3u6lRp17ilY8r1kP4i2+HDUwxlVqq3RTc943eAWSzGgpU1qhiKpqZ5CMyHReIYPHdo3Kg8v8edKtDqSVEyQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-dotall-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.18.6.tgz", + "integrity": "sha512-6S3jpun1eEbAxq7TdjLotAsl4WpQI9DxfkycRcKrjhQYzU87qpXdknpBg/e+TdcMehqGnLFi7tnFUBR02Vq6wg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-duplicate-keys": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.18.6.tgz", + "integrity": "sha512-NJU26U/208+sxYszf82nmGYqVF9QN8py2HFTblPT9hbawi8+1C5a9JubODLTGFuT0qlkqVinmkwOD13s0sZktg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-exponentiation-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.18.6.tgz", + "integrity": "sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-builder-binary-assignment-operator-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-flow-strip-types": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.18.6.tgz", + "integrity": "sha512-wE0xtA7csz+hw4fKPwxmu5jnzAsXPIO57XnRwzXP3T19jWh1BODnPGoG9xKYwvAwusP7iUktHayRFbMPGtODaQ==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-flow": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-for-of": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.18.6.tgz", + "integrity": "sha512-WAjoMf4wIiSsy88KmG7tgj2nFdEK7E46tArVtcgED7Bkj6Fg/tG5SbvNIOKxbFS2VFgNh6+iaPswBeQZm4ox8w==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-function-name": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.18.6.tgz", + "integrity": "sha512-kJha/Gbs5RjzIu0CxZwf5e3aTTSlhZnHMT8zPWnJMjNpLOUgqevg+PN5oMH68nMCXnfiMo4Bhgxqj59KHTlAnA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.18.6.tgz", + "integrity": "sha512-x3HEw0cJZVDoENXOp20HlypIHfl0zMIhMVZEBVTfmqbObIpsMxMbmU5nOEO8R7LYT+z5RORKPlTI5Hj4OsO9/Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-member-expression-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.18.6.tgz", + "integrity": "sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-amd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.18.6.tgz", + "integrity": "sha512-Pra5aXsmTsOnjM3IajS8rTaLCy++nGM4v3YR4esk5PCsyg9z8NA5oQLwxzMUtDBd8F+UmVza3VxoAaWCbzH1rg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-commonjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.18.6.tgz", + "integrity": "sha512-Qfv2ZOWikpvmedXQJDSbxNqy7Xr/j2Y8/KfijM0iJyKkBTmWuvCA1yeH1yDM7NJhBW/2aXxeucLj6i80/LAJ/Q==", + "dev": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-systemjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.18.6.tgz", + "integrity": "sha512-UbPYpXxLjTw6w6yXX2BYNxF3p6QY225wcTkfQCy3OMnSlS/C3xGtwUjEzGkldb/sy6PWLiCQ3NbYfjWUTI3t4g==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-identifier": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-umd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.18.6.tgz", + "integrity": "sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.18.6.tgz", + "integrity": "sha512-UmEOGF8XgaIqD74bC8g7iV3RYj8lMf0Bw7NJzvnS9qQhM4mg+1WHKotUIdjxgD2RGrgFLZZPCFPFj3P/kVDYhg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-new-target": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.18.6.tgz", + "integrity": "sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-object-super": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.18.6.tgz", + "integrity": "sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-parameters": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.18.6.tgz", + "integrity": "sha512-FjdqgMv37yVl/gwvzkcB+wfjRI8HQmc5EgOG9iGNvUY1ok+TjsoaMP7IqCDZBhkFcM5f3OPVMs6Dmp03C5k4/A==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-property-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.18.6.tgz", + "integrity": "sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-regenerator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.18.6.tgz", + "integrity": "sha512-poqRI2+qiSdeldcz4wTSTXBRryoq3Gc70ye7m7UD5Ww0nE29IXqMl6r7Nd15WBgRd74vloEMlShtH6CKxVzfmQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "regenerator-transform": "^0.15.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-reserved-words": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.18.6.tgz", + "integrity": "sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-shorthand-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.18.6.tgz", + "integrity": "sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-spread": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.18.6.tgz", + "integrity": "sha512-ayT53rT/ENF8WWexIRg9AiV9h0aIteyWn5ptfZTZQrjk/+f3WdrJGCY4c9wcgl2+MKkKPhzbYp97FTsquZpDCw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-sticky-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.18.6.tgz", + "integrity": "sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-template-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.18.6.tgz", + "integrity": "sha512-UuqlRrQmT2SWRvahW46cGSany0uTlcj8NYOS5sRGYi8FxPYPoLd5DDmMd32ZXEj2Jq+06uGVQKHxa/hJx2EzKw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typeof-symbol": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.18.6.tgz", + "integrity": "sha512-7m71iS/QhsPk85xSjFPovHPcH3H9qeyzsujhTc+vcdnsXavoWYJ74zx0lP5RhpC5+iDnVLO+PPMHzC11qels1g==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.18.6.tgz", + "integrity": "sha512-ijHNhzIrLj5lQCnI6aaNVRtGVuUZhOXFLRVFs7lLrkXTHip4FKty5oAuQdk4tywG0/WjXmjTfQCWmuzrvFer1w==", + "dev": true, + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-typescript": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-escapes": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.18.6.tgz", + "integrity": "sha512-XNRwQUXYMP7VLuy54cr/KS/WeL3AZeORhrmeZ7iewgu+X2eBqmpaLI/hzqr9ZxCeUoq0ASK4GUzSM0BDhZkLFw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.18.6.tgz", + "integrity": "sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-env": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.18.6.tgz", + "integrity": "sha512-WrthhuIIYKrEFAwttYzgRNQ5hULGmwTj+D6l7Zdfsv5M7IWV/OZbUfbeL++Qrzx1nVJwWROIFhCHRYQV4xbPNw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.18.6", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.18.6", + "@babel/plugin-proposal-async-generator-functions": "^7.18.6", + "@babel/plugin-proposal-class-properties": "^7.18.6", + "@babel/plugin-proposal-class-static-block": "^7.18.6", + "@babel/plugin-proposal-dynamic-import": "^7.18.6", + "@babel/plugin-proposal-export-namespace-from": "^7.18.6", + "@babel/plugin-proposal-json-strings": "^7.18.6", + "@babel/plugin-proposal-logical-assignment-operators": "^7.18.6", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", + "@babel/plugin-proposal-numeric-separator": "^7.18.6", + "@babel/plugin-proposal-object-rest-spread": "^7.18.6", + "@babel/plugin-proposal-optional-catch-binding": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.6", + "@babel/plugin-proposal-private-methods": "^7.18.6", + "@babel/plugin-proposal-private-property-in-object": "^7.18.6", + "@babel/plugin-proposal-unicode-property-regex": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3", + "@babel/plugin-syntax-import-assertions": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5", + "@babel/plugin-transform-arrow-functions": "^7.18.6", + "@babel/plugin-transform-async-to-generator": "^7.18.6", + "@babel/plugin-transform-block-scoped-functions": "^7.18.6", + "@babel/plugin-transform-block-scoping": "^7.18.6", + "@babel/plugin-transform-classes": "^7.18.6", + "@babel/plugin-transform-computed-properties": "^7.18.6", + "@babel/plugin-transform-destructuring": "^7.18.6", + "@babel/plugin-transform-dotall-regex": "^7.18.6", + "@babel/plugin-transform-duplicate-keys": "^7.18.6", + "@babel/plugin-transform-exponentiation-operator": "^7.18.6", + "@babel/plugin-transform-for-of": "^7.18.6", + "@babel/plugin-transform-function-name": "^7.18.6", + "@babel/plugin-transform-literals": "^7.18.6", + "@babel/plugin-transform-member-expression-literals": "^7.18.6", + "@babel/plugin-transform-modules-amd": "^7.18.6", + "@babel/plugin-transform-modules-commonjs": "^7.18.6", + "@babel/plugin-transform-modules-systemjs": "^7.18.6", + "@babel/plugin-transform-modules-umd": "^7.18.6", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.18.6", + "@babel/plugin-transform-new-target": "^7.18.6", + "@babel/plugin-transform-object-super": "^7.18.6", + "@babel/plugin-transform-parameters": "^7.18.6", + "@babel/plugin-transform-property-literals": "^7.18.6", + "@babel/plugin-transform-regenerator": "^7.18.6", + "@babel/plugin-transform-reserved-words": "^7.18.6", + "@babel/plugin-transform-shorthand-properties": "^7.18.6", + "@babel/plugin-transform-spread": "^7.18.6", + "@babel/plugin-transform-sticky-regex": "^7.18.6", + "@babel/plugin-transform-template-literals": "^7.18.6", + "@babel/plugin-transform-typeof-symbol": "^7.18.6", + "@babel/plugin-transform-unicode-escapes": "^7.18.6", + "@babel/plugin-transform-unicode-regex": "^7.18.6", + "@babel/preset-modules": "^0.1.5", + "@babel/types": "^7.18.6", + "babel-plugin-polyfill-corejs2": "^0.3.1", + "babel-plugin-polyfill-corejs3": "^0.5.2", + "babel-plugin-polyfill-regenerator": "^0.3.1", + "core-js-compat": "^3.22.1", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-flow": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-flow/-/preset-flow-7.18.6.tgz", + "integrity": "sha512-E7BDhL64W6OUqpuyHnSroLnqyRTcG6ZdOBl1OKI/QK/HJfplqK/S3sq1Cckx7oTodJ5yOXyfw7rEADJ6UjoQDQ==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-transform-flow-strip-types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-modules": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.5.tgz", + "integrity": "sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.4.4", + "@babel/plugin-transform-dotall-regex": "^7.4.4", + "@babel/types": "^7.4.4", + "esutils": "^2.0.2" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.18.6.tgz", + "integrity": "sha512-s9ik86kXBAnD760aybBucdpnLsAt0jK1xqJn2juOn9lkOvSHV60os5hxoVJsPzMQxvnUJFAlkont2DvvaYEBtQ==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-transform-typescript": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/register": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.18.6.tgz", + "integrity": "sha512-tkYtONzaO8rQubZzpBnvZPFcHgh8D9F55IjOsYton4X2IBoyRn2ZSWQqySTZnUn2guZbxbQiAB27hJEbvXamhQ==", + "dev": true, + "dependencies": { + "clone-deep": "^4.0.1", + "find-cache-dir": "^2.0.0", + "make-dir": "^2.1.0", + "pirates": "^4.0.5", + "source-map-support": "^0.5.16" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.6.tgz", + "integrity": "sha512-t9wi7/AW6XtKahAe20Yw0/mMljKq0B1r2fPdvaAdV/KPDZewFXdaaa6K7lxmZBZ8FBNpCiAT6iHPmd6QO9bKfQ==", + "dev": true, + "peer": true, + "dependencies": { + "regenerator-runtime": "^0.13.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.6.tgz", + "integrity": "sha512-JoDWzPe+wgBsTTgdnIma3iHNFC7YVJoPssVBDjiHfNlyt4YcunDtcDOUmfVDfCK5MfdsaIoX9PkijPhjH3nYUw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.18.6.tgz", + "integrity": "sha512-zS/OKyqmD7lslOtFqbscH6gMLFYOfG1YPqCKfAW5KrTeolKqvB8UelR49Fpr6y93kYkW2Ik00mT1LOGiAGvizw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/types": "^7.18.6", + "debug": "^4.1.0", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.18.6.tgz", + "integrity": "sha512-NdBNzPDwed30fZdDQtVR7ZgaO4UKjuaQFH9VArS+HMnurlOY0JWN+4ROlu/iapMFwjRQU4pOG4StZfDmulEwGA==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.18.6", + "to-fast-properties": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", + "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.0.0", + "@jridgewell/sourcemap-codec": "^1.4.10" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.8.tgz", + "integrity": "sha512-YK5G9LaddzGbcucK4c8h5tWFmMPBvRZ/uyWmN1/SbBdIvqGUdWGkJ5BAaccgs6XbzVLsqbPJrBSFwKv3kT9i7w==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.14", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.14.tgz", + "integrity": "sha512-bJWEfQ9lPTvm3SneWwRFVLzrh6nhjwqw7TUFFBEMzwvg7t7PCDenf2lDwqo4NQXzdpgBXyFgDWnQA+2vkruksQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/arr-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", + "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/arr-flatten": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-unique": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", + "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/assign-symbols": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", + "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ast-types": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.14.2.tgz", + "integrity": "sha512-O0yuUDnZeQDL+ncNGlJ78BiO4jnYI3bvMsD5prT0/nsgijG/LpNBIr63gTjVTNsiGkgQhiyCShTgxt8oXOrklA==", + "dev": true, + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/atob": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "dev": true, + "bin": { + "atob": "bin/atob.js" + }, + "engines": { + "node": ">= 4.5.0" + } + }, + "node_modules/babel-core": { + "version": "7.0.0-bridge.0", + "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz", + "integrity": "sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==", + "dev": true, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/babel-plugin-dynamic-import-node": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", + "dev": true, + "dependencies": { + "object.assign": "^4.1.0" + } + }, + "node_modules/babel-plugin-polyfill-corejs2": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.1.tgz", + "integrity": "sha512-v7/T6EQcNfVLfcN2X8Lulb7DjprieyLWJK/zOWH5DUYcAgex9sP3h25Q+DLsX9TloXe3y1O8l2q2Jv9q8UVB9w==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/compat-data": "^7.13.11", + "@babel/helper-define-polyfill-provider": "^0.3.1", + "semver": "^6.1.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/babel-plugin-polyfill-corejs3": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.5.2.tgz", + "integrity": "sha512-G3uJih0XWiID451fpeFaYGVuxHEjzKTHtc9uGFEjR6hHrvNzeS/PX+LLLcetJcytsB5m4j+K3o/EpXJNb/5IEQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.3.1", + "core-js-compat": "^3.21.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/babel-plugin-polyfill-regenerator": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.3.1.tgz", + "integrity": "sha512-Y2B06tvgHYt1x0yz17jGkGeeMr5FeKUu+ASJ+N6nB5lQ8Dapfg42i0OVrf8PNGJ3zKL4A23snMi1IRwrqqND7A==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.3.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/base": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "dev": true, + "dependencies": { + "cache-base": "^1.0.1", + "class-utils": "^0.3.5", + "component-emitter": "^1.2.1", + "define-property": "^1.0.0", + "isobject": "^3.0.1", + "mixin-deep": "^1.2.0", + "pascalcase": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, + "dependencies": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/braces/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/braces/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/browserslist": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.0.tgz", + "integrity": "sha512-UQxE0DIhRB5z/zDz9iA03BOfxaN2+GQdBYH/2WrSIWEUrnpzTPJbhqt+umq6r3acaPRTW1FNTkrcp0PXgtFkvA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001358", + "electron-to-chromium": "^1.4.164", + "node-releases": "^2.0.5", + "update-browserslist-db": "^1.0.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, + "node_modules/cache-base": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "dev": true, + "dependencies": { + "collection-visit": "^1.0.0", + "component-emitter": "^1.2.1", + "get-value": "^2.0.6", + "has-value": "^1.0.0", + "isobject": "^3.0.1", + "set-value": "^2.0.0", + "to-object-path": "^0.3.0", + "union-value": "^1.0.0", + "unset-value": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001359", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001359.tgz", + "integrity": "sha512-Xln/BAsPzEuiVLgJ2/45IaqD9jShtk3Y33anKb4+yLwQzws3+v6odKfpgES/cDEaZMLzSChpIGdbOYtH9MyuHw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + } + ] + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/class-utils": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "dev": true, + "dependencies": { + "arr-union": "^3.1.0", + "define-property": "^0.2.5", + "isobject": "^3.0.0", + "static-extend": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/collection-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", + "integrity": "sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==", + "dev": true, + "dependencies": { + "map-visit": "^1.0.0", + "object-visit": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true + }, + "node_modules/component-emitter": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/convert-source-map": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz", + "integrity": "sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.1" + } + }, + "node_modules/copy-descriptor": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", + "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/core-js-compat": { + "version": "3.23.3", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.23.3.tgz", + "integrity": "sha512-WSzUs2h2vvmKsacLHNTdpyOC9k43AEhcGoFlVgCY4L7aw98oSBKtPL6vD0/TqZjRWRQYdDSLkzZIni4Crbbiqw==", + "dev": true, + "peer": true, + "dependencies": { + "browserslist": "^4.21.0", + "semver": "7.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-js-compat/node_modules/semver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz", + "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==", + "dev": true, + "peer": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "dev": true, + "dependencies": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-property": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.2", + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.4.172", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.172.tgz", + "integrity": "sha512-yDoFfTJnqBAB6hSiPvzmsBJSrjOXJtHSJoqJdI/zSIh7DYupYnIOHt/bbPw/WE31BJjNTybDdNAs21gCMnTh0Q==", + "dev": true + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", + "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==", + "dev": true, + "dependencies": { + "debug": "^2.3.3", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "posix-character-classes": "^0.1.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/expand-brackets/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/extend-shallow": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", + "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==", + "dev": true, + "dependencies": { + "assign-symbols": "^1.0.0", + "is-extendable": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", + "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dev": true, + "dependencies": { + "array-unique": "^0.3.2", + "define-property": "^1.0.0", + "expand-brackets": "^2.1.4", + "extend-shallow": "^2.0.1", + "fragment-cache": "^0.2.1", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fill-range/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fill-range/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/flow-parser": { + "version": "0.181.1", + "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.181.1.tgz", + "integrity": "sha512-+Mx87/GkmF5+FHk8IXc5WppD/oC4wB+05MuIv7qmIMgThND3RhOBGl7Npyc2L7NLVenme00ZlwEKVieiMz4bqA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fragment-cache": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", + "integrity": "sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==", + "dev": true, + "dependencies": { + "map-cache": "^0.2.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-value": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", + "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "dev": true + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", + "integrity": "sha512-IBXk4GTsLYdQ7Rvt+GRBrFSVEkmuOUy4re0Xjd9kJSUQpnTrWR4/y9RpfexN9vkAPMFuQoeWKwqzPozRTlasGw==", + "dev": true, + "dependencies": { + "get-value": "^2.0.6", + "has-values": "^1.0.0", + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", + "integrity": "sha512-ODYZC64uqzmtfGMEAX/FvZiRyWLpAC3vYnNunURUnkGVTS+mI0smVsWaPydRBsE3g+ok7h960jChO8mFcWlHaQ==", + "dev": true, + "dependencies": { + "is-number": "^3.0.0", + "kind-of": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values/node_modules/kind-of": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", + "integrity": "sha512-24XsCxmEbRwEDbz/qz3stgin8TTzZ1ESR56OMCN0ujYg+vRutNSiOj9bHH9u85DKgXguraugV5sFuvbD4FW/hw==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true + }, + "node_modules/is-core-module": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", + "integrity": "sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A==", + "dev": true, + "peer": true, + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "node_modules/jscodeshift": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/jscodeshift/-/jscodeshift-0.13.1.tgz", + "integrity": "sha512-lGyiEbGOvmMRKgWk4vf+lUrCWO/8YR8sUR3FKF1Cq5fovjZDlIcw3Hu5ppLHAnEXshVffvaM0eyuY/AbOeYpnQ==", + "dev": true, + "dependencies": { + "@babel/core": "^7.13.16", + "@babel/parser": "^7.13.16", + "@babel/plugin-proposal-class-properties": "^7.13.0", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", + "@babel/plugin-proposal-optional-chaining": "^7.13.12", + "@babel/plugin-transform-modules-commonjs": "^7.13.8", + "@babel/preset-flow": "^7.13.13", + "@babel/preset-typescript": "^7.13.0", + "@babel/register": "^7.13.16", + "babel-core": "^7.0.0-bridge.0", + "chalk": "^4.1.2", + "flow-parser": "0.*", + "graceful-fs": "^4.2.4", + "micromatch": "^3.1.10", + "neo-async": "^2.5.0", + "node-dir": "^0.1.17", + "recast": "^0.20.4", + "temp": "^0.8.4", + "write-file-atomic": "^2.3.0" + }, + "bin": { + "jscodeshift": "bin/jscodeshift.js" + }, + "peerDependencies": { + "@babel/preset-env": "^7.1.6" + } + }, + "node_modules/jscodeshift/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jscodeshift/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/jscodeshift/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/jscodeshift/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/jscodeshift/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/jscodeshift/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true, + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "dev": true, + "peer": true + }, + "node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/map-cache": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", + "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/map-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", + "integrity": "sha512-4y7uGv8bd2WdM9vpQsiQNo41Ln1NvhvDRuVt0k2JZQ+ezN2uaQes7lZeZ+QQUHOLQAtDaBJ+7wCbi+ab/KFs+w==", + "dev": true, + "dependencies": { + "object-visit": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mixin-deep": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "dev": true, + "dependencies": { + "for-in": "^1.0.2", + "is-extendable": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/nanomatch": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "dev": true, + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "fragment-cache": "^0.2.1", + "is-windows": "^1.0.2", + "kind-of": "^6.0.2", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true + }, + "node_modules/node-dir": { + "version": "0.1.17", + "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", + "integrity": "sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==", + "dev": true, + "dependencies": { + "minimatch": "^3.0.2" + }, + "engines": { + "node": ">= 0.10.5" + } + }, + "node_modules/node-releases": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.5.tgz", + "integrity": "sha512-U9h1NLROZTq9uE1SNffn6WuPDg8icmi3ns4rEl/oTfIle4iLjTliCzgTsbaIFMq/Xn078/lfY/BL0GWZ+psK4Q==", + "dev": true + }, + "node_modules/object-copy": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", + "integrity": "sha512-79LYn6VAb63zgtmAteVOWo9Vdj71ZVBy3Pbse+VqxDpEP83XuujMrGqHIwAXJ5I/aM0zU7dIyIAhifVTPrNItQ==", + "dev": true, + "dependencies": { + "copy-descriptor": "^0.1.0", + "define-property": "^0.2.5", + "kind-of": "^3.0.3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/is-descriptor/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object-visit": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", + "integrity": "sha512-GBaMwwAVK9qbQN3Scdo0OyvgPW7l3lnaVMj84uTOZlswkX0KpF6fyDBJhtTthf7pymztoN36/KEr1DyhF96zEA==", + "dev": true, + "dependencies": { + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.pick": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", + "integrity": "sha512-tqa/UMy/CCoYmj+H5qc07qvSL9dqcs/WZENZ1JbtWBlATP+iVOe778gE6MSijnyCnORzDuX6hU+LA4SZ09YjFQ==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pascalcase": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", + "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "peer": true + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pirates": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz", + "integrity": "sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/posix-character-classes": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", + "integrity": "sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/recast": { + "version": "0.20.5", + "resolved": "https://registry.npmjs.org/recast/-/recast-0.20.5.tgz", + "integrity": "sha512-E5qICoPoNL4yU0H0NoBDntNB0Q5oMSNh9usFctYniLBluTthi3RsQVBXIJNbApOlvSwW/RGxIuokPcAc59J5fQ==", + "dev": true, + "dependencies": { + "ast-types": "0.14.2", + "esprima": "~4.0.0", + "source-map": "~0.6.1", + "tslib": "^2.0.1" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==", + "dev": true, + "peer": true + }, + "node_modules/regenerate-unicode-properties": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.0.1.tgz", + "integrity": "sha512-vn5DU6yg6h8hP/2OkQo3K7uVILvY4iu0oI4t3HFa81UPkhGJwkRwM10JEc3upjdhHjs/k8GJY1sRBhk5sr69Bw==", + "dev": true, + "peer": true, + "dependencies": { + "regenerate": "^1.4.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==", + "dev": true, + "peer": true + }, + "node_modules/regenerator-transform": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.0.tgz", + "integrity": "sha512-LsrGtPmbYg19bcPHwdtmXwbW+TqNvtY4riE3P83foeHRroMbH6/2ddFBfab3t7kbzc7v7p4wbkIecHImqt0QNg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/runtime": "^7.8.4" + } + }, + "node_modules/regex-not": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "dev": true, + "dependencies": { + "extend-shallow": "^3.0.2", + "safe-regex": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/regexpu-core": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-5.1.0.tgz", + "integrity": "sha512-bb6hk+xWd2PEOkj5It46A16zFMs2mv86Iwpdu94la4S3sJ7C973h2dHpYKwIBGaWSO7cIRJ+UX0IeMaWcO4qwA==", + "dev": true, + "peer": true, + "dependencies": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.0.1", + "regjsgen": "^0.6.0", + "regjsparser": "^0.8.2", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regjsgen": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.6.0.tgz", + "integrity": "sha512-ozE883Uigtqj3bx7OhL1KNbCzGyW2NQZPl6Hs09WTvCuZD5sTI4JY58bkbQWa/Y9hxIsvJ3M8Nbf7j54IqeZbA==", + "dev": true, + "peer": true + }, + "node_modules/regjsparser": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.8.4.tgz", + "integrity": "sha512-J3LABycON/VNEu3abOviqGHuB/LOtOQj8SKmfP9anY5GfAVw/SPjwzSjxGjbZXIxbGfqTHtJw58C2Li/WkStmA==", + "dev": true, + "peer": true, + "dependencies": { + "jsesc": "~0.5.0" + }, + "bin": { + "regjsparser": "bin/parser" + } + }, + "node_modules/regjsparser/node_modules/jsesc": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "integrity": "sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==", + "dev": true, + "peer": true, + "bin": { + "jsesc": "bin/jsesc" + } + }, + "node_modules/repeat-element": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.4.tgz", + "integrity": "sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/resolve": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "dev": true, + "peer": true, + "dependencies": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==", + "deprecated": "https://github.com/lydell/resolve-url#deprecated", + "dev": true + }, + "node_modules/ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "dev": true, + "engines": { + "node": ">=0.12" + } + }, + "node_modules/rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/safe-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha512-aJXcif4xnaNUzvUuC5gcb46oTS7zvg4jpMTnuqtrEPlR3vFr4pxtdTwaF1Qs3Enjn9HK+ZlwQui+a7z0SywIzg==", + "dev": true, + "dependencies": { + "ret": "~0.1.10" + } + }, + "node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/set-value": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "is-extendable": "^0.1.1", + "is-plain-object": "^2.0.3", + "split-string": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/set-value/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/set-value/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "node_modules/snapdragon": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "dev": true, + "dependencies": { + "base": "^0.11.1", + "debug": "^2.2.0", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "map-cache": "^0.2.2", + "source-map": "^0.5.6", + "source-map-resolve": "^0.5.0", + "use": "^3.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "dev": true, + "dependencies": { + "define-property": "^1.0.0", + "isobject": "^3.0.0", + "snapdragon-util": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-util": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "dev": true, + "dependencies": { + "kind-of": "^3.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-util/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/snapdragon/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/snapdragon/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-resolve": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", + "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "deprecated": "See https://github.com/lydell/source-map-resolve#deprecated", + "dev": true, + "dependencies": { + "atob": "^2.1.2", + "decode-uri-component": "^0.2.0", + "resolve-url": "^0.2.1", + "source-map-url": "^0.4.0", + "urix": "^0.1.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/source-map-url": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.1.tgz", + "integrity": "sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw==", + "deprecated": "See https://github.com/lydell/source-map-url#deprecated", + "dev": true + }, + "node_modules/split-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "dev": true, + "dependencies": { + "extend-shallow": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", + "integrity": "sha512-72E9+uLc27Mt718pMHt9VMNiAL4LMsmDbBva8mxWUCkT07fSzEGMYUCk0XWY6lp0j6RBAG4cJ3mWuZv2OE3s0g==", + "dev": true, + "dependencies": { + "define-property": "^0.2.5", + "object-copy": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/temp": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/temp/-/temp-0.8.4.tgz", + "integrity": "sha512-s0ZZzd0BzYv5tLSptZooSjK8oj6C+c19p7Vqta9+6NPOf7r+fxq0cJe6/oN4LTC79sy5NY8ucOJNgwsKCSbfqg==", + "dev": true, + "dependencies": { + "rimraf": "~2.6.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/to-object-path": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", + "integrity": "sha512-9mWHdnGRuh3onocaHzukyvCZhzvr6tiflAy/JRFXcJX0TjgfWA9pk9t8CMbzmBE4Jfw58pXbkngtBtqYxzNEyg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-object-path/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-regex": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "dev": true, + "dependencies": { + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "regex-not": "^1.0.2", + "safe-regex": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==", + "dev": true, + "dependencies": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, + "node_modules/unicode-canonical-property-names-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz", + "integrity": "sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "dev": true, + "peer": true, + "dependencies": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-value-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.0.0.tgz", + "integrity": "sha512-7Yhkc0Ye+t4PNYzOGKedDhXbYIBe1XEQYQxOPyhcXNMJ0WCABqqj6ckydd6pWRZTHV4GuCPKdBAUiMc60tsKVw==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-property-aliases-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.0.0.tgz", + "integrity": "sha512-5Zfuy9q/DFr4tfO7ZPeVXb1aPoeQSdeFMLpYuFebehDAhbuevLs5yxSZmIFN1tP5F9Wl4IpJrYojg85/zgyZHQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/union-value": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "dev": true, + "dependencies": { + "arr-union": "^3.1.0", + "get-value": "^2.0.6", + "is-extendable": "^0.1.1", + "set-value": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/union-value/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", + "integrity": "sha512-PcA2tsuGSF9cnySLHTLSh2qrQiJ70mn+r+Glzxv2TWZblxsxCC52BDlZoPCsz7STd9pN7EZetkWZBAvk4cgZdQ==", + "dev": true, + "dependencies": { + "has-value": "^0.3.1", + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-value": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", + "integrity": "sha512-gpG936j8/MzaeID5Yif+577c17TxaDmhuyVgSwtnL/q8UUTySg8Mecb+8Cf1otgLoD7DDH75axp86ER7LFsf3Q==", + "dev": true, + "dependencies": { + "get-value": "^2.0.3", + "has-values": "^0.1.4", + "isobject": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-value/node_modules/isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "dev": true, + "dependencies": { + "isarray": "1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-values": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", + "integrity": "sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.4.tgz", + "integrity": "sha512-jnmO2BEGUjsMOe/Fg9u0oczOe/ppIDZPebzccl1yDWGLFP16Pa1/RM5wEoKYPG2zstNcDuAStejyxsOuKINdGA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + }, + "bin": { + "browserslist-lint": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/urix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", + "integrity": "sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==", + "deprecated": "Please see https://github.com/lydell/urix#deprecated", + "dev": true + }, + "node_modules/use": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + } + }, + "dependencies": { + "@ampproject/remapping": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", + "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", + "dev": true, + "requires": { + "@jridgewell/gen-mapping": "^0.1.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@babel/code-frame": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", + "integrity": "sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==", + "dev": true, + "requires": { + "@babel/highlight": "^7.18.6" + } + }, + "@babel/compat-data": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.18.6.tgz", + "integrity": "sha512-tzulrgDT0QD6U7BJ4TKVk2SDDg7wlP39P9yAx1RfLy7vP/7rsDRlWVfbWxElslu56+r7QOhB2NSDsabYYruoZQ==", + "dev": true + }, + "@babel/core": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.18.6.tgz", + "integrity": "sha512-cQbWBpxcbbs/IUredIPkHiAGULLV8iwgNRMFzvbhEXISp4f3rUUXE5+TIw6KwUWUR3DwyI6gmBRnmAtYaWehwQ==", + "dev": true, + "requires": { + "@ampproject/remapping": "^2.1.0", + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helpers": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.1", + "semver": "^6.3.0" + } + }, + "@babel/generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.18.6.tgz", + "integrity": "sha512-AIwwoOS8axIC5MZbhNHRLKi3D+DMpvDf9XUcu3pIVAfOHFT45f4AoDAltRbHIQomCipkCZxrNkfpOEHhJz/VKw==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6", + "@jridgewell/gen-mapping": "^0.3.0", + "jsesc": "^2.5.1" + }, + "dependencies": { + "@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dev": true, + "requires": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + } + } + } + }, + "@babel/helper-annotate-as-pure": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz", + "integrity": "sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-builder-binary-assignment-operator-visitor": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.18.6.tgz", + "integrity": "sha512-KT10c1oWEpmrIRYnthbzHgoOf6B+Xd6a5yhdbNtdhtG7aO1or5HViuf1TQR36xY/QprXA5nvxO6nAjhJ4y38jw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-explode-assignable-expression": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-compilation-targets": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.18.6.tgz", + "integrity": "sha512-vFjbfhNCzqdeAtZflUFrG5YIFqGTqsctrtkZ1D/NB0mDW9TwW3GmmUepYY4G9wCET5rY5ugz4OGTcLd614IzQg==", + "dev": true, + "requires": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "browserslist": "^4.20.2", + "semver": "^6.3.0" + } + }, + "@babel/helper-create-class-features-plugin": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.18.6.tgz", + "integrity": "sha512-YfDzdnoxHGV8CzqHGyCbFvXg5QESPFkXlHtvdCkesLjjVMT2Adxe4FGUR5ChIb3DxSaXO12iIOCWoXdsUVwnqw==", + "dev": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-member-expression-to-functions": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6" + } + }, + "@babel/helper-create-regexp-features-plugin": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.18.6.tgz", + "integrity": "sha512-7LcpH1wnQLGrI+4v+nPp+zUvIkF9x0ddv1Hkdue10tg3gmRnLy97DXh4STiOf1qeIInyD69Qv5kKSZzKD8B/7A==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "regexpu-core": "^5.1.0" + } + }, + "@babel/helper-define-polyfill-provider": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.1.tgz", + "integrity": "sha512-J9hGMpJQmtWmj46B3kBHmL38UhJGhYX7eqkcq+2gsstyYt341HmPeWspihX43yVRA0mS+8GGk2Gckc7bY/HCmA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-compilation-targets": "^7.13.0", + "@babel/helper-module-imports": "^7.12.13", + "@babel/helper-plugin-utils": "^7.13.0", + "@babel/traverse": "^7.13.0", + "debug": "^4.1.1", + "lodash.debounce": "^4.0.8", + "resolve": "^1.14.2", + "semver": "^6.1.2" + } + }, + "@babel/helper-environment-visitor": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.6.tgz", + "integrity": "sha512-8n6gSfn2baOY+qlp+VSzsosjCVGFqWKmDF0cCWOybh52Dw3SEyoWR1KrhMJASjLwIEkkAufZ0xvr+SxLHSpy2Q==", + "dev": true + }, + "@babel/helper-explode-assignable-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz", + "integrity": "sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==", + "dev": true, + "peer": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-function-name": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.18.6.tgz", + "integrity": "sha512-0mWMxV1aC97dhjCah5U5Ua7668r5ZmSC2DLfH2EZnf9c3/dHZKiFa5pRLMH5tjSl471tY6496ZWk/kjNONBxhw==", + "dev": true, + "requires": { + "@babel/template": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-hoist-variables": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz", + "integrity": "sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-member-expression-to-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.18.6.tgz", + "integrity": "sha512-CeHxqwwipekotzPDUuJOfIMtcIHBuc7WAzLmTYWctVigqS5RktNMQ5bEwQSuGewzYnCtTWa3BARXeiLxDTv+Ng==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-module-imports": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz", + "integrity": "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-module-transforms": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.18.6.tgz", + "integrity": "sha512-L//phhB4al5uucwzlimruukHB3jRd5JGClwRMD/ROrVjXfLqovYnvQrK/JK36WYyVwGGO7OD3kMyVTjx+WVPhw==", + "dev": true, + "requires": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/helper-validator-identifier": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-optimise-call-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.18.6.tgz", + "integrity": "sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-plugin-utils": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.18.6.tgz", + "integrity": "sha512-gvZnm1YAAxh13eJdkb9EWHBnF3eAub3XTLCZEehHT2kWxiKVRL64+ae5Y6Ivne0mVHmMYKT+xWgZO+gQhuLUBg==", + "dev": true + }, + "@babel/helper-remap-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.18.6.tgz", + "integrity": "sha512-z5wbmV55TveUPZlCLZvxWHtrjuJd+8inFhk7DG0WW87/oJuGDcjDiu7HIvGcpf5464L6xKCg3vNkmlVVz9hwyQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-wrap-function": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-replace-supers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.18.6.tgz", + "integrity": "sha512-fTf7zoXnUGl9gF25fXCWE26t7Tvtyn6H4hkLSYhATwJvw2uYxd3aoXplMSe0g9XbwK7bmxNes7+FGO0rB/xC0g==", + "dev": true, + "requires": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-member-expression-to-functions": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-simple-access": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.18.6.tgz", + "integrity": "sha512-iNpIgTgyAvDQpDj76POqg+YEt8fPxx3yaNBg3S30dxNKm2SWfYhD0TGrK/Eu9wHpUW63VQU894TsTg+GLbUa1g==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.18.6.tgz", + "integrity": "sha512-4KoLhwGS9vGethZpAhYnMejWkX64wsnHPDwvOsKWU6Fg4+AlK2Jz3TyjQLMEPvz+1zemi/WBdkYxCD0bAfIkiw==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz", + "integrity": "sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.18.6.tgz", + "integrity": "sha512-MmetCkz9ej86nJQV+sFCxoGGrUbU3q02kgLciwkrt9QqEB7cP39oKEY0PakknEO0Gu20SskMRi+AYZ3b1TpN9g==", + "dev": true + }, + "@babel/helper-validator-option": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz", + "integrity": "sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==", + "dev": true + }, + "@babel/helper-wrap-function": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.18.6.tgz", + "integrity": "sha512-I5/LZfozwMNbwr/b1vhhuYD+J/mU+gfGAj5td7l5Rv9WYmH6i3Om69WGKNmlIpsVW/mF6O5bvTKbvDQZVgjqOw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-function-name": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helpers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.18.6.tgz", + "integrity": "sha512-vzSiiqbQOghPngUYt/zWGvK3LAsPhz55vc9XNN0xAl2gV4ieShI2OQli5duxWHD+72PZPTKAcfcZDE1Cwc5zsQ==", + "dev": true, + "requires": { + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/highlight": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz", + "integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.18.6", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, + "@babel/parser": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.18.6.tgz", + "integrity": "sha512-uQVSa9jJUe/G/304lXspfWVpKpK4euFLgGiMQFOCpM/bgcAdeoHwi/OQz23O9GK2osz26ZiXRRV9aV+Yl1O8tw==", + "dev": true + }, + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz", + "integrity": "sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.18.6.tgz", + "integrity": "sha512-Udgu8ZRgrBrttVz6A0EVL0SJ1z+RLbIeqsu632SA1hf0awEppD6TvdznoH+orIF8wtFFAV/Enmw9Y+9oV8TQcw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.6" + } + }, + "@babel/plugin-proposal-async-generator-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.18.6.tgz", + "integrity": "sha512-WAz4R9bvozx4qwf74M+sfqPMKfSqwM0phxPTR6iJIi8robgzXwkEgmeJG1gEKhm6sDqT/U9aV3lfcqybIpev8w==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4" + } + }, + "@babel/plugin-proposal-class-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz", + "integrity": "sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==", + "dev": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-proposal-class-static-block": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.18.6.tgz", + "integrity": "sha512-+I3oIiNxrCpup3Gi8n5IGMwj0gOCAjcJUSQEcotNnCCPMEnixawOQ+KeJPlgfjzx+FKQ1QSyZOWe7wmoJp7vhw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-class-static-block": "^7.14.5" + } + }, + "@babel/plugin-proposal-dynamic-import": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.18.6.tgz", + "integrity": "sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-dynamic-import": "^7.8.3" + } + }, + "@babel/plugin-proposal-export-namespace-from": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.6.tgz", + "integrity": "sha512-zr/QcUlUo7GPo6+X1wC98NJADqmy5QTFWWhqeQWiki4XHafJtLl/YMGkmRB2szDD2IYJCCdBTd4ElwhId9T7Xw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3" + } + }, + "@babel/plugin-proposal-json-strings": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.18.6.tgz", + "integrity": "sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3" + } + }, + "@babel/plugin-proposal-logical-assignment-operators": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.18.6.tgz", + "integrity": "sha512-zMo66azZth/0tVd7gmkxOkOjs2rpHyhpcFo565PUP37hSp6hSd9uUKIfTDFMz58BwqgQKhJ9YxtM5XddjXVn+Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4" + } + }, + "@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz", + "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3" + } + }, + "@babel/plugin-proposal-numeric-separator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz", + "integrity": "sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-numeric-separator": "^7.10.4" + } + }, + "@babel/plugin-proposal-object-rest-spread": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.18.6.tgz", + "integrity": "sha512-9yuM6wr4rIsKa1wlUAbZEazkCrgw2sMPEXCr4Rnwetu7cEW1NydkCWytLuYletbf8vFxdJxFhwEZqMpOx2eZyw==", + "dev": true, + "peer": true, + "requires": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-transform-parameters": "^7.18.6" + } + }, + "@babel/plugin-proposal-optional-catch-binding": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.18.6.tgz", + "integrity": "sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3" + } + }, + "@babel/plugin-proposal-optional-chaining": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.18.6.tgz", + "integrity": "sha512-PatI6elL5eMzoypFAiYDpYQyMtXTn+iMhuxxQt5mAXD4fEmKorpSI3PHd+i3JXBJN3xyA6MvJv7at23HffFHwA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6", + "@babel/plugin-syntax-optional-chaining": "^7.8.3" + } + }, + "@babel/plugin-proposal-private-methods": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz", + "integrity": "sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-proposal-private-property-in-object": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.18.6.tgz", + "integrity": "sha512-9Rysx7FOctvT5ouj5JODjAFAkgGoudQuLPamZb0v1TGLpapdNaftzifU8NTWQm0IRjqoYypdrSmyWgkocDQ8Dw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5" + } + }, + "@babel/plugin-proposal-unicode-property-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.18.6.tgz", + "integrity": "sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.12.13" + } + }, + "@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.14.5" + } + }, + "@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-export-namespace-from": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz", + "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.3" + } + }, + "@babel/plugin-syntax-flow": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz", + "integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-syntax-import-assertions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.18.6.tgz", + "integrity": "sha512-/DU3RXad9+bZwrgWJQKbr39gYbJpLJHezqEzRzi/BHRlJ9zsQb4CK2CA/5apllXNomwA1qHwzvHl+AdEmC5krQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.14.5" + } + }, + "@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.14.5" + } + }, + "@babel/plugin-syntax-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.18.6.tgz", + "integrity": "sha512-mAWAuq4rvOepWCBid55JuRNvpTNf2UGVgoz4JV0fXEKolsVZDzsa4NqCef758WZJj/GDu0gVGItjKFiClTAmZA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-arrow-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.18.6.tgz", + "integrity": "sha512-9S9X9RUefzrsHZmKMbDXxweEH+YlE8JJEuat9FdvW9Qh1cw7W64jELCtWNkPBPX5En45uy28KGvA/AySqUh8CQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.18.6.tgz", + "integrity": "sha512-ARE5wZLKnTgPW7/1ftQmSi1CmkqqHo2DNmtztFhvgtOWSDfq0Cq9/9L+KnZNYSNrydBekhW3rwShduf59RoXag==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6" + } + }, + "@babel/plugin-transform-block-scoped-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.18.6.tgz", + "integrity": "sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-block-scoping": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.18.6.tgz", + "integrity": "sha512-pRqwb91C42vs1ahSAWJkxOxU1RHWDn16XAa6ggQ72wjLlWyYeAcLvTtE0aM8ph3KNydy9CQF2nLYcjq1WysgxQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-classes": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.18.6.tgz", + "integrity": "sha512-XTg8XW/mKpzAF3actL554Jl/dOYoJtv3l8fxaEczpgz84IeeVf+T1u2CSvPHuZbt0w3JkIx4rdn/MRQI7mo0HQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "globals": "^11.1.0" + } + }, + "@babel/plugin-transform-computed-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.18.6.tgz", + "integrity": "sha512-9repI4BhNrR0KenoR9vm3/cIc1tSBIo+u1WVjKCAynahj25O8zfbiE6JtAtHPGQSs4yZ+bA8mRasRP+qc+2R5A==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-destructuring": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.18.6.tgz", + "integrity": "sha512-tgy3u6lRp17ilY8r1kP4i2+HDUwxlVqq3RTc943eAWSzGgpU1qhiKpqZ5CMyHReIYPHdo3Kg8v8edKtDqSVEyQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-dotall-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.18.6.tgz", + "integrity": "sha512-6S3jpun1eEbAxq7TdjLotAsl4WpQI9DxfkycRcKrjhQYzU87qpXdknpBg/e+TdcMehqGnLFi7tnFUBR02Vq6wg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-duplicate-keys": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.18.6.tgz", + "integrity": "sha512-NJU26U/208+sxYszf82nmGYqVF9QN8py2HFTblPT9hbawi8+1C5a9JubODLTGFuT0qlkqVinmkwOD13s0sZktg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-exponentiation-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.18.6.tgz", + "integrity": "sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-builder-binary-assignment-operator-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-flow-strip-types": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.18.6.tgz", + "integrity": "sha512-wE0xtA7csz+hw4fKPwxmu5jnzAsXPIO57XnRwzXP3T19jWh1BODnPGoG9xKYwvAwusP7iUktHayRFbMPGtODaQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-flow": "^7.18.6" + } + }, + "@babel/plugin-transform-for-of": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.18.6.tgz", + "integrity": "sha512-WAjoMf4wIiSsy88KmG7tgj2nFdEK7E46tArVtcgED7Bkj6Fg/tG5SbvNIOKxbFS2VFgNh6+iaPswBeQZm4ox8w==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-function-name": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.18.6.tgz", + "integrity": "sha512-kJha/Gbs5RjzIu0CxZwf5e3aTTSlhZnHMT8zPWnJMjNpLOUgqevg+PN5oMH68nMCXnfiMo4Bhgxqj59KHTlAnA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.18.6.tgz", + "integrity": "sha512-x3HEw0cJZVDoENXOp20HlypIHfl0zMIhMVZEBVTfmqbObIpsMxMbmU5nOEO8R7LYT+z5RORKPlTI5Hj4OsO9/Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-member-expression-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.18.6.tgz", + "integrity": "sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-modules-amd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.18.6.tgz", + "integrity": "sha512-Pra5aXsmTsOnjM3IajS8rTaLCy++nGM4v3YR4esk5PCsyg9z8NA5oQLwxzMUtDBd8F+UmVza3VxoAaWCbzH1rg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, + "@babel/plugin-transform-modules-commonjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.18.6.tgz", + "integrity": "sha512-Qfv2ZOWikpvmedXQJDSbxNqy7Xr/j2Y8/KfijM0iJyKkBTmWuvCA1yeH1yDM7NJhBW/2aXxeucLj6i80/LAJ/Q==", + "dev": true, + "requires": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, + "@babel/plugin-transform-modules-systemjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.18.6.tgz", + "integrity": "sha512-UbPYpXxLjTw6w6yXX2BYNxF3p6QY225wcTkfQCy3OMnSlS/C3xGtwUjEzGkldb/sy6PWLiCQ3NbYfjWUTI3t4g==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-identifier": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, + "@babel/plugin-transform-modules-umd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.18.6.tgz", + "integrity": "sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.18.6.tgz", + "integrity": "sha512-UmEOGF8XgaIqD74bC8g7iV3RYj8lMf0Bw7NJzvnS9qQhM4mg+1WHKotUIdjxgD2RGrgFLZZPCFPFj3P/kVDYhg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-new-target": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.18.6.tgz", + "integrity": "sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-object-super": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.18.6.tgz", + "integrity": "sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6" + } + }, + "@babel/plugin-transform-parameters": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.18.6.tgz", + "integrity": "sha512-FjdqgMv37yVl/gwvzkcB+wfjRI8HQmc5EgOG9iGNvUY1ok+TjsoaMP7IqCDZBhkFcM5f3OPVMs6Dmp03C5k4/A==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-property-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.18.6.tgz", + "integrity": "sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-regenerator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.18.6.tgz", + "integrity": "sha512-poqRI2+qiSdeldcz4wTSTXBRryoq3Gc70ye7m7UD5Ww0nE29IXqMl6r7Nd15WBgRd74vloEMlShtH6CKxVzfmQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "regenerator-transform": "^0.15.0" + } + }, + "@babel/plugin-transform-reserved-words": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.18.6.tgz", + "integrity": "sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-shorthand-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.18.6.tgz", + "integrity": "sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-spread": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.18.6.tgz", + "integrity": "sha512-ayT53rT/ENF8WWexIRg9AiV9h0aIteyWn5ptfZTZQrjk/+f3WdrJGCY4c9wcgl2+MKkKPhzbYp97FTsquZpDCw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6" + } + }, + "@babel/plugin-transform-sticky-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.18.6.tgz", + "integrity": "sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-template-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.18.6.tgz", + "integrity": "sha512-UuqlRrQmT2SWRvahW46cGSany0uTlcj8NYOS5sRGYi8FxPYPoLd5DDmMd32ZXEj2Jq+06uGVQKHxa/hJx2EzKw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-typeof-symbol": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.18.6.tgz", + "integrity": "sha512-7m71iS/QhsPk85xSjFPovHPcH3H9qeyzsujhTc+vcdnsXavoWYJ74zx0lP5RhpC5+iDnVLO+PPMHzC11qels1g==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.18.6.tgz", + "integrity": "sha512-ijHNhzIrLj5lQCnI6aaNVRtGVuUZhOXFLRVFs7lLrkXTHip4FKty5oAuQdk4tywG0/WjXmjTfQCWmuzrvFer1w==", + "dev": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-typescript": "^7.18.6" + } + }, + "@babel/plugin-transform-unicode-escapes": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.18.6.tgz", + "integrity": "sha512-XNRwQUXYMP7VLuy54cr/KS/WeL3AZeORhrmeZ7iewgu+X2eBqmpaLI/hzqr9ZxCeUoq0ASK4GUzSM0BDhZkLFw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-unicode-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.18.6.tgz", + "integrity": "sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/preset-env": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.18.6.tgz", + "integrity": "sha512-WrthhuIIYKrEFAwttYzgRNQ5hULGmwTj+D6l7Zdfsv5M7IWV/OZbUfbeL++Qrzx1nVJwWROIFhCHRYQV4xbPNw==", + "dev": true, + "peer": true, + "requires": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.18.6", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.18.6", + "@babel/plugin-proposal-async-generator-functions": "^7.18.6", + "@babel/plugin-proposal-class-properties": "^7.18.6", + "@babel/plugin-proposal-class-static-block": "^7.18.6", + "@babel/plugin-proposal-dynamic-import": "^7.18.6", + "@babel/plugin-proposal-export-namespace-from": "^7.18.6", + "@babel/plugin-proposal-json-strings": "^7.18.6", + "@babel/plugin-proposal-logical-assignment-operators": "^7.18.6", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", + "@babel/plugin-proposal-numeric-separator": "^7.18.6", + "@babel/plugin-proposal-object-rest-spread": "^7.18.6", + "@babel/plugin-proposal-optional-catch-binding": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.6", + "@babel/plugin-proposal-private-methods": "^7.18.6", + "@babel/plugin-proposal-private-property-in-object": "^7.18.6", + "@babel/plugin-proposal-unicode-property-regex": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3", + "@babel/plugin-syntax-import-assertions": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5", + "@babel/plugin-transform-arrow-functions": "^7.18.6", + "@babel/plugin-transform-async-to-generator": "^7.18.6", + "@babel/plugin-transform-block-scoped-functions": "^7.18.6", + "@babel/plugin-transform-block-scoping": "^7.18.6", + "@babel/plugin-transform-classes": "^7.18.6", + "@babel/plugin-transform-computed-properties": "^7.18.6", + "@babel/plugin-transform-destructuring": "^7.18.6", + "@babel/plugin-transform-dotall-regex": "^7.18.6", + "@babel/plugin-transform-duplicate-keys": "^7.18.6", + "@babel/plugin-transform-exponentiation-operator": "^7.18.6", + "@babel/plugin-transform-for-of": "^7.18.6", + "@babel/plugin-transform-function-name": "^7.18.6", + "@babel/plugin-transform-literals": "^7.18.6", + "@babel/plugin-transform-member-expression-literals": "^7.18.6", + "@babel/plugin-transform-modules-amd": "^7.18.6", + "@babel/plugin-transform-modules-commonjs": "^7.18.6", + "@babel/plugin-transform-modules-systemjs": "^7.18.6", + "@babel/plugin-transform-modules-umd": "^7.18.6", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.18.6", + "@babel/plugin-transform-new-target": "^7.18.6", + "@babel/plugin-transform-object-super": "^7.18.6", + "@babel/plugin-transform-parameters": "^7.18.6", + "@babel/plugin-transform-property-literals": "^7.18.6", + "@babel/plugin-transform-regenerator": "^7.18.6", + "@babel/plugin-transform-reserved-words": "^7.18.6", + "@babel/plugin-transform-shorthand-properties": "^7.18.6", + "@babel/plugin-transform-spread": "^7.18.6", + "@babel/plugin-transform-sticky-regex": "^7.18.6", + "@babel/plugin-transform-template-literals": "^7.18.6", + "@babel/plugin-transform-typeof-symbol": "^7.18.6", + "@babel/plugin-transform-unicode-escapes": "^7.18.6", + "@babel/plugin-transform-unicode-regex": "^7.18.6", + "@babel/preset-modules": "^0.1.5", + "@babel/types": "^7.18.6", + "babel-plugin-polyfill-corejs2": "^0.3.1", + "babel-plugin-polyfill-corejs3": "^0.5.2", + "babel-plugin-polyfill-regenerator": "^0.3.1", + "core-js-compat": "^3.22.1", + "semver": "^6.3.0" + } + }, + "@babel/preset-flow": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-flow/-/preset-flow-7.18.6.tgz", + "integrity": "sha512-E7BDhL64W6OUqpuyHnSroLnqyRTcG6ZdOBl1OKI/QK/HJfplqK/S3sq1Cckx7oTodJ5yOXyfw7rEADJ6UjoQDQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-transform-flow-strip-types": "^7.18.6" + } + }, + "@babel/preset-modules": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.5.tgz", + "integrity": "sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.4.4", + "@babel/plugin-transform-dotall-regex": "^7.4.4", + "@babel/types": "^7.4.4", + "esutils": "^2.0.2" + } + }, + "@babel/preset-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.18.6.tgz", + "integrity": "sha512-s9ik86kXBAnD760aybBucdpnLsAt0jK1xqJn2juOn9lkOvSHV60os5hxoVJsPzMQxvnUJFAlkont2DvvaYEBtQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-transform-typescript": "^7.18.6" + } + }, + "@babel/register": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.18.6.tgz", + "integrity": "sha512-tkYtONzaO8rQubZzpBnvZPFcHgh8D9F55IjOsYton4X2IBoyRn2ZSWQqySTZnUn2guZbxbQiAB27hJEbvXamhQ==", + "dev": true, + "requires": { + "clone-deep": "^4.0.1", + "find-cache-dir": "^2.0.0", + "make-dir": "^2.1.0", + "pirates": "^4.0.5", + "source-map-support": "^0.5.16" + } + }, + "@babel/runtime": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.6.tgz", + "integrity": "sha512-t9wi7/AW6XtKahAe20Yw0/mMljKq0B1r2fPdvaAdV/KPDZewFXdaaa6K7lxmZBZ8FBNpCiAT6iHPmd6QO9bKfQ==", + "dev": true, + "peer": true, + "requires": { + "regenerator-runtime": "^0.13.4" + } + }, + "@babel/template": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.6.tgz", + "integrity": "sha512-JoDWzPe+wgBsTTgdnIma3iHNFC7YVJoPssVBDjiHfNlyt4YcunDtcDOUmfVDfCK5MfdsaIoX9PkijPhjH3nYUw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/traverse": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.18.6.tgz", + "integrity": "sha512-zS/OKyqmD7lslOtFqbscH6gMLFYOfG1YPqCKfAW5KrTeolKqvB8UelR49Fpr6y93kYkW2Ik00mT1LOGiAGvizw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/types": "^7.18.6", + "debug": "^4.1.0", + "globals": "^11.1.0" + } + }, + "@babel/types": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.18.6.tgz", + "integrity": "sha512-NdBNzPDwed30fZdDQtVR7ZgaO4UKjuaQFH9VArS+HMnurlOY0JWN+4ROlu/iapMFwjRQU4pOG4StZfDmulEwGA==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.18.6", + "to-fast-properties": "^2.0.0" + } + }, + "@jridgewell/gen-mapping": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", + "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", + "dev": true, + "requires": { + "@jridgewell/set-array": "^1.0.0", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.8.tgz", + "integrity": "sha512-YK5G9LaddzGbcucK4c8h5tWFmMPBvRZ/uyWmN1/SbBdIvqGUdWGkJ5BAaccgs6XbzVLsqbPJrBSFwKv3kT9i7w==", + "dev": true + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.14", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.14.tgz", + "integrity": "sha512-bJWEfQ9lPTvm3SneWwRFVLzrh6nhjwqw7TUFFBEMzwvg7t7PCDenf2lDwqo4NQXzdpgBXyFgDWnQA+2vkruksQ==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "arr-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", + "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==", + "dev": true + }, + "arr-flatten": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "dev": true + }, + "arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "dev": true + }, + "array-unique": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", + "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==", + "dev": true + }, + "assign-symbols": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", + "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==", + "dev": true + }, + "ast-types": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.14.2.tgz", + "integrity": "sha512-O0yuUDnZeQDL+ncNGlJ78BiO4jnYI3bvMsD5prT0/nsgijG/LpNBIr63gTjVTNsiGkgQhiyCShTgxt8oXOrklA==", + "dev": true, + "requires": { + "tslib": "^2.0.1" + } + }, + "atob": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "dev": true + }, + "babel-core": { + "version": "7.0.0-bridge.0", + "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz", + "integrity": "sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==", + "dev": true, + "requires": {} + }, + "babel-plugin-dynamic-import-node": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", + "dev": true, + "requires": { + "object.assign": "^4.1.0" + } + }, + "babel-plugin-polyfill-corejs2": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.1.tgz", + "integrity": "sha512-v7/T6EQcNfVLfcN2X8Lulb7DjprieyLWJK/zOWH5DUYcAgex9sP3h25Q+DLsX9TloXe3y1O8l2q2Jv9q8UVB9w==", + "dev": true, + "peer": true, + "requires": { + "@babel/compat-data": "^7.13.11", + "@babel/helper-define-polyfill-provider": "^0.3.1", + "semver": "^6.1.1" + } + }, + "babel-plugin-polyfill-corejs3": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.5.2.tgz", + "integrity": "sha512-G3uJih0XWiID451fpeFaYGVuxHEjzKTHtc9uGFEjR6hHrvNzeS/PX+LLLcetJcytsB5m4j+K3o/EpXJNb/5IEQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-define-polyfill-provider": "^0.3.1", + "core-js-compat": "^3.21.0" + } + }, + "babel-plugin-polyfill-regenerator": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.3.1.tgz", + "integrity": "sha512-Y2B06tvgHYt1x0yz17jGkGeeMr5FeKUu+ASJ+N6nB5lQ8Dapfg42i0OVrf8PNGJ3zKL4A23snMi1IRwrqqND7A==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-define-polyfill-provider": "^0.3.1" + } + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "base": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "dev": true, + "requires": { + "cache-base": "^1.0.1", + "class-utils": "^0.3.5", + "component-emitter": "^1.2.1", + "define-property": "^1.0.0", + "isobject": "^3.0.1", + "mixin-deep": "^1.2.0", + "pascalcase": "^0.1.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.0" + } + } + } + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, + "requires": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "browserslist": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.0.tgz", + "integrity": "sha512-UQxE0DIhRB5z/zDz9iA03BOfxaN2+GQdBYH/2WrSIWEUrnpzTPJbhqt+umq6r3acaPRTW1FNTkrcp0PXgtFkvA==", + "dev": true, + "requires": { + "caniuse-lite": "^1.0.30001358", + "electron-to-chromium": "^1.4.164", + "node-releases": "^2.0.5", + "update-browserslist-db": "^1.0.0" + } + }, + "buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, + "cache-base": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "dev": true, + "requires": { + "collection-visit": "^1.0.0", + "component-emitter": "^1.2.1", + "get-value": "^2.0.6", + "has-value": "^1.0.0", + "isobject": "^3.0.1", + "set-value": "^2.0.0", + "to-object-path": "^0.3.0", + "union-value": "^1.0.0", + "unset-value": "^1.0.0" + } + }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "caniuse-lite": { + "version": "1.0.30001359", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001359.tgz", + "integrity": "sha512-Xln/BAsPzEuiVLgJ2/45IaqD9jShtk3Y33anKb4+yLwQzws3+v6odKfpgES/cDEaZMLzSChpIGdbOYtH9MyuHw==", + "dev": true + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "class-utils": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "dev": true, + "requires": { + "arr-union": "^3.1.0", + "define-property": "^0.2.5", + "isobject": "^3.0.0", + "static-extend": "^0.1.1" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + } + } + }, + "clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "requires": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + } + }, + "collection-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", + "integrity": "sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==", + "dev": true, + "requires": { + "map-visit": "^1.0.0", + "object-visit": "^1.0.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true + }, + "component-emitter": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "convert-source-map": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz", + "integrity": "sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.1" + } + }, + "copy-descriptor": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", + "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==", + "dev": true + }, + "core-js-compat": { + "version": "3.23.3", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.23.3.tgz", + "integrity": "sha512-WSzUs2h2vvmKsacLHNTdpyOC9k43AEhcGoFlVgCY4L7aw98oSBKtPL6vD0/TqZjRWRQYdDSLkzZIni4Crbbiqw==", + "dev": true, + "peer": true, + "requires": { + "browserslist": "^4.21.0", + "semver": "7.0.0" + }, + "dependencies": { + "semver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz", + "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==", + "dev": true, + "peer": true + } + } + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==", + "dev": true + }, + "define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "dev": true, + "requires": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + } + }, + "define-property": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.2", + "isobject": "^3.0.1" + } + }, + "electron-to-chromium": { + "version": "1.4.172", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.172.tgz", + "integrity": "sha512-yDoFfTJnqBAB6hSiPvzmsBJSrjOXJtHSJoqJdI/zSIh7DYupYnIOHt/bbPw/WE31BJjNTybDdNAs21gCMnTh0Q==", + "dev": true + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "peer": true + }, + "expand-brackets": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", + "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==", + "dev": true, + "requires": { + "debug": "^2.3.3", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "posix-character-classes": "^0.1.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + } + } + }, + "extend-shallow": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", + "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==", + "dev": true, + "requires": { + "assign-symbols": "^1.0.0", + "is-extendable": "^1.0.1" + } + }, + "extglob": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", + "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dev": true, + "requires": { + "array-unique": "^0.3.2", + "define-property": "^1.0.0", + "expand-brackets": "^2.1.4", + "extend-shallow": "^2.0.1", + "fragment-cache": "^0.2.1", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==", + "dev": true, + "requires": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + } + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "flow-parser": { + "version": "0.181.1", + "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.181.1.tgz", + "integrity": "sha512-+Mx87/GkmF5+FHk8IXc5WppD/oC4wB+05MuIv7qmIMgThND3RhOBGl7Npyc2L7NLVenme00ZlwEKVieiMz4bqA==", + "dev": true + }, + "for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "dev": true + }, + "fragment-cache": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", + "integrity": "sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==", + "dev": true, + "requires": { + "map-cache": "^0.2.2" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true + }, + "get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, + "get-value": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", + "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==", + "dev": true + }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "dev": true + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true + }, + "has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "dev": true, + "requires": { + "get-intrinsic": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true + }, + "has-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", + "integrity": "sha512-IBXk4GTsLYdQ7Rvt+GRBrFSVEkmuOUy4re0Xjd9kJSUQpnTrWR4/y9RpfexN9vkAPMFuQoeWKwqzPozRTlasGw==", + "dev": true, + "requires": { + "get-value": "^2.0.6", + "has-values": "^1.0.0", + "isobject": "^3.0.0" + } + }, + "has-values": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", + "integrity": "sha512-ODYZC64uqzmtfGMEAX/FvZiRyWLpAC3vYnNunURUnkGVTS+mI0smVsWaPydRBsE3g+ok7h960jChO8mFcWlHaQ==", + "dev": true, + "requires": { + "is-number": "^3.0.0", + "kind-of": "^4.0.0" + }, + "dependencies": { + "kind-of": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", + "integrity": "sha512-24XsCxmEbRwEDbz/qz3stgin8TTzZ1ESR56OMCN0ujYg+vRutNSiOj9bHH9u85DKgXguraugV5sFuvbD4FW/hw==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true + }, + "is-core-module": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", + "integrity": "sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A==", + "dev": true, + "peer": true, + "requires": { + "has": "^1.0.3" + } + }, + "is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + }, + "is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, + "requires": { + "is-plain-object": "^2.0.4" + } + }, + "is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "requires": { + "isobject": "^3.0.1" + } + }, + "is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "jscodeshift": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/jscodeshift/-/jscodeshift-0.13.1.tgz", + "integrity": "sha512-lGyiEbGOvmMRKgWk4vf+lUrCWO/8YR8sUR3FKF1Cq5fovjZDlIcw3Hu5ppLHAnEXshVffvaM0eyuY/AbOeYpnQ==", + "dev": true, + "requires": { + "@babel/core": "^7.13.16", + "@babel/parser": "^7.13.16", + "@babel/plugin-proposal-class-properties": "^7.13.0", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", + "@babel/plugin-proposal-optional-chaining": "^7.13.12", + "@babel/plugin-transform-modules-commonjs": "^7.13.8", + "@babel/preset-flow": "^7.13.13", + "@babel/preset-typescript": "^7.13.0", + "@babel/register": "^7.13.16", + "babel-core": "^7.0.0-bridge.0", + "chalk": "^4.1.2", + "flow-parser": "0.*", + "graceful-fs": "^4.2.4", + "micromatch": "^3.1.10", + "neo-async": "^2.5.0", + "node-dir": "^0.1.17", + "recast": "^0.20.4", + "temp": "^0.8.4", + "write-file-atomic": "^2.3.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true + }, + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true + }, + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "dev": true, + "peer": true + }, + "make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "requires": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + } + } + }, + "map-cache": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", + "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==", + "dev": true + }, + "map-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", + "integrity": "sha512-4y7uGv8bd2WdM9vpQsiQNo41Ln1NvhvDRuVt0k2JZQ+ezN2uaQes7lZeZ+QQUHOLQAtDaBJ+7wCbi+ab/KFs+w==", + "dev": true, + "requires": { + "object-visit": "^1.0.0" + } + }, + "micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, + "requires": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + } + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "mixin-deep": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "dev": true, + "requires": { + "for-in": "^1.0.2", + "is-extendable": "^1.0.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "nanomatch": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "dev": true, + "requires": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "fragment-cache": "^0.2.1", + "is-windows": "^1.0.2", + "kind-of": "^6.0.2", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + } + }, + "neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true + }, + "node-dir": { + "version": "0.1.17", + "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", + "integrity": "sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==", + "dev": true, + "requires": { + "minimatch": "^3.0.2" + } + }, + "node-releases": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.5.tgz", + "integrity": "sha512-U9h1NLROZTq9uE1SNffn6WuPDg8icmi3ns4rEl/oTfIle4iLjTliCzgTsbaIFMq/Xn078/lfY/BL0GWZ+psK4Q==", + "dev": true + }, + "object-copy": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", + "integrity": "sha512-79LYn6VAb63zgtmAteVOWo9Vdj71ZVBy3Pbse+VqxDpEP83XuujMrGqHIwAXJ5I/aM0zU7dIyIAhifVTPrNItQ==", + "dev": true, + "requires": { + "copy-descriptor": "^0.1.0", + "define-property": "^0.2.5", + "kind-of": "^3.0.3" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "dependencies": { + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + } + } + }, + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "object-visit": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", + "integrity": "sha512-GBaMwwAVK9qbQN3Scdo0OyvgPW7l3lnaVMj84uTOZlswkX0KpF6fyDBJhtTthf7pymztoN36/KEr1DyhF96zEA==", + "dev": true, + "requires": { + "isobject": "^3.0.0" + } + }, + "object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "dev": true, + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + } + }, + "object.pick": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", + "integrity": "sha512-tqa/UMy/CCoYmj+H5qc07qvSL9dqcs/WZENZ1JbtWBlATP+iVOe778gE6MSijnyCnORzDuX6hU+LA4SZ09YjFQ==", + "dev": true, + "requires": { + "isobject": "^3.0.1" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "pascalcase": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", + "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==", + "dev": true + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "dev": true + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "peer": true + }, + "picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true + }, + "pirates": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz", + "integrity": "sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==", + "dev": true + }, + "pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "requires": { + "find-up": "^3.0.0" + } + }, + "posix-character-classes": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", + "integrity": "sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==", + "dev": true + }, + "recast": { + "version": "0.20.5", + "resolved": "https://registry.npmjs.org/recast/-/recast-0.20.5.tgz", + "integrity": "sha512-E5qICoPoNL4yU0H0NoBDntNB0Q5oMSNh9usFctYniLBluTthi3RsQVBXIJNbApOlvSwW/RGxIuokPcAc59J5fQ==", + "dev": true, + "requires": { + "ast-types": "0.14.2", + "esprima": "~4.0.0", + "source-map": "~0.6.1", + "tslib": "^2.0.1" + } + }, + "regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==", + "dev": true, + "peer": true + }, + "regenerate-unicode-properties": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.0.1.tgz", + "integrity": "sha512-vn5DU6yg6h8hP/2OkQo3K7uVILvY4iu0oI4t3HFa81UPkhGJwkRwM10JEc3upjdhHjs/k8GJY1sRBhk5sr69Bw==", + "dev": true, + "peer": true, + "requires": { + "regenerate": "^1.4.2" + } + }, + "regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==", + "dev": true, + "peer": true + }, + "regenerator-transform": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.0.tgz", + "integrity": "sha512-LsrGtPmbYg19bcPHwdtmXwbW+TqNvtY4riE3P83foeHRroMbH6/2ddFBfab3t7kbzc7v7p4wbkIecHImqt0QNg==", + "dev": true, + "peer": true, + "requires": { + "@babel/runtime": "^7.8.4" + } + }, + "regex-not": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "dev": true, + "requires": { + "extend-shallow": "^3.0.2", + "safe-regex": "^1.1.0" + } + }, + "regexpu-core": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-5.1.0.tgz", + "integrity": "sha512-bb6hk+xWd2PEOkj5It46A16zFMs2mv86Iwpdu94la4S3sJ7C973h2dHpYKwIBGaWSO7cIRJ+UX0IeMaWcO4qwA==", + "dev": true, + "peer": true, + "requires": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.0.1", + "regjsgen": "^0.6.0", + "regjsparser": "^0.8.2", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.0.0" + } + }, + "regjsgen": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.6.0.tgz", + "integrity": "sha512-ozE883Uigtqj3bx7OhL1KNbCzGyW2NQZPl6Hs09WTvCuZD5sTI4JY58bkbQWa/Y9hxIsvJ3M8Nbf7j54IqeZbA==", + "dev": true, + "peer": true + }, + "regjsparser": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.8.4.tgz", + "integrity": "sha512-J3LABycON/VNEu3abOviqGHuB/LOtOQj8SKmfP9anY5GfAVw/SPjwzSjxGjbZXIxbGfqTHtJw58C2Li/WkStmA==", + "dev": true, + "peer": true, + "requires": { + "jsesc": "~0.5.0" + }, + "dependencies": { + "jsesc": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "integrity": "sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==", + "dev": true, + "peer": true + } + } + }, + "repeat-element": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.4.tgz", + "integrity": "sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ==", + "dev": true + }, + "repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "dev": true + }, + "resolve": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "dev": true, + "peer": true, + "requires": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, + "resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==", + "dev": true + }, + "ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "dev": true + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "safe-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha512-aJXcif4xnaNUzvUuC5gcb46oTS7zvg4jpMTnuqtrEPlR3vFr4pxtdTwaF1Qs3Enjn9HK+ZlwQui+a7z0SywIzg==", + "dev": true, + "requires": { + "ret": "~0.1.10" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "set-value": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "dev": true, + "requires": { + "extend-shallow": "^2.0.1", + "is-extendable": "^0.1.1", + "is-plain-object": "^2.0.3", + "split-string": "^3.0.1" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "requires": { + "kind-of": "^6.0.2" + } + }, + "signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "snapdragon": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "dev": true, + "requires": { + "base": "^0.11.1", + "debug": "^2.2.0", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "map-cache": "^0.2.2", + "source-map": "^0.5.6", + "source-map-resolve": "^0.5.0", + "use": "^3.1.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true + } + } + }, + "snapdragon-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "dev": true, + "requires": { + "define-property": "^1.0.0", + "isobject": "^3.0.0", + "snapdragon-util": "^3.0.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.0" + } + } + } + }, + "snapdragon-util": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "dev": true, + "requires": { + "kind-of": "^3.2.0" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "source-map-resolve": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", + "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "dev": true, + "requires": { + "atob": "^2.1.2", + "decode-uri-component": "^0.2.0", + "resolve-url": "^0.2.1", + "source-map-url": "^0.4.0", + "urix": "^0.1.0" + } + }, + "source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "source-map-url": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.1.tgz", + "integrity": "sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw==", + "dev": true + }, + "split-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "dev": true, + "requires": { + "extend-shallow": "^3.0.0" + } + }, + "static-extend": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", + "integrity": "sha512-72E9+uLc27Mt718pMHt9VMNiAL4LMsmDbBva8mxWUCkT07fSzEGMYUCk0XWY6lp0j6RBAG4cJ3mWuZv2OE3s0g==", + "dev": true, + "requires": { + "define-property": "^0.2.5", + "object-copy": "^0.1.0" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + } + } + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "peer": true + }, + "temp": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/temp/-/temp-0.8.4.tgz", + "integrity": "sha512-s0ZZzd0BzYv5tLSptZooSjK8oj6C+c19p7Vqta9+6NPOf7r+fxq0cJe6/oN4LTC79sy5NY8ucOJNgwsKCSbfqg==", + "dev": true, + "requires": { + "rimraf": "~2.6.2" + } + }, + "to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "dev": true + }, + "to-object-path": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", + "integrity": "sha512-9mWHdnGRuh3onocaHzukyvCZhzvr6tiflAy/JRFXcJX0TjgfWA9pk9t8CMbzmBE4Jfw58pXbkngtBtqYxzNEyg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "to-regex": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "dev": true, + "requires": { + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "regex-not": "^1.0.2", + "safe-regex": "^1.1.0" + } + }, + "to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==", + "dev": true, + "requires": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + } + }, + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, + "unicode-canonical-property-names-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz", + "integrity": "sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==", + "dev": true, + "peer": true + }, + "unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "dev": true, + "peer": true, + "requires": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + } + }, + "unicode-match-property-value-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.0.0.tgz", + "integrity": "sha512-7Yhkc0Ye+t4PNYzOGKedDhXbYIBe1XEQYQxOPyhcXNMJ0WCABqqj6ckydd6pWRZTHV4GuCPKdBAUiMc60tsKVw==", + "dev": true, + "peer": true + }, + "unicode-property-aliases-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.0.0.tgz", + "integrity": "sha512-5Zfuy9q/DFr4tfO7ZPeVXb1aPoeQSdeFMLpYuFebehDAhbuevLs5yxSZmIFN1tP5F9Wl4IpJrYojg85/zgyZHQ==", + "dev": true, + "peer": true + }, + "union-value": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "dev": true, + "requires": { + "arr-union": "^3.1.0", + "get-value": "^2.0.6", + "is-extendable": "^0.1.1", + "set-value": "^2.0.1" + }, + "dependencies": { + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "unset-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", + "integrity": "sha512-PcA2tsuGSF9cnySLHTLSh2qrQiJ70mn+r+Glzxv2TWZblxsxCC52BDlZoPCsz7STd9pN7EZetkWZBAvk4cgZdQ==", + "dev": true, + "requires": { + "has-value": "^0.3.1", + "isobject": "^3.0.0" + }, + "dependencies": { + "has-value": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", + "integrity": "sha512-gpG936j8/MzaeID5Yif+577c17TxaDmhuyVgSwtnL/q8UUTySg8Mecb+8Cf1otgLoD7DDH75axp86ER7LFsf3Q==", + "dev": true, + "requires": { + "get-value": "^2.0.3", + "has-values": "^0.1.4", + "isobject": "^2.0.0" + }, + "dependencies": { + "isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "dev": true, + "requires": { + "isarray": "1.0.0" + } + } + } + }, + "has-values": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", + "integrity": "sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==", + "dev": true + } + } + }, + "update-browserslist-db": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.4.tgz", + "integrity": "sha512-jnmO2BEGUjsMOe/Fg9u0oczOe/ppIDZPebzccl1yDWGLFP16Pa1/RM5wEoKYPG2zstNcDuAStejyxsOuKINdGA==", + "dev": true, + "requires": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + } + }, + "urix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", + "integrity": "sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==", + "dev": true + }, + "use": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "dev": true + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + } + } +} diff --git a/tools/esmify/package.json b/tools/esmify/package.json new file mode 100644 index 0000000000..152aaae5da --- /dev/null +++ b/tools/esmify/package.json @@ -0,0 +1,9 @@ +{ + "devDependencies": { + "jscodeshift": "^0.13.1" + }, + "scripts": { + "convert_module": "jscodeshift -t use-import-export-declarations.js --stdin --verbose=2", + "rewrite_imports": "jscodeshift -t import-to-import_esmodule.js --stdin --verbose=2" + } +} diff --git a/tools/esmify/static-import.js b/tools/esmify/static-import.js new file mode 100644 index 0000000000..e99bfb3380 --- /dev/null +++ b/tools/esmify/static-import.js @@ -0,0 +1,147 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +const _path = require("path"); +const { getESMFiles } = require(_path.resolve(__dirname, "./is-esmified.js")); +const { + esmifyExtension, + isString, + warnForPath, + isMemberExpressionWithIdentifiers, +} = require(_path.resolve(__dirname, "./utils.js")); + +function isTargetESM(resourceURI) { + if ("ESMIFY_TARGET_PREFIX" in process.env) { + const files = getESMFiles(resourceURI); + const targetPrefix = process.env.ESMIFY_TARGET_PREFIX; + for (const esm of files) { + if (esm.startsWith(targetPrefix)) { + return true; + } + } + + return false; + } + + return true; +} + +function isImportESModuleCall(node) { + return isMemberExpressionWithIdentifiers(node.callee, [ + "ChromeUtils", + "importESModule", + ]); +} + +// Replace `ChromeUtils.import`, `Cu.import`, and `ChromeUtils.importESModule` +// with static import if it's at the top-level of system ESM file. +function tryReplacingWithStaticImport( + jscodeshift, + inputFile, + path, + resourceURINode, + alwaysReplace +) { + if (!alwaysReplace && !inputFile.endsWith(".sys.mjs")) { + // Static import is available only in system ESM. + return false; + } + + // Check if it's at the top-level. + if (path.parent.node.type !== "VariableDeclarator") { + return false; + } + + if (path.parent.parent.node.type !== "VariableDeclaration") { + return false; + } + + const decls = path.parent.parent.node; + if (decls.declarations.length !== 1) { + return false; + } + + if (path.parent.parent.parent.node.type !== "Program") { + return false; + } + + if (path.node.arguments.length !== 1) { + return false; + } + + const resourceURI = resourceURINode.value; + + // Collect imported symbols. + const specs = []; + if (path.parent.node.id.type === "Identifier") { + specs.push(jscodeshift.importNamespaceSpecifier(path.parent.node.id)); + } else if (path.parent.node.id.type === "ObjectPattern") { + for (const prop of path.parent.node.id.properties) { + if (prop.shorthand) { + specs.push(jscodeshift.importSpecifier(prop.key)); + } else if (prop.value.type === "Identifier") { + specs.push(jscodeshift.importSpecifier(prop.key, prop.value)); + } else { + return false; + } + } + } else { + return false; + } + + // If this is `ChromeUtils.import` or `Cu.import`, replace the extension. + // no-op for `ChromeUtils.importESModule`. + resourceURINode.value = esmifyExtension(resourceURI); + + const e = jscodeshift.importDeclaration(specs, resourceURINode); + e.comments = path.parent.parent.node.comments; + path.parent.parent.node.comments = []; + path.parent.parent.replace(e); + + return true; +} + +function replaceImportESModuleCall( + inputFile, + jscodeshift, + path, + alwaysReplace +) { + if (path.node.arguments.length !== 1) { + warnForPath( + inputFile, + path, + `importESModule call should have only one argument` + ); + return; + } + + const resourceURINode = path.node.arguments[0]; + if (!isString(resourceURINode)) { + warnForPath(inputFile, path, `resource URI should be a string`); + return; + } + + if (!alwaysReplace) { + const resourceURI = resourceURINode.value; + if (!isTargetESM(resourceURI)) { + return; + } + } + + // If this cannot be replaced with static import, do nothing. + tryReplacingWithStaticImport( + jscodeshift, + inputFile, + path, + resourceURINode, + alwaysReplace + ); +} + +exports.isImportESModuleCall = isImportESModuleCall; +exports.tryReplacingWithStaticImport = tryReplacingWithStaticImport; +exports.replaceImportESModuleCall = replaceImportESModuleCall; diff --git a/tools/esmify/use-import-export-declarations.js b/tools/esmify/use-import-export-declarations.js new file mode 100644 index 0000000000..fe8c4dd286 --- /dev/null +++ b/tools/esmify/use-import-export-declarations.js @@ -0,0 +1,208 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// jscodeshift rule to replace EXPORTED_SYMBOLS with export declarations, +// and also convert existing ChromeUtils.importESModule to static import. + +/* eslint-env node */ + +const _path = require("path"); +const { + warnForPath, + getPrevStatement, + getNextStatement, +} = require(_path.resolve(__dirname, "./utils.js")); +const { + isImportESModuleCall, + replaceImportESModuleCall, +} = require(_path.resolve(__dirname, "./static-import.js")); + +module.exports = function (fileInfo, api) { + const { jscodeshift } = api; + const root = jscodeshift(fileInfo.source); + doTranslate(fileInfo.path, jscodeshift, root); + return root.toSource({ lineTerminator: "\n" }); +}; + +module.exports.doTranslate = doTranslate; + +// Move the comment for `path.node` to adjacent statement, keeping the position +// as much as possible. +function moveComments(inputFile, path) { + const next = getNextStatement(path); + if (next) { + if (next.comments) { + next.comments = [...path.node.comments, ...next.comments]; + } else { + next.comments = path.node.comments; + } + path.node.comments = []; + + return; + } + + const prev = getPrevStatement(path); + if (prev) { + path.node.comments.forEach(c => { + c.leading = false; + c.trailing = true; + }); + + if (prev.comments) { + prev.comments = [...prev.comments, ...path.node.comments]; + } else { + prev.comments = path.node.comments; + } + path.node.comments = []; + + return; + } + + warnForPath( + inputFile, + path, + `EXPORTED_SYMBOLS has comments and it cannot be preserved` + ); +} + +function collectAndRemoveExportedSymbols(inputFile, root) { + const nodes = root.findVariableDeclarators("EXPORTED_SYMBOLS"); + if (!nodes.length) { + throw Error(`EXPORTED_SYMBOLS not found`); + } + + let path = nodes.get(0); + const obj = nodes.get(0).node.init; + if (!obj) { + throw Error(`EXPORTED_SYMBOLS is not statically known`); + } + + if (path.parent.node.declarations.length !== 1) { + throw Error(`EXPORTED_SYMBOLS shouldn't be declared with other variables`); + } + + if (path.parent.node.comments && path.parent.node.comments.length) { + moveComments(inputFile, path.parent); + } + + path.parent.prune(); + + const EXPORTED_SYMBOLS = new Set(); + if (obj.type !== "ArrayExpression") { + throw Error(`EXPORTED_SYMBOLS is not statically known`); + } + + for (const elem of obj.elements) { + if (elem.type !== "Literal") { + throw Error(`EXPORTED_SYMBOLS is not statically known`); + } + var name = elem.value; + if (typeof name !== "string") { + throw Error(`EXPORTED_SYMBOLS item must be a string`); + } + EXPORTED_SYMBOLS.add(name); + } + + return EXPORTED_SYMBOLS; +} + +function isTopLevel(path) { + return path.parent.node.type === "Program"; +} + +function convertToExport(jscodeshift, path, name) { + const e = jscodeshift.exportNamedDeclaration(path.node); + e.comments = []; + e.comments = path.node.comments; + path.node.comments = []; + + path.replace(e); +} + +function doTranslate(inputFile, jscodeshift, root) { + const EXPORTED_SYMBOLS = collectAndRemoveExportedSymbols(inputFile, root); + + root.find(jscodeshift.FunctionDeclaration).forEach(path => { + if (!isTopLevel(path)) { + return; + } + const name = path.node.id.name; + if (!EXPORTED_SYMBOLS.has(name)) { + return; + } + EXPORTED_SYMBOLS.delete(name); + convertToExport(jscodeshift, path, name); + }); + + root.find(jscodeshift.ClassDeclaration).forEach(path => { + if (!isTopLevel(path)) { + return; + } + const name = path.node.id.name; + if (!EXPORTED_SYMBOLS.has(name)) { + return; + } + EXPORTED_SYMBOLS.delete(name); + convertToExport(jscodeshift, path, name); + }); + + root.find(jscodeshift.VariableDeclaration).forEach(path => { + if (!isTopLevel(path)) { + return; + } + + let exists = false; + let name; + for (const decl of path.node.declarations) { + if (decl.id.type === "Identifier") { + name = decl.id.name; + if (EXPORTED_SYMBOLS.has(name)) { + exists = true; + break; + } + } + + if (decl.id.type === "ObjectPattern") { + if (decl.id.properties.length === 1) { + const prop = decl.id.properties[0]; + if (prop.shorthand) { + if (prop.key.type === "Identifier") { + name = prop.key.name; + if (EXPORTED_SYMBOLS.has(name)) { + exists = true; + break; + } + } + } + } + } + } + if (!exists) { + return; + } + + if (path.node.declarations.length !== 1) { + throw Error( + `exported variable shouldn't be declared with other variables` + ); + } + + EXPORTED_SYMBOLS.delete(name); + convertToExport(jscodeshift, path, name); + }); + + if (EXPORTED_SYMBOLS.size !== 0) { + throw Error( + `exported symbols ${[...EXPORTED_SYMBOLS].join(", ")} not found` + ); + } + + root.find(jscodeshift.CallExpression).forEach(path => { + if (isImportESModuleCall(path.node)) { + // This file is not yet renamed. Skip the extension check. + // Also skip the isTargetESM. + replaceImportESModuleCall(inputFile, jscodeshift, path, true); + } + }); +} diff --git a/tools/esmify/utils.js b/tools/esmify/utils.js new file mode 100644 index 0000000000..801aab62af --- /dev/null +++ b/tools/esmify/utils.js @@ -0,0 +1,165 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Shared utility functions. + +/* eslint-env node */ + +function warnForPath(inputFile, path, message) { + const loc = path.node.loc; + console.log( + `WARNING: ${inputFile}:${loc.start.line}:${loc.start.column} : ${message}` + ); +} + +// Get the previous statement of `path.node` in `Program`. +function getPrevStatement(path) { + const parent = path.parent; + if (parent.node.type !== "Program") { + return null; + } + + const index = parent.node.body.findIndex(n => n == path.node); + if (index === -1) { + return null; + } + + if (index === 0) { + return null; + } + + return parent.node.body[index - 1]; +} + +// Get the next statement of `path.node` in `Program`. +function getNextStatement(path) { + const parent = path.parent; + if (parent.node.type !== "Program") { + return null; + } + + const index = parent.node.body.findIndex(n => n == path.node); + if (index === -1) { + return null; + } + + if (index + 1 == parent.node.body.length) { + return null; + } + + return parent.node.body[index + 1]; +} + +function isIdentifier(node, name) { + if (node.type !== "Identifier") { + return false; + } + if (node.name !== name) { + return false; + } + return true; +} + +function isString(node) { + return node.type === "Literal" && typeof node.value === "string"; +} + +const jsmExtPattern = /\.(jsm|js|jsm\.js)$/; + +function esmifyExtension(path) { + return path.replace(jsmExtPattern, ".sys.mjs"); +} + +// Given possible member expression, return the list of Identifier nodes in +// the source order. +// +// Returns an empty array if: +// * not a simple MemberExpression tree with Identifiers +// * there's computed property +function memberExpressionsToIdentifiers(memberExpr) { + let ids = []; + + function f(node) { + if (node.type !== "MemberExpression" || node.computed) { + return false; + } + + if (node.object.type === "Identifier") { + ids.push(node.object); + ids.push(node.property); + return true; + } + + if (!f(node.object)) { + return false; + } + ids.push(node.property); + return true; + } + + if (!f(memberExpr)) { + return []; + } + + return ids; +} + +// Returns true if the node is a simple MemberExpression tree with Identifiers +// matches expectedIDs. +function isMemberExpressionWithIdentifiers(node, expectedIDs) { + const actualIDs = memberExpressionsToIdentifiers(node); + if (actualIDs.length !== expectedIDs.length) { + return false; + } + + for (let i = 0; i < expectedIDs.length; i++) { + if (actualIDs[i].name !== expectedIDs[i]) { + return false; + } + } + + return true; +} + +// Rewrite the Identifiers of MemberExpression tree to toIDs. +// `node` must be a simple MemberExpression tree with Identifiers, and +// the length of Identifiers should match. +function rewriteMemberExpressionWithIdentifiers(node, toIDs) { + const actualIDs = memberExpressionsToIdentifiers(node); + for (let i = 0; i < toIDs.length; i++) { + actualIDs[i].name = toIDs[i]; + } +} + +// Create a simple MemberExpression tree with given Identifiers. +function createMemberExpressionWithIdentifiers(jscodeshift, ids) { + if (ids.length < 2) { + throw new Error("Unexpected length of ids for member expression"); + } + + if (ids.length > 2) { + return jscodeshift.memberExpression( + createMemberExpressionWithIdentifiers(jscodeshift, ids.slice(0, -1)), + jscodeshift.identifier(ids[ids.length - 1]) + ); + } + + return jscodeshift.memberExpression( + jscodeshift.identifier(ids[0]), + jscodeshift.identifier(ids[1]) + ); +} + +exports.warnForPath = warnForPath; +exports.getPrevStatement = getPrevStatement; +exports.getNextStatement = getNextStatement; +exports.isIdentifier = isIdentifier; +exports.isString = isString; +exports.jsmExtPattern = jsmExtPattern; +exports.esmifyExtension = esmifyExtension; +exports.isMemberExpressionWithIdentifiers = isMemberExpressionWithIdentifiers; +exports.rewriteMemberExpressionWithIdentifiers = + rewriteMemberExpressionWithIdentifiers; +exports.createMemberExpressionWithIdentifiers = + createMemberExpressionWithIdentifiers; diff --git a/tools/fuzzing/common/FuzzingMutate.cpp b/tools/fuzzing/common/FuzzingMutate.cpp new file mode 100644 index 0000000000..bb5e930125 --- /dev/null +++ b/tools/fuzzing/common/FuzzingMutate.cpp @@ -0,0 +1,32 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzingMutate.h" +#include "FuzzingTraits.h" + +namespace mozilla { +namespace fuzzing { + +/** + * Randomly mutates a byte inside |aData| by using bit manipulation. + */ +/* static */ +void FuzzingMutate::ChangeBit(uint8_t* aData, size_t aLength) { + size_t offset = RandomIntegerRange<size_t>(0, aLength); + aData[offset] ^= (1 << FuzzingTraits::Random(9)); +} + +/** + * Randomly replaces a byte inside |aData| with one in the range of [0, 255]. + */ +/* static */ +void FuzzingMutate::ChangeByte(uint8_t* aData, size_t aLength) { + size_t offset = RandomIntegerRange<size_t>(0, aLength); + aData[offset] = RandomInteger<unsigned char>(); +} + +} // namespace fuzzing +} // namespace mozilla diff --git a/tools/fuzzing/common/FuzzingMutate.h b/tools/fuzzing/common/FuzzingMutate.h new file mode 100644 index 0000000000..f24f557669 --- /dev/null +++ b/tools/fuzzing/common/FuzzingMutate.h @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_fuzzing_FuzzingMutate_h +#define mozilla_fuzzing_FuzzingMutate_h + +#include <random> + +namespace mozilla { +namespace fuzzing { + +class FuzzingMutate { + public: + static void ChangeBit(uint8_t* aData, size_t aLength); + static void ChangeByte(uint8_t* aData, size_t aLength); +}; + +} // namespace fuzzing +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/common/FuzzingTraits.cpp b/tools/fuzzing/common/FuzzingTraits.cpp new file mode 100644 index 0000000000..9e6ba3ac1d --- /dev/null +++ b/tools/fuzzing/common/FuzzingTraits.cpp @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <prinrval.h> +#include <thread> +#include <mutex> +#include "FuzzingTraits.h" + +namespace mozilla { +namespace fuzzing { + +/* static */ +unsigned int FuzzingTraits::Random(unsigned int aMax) { + MOZ_ASSERT(aMax > 0, "aMax needs to be bigger than 0"); + std::uniform_int_distribution<unsigned int> d(0, aMax); + return d(Rng()); +} + +/* static */ +bool FuzzingTraits::Sometimes(unsigned int aProbability) { + return FuzzingTraits::Random(aProbability) == 0; +} + +/* static */ +size_t FuzzingTraits::Frequency(const size_t aSize, const uint64_t aFactor) { + return RandomIntegerRange<size_t>(0, ceil(float(aSize) / aFactor)) + 1; +} + +/* static */ +std::mt19937_64& FuzzingTraits::Rng() { + static std::mt19937_64 rng; + static std::once_flag flag; + std::call_once(flag, [&] { rng.seed(PR_IntervalNow()); }); + return rng; +} + +} // namespace fuzzing +} // namespace mozilla diff --git a/tools/fuzzing/common/FuzzingTraits.h b/tools/fuzzing/common/FuzzingTraits.h new file mode 100644 index 0000000000..b8c9d76ab7 --- /dev/null +++ b/tools/fuzzing/common/FuzzingTraits.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_fuzzing_FuzzingTraits_h +#define mozilla_fuzzing_FuzzingTraits_h + +#include "mozilla/Assertions.h" +#include <cmath> +#include <random> +#include <type_traits> + +namespace mozilla { +namespace fuzzing { + +class FuzzingTraits { + public: + static unsigned int Random(unsigned int aMax); + static bool Sometimes(unsigned int aProbability); + /** + * Frequency() defines how many mutations of a kind shall be applied to a + * target buffer by using a user definable factor. The higher the factor, + * the less mutations are being made. + */ + static size_t Frequency(const size_t aSize, const uint64_t aFactor); + + static std::mt19937_64& Rng(); +}; + +/** + * RandomNumericLimit returns either the min or max limit of an arithmetic + * data type. + */ +template <typename T> +T RandomNumericLimit() { + static_assert(std::is_arithmetic_v<T> == true, + "T must be an arithmetic type"); + return FuzzingTraits::Sometimes(2) ? std::numeric_limits<T>::min() + : std::numeric_limits<T>::max(); +} + +/** + * RandomInteger generates negative and positive integers in 2**n increments. + */ +template <typename T> +T RandomInteger() { + static_assert(std::is_integral_v<T> == true, "T must be an integral type"); + double r = + static_cast<double>(FuzzingTraits::Random((sizeof(T) * CHAR_BIT) + 1)); + T x = static_cast<T>(pow(2.0, r)) - 1; + if (std::numeric_limits<T>::is_signed && FuzzingTraits::Sometimes(2)) { + return (x * -1) - 1; + } + return x; +} + +/** + * RandomIntegerRange returns a random integral within a [min, max] range. + */ +template <typename T> +T RandomIntegerRange(T min, T max) { + static_assert(std::is_integral_v<T> == true, "T must be an integral type"); + MOZ_ASSERT(min < max); + std::uniform_int_distribution<T> d(min, max); + return d(FuzzingTraits::Rng()); +} +/** + * uniform_int_distribution is undefined for char/uchar. Need to handle them + * separately. + */ +template <> +inline unsigned char RandomIntegerRange(unsigned char min, unsigned char max) { + MOZ_ASSERT(min < max); + std::uniform_int_distribution<unsigned short> d(min, max); + return static_cast<unsigned char>(d(FuzzingTraits::Rng())); +} +template <> +inline char RandomIntegerRange(char min, char max) { + MOZ_ASSERT(min < max); + std::uniform_int_distribution<short> d(min, max); + return static_cast<char>(d(FuzzingTraits::Rng())); +} + +/** + * RandomFloatingPointRange returns a random floating-point number within a + * [min, max] range. + */ +template <typename T> +T RandomFloatingPointRange(T min, T max) { + static_assert(std::is_floating_point_v<T> == true, + "T must be a floating point type"); + MOZ_ASSERT(min < max); + std::uniform_real_distribution<T> d( + min, std::nextafter(max, std::numeric_limits<T>::max())); + return d(FuzzingTraits::Rng()); +} + +/** + * RandomFloatingPoint returns a random floating-point number in 2**n + * increments. + */ +template <typename T> +T RandomFloatingPoint() { + static_assert(std::is_floating_point_v<T> == true, + "T must be a floating point type"); + int radix = RandomIntegerRange<int>(std::numeric_limits<T>::min_exponent, + std::numeric_limits<T>::max_exponent); + T x = static_cast<T>(pow(2.0, static_cast<double>(radix))); + return x * RandomFloatingPointRange<T>(-1.0, 1.0); +} + +} // namespace fuzzing +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/common/moz.build b/tools/fuzzing/common/moz.build new file mode 100644 index 0000000000..afa19eddb4 --- /dev/null +++ b/tools/fuzzing/common/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += ["FuzzingMutate.cpp", "FuzzingTraits.cpp"] + +EXPORTS += ["FuzzingMutate.h", "FuzzingTraits.h"] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/docs/fuzzing_interface.rst b/tools/fuzzing/docs/fuzzing_interface.rst new file mode 100644 index 0000000000..a3ab575524 --- /dev/null +++ b/tools/fuzzing/docs/fuzzing_interface.rst @@ -0,0 +1,505 @@ +Fuzzing Interface +================= + +The fuzzing interface is glue code living in mozilla-central in order to +make it easier for developers and security researchers to test C/C++ +code with either `libFuzzer <https://llvm.org/docs/LibFuzzer.html>`__ or +`afl-fuzz <http://lcamtuf.coredump.cx/afl/>`__. + +These fuzzing tools, are based on *compile-time instrumentation* to measure +things like branch coverage and more advanced heuristics per fuzzing test. +Doing so allows these tools to progress through code with little to no custom +logic/knowledge implemented in the fuzzer itself. Usually, the only thing +these tools need is a code "shim" that provides the entry point for the fuzzer +to the code to be tested. We call this additional code a *fuzzing target* and +the rest of this manual describes how to implement and work with these targets. + +As for the tools used with these targets, we currently recommend the use of +libFuzzer over afl-fuzz, as the latter is no longer maintained while libFuzzer +is being actively developed. Furthermore, libFuzzer has some advanced +instrumentation features (e.g. value profiling to deal with complicated +comparisons in code), making it overall more effective. + +What can be tested? +~~~~~~~~~~~~~~~~~~~ + +The interface can be used to test all C/C++ code that either ends up in +``libxul`` (more precisely, the gtest version of ``libxul``) **or** is +part of the JS engine. + +Note that this is not the right testing approach for testing the full +browser as a whole. It is rather meant for component-based testing +(especially as some components cannot be easily separated out of the +full build). + +.. note:: + + **Note:** If you are working on the JS engine (trying to reproduce a + bug or seeking to develop a new fuzzing target), then please also read + the :ref:`JS Engine Specifics Section <JS Engine Specifics>` at the end + of this documentation, as the JS engine offers additional options for + implementing and running fuzzing targets. + + +Reproducing bugs for existing fuzzing targets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are working on a bug that involves an existing fuzzing interface target, +you have two options for reproducing the issue: + + +Using existing builds +^^^^^^^^^^^^^^^^^^^^^ + +We have several fuzzing builds in CI that you can simply download. We recommend +using ``fuzzfetch`` for this purpose, as it makes downloading and unpacking +these builds much easier. + +You can install ``fuzzfetch`` from +`Github <https://github.com/MozillaSecurity/fuzzfetch>`__ or +`via pip <https://pypi.org/project/fuzzfetch/>`__. + +Afterwards, you can run + +:: + + $ python -m fuzzfetch -a --fuzzing --target gtest -n firefox-fuzzing + +to fetch the latest optimized build. Alternatively, we offer non-ASan debug builds +which you can download using + +:: + + $ python -m fuzzfetch -d --fuzzing --target gtest -n firefox-fuzzing + +In both commands, ``firefox-fuzzing`` indicates the name of the directory that +will be created for the download. + +Afterwards, you can reproduce the bug using + +:: + + $ FUZZER=TargetName firefox-fuzzing/firefox test.bin + +assuming that ``TargetName`` is the name of the fuzzing target specified in the +bug you are working on and ``test.bin`` is the attached testcase. + +.. note:: + + **Note:** You should not export the ``FUZZER`` variable permanently + in your shell, especially if you plan to do local builds. If the ``FUZZER`` + variable is exported, it will affect the build process. + +If the CI builds don't meet your requirements and you need a local build instead, +you can follow the steps below to create one: + +.. _Local build requirements and flags: + +Local build requirements and flags +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You will need a Linux environment with a recent Clang. Using the Clang downloaded +by ``./mach bootstrap`` or a newer version is recommended. + +The only build flag required to enable the fuzzing targets is ``--enable-fuzzing``, +so adding + +:: + + ac_add_options --enable-fuzzing + +to your ``.mozconfig`` is already sufficient for producing a fuzzing build. +However, for improved crash handling capabilities and to detect additional errors, +it is strongly recommended to combine libFuzzer with :ref:`AddressSanitizer <Address Sanitizer>` +at least for optimized builds and bugs requiring ASan to reproduce at all +(e.g. you are working on a bug where ASan reports a memory safety violation +of some sort). + +Once your build is complete, if you want to run gtests, you **must** additionally run + +:: + + $ ./mach gtest dontruntests + +to force the gtest libxul to be built. + +If you get the error ``error while loading shared libraries: libxul.so: cannot +open shared object file: No such file or directory``, you need to explicitly +set ``LD_LIBRARY_PATH`` to your build directory ``obj-dir`` before the command +to invoke the fuzzing. For example an IPC fuzzing invocation: + +:: + + $ LD_LIBRARY_PATH=/path/to/obj-dir/dist/bin/ MOZ_FUZZ_TESTFILE=/path/to/test.bin NYX_FUZZER="IPC_Generic" /path/to/obj-dir/dist/bin/firefox /path/to/testcase.html + +.. note:: + + **Note:** If you modify any code, please ensure that you run **both** build + commands to ensure that the gtest libxul is also rebuilt. It is a common mistake + to only run ``./mach build`` and miss the second command. + +Once these steps are complete, you can reproduce the bug locally using the same +steps as described above for the downloaded builds. + + +Developing new fuzzing targets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Developing a new fuzzing target using the fuzzing interface only requires a few steps. + + +Determine if the fuzzing interface is the right tool +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The fuzzing interface is not suitable for every kind of testing. In particular +if your testing requires the full browser to be running, then you might want to +look into other testing methods. + +The interface uses the ``ScopedXPCOM`` implementation to provide an environment +in which XPCOM is available and initialized. You can initialize further subsystems +that you might require, but you are responsible yourself for any kind of +initialization steps. + +There is (in theory) no limit as to how far you can take browser initialization. +However, the more subsystems are involved, the more problems might occur due to +non-determinism and loss of performance. + +If you are unsure if the fuzzing interface is the right approach for you or you +require help in evaluating what could be done for your particular task, please +don't hesitate to :ref:`contact us <Fuzzing#contact-us>`. + + +Develop the fuzzing code +^^^^^^^^^^^^^^^^^^^^^^^^ + +Where to put your fuzzing code +'''''''''''''''''''''''''''''' + +The code using the fuzzing interface usually lives in a separate directory +called ``fuzztest`` that is on the same level as gtests. If your component +has no gtests, then a subdirectory either in tests or in your main directory +will work. If such a directory does not exist yet in your component, then you +need to create one with a suitable ``moz.build``. See `the transport target +for an example <https://searchfox.org/mozilla-central/source/dom/media/webrtc/transport/fuzztest/moz.build>`__ + +In order to include the new subdirectory into the build process, you will +also have to modify the toplevel ``moz.build`` file accordingly. For this +purpose, you should add your directory to ``TEST_DIRS`` only if ``FUZZING_INTERFACES`` +is set. See again `the transport target for an example +<https://searchfox.org/mozilla-central/rev/de7676288a78b70d2b9927c79493adbf294faad5/media/mtransport/moz.build#18-24>`__. + +How your code should look like +'''''''''''''''''''''''''''''' + +In order to define your fuzzing target ``MyTarget``, you only need to implement 2 functions: + +1. A one-time initialization function. + + At startup, the fuzzing interface calls this function **once**, so this can + be used to perform one-time operations like initializing subsystems or parsing + extra fuzzing options. + + This function is the equivalent of the `LLVMFuzzerInitialize <https://llvm.org/docs/LibFuzzer.html#startup-initialization>`__ + function and has the same signature. However, with our fuzzing interface, + it won't be resolved by its name, so it can be defined ``static`` and called + whatever you prefer. Note that the function should always ``return 0`` and + can (except for the return), remain empty. + + For the sake of this documentation, we assume that you have ``static int FuzzingInitMyTarget(int* argc, char*** argv);`` + +2. The fuzzing iteration function. + + This is where the actual fuzzing happens, and this function is the equivalent + of `LLVMFuzzerTestOneInput <https://llvm.org/docs/LibFuzzer.html#fuzz-target>`__. + Again, the difference to the fuzzing interface is that the function won't be + resolved by its name. In addition, we offer two different possible signatures + for this function, either + + ``static int FuzzingRunMyTarget(const uint8_t* data, size_t size);`` + + or + + ``static int FuzzingRunMyTarget(nsCOMPtr<nsIInputStream> inputStream);`` + + The latter is just a wrapper around the first one for implementations that + usually work with streams. No matter which of the two signatures you choose + to work with, the only thing you need to implement inside the function + is the use of the provided data with your target implementation. This can + mean to simply feed the data to your target, using the data to drive operations + on the target API, or a mix of both. + + While doing so, you should avoid altering global state in a permanent way, + using additional sources of data/randomness or having code run beyond the + lifetime of the iteration function (e.g. on another thread), for one simple + reason: Coverage-guided fuzzing tools depend on the **deterministic** nature + of the iteration function. If the same input to this function does not lead + to the same execution when run twice (e.g. because the resulting state depends + on multiple successive calls or because of additional external influences), + then the tool will not be able to reproduce its fuzzing progress and perform + badly. Dealing with this restriction can be challenging e.g. when dealing + with asynchronous targets that run multi-threaded, but can usually be managed + by synchronizing execution on all threads at the end of the iteration function. + For implementations accumulating global state, it might be necessary to + (re)initialize this global state in each iteration, rather than doing it once + in the initialization function, even if this costs additional performance. + + Note that unlike the vanilla libFuzzer approach, you are allowed to ``return 1`` + in this function to indicate that an input is "bad". Doing so will cause + libFuzzer to discard the input, no matter if it generated new coverage or not. + This is particularly useful if you have means to internally detect and catch + bad testcase behavior such as timeouts/excessive resource usage etc. to avoid + these tests to end up in your corpus. + + +Once you have implemented the two functions, the only thing remaining is to +register them with the fuzzing interface. For this purpose, we offer two +macros, depending on which iteration function signature you used. If you +stuck to the classic signature using buffer and size, you can simply use + +:: + + #include "FuzzingInterface.h" + + // Your includes and code + + MOZ_FUZZING_INTERFACE_RAW(FuzzingInitMyTarget, FuzzingRunMyTarget, MyTarget); + +where ``MyTarget`` is the name of the target and will be used later to decide +at runtime which target should be used. + +If instead you went for the streaming interface, you need a different include, +but the macro invocation is quite similar: + +:: + + #include "FuzzingInterfaceStream.h" + + // Your includes and code + + MOZ_FUZZING_INTERFACE_STREAM(FuzzingInitMyTarget, FuzzingRunMyTarget, MyTarget); + +For a live example, see also the `implementation of the STUN fuzzing target +<https://searchfox.org/mozilla-central/source/dom/media/webrtc/transport/fuzztest/stun_parser_libfuzz.cpp>`__. + +Add instrumentation to the code being tested +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +libFuzzer requires that the code you are trying to test is instrumented +with special compiler flags. Fortunately, adding these on a per-directory basis +can be done just by including the following directive in each ``moz.build`` +file that builds code under test: + +:: + + # Add libFuzzer configuration directives + include('/tools/fuzzing/libfuzzer-config.mozbuild') + + +The include already does the appropriate configuration checks to be only +active in fuzzing builds, so you don't have to guard this in any way. + +.. note:: + + **Note:** This include modifies `CFLAGS` and `CXXFLAGS` accordingly + but this only works for source files defined in this particular + directory. The flags are **not** propagated to subdirectories automatically + and you have to ensure that each directory that builds source files + for your target has the include added to its ``moz.build`` file. + +By keeping the instrumentation limited to the parts that are actually being +tested using this tool, you not only increase the performance but also potentially +reduce the amount of noise that libFuzzer sees. + + +Build your code +^^^^^^^^^^^^^^^ + +See the :ref:`Build instructions above <Local build requirements and flags>` for instructions +how to modify your ``.mozconfig`` to create the appropriate build. + + +Running your code and building a corpus +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You need to set the following environment variable to enable running the +fuzzing code inside Firefox instead of the regular browser. + +- ``FUZZER=name`` + +Where ``name`` is the name of your fuzzing module that you specified +when calling the ``MOZ_FUZZING_INTERFACE_RAW`` macro. For the example +above, this would be ``MyTarget`` or ``StunParser`` for the live example. + +Now when you invoke the firefox binary in your build directory with the +``-help=1`` parameter, you should see the regular libFuzzer help. On +Linux for example: + +:: + + $ FUZZER=StunParser obj-asan/dist/bin/firefox -help=1 + +You should see an output similar to this: + +:: + + Running Fuzzer tests... + Usage: + + To run fuzzing pass 0 or more directories. + obj-asan/dist/bin/firefox [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ] + + To run individual tests without fuzzing pass 1 or more files: + obj-asan/dist/bin/firefox [-flag1=val1 [-flag2=val2 ...] ] file1 [file2 ...] + + Flags: (strictly in form -flag=value) + verbosity 1 Verbosity level. + seed 0 Random seed. If 0, seed is generated. + runs -1 Number of individual test runs (-1 for infinite runs). + max_len 0 Maximum length of the test input. If 0, libFuzzer tries to guess a good value based on the corpus and reports it. + ... + + +Reproducing a Crash +''''''''''''''''''' + +In order to reproduce a crash from a given test file, simply put the +file as the only argument on the command line, e.g. + +:: + + $ FUZZER=StunParser obj-asan/dist/bin/firefox test.bin + +This should reproduce the given problem. + + +FuzzManager and libFuzzer +''''''''''''''''''''''''' + +Our FuzzManager project comes with a harness for running libFuzzer with +an optional connection to a FuzzManager server instance. Note that this +connection is not mandatory, even without a server you can make use of +the local harness. + +You can find the harness +`here <https://github.com/MozillaSecurity/FuzzManager/tree/master/misc/afl-libfuzzer>`__. + +An example invocation for the harness to use with StunParser could look +like this: + +:: + + FUZZER=StunParser python /path/to/afl-libfuzzer-daemon.py --fuzzmanager \ + --stats libfuzzer-stunparser.stats --libfuzzer-auto-reduce-min 500 --libfuzzer-auto-reduce 30 \ + --tool libfuzzer-stunparser --libfuzzer --libfuzzer-instances 6 obj-asan/dist/bin/firefox \ + -max_len=256 -use_value_profile=1 -rss_limit_mb=3000 corpus-stunparser + +What this does is + +- run libFuzzer on the ``StunParser`` target with 6 parallel instances + using the corpus in the ``corpus-stunparser`` directory (with the + specified libFuzzer options such as ``-max_len`` and + ``-use_value_profile``) +- automatically reduce the corpus and restart if it grew by 30% (and + has at least 500 files) +- use FuzzManager (need a local ``.fuzzmanagerconf`` and a + ``firefox.fuzzmanagerconf`` binary configuration as described in the + FuzzManager manual) and submit crashes as ``libfuzzer-stunparser`` + tool +- write statistics to the ``libfuzzer-stunparser.stats`` file + +.. _JS Engine Specifics: + +JS Engine Specifics +~~~~~~~~~~~~~~~~~~~ + +The fuzzing interface can also be used for testing the JS engine, in fact there +are two separate options to implement and run fuzzing targets: + +Implementing in C++ +^^^^^^^^^^^^^^^^^^^ + +Similar to the fuzzing interface in Firefox, you can implement your target in +entirely C++ with very similar interfaces compared to what was described before. + +There are a few minor differences though: + +1. All of the fuzzing targets live in `js/src/fuzz-tests`. + +2. All of the code is linked into a separate binary called `fuzz-tests`, + similar to how all JSAPI tests end up in `jsapi-tests`. In order for this + binary to be built, you must build a JS shell with ``--enable-fuzzing`` + **and** ``--enable-tests``. Again, this can and should be combined with + AddressSanitizer for maximum effectiveness. This also means that there is no + need to (re)build gtests when dealing with a JS fuzzing target and using + a shell as part of a full browser build. + +3. The harness around the JS implementation already provides you with an + initialized ``JSContext`` and global object. You can access these in + your target by declaring + + ``extern JS::PersistentRootedObject gGlobal;`` + + and + + ``extern JSContext* gCx;`` + + but there is no obligation for you to use these. + +For a live example, see also the `implementation of the StructuredCloneReader target +<https://searchfox.org/mozilla-central/source/js/src/fuzz-tests/testStructuredCloneReader.cpp>`__. + + +Implementing in JS +^^^^^^^^^^^^^^^^^^ + +In addition to the C++ targets, you can also implement targets in JavaScript +using the JavaScript Runtime (JSRT) fuzzing approach. Using this approach is +not only much simpler (since you don't need to know anything about the +JSAPI or engine internals), but it also gives you full access to everything +defined in the JS shell, including handy functions such as ``timeout()``. + +Of course, this approach also comes with disadvantages: Calling into JS and +performing the fuzzing operations there costs performance. Also, there is more +chance for causing global side-effects or non-determinism compared to a +fairly isolated C++ target. + +As a rule of thumb, you should implement the target in JS if + +* you don't know C++ and/or how to use the JSAPI (after all, a JS fuzzing target is better than none), +* your target is expected to have lots of hangs/timeouts (you can catch these internally), +* or your target is not isolated enough for a C++ target and/or you need specific JS shell functions. + + +There is an `example target <https://searchfox.org/mozilla-central/source/js/src/shell/jsrtfuzzing/jsrtfuzzing-example.js>`__ +in-tree that shows roughly how to implement such a fuzzing target. + +To run such a target, you must run the ``js`` (shell) binary instead of the +``fuzz-tests`` binary and point the ``FUZZER`` variable to the file containing +your fuzzing target, e.g. + +:: + + $ FUZZER=/path/to/jsrtfuzzing-example.js obj-asan/dist/bin/js --fuzzing-safe --no-threads -- <libFuzzer options here> + +More elaborate targets can be found in `js/src/fuzz-tests/ <https://searchfox.org/mozilla-central/source/js/src/fuzz-tests/>`__. + +Troubleshooting +~~~~~~~~~~~~~~~ + + +Fuzzing Interface: Error: No testing callback found +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This error means that the fuzzing callback with the name you specified +using the ``FUZZER`` environment variable could not be found. Reasons +for are typically either a misspelled name or that your code wasn't +built (check your ``moz.build`` file and build log). + + +``mach build`` doesn't seem to update my fuzzing code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Keep in mind you always need to run both the ``mach build`` and +``mach gtest dontruntests`` commands in order to update your fuzzing +code. The latter rebuilds the gtest version of ``libxul``, containing +your code. diff --git a/tools/fuzzing/docs/index.rst b/tools/fuzzing/docs/index.rst new file mode 100644 index 0000000000..9a4e2d01c4 --- /dev/null +++ b/tools/fuzzing/docs/index.rst @@ -0,0 +1,438 @@ +Fuzzing +======= + +.. toctree:: + :maxdepth: 1 + :hidden: + :glob: + :reversed: + + * + +This section focuses on explaining the software testing technique called +“Fuzzing” or “Fuzz Testing” and its application to the Mozilla codebase. +The overall goal is to educate developers about the capabilities and +usefulness of fuzzing and also allow them to write their own fuzzing +targets. Note that not all fuzzing tools used at Mozilla are open +source. Some tools are for internal use only because they can easily +find critical security vulnerabilities. + +What is Fuzzing? +---------------- + +Fuzzing (or Fuzz Testing) is a technique to randomly use a program or +parts of it with the goal to uncover bugs. Random usage can have a wide +variety of forms, a few common ones are + +- random input data (e.g. file formats, network data, source code, etc.) + +- random API usage + +- random UI interaction + +with the first two being the most practical methods used in the field. +Of course, these methods are not entirely separate, combinations are +possible. Fuzzing is a great way to find quality issues, some of them +being also security issues. + +Random input data +~~~~~~~~~~~~~~~~~ + +This is probably the most obvious fuzzing method: You have code that +processes data and you provide it with random or mutated data, hoping +that it will uncover bugs in your implementation. Examples are media +formats like JPEG or H.264, but basically anything that involves +processing a “blob” of data can be a valuable target. Countless security +vulnerabilities in a variety of libraries and programs have been found +using this method (the AFLFuzz +`bug-o-rama <http://lcamtuf.coredump.cx/afl/#bugs>`__ gives a good +impression). + +Common tools for this task are e.g. +`libFuzzer <https://llvm.org/docs/LibFuzzer.html>`__ and +`AFLFuzz <http://lcamtuf.coredump.cx/afl/>`__, but also specialized +tools with custom logic like +`LangFuzz <https://www.usenix.org/system/files/conference/usenixsecurity12/sec12-final73.pdf>`__ +and `Avalanche <https://github.com/MozillaSecurity/avalanche>`__. + +Random API Usage +~~~~~~~~~~~~~~~~ + +Randomly testing APIs is especially helpful with parts of software that +expose a well-defined interface (see also :ref:`Well-defined +behavior and Safety <Well defined behaviour and safety>`). If this interface is additionally exposed to +untrusted parties/content, then this is a strong sign that random API +testing would be worthwhile here, also for security reasons. APIs can be +anything from C++ layer code to APIs offered in the browser. + +A good example for a fuzzing target here is the DOM (Document Object +Model) and various other browser APIs. The browser exposes a variety of +different APIs for working with documents, media, communication, +storage, etc. with a growing complexity. Each of these APIs has +potential bugs that can be uncovered with fuzzing. At Mozilla, we +currently use domino (internal tool) for this purpose. + +Random UI Interaction +~~~~~~~~~~~~~~~~~~~~~ + +A third way to test programs and in particular user interfaces is by +directly interacting with the UI in a random way, typically in +combination with other actions the program has to perform. Imagine for +example an automated browser that surfs through the web and randomly +performs actions such as scrolling, zooming and clicking links. The nice +thing about this approach is that you likely find many issues that the +end-user also experiences. However, this approach typically suffers from +bad reproducibility (see also :ref:`Reproducibility <Reproducibility>`) and is therefore +often of limited use. + +An example for a fuzzing tool using this technique is `Android +Monkey <https://developer.android.com/studio/test/monkey>`__. At +Mozilla however, we currently don’t make much use of this approach. + +Why Fuzzing Helps You +--------------------- + +Understanding the value of fuzzing for you as a developer and software +quality in general is important to justify the support this testing +method might need from you. When your component is fuzzed for the first +time there are two common things you will be confronted with: + +**Bug reports that don’t seem real bugs or not important:** Fuzzers +find all sorts of bugs in various corners of your component, even +obscure ones. This automatically leads to a larger number of bugs that +either don’t seem to be bugs (see also the :ref:`Well-defined behavior and +safety <Well defined behaviour and safety>` section below) or that don’t seem to be important bugs. + +Fixing these bugs is still important for the fuzzers because ignoring them +in fuzzing costs resources (performance, human resources) and might even +prevent the fuzzer from hitting other bugs. For example certain fuzzing tools +like libFuzzer run in-process and have to restart on every crash, involving a +costly re-read of the fuzzing samples. + +Also, as some of our code evolves quickly, a corner case might become a +hot code path in a few months. + +**New steps to reproduce:** Fuzzing tools are very likely to exercise +your component using different methods than an average end-user. A +common technique is modify existing parts of a program or write entirely +new code to yield a fuzzing "target". This target is specifically +designed to work with the fuzzing tools in use. Reproducing the reported +bugs might require you to learn these new steps to reproduce, including +building/acquiring that target and having the right environment. + +Both of these issues might seem like a waste of time in some cases, +however, realizing that both steps are a one-time investment for a +constant stream of valuable bug reports is paramount here. Helping your +security engineers to overcome these issues will ensure that future +regressions in your code can be detected at an earlier stage and in a +form that is more easily actionable. Especially if you are dealing with +regressions in your code already, fuzzing has the potential to make your +job as a developer easier. + +One of the best examples at Mozilla is the JavaScript engine. The JS +team has put great quite some effort into getting fuzzing started and +supporting our work. Here’s what Jan de Mooij, a senior platform +engineer for the JavaScript engine, has to say about it: + +*“Bugs in the engine can cause mysterious browser crashes and bugs that +are incredibly hard to track down. Fortunately, we don't have to deal +with these time consuming browser issues very often: usually the fuzzers +find a reliable shell test long before the bug makes it into a release. +Fuzzing is invaluable to us and I cannot imagine working on this project +without it.”* + +Levels of Fuzzing in Firefox/Gecko +---------------------------------- + +Applying fuzzing to e.g. Firefox happens at different "levels", similar +to the different types of automated tests we have: + +Full Browser Fuzzing +~~~~~~~~~~~~~~~~~~~~ + +The most obvious method of testing would be to test the full browser and +doing so is required for certain features like the DOM and other APIs. +The advantage here is that we have all the features of the browser +available and testing happens closely to what we actually ship. The +downside here though is that browser testing is by far the slowest of +all testing methods. In addition, it has the most amount of +non-determinism involved (resulting e.g. in intermittent testcases). +Browser fuzzing at Mozilla is largely done with the `Grizzly +framework <https://blog.mozilla.org/security/2019/07/10/grizzly/>`__ +(`meta bug <https://bugzilla.mozilla.org/show_bug.cgi?id=grizzly>`__) +and one of the most successful fuzzers is the Domino tool (`meta +bug <https://bugzilla.mozilla.org/show_bug.cgi?id=domino>`__). + +Summarizing, full browser fuzzing is the right technique to investigate +if your feature really requires it. Consider using other methods (see +below) if your code can be exercised in this way. + +The Fuzzing Interface +~~~~~~~~~~~~~~~~~~~~~ + +**Fuzzing Interface** + +The fuzzing interface is glue code living in mozilla-central in order to make it +easier for developers and security researchers to test C/C++ code with either libFuzzer or afl-fuzz. + +This interface offers a gtest (C++ unit test) level component based +fuzzing approach and is suitable for anything that could also be +tested/exercised using a gtest. This method is by far the fastest, but +usually limited to testing isolated components that can be instantiated +on this level. Utilizing this method requires you to write a fuzzing +target similar to writing a gtest. This target will automatically be +usable with libFuzzer and AFLFuzz. We offer a :ref:`comprehensive manual <Fuzzing Interface>` +that describes how to write and utilize your own target. + +A simple example here is the `SDP parser +target <https://searchfox.org/mozilla-central/rev/efdf9bb55789ea782ae3a431bda6be74a87b041e/media/webrtc/signaling/fuzztest/sdp_parser_libfuzz.cpp#30>`__, +which tests the SipccSdpParser in our codebase. + +Shell-based Fuzzing +~~~~~~~~~~~~~~~~~~~ + +Some of our fuzzing, e.g. JS Engine testing, happens in a separate shell +program. For JS, this is the JS shell also used for most of the JS tests +and development. In theory, xpcshell could also be used for testing but +so far, there has not been a use case for this (most things that can be +reached through xpcshell can also be tested on the gtest level). + +Identifying the right level of fuzzing is the first step towards +continuous fuzz testing of your code. + +Code/Process Requirements for Fuzzing +------------------------------------- + +In this section, we are going to discuss how code should be written in +order to yield optimal results with fuzzing. + +Defect Oracles +~~~~~~~~~~~~~~ + +Fuzzing is only effective if you are able to know when a problem has +been found. Crashes are typically problems if the unit being tested is +safe for fuzzing (see Well-defined behavior and Safety). But there are +many more problems that you would want to find, correctness issues, +corruptions that don’t necessarily crash etc. For this, you need an +*oracle* that tells you something is wrong. + +The simplest defect oracle is the assertion (ex: ``MOZ_ASSERT``). +Assertions are a very powerful instrument because they can be used to +determine if your program is performing correctly, even if the bug would +not lead to any sort of crash. They can encode arbitrarily complex +information about what is considered correct, information that might +otherwise only exist in the developers’ minds. + +External tools like the sanitizers (AddressSanitizer aka ASan, +ThreadSanitizer aka TSan, MemorySanitizer aka MSan and +UndefinedBehaviorSanitizer - UBSan) can also serve as oracles for +sometimes severe issues that would not necessarily crash. Making sure +that these tools can be used on your code is highly useful. + +Examples for bugs found with sanitizers are `bug +1419608 <https://bugzilla.mozilla.org/show_bug.cgi?id=1419608>`__, +`bug 1580288 <https://bugzilla.mozilla.org/show_bug.cgi?id=1580288>`__ +and `bug 922603 <https://bugzilla.mozilla.org/show_bug.cgi?id=922603>`__, +but since we started using sanitizers, we have found over 1000 bugs with +these tools. + +Another defect oracle can be a reference implementation. Comparing +program behavior (typically output) between two programs or two modes of +the same program that should produce the same outputs can find complex +correctness issues. This method is often called differential testing. + +One example where this is regularly used to find issues is the Mozilla +JavaScript engine: Running random programs with and without JIT +compilation enabled finds lots of problems with the JIT implementation. +One example for such a bug is `Bug +1404636 <https://bugzilla.mozilla.org/show_bug.cgi?id=1404636>`__. + +Component Decoupling +~~~~~~~~~~~~~~~~~~~~ + +Being able to test components in isolation can be an advantage for +fuzzing (both for performance and reproducibility). Clear boundaries +between different components and documentation that explains the +contracts usually help with this goal. Sometimes it might be useful to +mock a certain component that the target component is interacting with +and that is much harder if the components are tightly coupled and their +contracts unclear. Of course, this does not mean that one should only +test components in isolation. Sometimes, testing the interaction between +them is even desirable and does not hurt performance at all. + +Avoiding external I/O +~~~~~~~~~~~~~~~~~~~~~ + +External I/O like network or file interactions are bad for performance +and can introduce additional non-determinism. Providing interfaces to +process data directly from memory instead is usually much more helpful. + +.. _Well defined behaviour and safety: + +Well-defined Behavior and Safety +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This requirement mostly ties in where defect oracles ended and is one of +the most important problems seen in the wild nowadays with fuzzing. If a +part of your program’s behavior is unspecified, then this potentially +leads to bad times if the behavior is considered a defect by fuzzing. +For example, if your code has crashes that are not considered bugs, then +your code might be unsuitable for fuzzing. Your component should be +fuzzing safe, meaning that any defect oracle (e.g. assertion or crash) +triggered by the fuzzer is considered a bug. This important aspect is +often neglected. Be aware that any false positives cause both +performance degradation and additional manual work for your fuzzing +team. The Mozilla JS developers for example have implemented this +concept in a “--fuzzing-safe” switch which disables harmful functions. +Sometimes, crashes cannot be avoided for handling certain error +conditions. In such situations, it is important to mark these crashes in +a way the fuzzer can recognize and distinguish them from undesired +crashes. However, keep in mind that crashes in general can be disruptive +to the fuzzing process. Performance is an important aspect of fuzzing +and frequent crashes can severely degrade performance. + +.. _Reproducibility: + +Reproducibility +~~~~~~~~~~~~~~~ + +Being able to reproduce issues found with fuzzing is necessary for +several reasons: First, you as the developer probably want a test that +reproduces the issue so you can debug it better. Our feedback from most +developers is that traces without a reproducible test can help to find a +problem, but it makes the whole process very complicated. Some of these +non-reproducible bugs never get fixed. Second, having a reproducible +test also helps the triage process by allowing an automated bisection to +find the responsible developer. Last but not least, the test can be +added to a test suite, used for automated verification of fixes and even +serve as a basis for more fuzzing. + +Adding functionality to the program that improve reproducibility is +therefore a good idea in case non-reproducible issues are found. Some +examples are shown in the next section. + +While many problems with reproducibility are specific for the project +you are working on, there is one source of these problems that many +programs have in common: Threading. While some bugs only occur in the +first place due to concurrency, some other bugs would be perfectly +reproducible without threads, but are intermittent and hard to with +threading enabled. If the bug is indeed caused by a data race, then +tools like ThreadSanitizer will help and we are currently working on +making ThreadSanitizer usable on Firefox. For bugs that are not caused +by threading, it sometimes makes sense to be able to disable threading +or limit the amount of worker threads involved. + +Supporting Code +~~~~~~~~~~~~~~~ + +Some possibilities of what support implementations for fuzzing can do +have already been named in the previous sections: Additional defect +oracles and functionality to improve reproducibility and safety. In +fact, many features added specifically for fuzzing fit into one of these +categories. However, there’s room for more: Often, there are ways to +make it easier for fuzzers to exercise complex and hard to reach parts +of your code. For example, if a certain optimization feature is only +turned on under very specific conditions (that are not a requirement for +the optimization), then it makes sense to add a functionality to force +it on. Then, a fuzzer can hit the optimization code much more +frequently, increasing the chance to find issues. Some examples from +Firefox and SpiderMonkey: + +- The `FuzzingFunctions <https://searchfox.org/mozilla-central/rev/efdf9bb55789ea782ae3a431bda6be74a87b041e/dom/webidl/FuzzingFunctions.webidl#15>`__ + interface in the browser allows fuzzing tools to perform GC/CC, tune various + settings related to garbage collection or enable features like accessibility + mode. Being able to force a garbage collection at a specific time helped + identifying lots of problems in the past. + +- The --ion-eager and --baseline-eager flags for the JS shell force JIT + compilation at various stages, rather than using the builtin + heuristic to enable it only for hot functions. + +- The --no-threads flag disables all threading (if possible) in the JS shell. + This makes some bugs reproduce deterministically that would otherwise be + intermittent and harder to find. However, some bugs that only occur with + threading can’t be found with this option enabled. + +Another important feature that must be turned off for fuzzing is +checksums. Many file formats use checksums to validate a file before +processing it. If a checksum feature is still enabled, fuzzers are +likely never going to produce valid files. The same often holds for +cryptographic signatures. Being able to turn off the validation of these +features as part of a fuzzing switch is extremely helpful. + +An example for such a checksum can be found in the +`FlacDemuxer <https://searchfox.org/mozilla-central/rev/efdf9bb55789ea782ae3a431bda6be74a87b041e/dom/media/flac/FlacDemuxer.cpp#494>`__. + +Test Samples +~~~~~~~~~~~~ + +Some fuzzing strategies make use of existing data that is mutated to +produce the new random data. In fact, mutation-based strategies are +typically superior to others if the original samples are of good quality +because the originals carry a lot of semantics that the fuzzer does not +have to know about or implement. However, success here really stands and +falls with the quality of the samples. If the originals don’t cover +certain parts of the implementation, then the fuzzer will also have to +do more work to get there. + + +Fuzz Blockers +~~~~~~~~~~~~~ + +Fuzz blockers are issues that prevent fuzzers from being as +effective as possible. Depending on the fuzzer and its scope a fuzz blocker +in one area (or component) can impede performance in other areas and in +some cases block the fuzzer all together. Some examples are: + +- Frequent crashes - These can block code paths and waste compute + resources due to the need to relaunch the fuzzing target and handle + the results (regardless of whether it is ignored or reported). This can also + include assertions that are mostly benign in many cases are but easily + triggered by fuzzers. + +- Frequent hangs / timeouts - This includes any issue that slows down + or blocks execution of the fuzzer or the target. + +- Hard to bucket - This includes crashes such as stack overflows or any issue + that crashes in an inconsistent location. This also includes issues that + corrupt logs/debugger output or provide a broken/invalid crash report. + +- Broken builds - This is fairly straightforward, without up-to-date builds + fuzzers are unable to run or verify fixes. + +- Missing instrumentation - In some cases tools such as ASan are used as + defect oracles and are required by the fuzzing tools to allow for proper + automation. In other cases incomplete instrumentation can give a false sense + of stability or make investigating issues much more time consuming. Although + this is not necessarily blocking the fuzzers it should be prioritized + appropriately. + +Since these types of crashes harm the overall fuzzing progress, it is important +for them to be addressed in a timely manner. Even if the bug itself might seem +trivial and low priority for the product, it can still have devastating effects +on fuzzing and hence prevent finding other critical issues. + +Issues in Bugzilla are marked as fuzz blockers by adding “[fuzzblocker]” +to the “Whiteboard” field. A list of open issues marked as fuzz blockers +can be found on `Bugzilla <https://bugzilla.mozilla.org/buglist.cgi?cmdtype=dorem&remaction=run&namedcmd=fuzzblockers&sharer_id=486634>`__. + + +Documentation +~~~~~~~~~~~~~ + +It is important for the fuzzing team to know how your software, tests +and designs work. Even obvious tasks, like how a test program is +supposed to be invoked, which options are safe, etc. might be hard to +figure out for the person doing the testing, just as you are reading +this manual right now to find out what is important in fuzzing. + +Contact Us +~~~~~~~~~~ + +The fuzzing team can be reached at +`fuzzing@mozilla.com <mailto:fuzzing@mozilla.com>`__ or +`on Matrix <https://chat.mozilla.org/#/room/#fuzzing:mozilla.org>`__ +and will be happy to help you with any questions about fuzzing +you might have. We can help you find the right method of fuzzing for +your feature, collaborate on the implementation and provide the +infrastructure to run it and process the results accordingly. diff --git a/tools/fuzzing/interface/FuzzingInterface.cpp b/tools/fuzzing/interface/FuzzingInterface.cpp new file mode 100644 index 0000000000..f06ca68656 --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterface.cpp @@ -0,0 +1,30 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Common code for the unified fuzzing interface + */ + +#include <stdarg.h> +#include <stdlib.h> +#include "FuzzingInterface.h" + +namespace mozilla { + +#ifdef JS_STANDALONE +static bool fuzzing_verbose = !!getenv("MOZ_FUZZ_LOG"); +void fuzzing_log(const char* aFmt, ...) { + if (fuzzing_verbose) { + va_list ap; + va_start(ap, aFmt); + vfprintf(stderr, aFmt, ap); + va_end(ap); + } +} +#else +LazyLogModule gFuzzingLog("nsFuzzing"); +#endif + +} // namespace mozilla diff --git a/tools/fuzzing/interface/FuzzingInterface.h b/tools/fuzzing/interface/FuzzingInterface.h new file mode 100644 index 0000000000..792f0809ec --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterface.h @@ -0,0 +1,115 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface definitions for the unified fuzzing interface + */ + +#ifndef FuzzingInterface_h__ +#define FuzzingInterface_h__ + +#include <fstream> + +#ifdef LIBFUZZER +# include "FuzzerExtFunctions.h" +#endif + +#include "FuzzerRegistry.h" +#include "mozilla/Assertions.h" + +#ifndef JS_STANDALONE +# include "mozilla/Logging.h" +#endif + +namespace mozilla { + +#ifdef JS_STANDALONE +void fuzzing_log(const char* aFmt, ...); +# define MOZ_LOG_EXPAND_ARGS(...) __VA_ARGS__ + +# define FUZZING_LOG(args) fuzzing_log(MOZ_LOG_EXPAND_ARGS args); +#else +extern LazyLogModule gFuzzingLog; + +# define FUZZING_LOG(args) \ + MOZ_LOG(mozilla::gFuzzingLog, mozilla::LogLevel::Verbose, args) +#endif // JS_STANDALONE + +typedef int (*FuzzingTestFuncRaw)(const uint8_t*, size_t); + +#ifdef AFLFUZZ + +static int afl_interface_raw(const char* testFile, + FuzzingTestFuncRaw testFunc) { + char* buf = NULL; + + while (__AFL_LOOP(1000)) { + std::ifstream is; + is.open(testFile, std::ios::binary); + is.seekg(0, std::ios::end); + int len = is.tellg(); + is.seekg(0, std::ios::beg); + MOZ_RELEASE_ASSERT(len >= 0); + if (!len) { + is.close(); + continue; + } + buf = (char*)realloc(buf, len); + MOZ_RELEASE_ASSERT(buf); + is.read(buf, len); + is.close(); + testFunc((uint8_t*)buf, (size_t)len); + } + + free(buf); + + return 0; +} + +# define MOZ_AFL_INTERFACE_COMMON() \ + char* testFilePtr = getenv("MOZ_FUZZ_TESTFILE"); \ + if (!testFilePtr) { \ + fprintf(stderr, \ + "Must specify testfile in MOZ_FUZZ_TESTFILE environment " \ + "variable.\n"); \ + return 1; \ + } \ + /* Make a copy of testFilePtr so the testing function can safely call \ + * getenv \ + */ \ + std::string testFile(testFilePtr); + +# define MOZ_AFL_INTERFACE_RAW(initFunc, testFunc, moduleName) \ + static int afl_fuzz_##moduleName(const uint8_t* data, size_t size) { \ + MOZ_RELEASE_ASSERT(data == NULL && size == 0); \ + MOZ_AFL_INTERFACE_COMMON(); \ + return ::mozilla::afl_interface_raw(testFile.c_str(), testFunc); \ + } \ + static void __attribute__((constructor)) AFLRegister##moduleName() { \ + ::mozilla::FuzzerRegistry::getInstance().registerModule( \ + #moduleName, initFunc, afl_fuzz_##moduleName); \ + } +#else +# define MOZ_AFL_INTERFACE_RAW(initFunc, testFunc, moduleName) /* Nothing */ +#endif // AFLFUZZ + +#ifdef LIBFUZZER +# define MOZ_LIBFUZZER_INTERFACE_RAW(initFunc, testFunc, moduleName) \ + static void __attribute__((constructor)) LibFuzzerRegister##moduleName() { \ + ::mozilla::FuzzerRegistry::getInstance().registerModule( \ + #moduleName, initFunc, testFunc); \ + } +#else +# define MOZ_LIBFUZZER_INTERFACE_RAW(initFunc, testFunc, \ + moduleName) /* Nothing */ +#endif + +#define MOZ_FUZZING_INTERFACE_RAW(initFunc, testFunc, moduleName) \ + MOZ_LIBFUZZER_INTERFACE_RAW(initFunc, testFunc, moduleName); \ + MOZ_AFL_INTERFACE_RAW(initFunc, testFunc, moduleName); + +} // namespace mozilla + +#endif // FuzzingInterface_h__ diff --git a/tools/fuzzing/interface/FuzzingInterfaceStream.cpp b/tools/fuzzing/interface/FuzzingInterfaceStream.cpp new file mode 100644 index 0000000000..f2c5c891e9 --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterfaceStream.cpp @@ -0,0 +1,54 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface implementation for the unified fuzzing interface + */ + +#include "nsIFile.h" +#include "nsIPrefService.h" +#include "nsIProperties.h" + +#include "FuzzingInterfaceStream.h" + +#include "mozilla/Assertions.h" + +#ifndef JS_STANDALONE +# include "nsNetUtil.h" +#endif + +namespace mozilla { + +#ifdef AFLFUZZ + +void afl_interface_stream(const char* testFile, + FuzzingTestFuncStream testFunc) { + nsresult rv; + nsCOMPtr<nsIProperties> dirService = + do_GetService(NS_DIRECTORY_SERVICE_CONTRACTID); + MOZ_RELEASE_ASSERT(dirService != nullptr); + nsCOMPtr<nsIFile> file; + rv = dirService->Get(NS_OS_CURRENT_WORKING_DIR, NS_GET_IID(nsIFile), + getter_AddRefs(file)); + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + file->AppendNative(nsDependentCString(testFile)); + while (__AFL_LOOP(1000)) { + nsCOMPtr<nsIInputStream> inputStream; + rv = NS_NewLocalFileInputStream(getter_AddRefs(inputStream), file); + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + if (!NS_InputStreamIsBuffered(inputStream)) { + nsCOMPtr<nsIInputStream> bufStream; + rv = NS_NewBufferedInputStream(getter_AddRefs(bufStream), + inputStream.forget(), 1024); + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + inputStream = bufStream; + } + testFunc(inputStream.forget()); + } +} + +#endif + +} // namespace mozilla diff --git a/tools/fuzzing/interface/FuzzingInterfaceStream.h b/tools/fuzzing/interface/FuzzingInterfaceStream.h new file mode 100644 index 0000000000..1542020794 --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterfaceStream.h @@ -0,0 +1,90 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface definitions for the unified fuzzing interface with streaming + * support + */ + +#ifndef FuzzingInterfaceStream_h__ +#define FuzzingInterfaceStream_h__ + +#ifdef JS_STANDALONE +# error "FuzzingInterfaceStream.h cannot be used in JS standalone builds." +#endif + +#include "gtest/gtest.h" +#include "nsComponentManagerUtils.h" +#include "nsCOMPtr.h" +#include "nsIInputStream.h" + +#include "nsDirectoryServiceDefs.h" +#include "nsStreamUtils.h" +#include "nsStringStream.h" + +#include <fstream> + +#include "FuzzingInterface.h" + +namespace mozilla { + +typedef int (*FuzzingTestFuncStream)(nsCOMPtr<nsIInputStream>); + +#ifdef AFLFUZZ +void afl_interface_stream(const char* testFile, FuzzingTestFuncStream testFunc); + +# define MOZ_AFL_INTERFACE_COMMON(initFunc) \ + if (initFunc) initFunc(NULL, NULL); \ + char* testFilePtr = getenv("MOZ_FUZZ_TESTFILE"); \ + if (!testFilePtr) { \ + fprintf(stderr, \ + "Must specify testfile in MOZ_FUZZ_TESTFILE environment " \ + "variable.\n"); \ + return; \ + } \ + /* Make a copy of testFilePtr so the testing function can safely call \ + * getenv \ + */ \ + std::string testFile(testFilePtr); + +# define MOZ_AFL_INTERFACE_STREAM(initFunc, testFunc, moduleName) \ + TEST(AFL, moduleName) \ + { \ + MOZ_AFL_INTERFACE_COMMON(initFunc); \ + ::mozilla::afl_interface_stream(testFile.c_str(), testFunc); \ + } +#else +# define MOZ_AFL_INTERFACE_STREAM(initFunc, testFunc, moduleName) /* Nothing \ + */ +#endif + +#ifdef LIBFUZZER +# define MOZ_LIBFUZZER_INTERFACE_STREAM(initFunc, testFunc, moduleName) \ + static int LibFuzzerTest##moduleName(const uint8_t* data, size_t size) { \ + if (size > INT32_MAX) return 0; \ + nsCOMPtr<nsIInputStream> stream; \ + nsresult rv = NS_NewByteInputStream(getter_AddRefs(stream), \ + Span((const char*)data, size), \ + NS_ASSIGNMENT_DEPEND); \ + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); \ + testFunc(stream.forget()); \ + return 0; \ + } \ + static void __attribute__((constructor)) LibFuzzerRegister##moduleName() { \ + ::mozilla::FuzzerRegistry::getInstance().registerModule( \ + #moduleName, initFunc, LibFuzzerTest##moduleName); \ + } +#else +# define MOZ_LIBFUZZER_INTERFACE_STREAM(initFunc, testFunc, \ + moduleName) /* Nothing */ +#endif + +#define MOZ_FUZZING_INTERFACE_STREAM(initFunc, testFunc, moduleName) \ + MOZ_LIBFUZZER_INTERFACE_STREAM(initFunc, testFunc, moduleName); \ + MOZ_AFL_INTERFACE_STREAM(initFunc, testFunc, moduleName); + +} // namespace mozilla + +#endif // FuzzingInterfaceStream_h__ diff --git a/tools/fuzzing/interface/harness/FuzzerRunner.cpp b/tools/fuzzing/interface/harness/FuzzerRunner.cpp new file mode 100644 index 0000000000..fc2094aa59 --- /dev/null +++ b/tools/fuzzing/interface/harness/FuzzerRunner.cpp @@ -0,0 +1,91 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <cstdlib> + +#include "FuzzerRunner.h" +#include "mozilla/Attributes.h" +#include "prenv.h" + +#include "FuzzerTestHarness.h" + +namespace mozilla { + +// We use a static var 'fuzzerRunner' defined in nsAppRunner.cpp. +// fuzzerRunner is initialized to nullptr but if this file is linked in, +// then fuzzerRunner will be set here indicating that +// we want to call into either LibFuzzer's main or the AFL entrypoint. +class _InitFuzzer { + public: + _InitFuzzer() { fuzzerRunner = new FuzzerRunner(); } + void InitXPCOM() { mScopedXPCOM = new ScopedXPCOM("Fuzzer"); } + void DeinitXPCOM() { + if (mScopedXPCOM) delete mScopedXPCOM; + mScopedXPCOM = nullptr; + } + + private: + ScopedXPCOM* mScopedXPCOM; +} InitLibFuzzer; + +static void DeinitXPCOM() { InitLibFuzzer.DeinitXPCOM(); } + +int FuzzerRunner::Run(int* argc, char*** argv) { + /* + * libFuzzer uses exit() calls in several places instead of returning, + * so the destructor of ScopedXPCOM is not called in some cases. + * For fuzzing, this does not make a difference, but in debug builds + * when running a single testcase, this causes an assertion when destroying + * global linked lists. For this reason, we allocate ScopedXPCOM on the heap + * using the global InitLibFuzzer class, combined with an atexit call to + * destroy the ScopedXPCOM instance again. + */ + InitLibFuzzer.InitXPCOM(); + std::atexit(DeinitXPCOM); + + const char* fuzzerEnv = getenv("FUZZER"); + + if (!fuzzerEnv) { + fprintf(stderr, + "Must specify fuzzing target in FUZZER environment variable\n"); + exit(1); + } + + std::string moduleNameStr(fuzzerEnv); + FuzzerFunctions funcs = + FuzzerRegistry::getInstance().getModuleFunctions(moduleNameStr); + FuzzerInitFunc initFunc = funcs.first; + FuzzerTestingFunc testingFunc = funcs.second; + if (initFunc) { + int ret = initFunc(argc, argv); + if (ret) { + fprintf(stderr, "Fuzzing Interface: Error: Initialize callback failed\n"); + exit(1); + } + } + + if (!testingFunc) { + fprintf(stderr, "Fuzzing Interface: Error: No testing callback found\n"); + exit(1); + } + +#ifdef LIBFUZZER + int ret = mFuzzerDriver(argc, argv, testingFunc); +#else + // For AFL, testingFunc points to the entry function we need. + int ret = testingFunc(NULL, 0); +#endif + + InitLibFuzzer.DeinitXPCOM(); + return ret; +} + +#ifdef LIBFUZZER +void FuzzerRunner::setParams(LibFuzzerDriver aDriver) { + mFuzzerDriver = aDriver; +} +#endif + +} // namespace mozilla diff --git a/tools/fuzzing/interface/harness/FuzzerRunner.h b/tools/fuzzing/interface/harness/FuzzerRunner.h new file mode 100644 index 0000000000..6b19e751cd --- /dev/null +++ b/tools/fuzzing/interface/harness/FuzzerRunner.h @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzerRegistry.h" + +namespace mozilla { + +class FuzzerRunner { + public: + int Run(int* argc, char*** argv); + +#ifdef LIBFUZZER + void setParams(LibFuzzerDriver aDriver); + + private: + LibFuzzerDriver mFuzzerDriver; +#endif +}; + +extern FuzzerRunner* fuzzerRunner; + +} // namespace mozilla diff --git a/tools/fuzzing/interface/harness/FuzzerTestHarness.h b/tools/fuzzing/interface/harness/FuzzerTestHarness.h new file mode 100644 index 0000000000..d7bb1064cf --- /dev/null +++ b/tools/fuzzing/interface/harness/FuzzerTestHarness.h @@ -0,0 +1,256 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Test harness for XPCOM objects, providing a scoped XPCOM initializer, + * nsCOMPtr, nsRefPtr, do_CreateInstance, do_GetService, ns(Auto|C|)String, + * and stdio.h/stdlib.h. + */ + +#ifndef FuzzerTestHarness_h__ +#define FuzzerTestHarness_h__ + +#include "mozilla/ArrayUtils.h" +#include "mozilla/Attributes.h" + +#include "prenv.h" +#include "nsComponentManagerUtils.h" +#include "nsServiceManagerUtils.h" +#include "nsCOMPtr.h" +#include "nsString.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsDirectoryServiceDefs.h" +#include "nsDirectoryServiceUtils.h" +#include "nsIDirectoryService.h" +#include "nsIFile.h" +#include "nsIObserverService.h" +#include "nsIServiceManager.h" +#include "nsXULAppAPI.h" +#include "mozilla/AppShutdown.h" +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> + +namespace { + +static uint32_t gFailCount = 0; + +/** + * Prints the given failure message and arguments using printf, prepending + * "TEST-UNEXPECTED-FAIL " for the benefit of the test harness and + * appending "\n" to eliminate having to type it at each call site. + */ +MOZ_FORMAT_PRINTF(1, 2) void fail(const char* msg, ...) { + va_list ap; + + printf("TEST-UNEXPECTED-FAIL | "); + + va_start(ap, msg); + vprintf(msg, ap); + va_end(ap); + + putchar('\n'); + ++gFailCount; +} + +//----------------------------------------------------------------------------- + +class ScopedXPCOM final : public nsIDirectoryServiceProvider2 { + public: + NS_DECL_ISUPPORTS + + explicit ScopedXPCOM(const char* testName, + nsIDirectoryServiceProvider* dirSvcProvider = nullptr) + : mDirSvcProvider(dirSvcProvider) { + mTestName = testName; + printf("Running %s tests...\n", mTestName); + + nsresult rv = NS_InitXPCOM(&mServMgr, nullptr, this); + if (NS_FAILED(rv)) { + fail("NS_InitXPCOM returned failure code 0x%" PRIx32, + static_cast<uint32_t>(rv)); + mServMgr = nullptr; + return; + } + } + + ~ScopedXPCOM() { + // If we created a profile directory, we need to remove it. + if (mProfD) { + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdownNetTeardown); + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdownTeardown); + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdown); + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdownQM); + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdownTelemetry); + + if (NS_FAILED(mProfD->Remove(true))) { + NS_WARNING("Problem removing profile directory"); + } + + mProfD = nullptr; + } + + if (mServMgr) { + NS_RELEASE(mServMgr); + nsresult rv = NS_ShutdownXPCOM(nullptr); + if (NS_FAILED(rv)) { + fail("XPCOM shutdown failed with code 0x%" PRIx32, + static_cast<uint32_t>(rv)); + exit(1); + } + } + + printf("Finished running %s tests.\n", mTestName); + } + + already_AddRefed<nsIFile> GetProfileDirectory() { + if (mProfD) { + nsCOMPtr<nsIFile> copy = mProfD; + return copy.forget(); + } + + // Create a unique temporary folder to use for this test. + // Note that runcppunittests.py will run tests with a temp + // directory as the cwd, so just put something under that. + nsCOMPtr<nsIFile> profD; + nsresult rv = NS_GetSpecialDirectory(NS_OS_CURRENT_PROCESS_DIR, + getter_AddRefs(profD)); + NS_ENSURE_SUCCESS(rv, nullptr); + + rv = profD->Append(u"cpp-unit-profd"_ns); + NS_ENSURE_SUCCESS(rv, nullptr); + + rv = profD->CreateUnique(nsIFile::DIRECTORY_TYPE, 0755); + NS_ENSURE_SUCCESS(rv, nullptr); + + mProfD = profD; + return profD.forget(); + } + + already_AddRefed<nsIFile> GetGREDirectory() { + if (mGRED) { + nsCOMPtr<nsIFile> copy = mGRED; + return copy.forget(); + } + + char* env = PR_GetEnv("MOZ_XRE_DIR"); + nsCOMPtr<nsIFile> greD; + if (env) { + NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, getter_AddRefs(greD)); + } + + mGRED = greD; + return greD.forget(); + } + + already_AddRefed<nsIFile> GetGREBinDirectory() { + if (mGREBinD) { + nsCOMPtr<nsIFile> copy = mGREBinD; + return copy.forget(); + } + + nsCOMPtr<nsIFile> greD = GetGREDirectory(); + if (!greD) { + return greD.forget(); + } + greD->Clone(getter_AddRefs(mGREBinD)); + +#ifdef XP_MACOSX + nsAutoCString leafName; + mGREBinD->GetNativeLeafName(leafName); + if (leafName.EqualsLiteral("Resources")) { + mGREBinD->SetNativeLeafName("MacOS"_ns); + } +#endif + + nsCOMPtr<nsIFile> copy = mGREBinD; + return copy.forget(); + } + + //////////////////////////////////////////////////////////////////////////// + //// nsIDirectoryServiceProvider + + NS_IMETHODIMP GetFile(const char* aProperty, bool* _persistent, + nsIFile** _result) override { + // If we were supplied a directory service provider, ask it first. + if (mDirSvcProvider && NS_SUCCEEDED(mDirSvcProvider->GetFile( + aProperty, _persistent, _result))) { + return NS_OK; + } + + // Otherwise, the test harness provides some directories automatically. + if (0 == strcmp(aProperty, NS_APP_USER_PROFILE_50_DIR) || + 0 == strcmp(aProperty, NS_APP_USER_PROFILE_LOCAL_50_DIR) || + 0 == strcmp(aProperty, NS_APP_PROFILE_LOCAL_DIR_STARTUP)) { + nsCOMPtr<nsIFile> profD = GetProfileDirectory(); + NS_ENSURE_TRUE(profD, NS_ERROR_FAILURE); + + nsCOMPtr<nsIFile> clone; + nsresult rv = profD->Clone(getter_AddRefs(clone)); + NS_ENSURE_SUCCESS(rv, rv); + + *_persistent = true; + clone.forget(_result); + return NS_OK; + } else if (0 == strcmp(aProperty, NS_GRE_DIR)) { + nsCOMPtr<nsIFile> greD = GetGREDirectory(); + NS_ENSURE_TRUE(greD, NS_ERROR_FAILURE); + + *_persistent = true; + greD.forget(_result); + return NS_OK; + } else if (0 == strcmp(aProperty, NS_GRE_BIN_DIR)) { + nsCOMPtr<nsIFile> greBinD = GetGREBinDirectory(); + NS_ENSURE_TRUE(greBinD, NS_ERROR_FAILURE); + + *_persistent = true; + greBinD.forget(_result); + return NS_OK; + } + + return NS_ERROR_FAILURE; + } + + //////////////////////////////////////////////////////////////////////////// + //// nsIDirectoryServiceProvider2 + + NS_IMETHODIMP GetFiles(const char* aProperty, + nsISimpleEnumerator** _enum) override { + // If we were supplied a directory service provider, ask it first. + nsCOMPtr<nsIDirectoryServiceProvider2> provider = + do_QueryInterface(mDirSvcProvider); + if (provider && NS_SUCCEEDED(provider->GetFiles(aProperty, _enum))) { + return NS_OK; + } + + return NS_ERROR_FAILURE; + } + + private: + const char* mTestName; + nsIServiceManager* mServMgr; + nsCOMPtr<nsIDirectoryServiceProvider> mDirSvcProvider; + nsCOMPtr<nsIFile> mProfD; + nsCOMPtr<nsIFile> mGRED; + nsCOMPtr<nsIFile> mGREBinD; +}; + +NS_IMPL_QUERY_INTERFACE(ScopedXPCOM, nsIDirectoryServiceProvider, + nsIDirectoryServiceProvider2) + +NS_IMETHODIMP_(MozExternalRefCountType) +ScopedXPCOM::AddRef() { return 2; } + +NS_IMETHODIMP_(MozExternalRefCountType) +ScopedXPCOM::Release() { return 1; } + +} // namespace + +#endif // FuzzerTestHarness_h__ diff --git a/tools/fuzzing/interface/harness/moz.build b/tools/fuzzing/interface/harness/moz.build new file mode 100644 index 0000000000..0eff84c8aa --- /dev/null +++ b/tools/fuzzing/interface/harness/moz.build @@ -0,0 +1,16 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-runner") + +SOURCES += [ + "FuzzerRunner.cpp", +] +EXPORTS += [ + "FuzzerRunner.h", +] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/interface/moz.build b/tools/fuzzing/interface/moz.build new file mode 100644 index 0000000000..8a51007174 --- /dev/null +++ b/tools/fuzzing/interface/moz.build @@ -0,0 +1,32 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-interface") + +EXPORTS += [ + "FuzzingInterface.h", +] + +SOURCES += [ + "FuzzingInterface.cpp", +] + +if CONFIG["JS_STANDALONE"]: + FINAL_LIBRARY = "js" +else: + EXPORTS += [ + "FuzzingInterfaceStream.h", + ] + + SOURCES += [ + "FuzzingInterfaceStream.cpp", + ] + + DIRS += [ + "harness", + ] + + FINAL_LIBRARY = "xul-gtest" diff --git a/tools/fuzzing/ipc/IPCFuzzController.cpp b/tools/fuzzing/ipc/IPCFuzzController.cpp new file mode 100644 index 0000000000..54856e9f2c --- /dev/null +++ b/tools/fuzzing/ipc/IPCFuzzController.cpp @@ -0,0 +1,1328 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "IPCFuzzController.h" +#include "mozilla/Fuzzing.h" +#include "mozilla/SpinEventLoopUntil.h" +#include "mozilla/SyncRunnable.h" + +#include "nsIThread.h" +#include "nsThreadUtils.h" + +#include "mozilla/ipc/MessageChannel.h" +#include "mozilla/ipc/MessageLink.h" +#include "mozilla/ipc/ProtocolUtils.h" +#include "mozilla/ipc/NodeChannel.h" +#include "mozilla/ipc/NodeController.h" + +#include "mozilla/ipc/PIdleScheduler.h" +#include "mozilla/ipc/PBackground.h" +#include "mozilla/dom/PContent.h" + +#include <fstream> +#include <sstream> +#include <algorithm> + +using namespace mojo::core::ports; +using namespace mozilla::ipc; + +// Sync inject means that the actual fuzzing takes place on the I/O thread +// and hence it injects directly into the target NodeChannel. In async mode, +// we run the fuzzing on a separate thread and dispatch the runnable that +// injects the message back to the I/O thread. Both approaches seem to work +// and have advantages and disadvantages. Blocking the I/O thread means no +// IPC between other processes will interfere with our fuzzing in the meantime +// but blocking could also cause hangs when such IPC is required during the +// fuzzing runtime for some reason. +// #define MOZ_FUZZ_IPC_SYNC_INJECT 1 + +// Synchronize after each message rather than just after every constructor +// or at the end of the iteration. Doing so costs us some performance because +// we have to wait for each packet and process events on the main thread, +// but it is necessary when using `OnMessageError` to release on early errors. +#define MOZ_FUZZ_IPC_SYNC_AFTER_EACH_MSG 1 + +namespace mozilla { +namespace fuzzing { + +const uint32_t ipcDefaultTriggerMsg = dom::PContent::Msg_SignalFuzzingReady__ID; + +IPCFuzzController::IPCFuzzController() + : useLastPortName(false), + useLastActor(0), + mMutex("IPCFuzzController"), + mIPCTriggerMsg(ipcDefaultTriggerMsg) { + InitializeIPCTypes(); + + // We use 6 bits for port index selection without wrapping, so we just + // create 64 empty rows in our port matrix. Not all of these rows will + // be used though. + portNames.resize(64); + + // This is our port / toplevel actor ordering. Add new toplevel actors + // here to support them in the fuzzer. Do *NOT* change the order of + // these, as it will invalidate our fuzzing corpus. + portNameToIndex["PContent"] = 0; + portNameToIndex["PBackground"] = 1; + portNameToIndex["PBackgroundStarter"] = 2; + portNameToIndex["PCompositorManager"] = 3; + portNameToIndex["PImageBridge"] = 4; + portNameToIndex["PProcessHangMonitor"] = 5; + portNameToIndex["PProfiler"] = 6; + portNameToIndex["PVRManager"] = 7; + portNameToIndex["PCanvasManager"] = 8; + + // Used to select the n-th trigger message as a starting point for fuzzing + // in single message mode. A value of 1 will skip the first matching message + // and start fuzzing on the second message, and so on. + if (!!getenv("MOZ_FUZZ_IPC_TRIGGER_SINGLEMSG_WAIT")) { + mIPCTriggerSingleMsgWait = + atoi(getenv("MOZ_FUZZ_IPC_TRIGGER_SINGLEMSG_WAIT")); + } + + // When set, dump all IPC message at or above the specified size to files. + // Useful to collect samples of different types in one run. + if (!!getenv("MOZ_FUZZ_IPC_DUMP_ALL_MSGS_SIZE")) { + mIPCDumpAllMsgsSize.emplace( + atoi(getenv("MOZ_FUZZ_IPC_DUMP_ALL_MSGS_SIZE"))); + } +} + +// static +IPCFuzzController& IPCFuzzController::instance() { + static IPCFuzzController ifc; + return ifc; +} + +void IPCFuzzController::InitializeIPCTypes() { + const char* cons = "Constructor"; + size_t cons_len = strlen(cons); + + const char* targetNameTrigger = getenv("MOZ_FUZZ_IPC_TRIGGER"); + const char* targetNameDump = getenv("MOZ_FUZZ_IPC_DUMPMSG"); + + for (uint32_t start = 0; start < LastMsgIndex; ++start) { + uint32_t i; + for (i = (start << 16) + 1; i < ((start + 1) << 16); ++i) { + const char* name = IPC::StringFromIPCMessageType(i); + + if (name[0] == '<') break; + + if (targetNameTrigger && !strcmp(name, targetNameTrigger)) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [InitializeIPCTypes] Located trigger message (%s, %d)\n", + targetNameTrigger, i); + mIPCTriggerMsg = i; + } + + if (targetNameDump && !strcmp(name, targetNameDump)) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [InitializeIPCTypes] Located dump message (%s, %d)\n", + targetNameDump, i); + mIPCDumpMsg.emplace(i); + } + + size_t len = strlen(name); + if (len > cons_len && !memcmp(cons, name + len - cons_len, cons_len)) { + constructorTypes.insert(i); + } + } + + uint32_t msgCount = i - ((start << 16) + 1); + if (msgCount) { + validMsgTypes[(ProtocolId)start] = msgCount; + } + } +} + +bool IPCFuzzController::GetRandomIPCMessageType(ProtocolId pId, + uint16_t typeOffset, + uint32_t* type) { + auto pIdEntry = validMsgTypes.find(pId); + if (pIdEntry == validMsgTypes.end()) { + return false; + } + + *type = + ((uint32_t)pIdEntry->first << 16) + 1 + (typeOffset % pIdEntry->second); + + if (strstr(IPC::StringFromIPCMessageType(*type), "::Reply_")) { + *type = *type - 1; + } + + return true; +} + +void IPCFuzzController::OnActorConnected(IProtocol* protocol) { + if (!XRE_IsParentProcess()) { + return; + } + + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::OnActorConnected() Mutex try\n"); + + // Called on background threads and modifies `actorIds`. + MutexAutoLock lock(mMutex); + + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::OnActorConnected() Mutex locked\n"); + + static bool protoIdFilterInitialized = false; + static bool allowNewActors = false; + static std::string protoIdFilter; + if (!protoIdFilterInitialized) { + const char* protoIdFilterStr = getenv("MOZ_FUZZ_PROTOID_FILTER"); + if (protoIdFilterStr) { + protoIdFilter = std::string(protoIdFilterStr); + } + protoIdFilterInitialized = true; + } + +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF("INFO: [OnActorConnected] ActorID %d Protocol: %s\n", + protocol->Id(), protocol->GetProtocolName()); +#endif + + MessageChannel* channel = protocol->ToplevelProtocol()->GetIPCChannel(); + + Maybe<PortName> portName = channel->GetPortName(); + if (portName) { + if (!protoIdFilter.empty() && + (!Nyx::instance().started() || !allowNewActors) && + strcmp(protocol->GetProtocolName(), protoIdFilter.c_str()) && + !actorIds[*portName].empty()) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [OnActorConnected] ActorID %d Protocol: %s ignored due to " + "filter.\n", + protocol->Id(), protocol->GetProtocolName()); + return; + } else if (!protoIdFilter.empty() && + !strcmp(protocol->GetProtocolName(), protoIdFilter.c_str())) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [OnActorConnected] ActorID %d Protocol: %s matches target.\n", + protocol->Id(), protocol->GetProtocolName()); + } else if (!protoIdFilter.empty() && actorIds[*portName].empty()) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [OnActorConnected] ActorID %d Protocol: %s is toplevel " + "actor.\n", + protocol->Id(), protocol->GetProtocolName()); + } + + actorIds[*portName].emplace_back(protocol->Id(), protocol->GetProtocolId()); + + if (Nyx::instance().started() && protoIdFilter.empty()) { + // Fix the port we will be using for at least the next 5 messages + useLastPortName = true; + lastActorPortName = *portName; + + // Use this actor for the next 5 messages + useLastActor = 5; + } + } else { + MOZ_FUZZING_NYX_DEBUG("WARNING: No port name on actor?!\n"); + } +} + +void IPCFuzzController::OnActorDestroyed(IProtocol* protocol) { + if (!XRE_IsParentProcess()) { + return; + } + +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF("INFO: [OnActorDestroyed] ActorID %d Protocol: %s\n", + protocol->Id(), protocol->GetProtocolName()); +#endif + + MessageChannel* channel = protocol->ToplevelProtocol()->GetIPCChannel(); + + Maybe<PortName> portName = channel->GetPortName(); + if (portName) { + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::OnActorDestroyed() Mutex try\n"); + // Called on background threads and modifies `actorIds`. + MutexAutoLock lock(mMutex); + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::OnActorDestroyed() Mutex locked\n"); + + for (auto iter = actorIds[*portName].begin(); + iter != actorIds[*portName].end();) { + if (iter->first == protocol->Id() && + iter->second == protocol->GetProtocolId()) { + iter = actorIds[*portName].erase(iter); + } else { + ++iter; + } + } + } else { + MOZ_FUZZING_NYX_DEBUG("WARNING: No port name on destroyed actor?!\n"); + } +} + +void IPCFuzzController::AddToplevelActor(PortName name, ProtocolId protocolId) { + const char* protocolName = ProtocolIdToName(protocolId); + auto result = portNameToIndex.find(protocolName); + if (result == portNameToIndex.end()) { + MOZ_FUZZING_NYX_PRINTF( + "ERROR: [OnActorConnected] Unknown Top-Level Protocol: %s\n", + protocolName); + MOZ_FUZZING_NYX_ABORT("Unknown Top-Level Protocol\n"); + } + uint8_t portIndex = result->second; + portNames[portIndex].push_back(name); + portNameToProtocolName[name] = std::string(protocolName); +} + +bool IPCFuzzController::ObserveIPCMessage(mozilla::ipc::NodeChannel* channel, + IPC::Message& aMessage) { + if (!mozilla::fuzzing::Nyx::instance().is_enabled("IPC_Generic")) { + // Fuzzer is not enabled. + return true; + } + + if (!XRE_IsParentProcess()) { + // For now we only care about things in the parent process. + return true; + } + + if (aMessage.IsFuzzMsg()) { + // Don't observe our own messages. If this is the first fuzzing message, + // we also block further non-fuzzing communication on that node. + if (!channel->mBlockSendRecv) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [NodeChannel::OnMessageReceived] Blocking further " + "communication on Port %lu %lu (seen fuzz msg)\n", + channel->GetName().v1, channel->GetName().v2); + channel->mBlockSendRecv = true; + } + return true; + } else if (aMessage.type() == mIPCTriggerMsg && !Nyx::instance().started()) { + MOZ_FUZZING_NYX_PRINT("DEBUG: Ready message detected.\n"); + + if (!haveTargetNodeName && !!getenv("MOZ_FUZZ_PROTOID_FILTER")) { + // With a protocol filter set, we want to pin to the actor that + // received the ready message and stay there. We should do this here + // because OnActorConnected can be called even after the ready message + // has been received and potentially override the correct actor. + + // Get the port name associated with this message + Vector<char, 256, InfallibleAllocPolicy> footer; + if (!footer.initLengthUninitialized(aMessage.event_footer_size()) || + !aMessage.ReadFooter(footer.begin(), footer.length(), false)) { + MOZ_FUZZING_NYX_ABORT("ERROR: Failed to read message footer.\n"); + } + + UniquePtr<Event> event = + Event::Deserialize(footer.begin(), footer.length()); + + if (!event || event->type() != Event::kUserMessage) { + MOZ_FUZZING_NYX_ABORT("ERROR: Trigger message is not kUserMessage?!\n"); + } + + lastActorPortName = event->port_name(); + useLastPortName = true; + useLastActor = 1024; + } + + // TODO: This is specific to PContent fuzzing. If we later want to fuzz + // a different process pair, we need additional signals here. + OnChildReady(); + + // The ready message indicates the right node name for us to work with + // and we should only ever receive it once. + if (!haveTargetNodeName) { + targetNodeName = channel->GetName(); + haveTargetNodeName = true; + + // We can also use this message as the base template for other messages + if (!this->sampleHeader.initLengthUninitialized( + sizeof(IPC::Message::Header))) { + MOZ_FUZZING_NYX_ABORT("sampleHeader.initLengthUninitialized failed\n"); + } + + memcpy(sampleHeader.begin(), aMessage.header(), + sizeof(IPC::Message::Header)); + } + } else if (haveTargetNodeName && targetNodeName != channel->GetName()) { + // Not our node, no need to observe + return true; + } else if (Nyx::instance().started()) { + // When fuzzing is already started, we shouldn't observe messages anymore. + if (!channel->mBlockSendRecv) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [NodeChannel::OnMessageReceived] Blocking further " + "communication on Port %lu %lu (fuzzing started)\n", + channel->GetName().v1, channel->GetName().v2); + channel->mBlockSendRecv = true; + } + return false; + } + + Vector<char, 256, InfallibleAllocPolicy> footer; + + if (!footer.initLengthUninitialized(aMessage.event_footer_size())) { + MOZ_FUZZING_NYX_ABORT("footer.initLengthUninitialized failed\n"); + } + + if (!aMessage.ReadFooter(footer.begin(), footer.length(), false)) { + MOZ_FUZZING_NYX_ABORT("ERROR: ReadFooter() failed?!\n"); + } + + UniquePtr<Event> event = Event::Deserialize(footer.begin(), footer.length()); + + if (!event) { + MOZ_FUZZING_NYX_ABORT("ERROR: Failed to deserialize observed message?!\n"); + } + + if (event->type() == Event::kUserMessage) { + if (haveTargetNodeName && !fuzzingStartPending) { + bool missingActor = false; + + // Check if we have any entries in our port map that we haven't seen yet + // though `OnActorConnected`. That method is called on a background + // thread and this call will race with the I/O thread. + // + // However, with a custom MOZ_FUZZ_IPC_TRIGGER we assume we want to keep + // the port pinned so we don't have to wait at all. + if (mIPCTriggerMsg == ipcDefaultTriggerMsg) { + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::ObserveIPCMessage() Mutex try\n"); + // Called on the I/O thread and reads `portSeqNos`. + // + // IMPORTANT: We must give up any locks before entering `StartFuzzing`, + // as we will never return. This would cause a deadlock with new actors + // being created and `OnActorConnected` being called. + MutexAutoLock lock(mMutex); + + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::ObserveIPCMessage() Mutex locked\n"); + + for (auto iter = portSeqNos.begin(); iter != portSeqNos.end(); ++iter) { + auto result = actorIds.find(iter->first); + if (result == actorIds.end()) { + // Make sure we only wait for actors that belong to us. + auto result = portNodeName.find(iter->first); + if (result->second == targetNodeName) { + missingActor = true; + break; + } + } + } + } + + if (missingActor) { + MOZ_FUZZING_NYX_PRINT( + "INFO: Delaying fuzzing start, missing actors...\n"); + } else if (!childReady) { + MOZ_FUZZING_NYX_PRINT( + "INFO: Delaying fuzzing start, waiting for child...\n"); + } else { + fuzzingStartPending = true; + StartFuzzing(channel, aMessage); + + // In the async case, we return and can already block the relevant + // communication. + if (targetNodeName == channel->GetName()) { + if (!channel->mBlockSendRecv) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [NodeChannel::OnMessageReceived] Blocking further " + "communication on Port %lu %lu (fuzzing start pending)\n", + channel->GetName().v1, channel->GetName().v2); + channel->mBlockSendRecv = true; + } + + return false; + } + return true; + } + } + + // Add/update sequence numbers. We need to make sure to do this after our + // call to `StartFuzzing` because once we start fuzzing, the message will + // never actually be processed, so we run into a sequence number desync. + { + // Get the port name associated with this message + UserMessageEvent* userMsgEv = static_cast<UserMessageEvent*>(event.get()); + PortName name = event->port_name(); + + // Called on the I/O thread and modifies `portSeqNos`. + MutexAutoLock lock(mMutex); + portSeqNos.insert_or_assign( + name, std::pair<int32_t, uint64_t>(aMessage.seqno(), + userMsgEv->sequence_num())); + + portNodeName.insert_or_assign(name, channel->GetName()); + } + } + + return true; +} + +void IPCFuzzController::OnMessageError( + mozilla::ipc::HasResultCodes::Result code, const IPC::Message& aMsg) { + if (!mozilla::fuzzing::Nyx::instance().is_enabled("IPC_Generic")) { + // Fuzzer is not enabled. + return; + } + + if (!XRE_IsParentProcess()) { + // For now we only care about things in the parent process. + return; + } + + if (!aMsg.IsFuzzMsg()) { + // We should only act upon fuzzing messages. + return; + } + + switch (code) { + case ipc::HasResultCodes::MsgNotKnown: + // Seeing this error should be rare - one potential reason is if a sync + // message is sent as async and vice versa. Other than that, we shouldn't + // be generating this error at all. + Nyx::instance().handle_event("MOZ_IPC_UNKNOWN_TYPE", nullptr, 0, nullptr); +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF( + "WARNING: MOZ_IPC_UNKNOWN_TYPE for message type %s (%u) routed to " + "actor %d (sync %d)\n", + IPC::StringFromIPCMessageType(aMsg.type()), aMsg.type(), + aMsg.routing_id(), aMsg.is_sync()); +#endif + break; + case ipc::HasResultCodes::MsgNotAllowed: + Nyx::instance().handle_event("MOZ_IPC_NOTALLOWED_ERROR", nullptr, 0, + nullptr); + break; + case ipc::HasResultCodes::MsgPayloadError: + case ipc::HasResultCodes::MsgValueError: + Nyx::instance().handle_event("MOZ_IPC_DESERIALIZE_ERROR", nullptr, 0, + nullptr); + break; + case ipc::HasResultCodes::MsgProcessingError: + Nyx::instance().handle_event("MOZ_IPC_PROCESS_ERROR", nullptr, 0, + nullptr); + break; + case ipc::HasResultCodes::MsgRouteError: + Nyx::instance().handle_event("MOZ_IPC_ROUTE_ERROR", nullptr, 0, nullptr); + break; + default: + MOZ_FUZZING_NYX_ABORT("unknown Result code"); + } + + // Count this message as one iteration as well. + Nyx::instance().release(IPCFuzzController::instance().getMessageStopCount() + + 1); +} + +bool IPCFuzzController::MakeTargetDecision( + uint8_t portIndex, uint8_t portInstanceIndex, uint8_t actorIndex, + uint16_t typeOffset, PortName* name, int32_t* seqno, uint64_t* fseqno, + int32_t* actorId, uint32_t* type, bool* is_cons, bool update) { + // Every possible toplevel actor type has a fixed number that + // we assign to it in the constructor of this class. Here, we + // use the lower 6 bits to select this toplevel actor type. + // This approach has the advantage that the tests will always + // select the same toplevel actor type deterministically, + // independent of the order they appeared and independent + // of the type of fuzzing we are doing. + auto portInstances = portNames[portIndex & 0x3f]; + if (!portInstances.size()) { + return false; + } + + if (useLastActor) { + useLastActor--; + *name = lastActorPortName; + + MOZ_FUZZING_NYX_PRINT("DEBUG: MakeTargetDecision: Pinned to last actor.\n"); + + // Once we stop pinning to the last actor, we need to decide if we + // want to keep the pinning on the port itself. We use one of the + // unused upper bits of portIndex for this purpose. + if (!useLastActor && (portIndex & (1 << 7))) { + if (mIPCTriggerMsg == ipcDefaultTriggerMsg) { + MOZ_FUZZING_NYX_PRINT( + "DEBUG: MakeTargetDecision: Released pinning on last port.\n"); + useLastPortName = false; + } + } + } else if (useLastPortName) { + *name = lastActorPortName; + MOZ_FUZZING_NYX_PRINT("DEBUG: MakeTargetDecision: Pinned to last port.\n"); + } else { + *name = portInstances[portInstanceIndex % portInstances.size()]; + } + + // We should always have at least one actor per port + auto result = actorIds.find(*name); + if (result == actorIds.end()) { + MOZ_FUZZING_NYX_PRINT("ERROR: Couldn't find port in actors map?!\n"); + return false; + } + + // Find a random actor on this port + auto actors = result->second; + if (actors.empty()) { + MOZ_FUZZING_NYX_PRINT( + "ERROR: Couldn't find an actor for selected port?!\n"); + return false; + } + + auto seqNos = portSeqNos[*name]; + + // Hand out the correct sequence numbers + *seqno = seqNos.first - 1; + *fseqno = seqNos.second + 1; + + // If a type is already specified, we must be in preserveHeaderMode. + bool isPreserveHeader = *type; + + if (useLastActor) { + actorIndex = actors.size() - 1; + } else if (isPreserveHeader) { + // In preserveHeaderMode, we need to find an actor that matches the + // requested message type instead of any random actor. + uint16_t maybeProtocolId = *type >> 16; + if (maybeProtocolId >= IPCMessageStart::LastMsgIndex) { + // Not a valid protocol. + return false; + } + ProtocolId wantedProtocolId = static_cast<ProtocolId>(maybeProtocolId); + std::vector<uint32_t> allowedIndices; + for (uint32_t i = 0; i < actors.size(); ++i) { + if (actors[i].second == wantedProtocolId) { + allowedIndices.push_back(i); + } + } + + if (allowedIndices.empty()) { + return false; + } + + actorIndex = allowedIndices[actorIndex % allowedIndices.size()]; + } else { + actorIndex %= actors.size(); + } + + ActorIdPair ids = actors[actorIndex]; + *actorId = ids.first; + + // If the actor ID is 0, then we are talking to the toplevel actor + // of this port. Hence we must set the ID to MSG_ROUTING_CONTROL. + if (!*actorId) { + *actorId = MSG_ROUTING_CONTROL; + } + + if (!isPreserveHeader) { + // If msgType is already set, then we are in preserveHeaderMode + if (!this->GetRandomIPCMessageType(ids.second, typeOffset, type)) { + MOZ_FUZZING_NYX_PRINT("ERROR: GetRandomIPCMessageType failed?!\n"); + return false; + } + + *is_cons = false; + if (constructorTypes.find(*type) != constructorTypes.end()) { + *is_cons = true; + } + } + + MOZ_FUZZING_NYX_PRINTF( + "DEBUG: MakeTargetDecision: Top-Level Protocol: %s Protocol: %s msgType: " + "%s (%u), Actor Instance %u of %zu, actor ID: %d, PreservedHeader: %d\n", + portNameToProtocolName[*name].c_str(), ProtocolIdToName(ids.second), + IPC::StringFromIPCMessageType(*type), *type, actorIndex, actors.size(), + *actorId, isPreserveHeader); + + if (update) { + portSeqNos.insert_or_assign(*name, + std::pair<int32_t, uint64_t>(*seqno, *fseqno)); + } + + return true; +} + +void IPCFuzzController::OnMessageTaskStart() { messageStartCount++; } + +void IPCFuzzController::OnMessageTaskStop() { messageStopCount++; } + +void IPCFuzzController::OnPreFuzzMessageTaskRun() { messageTaskCount++; } +void IPCFuzzController::OnPreFuzzMessageTaskStop() { messageTaskCount--; } + +void IPCFuzzController::OnDropPeer(const char* reason = nullptr, + const char* file = nullptr, int line = 0) { + if (!XRE_IsParentProcess()) { + return; + } + + if (!Nyx::instance().started()) { + // It's possible to close a connection to some peer before we have even + // started fuzzing. We ignore these events until we are actually fuzzing. + return; + } + + MOZ_FUZZING_NYX_PRINT( + "ERROR: ======== END OF ITERATION (DROP_PEER) ========\n"); +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF("DEBUG: ======== %s:%d ========\n", file, line); +#endif + Nyx::instance().handle_event("MOZ_IPC_DROP_PEER", file, line, reason); + + if (Nyx::instance().is_replay()) { + // In replay mode, let's ignore drop peer to avoid races with it. + return; + } + + Nyx::instance().release(IPCFuzzController::instance().getMessageStopCount()); +} + +void IPCFuzzController::StartFuzzing(mozilla::ipc::NodeChannel* channel, + IPC::Message& aMessage) { + nodeChannel = channel; + + RefPtr<IPCFuzzLoop> runnable = new IPCFuzzLoop(); + +#if MOZ_FUZZ_IPC_SYNC_INJECT + runnable->Run(); +#else + nsCOMPtr<nsIThread> newThread; + nsresult rv = + NS_NewNamedThread("IPCFuzzLoop", getter_AddRefs(newThread), runnable); + + if (NS_FAILED(rv)) { + MOZ_FUZZING_NYX_ABORT("ERROR: [StartFuzzing] NS_NewNamedThread failed?!\n"); + } +#endif +} + +IPCFuzzController::IPCFuzzLoop::IPCFuzzLoop() + : mozilla::Runnable("IPCFuzzLoop") {} + +NS_IMETHODIMP IPCFuzzController::IPCFuzzLoop::Run() { + MOZ_FUZZING_NYX_DEBUG("DEBUG: BEGIN IPCFuzzLoop::Run()\n"); + + const size_t maxMsgSize = 2048; + const size_t controlLen = 16; + + Vector<char, 256, InfallibleAllocPolicy> buffer; + + RefPtr<NodeController> controller = NodeController::GetSingleton(); + + // TODO: The following code is full of data races. We need synchronization + // on the `IPCFuzzController` instance, because the I/O thread can call into + // this class via ObserveIPCMessages. The problem is that any such call + // must either be observed to update the sequence numbers, or the packet + // must be dropped already. + if (!IPCFuzzController::instance().haveTargetNodeName) { + MOZ_FUZZING_NYX_ABORT("ERROR: I don't have the target NodeName?!\n"); + } + + { + MOZ_FUZZING_NYX_DEBUG("DEBUG: IPCFuzzLoop::Run() Mutex try\n"); + // Called on the I/O thread and modifies `portSeqNos` and `actorIds`. + MutexAutoLock lock(IPCFuzzController::instance().mMutex); + MOZ_FUZZING_NYX_DEBUG("DEBUG: IPCFuzzLoop::Run() Mutex locked\n"); + + // The wait/delay logic in ObserveIPCMessage should ensure that we haven't + // seen any packets on ports for which we haven't received actor information + // yet, if those ports belong to our channel. However, we might also have + // seen ports not belonging to our channel, which we have to remove now. + for (auto iter = IPCFuzzController::instance().portSeqNos.begin(); + iter != IPCFuzzController::instance().portSeqNos.end();) { + auto result = IPCFuzzController::instance().actorIds.find(iter->first); + if (result == IPCFuzzController::instance().actorIds.end()) { + auto portNameResult = + IPCFuzzController::instance().portNodeName.find(iter->first); + if (portNameResult->second == + IPCFuzzController::instance().targetNodeName) { + MOZ_FUZZING_NYX_PRINT( + "ERROR: We should not have port map entries without a " + "corresponding " + "entry in our actors map\n"); + MOZ_REALLY_CRASH(__LINE__); + } else { + iter = IPCFuzzController::instance().portSeqNos.erase(iter); + } + } else { + ++iter; + } + } + + // TODO: Technically, at this point we only know that PContent (or whatever + // toplevel protocol we decided to synchronize on), is present. It might + // be possible that others aren't created yet and we are racing on this. + // + // Note: The delay logic mentioned above makes this less likely. Only actors + // which are created on-demand and which have not been referenced yet at all + // would be affected by such a race. + for (auto iter = IPCFuzzController::instance().actorIds.begin(); + iter != IPCFuzzController::instance().actorIds.end(); ++iter) { + bool isValidTarget = false; + Maybe<PortStatus> status; + PortRef ref = controller->GetPort(iter->first); + if (ref.is_valid()) { + status = controller->GetStatus(ref); + if (status) { + isValidTarget = status->peer_node_name == + IPCFuzzController::instance().targetNodeName; + } + } + + auto result = IPCFuzzController::instance().portSeqNos.find(iter->first); + if (result == IPCFuzzController::instance().portSeqNos.end()) { + if (isValidTarget) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: Using Port %lu %lu for protocol %s (*)\n", iter->first.v1, + iter->first.v2, ProtocolIdToName(iter->second[0].second)); + + // Normally the start sequence numbers would be -1 and 1, but our map + // does not record the next numbers, but the "last seen" state. So we + // have to adjust these so the next calculated sequence number pair + // matches the start sequence numbers. + IPCFuzzController::instance().portSeqNos.insert_or_assign( + iter->first, std::pair<int32_t, uint64_t>(0, 0)); + + IPCFuzzController::instance().AddToplevelActor( + iter->first, iter->second[0].second); + + } else { + MOZ_FUZZING_NYX_PRINTF( + "INFO: Removing Port %lu %lu for protocol %s (*)\n", + iter->first.v1, iter->first.v2, + ProtocolIdToName(iter->second[0].second)); + + // This toplevel actor does not belong to us, but we haven't added + // it to `portSeqNos`, so we don't have to remove it. + } + } else { + if (isValidTarget) { + MOZ_FUZZING_NYX_PRINTF("INFO: Using Port %lu %lu for protocol %s\n", + iter->first.v1, iter->first.v2, + ProtocolIdToName(iter->second[0].second)); + + IPCFuzzController::instance().AddToplevelActor( + iter->first, iter->second[0].second); + } else { + MOZ_FUZZING_NYX_PRINTF( + "INFO: Removing Port %lu %lu for protocol %s\n", iter->first.v1, + iter->first.v2, ProtocolIdToName(iter->second[0].second)); + + // This toplevel actor does not belong to us, so remove it. + IPCFuzzController::instance().portSeqNos.erase(result); + } + } + } + } + + IPCFuzzController::instance().runnableDone = false; + + SyncRunnable::DispatchToThread( + GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction("IPCFuzzController::StartFuzzing", [&]() -> void { + MOZ_FUZZING_NYX_PRINT("INFO: Main thread runnable start.\n"); + NS_ProcessPendingEvents(NS_GetCurrentThread()); + MOZ_FUZZING_NYX_PRINT("INFO: Main thread runnable done.\n"); + })); + + MOZ_FUZZING_NYX_PRINT("INFO: Performing snapshot...\n"); + Nyx::instance().start(); + + uint32_t expected_messages = 0; + + if (!buffer.initLengthUninitialized(maxMsgSize)) { + MOZ_FUZZING_NYX_ABORT("ERROR: Failed to initialize buffer!\n"); + } + + for (int i = 0; i < 3; ++i) { + // Grab enough data to potentially fill our everything except the footer. + uint32_t bufsize = + Nyx::instance().get_data((uint8_t*)buffer.begin(), buffer.length()); + + if (bufsize == 0xFFFFFFFF) { + // Done constructing + MOZ_FUZZING_NYX_DEBUG("Iteration complete: Out of data.\n"); + break; + } + + // Payload must be int aligned + bufsize -= bufsize % 4; + + // Need at least a header and the control bytes. + if (bufsize < sizeof(IPC::Message::Header) + controlLen) { + MOZ_FUZZING_NYX_DEBUG("INFO: Not enough data to craft IPC message.\n"); + continue; + } + + const uint8_t* controlData = (uint8_t*)buffer.begin(); + + char* ipcMsgData = buffer.begin() + controlLen; + size_t ipcMsgLen = bufsize - controlLen; + + bool preserveHeader = controlData[15] == 0xFF; + + if (!preserveHeader) { + // Copy the header of the original message + memcpy(ipcMsgData, IPCFuzzController::instance().sampleHeader.begin(), + sizeof(IPC::Message::Header)); + } + + IPC::Message::Header* ipchdr = (IPC::Message::Header*)ipcMsgData; + + ipchdr->payload_size = ipcMsgLen - sizeof(IPC::Message::Header); + + PortName new_port_name; + int32_t new_seqno; + uint64_t new_fseqno; + + int32_t actorId; + uint32_t msgType = 0; + bool isConstructor = false; + // Control Data Layout (16 byte) + // Byte 0 - Port Index (selects out of the valid ports seen) + // Byte 1 - Actor Index (selects one of the actors for that port) + // Byte 2 - Type Offset (select valid type for the specified actor) + // Byte 3 - ^- continued + // Byte 4 - Sync Bit + // Byte 5 - Optionally select a particular instance of the selected + // port type. Some toplevel protocols can have multiple + // instances running at the same time. + // + // Byte 15 - If set to 0xFF, skip overwriting the header, leave fields + // like message type intact and only set target actor and + // other fields that are dynamic. + + uint8_t portIndex = controlData[0]; + uint8_t actorIndex = controlData[1]; + uint16_t typeOffset = *(uint16_t*)(&controlData[2]); + bool isSync = controlData[4] > 127; + uint8_t portInstanceIndex = controlData[5]; + + UniquePtr<IPC::Message> msg(new IPC::Message(ipcMsgData, ipcMsgLen)); + + if (preserveHeader) { + isConstructor = msg->is_constructor(); + isSync = msg->is_sync(); + msgType = msg->header()->type; + + if (!msgType) { + // msgType == 0 is used to indicate to MakeTargetDecision that we are + // not in preserve header mode. It's not a valid message type in any + // case and we can error out early. + Nyx::instance().release( + IPCFuzzController::instance().getMessageStopCount()); + } + } + + if (!IPCFuzzController::instance().MakeTargetDecision( + portIndex, portInstanceIndex, actorIndex, typeOffset, + &new_port_name, &new_seqno, &new_fseqno, &actorId, &msgType, + &isConstructor)) { + MOZ_FUZZING_NYX_DEBUG("DEBUG: MakeTargetDecision returned false.\n"); + continue; + } + + if (Nyx::instance().is_replay()) { + MOZ_FUZZING_NYX_PRINT("INFO: Replaying IPC packet with payload:\n"); + for (uint32_t i = 0; i < ipcMsgLen - sizeof(IPC::Message::Header); ++i) { + if (i % 16 == 0) { + MOZ_FUZZING_NYX_PRINT("\n "); + } + + MOZ_FUZZING_NYX_PRINTF( + "0x%02X ", + (unsigned char)(ipcMsgData[sizeof(IPC::Message::Header) + i])); + } + MOZ_FUZZING_NYX_PRINT("\n"); + } + + if (isConstructor) { + MOZ_FUZZING_NYX_DEBUG("DEBUG: Sending constructor message...\n"); + msg->header()->flags.SetConstructor(); + } + + if (!isConstructor && isSync) { + MOZ_FUZZING_NYX_DEBUG("INFO: Sending sync message...\n"); + msg->header()->flags.SetSync(); + } + + msg->set_seqno(new_seqno); + msg->set_routing_id(actorId); + + if (!preserveHeader) { + // TODO: There is no setter for this. + msg->header()->type = msgType; + } + + // Create the footer + auto messageEvent = MakeUnique<UserMessageEvent>(0); + messageEvent->set_port_name(new_port_name); + messageEvent->set_sequence_num(new_fseqno); + + Vector<char, 256, InfallibleAllocPolicy> footerBuffer; + (void)footerBuffer.initLengthUninitialized( + messageEvent->GetSerializedSize()); + messageEvent->Serialize(footerBuffer.begin()); + + msg->WriteFooter(footerBuffer.begin(), footerBuffer.length()); + msg->set_event_footer_size(footerBuffer.length()); + + // This marks the message as a fuzzing message. Without this, it will + // be ignored by MessageTask and also not even scheduled by NodeChannel + // in asynchronous mode. We use this to ignore any IPC activity that + // happens just while we are fuzzing. + msg->SetFuzzMsg(); + +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF( + "DEBUG: OnEventMessage iteration %d, EVS: %u Payload: %u.\n", i, + ipchdr->event_footer_size, ipchdr->payload_size); +#endif + +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF("DEBUG: OnEventMessage: Port %lu %lu. Actor %d\n", + new_port_name.v1, new_port_name.v2, actorId); + MOZ_FUZZING_NYX_PRINTF( + "DEBUG: OnEventMessage: Flags: %u TxID: %d Handles: %u\n", + msg->header()->flags, msg->header()->txid, msg->header()->num_handles); +#endif + + // The number of messages we expect to see stopped. + expected_messages++; + +#if MOZ_FUZZ_IPC_SYNC_INJECT + // For synchronous injection, we just call OnMessageReceived directly. + IPCFuzzController::instance().nodeChannel->OnMessageReceived( + std::move(msg)); +#else + // For asynchronous injection, we have to post to the I/O thread instead. + XRE_GetIOMessageLoop()->PostTask(NS_NewRunnableFunction( + "NodeChannel::OnMessageReceived", + [msg = std::move(msg), + nodeChannel = + RefPtr{IPCFuzzController::instance().nodeChannel}]() mutable { + int32_t msgType = msg->header()->type; + + // By default, we sync on the target thread of the receiving actor. + bool syncOnIOThread = false; + + switch (msgType) { + case DATA_PIPE_CLOSED_MESSAGE_TYPE: + case DATA_PIPE_BYTES_CONSUMED_MESSAGE_TYPE: + case ACCEPT_INVITE_MESSAGE_TYPE: + case REQUEST_INTRODUCTION_MESSAGE_TYPE: + case INTRODUCE_MESSAGE_TYPE: + case BROADCAST_MESSAGE_TYPE: + // This set of special messages will not be routed to actors and + // therefore we won't see these as stopped messages later. These + // messages are either used by NodeChannel, DataPipe or + // MessageChannel without creating MessageTasks. As such, the best + // we can do is synchronize on this thread. We do this by + // emulating the MessageTaskStart/Stop behavior that normal event + // messages have. + syncOnIOThread = true; + break; + default: + // Synchronization will happen in MessageChannel. Note that this + // also applies to certain special message types, as long as they + // are received by actors and not intercepted earlier. + break; + } + + if (syncOnIOThread) { + mozilla::fuzzing::IPCFuzzController::instance() + .OnMessageTaskStart(); + } + + nodeChannel->OnMessageReceived(std::move(msg)); + + if (syncOnIOThread) { + mozilla::fuzzing::IPCFuzzController::instance().OnMessageTaskStop(); + + // Don't continue for now after sending such a special message. + // It can cause ports to go away and further messages can time out. + Nyx::instance().release( + IPCFuzzController::instance().getMessageStopCount()); + } + })); +#endif + +#ifdef MOZ_FUZZ_IPC_SYNC_AFTER_EACH_MSG + MOZ_FUZZING_NYX_DEBUG("DEBUG: Synchronizing after message...\n"); + IPCFuzzController::instance().SynchronizeOnMessageExecution( + expected_messages); + + SyncRunnable::DispatchToThread( + GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction( + "IPCFuzzController::StartFuzzing", [&]() -> void { + MOZ_FUZZING_NYX_DEBUG("DEBUG: Main thread runnable start.\n"); + NS_ProcessPendingEvents(NS_GetCurrentThread()); + MOZ_FUZZING_NYX_DEBUG("DEBUG: Main thread runnable done.\n"); + })); +#else + + if (isConstructor) { + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: Synchronizing due to constructor message...\n"); + IPCFuzzController::instance().SynchronizeOnMessageExecution( + expected_messages); + } +#endif + } + +#ifndef MOZ_FUZZ_IPC_SYNC_AFTER_EACH_MSG + MOZ_FUZZING_NYX_DEBUG("DEBUG: Synchronizing due to end of iteration...\n"); + IPCFuzzController::instance().SynchronizeOnMessageExecution( + expected_messages); + + SyncRunnable::DispatchToThread( + GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction("IPCFuzzController::StartFuzzing", [&]() -> void { + MOZ_FUZZING_NYX_DEBUG("DEBUG: Main thread runnable start.\n"); + NS_ProcessPendingEvents(NS_GetCurrentThread()); + MOZ_FUZZING_NYX_DEBUG("DEBUG: Main thread runnable done.\n"); + })); +#endif + + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: ======== END OF ITERATION (RELEASE) ========\n"); + + Nyx::instance().release(IPCFuzzController::instance().getMessageStopCount()); + + // Never reached. + return NS_OK; +} + +void IPCFuzzController::SynchronizeOnMessageExecution( + uint32_t expected_messages) { + // This synchronization will work in both the sync and async case. + // For the async case, it is important to wait for the exact stop count + // because the message task is not even started potentially when we + // read this loop. + int hang_timeout = 10 * 1000; + while (IPCFuzzController::instance().getMessageStopCount() != + expected_messages) { +#ifdef FUZZ_DEBUG + uint32_t count_stopped = + IPCFuzzController::instance().getMessageStopCount(); + uint32_t count_live = IPCFuzzController::instance().getMessageStartCount(); + MOZ_FUZZING_NYX_PRINTF( + "DEBUG: Post Constructor: %d stopped messages (%d live, %d " + "expected)!\n", + count_stopped, count_live, expected_messages); +#endif + PR_Sleep(PR_MillisecondsToInterval(50)); + hang_timeout -= 50; + + if (hang_timeout <= 0) { + Nyx::instance().handle_event("MOZ_TIMEOUT", nullptr, 0, nullptr); + MOZ_FUZZING_NYX_PRINT( + "ERROR: ======== END OF ITERATION (TIMEOUT) ========\n"); + Nyx::instance().release( + IPCFuzzController::instance().getMessageStopCount()); + } + } +} + +static void dumpIPCMessageToFile(const UniquePtr<IPC::Message>& aMsg, + uint32_t aDumpCount, bool aUseNyx = false) { + if (Nyx::instance().is_replay()) { + return; + } + + std::stringstream dumpFilename; + std::string msgName(IPC::StringFromIPCMessageType(aMsg->type())); + std::replace(msgName.begin(), msgName.end(), ':', '_'); + + if (aUseNyx) { + dumpFilename << "seeds/"; + } + + dumpFilename << msgName << aDumpCount << ".bin"; + + Pickle::BufferList::IterImpl iter(aMsg->Buffers()); + Vector<char, 256, InfallibleAllocPolicy> dumpBuffer; + if (!dumpBuffer.initLengthUninitialized(sizeof(IPC::Message::Header) + + aMsg->Buffers().Size())) { + MOZ_FUZZING_NYX_ABORT("dumpBuffer.initLengthUninitialized failed\n"); + } + if (!aMsg->Buffers().ReadBytes( + iter, + reinterpret_cast<char*>(dumpBuffer.begin() + + sizeof(IPC::Message::Header)), + dumpBuffer.length() - sizeof(IPC::Message::Header))) { + MOZ_FUZZING_NYX_ABORT("ReadBytes failed\n"); + } + memcpy(dumpBuffer.begin(), aMsg->header(), sizeof(IPC::Message::Header)); + + if (aUseNyx) { + MOZ_FUZZING_NYX_PRINTF("INFO: Calling dump_file: %s Size: %zu\n", + dumpFilename.str().c_str(), dumpBuffer.length()); + Nyx::instance().dump_file(reinterpret_cast<char*>(dumpBuffer.begin()), + dumpBuffer.length(), dumpFilename.str().c_str()); + } else { + std::fstream file; + file.open(dumpFilename.str(), std::ios::out | std::ios::binary); + file.write(reinterpret_cast<char*>(dumpBuffer.begin()), + dumpBuffer.length()); + file.close(); + } +} + +UniquePtr<IPC::Message> IPCFuzzController::replaceIPCMessage( + UniquePtr<IPC::Message> aMsg) { + if (!mozilla::fuzzing::Nyx::instance().is_enabled("IPC_SingleMessage")) { + // Fuzzer is not enabled. + return aMsg; + } + + if (!XRE_IsParentProcess()) { + // For now we only care about things in the parent process. + return aMsg; + } + + static bool dumpFilterInitialized = false; + static std::string dumpFilter; + if (!dumpFilterInitialized) { + const char* dumpFilterStr = getenv("MOZ_FUZZ_DUMP_FILTER"); + if (dumpFilterStr) { + dumpFilter = std::string(dumpFilterStr); + } + dumpFilterInitialized = true; + } + + if (aMsg->type() != mIPCTriggerMsg) { + if ((mIPCDumpMsg && aMsg->type() == mIPCDumpMsg.value()) || + (mIPCDumpAllMsgsSize.isSome() && + aMsg->Buffers().Size() >= mIPCDumpAllMsgsSize.value())) { + if (!dumpFilter.empty()) { + std::string msgName(IPC::StringFromIPCMessageType(aMsg->type())); + if (msgName.find(dumpFilter) != std::string::npos) { + dumpIPCMessageToFile(aMsg, mIPCDumpCount); + mIPCDumpCount++; + } + } else { + dumpIPCMessageToFile(aMsg, mIPCDumpCount); + mIPCDumpCount++; + } + } + + // Not the trigger message. Output additional information here for + // automation purposes. This shouldn't be an issue as we will only + // output these messages until we take a snapshot. + MOZ_FUZZING_NYX_PRINTF("INFO: [OnIPCMessage] Message: %s Size: %u\n", + IPC::StringFromIPCMessageType(aMsg->type()), + aMsg->header()->payload_size); + return aMsg; + } else { + // Dump the trigger message through Nyx in case we want to use it + // as a seed to AFL++ outside of the VM. + dumpIPCMessageToFile(aMsg, mIPCDumpCount, true /* aUseNyx */); + mIPCDumpCount++; + if (mIPCTriggerSingleMsgWait > 0) { + mIPCTriggerSingleMsgWait--; + return aMsg; + } + } + + const size_t maxMsgSize = 4096; + + Vector<char, 256, InfallibleAllocPolicy> buffer; + if (!buffer.initLengthUninitialized(maxMsgSize)) { + MOZ_FUZZING_NYX_ABORT("ERROR: Failed to initialize buffer!\n"); + } + + char* ipcMsgData = buffer.begin(); + + // // + // *** Snapshot Point *** // + // // + MOZ_FUZZING_NYX_PRINT("INFO: Performing snapshot...\n"); + Nyx::instance().start(); + + IPCFuzzController::instance().useLastActor = 0; + IPCFuzzController::instance().useLastPortName = false; + + MOZ_FUZZING_NYX_DEBUG("DEBUG: Requesting data...\n"); + + // Grab enough data to send at most `maxMsgSize` bytes + uint32_t bufsize = + Nyx::instance().get_raw_data((uint8_t*)buffer.begin(), buffer.length()); + + if (bufsize == 0xFFFFFFFF) { + MOZ_FUZZING_NYX_DEBUG("Nyx: Out of data.\n"); + Nyx::instance().release(0); + } + +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF("DEBUG: Got buffer of size %u...\n", bufsize); +#endif + + // Payload must be int aligned + bufsize -= bufsize % 4; + + // Need at least a header and the control bytes. + if (bufsize < sizeof(IPC::Message::Header)) { + MOZ_FUZZING_NYX_DEBUG("INFO: Not enough data to craft IPC message.\n"); + Nyx::instance().release(0); + } + + buffer.shrinkTo(bufsize); + + // Copy the header of the original message + memcpy(ipcMsgData, aMsg->header(), sizeof(IPC::Message::Header)); + IPC::Message::Header* ipchdr = (IPC::Message::Header*)ipcMsgData; + + size_t ipcMsgLen = buffer.length(); + ipchdr->payload_size = ipcMsgLen - sizeof(IPC::Message::Header); + + if (Nyx::instance().is_replay()) { + MOZ_FUZZING_NYX_PRINT("INFO: Replaying IPC packet with payload:\n"); + for (uint32_t i = 0; i < ipcMsgLen - sizeof(IPC::Message::Header); ++i) { + if (i % 16 == 0) { + MOZ_FUZZING_NYX_PRINT("\n "); + } + + MOZ_FUZZING_NYX_PRINTF( + "0x%02X ", + (unsigned char)(ipcMsgData[sizeof(IPC::Message::Header) + i])); + } + MOZ_FUZZING_NYX_PRINT("\n"); + } + + UniquePtr<IPC::Message> msg(new IPC::Message(ipcMsgData, ipcMsgLen)); + + // This marks the message as a fuzzing message. Without this, it will + // be ignored by MessageTask and also not even scheduled by NodeChannel + // in asynchronous mode. We use this to ignore any IPC activity that + // happens just while we are fuzzing. + msg->SetFuzzMsg(); + + return msg; +} + +void IPCFuzzController::syncAfterReplace() { + if (!mozilla::fuzzing::Nyx::instance().is_enabled("IPC_SingleMessage")) { + // Fuzzer is not enabled. + return; + } + + if (!XRE_IsParentProcess()) { + // For now we only care about things in the parent process. + return; + } + + if (!Nyx::instance().started()) { + // Not started yet + return; + } + + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: ======== END OF ITERATION (RELEASE) ========\n"); + + Nyx::instance().release(1); +} + +} // namespace fuzzing +} // namespace mozilla diff --git a/tools/fuzzing/ipc/IPCFuzzController.h b/tools/fuzzing/ipc/IPCFuzzController.h new file mode 100644 index 0000000000..756a68f38f --- /dev/null +++ b/tools/fuzzing/ipc/IPCFuzzController.h @@ -0,0 +1,216 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_ipc_IPCFuzzController_h +#define mozilla_ipc_IPCFuzzController_h + +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/HashTable.h" +#include "mozilla/Mutex.h" +#include "mozilla/fuzzing/Nyx.h" +#include "mozilla/ipc/MessageLink.h" + +#include "nsIRunnable.h" +#include "nsThreadUtils.h" + +#include "chrome/common/ipc_message.h" +#include "mojo/core/ports/name.h" +#include "mojo/core/ports/event.h" + +#include "IPCMessageStart.h" + +#include <unordered_map> +#include <unordered_set> +#include <string> +#include <utility> +#include <vector> + +#define MOZ_FUZZING_IPC_DROP_PEER(aReason) \ + mozilla::fuzzing::IPCFuzzController::instance().OnDropPeer( \ + aReason, __FILE__, __LINE__); + +#define MOZ_FUZZING_IPC_MT_CTOR() \ + mozilla::fuzzing::IPCFuzzController::instance().OnMessageTaskStart(); + +#define MOZ_FUZZING_IPC_MT_STOP() \ + mozilla::fuzzing::IPCFuzzController::instance().OnMessageTaskStop(); + +#define MOZ_FUZZING_IPC_PRE_FUZZ_MT_RUN() \ + mozilla::fuzzing::IPCFuzzController::instance().OnPreFuzzMessageTaskRun(); + +#define MOZ_FUZZING_IPC_PRE_FUZZ_MT_STOP() \ + mozilla::fuzzing::IPCFuzzController::instance().OnPreFuzzMessageTaskStop(); + +namespace mozilla { + +namespace ipc { +// We can't include ProtocolUtils.h here +class IProtocol; +typedef IPCMessageStart ProtocolId; + +class NodeChannel; +} // namespace ipc + +namespace fuzzing { + +class IPCFuzzController { + typedef std::pair<int32_t, uint64_t> SeqNoPair; + + typedef std::pair<int32_t, mozilla::ipc::ProtocolId> ActorIdPair; + + class IPCFuzzLoop final : public Runnable { + friend class IPCFuzzController; + + public: + NS_DECL_NSIRUNNABLE + + IPCFuzzLoop(); + + private: + ~IPCFuzzLoop() = default; + }; + + public: + static IPCFuzzController& instance(); + + void InitializeIPCTypes(); + bool GetRandomIPCMessageType(mozilla::ipc::ProtocolId pId, + uint16_t typeOffset, uint32_t* type); + + bool ObserveIPCMessage(mozilla::ipc::NodeChannel* channel, + IPC::Message& aMessage); + bool MakeTargetDecision(uint8_t portIndex, uint8_t portInstanceIndex, + uint8_t actorIndex, uint16_t typeOffset, + mojo::core::ports::PortName* name, int32_t* seqno, + uint64_t* fseqno, int32_t* actorId, uint32_t* type, + bool* is_cons, bool update = true); + + void OnActorConnected(mozilla::ipc::IProtocol* protocol); + void OnActorDestroyed(mozilla::ipc::IProtocol* protocol); + void OnMessageError(mozilla::ipc::HasResultCodes::Result code, + const IPC::Message& aMsg); + void OnDropPeer(const char* reason, const char* file, int line); + void OnMessageTaskStart(); + void OnMessageTaskStop(); + void OnPreFuzzMessageTaskRun(); + void OnPreFuzzMessageTaskStop(); + void OnChildReady() { childReady = true; } + void OnRunnableDone() { runnableDone = true; } + + uint32_t getPreFuzzMessageTaskCount() { return messageTaskCount; }; + uint32_t getMessageStartCount() { return messageStartCount; }; + uint32_t getMessageStopCount() { return messageStopCount; }; + + void StartFuzzing(mozilla::ipc::NodeChannel* channel, IPC::Message& aMessage); + + void SynchronizeOnMessageExecution(uint32_t expected_messages); + void AddToplevelActor(mojo::core::ports::PortName name, + mozilla::ipc::ProtocolId protocolId); + + // Used for the IPC_SingleMessage fuzzer + UniquePtr<IPC::Message> replaceIPCMessage(UniquePtr<IPC::Message> aMsg); + void syncAfterReplace(); + + private: + // This is a mapping from port name to a pair of last seen sequence numbers. + std::unordered_map<mojo::core::ports::PortName, SeqNoPair> portSeqNos; + + // This is a mapping from port name to node name. + std::unordered_map<mojo::core::ports::PortName, mojo::core::ports::NodeName> + portNodeName; + + // This is a mapping from port name to protocol name, purely for debugging. + std::unordered_map<mojo::core::ports::PortName, std::string> + portNameToProtocolName; + + // This maps each ProtocolId (IPCMessageStart) to the number of valid + // messages for that particular type. + std::unordered_map<mozilla::ipc::ProtocolId, uint32_t> validMsgTypes; + + // This is a mapping from port name to pairs of actor Id and ProtocolId. + std::unordered_map<mojo::core::ports::PortName, std::vector<ActorIdPair>> + actorIds; + + // If set, `lastActorPortName` is valid and fuzzing is pinned to this port. + Atomic<bool> useLastPortName; + + // Last port where a new actor appeared. Only valid with `useLastPortName`. + mojo::core::ports::PortName lastActorPortName; + + // Counter to indicate how long fuzzing should stay pinned to the last + // actor that appeared on `lastActorPortName`. + Atomic<uint32_t> useLastActor; + + // This is the deterministic ordering of toplevel actors for fuzzing. + // In this matrix, each row (toplevel index) corresponds to one toplevel + // actor *type* while each entry in that row is an instance of that type, + // since some actors usually have multiple instances alive while others + // don't. For the exact ordering, please check the constructor for this + // class. + std::vector<std::vector<mojo::core::ports::PortName>> portNames; + std::unordered_map<std::string, uint8_t> portNameToIndex; + + // This is a set of all types that are constructors. + std::unordered_set<uint32_t> constructorTypes; + + // This is the name of the target node. We select one Node based on a + // particular toplevel actor and then use this to pull in additional + // toplevel actors that are on the same node (i.e. belong to the same + // process pair). + mojo::core::ports::NodeName targetNodeName; + bool haveTargetNodeName = false; + + // This indicates that we have started the fuzzing thread and fuzzing will + // begin shortly. + bool fuzzingStartPending = false; + + // This is used as a signal from other threads that runnables we dispatched + // are completed. Right now we use this only when dispatching to the main + // thread to await the completion of all pending events. + Atomic<bool> runnableDone; + + // This is used to signal that the other process we are talking to is ready + // to start fuzzing. In the case of Parent <-> Child, a special IPC message + // is used to signal this. We might not be able to start fuzzing immediately + // hough if not all toplevel actors have been created yet. + Atomic<bool> childReady; + + // Current amount of pending message tasks. + Atomic<uint32_t> messageStartCount; + Atomic<uint32_t> messageStopCount; + + Atomic<uint32_t> messageTaskCount; + + Vector<char, 256, InfallibleAllocPolicy> sampleHeader; + + mozilla::ipc::NodeChannel* nodeChannel = nullptr; + + // This class is used both on the I/O and background threads as well as + // its own fuzzing thread. Those methods that alter non-threadsafe data + // structures need to aquire this mutex first. + Mutex mMutex; // MOZ_UNANNOTATED; + + // Can be used to specify a non-standard trigger message, e.h. to target + // a specific actor. + uint32_t mIPCTriggerMsg; + + // Used to dump IPC messages in single message mode + Maybe<uint32_t> mIPCDumpMsg; + Maybe<uint32_t> mIPCDumpAllMsgsSize; + uint32_t mIPCDumpCount = 0; + + // Used to select a particular packet instance in single message mode + uint32_t mIPCTriggerSingleMsgWait = 0; + + IPCFuzzController(); + NYX_DISALLOW_COPY_AND_ASSIGN(IPCFuzzController); +}; + +} // namespace fuzzing +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/ipc/moz.build b/tools/fuzzing/ipc/moz.build new file mode 100644 index 0000000000..001c67ed37 --- /dev/null +++ b/tools/fuzzing/ipc/moz.build @@ -0,0 +1,27 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-ipc-protocol") + +LOCAL_INCLUDES += [ + "/dom/base", + "/dom/ipc", +] + +SOURCES += [ + "IPCFuzzController.cpp", +] + +EXPORTS.mozilla.fuzzing += [ + "IPCFuzzController.h", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul" + +# Add libFuzzer configuration directives +include("/tools/fuzzing/libfuzzer-config.mozbuild") diff --git a/tools/fuzzing/libfuzzer-config.mozbuild b/tools/fuzzing/libfuzzer-config.mozbuild new file mode 100644 index 0000000000..e7a40c89ac --- /dev/null +++ b/tools/fuzzing/libfuzzer-config.mozbuild @@ -0,0 +1,13 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +include("libfuzzer-flags.mozbuild") + +if CONFIG["FUZZING"]: + if CONFIG["LIBFUZZER"]: + # Add trace-pc coverage for libfuzzer + CFLAGS += libfuzzer_flags + CXXFLAGS += libfuzzer_flags diff --git a/tools/fuzzing/libfuzzer-flags.mozbuild b/tools/fuzzing/libfuzzer-flags.mozbuild new file mode 100644 index 0000000000..258f4a68c2 --- /dev/null +++ b/tools/fuzzing/libfuzzer-flags.mozbuild @@ -0,0 +1,10 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +libfuzzer_flags = [] + +if CONFIG["LIBFUZZER_FLAGS"]: + libfuzzer_flags += CONFIG["LIBFUZZER_FLAGS"] diff --git a/tools/fuzzing/libfuzzer/FuzzerBuiltins.h b/tools/fuzzing/libfuzzer/FuzzerBuiltins.h new file mode 100644 index 0000000000..4c0ada8266 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerBuiltins.h @@ -0,0 +1,35 @@ +//===- FuzzerBuiltins.h - Internal header for builtins ----------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos around builtin functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_H +#define LLVM_FUZZER_BUILTINS_H + +#include "FuzzerPlatform.h" + +#if !LIBFUZZER_MSVC +#include <cstdint> + +#define GET_CALLER_PC() __builtin_return_address(0) + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); } +inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } +inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } + +inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); } +inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); } +inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); } + +} // namespace fuzzer + +#endif // !LIBFUZZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_H diff --git a/tools/fuzzing/libfuzzer/FuzzerBuiltinsMsvc.h b/tools/fuzzing/libfuzzer/FuzzerBuiltinsMsvc.h new file mode 100644 index 0000000000..c5bec9787d --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerBuiltinsMsvc.h @@ -0,0 +1,72 @@ +//===- FuzzerBuiltinsMSVC.h - Internal header for builtins ------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos that use intrinsics instead of builtin functions +// which cannot be compiled by MSVC. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_MSVC_H +#define LLVM_FUZZER_BUILTINS_MSVC_H + +#include "FuzzerPlatform.h" + +#if LIBFUZZER_MSVC +#include <intrin.h> +#include <cstdint> +#include <cstdlib> + +// __builtin_return_address() cannot be compiled with MSVC. Use the equivalent +// from <intrin.h> +#define GET_CALLER_PC() _ReturnAddress() + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +// Use alternatives to __builtin functions from <stdlib.h> and <intrin.h> on +// Windows since the builtins are not supported by MSVC. +inline uint16_t Bswap(uint16_t x) { return _byteswap_ushort(x); } +inline uint32_t Bswap(uint32_t x) { return _byteswap_ulong(x); } +inline uint64_t Bswap(uint64_t x) { return _byteswap_uint64(x); } + +// The functions below were mostly copied from +// compiler-rt/lib/builtins/int_lib.h which defines the __builtin functions used +// outside of Windows. +inline uint32_t Clzll(uint64_t X) { + unsigned long LeadZeroIdx = 0; + +#if !defined(_M_ARM) && !defined(_M_X64) + // Scan the high 32 bits. + if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X >> 32))) + return static_cast<int>(63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB. + // Scan the low 32 bits. + if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X))) + return static_cast<int>(63 - LeadZeroIdx); + +#else + if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; +#endif + return 64; +} + +inline uint32_t Clz(uint32_t X) { + unsigned long LeadZeroIdx = 0; + if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx; + return 32; +} + +inline int Popcountll(unsigned long long X) { +#if !defined(_M_ARM) && !defined(_M_X64) + return __popcnt(X) + __popcnt(X >> 32); +#else + return __popcnt64(X); +#endif +} + +} // namespace fuzzer + +#endif // LIBFUZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_MSVC_H diff --git a/tools/fuzzing/libfuzzer/FuzzerCommand.h b/tools/fuzzing/libfuzzer/FuzzerCommand.h new file mode 100644 index 0000000000..87308864af --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerCommand.h @@ -0,0 +1,178 @@ +//===- FuzzerCommand.h - Interface representing a process -------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// FuzzerCommand represents a command to run in a subprocess. It allows callers +// to manage command line arguments and output and error streams. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_COMMAND_H +#define LLVM_FUZZER_COMMAND_H + +#include "FuzzerDefs.h" +#include "FuzzerIO.h" + +#include <algorithm> +#include <sstream> +#include <string> +#include <vector> + +namespace fuzzer { + +class Command final { +public: + // This command line flag is used to indicate that the remaining command line + // is immutable, meaning this flag effectively marks the end of the mutable + // argument list. + static inline const char *ignoreRemainingArgs() { + return "-ignore_remaining_args=1"; + } + + Command() : CombinedOutAndErr(false) {} + + explicit Command(const Vector<std::string> &ArgsToAdd) + : Args(ArgsToAdd), CombinedOutAndErr(false) {} + + explicit Command(const Command &Other) + : Args(Other.Args), CombinedOutAndErr(Other.CombinedOutAndErr), + OutputFile(Other.OutputFile) {} + + Command &operator=(const Command &Other) { + Args = Other.Args; + CombinedOutAndErr = Other.CombinedOutAndErr; + OutputFile = Other.OutputFile; + return *this; + } + + ~Command() {} + + // Returns true if the given Arg is present in Args. Only checks up to + // "-ignore_remaining_args=1". + bool hasArgument(const std::string &Arg) const { + auto i = endMutableArgs(); + return std::find(Args.begin(), i, Arg) != i; + } + + // Gets all of the current command line arguments, **including** those after + // "-ignore-remaining-args=1". + const Vector<std::string> &getArguments() const { return Args; } + + // Adds the given argument before "-ignore_remaining_args=1", or at the end + // if that flag isn't present. + void addArgument(const std::string &Arg) { + Args.insert(endMutableArgs(), Arg); + } + + // Adds all given arguments before "-ignore_remaining_args=1", or at the end + // if that flag isn't present. + void addArguments(const Vector<std::string> &ArgsToAdd) { + Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end()); + } + + // Removes the given argument from the command argument list. Ignores any + // occurrences after "-ignore_remaining_args=1", if present. + void removeArgument(const std::string &Arg) { + auto i = endMutableArgs(); + Args.erase(std::remove(Args.begin(), i, Arg), i); + } + + // Like hasArgument, but checks for "-[Flag]=...". + bool hasFlag(const std::string &Flag) const { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + return std::any_of(Args.begin(), endMutableArgs(), IsMatch); + } + + // Returns the value of the first instance of a given flag, or an empty string + // if the flag isn't present. Ignores any occurrences after + // "-ignore_remaining_args=1", if present. + std::string getFlagValue(const std::string &Flag) const { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + auto i = endMutableArgs(); + auto j = std::find_if(Args.begin(), i, IsMatch); + std::string result; + if (j != i) { + result = j->substr(Arg.length()); + } + return result; + } + + // Like AddArgument, but adds "-[Flag]=[Value]". + void addFlag(const std::string &Flag, const std::string &Value) { + addArgument("-" + Flag + "=" + Value); + } + + // Like RemoveArgument, but removes "-[Flag]=...". + void removeFlag(const std::string &Flag) { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + auto i = endMutableArgs(); + Args.erase(std::remove_if(Args.begin(), i, IsMatch), i); + } + + // Returns whether the command's stdout is being written to an output file. + bool hasOutputFile() const { return !OutputFile.empty(); } + + // Returns the currently set output file. + const std::string &getOutputFile() const { return OutputFile; } + + // Configures the command to redirect its output to the name file. + void setOutputFile(const std::string &FileName) { OutputFile = FileName; } + + // Returns whether the command's stderr is redirected to stdout. + bool isOutAndErrCombined() const { return CombinedOutAndErr; } + + // Sets whether to redirect the command's stderr to its stdout. + void combineOutAndErr(bool combine = true) { CombinedOutAndErr = combine; } + + // Returns a string representation of the command. On many systems this will + // be the equivalent command line. + std::string toString() const { + std::stringstream SS; + for (auto arg : getArguments()) + SS << arg << " "; + if (hasOutputFile()) + SS << ">" << getOutputFile() << " "; + if (isOutAndErrCombined()) + SS << "2>&1 "; + std::string result = SS.str(); + if (!result.empty()) + result = result.substr(0, result.length() - 1); + return result; + } + +private: + Command(Command &&Other) = delete; + Command &operator=(Command &&Other) = delete; + + Vector<std::string>::iterator endMutableArgs() { + return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); + } + + Vector<std::string>::const_iterator endMutableArgs() const { + return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); + } + + // The command arguments. Args[0] is the command name. + Vector<std::string> Args; + + // True indicates stderr is redirected to stdout. + bool CombinedOutAndErr; + + // If not empty, stdout is redirected to the named file. + std::string OutputFile; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_COMMAND_H diff --git a/tools/fuzzing/libfuzzer/FuzzerCorpus.h b/tools/fuzzing/libfuzzer/FuzzerCorpus.h new file mode 100644 index 0000000000..54d1e09ec6 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerCorpus.h @@ -0,0 +1,533 @@ +//===- FuzzerCorpus.h - Internal header for the Fuzzer ----------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::InputCorpus +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_CORPUS +#define LLVM_FUZZER_CORPUS + +#include "FuzzerDataFlowTrace.h" +#include "FuzzerDefs.h" +#include "FuzzerIO.h" +#include "FuzzerRandom.h" +#include "FuzzerSHA1.h" +#include "FuzzerTracePC.h" +#include <algorithm> +#include <numeric> +#include <random> +#include <unordered_set> + +namespace fuzzer { + +struct InputInfo { + Unit U; // The actual input data. + uint8_t Sha1[kSHA1NumBytes]; // Checksum. + // Number of features that this input has and no smaller input has. + size_t NumFeatures = 0; + size_t Tmp = 0; // Used by ValidateFeatureSet. + // Stats. + size_t NumExecutedMutations = 0; + size_t NumSuccessfullMutations = 0; + bool MayDeleteFile = false; + bool Reduced = false; + bool HasFocusFunction = false; + Vector<uint32_t> UniqFeatureSet; + Vector<uint8_t> DataFlowTraceForFocusFunction; + // Power schedule. + bool NeedsEnergyUpdate = false; + double Energy = 0.0; + size_t SumIncidence = 0; + Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs; + + // Delete feature Idx and its frequency from FeatureFreqs. + bool DeleteFeatureFreq(uint32_t Idx) { + if (FeatureFreqs.empty()) + return false; + + // Binary search over local feature frequencies sorted by index. + auto Lower = std::lower_bound(FeatureFreqs.begin(), FeatureFreqs.end(), + std::pair<uint32_t, uint16_t>(Idx, 0)); + + if (Lower != FeatureFreqs.end() && Lower->first == Idx) { + FeatureFreqs.erase(Lower); + return true; + } + return false; + } + + // Assign more energy to a high-entropy seed, i.e., that reveals more + // information about the globally rare features in the neighborhood + // of the seed. Since we do not know the entropy of a seed that has + // never been executed we assign fresh seeds maximum entropy and + // let II->Energy approach the true entropy from above. + void UpdateEnergy(size_t GlobalNumberOfFeatures) { + Energy = 0.0; + SumIncidence = 0; + + // Apply add-one smoothing to locally discovered features. + for (auto F : FeatureFreqs) { + size_t LocalIncidence = F.second + 1; + Energy -= LocalIncidence * logl(LocalIncidence); + SumIncidence += LocalIncidence; + } + + // Apply add-one smoothing to locally undiscovered features. + // PreciseEnergy -= 0; // since logl(1.0) == 0) + SumIncidence += (GlobalNumberOfFeatures - FeatureFreqs.size()); + + // Add a single locally abundant feature apply add-one smoothing. + size_t AbdIncidence = NumExecutedMutations + 1; + Energy -= AbdIncidence * logl(AbdIncidence); + SumIncidence += AbdIncidence; + + // Normalize. + if (SumIncidence != 0) + Energy = (Energy / SumIncidence) + logl(SumIncidence); + } + + // Increment the frequency of the feature Idx. + void UpdateFeatureFrequency(uint32_t Idx) { + NeedsEnergyUpdate = true; + + // The local feature frequencies is an ordered vector of pairs. + // If there are no local feature frequencies, push_back preserves order. + // Set the feature frequency for feature Idx32 to 1. + if (FeatureFreqs.empty()) { + FeatureFreqs.push_back(std::pair<uint32_t, uint16_t>(Idx, 1)); + return; + } + + // Binary search over local feature frequencies sorted by index. + auto Lower = std::lower_bound(FeatureFreqs.begin(), FeatureFreqs.end(), + std::pair<uint32_t, uint16_t>(Idx, 0)); + + // If feature Idx32 already exists, increment its frequency. + // Otherwise, insert a new pair right after the next lower index. + if (Lower != FeatureFreqs.end() && Lower->first == Idx) { + Lower->second++; + } else { + FeatureFreqs.insert(Lower, std::pair<uint32_t, uint16_t>(Idx, 1)); + } + } +}; + +struct EntropicOptions { + bool Enabled; + size_t NumberOfRarestFeatures; + size_t FeatureFrequencyThreshold; +}; + +class InputCorpus { + static const uint32_t kFeatureSetSize = 1 << 21; + static const uint8_t kMaxMutationFactor = 20; + static const size_t kSparseEnergyUpdates = 100; + + size_t NumExecutedMutations = 0; + + EntropicOptions Entropic; + +public: + InputCorpus(const std::string &OutputCorpus, EntropicOptions Entropic) + : Entropic(Entropic), OutputCorpus(OutputCorpus) { + memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature)); + memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature)); + } + ~InputCorpus() { + for (auto II : Inputs) + delete II; + } + size_t size() const { return Inputs.size(); } + size_t SizeInBytes() const { + size_t Res = 0; + for (auto II : Inputs) + Res += II->U.size(); + return Res; + } + size_t NumActiveUnits() const { + size_t Res = 0; + for (auto II : Inputs) + Res += !II->U.empty(); + return Res; + } + size_t MaxInputSize() const { + size_t Res = 0; + for (auto II : Inputs) + Res = std::max(Res, II->U.size()); + return Res; + } + void IncrementNumExecutedMutations() { NumExecutedMutations++; } + + size_t NumInputsThatTouchFocusFunction() { + return std::count_if(Inputs.begin(), Inputs.end(), [](const InputInfo *II) { + return II->HasFocusFunction; + }); + } + + size_t NumInputsWithDataFlowTrace() { + return std::count_if(Inputs.begin(), Inputs.end(), [](const InputInfo *II) { + return !II->DataFlowTraceForFocusFunction.empty(); + }); + } + + bool empty() const { return Inputs.empty(); } + const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; } + InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, + bool HasFocusFunction, + const Vector<uint32_t> &FeatureSet, + const DataFlowTrace &DFT, const InputInfo *BaseII) { + assert(!U.empty()); + if (FeatureDebug) + Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures); + Inputs.push_back(new InputInfo()); + InputInfo &II = *Inputs.back(); + II.U = U; + II.NumFeatures = NumFeatures; + II.MayDeleteFile = MayDeleteFile; + II.UniqFeatureSet = FeatureSet; + II.HasFocusFunction = HasFocusFunction; + // Assign maximal energy to the new seed. + II.Energy = RareFeatures.empty() ? 1.0 : log(RareFeatures.size()); + II.SumIncidence = RareFeatures.size(); + II.NeedsEnergyUpdate = false; + std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end()); + ComputeSHA1(U.data(), U.size(), II.Sha1); + auto Sha1Str = Sha1ToString(II.Sha1); + Hashes.insert(Sha1Str); + if (HasFocusFunction) + if (auto V = DFT.Get(Sha1Str)) + II.DataFlowTraceForFocusFunction = *V; + // This is a gross heuristic. + // Ideally, when we add an element to a corpus we need to know its DFT. + // But if we don't, we'll use the DFT of its base input. + if (II.DataFlowTraceForFocusFunction.empty() && BaseII) + II.DataFlowTraceForFocusFunction = BaseII->DataFlowTraceForFocusFunction; + DistributionNeedsUpdate = true; + PrintCorpus(); + // ValidateFeatureSet(); + return &II; + } + + // Debug-only + void PrintUnit(const Unit &U) { + if (!FeatureDebug) return; + for (uint8_t C : U) { + if (C != 'F' && C != 'U' && C != 'Z') + C = '.'; + Printf("%c", C); + } + } + + // Debug-only + void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) { + if (!FeatureDebug) return; + Printf("{"); + for (uint32_t Feature: FeatureSet) + Printf("%u,", Feature); + Printf("}"); + } + + // Debug-only + void PrintCorpus() { + if (!FeatureDebug) return; + Printf("======= CORPUS:\n"); + int i = 0; + for (auto II : Inputs) { + if (std::find(II->U.begin(), II->U.end(), 'F') != II->U.end()) { + Printf("[%2d] ", i); + Printf("%s sz=%zd ", Sha1ToString(II->Sha1).c_str(), II->U.size()); + PrintUnit(II->U); + Printf(" "); + PrintFeatureSet(II->UniqFeatureSet); + Printf("\n"); + } + i++; + } + } + + void Replace(InputInfo *II, const Unit &U) { + assert(II->U.size() > U.size()); + Hashes.erase(Sha1ToString(II->Sha1)); + DeleteFile(*II); + ComputeSHA1(U.data(), U.size(), II->Sha1); + Hashes.insert(Sha1ToString(II->Sha1)); + II->U = U; + II->Reduced = true; + DistributionNeedsUpdate = true; + } + + bool HasUnit(const Unit &U) { return Hashes.count(Hash(U)); } + bool HasUnit(const std::string &H) { return Hashes.count(H); } + InputInfo &ChooseUnitToMutate(Random &Rand) { + InputInfo &II = *Inputs[ChooseUnitIdxToMutate(Rand)]; + assert(!II.U.empty()); + return II; + } + + // Returns an index of random unit from the corpus to mutate. + size_t ChooseUnitIdxToMutate(Random &Rand) { + UpdateCorpusDistribution(Rand); + size_t Idx = static_cast<size_t>(CorpusDistribution(Rand)); + assert(Idx < Inputs.size()); + return Idx; + } + + void PrintStats() { + for (size_t i = 0; i < Inputs.size(); i++) { + const auto &II = *Inputs[i]; + Printf(" [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i, + Sha1ToString(II.Sha1).c_str(), II.U.size(), + II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction); + } + } + + void PrintFeatureSet() { + for (size_t i = 0; i < kFeatureSetSize; i++) { + if(size_t Sz = GetFeature(i)) + Printf("[%zd: id %zd sz%zd] ", i, SmallestElementPerFeature[i], Sz); + } + Printf("\n\t"); + for (size_t i = 0; i < Inputs.size(); i++) + if (size_t N = Inputs[i]->NumFeatures) + Printf(" %zd=>%zd ", i, N); + Printf("\n"); + } + + void DeleteFile(const InputInfo &II) { + if (!OutputCorpus.empty() && II.MayDeleteFile) + RemoveFile(DirPlusFile(OutputCorpus, Sha1ToString(II.Sha1))); + } + + void DeleteInput(size_t Idx) { + InputInfo &II = *Inputs[Idx]; + DeleteFile(II); + Unit().swap(II.U); + II.Energy = 0.0; + II.NeedsEnergyUpdate = false; + DistributionNeedsUpdate = true; + if (FeatureDebug) + Printf("EVICTED %zd\n", Idx); + } + + void AddRareFeature(uint32_t Idx) { + // Maintain *at least* TopXRarestFeatures many rare features + // and all features with a frequency below ConsideredRare. + // Remove all other features. + while (RareFeatures.size() > Entropic.NumberOfRarestFeatures && + FreqOfMostAbundantRareFeature > Entropic.FeatureFrequencyThreshold) { + + // Find most and second most abbundant feature. + uint32_t MostAbundantRareFeatureIndices[2] = {RareFeatures[0], + RareFeatures[0]}; + size_t Delete = 0; + for (size_t i = 0; i < RareFeatures.size(); i++) { + uint32_t Idx2 = RareFeatures[i]; + if (GlobalFeatureFreqs[Idx2] >= + GlobalFeatureFreqs[MostAbundantRareFeatureIndices[0]]) { + MostAbundantRareFeatureIndices[1] = MostAbundantRareFeatureIndices[0]; + MostAbundantRareFeatureIndices[0] = Idx2; + Delete = i; + } + } + + // Remove most abundant rare feature. + RareFeatures[Delete] = RareFeatures.back(); + RareFeatures.pop_back(); + + for (auto II : Inputs) { + if (II->DeleteFeatureFreq(MostAbundantRareFeatureIndices[0])) + II->NeedsEnergyUpdate = true; + } + + // Set 2nd most abundant as the new most abundant feature count. + FreqOfMostAbundantRareFeature = + GlobalFeatureFreqs[MostAbundantRareFeatureIndices[1]]; + } + + // Add rare feature, handle collisions, and update energy. + RareFeatures.push_back(Idx); + GlobalFeatureFreqs[Idx] = 0; + for (auto II : Inputs) { + II->DeleteFeatureFreq(Idx); + + // Apply add-one smoothing to this locally undiscovered feature. + // Zero energy seeds will never be fuzzed and remain zero energy. + if (II->Energy > 0.0) { + II->SumIncidence += 1; + II->Energy += logl(II->SumIncidence) / II->SumIncidence; + } + } + + DistributionNeedsUpdate = true; + } + + bool AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) { + assert(NewSize); + Idx = Idx % kFeatureSetSize; + uint32_t OldSize = GetFeature(Idx); + if (OldSize == 0 || (Shrink && OldSize > NewSize)) { + if (OldSize > 0) { + size_t OldIdx = SmallestElementPerFeature[Idx]; + InputInfo &II = *Inputs[OldIdx]; + assert(II.NumFeatures > 0); + II.NumFeatures--; + if (II.NumFeatures == 0) + DeleteInput(OldIdx); + } else { + NumAddedFeatures++; + if (Entropic.Enabled) + AddRareFeature((uint32_t)Idx); + } + NumUpdatedFeatures++; + if (FeatureDebug) + Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize); + SmallestElementPerFeature[Idx] = Inputs.size(); + InputSizesPerFeature[Idx] = NewSize; + return true; + } + return false; + } + + // Increment frequency of feature Idx globally and locally. + void UpdateFeatureFrequency(InputInfo *II, size_t Idx) { + uint32_t Idx32 = Idx % kFeatureSetSize; + + // Saturated increment. + if (GlobalFeatureFreqs[Idx32] == 0xFFFF) + return; + uint16_t Freq = GlobalFeatureFreqs[Idx32]++; + + // Skip if abundant. + if (Freq > FreqOfMostAbundantRareFeature || + std::find(RareFeatures.begin(), RareFeatures.end(), Idx32) == + RareFeatures.end()) + return; + + // Update global frequencies. + if (Freq == FreqOfMostAbundantRareFeature) + FreqOfMostAbundantRareFeature++; + + // Update local frequencies. + if (II) + II->UpdateFeatureFrequency(Idx32); + } + + size_t NumFeatures() const { return NumAddedFeatures; } + size_t NumFeatureUpdates() const { return NumUpdatedFeatures; } + +private: + + static const bool FeatureDebug = false; + + size_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } + + void ValidateFeatureSet() { + if (FeatureDebug) + PrintFeatureSet(); + for (size_t Idx = 0; Idx < kFeatureSetSize; Idx++) + if (GetFeature(Idx)) + Inputs[SmallestElementPerFeature[Idx]]->Tmp++; + for (auto II: Inputs) { + if (II->Tmp != II->NumFeatures) + Printf("ZZZ %zd %zd\n", II->Tmp, II->NumFeatures); + assert(II->Tmp == II->NumFeatures); + II->Tmp = 0; + } + } + + // Updates the probability distribution for the units in the corpus. + // Must be called whenever the corpus or unit weights are changed. + // + // Hypothesis: inputs that maximize information about globally rare features + // are interesting. + void UpdateCorpusDistribution(Random &Rand) { + // Skip update if no seeds or rare features were added/deleted. + // Sparse updates for local change of feature frequencies, + // i.e., randomly do not skip. + if (!DistributionNeedsUpdate && + (!Entropic.Enabled || Rand(kSparseEnergyUpdates))) + return; + + DistributionNeedsUpdate = false; + + size_t N = Inputs.size(); + assert(N); + Intervals.resize(N + 1); + Weights.resize(N); + std::iota(Intervals.begin(), Intervals.end(), 0); + + bool VanillaSchedule = true; + if (Entropic.Enabled) { + for (auto II : Inputs) { + if (II->NeedsEnergyUpdate && II->Energy != 0.0) { + II->NeedsEnergyUpdate = false; + II->UpdateEnergy(RareFeatures.size()); + } + } + + for (size_t i = 0; i < N; i++) { + + if (Inputs[i]->NumFeatures == 0) { + // If the seed doesn't represent any features, assign zero energy. + Weights[i] = 0.; + } else if (Inputs[i]->NumExecutedMutations / kMaxMutationFactor > + NumExecutedMutations / Inputs.size()) { + // If the seed was fuzzed a lot more than average, assign zero energy. + Weights[i] = 0.; + } else { + // Otherwise, simply assign the computed energy. + Weights[i] = Inputs[i]->Energy; + } + + // If energy for all seeds is zero, fall back to vanilla schedule. + if (Weights[i] > 0.0) + VanillaSchedule = false; + } + } + + if (VanillaSchedule) { + for (size_t i = 0; i < N; i++) + Weights[i] = Inputs[i]->NumFeatures + ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1) + : 0.; + } + + if (FeatureDebug) { + for (size_t i = 0; i < N; i++) + Printf("%zd ", Inputs[i]->NumFeatures); + Printf("SCORE\n"); + for (size_t i = 0; i < N; i++) + Printf("%f ", Weights[i]); + Printf("Weights\n"); + } + CorpusDistribution = std::piecewise_constant_distribution<double>( + Intervals.begin(), Intervals.end(), Weights.begin()); + } + std::piecewise_constant_distribution<double> CorpusDistribution; + + Vector<double> Intervals; + Vector<double> Weights; + + std::unordered_set<std::string> Hashes; + Vector<InputInfo*> Inputs; + + size_t NumAddedFeatures = 0; + size_t NumUpdatedFeatures = 0; + uint32_t InputSizesPerFeature[kFeatureSetSize]; + uint32_t SmallestElementPerFeature[kFeatureSetSize]; + + bool DistributionNeedsUpdate = true; + uint16_t FreqOfMostAbundantRareFeature = 0; + uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {}; + Vector<uint32_t> RareFeatures; + + std::string OutputCorpus; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_CORPUS diff --git a/tools/fuzzing/libfuzzer/FuzzerCrossOver.cpp b/tools/fuzzing/libfuzzer/FuzzerCrossOver.cpp new file mode 100644 index 0000000000..83d9f8d47c --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerCrossOver.cpp @@ -0,0 +1,51 @@ +//===- FuzzerCrossOver.cpp - Cross over two test inputs -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Cross over test inputs. +//===----------------------------------------------------------------------===// + +#include "FuzzerDefs.h" +#include "FuzzerMutate.h" +#include "FuzzerRandom.h" +#include <cstring> + +namespace fuzzer { + +// Cross Data1 and Data2, store the result (up to MaxOutSize bytes) in Out. +size_t MutationDispatcher::CrossOver(const uint8_t *Data1, size_t Size1, + const uint8_t *Data2, size_t Size2, + uint8_t *Out, size_t MaxOutSize) { + assert(Size1 || Size2); + MaxOutSize = Rand(MaxOutSize) + 1; + size_t OutPos = 0; + size_t Pos1 = 0; + size_t Pos2 = 0; + size_t *InPos = &Pos1; + size_t InSize = Size1; + const uint8_t *Data = Data1; + bool CurrentlyUsingFirstData = true; + while (OutPos < MaxOutSize && (Pos1 < Size1 || Pos2 < Size2)) { + // Merge a part of Data into Out. + size_t OutSizeLeft = MaxOutSize - OutPos; + if (*InPos < InSize) { + size_t InSizeLeft = InSize - *InPos; + size_t MaxExtraSize = std::min(OutSizeLeft, InSizeLeft); + size_t ExtraSize = Rand(MaxExtraSize) + 1; + memcpy(Out + OutPos, Data + *InPos, ExtraSize); + OutPos += ExtraSize; + (*InPos) += ExtraSize; + } + // Use the other input data on the next iteration. + InPos = CurrentlyUsingFirstData ? &Pos2 : &Pos1; + InSize = CurrentlyUsingFirstData ? Size2 : Size1; + Data = CurrentlyUsingFirstData ? Data2 : Data1; + CurrentlyUsingFirstData = !CurrentlyUsingFirstData; + } + return OutPos; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp new file mode 100644 index 0000000000..06ea287a3c --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp @@ -0,0 +1,295 @@ +//===- FuzzerDataFlowTrace.cpp - DataFlowTrace ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::DataFlowTrace +//===----------------------------------------------------------------------===// + +#include "FuzzerDataFlowTrace.h" + +#include "FuzzerCommand.h" +#include "FuzzerIO.h" +#include "FuzzerRandom.h" +#include "FuzzerSHA1.h" +#include "FuzzerUtil.h" + +#include <cstdlib> +#include <fstream> +#include <numeric> +#include <queue> +#include <sstream> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +namespace fuzzer { +static const char *kFunctionsTxt = "functions.txt"; + +bool BlockCoverage::AppendCoverage(const std::string &S) { + std::stringstream SS(S); + return AppendCoverage(SS); +} + +// Coverage lines have this form: +// CN X Y Z T +// where N is the number of the function, T is the total number of instrumented +// BBs, and X,Y,Z, if present, are the indecies of covered BB. +// BB #0, which is the entry block, is not explicitly listed. +bool BlockCoverage::AppendCoverage(std::istream &IN) { + std::string L; + while (std::getline(IN, L, '\n')) { + if (L.empty()) + continue; + std::stringstream SS(L.c_str() + 1); + size_t FunctionId = 0; + SS >> FunctionId; + if (L[0] == 'F') { + FunctionsWithDFT.insert(FunctionId); + continue; + } + if (L[0] != 'C') continue; + Vector<uint32_t> CoveredBlocks; + while (true) { + uint32_t BB = 0; + SS >> BB; + if (!SS) break; + CoveredBlocks.push_back(BB); + } + if (CoveredBlocks.empty()) return false; + uint32_t NumBlocks = CoveredBlocks.back(); + CoveredBlocks.pop_back(); + for (auto BB : CoveredBlocks) + if (BB >= NumBlocks) return false; + auto It = Functions.find(FunctionId); + auto &Counters = + It == Functions.end() + ? Functions.insert({FunctionId, Vector<uint32_t>(NumBlocks)}) + .first->second + : It->second; + + if (Counters.size() != NumBlocks) return false; // wrong number of blocks. + + Counters[0]++; + for (auto BB : CoveredBlocks) + Counters[BB]++; + } + return true; +} + +// Assign weights to each function. +// General principles: +// * any uncovered function gets weight 0. +// * a function with lots of uncovered blocks gets bigger weight. +// * a function with a less frequently executed code gets bigger weight. +Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { + Vector<double> Res(NumFunctions); + for (auto It : Functions) { + auto FunctionID = It.first; + auto Counters = It.second; + assert(FunctionID < NumFunctions); + auto &Weight = Res[FunctionID]; + // Give higher weight if the function has a DFT. + Weight = FunctionsWithDFT.count(FunctionID) ? 1000. : 1; + // Give higher weight to functions with less frequently seen basic blocks. + Weight /= SmallestNonZeroCounter(Counters); + // Give higher weight to functions with the most uncovered basic blocks. + Weight *= NumberOfUncoveredBlocks(Counters) + 1; + } + return Res; +} + +int DataFlowTrace::ReadCoverage(const std::string &DirPath) { + Vector<SizedFile> Files; + int Res = GetSizedFilesFromDir(DirPath, &Files); + if (Res != 0) + return Res; + for (auto &SF : Files) { + auto Name = Basename(SF.File); + if (Name == kFunctionsTxt) continue; + if (!CorporaHashes.count(Name)) continue; + std::ifstream IF(SF.File); + Coverage.AppendCoverage(IF); + } + return 0; +} + +static void DFTStringAppendToVector(Vector<uint8_t> *DFT, + const std::string &DFTString) { + assert(DFT->size() == DFTString.size()); + for (size_t I = 0, Len = DFT->size(); I < Len; I++) + (*DFT)[I] = DFTString[I] == '1'; +} + +// converts a string of '0' and '1' into a Vector<uint8_t> +static Vector<uint8_t> DFTStringToVector(const std::string &DFTString) { + Vector<uint8_t> DFT(DFTString.size()); + DFTStringAppendToVector(&DFT, DFTString); + return DFT; +} + +static bool ParseError(const char *Err, const std::string &Line) { + Printf("DataFlowTrace: parse error: %s: Line: %s\n", Err, Line.c_str()); + return false; +} + +// TODO(metzman): replace std::string with std::string_view for +// better performance. Need to figure our how to use string_view on Windows. +static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum, + std::string *DFTString) { + if (!Line.empty() && Line[0] != 'F') + return false; // Ignore coverage. + size_t SpacePos = Line.find(' '); + if (SpacePos == std::string::npos) + return ParseError("no space in the trace line", Line); + if (Line.empty() || Line[0] != 'F') + return ParseError("the trace line doesn't start with 'F'", Line); + *FunctionNum = std::atol(Line.c_str() + 1); + const char *Beg = Line.c_str() + SpacePos + 1; + const char *End = Line.c_str() + Line.size(); + assert(Beg < End); + size_t Len = End - Beg; + for (size_t I = 0; I < Len; I++) { + if (Beg[I] != '0' && Beg[I] != '1') + return ParseError("the trace should contain only 0 or 1", Line); + } + *DFTString = Beg; + return true; +} + +int DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + Vector<SizedFile> &CorporaFiles, Random &Rand) { + if (DirPath.empty()) return 0; + Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str()); + Vector<SizedFile> Files; + int Res = GetSizedFilesFromDir(DirPath, &Files); + if (Res != 0) + return Res; + std::string L; + size_t FocusFuncIdx = SIZE_MAX; + Vector<std::string> FunctionNames; + + // Collect the hashes of the corpus files. + for (auto &SF : CorporaFiles) + CorporaHashes.insert(Hash(FileToVector(SF.File))); + + // Read functions.txt + std::ifstream IF(DirPlusFile(DirPath, kFunctionsTxt)); + size_t NumFunctions = 0; + while (std::getline(IF, L, '\n')) { + FunctionNames.push_back(L); + NumFunctions++; + if (*FocusFunction == L) + FocusFuncIdx = NumFunctions - 1; + } + if (!NumFunctions) + return 0; + + if (*FocusFunction == "auto") { + // AUTOFOCUS works like this: + // * reads the coverage data from the DFT files. + // * assigns weights to functions based on coverage. + // * chooses a random function according to the weights. + Res = ReadCoverage(DirPath); + if (Res != 0) + return Res; + auto Weights = Coverage.FunctionWeights(NumFunctions); + Vector<double> Intervals(NumFunctions + 1); + std::iota(Intervals.begin(), Intervals.end(), 0); + auto Distribution = std::piecewise_constant_distribution<double>( + Intervals.begin(), Intervals.end(), Weights.begin()); + FocusFuncIdx = static_cast<size_t>(Distribution(Rand)); + *FocusFunction = FunctionNames[FocusFuncIdx]; + assert(FocusFuncIdx < NumFunctions); + Printf("INFO: AUTOFOCUS: %zd %s\n", FocusFuncIdx, + FunctionNames[FocusFuncIdx].c_str()); + for (size_t i = 0; i < NumFunctions; i++) { + if (!Weights[i]) continue; + Printf(" [%zd] W %g\tBB-tot %u\tBB-cov %u\tEntryFreq %u:\t%s\n", i, + Weights[i], Coverage.GetNumberOfBlocks(i), + Coverage.GetNumberOfCoveredBlocks(i), Coverage.GetCounter(i, 0), + FunctionNames[i].c_str()); + } + } + + if (!NumFunctions || FocusFuncIdx == SIZE_MAX || Files.size() <= 1) + return 0; + + // Read traces. + size_t NumTraceFiles = 0; + size_t NumTracesWithFocusFunction = 0; + for (auto &SF : Files) { + auto Name = Basename(SF.File); + if (Name == kFunctionsTxt) continue; + if (!CorporaHashes.count(Name)) continue; // not in the corpus. + NumTraceFiles++; + // Printf("=== %s\n", Name.c_str()); + std::ifstream IF(SF.File); + while (std::getline(IF, L, '\n')) { + size_t FunctionNum = 0; + std::string DFTString; + if (ParseDFTLine(L, &FunctionNum, &DFTString) && + FunctionNum == FocusFuncIdx) { + NumTracesWithFocusFunction++; + + if (FunctionNum >= NumFunctions) { + ParseError("N is greater than the number of functions", L); + return 0; + } + Traces[Name] = DFTStringToVector(DFTString); + // Print just a few small traces. + if (NumTracesWithFocusFunction <= 3 && DFTString.size() <= 16) + Printf("%s => |%s|\n", Name.c_str(), std::string(DFTString).c_str()); + break; // No need to parse the following lines. + } + } + } + Printf("INFO: DataFlowTrace: %zd trace files, %zd functions, " + "%zd traces with focus function\n", + NumTraceFiles, NumFunctions, NumTracesWithFocusFunction); + return 0; +} + +int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, + const Vector<SizedFile> &CorporaFiles) { + Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n", + DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size()); + if (CorporaFiles.empty()) { + Printf("ERROR: can't collect data flow without corpus provided."); + return 1; + } + + static char DFSanEnv[] = "DFSAN_OPTIONS=warn_unimplemented=0"; + putenv(DFSanEnv); + MkDir(DirPath); + for (auto &F : CorporaFiles) { + // For every input F we need to collect the data flow and the coverage. + // Data flow collection may fail if we request too many DFSan tags at once. + // So, we start from requesting all tags in range [0,Size) and if that fails + // we then request tags in [0,Size/2) and [Size/2, Size), and so on. + // Function number => DFT. + auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File))); + std::unordered_map<size_t, Vector<uint8_t>> DFTMap; + std::unordered_set<std::string> Cov; + Command Cmd; + Cmd.addArgument(DFTBinary); + Cmd.addArgument(F.File); + Cmd.addArgument(OutPath); + Printf("CMD: %s\n", Cmd.toString().c_str()); + ExecuteCommand(Cmd); + } + // Write functions.txt if it's currently empty or doesn't exist. + auto FunctionsTxtPath = DirPlusFile(DirPath, kFunctionsTxt); + if (FileToString(FunctionsTxtPath).empty()) { + Command Cmd; + Cmd.addArgument(DFTBinary); + Cmd.setOutputFile(FunctionsTxtPath); + ExecuteCommand(Cmd); + } + return 0; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h new file mode 100644 index 0000000000..767bad24f1 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h @@ -0,0 +1,135 @@ +//===- FuzzerDataFlowTrace.h - Internal header for the Fuzzer ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::DataFlowTrace; reads and handles a data-flow trace. +// +// A data flow trace is generated by e.g. dataflow/DataFlow.cpp +// and is stored on disk in a separate directory. +// +// The trace dir contains a file 'functions.txt' which lists function names, +// oner per line, e.g. +// ==> functions.txt <== +// Func2 +// LLVMFuzzerTestOneInput +// Func1 +// +// All other files in the dir are the traces, see dataflow/DataFlow.cpp. +// The name of the file is sha1 of the input used to generate the trace. +// +// Current status: +// the data is parsed and the summary is printed, but the data is not yet +// used in any other way. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_DATA_FLOW_TRACE +#define LLVM_FUZZER_DATA_FLOW_TRACE + +#include "FuzzerDefs.h" +#include "FuzzerIO.h" + +#include <unordered_map> +#include <unordered_set> +#include <vector> +#include <string> + +namespace fuzzer { + +int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, + const Vector<SizedFile> &CorporaFiles); + +class BlockCoverage { + public: + bool AppendCoverage(std::istream &IN); + bool AppendCoverage(const std::string &S); + + size_t NumCoveredFunctions() const { return Functions.size(); } + + uint32_t GetCounter(size_t FunctionId, size_t BasicBlockId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + if (BasicBlockId < Counters.size()) + return Counters[BasicBlockId]; + return 0; + } + + uint32_t GetNumberOfBlocks(size_t FunctionId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + return Counters.size(); + } + + uint32_t GetNumberOfCoveredBlocks(size_t FunctionId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + uint32_t Result = 0; + for (auto Cnt: Counters) + if (Cnt) + Result++; + return Result; + } + + Vector<double> FunctionWeights(size_t NumFunctions) const; + void clear() { Functions.clear(); } + + private: + + typedef Vector<uint32_t> CoverageVector; + + uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const { + uint32_t Res = 0; + for (auto Cnt : Counters) + if (Cnt) + Res++; + return Res; + } + + uint32_t NumberOfUncoveredBlocks(const CoverageVector &Counters) const { + return Counters.size() - NumberOfCoveredBlocks(Counters); + } + + uint32_t SmallestNonZeroCounter(const CoverageVector &Counters) const { + assert(!Counters.empty()); + uint32_t Res = Counters[0]; + for (auto Cnt : Counters) + if (Cnt) + Res = Min(Res, Cnt); + assert(Res); + return Res; + } + + // Function ID => vector of counters. + // Each counter represents how many input files trigger the given basic block. + std::unordered_map<size_t, CoverageVector> Functions; + // Functions that have DFT entry. + std::unordered_set<size_t> FunctionsWithDFT; +}; + +class DataFlowTrace { + public: + int ReadCoverage(const std::string &DirPath); + int Init(const std::string &DirPath, std::string *FocusFunction, + Vector<SizedFile> &CorporaFiles, Random &Rand); + void Clear() { Traces.clear(); } + const Vector<uint8_t> *Get(const std::string &InputSha1) const { + auto It = Traces.find(InputSha1); + if (It != Traces.end()) + return &It->second; + return nullptr; + } + + private: + // Input's sha1 => DFT for the FocusFunction. + std::unordered_map<std::string, Vector<uint8_t> > Traces; + BlockCoverage Coverage; + std::unordered_set<std::string> CorporaHashes; +}; +} // namespace fuzzer + +#endif // LLVM_FUZZER_DATA_FLOW_TRACE diff --git a/tools/fuzzing/libfuzzer/FuzzerDefs.h b/tools/fuzzing/libfuzzer/FuzzerDefs.h new file mode 100644 index 0000000000..3952ac51ef --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDefs.h @@ -0,0 +1,75 @@ +//===- FuzzerDefs.h - Internal header for the Fuzzer ------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Basic definitions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_DEFS_H +#define LLVM_FUZZER_DEFS_H + +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <memory> +#include <set> +#include <string> +#include <vector> + + +namespace fuzzer { + +template <class T> T Min(T a, T b) { return a < b ? a : b; } +template <class T> T Max(T a, T b) { return a > b ? a : b; } + +class Random; +class Dictionary; +class DictionaryEntry; +class MutationDispatcher; +struct FuzzingOptions; +class InputCorpus; +struct InputInfo; +struct ExternalFunctions; + +// Global interface to functions that may or may not be available. +extern ExternalFunctions *EF; + +// We are using a custom allocator to give a different symbol name to STL +// containers in order to avoid ODR violations. +template<typename T> + class fuzzer_allocator: public std::allocator<T> { + public: + fuzzer_allocator() = default; + + template<class U> + explicit fuzzer_allocator(const fuzzer_allocator<U>&) {} + + template<class Other> + struct rebind { typedef fuzzer_allocator<Other> other; }; + }; + +template<typename T> +using Vector = std::vector<T, fuzzer_allocator<T>>; + +template<typename T> +using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>; + +typedef Vector<uint8_t> Unit; +typedef Vector<Unit> UnitVector; +typedef int (*UserCallback)(const uint8_t *Data, size_t Size); + +int FuzzerDriver(int *argc, char ***argv, UserCallback Callback); + +uint8_t *ExtraCountersBegin(); +uint8_t *ExtraCountersEnd(); +void ClearExtraCounters(); + +extern bool RunningUserCallback; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_DEFS_H diff --git a/tools/fuzzing/libfuzzer/FuzzerDictionary.h b/tools/fuzzing/libfuzzer/FuzzerDictionary.h new file mode 100644 index 0000000000..301c5d9afe --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDictionary.h @@ -0,0 +1,118 @@ +//===- FuzzerDictionary.h - Internal header for the Fuzzer ------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::Dictionary +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_DICTIONARY_H +#define LLVM_FUZZER_DICTIONARY_H + +#include "FuzzerDefs.h" +#include "FuzzerIO.h" +#include "FuzzerUtil.h" +#include <algorithm> +#include <limits> + +namespace fuzzer { +// A simple POD sized array of bytes. +template <size_t kMaxSizeT> class FixedWord { +public: + static const size_t kMaxSize = kMaxSizeT; + FixedWord() {} + FixedWord(const uint8_t *B, uint8_t S) { Set(B, S); } + + void Set(const uint8_t *B, uint8_t S) { + assert(S <= kMaxSize); + memcpy(Data, B, S); + Size = S; + } + + bool operator==(const FixedWord<kMaxSize> &w) const { + return Size == w.Size && 0 == memcmp(Data, w.Data, Size); + } + + static size_t GetMaxSize() { return kMaxSize; } + const uint8_t *data() const { return Data; } + uint8_t size() const { return Size; } + +private: + uint8_t Size = 0; + uint8_t Data[kMaxSize]; +}; + +typedef FixedWord<64> Word; + +class DictionaryEntry { + public: + DictionaryEntry() {} + DictionaryEntry(Word W) : W(W) {} + DictionaryEntry(Word W, size_t PositionHint) : W(W), PositionHint(PositionHint) {} + const Word &GetW() const { return W; } + + bool HasPositionHint() const { return PositionHint != std::numeric_limits<size_t>::max(); } + size_t GetPositionHint() const { + assert(HasPositionHint()); + return PositionHint; + } + void IncUseCount() { UseCount++; } + void IncSuccessCount() { SuccessCount++; } + size_t GetUseCount() const { return UseCount; } + size_t GetSuccessCount() const {return SuccessCount; } + + void Print(const char *PrintAfter = "\n") { + PrintASCII(W.data(), W.size()); + if (HasPositionHint()) + Printf("@%zd", GetPositionHint()); + Printf("%s", PrintAfter); + } + +private: + Word W; + size_t PositionHint = std::numeric_limits<size_t>::max(); + size_t UseCount = 0; + size_t SuccessCount = 0; +}; + +class Dictionary { + public: + static const size_t kMaxDictSize = 1 << 14; + + bool ContainsWord(const Word &W) const { + return std::any_of(begin(), end(), [&](const DictionaryEntry &DE) { + return DE.GetW() == W; + }); + } + const DictionaryEntry *begin() const { return &DE[0]; } + const DictionaryEntry *end() const { return begin() + Size; } + DictionaryEntry & operator[] (size_t Idx) { + assert(Idx < Size); + return DE[Idx]; + } + void push_back(DictionaryEntry DE) { + if (Size < kMaxDictSize) + this->DE[Size++] = DE; + } + void clear() { Size = 0; } + bool empty() const { return Size == 0; } + size_t size() const { return Size; } + +private: + DictionaryEntry DE[kMaxDictSize]; + size_t Size = 0; +}; + +// Parses one dictionary entry. +// If successful, write the enty to Unit and returns true, +// otherwise returns false. +bool ParseOneDictionaryEntry(const std::string &Str, Unit *U); +// Parses the dictionary file, fills Units, returns true iff all lines +// were parsed successfully. +bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units); + +} // namespace fuzzer + +#endif // LLVM_FUZZER_DICTIONARY_H diff --git a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp new file mode 100644 index 0000000000..bedad16efa --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp @@ -0,0 +1,888 @@ +//===- FuzzerDriver.cpp - FuzzerDriver function and flags -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// FuzzerDriver and flag parsing. +//===----------------------------------------------------------------------===// + +#include "FuzzerCommand.h" +#include "FuzzerCorpus.h" +#include "FuzzerFork.h" +#include "FuzzerIO.h" +#include "FuzzerInterface.h" +#include "FuzzerInternal.h" +#include "FuzzerMerge.h" +#include "FuzzerMutate.h" +#include "FuzzerPlatform.h" +#include "FuzzerRandom.h" +#include "FuzzerTracePC.h" +#include <algorithm> +#include <atomic> +#include <chrono> +#include <cstdlib> +#include <cstring> +#include <mutex> +#include <string> +#include <thread> +#include <fstream> + +// This function should be present in the libFuzzer so that the client +// binary can test for its existence. +#if LIBFUZZER_MSVC +extern "C" void __libfuzzer_is_present() {} +#if defined(_M_IX86) || defined(__i386__) +#pragma comment(linker, "/include:___libfuzzer_is_present") +#else +#pragma comment(linker, "/include:__libfuzzer_is_present") +#endif +#else +extern "C" __attribute__((used)) void __libfuzzer_is_present() {} +#endif // LIBFUZZER_MSVC + +namespace fuzzer { + +// Program arguments. +struct FlagDescription { + const char *Name; + const char *Description; + int Default; + int *IntFlag; + const char **StrFlag; + unsigned int *UIntFlag; +}; + +struct { +#define FUZZER_DEPRECATED_FLAG(Name) +#define FUZZER_FLAG_INT(Name, Default, Description) int Name; +#define FUZZER_FLAG_UNSIGNED(Name, Default, Description) unsigned int Name; +#define FUZZER_FLAG_STRING(Name, Description) const char *Name; +#include "FuzzerFlags.def" +#undef FUZZER_DEPRECATED_FLAG +#undef FUZZER_FLAG_INT +#undef FUZZER_FLAG_UNSIGNED +#undef FUZZER_FLAG_STRING +} Flags; + +static const FlagDescription FlagDescriptions [] { +#define FUZZER_DEPRECATED_FLAG(Name) \ + {#Name, "Deprecated; don't use", 0, nullptr, nullptr, nullptr}, +#define FUZZER_FLAG_INT(Name, Default, Description) \ + {#Name, Description, Default, &Flags.Name, nullptr, nullptr}, +#define FUZZER_FLAG_UNSIGNED(Name, Default, Description) \ + {#Name, Description, static_cast<int>(Default), \ + nullptr, nullptr, &Flags.Name}, +#define FUZZER_FLAG_STRING(Name, Description) \ + {#Name, Description, 0, nullptr, &Flags.Name, nullptr}, +#include "FuzzerFlags.def" +#undef FUZZER_DEPRECATED_FLAG +#undef FUZZER_FLAG_INT +#undef FUZZER_FLAG_UNSIGNED +#undef FUZZER_FLAG_STRING +}; + +static const size_t kNumFlags = + sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]); + +static Vector<std::string> *Inputs; +static std::string *ProgName; + +static void PrintHelp() { + Printf("Usage:\n"); + auto Prog = ProgName->c_str(); + Printf("\nTo run fuzzing pass 0 or more directories.\n"); + Printf("%s [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ]\n", Prog); + + Printf("\nTo run individual tests without fuzzing pass 1 or more files:\n"); + Printf("%s [-flag1=val1 [-flag2=val2 ...] ] file1 [file2 ...]\n", Prog); + + Printf("\nFlags: (strictly in form -flag=value)\n"); + size_t MaxFlagLen = 0; + for (size_t F = 0; F < kNumFlags; F++) + MaxFlagLen = std::max(strlen(FlagDescriptions[F].Name), MaxFlagLen); + + for (size_t F = 0; F < kNumFlags; F++) { + const auto &D = FlagDescriptions[F]; + if (strstr(D.Description, "internal flag") == D.Description) continue; + Printf(" %s", D.Name); + for (size_t i = 0, n = MaxFlagLen - strlen(D.Name); i < n; i++) + Printf(" "); + Printf("\t"); + Printf("%d\t%s\n", D.Default, D.Description); + } + Printf("\nFlags starting with '--' will be ignored and " + "will be passed verbatim to subprocesses.\n"); +} + +static const char *FlagValue(const char *Param, const char *Name) { + size_t Len = strlen(Name); + if (Param[0] == '-' && strstr(Param + 1, Name) == Param + 1 && + Param[Len + 1] == '=') + return &Param[Len + 2]; + return nullptr; +} + +// Avoid calling stol as it triggers a bug in clang/glibc build. +static long MyStol(const char *Str) { + long Res = 0; + long Sign = 1; + if (*Str == '-') { + Str++; + Sign = -1; + } + for (size_t i = 0; Str[i]; i++) { + char Ch = Str[i]; + if (Ch < '0' || Ch > '9') + return Res; + Res = Res * 10 + (Ch - '0'); + } + return Res * Sign; +} + +static bool ParseOneFlag(const char *Param) { + if (Param[0] != '-') return false; + if (Param[1] == '-') { + static bool PrintedWarning = false; + if (!PrintedWarning) { + PrintedWarning = true; + Printf("INFO: libFuzzer ignores flags that start with '--'\n"); + } + for (size_t F = 0; F < kNumFlags; F++) + if (FlagValue(Param + 1, FlagDescriptions[F].Name)) + Printf("WARNING: did you mean '%s' (single dash)?\n", Param + 1); + return true; + } + for (size_t F = 0; F < kNumFlags; F++) { + const char *Name = FlagDescriptions[F].Name; + const char *Str = FlagValue(Param, Name); + if (Str) { + if (FlagDescriptions[F].IntFlag) { + int Val = MyStol(Str); + *FlagDescriptions[F].IntFlag = Val; + if (Flags.verbosity >= 2) + Printf("Flag: %s %d\n", Name, Val); + return true; + } else if (FlagDescriptions[F].UIntFlag) { + unsigned int Val = std::stoul(Str); + *FlagDescriptions[F].UIntFlag = Val; + if (Flags.verbosity >= 2) + Printf("Flag: %s %u\n", Name, Val); + return true; + } else if (FlagDescriptions[F].StrFlag) { + *FlagDescriptions[F].StrFlag = Str; + if (Flags.verbosity >= 2) + Printf("Flag: %s %s\n", Name, Str); + return true; + } else { // Deprecated flag. + Printf("Flag: %s: deprecated, don't use\n", Name); + return true; + } + } + } + Printf("\n\nWARNING: unrecognized flag '%s'; " + "use -help=1 to list all flags\n\n", Param); + return true; +} + +// We don't use any library to minimize dependencies. +static void ParseFlags(const Vector<std::string> &Args, + const ExternalFunctions *EF) { + for (size_t F = 0; F < kNumFlags; F++) { + if (FlagDescriptions[F].IntFlag) + *FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default; + if (FlagDescriptions[F].UIntFlag) + *FlagDescriptions[F].UIntFlag = + static_cast<unsigned int>(FlagDescriptions[F].Default); + if (FlagDescriptions[F].StrFlag) + *FlagDescriptions[F].StrFlag = nullptr; + } + + // Disable len_control by default, if LLVMFuzzerCustomMutator is used. + if (EF->LLVMFuzzerCustomMutator) { + Flags.len_control = 0; + Printf("INFO: found LLVMFuzzerCustomMutator (%p). " + "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator); + } + + Inputs = new Vector<std::string>; + for (size_t A = 1; A < Args.size(); A++) { + if (ParseOneFlag(Args[A].c_str())) { + if (Flags.ignore_remaining_args) + break; + continue; + } + Inputs->push_back(Args[A]); + } +} + +static std::mutex Mu; + +static void PulseThread() { + while (true) { + SleepSeconds(600); + std::lock_guard<std::mutex> Lock(Mu); + Printf("pulse...\n"); + } +} + +static void WorkerThread(const Command &BaseCmd, std::atomic<unsigned> *Counter, + unsigned NumJobs, std::atomic<bool> *HasErrors) { + while (true) { + unsigned C = (*Counter)++; + if (C >= NumJobs) break; + std::string Log = "fuzz-" + std::to_string(C) + ".log"; + Command Cmd(BaseCmd); + Cmd.setOutputFile(Log); + Cmd.combineOutAndErr(); + if (Flags.verbosity) { + std::string CommandLine = Cmd.toString(); + Printf("%s\n", CommandLine.c_str()); + } + int ExitCode = ExecuteCommand(Cmd); + if (ExitCode != 0) + *HasErrors = true; + std::lock_guard<std::mutex> Lock(Mu); + Printf("================== Job %u exited with exit code %d ============\n", + C, ExitCode); + fuzzer::CopyFileToErr(Log); + } +} + +std::string CloneArgsWithoutX(const Vector<std::string> &Args, + const char *X1, const char *X2) { + std::string Cmd; + for (auto &S : Args) { + if (FlagValue(S.c_str(), X1) || FlagValue(S.c_str(), X2)) + continue; + Cmd += S + " "; + } + return Cmd; +} + +static int RunInMultipleProcesses(const Vector<std::string> &Args, + unsigned NumWorkers, unsigned NumJobs) { + std::atomic<unsigned> Counter(0); + std::atomic<bool> HasErrors(false); + Command Cmd(Args); + Cmd.removeFlag("jobs"); + Cmd.removeFlag("workers"); + Vector<std::thread> V; + std::thread Pulse(PulseThread); + Pulse.detach(); + for (unsigned i = 0; i < NumWorkers; i++) + V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); + for (auto &T : V) + T.join(); + return HasErrors ? 1 : 0; +} + +static void RssThread(Fuzzer *F, size_t RssLimitMb) { + while (true) { + SleepSeconds(1); + size_t Peak = GetPeakRSSMb(); + if (Peak > RssLimitMb) + F->RssLimitCallback(); + } +} + +static void StartRssThread(Fuzzer *F, size_t RssLimitMb) { + if (!RssLimitMb) + return; + std::thread T(RssThread, F, RssLimitMb); + T.detach(); +} + +int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) { + Unit U = FileToVector(InputFilePath); + if (MaxLen && MaxLen < U.size()) + U.resize(MaxLen); + F->ExecuteCallback(U.data(), U.size()); + F->TryDetectingAMemoryLeak(U.data(), U.size(), true); + return 0; +} + +static bool AllInputsAreFiles() { + if (Inputs->empty()) return false; + for (auto &Path : *Inputs) + if (!IsFile(Path)) + return false; + return true; +} + +static std::string GetDedupTokenFromCmdOutput(const std::string &S) { + auto Beg = S.find("DEDUP_TOKEN:"); + if (Beg == std::string::npos) + return ""; + auto End = S.find('\n', Beg); + if (End == std::string::npos) + return ""; + return S.substr(Beg, End - Beg); +} + +int CleanseCrashInput(const Vector<std::string> &Args, + const FuzzingOptions &Options) { + if (Inputs->size() != 1 || !Flags.exact_artifact_path) { + Printf("ERROR: -cleanse_crash should be given one input file and" + " -exact_artifact_path\n"); + return 1; + } + std::string InputFilePath = Inputs->at(0); + std::string OutputFilePath = Flags.exact_artifact_path; + Command Cmd(Args); + Cmd.removeFlag("cleanse_crash"); + + assert(Cmd.hasArgument(InputFilePath)); + Cmd.removeArgument(InputFilePath); + + auto TmpFilePath = TempPath("CleanseCrashInput", ".repro"); + Cmd.addArgument(TmpFilePath); + Cmd.setOutputFile(getDevNull()); + Cmd.combineOutAndErr(); + + std::string CurrentFilePath = InputFilePath; + auto U = FileToVector(CurrentFilePath); + size_t Size = U.size(); + + const Vector<uint8_t> ReplacementBytes = {' ', 0xff}; + for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) { + bool Changed = false; + for (size_t Idx = 0; Idx < Size; Idx++) { + Printf("CLEANSE[%d]: Trying to replace byte %zd of %zd\n", NumAttempts, + Idx, Size); + uint8_t OriginalByte = U[Idx]; + if (ReplacementBytes.end() != std::find(ReplacementBytes.begin(), + ReplacementBytes.end(), + OriginalByte)) + continue; + for (auto NewByte : ReplacementBytes) { + U[Idx] = NewByte; + WriteToFile(U, TmpFilePath); + auto ExitCode = ExecuteCommand(Cmd); + RemoveFile(TmpFilePath); + if (!ExitCode) { + U[Idx] = OriginalByte; + } else { + Changed = true; + Printf("CLEANSE: Replaced byte %zd with 0x%x\n", Idx, NewByte); + WriteToFile(U, OutputFilePath); + break; + } + } + } + if (!Changed) break; + } + return 0; +} + +int MinimizeCrashInput(const Vector<std::string> &Args, + const FuzzingOptions &Options) { + if (Inputs->size() != 1) { + Printf("ERROR: -minimize_crash should be given one input file\n"); + return 1; + } + std::string InputFilePath = Inputs->at(0); + Command BaseCmd(Args); + BaseCmd.removeFlag("minimize_crash"); + BaseCmd.removeFlag("exact_artifact_path"); + assert(BaseCmd.hasArgument(InputFilePath)); + BaseCmd.removeArgument(InputFilePath); + if (Flags.runs <= 0 && Flags.max_total_time == 0) { + Printf("INFO: you need to specify -runs=N or " + "-max_total_time=N with -minimize_crash=1\n" + "INFO: defaulting to -max_total_time=600\n"); + BaseCmd.addFlag("max_total_time", "600"); + } + + BaseCmd.combineOutAndErr(); + + std::string CurrentFilePath = InputFilePath; + while (true) { + Unit U = FileToVector(CurrentFilePath); + Printf("CRASH_MIN: minimizing crash input: '%s' (%zd bytes)\n", + CurrentFilePath.c_str(), U.size()); + + Command Cmd(BaseCmd); + Cmd.addArgument(CurrentFilePath); + + Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str()); + std::string CmdOutput; + bool Success = ExecuteCommand(Cmd, &CmdOutput); + if (Success) { + Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); + return 1; + } + Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " + "it further\n", + CurrentFilePath.c_str(), U.size()); + auto DedupToken1 = GetDedupTokenFromCmdOutput(CmdOutput); + if (!DedupToken1.empty()) + Printf("CRASH_MIN: DedupToken1: %s\n", DedupToken1.c_str()); + + std::string ArtifactPath = + Flags.exact_artifact_path + ? Flags.exact_artifact_path + : Options.ArtifactPrefix + "minimized-from-" + Hash(U); + Cmd.addFlag("minimize_crash_internal_step", "1"); + Cmd.addFlag("exact_artifact_path", ArtifactPath); + Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str()); + CmdOutput.clear(); + Success = ExecuteCommand(Cmd, &CmdOutput); + Printf("%s", CmdOutput.c_str()); + if (Success) { + if (Flags.exact_artifact_path) { + CurrentFilePath = Flags.exact_artifact_path; + WriteToFile(U, CurrentFilePath); + } + Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n", + CurrentFilePath.c_str(), U.size()); + break; + } + auto DedupToken2 = GetDedupTokenFromCmdOutput(CmdOutput); + if (!DedupToken2.empty()) + Printf("CRASH_MIN: DedupToken2: %s\n", DedupToken2.c_str()); + + if (DedupToken1 != DedupToken2) { + if (Flags.exact_artifact_path) { + CurrentFilePath = Flags.exact_artifact_path; + WriteToFile(U, CurrentFilePath); + } + Printf("CRASH_MIN: mismatch in dedup tokens" + " (looks like a different bug). Won't minimize further\n"); + break; + } + + CurrentFilePath = ArtifactPath; + Printf("*********************************\n"); + } + return 0; +} + +int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { + assert(Inputs->size() == 1); + std::string InputFilePath = Inputs->at(0); + Unit U = FileToVector(InputFilePath); + Printf("INFO: Starting MinimizeCrashInputInternalStep: %zd\n", U.size()); + if (U.size() < 2) { + Printf("INFO: The input is small enough, exiting\n"); + return 0; + } + F->SetMaxInputLen(U.size()); + F->SetMaxMutationLen(U.size() - 1); + F->MinimizeCrashLoop(U); + Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n"); + return 0; +} + +int Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, + const Vector<std::string> &Corpora, const char *CFPathOrNull) { + if (Corpora.size() < 2) { + Printf("INFO: Merge requires two or more corpus dirs\n"); + return 0; + } + + Vector<SizedFile> OldCorpus, NewCorpus; + int Res = GetSizedFilesFromDir(Corpora[0], &OldCorpus); + if (Res != 0) + return Res; + for (size_t i = 1; i < Corpora.size(); i++) { + Res = GetSizedFilesFromDir(Corpora[i], &NewCorpus); + if (Res != 0) + return Res; + } + std::sort(OldCorpus.begin(), OldCorpus.end()); + std::sort(NewCorpus.begin(), NewCorpus.end()); + + std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt"); + Vector<std::string> NewFiles; + Set<uint32_t> NewFeatures, NewCov; + Res = CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, + {}, &NewCov, CFPath, true); + if (Res != 0) + return Res; + + if (F->isGracefulExitRequested()) + return 0; + for (auto &Path : NewFiles) + F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); + // We are done, delete the control file if it was a temporary one. + if (!Flags.merge_control_file) + RemoveFile(CFPath); + + return 0; +} + +int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, + UnitVector& Corpus) { + Printf("Started dictionary minimization (up to %d tests)\n", + Dict.size() * Corpus.size() * 2); + + // Scores and usage count for each dictionary unit. + Vector<int> Scores(Dict.size()); + Vector<int> Usages(Dict.size()); + + Vector<size_t> InitialFeatures; + Vector<size_t> ModifiedFeatures; + for (auto &C : Corpus) { + // Get coverage for the testcase without modifications. + F->ExecuteCallback(C.data(), C.size()); + InitialFeatures.clear(); + TPC.CollectFeatures([&](size_t Feature) { + InitialFeatures.push_back(Feature); + }); + + for (size_t i = 0; i < Dict.size(); ++i) { + Vector<uint8_t> Data = C; + auto StartPos = std::search(Data.begin(), Data.end(), + Dict[i].begin(), Dict[i].end()); + // Skip dictionary unit, if the testcase does not contain it. + if (StartPos == Data.end()) + continue; + + ++Usages[i]; + while (StartPos != Data.end()) { + // Replace all occurrences of dictionary unit in the testcase. + auto EndPos = StartPos + Dict[i].size(); + for (auto It = StartPos; It != EndPos; ++It) + *It ^= 0xFF; + + StartPos = std::search(EndPos, Data.end(), + Dict[i].begin(), Dict[i].end()); + } + + // Get coverage for testcase with masked occurrences of dictionary unit. + F->ExecuteCallback(Data.data(), Data.size()); + ModifiedFeatures.clear(); + TPC.CollectFeatures([&](size_t Feature) { + ModifiedFeatures.push_back(Feature); + }); + + if (InitialFeatures == ModifiedFeatures) + --Scores[i]; + else + Scores[i] += 2; + } + } + + Printf("###### Useless dictionary elements. ######\n"); + for (size_t i = 0; i < Dict.size(); ++i) { + // Dictionary units with positive score are treated as useful ones. + if (Scores[i] > 0) + continue; + + Printf("\""); + PrintASCII(Dict[i].data(), Dict[i].size(), "\""); + Printf(" # Score: %d, Used: %d\n", Scores[i], Usages[i]); + } + Printf("###### End of useless dictionary elements. ######\n"); + return 0; +} + +int ParseSeedInuts(const char *seed_inputs, Vector<std::string> &Files) { + // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file + if (!seed_inputs) return 0; + std::string SeedInputs; + if (Flags.seed_inputs[0] == '@') + SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. + else + SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. + if (SeedInputs.empty()) { + Printf("seed_inputs is empty or @file does not exist.\n"); + return 1; + } + // Parse SeedInputs. + size_t comma_pos = 0; + while ((comma_pos = SeedInputs.find_last_of(',')) != std::string::npos) { + Files.push_back(SeedInputs.substr(comma_pos + 1)); + SeedInputs = SeedInputs.substr(0, comma_pos); + } + Files.push_back(SeedInputs); + return 0; +} + +static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs, + const Vector<std::string> &ExtraSeedFiles) { + Vector<SizedFile> SizedFiles; + size_t LastNumFiles = 0; + for (auto &Dir : CorpusDirs) { + GetSizedFilesFromDir(Dir, &SizedFiles); + Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles, + Dir.c_str()); + LastNumFiles = SizedFiles.size(); + } + for (auto &File : ExtraSeedFiles) + if (auto Size = FileSize(File)) + SizedFiles.push_back({File, Size}); + return SizedFiles; +} + +int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + using namespace fuzzer; + assert(argc && argv && "Argument pointers cannot be nullptr"); + std::string Argv0((*argv)[0]); + if (!EF) + EF = new ExternalFunctions(); + if (EF->LLVMFuzzerInitialize) + EF->LLVMFuzzerInitialize(argc, argv); + if (EF->__msan_scoped_disable_interceptor_checks) + EF->__msan_scoped_disable_interceptor_checks(); + const Vector<std::string> Args(*argv, *argv + *argc); + assert(!Args.empty()); + ProgName = new std::string(Args[0]); + if (Argv0 != *ProgName) { + Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n"); + return 1; + } + ParseFlags(Args, EF); + if (Flags.help) { + PrintHelp(); + return 0; + } + + if (Flags.close_fd_mask & 2) + DupAndCloseStderr(); + if (Flags.close_fd_mask & 1) + CloseStdout(); + + if (Flags.jobs > 0 && Flags.workers == 0) { + Flags.workers = std::min(NumberOfCpuCores() / 2, Flags.jobs); + if (Flags.workers > 1) + Printf("Running %u workers\n", Flags.workers); + } + + if (Flags.workers > 0 && Flags.jobs > 0) + return RunInMultipleProcesses(Args, Flags.workers, Flags.jobs); + + FuzzingOptions Options; + Options.Verbosity = Flags.verbosity; + Options.MaxLen = Flags.max_len; + Options.LenControl = Flags.len_control; + Options.UnitTimeoutSec = Flags.timeout; + Options.ErrorExitCode = Flags.error_exitcode; + Options.TimeoutExitCode = Flags.timeout_exitcode; + Options.IgnoreTimeouts = Flags.ignore_timeouts; + Options.IgnoreOOMs = Flags.ignore_ooms; + Options.IgnoreCrashes = Flags.ignore_crashes; + Options.MaxTotalTimeSec = Flags.max_total_time; + Options.DoCrossOver = Flags.cross_over; + Options.MutateDepth = Flags.mutate_depth; + Options.ReduceDepth = Flags.reduce_depth; + Options.UseCounters = Flags.use_counters; + Options.UseMemmem = Flags.use_memmem; + Options.UseCmp = Flags.use_cmp; + Options.UseValueProfile = Flags.use_value_profile; + Options.Shrink = Flags.shrink; + Options.ReduceInputs = Flags.reduce_inputs; + Options.ShuffleAtStartUp = Flags.shuffle; + Options.PreferSmall = Flags.prefer_small; + Options.ReloadIntervalSec = Flags.reload; + Options.OnlyASCII = Flags.only_ascii; + Options.DetectLeaks = Flags.detect_leaks; + Options.PurgeAllocatorIntervalSec = Flags.purge_allocator_interval; + Options.TraceMalloc = Flags.trace_malloc; + Options.RssLimitMb = Flags.rss_limit_mb; + Options.MallocLimitMb = Flags.malloc_limit_mb; + if (!Options.MallocLimitMb) + Options.MallocLimitMb = Options.RssLimitMb; + if (Flags.runs >= 0) + Options.MaxNumberOfRuns = Flags.runs; + if (!Inputs->empty() && !Flags.minimize_crash_internal_step) + Options.OutputCorpus = (*Inputs)[0]; + Options.ReportSlowUnits = Flags.report_slow_units; + if (Flags.artifact_prefix) + Options.ArtifactPrefix = Flags.artifact_prefix; + if (Flags.exact_artifact_path) + Options.ExactArtifactPath = Flags.exact_artifact_path; + Vector<Unit> Dictionary; + if (Flags.dict) + if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) + return 1; + if (Flags.verbosity > 0 && !Dictionary.empty()) + Printf("Dictionary: %zd entries\n", Dictionary.size()); + bool RunIndividualFiles = AllInputsAreFiles(); + Options.SaveArtifacts = + !RunIndividualFiles || Flags.minimize_crash_internal_step; + Options.PrintNewCovPcs = Flags.print_pcs; + Options.PrintNewCovFuncs = Flags.print_funcs; + Options.PrintFinalStats = Flags.print_final_stats; + Options.PrintCorpusStats = Flags.print_corpus_stats; + Options.PrintCoverage = Flags.print_coverage; + if (Flags.exit_on_src_pos) + Options.ExitOnSrcPos = Flags.exit_on_src_pos; + if (Flags.exit_on_item) + Options.ExitOnItem = Flags.exit_on_item; + if (Flags.focus_function) + Options.FocusFunction = Flags.focus_function; + if (Flags.data_flow_trace) + Options.DataFlowTrace = Flags.data_flow_trace; + if (Flags.features_dir) + Options.FeaturesDir = Flags.features_dir; + if (Flags.collect_data_flow) + Options.CollectDataFlow = Flags.collect_data_flow; + if (Flags.stop_file) + Options.StopFile = Flags.stop_file; + Options.Entropic = Flags.entropic; + Options.EntropicFeatureFrequencyThreshold = + (size_t)Flags.entropic_feature_frequency_threshold; + Options.EntropicNumberOfRarestFeatures = + (size_t)Flags.entropic_number_of_rarest_features; + if (Options.Entropic) { + if (!Options.FocusFunction.empty()) { + Printf("ERROR: The parameters `--entropic` and `--focus_function` cannot " + "be used together.\n"); + return 1; + } + Printf("INFO: Running with entropic power schedule (0x%X, %d).\n", + Options.EntropicFeatureFrequencyThreshold, + Options.EntropicNumberOfRarestFeatures); + } + struct EntropicOptions Entropic; + Entropic.Enabled = Options.Entropic; + Entropic.FeatureFrequencyThreshold = + Options.EntropicFeatureFrequencyThreshold; + Entropic.NumberOfRarestFeatures = Options.EntropicNumberOfRarestFeatures; + + unsigned Seed = Flags.seed; + // Initialize Seed. + if (Seed == 0) + Seed = + std::chrono::system_clock::now().time_since_epoch().count() + GetPid(); + if (Flags.verbosity) + Printf("INFO: Seed: %u\n", Seed); + + if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) { + if (RunIndividualFiles) + return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, + ReadCorpora({}, *Inputs)); + else + return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, + ReadCorpora(*Inputs, {})); + } + + Random Rand(Seed); + auto *MD = new MutationDispatcher(Rand, Options); + auto *Corpus = new InputCorpus(Options.OutputCorpus, Entropic); + auto *F = new Fuzzer(Callback, *Corpus, *MD, Options); + + for (auto &U: Dictionary) + if (U.size() <= Word::GetMaxSize()) + MD->AddWordToManualDictionary(Word(U.data(), U.size())); + + // Threads are only supported by Chrome. Don't use them with emscripten + // for now. +#if !LIBFUZZER_EMSCRIPTEN + StartRssThread(F, Flags.rss_limit_mb); +#endif // LIBFUZZER_EMSCRIPTEN + + Options.HandleAbrt = Flags.handle_abrt; + Options.HandleBus = Flags.handle_bus; + Options.HandleFpe = Flags.handle_fpe; + Options.HandleIll = Flags.handle_ill; + Options.HandleInt = Flags.handle_int; + Options.HandleSegv = Flags.handle_segv; + Options.HandleTerm = Flags.handle_term; + Options.HandleXfsz = Flags.handle_xfsz; + Options.HandleUsr1 = Flags.handle_usr1; + Options.HandleUsr2 = Flags.handle_usr2; + SetSignalHandler(Options); + + std::atexit(Fuzzer::StaticExitCallback); + + if (Flags.minimize_crash) + return MinimizeCrashInput(Args, Options); + + if (Flags.minimize_crash_internal_step) + return MinimizeCrashInputInternalStep(F, Corpus); + + if (Flags.cleanse_crash) + return CleanseCrashInput(Args, Options); + + if (RunIndividualFiles) { + Options.SaveArtifacts = false; + int Runs = std::max(1, Flags.runs); + Printf("%s: Running %zd inputs %d time(s) each.\n", ProgName->c_str(), + Inputs->size(), Runs); + for (auto &Path : *Inputs) { + auto StartTime = system_clock::now(); + Printf("Running: %s\n", Path.c_str()); + for (int Iter = 0; Iter < Runs; Iter++) + RunOneTest(F, Path.c_str(), Options.MaxLen); + auto StopTime = system_clock::now(); + auto MS = duration_cast<milliseconds>(StopTime - StartTime).count(); + Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS); + } + Printf("***\n" + "*** NOTE: fuzzing was not performed, you have only\n" + "*** executed the target code on a fixed set of inputs.\n" + "***\n"); + F->PrintFinalStats(); + return 0; + } + + if (Flags.fork) + return FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + + if (Flags.merge) + return Merge(F, Options, Args, *Inputs, Flags.merge_control_file); + + if (Flags.merge_inner) { + const size_t kDefaultMaxMergeLen = 1 << 20; + if (Options.MaxLen == 0) + F->SetMaxInputLen(kDefaultMaxMergeLen); + assert(Flags.merge_control_file); + return F->CrashResistantMergeInternalStep(Flags.merge_control_file); + } + + if (Flags.analyze_dict) { + size_t MaxLen = INT_MAX; // Large max length. + UnitVector InitialCorpus; + for (auto &Inp : *Inputs) { + Printf("Loading corpus dir: %s\n", Inp.c_str()); + ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, + MaxLen, /*ExitOnError=*/false); + } + + if (Dictionary.empty() || Inputs->empty()) { + Printf("ERROR: can't analyze dict without dict and corpus provided\n"); + return 1; + } + if (AnalyzeDictionary(F, Dictionary, InitialCorpus)) { + Printf("Dictionary analysis failed\n"); + return 1; + } + Printf("Dictionary analysis succeeded\n"); + return 0; + } + + { + Vector<std::string> Files; + int Res = ParseSeedInuts(Flags.seed_inputs, Files); + if (Res != 0) + return Res; + auto CorporaFiles = ReadCorpora(*Inputs, Files); + Res = F->Loop(CorporaFiles); + if (Res != 0) + return Res; + if (F->isGracefulExitRequested()) + return 0; + } + + if (Flags.verbosity) + Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), + F->secondsSinceProcessStartUp()); + F->PrintFinalStats(); + + return 0; // Don't let F destroy itself. +} + +extern "C" ATTRIBUTE_INTERFACE int +LLVMFuzzerRunDriver(int *argc, char ***argv, + int (*UserCb)(const uint8_t *Data, size_t Size)) { + return FuzzerDriver(argc, argv, UserCb); +} + +// Storage for global ExternalFunctions object. +ExternalFunctions *EF = nullptr; + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctions.def b/tools/fuzzing/libfuzzer/FuzzerExtFunctions.def new file mode 100644 index 0000000000..51edf8444e --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctions.def @@ -0,0 +1,50 @@ +//===- FuzzerExtFunctions.def - External functions --------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// This defines the external function pointers that +// ``fuzzer::ExternalFunctions`` should contain and try to initialize. The +// EXT_FUNC macro must be defined at the point of inclusion. The signature of +// the macro is: +// +// EXT_FUNC(<name>, <return_type>, <function_signature>, <warn_if_missing>) +//===----------------------------------------------------------------------===// + +// Optional user functions +EXT_FUNC(LLVMFuzzerInitialize, int, (int *argc, char ***argv), false); +EXT_FUNC(LLVMFuzzerCustomMutator, size_t, + (uint8_t *Data, size_t Size, size_t MaxSize, unsigned int Seed), + false); +EXT_FUNC(LLVMFuzzerCustomCrossOver, size_t, + (const uint8_t *Data1, size_t Size1, + const uint8_t *Data2, size_t Size2, + uint8_t *Out, size_t MaxOutSize, unsigned int Seed), + false); + +// Sanitizer functions +EXT_FUNC(__lsan_enable, void, (), false); +EXT_FUNC(__lsan_disable, void, (), false); +EXT_FUNC(__lsan_do_recoverable_leak_check, int, (), false); +EXT_FUNC(__sanitizer_acquire_crash_state, int, (), true); +EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int, + (void (*malloc_hook)(const volatile void *, size_t), + void (*free_hook)(const volatile void *)), + false); +EXT_FUNC(__sanitizer_log_write, void, (const char *buf, size_t len), false); +EXT_FUNC(__sanitizer_purge_allocator, void, (), false); +EXT_FUNC(__sanitizer_print_memory_profile, void, (size_t, size_t), false); +EXT_FUNC(__sanitizer_print_stack_trace, void, (), true); +EXT_FUNC(__sanitizer_symbolize_pc, void, + (void *, const char *fmt, char *out_buf, size_t out_buf_size), false); +EXT_FUNC(__sanitizer_get_module_and_offset_for_pc, int, + (void *pc, char *module_path, + size_t module_path_len,void **pc_offset), false); +EXT_FUNC(__sanitizer_set_death_callback, void, (void (*)(void)), true); +EXT_FUNC(__sanitizer_set_report_fd, void, (void*), false); +EXT_FUNC(__msan_scoped_disable_interceptor_checks, void, (), false); +EXT_FUNC(__msan_scoped_enable_interceptor_checks, void, (), false); +EXT_FUNC(__msan_unpoison, void, (const volatile void *, size_t size), false); +EXT_FUNC(__msan_unpoison_param, void, (size_t n), false); diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctions.h b/tools/fuzzing/libfuzzer/FuzzerExtFunctions.h new file mode 100644 index 0000000000..c88aac4e67 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctions.h @@ -0,0 +1,34 @@ +//===- FuzzerExtFunctions.h - Interface to external functions ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Defines an interface to (possibly optional) functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_EXT_FUNCTIONS_H +#define LLVM_FUZZER_EXT_FUNCTIONS_H + +#include <stddef.h> +#include <stdint.h> + +namespace fuzzer { + +struct ExternalFunctions { + // Initialize function pointers. Functions that are not available will be set + // to nullptr. Do not call this constructor before ``main()`` has been + // entered. + ExternalFunctions(); + +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + RETURN_TYPE(*NAME) FUNC_SIG = nullptr + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +}; +} // namespace fuzzer + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsym.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsym.cpp new file mode 100644 index 0000000000..95233d2a10 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsym.cpp @@ -0,0 +1,51 @@ +//===- FuzzerExtFunctionsDlsym.cpp - Interface to external functions ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Implementation for operating systems that support dlsym(). We only use it on +// Apple platforms for now. We don't use this approach on Linux because it +// requires that clients of LibFuzzer pass ``--export-dynamic`` to the linker. +// That is a complication we don't wish to expose to clients right now. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_APPLE + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include <dlfcn.h> + +using namespace fuzzer; + +template <typename T> +static T GetFnPtr(const char *FnName, bool WarnIfMissing) { + dlerror(); // Clear any previous errors. + void *Fn = dlsym(RTLD_DEFAULT, FnName); + if (Fn == nullptr) { + if (WarnIfMissing) { + const char *ErrorMsg = dlerror(); + Printf("WARNING: Failed to find function \"%s\".", FnName); + if (ErrorMsg) + Printf(" Reason %s.", ErrorMsg); + Printf("\n"); + } + } + return reinterpret_cast<T>(Fn); +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = GetFnPtr<decltype(ExternalFunctions::NAME)>(#NAME, WARN) + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif // LIBFUZZER_APPLE diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp new file mode 100644 index 0000000000..24ddc57d47 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp @@ -0,0 +1,54 @@ +//===- FuzzerExtFunctionsWeak.cpp - Interface to external functions -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Implementation for Linux. This relies on the linker's support for weak +// symbols. We don't use this approach on Apple platforms because it requires +// clients of LibFuzzer to pass ``-U _<symbol_name>`` to the linker to allow +// weak symbols to be undefined. That is a complication we don't want to expose +// to clients right now. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUCHSIA || \ + LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" + +extern "C" { +// Declare these symbols as weak to allow them to be optionally defined. +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + __attribute__((weak, visibility("default"))) RETURN_TYPE NAME FUNC_SIG + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +using namespace fuzzer; + +static void CheckFnPtr(void *FnPtr, const char *FnName, bool WarnIfMissing) { + if (FnPtr == nullptr && WarnIfMissing) { + Printf("WARNING: Failed to find function \"%s\".\n", FnName); + } +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = ::NAME; \ + CheckFnPtr(reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(::NAME)), \ + #NAME, WARN); + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWindows.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWindows.cpp new file mode 100644 index 0000000000..688bad1d51 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWindows.cpp @@ -0,0 +1,82 @@ +//=== FuzzerExtWindows.cpp - Interface to external functions --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Implementation of FuzzerExtFunctions for Windows. Uses alternatename when +// compiled with MSVC. Uses weak aliases when compiled with clang. Unfortunately +// the method each compiler supports is not supported by the other. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" + +using namespace fuzzer; + +// Intermediate macro to ensure the parameter is expanded before stringified. +#define STRINGIFY_(A) #A +#define STRINGIFY(A) STRINGIFY_(A) + +#if LIBFUZZER_MSVC +// Copied from compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h +#if defined(_M_IX86) || defined(__i386__) +#define WIN_SYM_PREFIX "_" +#else +#define WIN_SYM_PREFIX +#endif + +// Declare external functions as having alternativenames, so that we can +// determine if they are not defined. +#define EXTERNAL_FUNC(Name, Default) \ + __pragma(comment(linker, "/alternatename:" WIN_SYM_PREFIX STRINGIFY( \ + Name) "=" WIN_SYM_PREFIX STRINGIFY(Default))) +#else +// Declare external functions as weak to allow them to default to a specified +// function if not defined explicitly. We must use weak symbols because clang's +// support for alternatename is not 100%, see +// https://bugs.llvm.org/show_bug.cgi?id=40218 for more details. +#define EXTERNAL_FUNC(Name, Default) \ + __attribute__((weak, alias(STRINGIFY(Default)))) +#endif // LIBFUZZER_MSVC + +extern "C" { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + RETURN_TYPE NAME##Def FUNC_SIG { \ + Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ + exit(1); \ + } \ + EXTERNAL_FUNC(NAME, NAME##Def) RETURN_TYPE NAME FUNC_SIG + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +template <typename T> +static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { + if (Fun == FunDef) { + if (WarnIfMissing) + Printf("WARNING: Failed to find function \"%s\".\n", FnName); + return nullptr; + } + return Fun; +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = GetFnPtr<decltype(::NAME)>(::NAME, ::NAME##Def, #NAME, WARN); + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp b/tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp new file mode 100644 index 0000000000..d36beba1b1 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp @@ -0,0 +1,42 @@ +//===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Extra coverage counters defined by user code. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" +#include <cstdint> + +#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ + LIBFUZZER_OPENBSD || LIBFUZZER_FUCHSIA || LIBFUZZER_EMSCRIPTEN +__attribute__((weak)) extern uint8_t __start___libfuzzer_extra_counters; +__attribute__((weak)) extern uint8_t __stop___libfuzzer_extra_counters; + +namespace fuzzer { +uint8_t *ExtraCountersBegin() { return &__start___libfuzzer_extra_counters; } +uint8_t *ExtraCountersEnd() { return &__stop___libfuzzer_extra_counters; } +ATTRIBUTE_NO_SANITIZE_ALL +void ClearExtraCounters() { // hand-written memset, don't asan-ify. + uintptr_t *Beg = reinterpret_cast<uintptr_t*>(ExtraCountersBegin()); + uintptr_t *End = reinterpret_cast<uintptr_t*>(ExtraCountersEnd()); + for (; Beg < End; Beg++) { + *Beg = 0; + __asm__ __volatile__("" : : : "memory"); + } +} + +} // namespace fuzzer + +#else +// TODO: implement for other platforms. +namespace fuzzer { +uint8_t *ExtraCountersBegin() { return nullptr; } +uint8_t *ExtraCountersEnd() { return nullptr; } +void ClearExtraCounters() {} +} // namespace fuzzer + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerFlags.def b/tools/fuzzing/libfuzzer/FuzzerFlags.def new file mode 100644 index 0000000000..832224a705 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerFlags.def @@ -0,0 +1,169 @@ +//===- FuzzerFlags.def - Run-time flags -------------------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Flags. FUZZER_FLAG_INT/FUZZER_FLAG_STRING macros should be defined at the +// point of inclusion. We are not using any flag parsing library for better +// portability and independence. +//===----------------------------------------------------------------------===// +FUZZER_FLAG_INT(verbosity, 1, "Verbosity level.") +FUZZER_FLAG_UNSIGNED(seed, 0, "Random seed. If 0, seed is generated.") +FUZZER_FLAG_INT(runs, -1, + "Number of individual test runs (-1 for infinite runs).") +FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. " + "If 0, libFuzzer tries to guess a good value based on the corpus " + "and reports it. ") +FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " + "then try larger inputs over time. Specifies the rate at which the length " + "limit is increased (smaller == faster). If 0, immediately try inputs with " + "size up to max_len. Default value is 0, if LLVMFuzzerCustomMutator is used.") +FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " + "to use as an additional seed corpus. Alternatively, an \"@\" followed by " + "the name of a file containing the comma-separated list.") +FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") +FUZZER_FLAG_INT(mutate_depth, 5, + "Apply this number of consecutive mutations to each input.") +FUZZER_FLAG_INT(reduce_depth, 0, "Experimental/internal. " + "Reduce depth if mutations lose unique features") +FUZZER_FLAG_INT(shuffle, 1, "Shuffle inputs at startup") +FUZZER_FLAG_INT(prefer_small, 1, + "If 1, always prefer smaller inputs during the corpus shuffle.") +FUZZER_FLAG_INT( + timeout, 1200, + "Timeout in seconds (if positive). " + "If one unit runs more than this number of seconds the process will abort.") +FUZZER_FLAG_INT(error_exitcode, 77, "When libFuzzer itself reports a bug " + "this exit code will be used.") +FUZZER_FLAG_INT(timeout_exitcode, 70, "When libFuzzer reports a timeout " + "this exit code will be used.") +FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " + "time in seconds to run the fuzzer.") +FUZZER_FLAG_INT(help, 0, "Print help.") +FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " + "in a subprocess") +FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") +FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode") +FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode") +FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " + "merged into the 1-st corpus. Only interesting units will be taken. " + "This flag can be used to minimize a corpus.") +FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists") +FUZZER_FLAG_STRING(merge_inner, "internal flag") +FUZZER_FLAG_STRING(merge_control_file, + "Specify a control file used for the merge process. " + "If a merge process gets killed it tries to leave this file " + "in a state suitable for resuming the merge. " + "By default a temporary file will be used." + "The same file can be used for multistep merge process.") +FUZZER_FLAG_INT(minimize_crash, 0, "If 1, minimizes the provided" + " crash input. Use with -runs=N or -max_total_time=N to limit " + "the number attempts." + " Use with -exact_artifact_path to specify the output." + " Combine with ASAN_OPTIONS=dedup_token_length=3 (or similar) to ensure that" + " the minimized input triggers the same crash." + ) +FUZZER_FLAG_INT(cleanse_crash, 0, "If 1, tries to cleanse the provided" + " crash input to make it contain fewer original bytes." + " Use with -exact_artifact_path to specify the output." + ) +FUZZER_FLAG_INT(minimize_crash_internal_step, 0, "internal flag") +FUZZER_FLAG_STRING(features_dir, "internal flag. Used to dump feature sets on disk." + "Every time a new input is added to the corpus, a corresponding file in the features_dir" + " is created containing the unique features of that input." + " Features are stored in binary format.") +FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters") +FUZZER_FLAG_INT(use_memmem, 1, + "Use hints from intercepting memmem, strstr, etc") +FUZZER_FLAG_INT(use_value_profile, 0, + "Experimental. Use value profile to guide fuzzing.") +FUZZER_FLAG_INT(use_cmp, 1, "Use CMP traces to guide mutations") +FUZZER_FLAG_INT(shrink, 0, "Experimental. Try to shrink corpus inputs.") +FUZZER_FLAG_INT(reduce_inputs, 1, + "Try to reduce the size of inputs while preserving their full feature sets") +FUZZER_FLAG_UNSIGNED(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn" + " this number of jobs in separate worker processes" + " with stdout/stderr redirected to fuzz-JOB.log.") +FUZZER_FLAG_UNSIGNED(workers, 0, + "Number of simultaneous worker processes to run the jobs." + " If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.") +FUZZER_FLAG_INT(reload, 1, + "Reload the main corpus every <N> seconds to get new units" + " discovered by other processes. If 0, disabled") +FUZZER_FLAG_INT(report_slow_units, 10, + "Report slowest units if they run for more than this number of seconds.") +FUZZER_FLAG_INT(only_ascii, 0, + "If 1, generate only ASCII (isprint+isspace) inputs.") +FUZZER_FLAG_STRING(dict, "Experimental. Use the dictionary file.") +FUZZER_FLAG_STRING(artifact_prefix, "Write fuzzing artifacts (crash, " + "timeout, or slow inputs) as " + "$(artifact_prefix)file") +FUZZER_FLAG_STRING(exact_artifact_path, + "Write the single artifact on failure (crash, timeout) " + "as $(exact_artifact_path). This overrides -artifact_prefix " + "and will not use checksum in the file name. Do not " + "use the same path for several parallel processes.") +FUZZER_FLAG_INT(print_pcs, 0, "If 1, print out newly covered PCs.") +FUZZER_FLAG_INT(print_funcs, 2, "If >=1, print out at most this number of " + "newly covered functions.") +FUZZER_FLAG_INT(print_final_stats, 0, "If 1, print statistics at exit.") +FUZZER_FLAG_INT(print_corpus_stats, 0, + "If 1, print statistics on corpus elements at exit.") +FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" + " at exit.") +FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated.") +FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") +FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") +FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.") +FUZZER_FLAG_INT(handle_ill, 1, "If 1, try to intercept SIGILL.") +FUZZER_FLAG_INT(handle_fpe, 1, "If 1, try to intercept SIGFPE.") +FUZZER_FLAG_INT(handle_int, 1, "If 1, try to intercept SIGINT.") +FUZZER_FLAG_INT(handle_term, 1, "If 1, try to intercept SIGTERM.") +FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.") +FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.") +FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.") +FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; " + "if 2, close stderr; if 3, close both. " + "Be careful, this will also close e.g. stderr of asan.") +FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled " + "try to detect memory leaks during fuzzing (i.e. not only at shut down).") +FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and " + "quarantines every <N> seconds. When rss_limit_mb is specified (>0), " + "purging starts when RSS exceeds 50% of rss_limit_mb. Pass " + "purge_allocator_interval=-1 to disable this functionality.") +FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. " + "If >= 2 will also print stack traces.") +FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon" + "reaching this limit of RSS memory usage.") +FUZZER_FLAG_INT(malloc_limit_mb, 0, "If non-zero, the fuzzer will exit " + "if the target tries to allocate this number of Mb with one malloc call. " + "If zero (default) same limit as rss_limit_mb is applied.") +FUZZER_FLAG_STRING(exit_on_src_pos, "Exit if a newly found PC originates" + " from the given source location. Example: -exit_on_src_pos=foo.cc:123. " + "Used primarily for testing libFuzzer itself.") +FUZZER_FLAG_STRING(exit_on_item, "Exit if an item with a given sha1 sum" + " was added to the corpus. " + "Used primarily for testing libFuzzer itself.") +FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " + "after this one. Useful for fuzzers that need to do their own " + "argument parsing.") +FUZZER_FLAG_STRING(focus_function, "Experimental. " + "Fuzzing will focus on inputs that trigger calls to this function. " + "If -focus_function=auto and -data_flow_trace is used, libFuzzer " + "will choose the focus functions automatically.") +FUZZER_FLAG_INT(entropic, 0, "Experimental. Enables entropic power schedule.") +FUZZER_FLAG_INT(entropic_feature_frequency_threshold, 0xFF, "Experimental. If " + "entropic is enabled, all features which are observed less often than " + "the specified value are considered as rare.") +FUZZER_FLAG_INT(entropic_number_of_rarest_features, 100, "Experimental. If " + "entropic is enabled, we keep track of the frequencies only for the " + "Top-X least abundant features (union features that are considered as " + "rare).") + +FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") +FUZZER_DEPRECATED_FLAG(use_clang_coverage) +FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") +FUZZER_FLAG_STRING(collect_data_flow, + "Experimental: collect the data flow trace") diff --git a/tools/fuzzing/libfuzzer/FuzzerFork.cpp b/tools/fuzzing/libfuzzer/FuzzerFork.cpp new file mode 100644 index 0000000000..ee2a99a250 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerFork.cpp @@ -0,0 +1,427 @@ +//===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Spawn and orchestrate separate fuzzing processes. +//===----------------------------------------------------------------------===// + +#include "FuzzerCommand.h" +#include "FuzzerFork.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerMerge.h" +#include "FuzzerSHA1.h" +#include "FuzzerTracePC.h" +#include "FuzzerUtil.h" + +#include <atomic> +#include <chrono> +#include <condition_variable> +#include <fstream> +#include <memory> +#include <mutex> +#include <queue> +#include <sstream> +#include <thread> + +namespace fuzzer { + +struct Stats { + size_t number_of_executed_units = 0; + size_t peak_rss_mb = 0; + size_t average_exec_per_sec = 0; +}; + +static Stats ParseFinalStatsFromLog(const std::string &LogPath) { + std::ifstream In(LogPath); + std::string Line; + Stats Res; + struct { + const char *Name; + size_t *Var; + } NameVarPairs[] = { + {"stat::number_of_executed_units:", &Res.number_of_executed_units}, + {"stat::peak_rss_mb:", &Res.peak_rss_mb}, + {"stat::average_exec_per_sec:", &Res.average_exec_per_sec}, + {nullptr, nullptr}, + }; + while (std::getline(In, Line, '\n')) { + if (Line.find("stat::") != 0) continue; + std::istringstream ISS(Line); + std::string Name; + size_t Val; + ISS >> Name >> Val; + for (size_t i = 0; NameVarPairs[i].Name; i++) + if (Name == NameVarPairs[i].Name) + *NameVarPairs[i].Var = Val; + } + return Res; +} + +struct FuzzJob { + // Inputs. + Command Cmd; + std::string CorpusDir; + std::string FeaturesDir; + std::string LogPath; + std::string SeedListPath; + std::string CFPath; + size_t JobId; + + int DftTimeInSeconds = 0; + + // Fuzzing Outputs. + int ExitCode; + + ~FuzzJob() { + RemoveFile(CFPath); + RemoveFile(LogPath); + RemoveFile(SeedListPath); + RmDirRecursive(CorpusDir); + RmDirRecursive(FeaturesDir); + } +}; + +struct GlobalEnv { + Vector<std::string> Args; + Vector<std::string> CorpusDirs; + std::string MainCorpusDir; + std::string TempDir; + std::string DFTDir; + std::string DataFlowBinary; + Set<uint32_t> Features, Cov; + Set<std::string> FilesWithDFT; + Vector<std::string> Files; + Random *Rand; + std::chrono::system_clock::time_point ProcessStartTime; + int Verbosity = 0; + + size_t NumTimeouts = 0; + size_t NumOOMs = 0; + size_t NumCrashes = 0; + + + size_t NumRuns = 0; + + std::string StopFile() { return DirPlusFile(TempDir, "STOP"); } + + size_t secondsSinceProcessStartUp() const { + return std::chrono::duration_cast<std::chrono::seconds>( + std::chrono::system_clock::now() - ProcessStartTime) + .count(); + } + + FuzzJob *CreateNewJob(size_t JobId) { + Command Cmd(Args); + Cmd.removeFlag("fork"); + Cmd.removeFlag("runs"); + Cmd.removeFlag("collect_data_flow"); + for (auto &C : CorpusDirs) // Remove all corpora from the args. + Cmd.removeArgument(C); + Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. + Cmd.addFlag("print_final_stats", "1"); + Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing. + Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId))); + Cmd.addFlag("stop_file", StopFile()); + if (!DataFlowBinary.empty()) { + Cmd.addFlag("data_flow_trace", DFTDir); + if (!Cmd.hasFlag("focus_function")) + Cmd.addFlag("focus_function", "auto"); + } + auto Job = new FuzzJob; + std::string Seeds; + if (size_t CorpusSubsetSize = + std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { + auto Time1 = std::chrono::system_clock::now(); + for (size_t i = 0; i < CorpusSubsetSize; i++) { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + auto Time2 = std::chrono::system_clock::now(); + Job->DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count(); + } + if (!Seeds.empty()) { + Job->SeedListPath = + DirPlusFile(TempDir, std::to_string(JobId) + ".seeds"); + WriteToFile(Seeds, Job->SeedListPath); + Cmd.addFlag("seed_inputs", "@" + Job->SeedListPath); + } + Job->LogPath = DirPlusFile(TempDir, std::to_string(JobId) + ".log"); + Job->CorpusDir = DirPlusFile(TempDir, "C" + std::to_string(JobId)); + Job->FeaturesDir = DirPlusFile(TempDir, "F" + std::to_string(JobId)); + Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge"); + Job->JobId = JobId; + + + Cmd.addArgument(Job->CorpusDir); + Cmd.addFlag("features_dir", Job->FeaturesDir); + + for (auto &D : {Job->CorpusDir, Job->FeaturesDir}) { + RmDirRecursive(D); + MkDir(D); + } + + Cmd.setOutputFile(Job->LogPath); + Cmd.combineOutAndErr(); + + Job->Cmd = Cmd; + + if (Verbosity >= 2) + Printf("Job %zd/%p Created: %s\n", JobId, Job, + Job->Cmd.toString().c_str()); + // Start from very short runs and gradually increase them. + return Job; + } + + int RunOneMergeJob(FuzzJob *Job) { + auto Stats = ParseFinalStatsFromLog(Job->LogPath); + NumRuns += Stats.number_of_executed_units; + + Vector<SizedFile> TempFiles, MergeCandidates; + // Read all newly created inputs and their feature sets. + // Choose only those inputs that have new features. + int Res = GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); + if (Res != 0) + return Res; + std::sort(TempFiles.begin(), TempFiles.end()); + for (auto &F : TempFiles) { + auto FeatureFile = F.File; + FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir); + auto FeatureBytes = FileToVector(FeatureFile, 0, false); + assert((FeatureBytes.size() % sizeof(uint32_t)) == 0); + Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); + memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size()); + for (auto Ft : NewFeatures) { + if (!Features.count(Ft)) { + MergeCandidates.push_back(F); + break; + } + } + } + // if (!FilesToAdd.empty() || Job->ExitCode != 0) + Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd " + "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n", + NumRuns, Cov.size(), Features.size(), Files.size(), + Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, + secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds); + + if (MergeCandidates.empty()) return 0; + + Vector<std::string> FilesToAdd; + Set<uint32_t> NewFeatures, NewCov; + CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, + &NewFeatures, Cov, &NewCov, Job->CFPath, false); + if (Fuzzer::isGracefulExitRequested()) + return 0; + for (auto &Path : FilesToAdd) { + auto U = FileToVector(Path); + auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); + WriteToFile(U, NewPath); + Files.push_back(NewPath); + } + Features.insert(NewFeatures.begin(), NewFeatures.end()); + Cov.insert(NewCov.begin(), NewCov.end()); + for (auto Idx : NewCov) + if (auto *TE = TPC.PCTableEntryByIdx(Idx)) + if (TPC.PcIsFuncEntry(TE)) + PrintPC(" NEW_FUNC: %p %F %L\n", "", + TPC.GetNextInstructionPc(TE->PC)); + return 0; + } + + + void CollectDFT(const std::string &InputPath) { + if (DataFlowBinary.empty()) return; + if (!FilesWithDFT.insert(InputPath).second) return; + Command Cmd(Args); + Cmd.removeFlag("fork"); + Cmd.removeFlag("runs"); + Cmd.addFlag("data_flow_trace", DFTDir); + Cmd.addArgument(InputPath); + for (auto &C : CorpusDirs) // Remove all corpora from the args. + Cmd.removeArgument(C); + Cmd.setOutputFile(DirPlusFile(TempDir, "dft.log")); + Cmd.combineOutAndErr(); + // Printf("CollectDFT: %s\n", Cmd.toString().c_str()); + ExecuteCommand(Cmd); + } + +}; + +struct JobQueue { + std::queue<FuzzJob *> Qu; + std::mutex Mu; + std::condition_variable Cv; + + void Push(FuzzJob *Job) { + { + std::lock_guard<std::mutex> Lock(Mu); + Qu.push(Job); + } + Cv.notify_one(); + } + FuzzJob *Pop() { + std::unique_lock<std::mutex> Lk(Mu); + // std::lock_guard<std::mutex> Lock(Mu); + Cv.wait(Lk, [&]{return !Qu.empty();}); + assert(!Qu.empty()); + auto Job = Qu.front(); + Qu.pop(); + return Job; + } +}; + +void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { + while (auto Job = FuzzQ->Pop()) { + // Printf("WorkerThread: job %p\n", Job); + Job->ExitCode = ExecuteCommand(Job->Cmd); + MergeQ->Push(Job); + } +} + +// This is just a skeleton of an experimental -fork=1 feature. +int FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector<std::string> &Args, + const Vector<std::string> &CorpusDirs, int NumJobs) { + Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); + + GlobalEnv Env; + Env.Args = Args; + Env.CorpusDirs = CorpusDirs; + Env.Rand = &Rand; + Env.Verbosity = Options.Verbosity; + Env.ProcessStartTime = std::chrono::system_clock::now(); + Env.DataFlowBinary = Options.CollectDataFlow; + + Vector<SizedFile> SeedFiles; + int Res; + for (auto &Dir : CorpusDirs) { + Res = GetSizedFilesFromDir(Dir, &SeedFiles); + if (Res != 0) + return Res; + } + std::sort(SeedFiles.begin(), SeedFiles.end()); + Env.TempDir = TempPath("FuzzWithFork", ".dir"); + Env.DFTDir = DirPlusFile(Env.TempDir, "DFT"); + RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs. + MkDir(Env.TempDir); + MkDir(Env.DFTDir); + + + if (CorpusDirs.empty()) + MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); + else + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + Res = CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, + {}, &Env.Cov, + CFPath, false); + if (Res != 0) + return Res; + if (Fuzzer::isGracefulExitRequested()) + return 0; + + RemoveFile(CFPath); + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, + Env.Files.size(), Env.TempDir.c_str()); + + int ExitCode = 0; + + JobQueue FuzzQ, MergeQ; + + auto StopJobs = [&]() { + for (int i = 0; i < NumJobs; i++) + FuzzQ.Push(nullptr); + MergeQ.Push(nullptr); + WriteToFile(Unit({1}), Env.StopFile()); + }; + + size_t JobId = 1; + Vector<std::thread> Threads; + for (int t = 0; t < NumJobs; t++) { + Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + + while (true) { + std::unique_ptr<FuzzJob> Job(MergeQ.Pop()); + if (!Job) + break; + ExitCode = Job->ExitCode; + if (ExitCode == Options.InterruptExitCode) { + Printf("==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid()); + StopJobs(); + break; + } + if (Fuzzer::MaybeExitGracefully()) + return 0; + + Res = Env.RunOneMergeJob(Job.get()); + if (Res != 0) + return Res; + if (Fuzzer::isGracefulExitRequested()) + return 0; + + // Continue if our crash is one of the ignorred ones. + if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) + Env.NumTimeouts++; + else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode) + Env.NumOOMs++; + else if (ExitCode != 0) { + Env.NumCrashes++; + if (Options.IgnoreCrashes) { + std::ifstream In(Job->LogPath); + std::string Line; + while (std::getline(In, Line, '\n')) + if (Line.find("ERROR:") != Line.npos || + Line.find("runtime error:") != Line.npos) + Printf("%s\n", Line.c_str()); + } else { + // And exit if we don't ignore this crash. + Printf("INFO: log from the inner process:\n%s", + FileToString(Job->LogPath).c_str()); + StopJobs(); + break; + } + } + + // Stop if we are over the time budget. + // This is not precise, since other threads are still running + // and we will wait while joining them. + // We also don't stop instantly: other jobs need to finish. + if (Options.MaxTotalTimeSec > 0 && + Env.secondsSinceProcessStartUp() >= (size_t)Options.MaxTotalTimeSec) { + Printf("INFO: fuzzed for %zd seconds, wrapping up soon\n", + Env.secondsSinceProcessStartUp()); + StopJobs(); + break; + } + if (Env.NumRuns >= Options.MaxNumberOfRuns) { + Printf("INFO: fuzzed for %zd iterations, wrapping up soon\n", + Env.NumRuns); + StopJobs(); + break; + } + + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + + for (auto &T : Threads) + T.join(); + + // The workers have terminated. Don't try to remove the directory before they + // terminate to avoid a race condition preventing cleanup on Windows. + RmDirRecursive(Env.TempDir); + + // Use the exit code from the last child process. + Printf("INFO: exiting: %d time: %zds\n", ExitCode, + Env.secondsSinceProcessStartUp()); + return ExitCode; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerFork.h b/tools/fuzzing/libfuzzer/FuzzerFork.h new file mode 100644 index 0000000000..1352171ad4 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerFork.h @@ -0,0 +1,24 @@ +//===- FuzzerFork.h - run fuzzing in sub-processes --------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_FORK_H +#define LLVM_FUZZER_FORK_H + +#include "FuzzerDefs.h" +#include "FuzzerOptions.h" +#include "FuzzerRandom.h" + +#include <string> + +namespace fuzzer { +int FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector<std::string> &Args, + const Vector<std::string> &CorpusDirs, int NumJobs); +} // namespace fuzzer + +#endif // LLVM_FUZZER_FORK_H diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.cpp b/tools/fuzzing/libfuzzer/FuzzerIO.cpp new file mode 100644 index 0000000000..6be2be67c6 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerIO.cpp @@ -0,0 +1,165 @@ +//===- FuzzerIO.cpp - IO utils. -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// IO functions. +//===----------------------------------------------------------------------===// + +#include "mozilla/Unused.h" +#include "FuzzerDefs.h" +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "FuzzerUtil.h" +#include <algorithm> +#include <cstdarg> +#include <fstream> +#include <iterator> +#include <sys/stat.h> +#include <sys/types.h> + +namespace fuzzer { + +static FILE *OutputFile = stderr; + +long GetEpoch(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return 0; // Can't stat, be conservative. + return St.st_mtime; +} + +Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { + std::ifstream T(Path, std::ios::binary); + if (ExitOnError && !T) { + Printf("No such directory: %s; exiting\n", Path.c_str()); + exit(1); + } + + T.seekg(0, T.end); + auto EndPos = T.tellg(); + if (EndPos < 0) return {}; + size_t FileLen = EndPos; + if (MaxSize) + FileLen = std::min(FileLen, MaxSize); + + T.seekg(0, T.beg); + Unit Res(FileLen); + T.read(reinterpret_cast<char *>(Res.data()), FileLen); + return Res; +} + +std::string FileToString(const std::string &Path) { + std::ifstream T(Path, std::ios::binary); + return std::string((std::istreambuf_iterator<char>(T)), + std::istreambuf_iterator<char>()); +} + +void CopyFileToErr(const std::string &Path) { + Printf("%s", FileToString(Path).c_str()); +} + +void WriteToFile(const Unit &U, const std::string &Path) { + WriteToFile(U.data(), U.size(), Path); +} + +void WriteToFile(const std::string &Data, const std::string &Path) { + WriteToFile(reinterpret_cast<const uint8_t *>(Data.c_str()), Data.size(), + Path); +} + +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path) { + // Use raw C interface because this function may be called from a sig handler. + FILE *Out = fopen(Path.c_str(), "wb"); + if (!Out) return; + mozilla::Unused << fwrite(Data, sizeof(Data[0]), Size, Out); + fclose(Out); +} + +void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, + long *Epoch, size_t MaxSize, bool ExitOnError) { + long E = Epoch ? *Epoch : 0; + Vector<std::string> Files; + int Res = ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); + if (ExitOnError && Res != 0) + exit(Res); + size_t NumLoaded = 0; + for (size_t i = 0; i < Files.size(); i++) { + auto &X = Files[i]; + if (Epoch && GetEpoch(X) < E) continue; + NumLoaded++; + if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024) + Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path); + auto S = FileToVector(X, MaxSize, ExitOnError); + if (!S.empty()) + V->push_back(S); + } +} + + +int GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) { + Vector<std::string> Files; + int Res = ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); + if (Res != 0) + return Res; + for (auto &File : Files) + if (size_t Size = FileSize(File)) + V->push_back({File, Size}); + return 0; +} + +std::string DirPlusFile(const std::string &DirPath, + const std::string &FileName) { + return DirPath + GetSeparator() + FileName; +} + +void DupAndCloseStderr() { + int OutputFd = DuplicateFile(2); + if (OutputFd >= 0) { + FILE *NewOutputFile = OpenFile(OutputFd, "w"); + if (NewOutputFile) { + OutputFile = NewOutputFile; + if (EF->__sanitizer_set_report_fd) + EF->__sanitizer_set_report_fd( + reinterpret_cast<void *>(GetHandleFromFd(OutputFd))); + DiscardOutput(2); + } + } +} + +void CloseStdout() { + DiscardOutput(1); +} + +void Printf(const char *Fmt, ...) { + va_list ap; + va_start(ap, Fmt); + vfprintf(OutputFile, Fmt, ap); + va_end(ap); + fflush(OutputFile); +} + +void VPrintf(bool Verbose, const char *Fmt, ...) { + if (!Verbose) return; + va_list ap; + va_start(ap, Fmt); + vfprintf(OutputFile, Fmt, ap); + va_end(ap); + fflush(OutputFile); +} + +void RmDirRecursive(const std::string &Dir) { + IterateDirRecursive( + Dir, [](const std::string &Path) {}, + [](const std::string &Path) { RmDir(Path); }, + [](const std::string &Path) { RemoveFile(Path); }); +} + +std::string TempPath(const char *Prefix, const char *Extension) { + return DirPlusFile(TmpDir(), std::string("libFuzzerTemp.") + Prefix + + std::to_string(GetPid()) + Extension); +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.h b/tools/fuzzing/libfuzzer/FuzzerIO.h new file mode 100644 index 0000000000..6c90ba6373 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerIO.h @@ -0,0 +1,106 @@ +//===- FuzzerIO.h - Internal header for IO utils ----------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// IO interface. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_IO_H +#define LLVM_FUZZER_IO_H + +#include "FuzzerDefs.h" + +namespace fuzzer { + +long GetEpoch(const std::string &Path); + +Unit FileToVector(const std::string &Path, size_t MaxSize = 0, + bool ExitOnError = true); + +std::string FileToString(const std::string &Path); + +void CopyFileToErr(const std::string &Path); + +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path); +// Write Data.c_str() to the file without terminating null character. +void WriteToFile(const std::string &Data, const std::string &Path); +void WriteToFile(const Unit &U, const std::string &Path); + +void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, + long *Epoch, size_t MaxSize, bool ExitOnError); + +// Returns "Dir/FileName" or equivalent for the current OS. +std::string DirPlusFile(const std::string &DirPath, + const std::string &FileName); + +// Returns the name of the dir, similar to the 'dirname' utility. +std::string DirName(const std::string &FileName); + +// Returns path to a TmpDir. +std::string TmpDir(); + +std::string TempPath(const char *Prefix, const char *Extension); + +bool IsInterestingCoverageFile(const std::string &FileName); + +void DupAndCloseStderr(); + +void CloseStdout(); + +void Printf(const char *Fmt, ...); +void VPrintf(bool Verbose, const char *Fmt, ...); + +// Print using raw syscalls, useful when printing at early init stages. +void RawPrint(const char *Str); + +// Platform specific functions: +bool IsFile(const std::string &Path); +size_t FileSize(const std::string &Path); + +int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector<std::string> *V, bool TopDir); + +void RmDirRecursive(const std::string &Dir); + +// Iterate files and dirs inside Dir, recursively. +// Call DirPreCallback/DirPostCallback on dirs before/after +// calling FileCallback on files. +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)); + +struct SizedFile { + std::string File; + size_t Size; + bool operator<(const SizedFile &B) const { return Size < B.Size; } +}; + +int GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V); + +char GetSeparator(); +// Similar to the basename utility: returns the file name w/o the dir prefix. +std::string Basename(const std::string &Path); + +FILE* OpenFile(int Fd, const char *Mode); + +int CloseFile(int Fd); + +int DuplicateFile(int Fd); + +void RemoveFile(const std::string &Path); +void RenameFile(const std::string &OldPath, const std::string &NewPath); + +intptr_t GetHandleFromFd(int fd); + +void MkDir(const std::string &Path); +void RmDir(const std::string &Path); + +const std::string &getDevNull(); + +} // namespace fuzzer + +#endif // LLVM_FUZZER_IO_H diff --git a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp new file mode 100644 index 0000000000..1a50295c01 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp @@ -0,0 +1,181 @@ +//===- FuzzerIOPosix.cpp - IO utils for Posix. ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// IO functions implementation using Posix API. +//===----------------------------------------------------------------------===// +#include "mozilla/Unused.h" +#include "FuzzerPlatform.h" +#if LIBFUZZER_POSIX || LIBFUZZER_FUCHSIA + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include <cstdarg> +#include <cstdio> +#include <dirent.h> +#include <fstream> +#include <iterator> +#include <libgen.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +namespace fuzzer { + +bool IsFile(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return false; + return S_ISREG(St.st_mode); +} + +static bool IsDirectory(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return false; + return S_ISDIR(St.st_mode); +} + +size_t FileSize(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return 0; + return St.st_size; +} + +std::string Basename(const std::string &Path) { + size_t Pos = Path.rfind(GetSeparator()); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + +int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector<std::string> *V, bool TopDir) { + auto E = GetEpoch(Dir); + if (Epoch) + if (E && *Epoch >= E) return 0; + + DIR *D = opendir(Dir.c_str()); + if (!D) { + Printf("%s: %s; exiting\n", strerror(errno), Dir.c_str()); + return 1; + } + while (auto E = readdir(D)) { + std::string Path = DirPlusFile(Dir, E->d_name); + if (E->d_type == DT_REG || E->d_type == DT_LNK || + (E->d_type == DT_UNKNOWN && IsFile(Path))) + V->push_back(Path); + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && + *E->d_name != '.') { + int Res = ListFilesInDirRecursive(Path, Epoch, V, false); + if (Res != 0) + return Res; + } + } + closedir(D); + if (Epoch && TopDir) + *Epoch = E; + return 0; +} + + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + DirPreCallback(Dir); + DIR *D = opendir(Dir.c_str()); + if (!D) return; + while (auto E = readdir(D)) { + std::string Path = DirPlusFile(Dir, E->d_name); + if (E->d_type == DT_REG || E->d_type == DT_LNK || + (E->d_type == DT_UNKNOWN && IsFile(Path))) + FileCallback(Path); + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && + *E->d_name != '.') + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } + closedir(D); + DirPostCallback(Dir); +} + +char GetSeparator() { + return '/'; +} + +FILE* OpenFile(int Fd, const char* Mode) { + return fdopen(Fd, Mode); +} + +int CloseFile(int fd) { + return close(fd); +} + +int DuplicateFile(int Fd) { + return dup(Fd); +} + +void RemoveFile(const std::string &Path) { + unlink(Path.c_str()); +} + +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + +intptr_t GetHandleFromFd(int fd) { + return static_cast<intptr_t>(fd); +} + +std::string DirName(const std::string &FileName) { + char *Tmp = new char[FileName.size() + 1]; + memcpy(Tmp, FileName.c_str(), FileName.size() + 1); + std::string Res = dirname(Tmp); + delete [] Tmp; + return Res; +} + +std::string TmpDir() { + if (auto Env = getenv("TMPDIR")) + return Env; + return "/tmp"; +} + +bool IsInterestingCoverageFile(const std::string &FileName) { + if (FileName.find("compiler-rt/lib/") != std::string::npos) + return false; // sanitizer internal. + if (FileName.find("/usr/lib/") != std::string::npos) + return false; + if (FileName.find("/usr/include/") != std::string::npos) + return false; + if (FileName == "<null>") + return false; + return true; +} + +void RawPrint(const char *Str) { + mozilla::Unused << write(2, Str, strlen(Str)); +} + +void MkDir(const std::string &Path) { + mkdir(Path.c_str(), 0700); +} + +void RmDir(const std::string &Path) { + rmdir(Path.c_str()); +} + +const std::string &getDevNull() { + static const std::string devNull = "/dev/null"; + return devNull; +} + +} // namespace fuzzer + +#endif // LIBFUZZER_POSIX diff --git a/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp b/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp new file mode 100644 index 0000000000..0e977bd025 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp @@ -0,0 +1,417 @@ +//===- FuzzerIOWindows.cpp - IO utils for Windows. ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// IO functions implementation for Windows. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include <cstdarg> +#include <cstdio> +#include <fstream> +#include <io.h> +#include <iterator> +#include <sys/stat.h> +#include <sys/types.h> +#include <windows.h> + +namespace fuzzer { + +static bool IsFile(const std::string &Path, const DWORD &FileAttributes) { + + if (FileAttributes & FILE_ATTRIBUTE_NORMAL) + return true; + + if (FileAttributes & FILE_ATTRIBUTE_DIRECTORY) + return false; + + HANDLE FileHandle( + CreateFileA(Path.c_str(), 0, FILE_SHARE_READ, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, 0)); + + if (FileHandle == INVALID_HANDLE_VALUE) { + Printf("CreateFileA() failed for \"%s\" (Error code: %lu).\n", Path.c_str(), + GetLastError()); + return false; + } + + DWORD FileType = GetFileType(FileHandle); + + if (FileType == FILE_TYPE_UNKNOWN) { + Printf("GetFileType() failed for \"%s\" (Error code: %lu).\n", Path.c_str(), + GetLastError()); + CloseHandle(FileHandle); + return false; + } + + if (FileType != FILE_TYPE_DISK) { + CloseHandle(FileHandle); + return false; + } + + CloseHandle(FileHandle); + return true; +} + +bool IsFile(const std::string &Path) { + DWORD Att = GetFileAttributesA(Path.c_str()); + + if (Att == INVALID_FILE_ATTRIBUTES) { + Printf("GetFileAttributesA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), GetLastError()); + return false; + } + + return IsFile(Path, Att); +} + +static bool IsDir(DWORD FileAttrs) { + if (FileAttrs == INVALID_FILE_ATTRIBUTES) return false; + return FileAttrs & FILE_ATTRIBUTE_DIRECTORY; +} + +std::string Basename(const std::string &Path) { + size_t Pos = Path.find_last_of("/\\"); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + +size_t FileSize(const std::string &Path) { + WIN32_FILE_ATTRIBUTE_DATA attr; + if (!GetFileAttributesExA(Path.c_str(), GetFileExInfoStandard, &attr)) { + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) + Printf("GetFileAttributesExA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), LastError); + return 0; + } + ULARGE_INTEGER size; + size.HighPart = attr.nFileSizeHigh; + size.LowPart = attr.nFileSizeLow; + return size.QuadPart; +} + +int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector<std::string> *V, bool TopDir) { + int Res; + auto E = GetEpoch(Dir); + if (Epoch) + if (E && *Epoch >= E) return 0; + + std::string Path(Dir); + assert(!Path.empty()); + if (Path.back() != '\\') + Path.push_back('\\'); + Path.push_back('*'); + + // Get the first directory entry. + WIN32_FIND_DATAA FindInfo; + HANDLE FindHandle(FindFirstFileA(Path.c_str(), &FindInfo)); + if (FindHandle == INVALID_HANDLE_VALUE) + { + if (GetLastError() == ERROR_FILE_NOT_FOUND) + return 0; + Printf("No such file or directory: %s; exiting\n", Dir.c_str()); + return 1; + } + + do { + std::string FileName = DirPlusFile(Dir, FindInfo.cFileName); + + if (FindInfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + size_t FilenameLen = strlen(FindInfo.cFileName); + if ((FilenameLen == 1 && FindInfo.cFileName[0] == '.') || + (FilenameLen == 2 && FindInfo.cFileName[0] == '.' && + FindInfo.cFileName[1] == '.')) + continue; + + int Res = ListFilesInDirRecursive(FileName, Epoch, V, false); + if (Res != 0) + return Res; + } + else if (IsFile(FileName, FindInfo.dwFileAttributes)) + V->push_back(FileName); + } while (FindNextFileA(FindHandle, &FindInfo)); + + DWORD LastError = GetLastError(); + if (LastError != ERROR_NO_MORE_FILES) + Printf("FindNextFileA failed (Error code: %lu).\n", LastError); + + FindClose(FindHandle); + + if (Epoch && TopDir) + *Epoch = E; + return 0; +} + + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + // TODO(metzman): Implement ListFilesInDirRecursive via this function. + DirPreCallback(Dir); + + DWORD DirAttrs = GetFileAttributesA(Dir.c_str()); + if (!IsDir(DirAttrs)) return; + + std::string TargetDir(Dir); + assert(!TargetDir.empty()); + if (TargetDir.back() != '\\') TargetDir.push_back('\\'); + TargetDir.push_back('*'); + + WIN32_FIND_DATAA FindInfo; + // Find the directory's first file. + HANDLE FindHandle = FindFirstFileA(TargetDir.c_str(), &FindInfo); + if (FindHandle == INVALID_HANDLE_VALUE) { + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) { + // If the directory isn't empty, then something abnormal is going on. + Printf("FindFirstFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + } + return; + } + + do { + std::string Path = DirPlusFile(Dir, FindInfo.cFileName); + DWORD PathAttrs = FindInfo.dwFileAttributes; + if (IsDir(PathAttrs)) { + // Is Path the current directory (".") or the parent ("..")? + if (strcmp(FindInfo.cFileName, ".") == 0 || + strcmp(FindInfo.cFileName, "..") == 0) + continue; + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } else if (PathAttrs != INVALID_FILE_ATTRIBUTES) { + FileCallback(Path); + } + } while (FindNextFileA(FindHandle, &FindInfo)); + + DWORD LastError = GetLastError(); + if (LastError != ERROR_NO_MORE_FILES) + Printf("FindNextFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + + FindClose(FindHandle); + DirPostCallback(Dir); +} + +char GetSeparator() { + return '\\'; +} + +FILE* OpenFile(int Fd, const char* Mode) { + return _fdopen(Fd, Mode); +} + +int CloseFile(int Fd) { + return _close(Fd); +} + +int DuplicateFile(int Fd) { + return _dup(Fd); +} + +void RemoveFile(const std::string &Path) { + _unlink(Path.c_str()); +} + +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + +intptr_t GetHandleFromFd(int fd) { + return _get_osfhandle(fd); +} + +static bool IsSeparator(char C) { + return C == '\\' || C == '/'; +} + +// Parse disk designators, like "C:\". If Relative == true, also accepts: "C:". +// Returns number of characters considered if successful. +static size_t ParseDrive(const std::string &FileName, const size_t Offset, + bool Relative = true) { + if (Offset + 1 >= FileName.size() || FileName[Offset + 1] != ':') + return 0; + if (Offset + 2 >= FileName.size() || !IsSeparator(FileName[Offset + 2])) { + if (!Relative) // Accept relative path? + return 0; + else + return 2; + } + return 3; +} + +// Parse a file name, like: SomeFile.txt +// Returns number of characters considered if successful. +static size_t ParseFileName(const std::string &FileName, const size_t Offset) { + size_t Pos = Offset; + const size_t End = FileName.size(); + for(; Pos < End && !IsSeparator(FileName[Pos]); ++Pos) + ; + return Pos - Offset; +} + +// Parse a directory ending in separator, like: `SomeDir\` +// Returns number of characters considered if successful. +static size_t ParseDir(const std::string &FileName, const size_t Offset) { + size_t Pos = Offset; + const size_t End = FileName.size(); + if (Pos >= End || IsSeparator(FileName[Pos])) + return 0; + for(; Pos < End && !IsSeparator(FileName[Pos]); ++Pos) + ; + if (Pos >= End) + return 0; + ++Pos; // Include separator. + return Pos - Offset; +} + +// Parse a servername and share, like: `SomeServer\SomeShare\` +// Returns number of characters considered if successful. +static size_t ParseServerAndShare(const std::string &FileName, + const size_t Offset) { + size_t Pos = Offset, Res; + if (!(Res = ParseDir(FileName, Pos))) + return 0; + Pos += Res; + if (!(Res = ParseDir(FileName, Pos))) + return 0; + Pos += Res; + return Pos - Offset; +} + +// Parse the given Ref string from the position Offset, to exactly match the given +// string Patt. +// Returns number of characters considered if successful. +static size_t ParseCustomString(const std::string &Ref, size_t Offset, + const char *Patt) { + size_t Len = strlen(Patt); + if (Offset + Len > Ref.size()) + return 0; + return Ref.compare(Offset, Len, Patt) == 0 ? Len : 0; +} + +// Parse a location, like: +// \\?\UNC\Server\Share\ \\?\C:\ \\Server\Share\ \ C:\ C: +// Returns number of characters considered if successful. +static size_t ParseLocation(const std::string &FileName) { + size_t Pos = 0, Res; + + if ((Res = ParseCustomString(FileName, Pos, R"(\\?\)"))) { + Pos += Res; + if ((Res = ParseCustomString(FileName, Pos, R"(UNC\)"))) { + Pos += Res; + if ((Res = ParseServerAndShare(FileName, Pos))) + return Pos + Res; + return 0; + } + if ((Res = ParseDrive(FileName, Pos, false))) + return Pos + Res; + return 0; + } + + if (Pos < FileName.size() && IsSeparator(FileName[Pos])) { + ++Pos; + if (Pos < FileName.size() && IsSeparator(FileName[Pos])) { + ++Pos; + if ((Res = ParseServerAndShare(FileName, Pos))) + return Pos + Res; + return 0; + } + return Pos; + } + + if ((Res = ParseDrive(FileName, Pos))) + return Pos + Res; + + return Pos; +} + +std::string DirName(const std::string &FileName) { + size_t LocationLen = ParseLocation(FileName); + size_t DirLen = 0, Res; + while ((Res = ParseDir(FileName, LocationLen + DirLen))) + DirLen += Res; + size_t FileLen = ParseFileName(FileName, LocationLen + DirLen); + + if (LocationLen + DirLen + FileLen != FileName.size()) { + Printf("DirName() failed for \"%s\", invalid path.\n", FileName.c_str()); + exit(1); + } + + if (DirLen) { + --DirLen; // Remove trailing separator. + if (!FileLen) { // Path ended in separator. + assert(DirLen); + // Remove file name from Dir. + while (DirLen && !IsSeparator(FileName[LocationLen + DirLen - 1])) + --DirLen; + if (DirLen) // Remove trailing separator. + --DirLen; + } + } + + if (!LocationLen) { // Relative path. + if (!DirLen) + return "."; + return std::string(".\\").append(FileName, 0, DirLen); + } + + return FileName.substr(0, LocationLen + DirLen); +} + +std::string TmpDir() { + std::string Tmp; + Tmp.resize(MAX_PATH + 1); + DWORD Size = GetTempPathA(Tmp.size(), &Tmp[0]); + if (Size == 0) { + Printf("Couldn't get Tmp path.\n"); + exit(1); + } + Tmp.resize(Size); + return Tmp; +} + +bool IsInterestingCoverageFile(const std::string &FileName) { + if (FileName.find("Program Files") != std::string::npos) + return false; + if (FileName.find("compiler-rt\\lib\\") != std::string::npos) + return false; // sanitizer internal. + if (FileName == "<null>") + return false; + return true; +} + +void RawPrint(const char *Str) { + _write(2, Str, strlen(Str)); +} + +void MkDir(const std::string &Path) { + if (CreateDirectoryA(Path.c_str(), nullptr)) return; + Printf("CreateDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +void RmDir(const std::string &Path) { + if (RemoveDirectoryA(Path.c_str())) return; + Printf("RemoveDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +const std::string &getDevNull() { + static const std::string devNull = "NUL"; + return devNull; +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerInterceptors.cpp b/tools/fuzzing/libfuzzer/FuzzerInterceptors.cpp new file mode 100644 index 0000000000..a1a64780de --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerInterceptors.cpp @@ -0,0 +1,235 @@ +//===-- FuzzerInterceptors.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Intercept certain libc functions to aid fuzzing. +// Linked only when other RTs that define their own interceptors are not linked. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" + +#if LIBFUZZER_LINUX + +#define GET_CALLER_PC() __builtin_return_address(0) + +#define PTR_TO_REAL(x) real_##x +#define REAL(x) __interception::PTR_TO_REAL(x) +#define FUNC_TYPE(x) x##_type +#define DEFINE_REAL(ret_type, func, ...) \ + typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ + namespace __interception { \ + FUNC_TYPE(func) PTR_TO_REAL(func); \ + } + +#include <cassert> +#include <cstdint> +#include <dlfcn.h> // for dlsym() +#include <sanitizer/common_interface_defs.h> + +static void *getFuncAddr(const char *name, uintptr_t wrapper_addr) { + void *addr = dlsym(RTLD_NEXT, name); + if (!addr) { + // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is + // later in the library search order than the DSO that we are trying to + // intercept, which means that we cannot intercept this function. We still + // want the address of the real definition, though, so look it up using + // RTLD_DEFAULT. + addr = dlsym(RTLD_DEFAULT, name); + + // In case `name' is not loaded, dlsym ends up finding the actual wrapper. + // We don't want to intercept the wrapper and have it point to itself. + if (reinterpret_cast<uintptr_t>(addr) == wrapper_addr) + addr = nullptr; + } + return addr; +} + +static int FuzzerInited = 0; +static bool FuzzerInitIsRunning; + +static void fuzzerInit(); + +static void ensureFuzzerInited() { + assert(!FuzzerInitIsRunning); + if (!FuzzerInited) { + fuzzerInit(); + } +} + +static int internal_strcmp_strncmp(const char *s1, const char *s2, bool strncmp, + size_t n) { + size_t i = 0; + while (true) { + if (strncmp) { + if (i == n) + break; + i++; + } + unsigned c1 = *s1; + unsigned c2 = *s2; + if (c1 != c2) + return (c1 < c2) ? -1 : 1; + if (c1 == 0) + break; + s1++; + s2++; + } + return 0; +} + +static int internal_strncmp(const char *s1, const char *s2, size_t n) { + return internal_strcmp_strncmp(s1, s2, true, n); +} + +static int internal_strcmp(const char *s1, const char *s2) { + return internal_strcmp_strncmp(s1, s2, false, 0); +} + +static int internal_memcmp(const void *s1, const void *s2, size_t n) { + const uint8_t *t1 = static_cast<const uint8_t *>(s1); + const uint8_t *t2 = static_cast<const uint8_t *>(s2); + for (size_t i = 0; i < n; ++i, ++t1, ++t2) + if (*t1 != *t2) + return *t1 < *t2 ? -1 : 1; + return 0; +} + +static size_t internal_strlen(const char *s) { + size_t i = 0; + while (s[i]) + i++; + return i; +} + +static char *internal_strstr(const char *haystack, const char *needle) { + // This is O(N^2), but we are not using it in hot places. + size_t len1 = internal_strlen(haystack); + size_t len2 = internal_strlen(needle); + if (len1 < len2) + return nullptr; + for (size_t pos = 0; pos <= len1 - len2; pos++) { + if (internal_memcmp(haystack + pos, needle, len2) == 0) + return const_cast<char *>(haystack) + pos; + } + return nullptr; +} + +extern "C" { + +DEFINE_REAL(int, bcmp, const void *, const void *, size_t) +DEFINE_REAL(int, memcmp, const void *, const void *, size_t) +DEFINE_REAL(int, strncmp, const char *, const char *, size_t) +DEFINE_REAL(int, strcmp, const char *, const char *) +DEFINE_REAL(int, strncasecmp, const char *, const char *, size_t) +DEFINE_REAL(int, strcasecmp, const char *, const char *) +DEFINE_REAL(char *, strstr, const char *, const char *) +DEFINE_REAL(char *, strcasestr, const char *, const char *) +DEFINE_REAL(void *, memmem, const void *, size_t, const void *, size_t) + +ATTRIBUTE_INTERFACE int bcmp(const char *s1, const char *s2, size_t n) { + if (!FuzzerInited) + return internal_memcmp(s1, s2, n); + int result = REAL(bcmp)(s1, s2, n); + __sanitizer_weak_hook_memcmp(GET_CALLER_PC(), s1, s2, n, result); + return result; +} + +ATTRIBUTE_INTERFACE int memcmp(const void *s1, const void *s2, size_t n) { + if (!FuzzerInited) + return internal_memcmp(s1, s2, n); + int result = REAL(memcmp)(s1, s2, n); + __sanitizer_weak_hook_memcmp(GET_CALLER_PC(), s1, s2, n, result); + return result; +} + +ATTRIBUTE_INTERFACE int strncmp(const char *s1, const char *s2, size_t n) { + if (!FuzzerInited) + return internal_strncmp(s1, s2, n); + int result = REAL(strncmp)(s1, s2, n); + __sanitizer_weak_hook_strncmp(GET_CALLER_PC(), s1, s2, n, result); + return result; +} + +ATTRIBUTE_INTERFACE int strcmp(const char *s1, const char *s2) { + if (!FuzzerInited) + return internal_strcmp(s1, s2); + int result = REAL(strcmp)(s1, s2); + __sanitizer_weak_hook_strcmp(GET_CALLER_PC(), s1, s2, result); + return result; +} + +ATTRIBUTE_INTERFACE int strncasecmp(const char *s1, const char *s2, size_t n) { + ensureFuzzerInited(); + int result = REAL(strncasecmp)(s1, s2, n); + __sanitizer_weak_hook_strncasecmp(GET_CALLER_PC(), s1, s2, n, result); + return result; +} + +ATTRIBUTE_INTERFACE int strcasecmp(const char *s1, const char *s2) { + ensureFuzzerInited(); + int result = REAL(strcasecmp)(s1, s2); + __sanitizer_weak_hook_strcasecmp(GET_CALLER_PC(), s1, s2, result); + return result; +} + +ATTRIBUTE_INTERFACE char *strstr(const char *s1, const char *s2) { + if (!FuzzerInited) + return internal_strstr(s1, s2); + char *result = REAL(strstr)(s1, s2); + __sanitizer_weak_hook_strstr(GET_CALLER_PC(), s1, s2, result); + return result; +} + +ATTRIBUTE_INTERFACE char *strcasestr(const char *s1, const char *s2) { + ensureFuzzerInited(); + char *result = REAL(strcasestr)(s1, s2); + __sanitizer_weak_hook_strcasestr(GET_CALLER_PC(), s1, s2, result); + return result; +} + +ATTRIBUTE_INTERFACE +void *memmem(const void *s1, size_t len1, const void *s2, size_t len2) { + ensureFuzzerInited(); + void *result = REAL(memmem)(s1, len1, s2, len2); + __sanitizer_weak_hook_memmem(GET_CALLER_PC(), s1, len1, s2, len2, result); + return result; +} + +__attribute__((section(".preinit_array"), + used)) static void (*__local_fuzzer_preinit)(void) = fuzzerInit; + +} // extern "C" + +static void fuzzerInit() { + assert(!FuzzerInitIsRunning); + if (FuzzerInited) + return; + FuzzerInitIsRunning = true; + + REAL(bcmp) = reinterpret_cast<memcmp_type>( + getFuncAddr("bcmp", reinterpret_cast<uintptr_t>(&bcmp))); + REAL(memcmp) = reinterpret_cast<memcmp_type>( + getFuncAddr("memcmp", reinterpret_cast<uintptr_t>(&memcmp))); + REAL(strncmp) = reinterpret_cast<strncmp_type>( + getFuncAddr("strncmp", reinterpret_cast<uintptr_t>(&strncmp))); + REAL(strcmp) = reinterpret_cast<strcmp_type>( + getFuncAddr("strcmp", reinterpret_cast<uintptr_t>(&strcmp))); + REAL(strncasecmp) = reinterpret_cast<strncasecmp_type>( + getFuncAddr("strncasecmp", reinterpret_cast<uintptr_t>(&strncasecmp))); + REAL(strcasecmp) = reinterpret_cast<strcasecmp_type>( + getFuncAddr("strcasecmp", reinterpret_cast<uintptr_t>(&strcasecmp))); + REAL(strstr) = reinterpret_cast<strstr_type>( + getFuncAddr("strstr", reinterpret_cast<uintptr_t>(&strstr))); + REAL(strcasestr) = reinterpret_cast<strcasestr_type>( + getFuncAddr("strcasestr", reinterpret_cast<uintptr_t>(&strcasestr))); + REAL(memmem) = reinterpret_cast<memmem_type>( + getFuncAddr("memmem", reinterpret_cast<uintptr_t>(&memmem))); + + FuzzerInitIsRunning = false; + FuzzerInited = 1; +} + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerInterface.h b/tools/fuzzing/libfuzzer/FuzzerInterface.h new file mode 100644 index 0000000000..4f62822eac --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerInterface.h @@ -0,0 +1,79 @@ +//===- FuzzerInterface.h - Interface header for the Fuzzer ------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Define the interface between libFuzzer and the library being tested. +//===----------------------------------------------------------------------===// + +// NOTE: the libFuzzer interface is thin and in the majority of cases +// you should not include this file into your target. In 95% of cases +// all you need is to define the following function in your file: +// extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); + +// WARNING: keep the interface in C. + +#ifndef LLVM_FUZZER_INTERFACE_H +#define LLVM_FUZZER_INTERFACE_H + +#include <stddef.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +// Define FUZZER_INTERFACE_VISIBILITY to set default visibility in a way that +// doesn't break MSVC. +#if defined(_WIN32) +#define FUZZER_INTERFACE_VISIBILITY __declspec(dllexport) +#else +#define FUZZER_INTERFACE_VISIBILITY __attribute__((visibility("default"))) +#endif + +// Mandatory user-provided target function. +// Executes the code under test with [Data, Data+Size) as the input. +// libFuzzer will invoke this function *many* times with different inputs. +// Must return 0. +FUZZER_INTERFACE_VISIBILITY int +LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); + +// Optional user-provided initialization function. +// If provided, this function will be called by libFuzzer once at startup. +// It may read and modify argc/argv. +// Must return 0. +FUZZER_INTERFACE_VISIBILITY int LLVMFuzzerInitialize(int *argc, char ***argv); + +// Optional user-provided custom mutator. +// Mutates raw data in [Data, Data+Size) inplace. +// Returns the new size, which is not greater than MaxSize. +// Given the same Seed produces the same mutation. +FUZZER_INTERFACE_VISIBILITY size_t +LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, + unsigned int Seed); + +// Optional user-provided custom cross-over function. +// Combines pieces of Data1 & Data2 together into Out. +// Returns the new size, which is not greater than MaxOutSize. +// Should produce the same mutation given the same Seed. +FUZZER_INTERFACE_VISIBILITY size_t +LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, + const uint8_t *Data2, size_t Size2, uint8_t *Out, + size_t MaxOutSize, unsigned int Seed); + +// Experimental, may go away in future. +// libFuzzer-provided function to be used inside LLVMFuzzerCustomMutator. +// Mutates raw data in [Data, Data+Size) inplace. +// Returns the new size, which is not greater than MaxSize. +FUZZER_INTERFACE_VISIBILITY size_t +LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize); + +#undef FUZZER_INTERFACE_VISIBILITY + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // LLVM_FUZZER_INTERFACE_H diff --git a/tools/fuzzing/libfuzzer/FuzzerInternal.h b/tools/fuzzing/libfuzzer/FuzzerInternal.h new file mode 100644 index 0000000000..cc2650b58e --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h @@ -0,0 +1,175 @@ +//===- FuzzerInternal.h - Internal header for the Fuzzer --------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Define the main class fuzzer::Fuzzer and most functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_INTERNAL_H +#define LLVM_FUZZER_INTERNAL_H + +#include "FuzzerDataFlowTrace.h" +#include "FuzzerDefs.h" +#include "FuzzerExtFunctions.h" +#include "FuzzerInterface.h" +#include "FuzzerOptions.h" +#include "FuzzerSHA1.h" +#include "FuzzerValueBitMap.h" +#include <algorithm> +#include <atomic> +#include <chrono> +#include <climits> +#include <cstdlib> +#include <string.h> + +namespace fuzzer { + +using namespace std::chrono; + +class Fuzzer { +public: + + Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, + FuzzingOptions Options); + ~Fuzzer(); + int Loop(Vector<SizedFile> &CorporaFiles); + int ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles); + void MinimizeCrashLoop(const Unit &U); + void RereadOutputCorpus(size_t MaxSize); + + size_t secondsSinceProcessStartUp() { + return duration_cast<seconds>(system_clock::now() - ProcessStartTime) + .count(); + } + + bool TimedOut() { + return Options.MaxTotalTimeSec > 0 && + secondsSinceProcessStartUp() > + static_cast<size_t>(Options.MaxTotalTimeSec); + } + + size_t execPerSec() { + size_t Seconds = secondsSinceProcessStartUp(); + return Seconds ? TotalNumberOfRuns / Seconds : 0; + } + + size_t getTotalNumberOfRuns() { return TotalNumberOfRuns; } + + static void StaticAlarmCallback(); + static void StaticCrashSignalCallback(); + static void StaticExitCallback(); + static void StaticInterruptCallback(); + static void StaticFileSizeExceedCallback(); + static void StaticGracefulExitCallback(); + + static void GracefullyExit(); + static bool isGracefulExitRequested(); + + int ExecuteCallback(const uint8_t *Data, size_t Size); + bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, + InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); + + // Merge Corpora[1:] into Corpora[0]. + void Merge(const Vector<std::string> &Corpora); + int CrashResistantMergeInternalStep(const std::string &ControlFilePath); + MutationDispatcher &GetMD() { return MD; } + void PrintFinalStats(); + void SetMaxInputLen(size_t MaxInputLen); + void SetMaxMutationLen(size_t MaxMutationLen); + void RssLimitCallback(); + + bool InFuzzingThread() const { return IsMyThread; } + size_t GetCurrentUnitInFuzzingThead(const uint8_t **Data) const; + void TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size, + bool DuringInitialCorpusExecution); + + void HandleMalloc(size_t Size); + static bool MaybeExitGracefully(); + std::string WriteToOutputCorpus(const Unit &U); + +private: + void AlarmCallback(); + void CrashCallback(); + void ExitCallback(); + void CrashOnOverwrittenData(); + void InterruptCallback(); + bool MutateAndTestOne(); + void PurgeAllocator(); + void ReportNewCoverage(InputInfo *II, const Unit &U); + void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size); + void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix); + void PrintStats(const char *Where, const char *End = "\n", size_t Units = 0, + size_t Features = 0); + void PrintStatusForNewUnit(const Unit &U, const char *Text); + void CheckExitOnSrcPosOrItem(); + + static void StaticDeathCallback(); + void DumpCurrentUnit(const char *Prefix); + void DeathCallback(); + + void AllocateCurrentUnitData(); + uint8_t *CurrentUnitData = nullptr; + std::atomic<size_t> CurrentUnitSize; + uint8_t BaseSha1[kSHA1NumBytes]; // Checksum of the base unit. + + bool GracefulExitRequested = false; + + size_t TotalNumberOfRuns = 0; + size_t NumberOfNewUnitsAdded = 0; + + size_t LastCorpusUpdateRun = 0; + + bool HasMoreMallocsThanFrees = false; + size_t NumberOfLeakDetectionAttempts = 0; + + system_clock::time_point LastAllocatorPurgeAttemptTime = system_clock::now(); + + UserCallback CB; + InputCorpus &Corpus; + MutationDispatcher &MD; + FuzzingOptions Options; + DataFlowTrace DFT; + + system_clock::time_point ProcessStartTime = system_clock::now(); + system_clock::time_point UnitStartTime, UnitStopTime; + long TimeOfLongestUnitInSeconds = 0; + long EpochOfLastReadOfOutputCorpus = 0; + + size_t MaxInputLen = 0; + size_t MaxMutationLen = 0; + size_t TmpMaxMutationLen = 0; + + Vector<uint32_t> UniqFeatureSetTmp; + + // Need to know our own thread. + static thread_local bool IsMyThread; +}; + +struct ScopedEnableMsanInterceptorChecks { + ScopedEnableMsanInterceptorChecks() { + if (EF->__msan_scoped_enable_interceptor_checks) + EF->__msan_scoped_enable_interceptor_checks(); + } + ~ScopedEnableMsanInterceptorChecks() { + if (EF->__msan_scoped_disable_interceptor_checks) + EF->__msan_scoped_disable_interceptor_checks(); + } +}; + +struct ScopedDisableMsanInterceptorChecks { + ScopedDisableMsanInterceptorChecks() { + if (EF->__msan_scoped_disable_interceptor_checks) + EF->__msan_scoped_disable_interceptor_checks(); + } + ~ScopedDisableMsanInterceptorChecks() { + if (EF->__msan_scoped_enable_interceptor_checks) + EF->__msan_scoped_enable_interceptor_checks(); + } +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_INTERNAL_H diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp new file mode 100644 index 0000000000..e7dfc187db --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp @@ -0,0 +1,901 @@ +//===- FuzzerLoop.cpp - Fuzzer's main loop --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Fuzzer's main loop. +//===----------------------------------------------------------------------===// + +#include "FuzzerCorpus.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerMutate.h" +#include "FuzzerPlatform.h" +#include "FuzzerRandom.h" +#include "FuzzerTracePC.h" +#include <algorithm> +#include <cstring> +#include <memory> +#include <mutex> +#include <set> + +#if defined(__has_include) +#if __has_include(<sanitizer / lsan_interface.h>) +#include <sanitizer/lsan_interface.h> +#endif +#endif + +#define NO_SANITIZE_MEMORY +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#undef NO_SANITIZE_MEMORY +#define NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +#endif +#endif + +namespace fuzzer { +static const size_t kMaxUnitSizeToPrint = 256; + +thread_local bool Fuzzer::IsMyThread; + +bool RunningUserCallback = false; + +// Only one Fuzzer per process. +static Fuzzer *F; + +// Leak detection is expensive, so we first check if there were more mallocs +// than frees (using the sanitizer malloc hooks) and only then try to call lsan. +struct MallocFreeTracer { + void Start(int TraceLevel) { + this->TraceLevel = TraceLevel; + if (TraceLevel) + Printf("MallocFreeTracer: START\n"); + Mallocs = 0; + Frees = 0; + } + // Returns true if there were more mallocs than frees. + bool Stop() { + if (TraceLevel) + Printf("MallocFreeTracer: STOP %zd %zd (%s)\n", Mallocs.load(), + Frees.load(), Mallocs == Frees ? "same" : "DIFFERENT"); + bool Result = Mallocs > Frees; + Mallocs = 0; + Frees = 0; + TraceLevel = 0; + return Result; + } + std::atomic<size_t> Mallocs; + std::atomic<size_t> Frees; + int TraceLevel = 0; + + std::recursive_mutex TraceMutex; + bool TraceDisabled = false; +}; + +static MallocFreeTracer AllocTracer; + +// Locks printing and avoids nested hooks triggered from mallocs/frees in +// sanitizer. +class TraceLock { +public: + TraceLock() : Lock(AllocTracer.TraceMutex) { + AllocTracer.TraceDisabled = !AllocTracer.TraceDisabled; + } + ~TraceLock() { AllocTracer.TraceDisabled = !AllocTracer.TraceDisabled; } + + bool IsDisabled() const { + // This is already inverted value. + return !AllocTracer.TraceDisabled; + } + +private: + std::lock_guard<std::recursive_mutex> Lock; +}; + +ATTRIBUTE_NO_SANITIZE_MEMORY +void MallocHook(const volatile void *ptr, size_t size) { + size_t N = AllocTracer.Mallocs++; + F->HandleMalloc(size); + if (int TraceLevel = AllocTracer.TraceLevel) { + TraceLock Lock; + if (Lock.IsDisabled()) + return; + Printf("MALLOC[%zd] %p %zd\n", N, ptr, size); + if (TraceLevel >= 2 && EF) + PrintStackTrace(); + } +} + +ATTRIBUTE_NO_SANITIZE_MEMORY +void FreeHook(const volatile void *ptr) { + size_t N = AllocTracer.Frees++; + if (int TraceLevel = AllocTracer.TraceLevel) { + TraceLock Lock; + if (Lock.IsDisabled()) + return; + Printf("FREE[%zd] %p\n", N, ptr); + if (TraceLevel >= 2 && EF) + PrintStackTrace(); + } +} + +// Crash on a single malloc that exceeds the rss limit. +void Fuzzer::HandleMalloc(size_t Size) { + if (!Options.MallocLimitMb || (Size >> 20) < (size_t)Options.MallocLimitMb) + return; + Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(), + Size); + Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n"); + PrintStackTrace(); + DumpCurrentUnit("oom-"); + Printf("SUMMARY: libFuzzer: out-of-memory\n"); + PrintFinalStats(); + _Exit(Options.OOMExitCode); // Stop right now. +} + +Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, + FuzzingOptions Options) + : CB(CB), Corpus(Corpus), MD(MD), Options(Options) { + if (EF->__sanitizer_set_death_callback) + EF->__sanitizer_set_death_callback(StaticDeathCallback); + assert(!F); + F = this; + TPC.ResetMaps(); + IsMyThread = true; + if (Options.DetectLeaks && EF->__sanitizer_install_malloc_and_free_hooks) + EF->__sanitizer_install_malloc_and_free_hooks(MallocHook, FreeHook); + TPC.SetUseCounters(Options.UseCounters); + TPC.SetUseValueProfileMask(Options.UseValueProfile); + + if (Options.Verbosity) + TPC.PrintModuleInfo(); + if (!Options.OutputCorpus.empty() && Options.ReloadIntervalSec) + EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus); + MaxInputLen = MaxMutationLen = Options.MaxLen; + TmpMaxMutationLen = 0; // Will be set once we load the corpus. + AllocateCurrentUnitData(); + CurrentUnitSize = 0; + memset(BaseSha1, 0, sizeof(BaseSha1)); +} + +Fuzzer::~Fuzzer() {} + +void Fuzzer::AllocateCurrentUnitData() { + if (CurrentUnitData || MaxInputLen == 0) + return; + CurrentUnitData = new uint8_t[MaxInputLen]; +} + +void Fuzzer::StaticDeathCallback() { + assert(F); + F->DeathCallback(); +} + +void Fuzzer::DumpCurrentUnit(const char *Prefix) { + if (!CurrentUnitData) + return; // Happens when running individual inputs. + ScopedDisableMsanInterceptorChecks S; + MD.PrintMutationSequence(); + Printf("; base unit: %s\n", Sha1ToString(BaseSha1).c_str()); + size_t UnitSize = CurrentUnitSize; + if (UnitSize <= kMaxUnitSizeToPrint) { + PrintHexArray(CurrentUnitData, UnitSize, "\n"); + PrintASCII(CurrentUnitData, UnitSize, "\n"); + } + WriteUnitToFileWithPrefix({CurrentUnitData, CurrentUnitData + UnitSize}, + Prefix); +} + +NO_SANITIZE_MEMORY +void Fuzzer::DeathCallback() { + DumpCurrentUnit("crash-"); + PrintFinalStats(); +} + +void Fuzzer::StaticAlarmCallback() { + assert(F); + F->AlarmCallback(); +} + +void Fuzzer::StaticCrashSignalCallback() { + assert(F); + F->CrashCallback(); +} + +void Fuzzer::StaticExitCallback() { + assert(F); + F->ExitCallback(); +} + +void Fuzzer::StaticInterruptCallback() { + assert(F); + F->InterruptCallback(); +} + +void Fuzzer::StaticGracefulExitCallback() { + assert(F); + F->GracefulExitRequested = true; + Printf("INFO: signal received, trying to exit gracefully\n"); +} + +void Fuzzer::StaticFileSizeExceedCallback() { + Printf("==%lu== ERROR: libFuzzer: file size exceeded\n", GetPid()); + exit(1); +} + +void Fuzzer::CrashCallback() { + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; + Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid()); + PrintStackTrace(); + Printf("NOTE: libFuzzer has rudimentary signal handlers.\n" + " Combine libFuzzer with AddressSanitizer or similar for better " + "crash reports.\n"); + Printf("SUMMARY: libFuzzer: deadly signal\n"); + DumpCurrentUnit("crash-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); // Stop right now. +} + +void Fuzzer::ExitCallback() { + if (!RunningUserCallback) + return; // This exit did not come from the user callback + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; + Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid()); + PrintStackTrace(); + Printf("SUMMARY: libFuzzer: fuzz target exited\n"); + DumpCurrentUnit("crash-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); +} + +bool Fuzzer::MaybeExitGracefully() { + if (!F->GracefulExitRequested) return false; + Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid()); + RmDirRecursive(TempPath("FuzzWithFork", ".dir")); + F->PrintFinalStats(); + return true; +} + +void Fuzzer::GracefullyExit() { + F->GracefulExitRequested = true; +} + +bool Fuzzer::isGracefulExitRequested() { + return F->GracefulExitRequested; +} + +void Fuzzer::InterruptCallback() { + Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); + PrintFinalStats(); + ScopedDisableMsanInterceptorChecks S; // RmDirRecursive may call opendir(). + RmDirRecursive(TempPath("FuzzWithFork", ".dir")); + // Stop right now, don't perform any at-exit actions. + _Exit(Options.InterruptExitCode); +} + +NO_SANITIZE_MEMORY +void Fuzzer::AlarmCallback() { + assert(Options.UnitTimeoutSec > 0); + // In Windows and Fuchsia, Alarm callback is executed by a different thread. + // NetBSD's current behavior needs this change too. +#if !LIBFUZZER_WINDOWS && !LIBFUZZER_NETBSD && !LIBFUZZER_FUCHSIA + if (!InFuzzingThread()) + return; +#endif + if (!RunningUserCallback) + return; // We have not started running units yet. + size_t Seconds = + duration_cast<seconds>(system_clock::now() - UnitStartTime).count(); + if (Seconds == 0) + return; + if (Options.Verbosity >= 2) + Printf("AlarmCallback %zd\n", Seconds); + if (Seconds >= (size_t)Options.UnitTimeoutSec) { + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; + Printf("ALARM: working on the last Unit for %zd seconds\n", Seconds); + Printf(" and the timeout value is %d (use -timeout=N to change)\n", + Options.UnitTimeoutSec); + DumpCurrentUnit("timeout-"); + Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(), + Seconds); + PrintStackTrace(); + Printf("SUMMARY: libFuzzer: timeout\n"); + PrintFinalStats(); + _Exit(Options.TimeoutExitCode); // Stop right now. + } +} + +void Fuzzer::RssLimitCallback() { + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; + Printf( + "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n", + GetPid(), GetPeakRSSMb(), Options.RssLimitMb); + Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n"); + PrintMemoryProfile(); + DumpCurrentUnit("oom-"); + Printf("SUMMARY: libFuzzer: out-of-memory\n"); + PrintFinalStats(); + _Exit(Options.OOMExitCode); // Stop right now. +} + +void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units, + size_t Features) { + size_t ExecPerSec = execPerSec(); + if (!Options.Verbosity) + return; + Printf("#%zd\t%s", TotalNumberOfRuns, Where); + if (size_t N = TPC.GetTotalPCCoverage()) + Printf(" cov: %zd", N); + if (size_t N = Features ? Features : Corpus.NumFeatures()) + Printf(" ft: %zd", N); + if (!Corpus.empty()) { + Printf(" corp: %zd", Corpus.NumActiveUnits()); + if (size_t N = Corpus.SizeInBytes()) { + if (N < (1 << 14)) + Printf("/%zdb", N); + else if (N < (1 << 24)) + Printf("/%zdKb", N >> 10); + else + Printf("/%zdMb", N >> 20); + } + if (size_t FF = Corpus.NumInputsThatTouchFocusFunction()) + Printf(" focus: %zd", FF); + } + if (TmpMaxMutationLen) + Printf(" lim: %zd", TmpMaxMutationLen); + if (Units) + Printf(" units: %zd", Units); + + Printf(" exec/s: %zd", ExecPerSec); + Printf(" rss: %zdMb", GetPeakRSSMb()); + Printf("%s", End); +} + +void Fuzzer::PrintFinalStats() { + if (Options.PrintCoverage) + TPC.PrintCoverage(); + if (Options.PrintCorpusStats) + Corpus.PrintStats(); + if (!Options.PrintFinalStats) + return; + size_t ExecPerSec = execPerSec(); + Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns); + Printf("stat::average_exec_per_sec: %zd\n", ExecPerSec); + Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded); + Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds); + Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb()); +} + +void Fuzzer::SetMaxInputLen(size_t MaxInputLen) { + assert(this->MaxInputLen == 0); // Can only reset MaxInputLen from 0 to non-0. + assert(MaxInputLen); + this->MaxInputLen = MaxInputLen; + this->MaxMutationLen = MaxInputLen; + AllocateCurrentUnitData(); + Printf("INFO: -max_len is not provided; " + "libFuzzer will not generate inputs larger than %zd bytes\n", + MaxInputLen); +} + +void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { + assert(MaxMutationLen && MaxMutationLen <= MaxInputLen); + this->MaxMutationLen = MaxMutationLen; +} + +void Fuzzer::CheckExitOnSrcPosOrItem() { + if (!Options.ExitOnSrcPos.empty()) { + static auto *PCsSet = new Set<uintptr_t>; + auto HandlePC = [&](const TracePC::PCTableEntry *TE) { + if (!PCsSet->insert(TE->PC).second) + return; + std::string Descr = DescribePC("%F %L", TE->PC + 1); + if (Descr.find(Options.ExitOnSrcPos) != std::string::npos) { + Printf("INFO: found line matching '%s', exiting.\n", + Options.ExitOnSrcPos.c_str()); + _Exit(0); + } + }; + TPC.ForEachObservedPC(HandlePC); + } + if (!Options.ExitOnItem.empty()) { + if (Corpus.HasUnit(Options.ExitOnItem)) { + Printf("INFO: found item with checksum '%s', exiting.\n", + Options.ExitOnItem.c_str()); + _Exit(0); + } + } +} + +void Fuzzer::RereadOutputCorpus(size_t MaxSize) { + if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) + return; + Vector<Unit> AdditionalCorpus; + ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus, + &EpochOfLastReadOfOutputCorpus, MaxSize, + /*ExitOnError*/ false); + if (Options.Verbosity >= 2) + Printf("Reload: read %zd new units.\n", AdditionalCorpus.size()); + bool Reloaded = false; + for (auto &U : AdditionalCorpus) { + if (U.size() > MaxSize) + U.resize(MaxSize); + if (!Corpus.HasUnit(U)) { + if (RunOne(U.data(), U.size())) { + CheckExitOnSrcPosOrItem(); + Reloaded = true; + } + } + } + if (Reloaded) + PrintStats("RELOAD"); +} + +void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { + auto TimeOfUnit = + duration_cast<seconds>(UnitStopTime - UnitStartTime).count(); + if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && + secondsSinceProcessStartUp() >= 2) + PrintStats("pulse "); + if (TimeOfUnit > TimeOfLongestUnitInSeconds * 1.1 && + TimeOfUnit >= Options.ReportSlowUnits) { + TimeOfLongestUnitInSeconds = TimeOfUnit; + Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds); + WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-"); + } +} + +static void WriteFeatureSetToFile(const std::string &FeaturesDir, + const std::string &FileName, + const Vector<uint32_t> &FeatureSet) { + if (FeaturesDir.empty() || FeatureSet.empty()) return; + WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()), + FeatureSet.size() * sizeof(FeatureSet[0]), + DirPlusFile(FeaturesDir, FileName)); +} + +static void RenameFeatureSetFile(const std::string &FeaturesDir, + const std::string &OldFile, + const std::string &NewFile) { + if (FeaturesDir.empty()) return; + RenameFile(DirPlusFile(FeaturesDir, OldFile), + DirPlusFile(FeaturesDir, NewFile)); +} + +bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, + InputInfo *II, bool *FoundUniqFeatures) { + if (!Size) + return false; + + if (ExecuteCallback(Data, Size) > 0) { + return false; + } + + UniqFeatureSetTmp.clear(); + size_t FoundUniqFeaturesOfII = 0; + size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); + TPC.CollectFeatures([&](size_t Feature) { + if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + UniqFeatureSetTmp.push_back(Feature); + if (Options.Entropic) + Corpus.UpdateFeatureFrequency(II, Feature); + if (Options.ReduceInputs && II) + if (std::binary_search(II->UniqFeatureSet.begin(), + II->UniqFeatureSet.end(), Feature)) + FoundUniqFeaturesOfII++; + }); + if (FoundUniqFeatures) + *FoundUniqFeatures = FoundUniqFeaturesOfII; + PrintPulseAndReportSlowInput(Data, Size); + size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; + if (NumNewFeatures) { + TPC.UpdateObservedPCs(); + auto NewII = Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, + MayDeleteFile, TPC.ObservedFocusFunction(), + UniqFeatureSetTmp, DFT, II); + WriteFeatureSetToFile(Options.FeaturesDir, Sha1ToString(NewII->Sha1), + NewII->UniqFeatureSet); + return true; + } + if (II && FoundUniqFeaturesOfII && + II->DataFlowTraceForFocusFunction.empty() && + FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && + II->U.size() > Size) { + auto OldFeaturesFile = Sha1ToString(II->Sha1); + Corpus.Replace(II, {Data, Data + Size}); + RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile, + Sha1ToString(II->Sha1)); + return true; + } + return false; +} + +size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const { + assert(InFuzzingThread()); + *Data = CurrentUnitData; + return CurrentUnitSize; +} + +void Fuzzer::CrashOnOverwrittenData() { + Printf("==%d== ERROR: libFuzzer: fuzz target overwrites its const input\n", + GetPid()); + PrintStackTrace(); + Printf("SUMMARY: libFuzzer: overwrites-const-input\n"); + DumpCurrentUnit("crash-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); // Stop right now. +} + +// Compare two arrays, but not all bytes if the arrays are large. +static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) { + const size_t Limit = 64; + if (Size <= 64) + return !memcmp(A, B, Size); + // Compare first and last Limit/2 bytes. + return !memcmp(A, B, Limit / 2) && + !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); +} + +int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { + TPC.RecordInitialStack(); + TotalNumberOfRuns++; + assert(InFuzzingThread()); + // We copy the contents of Unit into a separate heap buffer + // so that we reliably find buffer overflows in it. + uint8_t *DataCopy = new uint8_t[Size]; + memcpy(DataCopy, Data, Size); + if (EF->__msan_unpoison) + EF->__msan_unpoison(DataCopy, Size); + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(2); + if (CurrentUnitData && CurrentUnitData != Data) + memcpy(CurrentUnitData, Data, Size); + CurrentUnitSize = Size; + int Res = 0; + { + ScopedEnableMsanInterceptorChecks S; + AllocTracer.Start(Options.TraceMalloc); + UnitStartTime = system_clock::now(); + TPC.ResetMaps(); + RunningUserCallback = true; + Res = CB(DataCopy, Size); + RunningUserCallback = false; + UnitStopTime = system_clock::now(); + assert(Res >= 0); + HasMoreMallocsThanFrees = AllocTracer.Stop(); + } + if (!LooseMemeq(DataCopy, Data, Size)) + CrashOnOverwrittenData(); + CurrentUnitSize = 0; + delete[] DataCopy; + return Res; +} + +std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { + if (Options.OnlyASCII) + assert(IsASCII(U)); + if (Options.OutputCorpus.empty()) + return ""; + std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); + WriteToFile(U, Path); + if (Options.Verbosity >= 2) + Printf("Written %zd bytes to %s\n", U.size(), Path.c_str()); + return Path; +} + +void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) { + if (!Options.SaveArtifacts) + return; + std::string Path = Options.ArtifactPrefix + Prefix + Hash(U); + if (!Options.ExactArtifactPath.empty()) + Path = Options.ExactArtifactPath; // Overrides ArtifactPrefix. + WriteToFile(U, Path); + Printf("artifact_prefix='%s'; Test unit written to %s\n", + Options.ArtifactPrefix.c_str(), Path.c_str()); + if (U.size() <= kMaxUnitSizeToPrint) + Printf("Base64: %s\n", Base64(U).c_str()); +} + +void Fuzzer::PrintStatusForNewUnit(const Unit &U, const char *Text) { + if (!Options.PrintNEW) + return; + PrintStats(Text, ""); + if (Options.Verbosity) { + Printf(" L: %zd/%zd ", U.size(), Corpus.MaxInputSize()); + MD.PrintMutationSequence(); + Printf("\n"); + } +} + +void Fuzzer::ReportNewCoverage(InputInfo *II, const Unit &U) { + II->NumSuccessfullMutations++; + MD.RecordSuccessfulMutationSequence(); + PrintStatusForNewUnit(U, II->Reduced ? "REDUCE" : "NEW "); + WriteToOutputCorpus(U); + NumberOfNewUnitsAdded++; + CheckExitOnSrcPosOrItem(); // Check only after the unit is saved to corpus. + LastCorpusUpdateRun = TotalNumberOfRuns; +} + +// Tries detecting a memory leak on the particular input that we have just +// executed before calling this function. +void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size, + bool DuringInitialCorpusExecution) { + if (!HasMoreMallocsThanFrees) + return; // mallocs==frees, a leak is unlikely. + if (!Options.DetectLeaks) + return; + if (!DuringInitialCorpusExecution && + TotalNumberOfRuns >= Options.MaxNumberOfRuns) + return; + if (!&(EF->__lsan_enable) || !&(EF->__lsan_disable) || + !(EF->__lsan_do_recoverable_leak_check)) + return; // No lsan. + // Run the target once again, but with lsan disabled so that if there is + // a real leak we do not report it twice. + EF->__lsan_disable(); + ExecuteCallback(Data, Size); + EF->__lsan_enable(); + if (!HasMoreMallocsThanFrees) + return; // a leak is unlikely. + if (NumberOfLeakDetectionAttempts++ > 1000) { + Options.DetectLeaks = false; + Printf("INFO: libFuzzer disabled leak detection after every mutation.\n" + " Most likely the target function accumulates allocated\n" + " memory in a global state w/o actually leaking it.\n" + " You may try running this binary with -trace_malloc=[12]" + " to get a trace of mallocs and frees.\n" + " If LeakSanitizer is enabled in this process it will still\n" + " run on the process shutdown.\n"); + return; + } + // Now perform the actual lsan pass. This is expensive and we must ensure + // we don't call it too often. + if (EF->__lsan_do_recoverable_leak_check()) { // Leak is found, report it. + if (DuringInitialCorpusExecution) + Printf("\nINFO: a leak has been found in the initial corpus.\n\n"); + Printf("INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.\n\n"); + CurrentUnitSize = Size; + DumpCurrentUnit("leak-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on. + } +} + +bool Fuzzer::MutateAndTestOne() { + MD.StartMutationSequence(); + + auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); + if (Options.DoCrossOver) + MD.SetCrossOverWith(&Corpus.ChooseUnitToMutate(MD.GetRand()).U); + const auto &U = II.U; + memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1)); + assert(CurrentUnitData); + size_t Size = U.size(); + assert(Size <= MaxInputLen && "Oversized Unit"); + memcpy(CurrentUnitData, U.data(), Size); + + assert(MaxMutationLen > 0); + + size_t CurrentMaxMutationLen = + Min(MaxMutationLen, Max(U.size(), TmpMaxMutationLen)); + assert(CurrentMaxMutationLen > 0); + + for (int i = 0; i < Options.MutateDepth; i++) { + if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) + break; + if (MaybeExitGracefully()) return true; + size_t NewSize = 0; + if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() && + Size <= CurrentMaxMutationLen) + NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size, + II.DataFlowTraceForFocusFunction); + + // If MutateWithMask either failed or wasn't called, call default Mutate. + if (!NewSize) + NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); + + if (!NewSize) + continue; + + assert(NewSize > 0 && "Mutator returned empty unit"); + assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); + Size = NewSize; + II.NumExecutedMutations++; + Corpus.IncrementNumExecutedMutations(); + + bool FoundUniqFeatures = false; + bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II, + &FoundUniqFeatures); + TryDetectingAMemoryLeak(CurrentUnitData, Size, + /*DuringInitialCorpusExecution*/ false); + if (NewCov) { + ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size}); + break; // We will mutate this input more in the next rounds. + } + if (Options.ReduceDepth && !FoundUniqFeatures) + break; + } + + II.NeedsEnergyUpdate = true; + return false; +} + +void Fuzzer::PurgeAllocator() { + if (Options.PurgeAllocatorIntervalSec < 0 || !EF->__sanitizer_purge_allocator) + return; + if (duration_cast<seconds>(system_clock::now() - + LastAllocatorPurgeAttemptTime) + .count() < Options.PurgeAllocatorIntervalSec) + return; + + if (Options.RssLimitMb <= 0 || + GetPeakRSSMb() > static_cast<size_t>(Options.RssLimitMb) / 2) + EF->__sanitizer_purge_allocator(); + + LastAllocatorPurgeAttemptTime = system_clock::now(); +} + +int Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { + const size_t kMaxSaneLen = 1 << 20; + const size_t kMinDefaultLen = 4096; + size_t MaxSize = 0; + size_t MinSize = -1; + size_t TotalSize = 0; + for (auto &File : CorporaFiles) { + MaxSize = Max(File.Size, MaxSize); + MinSize = Min(File.Size, MinSize); + TotalSize += File.Size; + } + if (Options.MaxLen == 0) + SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxSize), kMaxSaneLen)); + assert(MaxInputLen > 0); + + // Test the callback with empty input and never try it again. + uint8_t dummy = 0; + ExecuteCallback(&dummy, 0); + + if (CorporaFiles.empty()) { + Printf("INFO: A corpus is not provided, starting from an empty corpus\n"); + Unit U({'\n'}); // Valid ASCII input. + RunOne(U.data(), U.size()); + } else { + Printf("INFO: seed corpus: files: %zd min: %zdb max: %zdb total: %zdb" + " rss: %zdMb\n", + CorporaFiles.size(), MinSize, MaxSize, TotalSize, GetPeakRSSMb()); + if (Options.ShuffleAtStartUp) + std::shuffle(CorporaFiles.begin(), CorporaFiles.end(), MD.GetRand()); + + if (Options.PreferSmall) { + std::stable_sort(CorporaFiles.begin(), CorporaFiles.end()); + assert(CorporaFiles.front().Size <= CorporaFiles.back().Size); + } + + // Load and execute inputs one by one. + for (auto &SF : CorporaFiles) { + auto U = FileToVector(SF.File, MaxInputLen, /*ExitOnError=*/false); + assert(U.size() <= MaxInputLen); + RunOne(U.data(), U.size()); + CheckExitOnSrcPosOrItem(); + TryDetectingAMemoryLeak(U.data(), U.size(), + /*DuringInitialCorpusExecution*/ true); + } + } + + PrintStats("INITED"); + if (!Options.FocusFunction.empty()) { + Printf("INFO: %zd/%zd inputs touch the focus function\n", + Corpus.NumInputsThatTouchFocusFunction(), Corpus.size()); + if (!Options.DataFlowTrace.empty()) + Printf("INFO: %zd/%zd inputs have the Data Flow Trace\n", + Corpus.NumInputsWithDataFlowTrace(), + Corpus.NumInputsThatTouchFocusFunction()); + } + + if (Corpus.empty() && Options.MaxNumberOfRuns) { + Printf("ERROR: no interesting inputs were found. " + "Is the code instrumented for coverage? Exiting.\n"); + return 1; + } + return 0; +} + +int Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) { + auto FocusFunctionOrAuto = Options.FocusFunction; + int Res = DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, + MD.GetRand()); + if (Res != 0) + return Res; + Res = TPC.SetFocusFunction(FocusFunctionOrAuto); + if (Res != 0) + return Res; + Res = ReadAndExecuteSeedCorpora(CorporaFiles); + if (Res != 0) + return Res; + DFT.Clear(); // No need for DFT any more. + TPC.SetPrintNewPCs(Options.PrintNewCovPcs); + TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs); + system_clock::time_point LastCorpusReload = system_clock::now(); + + TmpMaxMutationLen = + Min(MaxMutationLen, Max(size_t(4), Corpus.MaxInputSize())); + + while (true) { + auto Now = system_clock::now(); + if (!Options.StopFile.empty() && + !FileToVector(Options.StopFile, 1, false).empty()) + break; + if (duration_cast<seconds>(Now - LastCorpusReload).count() >= + Options.ReloadIntervalSec) { + RereadOutputCorpus(MaxInputLen); + LastCorpusReload = system_clock::now(); + } + if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) + break; + if (TimedOut()) + break; + + // Update TmpMaxMutationLen + if (Options.LenControl) { + if (TmpMaxMutationLen < MaxMutationLen && + TotalNumberOfRuns - LastCorpusUpdateRun > + Options.LenControl * Log(TmpMaxMutationLen)) { + TmpMaxMutationLen = + Min(MaxMutationLen, TmpMaxMutationLen + Log(TmpMaxMutationLen)); + LastCorpusUpdateRun = TotalNumberOfRuns; + } + } else { + TmpMaxMutationLen = MaxMutationLen; + } + + // Perform several mutations and runs. + if (MutateAndTestOne()) + return 0; + + PurgeAllocator(); + } + + PrintStats("DONE ", "\n"); + MD.PrintRecommendedDictionary(); + return 0; +} + +void Fuzzer::MinimizeCrashLoop(const Unit &U) { + if (U.size() <= 1) + return; + while (!TimedOut() && TotalNumberOfRuns < Options.MaxNumberOfRuns) { + MD.StartMutationSequence(); + memcpy(CurrentUnitData, U.data(), U.size()); + for (int i = 0; i < Options.MutateDepth; i++) { + size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen); + assert(NewSize <= MaxMutationLen); + if (!NewSize) + continue; + ExecuteCallback(CurrentUnitData, NewSize); + PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); + TryDetectingAMemoryLeak(CurrentUnitData, NewSize, + /*DuringInitialCorpusExecution*/ false); + } + } +} + +} // namespace fuzzer + +extern "C" { + +ATTRIBUTE_INTERFACE size_t +LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { + assert(fuzzer::F); + return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize); +} + +} // extern "C" diff --git a/tools/fuzzing/libfuzzer/FuzzerMain.cpp b/tools/fuzzing/libfuzzer/FuzzerMain.cpp new file mode 100644 index 0000000000..75f2f8e75c --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMain.cpp @@ -0,0 +1,21 @@ +//===- FuzzerMain.cpp - main() function and flags -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// main() and flags. +//===----------------------------------------------------------------------===// + +#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" + +extern "C" { +// This function should be defined by the user. +int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); +} // extern "C" + +ATTRIBUTE_INTERFACE int main(int argc, char **argv) { + return fuzzer::FuzzerDriver(&argc, &argv, LLVMFuzzerTestOneInput); +} diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp new file mode 100644 index 0000000000..0a185c7325 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp @@ -0,0 +1,419 @@ +//===- FuzzerMerge.cpp - merging corpora ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Merging corpora. +//===----------------------------------------------------------------------===// + +#include "FuzzerCommand.h" +#include "FuzzerMerge.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerTracePC.h" +#include "FuzzerUtil.h" + +#include <fstream> +#include <iterator> +#include <set> +#include <sstream> +#include <unordered_set> + +namespace fuzzer { + +bool Merger::Parse(const std::string &Str, bool ParseCoverage) { + std::istringstream SS(Str); + return Parse(SS, ParseCoverage); +} + +int Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { + if (!Parse(IS, ParseCoverage)) { + Printf("MERGE: failed to parse the control file (unexpected error)\n"); + return 1; + } + return 0; +} + +// The control file example: +// +// 3 # The number of inputs +// 1 # The number of inputs in the first corpus, <= the previous number +// file0 +// file1 +// file2 # One file name per line. +// STARTED 0 123 # FileID, file size +// FT 0 1 4 6 8 # FileID COV1 COV2 ... +// COV 0 7 8 9 # FileID COV1 COV1 +// STARTED 1 456 # If FT is missing, the input crashed while processing. +// STARTED 2 567 +// FT 2 8 9 +// COV 2 11 12 +bool Merger::Parse(std::istream &IS, bool ParseCoverage) { + LastFailure.clear(); + std::string Line; + + // Parse NumFiles. + if (!std::getline(IS, Line, '\n')) return false; + std::istringstream L1(Line); + size_t NumFiles = 0; + L1 >> NumFiles; + if (NumFiles == 0 || NumFiles > 10000000) return false; + + // Parse NumFilesInFirstCorpus. + if (!std::getline(IS, Line, '\n')) return false; + std::istringstream L2(Line); + NumFilesInFirstCorpus = NumFiles + 1; + L2 >> NumFilesInFirstCorpus; + if (NumFilesInFirstCorpus > NumFiles) return false; + + // Parse file names. + Files.resize(NumFiles); + for (size_t i = 0; i < NumFiles; i++) + if (!std::getline(IS, Files[i].Name, '\n')) + return false; + + // Parse STARTED, FT, and COV lines. + size_t ExpectedStartMarker = 0; + const size_t kInvalidStartMarker = -1; + size_t LastSeenStartMarker = kInvalidStartMarker; + Vector<uint32_t> TmpFeatures; + Set<uint32_t> PCs; + while (std::getline(IS, Line, '\n')) { + std::istringstream ISS1(Line); + std::string Marker; + size_t N; + ISS1 >> Marker; + ISS1 >> N; + if (Marker == "STARTED") { + // STARTED FILE_ID FILE_SIZE + if (ExpectedStartMarker != N) + return false; + ISS1 >> Files[ExpectedStartMarker].Size; + LastSeenStartMarker = ExpectedStartMarker; + assert(ExpectedStartMarker < Files.size()); + ExpectedStartMarker++; + } else if (Marker == "FT") { + // FT FILE_ID COV1 COV2 COV3 ... + size_t CurrentFileIdx = N; + if (CurrentFileIdx != LastSeenStartMarker) + return false; + LastSeenStartMarker = kInvalidStartMarker; + if (ParseCoverage) { + TmpFeatures.clear(); // use a vector from outer scope to avoid resizes. + while (ISS1 >> N) + TmpFeatures.push_back(N); + std::sort(TmpFeatures.begin(), TmpFeatures.end()); + Files[CurrentFileIdx].Features = TmpFeatures; + } + } else if (Marker == "COV") { + size_t CurrentFileIdx = N; + if (ParseCoverage) + while (ISS1 >> N) + if (PCs.insert(N).second) + Files[CurrentFileIdx].Cov.push_back(N); + } else { + return false; + } + } + if (LastSeenStartMarker != kInvalidStartMarker) + LastFailure = Files[LastSeenStartMarker].Name; + + FirstNotProcessedFile = ExpectedStartMarker; + return true; +} + +size_t Merger::ApproximateMemoryConsumption() const { + size_t Res = 0; + for (const auto &F: Files) + Res += sizeof(F) + F.Features.size() * sizeof(F.Features[0]); + return Res; +} + +// Decides which files need to be merged (add those to NewFiles). +// Returns the number of new features added. +size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, + Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, + Vector<std::string> *NewFiles) { + NewFiles->clear(); + assert(NumFilesInFirstCorpus <= Files.size()); + Set<uint32_t> AllFeatures = InitialFeatures; + + // What features are in the initial corpus? + for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { + auto &Cur = Files[i].Features; + AllFeatures.insert(Cur.begin(), Cur.end()); + } + // Remove all features that we already know from all other inputs. + for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { + auto &Cur = Files[i].Features; + Vector<uint32_t> Tmp; + std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(), + AllFeatures.end(), std::inserter(Tmp, Tmp.begin())); + Cur.swap(Tmp); + } + + // Sort. Give preference to + // * smaller files + // * files with more features. + std::sort(Files.begin() + NumFilesInFirstCorpus, Files.end(), + [&](const MergeFileInfo &a, const MergeFileInfo &b) -> bool { + if (a.Size != b.Size) + return a.Size < b.Size; + return a.Features.size() > b.Features.size(); + }); + + // One greedy pass: add the file's features to AllFeatures. + // If new features were added, add this file to NewFiles. + for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { + auto &Cur = Files[i].Features; + // Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(), + // Files[i].Size, Cur.size()); + bool FoundNewFeatures = false; + for (auto Fe: Cur) { + if (AllFeatures.insert(Fe).second) { + FoundNewFeatures = true; + NewFeatures->insert(Fe); + } + } + if (FoundNewFeatures) + NewFiles->push_back(Files[i].Name); + for (auto Cov : Files[i].Cov) + if (InitialCov.find(Cov) == InitialCov.end()) + NewCov->insert(Cov); + } + return NewFeatures->size(); +} + +Set<uint32_t> Merger::AllFeatures() const { + Set<uint32_t> S; + for (auto &File : Files) + S.insert(File.Features.begin(), File.Features.end()); + return S; +} + +// Inner process. May crash if the target crashes. +int Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { + Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); + Merger M; + std::ifstream IF(CFPath); + int Res = M.ParseOrExit(IF, false); + if (Res != 0) + return Res; + IF.close(); + if (!M.LastFailure.empty()) + Printf("MERGE-INNER: '%s' caused a failure at the previous merge step\n", + M.LastFailure.c_str()); + + Printf("MERGE-INNER: %zd total files;" + " %zd processed earlier; will process %zd files now\n", + M.Files.size(), M.FirstNotProcessedFile, + M.Files.size() - M.FirstNotProcessedFile); + + std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); + Set<size_t> AllFeatures; + auto PrintStatsWrapper = [this, &AllFeatures](const char* Where) { + this->PrintStats(Where, "\n", 0, AllFeatures.size()); + }; + Set<const TracePC::PCTableEntry *> AllPCs; + for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { + if (Fuzzer::MaybeExitGracefully()) + return 0; + auto U = FileToVector(M.Files[i].Name); + if (U.size() > MaxInputLen) { + U.resize(MaxInputLen); + U.shrink_to_fit(); + } + + // Write the pre-run marker. + OF << "STARTED " << i << " " << U.size() << "\n"; + OF.flush(); // Flush is important since Command::Execute may crash. + // Run. + TPC.ResetMaps(); + if (ExecuteCallback(U.data(), U.size()) > 0) { + continue; + } + // Collect coverage. We are iterating over the files in this order: + // * First, files in the initial corpus ordered by size, smallest first. + // * Then, all other files, smallest first. + // So it makes no sense to record all features for all files, instead we + // only record features that were not seen before. + Set<size_t> UniqFeatures; + TPC.CollectFeatures([&](size_t Feature) { + if (AllFeatures.insert(Feature).second) + UniqFeatures.insert(Feature); + }); + TPC.UpdateObservedPCs(); + // Show stats. + if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) + PrintStatsWrapper("pulse "); + if (TotalNumberOfRuns == M.NumFilesInFirstCorpus) + PrintStatsWrapper("LOADED"); + // Write the post-run marker and the coverage. + OF << "FT " << i; + for (size_t F : UniqFeatures) + OF << " " << F; + OF << "\n"; + OF << "COV " << i; + TPC.ForEachObservedPC([&](const TracePC::PCTableEntry *TE) { + if (AllPCs.insert(TE).second) + OF << " " << TPC.PCTableEntryIdx(TE); + }); + OF << "\n"; + OF.flush(); + } + PrintStatsWrapper("DONE "); + return 0; +} + +static int WriteNewControlFile(const std::string &CFPath, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, + const Vector<MergeFileInfo> &KnownFiles, + size_t &NumFiles) { + std::unordered_set<std::string> FilesToSkip; + for (auto &SF: KnownFiles) + FilesToSkip.insert(SF.Name); + + Vector<std::string> FilesToUse; + auto MaybeUseFile = [=, &FilesToUse](std::string Name) { + if (FilesToSkip.find(Name) == FilesToSkip.end()) + FilesToUse.push_back(Name); + }; + for (auto &SF: OldCorpus) + MaybeUseFile(SF.File); + auto FilesToUseFromOldCorpus = FilesToUse.size(); + for (auto &SF: NewCorpus) + MaybeUseFile(SF.File); + + RemoveFile(CFPath); + std::ofstream ControlFile(CFPath); + ControlFile << FilesToUse.size() << "\n"; + ControlFile << FilesToUseFromOldCorpus << "\n"; + for (auto &FN: FilesToUse) + ControlFile << FN << "\n"; + + if (!ControlFile) { + Printf("MERGE-OUTER: failed to write to the control file: %s\n", + CFPath.c_str()); + return 1; + } + + NumFiles = FilesToUse.size(); + return 0; +} + +// Outer process. Does not call the target code and thus should not fail. +int CrashResistantMerge(const Vector<std::string> &Args, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, + Vector<std::string> *NewFiles, + const Set<uint32_t> &InitialFeatures, + Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, + Set<uint32_t> *NewCov, + const std::string &CFPath, + bool V /*Verbose*/) { + if (NewCorpus.empty() && OldCorpus.empty()) return 0; // Nothing to merge. + size_t NumAttempts = 0; + int Res; + Vector<MergeFileInfo> KnownFiles; + if (FileSize(CFPath)) { + VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", + CFPath.c_str()); + Merger M; + std::ifstream IF(CFPath); + if (M.Parse(IF, /*ParseCoverage=*/true)) { + VPrintf(V, "MERGE-OUTER: control file ok, %zd files total," + " first not processed file %zd\n", + M.Files.size(), M.FirstNotProcessedFile); + if (!M.LastFailure.empty()) + VPrintf(V, "MERGE-OUTER: '%s' will be skipped as unlucky " + "(merge has stumbled on it the last time)\n", + M.LastFailure.c_str()); + if (M.FirstNotProcessedFile >= M.Files.size()) { + // Merge has already been completed with the given merge control file. + if (M.Files.size() == OldCorpus.size() + NewCorpus.size()) { + VPrintf( + V, + "MERGE-OUTER: nothing to do, merge has been completed before\n"); + Fuzzer::GracefullyExit(); + return 0; + } + + // Number of input files likely changed, start merge from scratch, but + // reuse coverage information from the given merge control file. + VPrintf( + V, + "MERGE-OUTER: starting merge from scratch, but reusing coverage " + "information from the given control file\n"); + KnownFiles = M.Files; + } else { + // There is a merge in progress, continue. + NumAttempts = M.Files.size() - M.FirstNotProcessedFile; + } + } else { + VPrintf(V, "MERGE-OUTER: bad control file, will overwrite it\n"); + } + } + + if (!NumAttempts) { + // The supplied control file is empty or bad, create a fresh one. + VPrintf(V, "MERGE-OUTER: " + "%zd files, %zd in the initial corpus, %zd processed earlier\n", + OldCorpus.size() + NewCorpus.size(), OldCorpus.size(), + KnownFiles.size()); + Res = WriteNewControlFile(CFPath, OldCorpus, NewCorpus, KnownFiles, NumAttempts); + if (Res != 0) + return Res; + } + + // Execute the inner process until it passes. + // Every inner process should execute at least one input. + Command BaseCmd(Args); + BaseCmd.removeFlag("merge"); + BaseCmd.removeFlag("fork"); + BaseCmd.removeFlag("collect_data_flow"); + for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { + if (Fuzzer::MaybeExitGracefully()) + return 0; + VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); + Command Cmd(BaseCmd); + Cmd.addFlag("merge_control_file", CFPath); + Cmd.addFlag("merge_inner", "1"); + if (!V) { + Cmd.setOutputFile(getDevNull()); + Cmd.combineOutAndErr(); + } + auto ExitCode = ExecuteCommand(Cmd); + if (!ExitCode) { + VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); + break; + } + } + // Read the control file and do the merge. + Merger M; + std::ifstream IF(CFPath); + IF.seekg(0, IF.end); + VPrintf(V, "MERGE-OUTER: the control file has %zd bytes\n", + (size_t)IF.tellg()); + IF.seekg(0, IF.beg); + Res = M.ParseOrExit(IF, true); + if (Res != 0) + return Res; + IF.close(); + VPrintf(V, + "MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", + M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); + + M.Files.insert(M.Files.end(), KnownFiles.begin(), KnownFiles.end()); + M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " + "%zd new coverage edges\n", + NewFiles->size(), NewFeatures->size(), NewCov->size()); + return 0; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.h b/tools/fuzzing/libfuzzer/FuzzerMerge.h new file mode 100644 index 0000000000..6dc1c4c45a --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMerge.h @@ -0,0 +1,87 @@ +//===- FuzzerMerge.h - merging corpa ----------------------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Merging Corpora. +// +// The task: +// Take the existing corpus (possibly empty) and merge new inputs into +// it so that only inputs with new coverage ('features') are added. +// The process should tolerate the crashes, OOMs, leaks, etc. +// +// Algorithm: +// The outer process collects the set of files and writes their names +// into a temporary "control" file, then repeatedly launches the inner +// process until all inputs are processed. +// The outer process does not actually execute the target code. +// +// The inner process reads the control file and sees a) list of all the inputs +// and b) the last processed input. Then it starts processing the inputs one +// by one. Before processing every input it writes one line to control file: +// STARTED INPUT_ID INPUT_SIZE +// After processing an input it writes the following lines: +// FT INPUT_ID Feature1 Feature2 Feature3 ... +// COV INPUT_ID Coverage1 Coverage2 Coverage3 ... +// If a crash happens while processing an input the last line in the control +// file will be "STARTED INPUT_ID" and so the next process will know +// where to resume. +// +// Once all inputs are processed by the inner process(es) the outer process +// reads the control files and does the merge based entirely on the contents +// of control file. +// It uses a single pass greedy algorithm choosing first the smallest inputs +// within the same size the inputs that have more new features. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_MERGE_H +#define LLVM_FUZZER_MERGE_H + +#include "FuzzerDefs.h" + +#include <istream> +#include <ostream> +#include <set> +#include <vector> + +namespace fuzzer { + +struct MergeFileInfo { + std::string Name; + size_t Size = 0; + Vector<uint32_t> Features, Cov; +}; + +struct Merger { + Vector<MergeFileInfo> Files; + size_t NumFilesInFirstCorpus = 0; + size_t FirstNotProcessedFile = 0; + std::string LastFailure; + + bool Parse(std::istream &IS, bool ParseCoverage); + bool Parse(const std::string &Str, bool ParseCoverage); + int ParseOrExit(std::istream &IS, bool ParseCoverage); + size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, + Vector<std::string> *NewFiles); + size_t ApproximateMemoryConsumption() const; + Set<uint32_t> AllFeatures() const; +}; + +int CrashResistantMerge(const Vector<std::string> &Args, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, + Vector<std::string> *NewFiles, + const Set<uint32_t> &InitialFeatures, + Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, + Set<uint32_t> *NewCov, + const std::string &CFPath, + bool Verbose); + +} // namespace fuzzer + +#endif // LLVM_FUZZER_MERGE_H diff --git a/tools/fuzzing/libfuzzer/FuzzerMutate.cpp b/tools/fuzzing/libfuzzer/FuzzerMutate.cpp new file mode 100644 index 0000000000..29541eac5d --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMutate.cpp @@ -0,0 +1,562 @@ +//===- FuzzerMutate.cpp - Mutate a test input -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Mutate a test input. +//===----------------------------------------------------------------------===// + +#include "FuzzerDefs.h" +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "FuzzerMutate.h" +#include "FuzzerOptions.h" +#include "FuzzerTracePC.h" + +namespace fuzzer { + +const size_t Dictionary::kMaxDictSize; + +static void PrintASCII(const Word &W, const char *PrintAfter) { + PrintASCII(W.data(), W.size(), PrintAfter); +} + +MutationDispatcher::MutationDispatcher(Random &Rand, + const FuzzingOptions &Options) + : Rand(Rand), Options(Options) { + DefaultMutators.insert( + DefaultMutators.begin(), + { + {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"}, + {&MutationDispatcher::Mutate_InsertByte, "InsertByte"}, + {&MutationDispatcher::Mutate_InsertRepeatedBytes, + "InsertRepeatedBytes"}, + {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"}, + {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, + {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, + {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, + {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, + {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, + {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, + {&MutationDispatcher::Mutate_AddWordFromManualDictionary, + "ManualDict"}, + {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, + "PersAutoDict"}, + }); + if(Options.UseCmp) + DefaultMutators.push_back( + {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); + + if (EF->LLVMFuzzerCustomMutator) + Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); + else + Mutators = DefaultMutators; + + if (EF->LLVMFuzzerCustomCrossOver) + Mutators.push_back( + {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); +} + +static char RandCh(Random &Rand) { + if (Rand.RandBool()) return Rand(256); + const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00"; + return Special[Rand(sizeof(Special) - 1)]; +} + +size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, + size_t MaxSize) { + return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand()); +} + +size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size == 0) + return 0; + if (!CrossOverWith) return 0; + const Unit &Other = *CrossOverWith; + if (Other.empty()) + return 0; + CustomCrossOverInPlaceHere.resize(MaxSize); + auto &U = CustomCrossOverInPlaceHere; + size_t NewSize = EF->LLVMFuzzerCustomCrossOver( + Data, Size, Other.data(), Other.size(), U.data(), U.size(), Rand.Rand()); + if (!NewSize) + return 0; + assert(NewSize <= MaxSize && "CustomCrossOver returned overisized unit"); + memcpy(Data, U.data(), NewSize); + return NewSize; +} + +size_t MutationDispatcher::Mutate_ShuffleBytes(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize || Size == 0) return 0; + size_t ShuffleAmount = + Rand(std::min(Size, (size_t)8)) + 1; // [1,8] and <= Size. + size_t ShuffleStart = Rand(Size - ShuffleAmount); + assert(ShuffleStart + ShuffleAmount <= Size); + std::shuffle(Data + ShuffleStart, Data + ShuffleStart + ShuffleAmount, Rand); + return Size; +} + +size_t MutationDispatcher::Mutate_EraseBytes(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size <= 1) return 0; + size_t N = Rand(Size / 2) + 1; + assert(N < Size); + size_t Idx = Rand(Size - N + 1); + // Erase Data[Idx:Idx+N]. + memmove(Data + Idx, Data + Idx + N, Size - Idx - N); + // Printf("Erase: %zd %zd => %zd; Idx %zd\n", N, Size, Size - N, Idx); + return Size - N; +} + +size_t MutationDispatcher::Mutate_InsertByte(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size >= MaxSize) return 0; + size_t Idx = Rand(Size + 1); + // Insert new value at Data[Idx]. + memmove(Data + Idx + 1, Data + Idx, Size - Idx); + Data[Idx] = RandCh(Rand); + return Size + 1; +} + +size_t MutationDispatcher::Mutate_InsertRepeatedBytes(uint8_t *Data, + size_t Size, + size_t MaxSize) { + const size_t kMinBytesToInsert = 3; + if (Size + kMinBytesToInsert >= MaxSize) return 0; + size_t MaxBytesToInsert = std::min(MaxSize - Size, (size_t)128); + size_t N = Rand(MaxBytesToInsert - kMinBytesToInsert + 1) + kMinBytesToInsert; + assert(Size + N <= MaxSize && N); + size_t Idx = Rand(Size + 1); + // Insert new values at Data[Idx]. + memmove(Data + Idx + N, Data + Idx, Size - Idx); + // Give preference to 0x00 and 0xff. + uint8_t Byte = Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255); + for (size_t i = 0; i < N; i++) + Data[Idx + i] = Byte; + return Size + N; +} + +size_t MutationDispatcher::Mutate_ChangeByte(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + size_t Idx = Rand(Size); + Data[Idx] = RandCh(Rand); + return Size; +} + +size_t MutationDispatcher::Mutate_ChangeBit(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + size_t Idx = Rand(Size); + Data[Idx] ^= 1 << Rand(8); + return Size; +} + +size_t MutationDispatcher::Mutate_AddWordFromManualDictionary(uint8_t *Data, + size_t Size, + size_t MaxSize) { + return AddWordFromDictionary(ManualDictionary, Data, Size, MaxSize); +} + +size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size, + size_t MaxSize, + DictionaryEntry &DE) { + const Word &W = DE.GetW(); + bool UsePositionHint = DE.HasPositionHint() && + DE.GetPositionHint() + W.size() < Size && + Rand.RandBool(); + if (Rand.RandBool()) { // Insert W. + if (Size + W.size() > MaxSize) return 0; + size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1); + memmove(Data + Idx + W.size(), Data + Idx, Size - Idx); + memcpy(Data + Idx, W.data(), W.size()); + Size += W.size(); + } else { // Overwrite some bytes with W. + if (W.size() > Size) return 0; + size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size()); + memcpy(Data + Idx, W.data(), W.size()); + } + return Size; +} + +// Somewhere in the past we have observed a comparison instructions +// with arguments Arg1 Arg2. This function tries to guess a dictionary +// entry that will satisfy that comparison. +// It first tries to find one of the arguments (possibly swapped) in the +// input and if it succeeds it creates a DE with a position hint. +// Otherwise it creates a DE with one of the arguments w/o a position hint. +DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( + const void *Arg1, const void *Arg2, + const void *Arg1Mutation, const void *Arg2Mutation, + size_t ArgSize, const uint8_t *Data, + size_t Size) { + bool HandleFirst = Rand.RandBool(); + const void *ExistingBytes, *DesiredBytes; + Word W; + const uint8_t *End = Data + Size; + for (int Arg = 0; Arg < 2; Arg++) { + ExistingBytes = HandleFirst ? Arg1 : Arg2; + DesiredBytes = HandleFirst ? Arg2Mutation : Arg1Mutation; + HandleFirst = !HandleFirst; + W.Set(reinterpret_cast<const uint8_t*>(DesiredBytes), ArgSize); + const size_t kMaxNumPositions = 8; + size_t Positions[kMaxNumPositions]; + size_t NumPositions = 0; + for (const uint8_t *Cur = Data; + Cur < End && NumPositions < kMaxNumPositions; Cur++) { + Cur = + (const uint8_t *)SearchMemory(Cur, End - Cur, ExistingBytes, ArgSize); + if (!Cur) break; + Positions[NumPositions++] = Cur - Data; + } + if (!NumPositions) continue; + return DictionaryEntry(W, Positions[Rand(NumPositions)]); + } + DictionaryEntry DE(W); + return DE; +} + + +template <class T> +DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( + T Arg1, T Arg2, const uint8_t *Data, size_t Size) { + if (Rand.RandBool()) Arg1 = Bswap(Arg1); + if (Rand.RandBool()) Arg2 = Bswap(Arg2); + T Arg1Mutation = Arg1 + Rand(-1, 1); + T Arg2Mutation = Arg2 + Rand(-1, 1); + return MakeDictionaryEntryFromCMP(&Arg1, &Arg2, &Arg1Mutation, &Arg2Mutation, + sizeof(Arg1), Data, Size); +} + +DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( + const Word &Arg1, const Word &Arg2, const uint8_t *Data, size_t Size) { + return MakeDictionaryEntryFromCMP(Arg1.data(), Arg2.data(), Arg1.data(), + Arg2.data(), Arg1.size(), Data, Size); +} + +size_t MutationDispatcher::Mutate_AddWordFromTORC( + uint8_t *Data, size_t Size, size_t MaxSize) { + Word W; + DictionaryEntry DE; + switch (Rand(4)) { + case 0: { + auto X = TPC.TORC8.Get(Rand.Rand()); + DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); + } break; + case 1: { + auto X = TPC.TORC4.Get(Rand.Rand()); + if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool()) + DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data, Size); + else + DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); + } break; + case 2: { + auto X = TPC.TORCW.Get(Rand.Rand()); + DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); + } break; + case 3: if (Options.UseMemmem) { + auto X = TPC.MMT.Get(Rand.Rand()); + DE = DictionaryEntry(X); + } break; + default: + assert(0); + } + if (!DE.GetW().size()) return 0; + Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE); + if (!Size) return 0; + DictionaryEntry &DERef = + CmpDictionaryEntriesDeque[CmpDictionaryEntriesDequeIdx++ % + kCmpDictionaryEntriesDequeSize]; + DERef = DE; + CurrentDictionaryEntrySequence.push_back(&DERef); + return Size; +} + +size_t MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary( + uint8_t *Data, size_t Size, size_t MaxSize) { + return AddWordFromDictionary(PersistentAutoDictionary, Data, Size, MaxSize); +} + +size_t MutationDispatcher::AddWordFromDictionary(Dictionary &D, uint8_t *Data, + size_t Size, size_t MaxSize) { + if (Size > MaxSize) return 0; + if (D.empty()) return 0; + DictionaryEntry &DE = D[Rand(D.size())]; + Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE); + if (!Size) return 0; + DE.IncUseCount(); + CurrentDictionaryEntrySequence.push_back(&DE); + return Size; +} + +// Overwrites part of To[0,ToSize) with a part of From[0,FromSize). +// Returns ToSize. +size_t MutationDispatcher::CopyPartOf(const uint8_t *From, size_t FromSize, + uint8_t *To, size_t ToSize) { + // Copy From[FromBeg, FromBeg + CopySize) into To[ToBeg, ToBeg + CopySize). + size_t ToBeg = Rand(ToSize); + size_t CopySize = Rand(ToSize - ToBeg) + 1; + assert(ToBeg + CopySize <= ToSize); + CopySize = std::min(CopySize, FromSize); + size_t FromBeg = Rand(FromSize - CopySize + 1); + assert(FromBeg + CopySize <= FromSize); + memmove(To + ToBeg, From + FromBeg, CopySize); + return ToSize; +} + +// Inserts part of From[0,ToSize) into To. +// Returns new size of To on success or 0 on failure. +size_t MutationDispatcher::InsertPartOf(const uint8_t *From, size_t FromSize, + uint8_t *To, size_t ToSize, + size_t MaxToSize) { + if (ToSize >= MaxToSize) return 0; + size_t AvailableSpace = MaxToSize - ToSize; + size_t MaxCopySize = std::min(AvailableSpace, FromSize); + size_t CopySize = Rand(MaxCopySize) + 1; + size_t FromBeg = Rand(FromSize - CopySize + 1); + assert(FromBeg + CopySize <= FromSize); + size_t ToInsertPos = Rand(ToSize + 1); + assert(ToInsertPos + CopySize <= MaxToSize); + size_t TailSize = ToSize - ToInsertPos; + if (To == From) { + MutateInPlaceHere.resize(MaxToSize); + memcpy(MutateInPlaceHere.data(), From + FromBeg, CopySize); + memmove(To + ToInsertPos + CopySize, To + ToInsertPos, TailSize); + memmove(To + ToInsertPos, MutateInPlaceHere.data(), CopySize); + } else { + memmove(To + ToInsertPos + CopySize, To + ToInsertPos, TailSize); + memmove(To + ToInsertPos, From + FromBeg, CopySize); + } + return ToSize + CopySize; +} + +size_t MutationDispatcher::Mutate_CopyPart(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize || Size == 0) return 0; + // If Size == MaxSize, `InsertPartOf(...)` will + // fail so there's no point using it in this case. + if (Size == MaxSize || Rand.RandBool()) + return CopyPartOf(Data, Size, Data, Size); + else + return InsertPartOf(Data, Size, Data, Size, MaxSize); +} + +size_t MutationDispatcher::Mutate_ChangeASCIIInteger(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + size_t B = Rand(Size); + while (B < Size && !isdigit(Data[B])) B++; + if (B == Size) return 0; + size_t E = B; + while (E < Size && isdigit(Data[E])) E++; + assert(B < E); + // now we have digits in [B, E). + // strtol and friends don't accept non-zero-teminated data, parse it manually. + uint64_t Val = Data[B] - '0'; + for (size_t i = B + 1; i < E; i++) + Val = Val * 10 + Data[i] - '0'; + + // Mutate the integer value. + switch(Rand(5)) { + case 0: Val++; break; + case 1: Val--; break; + case 2: Val /= 2; break; + case 3: Val *= 2; break; + case 4: Val = Rand(Val * Val); break; + default: assert(0); + } + // Just replace the bytes with the new ones, don't bother moving bytes. + for (size_t i = B; i < E; i++) { + size_t Idx = E + B - i - 1; + assert(Idx >= B && Idx < E); + Data[Idx] = (Val % 10) + '0'; + Val /= 10; + } + return Size; +} + +template<class T> +size_t ChangeBinaryInteger(uint8_t *Data, size_t Size, Random &Rand) { + if (Size < sizeof(T)) return 0; + size_t Off = Rand(Size - sizeof(T) + 1); + assert(Off + sizeof(T) <= Size); + T Val; + if (Off < 64 && !Rand(4)) { + Val = Size; + if (Rand.RandBool()) + Val = Bswap(Val); + } else { + memcpy(&Val, Data + Off, sizeof(Val)); + T Add = Rand(21); + Add -= 10; + if (Rand.RandBool()) + Val = Bswap(T(Bswap(Val) + Add)); // Add assuming different endiannes. + else + Val = Val + Add; // Add assuming current endiannes. + if (Add == 0 || Rand.RandBool()) // Maybe negate. + Val = -Val; + } + memcpy(Data + Off, &Val, sizeof(Val)); + return Size; +} + +size_t MutationDispatcher::Mutate_ChangeBinaryInteger(uint8_t *Data, + size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + switch (Rand(4)) { + case 3: return ChangeBinaryInteger<uint64_t>(Data, Size, Rand); + case 2: return ChangeBinaryInteger<uint32_t>(Data, Size, Rand); + case 1: return ChangeBinaryInteger<uint16_t>(Data, Size, Rand); + case 0: return ChangeBinaryInteger<uint8_t>(Data, Size, Rand); + default: assert(0); + } + return 0; +} + +size_t MutationDispatcher::Mutate_CrossOver(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + if (Size == 0) return 0; + if (!CrossOverWith) return 0; + const Unit &O = *CrossOverWith; + if (O.empty()) return 0; + MutateInPlaceHere.resize(MaxSize); + auto &U = MutateInPlaceHere; + size_t NewSize = 0; + switch(Rand(3)) { + case 0: + NewSize = CrossOver(Data, Size, O.data(), O.size(), U.data(), U.size()); + break; + case 1: + NewSize = InsertPartOf(O.data(), O.size(), U.data(), U.size(), MaxSize); + if (!NewSize) + NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size()); + break; + case 2: + NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size()); + break; + default: assert(0); + } + assert(NewSize > 0 && "CrossOver returned empty unit"); + assert(NewSize <= MaxSize && "CrossOver returned overisized unit"); + memcpy(Data, U.data(), NewSize); + return NewSize; +} + +void MutationDispatcher::StartMutationSequence() { + CurrentMutatorSequence.clear(); + CurrentDictionaryEntrySequence.clear(); +} + +// Copy successful dictionary entries to PersistentAutoDictionary. +void MutationDispatcher::RecordSuccessfulMutationSequence() { + for (auto DE : CurrentDictionaryEntrySequence) { + // PersistentAutoDictionary.AddWithSuccessCountOne(DE); + DE->IncSuccessCount(); + assert(DE->GetW().size()); + // Linear search is fine here as this happens seldom. + if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) + PersistentAutoDictionary.push_back({DE->GetW(), 1}); + } +} + +void MutationDispatcher::PrintRecommendedDictionary() { + Vector<DictionaryEntry> V; + for (auto &DE : PersistentAutoDictionary) + if (!ManualDictionary.ContainsWord(DE.GetW())) + V.push_back(DE); + if (V.empty()) return; + Printf("###### Recommended dictionary. ######\n"); + for (auto &DE: V) { + assert(DE.GetW().size()); + Printf("\""); + PrintASCII(DE.GetW(), "\""); + Printf(" # Uses: %zd\n", DE.GetUseCount()); + } + Printf("###### End of recommended dictionary. ######\n"); +} + +void MutationDispatcher::PrintMutationSequence() { + Printf("MS: %zd ", CurrentMutatorSequence.size()); + for (auto M : CurrentMutatorSequence) + Printf("%s-", M.Name); + if (!CurrentDictionaryEntrySequence.empty()) { + Printf(" DE: "); + for (auto DE : CurrentDictionaryEntrySequence) { + Printf("\""); + PrintASCII(DE->GetW(), "\"-"); + } + } +} + +size_t MutationDispatcher::Mutate(uint8_t *Data, size_t Size, size_t MaxSize) { + return MutateImpl(Data, Size, MaxSize, Mutators); +} + +size_t MutationDispatcher::DefaultMutate(uint8_t *Data, size_t Size, + size_t MaxSize) { + return MutateImpl(Data, Size, MaxSize, DefaultMutators); +} + +// Mutates Data in place, returns new size. +size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, + size_t MaxSize, + Vector<Mutator> &Mutators) { + assert(MaxSize > 0); + // Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize), + // in which case they will return 0. + // Try several times before returning un-mutated data. + for (int Iter = 0; Iter < 100; Iter++) { + auto M = Mutators[Rand(Mutators.size())]; + size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize); + if (NewSize && NewSize <= MaxSize) { + if (Options.OnlyASCII) + ToASCII(Data, NewSize); + CurrentMutatorSequence.push_back(M); + return NewSize; + } + } + *Data = ' '; + return 1; // Fallback, should not happen frequently. +} + +// Mask represents the set of Data bytes that are worth mutating. +size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, + size_t MaxSize, + const Vector<uint8_t> &Mask) { + size_t MaskedSize = std::min(Size, Mask.size()); + // * Copy the worthy bytes into a temporary array T + // * Mutate T + // * Copy T back. + // This is totally unoptimized. + auto &T = MutateWithMaskTemp; + if (T.size() < Size) + T.resize(Size); + size_t OneBits = 0; + for (size_t I = 0; I < MaskedSize; I++) + if (Mask[I]) + T[OneBits++] = Data[I]; + + if (!OneBits) return 0; + assert(!T.empty()); + size_t NewSize = Mutate(T.data(), OneBits, OneBits); + assert(NewSize <= OneBits); + (void)NewSize; + // Even if NewSize < OneBits we still use all OneBits bytes. + for (size_t I = 0, J = 0; I < MaskedSize; I++) + if (Mask[I]) + Data[I] = T[J++]; + return Size; +} + +void MutationDispatcher::AddWordToManualDictionary(const Word &W) { + ManualDictionary.push_back( + {W, std::numeric_limits<size_t>::max()}); +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerMutate.h b/tools/fuzzing/libfuzzer/FuzzerMutate.h new file mode 100644 index 0000000000..6cbce80276 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMutate.h @@ -0,0 +1,156 @@ +//===- FuzzerMutate.h - Internal header for the Fuzzer ----------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::MutationDispatcher +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_MUTATE_H +#define LLVM_FUZZER_MUTATE_H + +#include "FuzzerDefs.h" +#include "FuzzerDictionary.h" +#include "FuzzerOptions.h" +#include "FuzzerRandom.h" + +namespace fuzzer { + +class MutationDispatcher { +public: + MutationDispatcher(Random &Rand, const FuzzingOptions &Options); + ~MutationDispatcher() {} + /// Indicate that we are about to start a new sequence of mutations. + void StartMutationSequence(); + /// Print the current sequence of mutations. + void PrintMutationSequence(); + /// Indicate that the current sequence of mutations was successful. + void RecordSuccessfulMutationSequence(); + /// Mutates data by invoking user-provided mutator. + size_t Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by invoking user-provided crossover. + size_t Mutate_CustomCrossOver(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by shuffling bytes. + size_t Mutate_ShuffleBytes(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by erasing bytes. + size_t Mutate_EraseBytes(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by inserting a byte. + size_t Mutate_InsertByte(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by inserting several repeated bytes. + size_t Mutate_InsertRepeatedBytes(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by chanding one byte. + size_t Mutate_ChangeByte(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by chanding one bit. + size_t Mutate_ChangeBit(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by copying/inserting a part of data into a different place. + size_t Mutate_CopyPart(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Mutates data by adding a word from the manual dictionary. + size_t Mutate_AddWordFromManualDictionary(uint8_t *Data, size_t Size, + size_t MaxSize); + + /// Mutates data by adding a word from the TORC. + size_t Mutate_AddWordFromTORC(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Mutates data by adding a word from the persistent automatic dictionary. + size_t Mutate_AddWordFromPersistentAutoDictionary(uint8_t *Data, size_t Size, + size_t MaxSize); + + /// Tries to find an ASCII integer in Data, changes it to another ASCII int. + size_t Mutate_ChangeASCIIInteger(uint8_t *Data, size_t Size, size_t MaxSize); + /// Change a 1-, 2-, 4-, or 8-byte integer in interesting ways. + size_t Mutate_ChangeBinaryInteger(uint8_t *Data, size_t Size, size_t MaxSize); + + /// CrossOver Data with CrossOverWith. + size_t Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Applies one of the configured mutations. + /// Returns the new size of data which could be up to MaxSize. + size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Applies one of the configured mutations to the bytes of Data + /// that have '1' in Mask. + /// Mask.size() should be >= Size. + size_t MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, + const Vector<uint8_t> &Mask); + + /// Applies one of the default mutations. Provided as a service + /// to mutation authors. + size_t DefaultMutate(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Creates a cross-over of two pieces of Data, returns its size. + size_t CrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2, + size_t Size2, uint8_t *Out, size_t MaxOutSize); + + void AddWordToManualDictionary(const Word &W); + + void PrintRecommendedDictionary(); + + void SetCrossOverWith(const Unit *U) { CrossOverWith = U; } + + Random &GetRand() { return Rand; } + + private: + struct Mutator { + size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); + const char *Name; + }; + + size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, + size_t MaxSize); + size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, + Vector<Mutator> &Mutators); + + size_t InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, + size_t ToSize, size_t MaxToSize); + size_t CopyPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, + size_t ToSize); + size_t ApplyDictionaryEntry(uint8_t *Data, size_t Size, size_t MaxSize, + DictionaryEntry &DE); + + template <class T> + DictionaryEntry MakeDictionaryEntryFromCMP(T Arg1, T Arg2, + const uint8_t *Data, size_t Size); + DictionaryEntry MakeDictionaryEntryFromCMP(const Word &Arg1, const Word &Arg2, + const uint8_t *Data, size_t Size); + DictionaryEntry MakeDictionaryEntryFromCMP(const void *Arg1, const void *Arg2, + const void *Arg1Mutation, + const void *Arg2Mutation, + size_t ArgSize, + const uint8_t *Data, size_t Size); + + Random &Rand; + const FuzzingOptions Options; + + // Dictionary provided by the user via -dict=DICT_FILE. + Dictionary ManualDictionary; + // Temporary dictionary modified by the fuzzer itself, + // recreated periodically. + Dictionary TempAutoDictionary; + // Persistent dictionary modified by the fuzzer, consists of + // entries that led to successful discoveries in the past mutations. + Dictionary PersistentAutoDictionary; + + Vector<DictionaryEntry *> CurrentDictionaryEntrySequence; + + static const size_t kCmpDictionaryEntriesDequeSize = 16; + DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; + size_t CmpDictionaryEntriesDequeIdx = 0; + + const Unit *CrossOverWith = nullptr; + Vector<uint8_t> MutateInPlaceHere; + Vector<uint8_t> MutateWithMaskTemp; + // CustomCrossOver needs its own buffer as a custom implementation may call + // LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere. + Vector<uint8_t> CustomCrossOverInPlaceHere; + + Vector<Mutator> Mutators; + Vector<Mutator> DefaultMutators; + Vector<Mutator> CurrentMutatorSequence; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_MUTATE_H diff --git a/tools/fuzzing/libfuzzer/FuzzerOptions.h b/tools/fuzzing/libfuzzer/FuzzerOptions.h new file mode 100644 index 0000000000..9d975bd61f --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerOptions.h @@ -0,0 +1,85 @@ +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::FuzzingOptions +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_OPTIONS_H +#define LLVM_FUZZER_OPTIONS_H + +#include "FuzzerDefs.h" + +namespace fuzzer { + +struct FuzzingOptions { + int Verbosity = 1; + size_t MaxLen = 0; + size_t LenControl = 1000; + int UnitTimeoutSec = 300; + int TimeoutExitCode = 70; + int OOMExitCode = 71; + int InterruptExitCode = 72; + int ErrorExitCode = 77; + bool IgnoreTimeouts = true; + bool IgnoreOOMs = true; + bool IgnoreCrashes = false; + int MaxTotalTimeSec = 0; + int RssLimitMb = 0; + int MallocLimitMb = 0; + bool DoCrossOver = true; + int MutateDepth = 5; + bool ReduceDepth = false; + bool UseCounters = false; + bool UseMemmem = true; + bool UseCmp = false; + int UseValueProfile = false; + bool Shrink = false; + bool ReduceInputs = false; + int ReloadIntervalSec = 1; + bool ShuffleAtStartUp = true; + bool PreferSmall = true; + size_t MaxNumberOfRuns = -1L; + int ReportSlowUnits = 10; + bool OnlyASCII = false; + bool Entropic = false; + size_t EntropicFeatureFrequencyThreshold = 0xFF; + size_t EntropicNumberOfRarestFeatures = 100; + std::string OutputCorpus; + std::string ArtifactPrefix = "./"; + std::string ExactArtifactPath; + std::string ExitOnSrcPos; + std::string ExitOnItem; + std::string FocusFunction; + std::string DataFlowTrace; + std::string CollectDataFlow; + std::string FeaturesDir; + std::string StopFile; + bool SaveArtifacts = true; + bool PrintNEW = true; // Print a status line when new units are found; + bool PrintNewCovPcs = false; + int PrintNewCovFuncs = 0; + bool PrintFinalStats = false; + bool PrintCorpusStats = false; + bool PrintCoverage = false; + bool DumpCoverage = false; + bool DetectLeaks = true; + int PurgeAllocatorIntervalSec = 1; + int TraceMalloc = 0; + bool HandleAbrt = false; + bool HandleBus = false; + bool HandleFpe = false; + bool HandleIll = false; + bool HandleInt = false; + bool HandleSegv = false; + bool HandleTerm = false; + bool HandleXfsz = false; + bool HandleUsr1 = false; + bool HandleUsr2 = false; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_OPTIONS_H diff --git a/tools/fuzzing/libfuzzer/FuzzerPlatform.h b/tools/fuzzing/libfuzzer/FuzzerPlatform.h new file mode 100644 index 0000000000..8befdb882c --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerPlatform.h @@ -0,0 +1,163 @@ +//===-- FuzzerPlatform.h --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Common platform macros. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_PLATFORM_H +#define LLVM_FUZZER_PLATFORM_H + +// Platform detection. +#ifdef __linux__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 1 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __APPLE__ +#define LIBFUZZER_APPLE 1 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __NetBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 1 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __FreeBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 1 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __OpenBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 1 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif _WIN32 +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 1 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __Fuchsia__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 1 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __EMSCRIPTEN__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 1 +#else +#error "Support for your platform has not been implemented" +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +// MSVC compiler is being used. +#define LIBFUZZER_MSVC 1 +#else +#define LIBFUZZER_MSVC 0 +#endif + +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#define LIBFUZZER_POSIX \ + (LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD || \ + LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN) + +#ifdef __x86_64 +#if __has_attribute(target) +#define ATTRIBUTE_TARGET_POPCNT __attribute__((target("popcnt"))) +#else +#define ATTRIBUTE_TARGET_POPCNT +#endif +#else +#define ATTRIBUTE_TARGET_POPCNT +#endif + +#ifdef __clang__ // avoid gcc warning. +#if __has_attribute(no_sanitize) +#define ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) +#else +#define ATTRIBUTE_NO_SANITIZE_MEMORY +#endif +#define ALWAYS_INLINE __attribute__((always_inline)) +#else +#define ATTRIBUTE_NO_SANITIZE_MEMORY +#define ALWAYS_INLINE +#endif // __clang__ + +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_NO_SANITIZE_ADDRESS +#else +#define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +#endif + +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_ALIGNED(X) __declspec(align(X)) +#define ATTRIBUTE_INTERFACE __declspec(dllexport) +// This is used for __sancov_lowest_stack which is needed for +// -fsanitize-coverage=stack-depth. That feature is not yet available on +// Windows, so make the symbol static to avoid linking errors. +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC static +#define ATTRIBUTE_NOINLINE __declspec(noinline) +#else +#define ATTRIBUTE_ALIGNED(X) __attribute__((aligned(X))) +#define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ + ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) thread_local + +#define ATTRIBUTE_NOINLINE __attribute__((noinline)) +#endif + +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_ADDRESS +#elif __has_feature(memory_sanitizer) +#define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_MEMORY +#else +#define ATTRIBUTE_NO_SANITIZE_ALL +#endif +#else +#define ATTRIBUTE_NO_SANITIZE_ALL +#endif + +#endif // LLVM_FUZZER_PLATFORM_H diff --git a/tools/fuzzing/libfuzzer/FuzzerRandom.h b/tools/fuzzing/libfuzzer/FuzzerRandom.h new file mode 100644 index 0000000000..659283eee2 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerRandom.h @@ -0,0 +1,38 @@ +//===- FuzzerRandom.h - Internal header for the Fuzzer ----------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::Random +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_RANDOM_H +#define LLVM_FUZZER_RANDOM_H + +#include <random> + +namespace fuzzer { +class Random : public std::minstd_rand { + public: + Random(unsigned int seed) : std::minstd_rand(seed) {} + result_type operator()() { return this->std::minstd_rand::operator()(); } + size_t Rand() { return this->operator()(); } + size_t RandBool() { return Rand() % 2; } + size_t SkewTowardsLast(size_t n) { + size_t T = this->operator()(n * n); + size_t Res = sqrt(T); + return Res; + } + size_t operator()(size_t n) { return n ? Rand() % n : 0; } + intptr_t operator()(intptr_t From, intptr_t To) { + assert(From < To); + intptr_t RangeSize = To - From + 1; + return operator()(RangeSize) + From; + } +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_RANDOM_H diff --git a/tools/fuzzing/libfuzzer/FuzzerSHA1.cpp b/tools/fuzzing/libfuzzer/FuzzerSHA1.cpp new file mode 100644 index 0000000000..2005dc7003 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerSHA1.cpp @@ -0,0 +1,223 @@ +//===- FuzzerSHA1.h - Private copy of the SHA1 implementation ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// This code is taken from public domain +// (http://oauth.googlecode.com/svn/code/c/liboauth/src/sha1.c) +// and modified by adding anonymous namespace, adding an interface +// function fuzzer::ComputeSHA1() and removing unnecessary code. +// +// lib/Fuzzer can not use SHA1 implementation from openssl because +// openssl may not be available and because we may be fuzzing openssl itself. +// For the same reason we do not want to depend on SHA1 from LLVM tree. +//===----------------------------------------------------------------------===// + +#include "FuzzerSHA1.h" +#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" + +/* This code is public-domain - it is based on libcrypt + * placed in the public domain by Wei Dai and other contributors. + */ + +#include <iomanip> +#include <sstream> +#include <stdint.h> +#include <string.h> + +namespace { // Added for LibFuzzer + +#ifdef __BIG_ENDIAN__ +# define SHA_BIG_ENDIAN +// Windows is always little endian and MSVC doesn't have <endian.h> +#elif defined __LITTLE_ENDIAN__ || LIBFUZZER_WINDOWS +/* override */ +#elif defined __BYTE_ORDER +# if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define SHA_BIG_ENDIAN +# endif +#else // ! defined __LITTLE_ENDIAN__ +# include <endian.h> // machine/endian.h +# if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define SHA_BIG_ENDIAN +# endif +#endif + + +/* header */ + +#define HASH_LENGTH 20 +#define BLOCK_LENGTH 64 + +typedef struct sha1nfo { + uint32_t buffer[BLOCK_LENGTH/4]; + uint32_t state[HASH_LENGTH/4]; + uint32_t byteCount; + uint8_t bufferOffset; + uint8_t keyBuffer[BLOCK_LENGTH]; + uint8_t innerHash[HASH_LENGTH]; +} sha1nfo; + +/* public API - prototypes - TODO: doxygen*/ + +/** + */ +void sha1_init(sha1nfo *s); +/** + */ +void sha1_writebyte(sha1nfo *s, uint8_t data); +/** + */ +void sha1_write(sha1nfo *s, const char *data, size_t len); +/** + */ +uint8_t* sha1_result(sha1nfo *s); + + +/* code */ +#define SHA1_K0 0x5a827999 +#define SHA1_K20 0x6ed9eba1 +#define SHA1_K40 0x8f1bbcdc +#define SHA1_K60 0xca62c1d6 + +void sha1_init(sha1nfo *s) { + s->state[0] = 0x67452301; + s->state[1] = 0xefcdab89; + s->state[2] = 0x98badcfe; + s->state[3] = 0x10325476; + s->state[4] = 0xc3d2e1f0; + s->byteCount = 0; + s->bufferOffset = 0; +} + +uint32_t sha1_rol32(uint32_t number, uint8_t bits) { + return ((number << bits) | (number >> (32-bits))); +} + +void sha1_hashBlock(sha1nfo *s) { + uint8_t i; + uint32_t a,b,c,d,e,t; + + a=s->state[0]; + b=s->state[1]; + c=s->state[2]; + d=s->state[3]; + e=s->state[4]; + for (i=0; i<80; i++) { + if (i>=16) { + t = s->buffer[(i+13)&15] ^ s->buffer[(i+8)&15] ^ s->buffer[(i+2)&15] ^ s->buffer[i&15]; + s->buffer[i&15] = sha1_rol32(t,1); + } + if (i<20) { + t = (d ^ (b & (c ^ d))) + SHA1_K0; + } else if (i<40) { + t = (b ^ c ^ d) + SHA1_K20; + } else if (i<60) { + t = ((b & c) | (d & (b | c))) + SHA1_K40; + } else { + t = (b ^ c ^ d) + SHA1_K60; + } + t+=sha1_rol32(a,5) + e + s->buffer[i&15]; + e=d; + d=c; + c=sha1_rol32(b,30); + b=a; + a=t; + } + s->state[0] += a; + s->state[1] += b; + s->state[2] += c; + s->state[3] += d; + s->state[4] += e; +} + +void sha1_addUncounted(sha1nfo *s, uint8_t data) { + uint8_t * const b = (uint8_t*) s->buffer; +#ifdef SHA_BIG_ENDIAN + b[s->bufferOffset] = data; +#else + b[s->bufferOffset ^ 3] = data; +#endif + s->bufferOffset++; + if (s->bufferOffset == BLOCK_LENGTH) { + sha1_hashBlock(s); + s->bufferOffset = 0; + } +} + +void sha1_writebyte(sha1nfo *s, uint8_t data) { + ++s->byteCount; + sha1_addUncounted(s, data); +} + +void sha1_write(sha1nfo *s, const char *data, size_t len) { + for (;len--;) sha1_writebyte(s, (uint8_t) *data++); +} + +void sha1_pad(sha1nfo *s) { + // Implement SHA-1 padding (fips180-2 §5.1.1) + + // Pad with 0x80 followed by 0x00 until the end of the block + sha1_addUncounted(s, 0x80); + while (s->bufferOffset != 56) sha1_addUncounted(s, 0x00); + + // Append length in the last 8 bytes + sha1_addUncounted(s, 0); // We're only using 32 bit lengths + sha1_addUncounted(s, 0); // But SHA-1 supports 64 bit lengths + sha1_addUncounted(s, 0); // So zero pad the top bits + sha1_addUncounted(s, s->byteCount >> 29); // Shifting to multiply by 8 + sha1_addUncounted(s, s->byteCount >> 21); // as SHA-1 supports bitstreams as well as + sha1_addUncounted(s, s->byteCount >> 13); // byte. + sha1_addUncounted(s, s->byteCount >> 5); + sha1_addUncounted(s, s->byteCount << 3); +} + +uint8_t* sha1_result(sha1nfo *s) { + // Pad to complete the last block + sha1_pad(s); + +#ifndef SHA_BIG_ENDIAN + // Swap byte order back + int i; + for (i=0; i<5; i++) { + s->state[i]= + (((s->state[i])<<24)& 0xff000000) + | (((s->state[i])<<8) & 0x00ff0000) + | (((s->state[i])>>8) & 0x0000ff00) + | (((s->state[i])>>24)& 0x000000ff); + } +#endif + + // Return pointer to hash (20 characters) + return (uint8_t*) s->state; +} + +} // namespace; Added for LibFuzzer + +namespace fuzzer { + +// The rest is added for LibFuzzer +void ComputeSHA1(const uint8_t *Data, size_t Len, uint8_t *Out) { + sha1nfo s; + sha1_init(&s); + sha1_write(&s, (const char*)Data, Len); + memcpy(Out, sha1_result(&s), HASH_LENGTH); +} + +std::string Sha1ToString(const uint8_t Sha1[kSHA1NumBytes]) { + std::stringstream SS; + for (int i = 0; i < kSHA1NumBytes; i++) + SS << std::hex << std::setfill('0') << std::setw(2) << (unsigned)Sha1[i]; + return SS.str(); +} + +std::string Hash(const Unit &U) { + uint8_t Hash[kSHA1NumBytes]; + ComputeSHA1(U.data(), U.size(), Hash); + return Sha1ToString(Hash); +} + +} diff --git a/tools/fuzzing/libfuzzer/FuzzerSHA1.h b/tools/fuzzing/libfuzzer/FuzzerSHA1.h new file mode 100644 index 0000000000..05cbacda87 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerSHA1.h @@ -0,0 +1,32 @@ +//===- FuzzerSHA1.h - Internal header for the SHA1 utils --------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// SHA1 utils. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_SHA1_H +#define LLVM_FUZZER_SHA1_H + +#include "FuzzerDefs.h" +#include <cstddef> +#include <stdint.h> + +namespace fuzzer { + +// Private copy of SHA1 implementation. +static const int kSHA1NumBytes = 20; + +// Computes SHA1 hash of 'Len' bytes in 'Data', writes kSHA1NumBytes to 'Out'. +void ComputeSHA1(const uint8_t *Data, size_t Len, uint8_t *Out); + +std::string Sha1ToString(const uint8_t Sha1[kSHA1NumBytes]); + +std::string Hash(const Unit &U); + +} // namespace fuzzer + +#endif // LLVM_FUZZER_SHA1_H diff --git a/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp b/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp new file mode 100644 index 0000000000..fbceda39bc --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp @@ -0,0 +1,657 @@ +//===- FuzzerTracePC.cpp - PC tracing--------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Trace PCs. +// This module implements __sanitizer_cov_trace_pc_guard[_init], +// the callback required for -fsanitize-coverage=trace-pc-guard instrumentation. +// +//===----------------------------------------------------------------------===// + +#include "FuzzerTracePC.h" +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" +#include "FuzzerCorpus.h" +#include "FuzzerDefs.h" +#include "FuzzerDictionary.h" +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "FuzzerPlatform.h" +#include "FuzzerUtil.h" +#include "FuzzerValueBitMap.h" +#include <set> + +// Used by -fsanitize-coverage=stack-depth to track stack depth +ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC uintptr_t __sancov_lowest_stack; + +namespace fuzzer { + +TracePC TPC; + +size_t TracePC::GetTotalPCCoverage() { + return ObservedPCs.size(); +} + + +void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) { + if (Start == Stop) return; + if (NumModules && + Modules[NumModules - 1].Start() == Start) + return; + assert(NumModules < + sizeof(Modules) / sizeof(Modules[0])); + auto &M = Modules[NumModules++]; + uint8_t *AlignedStart = RoundUpByPage(Start); + uint8_t *AlignedStop = RoundDownByPage(Stop); + size_t NumFullPages = AlignedStop > AlignedStart ? + (AlignedStop - AlignedStart) / PageSize() : 0; + bool NeedFirst = Start < AlignedStart || !NumFullPages; + bool NeedLast = Stop > AlignedStop && AlignedStop >= AlignedStart; + M.NumRegions = NumFullPages + NeedFirst + NeedLast;; + assert(M.NumRegions > 0); + M.Regions = new Module::Region[M.NumRegions]; + assert(M.Regions); + size_t R = 0; + if (NeedFirst) + M.Regions[R++] = {Start, std::min(Stop, AlignedStart), true, false}; + for (uint8_t *P = AlignedStart; P < AlignedStop; P += PageSize()) + M.Regions[R++] = {P, P + PageSize(), true, true}; + if (NeedLast) + M.Regions[R++] = {AlignedStop, Stop, true, false}; + assert(R == M.NumRegions); + assert(M.Size() == (size_t)(Stop - Start)); + assert(M.Stop() == Stop); + assert(M.Start() == Start); + NumInline8bitCounters += M.Size(); +} + +void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) { + const PCTableEntry *B = reinterpret_cast<const PCTableEntry *>(Start); + const PCTableEntry *E = reinterpret_cast<const PCTableEntry *>(Stop); + if (NumPCTables && ModulePCTable[NumPCTables - 1].Start == B) return; + assert(NumPCTables < sizeof(ModulePCTable) / sizeof(ModulePCTable[0])); + ModulePCTable[NumPCTables++] = {B, E}; + NumPCsInPCTables += E - B; +} + +void TracePC::PrintModuleInfo() { + if (NumModules) { + Printf("INFO: Loaded %zd modules (%zd inline 8-bit counters): ", + NumModules, NumInline8bitCounters); + for (size_t i = 0; i < NumModules; i++) + Printf("%zd [%p, %p), ", Modules[i].Size(), Modules[i].Start(), + Modules[i].Stop()); + Printf("\n"); + } + if (NumPCTables) { + Printf("INFO: Loaded %zd PC tables (%zd PCs): ", NumPCTables, + NumPCsInPCTables); + for (size_t i = 0; i < NumPCTables; i++) { + Printf("%zd [%p,%p), ", ModulePCTable[i].Stop - ModulePCTable[i].Start, + ModulePCTable[i].Start, ModulePCTable[i].Stop); + } + Printf("\n"); + + if (NumInline8bitCounters && NumInline8bitCounters != NumPCsInPCTables) { + Printf("ERROR: The size of coverage PC tables does not match the\n" + "number of instrumented PCs. This might be a compiler bug,\n" + "please contact the libFuzzer developers.\n" + "Also check https://bugs.llvm.org/show_bug.cgi?id=34636\n" + "for possible workarounds (tl;dr: don't use the old GNU ld)\n"); + _Exit(1); + } + } + if (size_t NumExtraCounters = ExtraCountersEnd() - ExtraCountersBegin()) + Printf("INFO: %zd Extra Counters\n", NumExtraCounters); +} + +ATTRIBUTE_NO_SANITIZE_ALL +void TracePC::HandleCallerCallee(uintptr_t Caller, uintptr_t Callee) { + const uintptr_t kBits = 12; + const uintptr_t kMask = (1 << kBits) - 1; + uintptr_t Idx = (Caller & kMask) | ((Callee & kMask) << kBits); + ValueProfileMap.AddValueModPrime(Idx); +} + +/// \return the address of the previous instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.h` +inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { +#if defined(__arm__) + // T32 (Thumb) branch instructions might be 16 or 32 bit long, + // so we return (pc-2) in that case in order to be safe. + // For A32 mode we return (pc-4) because all instructions are 32 bit long. + return (PC - 3) & (~1); +#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__) + // PCs are always 4 byte aligned. + return PC - 4; +#elif defined(__sparc__) || defined(__mips__) + return PC - 8; +#else + return PC - 1; +#endif +} + +/// \return the address of the next instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.cpp` +ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { +#if defined(__mips__) + return PC + 8; +#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ + defined(__aarch64__) + return PC + 4; +#else + return PC + 1; +#endif +} + +void TracePC::UpdateObservedPCs() { + Vector<uintptr_t> CoveredFuncs; + auto ObservePC = [&](const PCTableEntry *TE) { + if (ObservedPCs.insert(TE).second && DoPrintNewPCs) { + PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", + GetNextInstructionPc(TE->PC)); + Printf("\n"); + } + }; + + auto Observe = [&](const PCTableEntry *TE) { + if (PcIsFuncEntry(TE)) + if (++ObservedFuncs[TE->PC] == 1 && NumPrintNewFuncs) + CoveredFuncs.push_back(TE->PC); + ObservePC(TE); + }; + + if (NumPCsInPCTables) { + if (NumInline8bitCounters == NumPCsInPCTables) { + for (size_t i = 0; i < NumModules; i++) { + auto &M = Modules[i]; + assert(M.Size() == + (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); + for (size_t r = 0; r < M.NumRegions; r++) { + auto &R = M.Regions[r]; + if (!R.Enabled) continue; + for (uint8_t *P = R.Start; P < R.Stop; P++) + if (*P) + Observe(&ModulePCTable[i].Start[M.Idx(P)]); + } + } + } + } + + for (size_t i = 0, N = Min(CoveredFuncs.size(), NumPrintNewFuncs); i < N; + i++) { + Printf("\tNEW_FUNC[%zd/%zd]: ", i + 1, CoveredFuncs.size()); + PrintPC("%p %F %L", "%p", GetNextInstructionPc(CoveredFuncs[i])); + Printf("\n"); + } +} + +uintptr_t TracePC::PCTableEntryIdx(const PCTableEntry *TE) { + size_t TotalTEs = 0; + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + if (TE >= M.Start && TE < M.Stop) + return TotalTEs + TE - M.Start; + TotalTEs += M.Stop - M.Start; + } + assert(0); + return 0; +} + +const TracePC::PCTableEntry *TracePC::PCTableEntryByIdx(uintptr_t Idx) { + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + size_t Size = M.Stop - M.Start; + if (Idx < Size) return &M.Start[Idx]; + Idx -= Size; + } + return nullptr; +} + +static std::string GetModuleName(uintptr_t PC) { + char ModulePathRaw[4096] = ""; // What's PATH_MAX in portable C++? + void *OffsetRaw = nullptr; + if (!EF->__sanitizer_get_module_and_offset_for_pc( + reinterpret_cast<void *>(PC), ModulePathRaw, + sizeof(ModulePathRaw), &OffsetRaw)) + return ""; + return ModulePathRaw; +} + +template<class CallBack> +void TracePC::IterateCoveredFunctions(CallBack CB) { + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + assert(M.Start < M.Stop); + auto ModuleName = GetModuleName(M.Start->PC); + for (auto NextFE = M.Start; NextFE < M.Stop; ) { + auto FE = NextFE; + assert(PcIsFuncEntry(FE) && "Not a function entry point"); + do { + NextFE++; + } while (NextFE < M.Stop && !(PcIsFuncEntry(NextFE))); + CB(FE, NextFE, ObservedFuncs[FE->PC]); + } + } +} + +int TracePC::SetFocusFunction(const std::string &FuncName) { + // This function should be called once. + assert(!FocusFunctionCounterPtr); + // "auto" is not a valid function name. If this function is called with "auto" + // that means the auto focus functionality failed. + if (FuncName.empty() || FuncName == "auto") + return 0; + for (size_t M = 0; M < NumModules; M++) { + auto &PCTE = ModulePCTable[M]; + size_t N = PCTE.Stop - PCTE.Start; + for (size_t I = 0; I < N; I++) { + if (!(PcIsFuncEntry(&PCTE.Start[I]))) continue; // not a function entry. + auto Name = DescribePC("%F", GetNextInstructionPc(PCTE.Start[I].PC)); + if (Name[0] == 'i' && Name[1] == 'n' && Name[2] == ' ') + Name = Name.substr(3, std::string::npos); + if (FuncName != Name) continue; + Printf("INFO: Focus function is set to '%s'\n", Name.c_str()); + FocusFunctionCounterPtr = Modules[M].Start() + I; + return 0; + } + } + + Printf("ERROR: Failed to set focus function. Make sure the function name is " + "valid (%s) and symbolization is enabled.\n", FuncName.c_str()); + return 1; +} + +bool TracePC::ObservedFocusFunction() { + return FocusFunctionCounterPtr && *FocusFunctionCounterPtr; +} + +void TracePC::PrintCoverage() { + if (!EF->__sanitizer_symbolize_pc || + !EF->__sanitizer_get_module_and_offset_for_pc) { + Printf("INFO: __sanitizer_symbolize_pc or " + "__sanitizer_get_module_and_offset_for_pc is not available," + " not printing coverage\n"); + return; + } + Printf("COVERAGE:\n"); + auto CoveredFunctionCallback = [&](const PCTableEntry *First, + const PCTableEntry *Last, + uintptr_t Counter) { + assert(First < Last); + auto VisualizePC = GetNextInstructionPc(First->PC); + std::string FileStr = DescribePC("%s", VisualizePC); + if (!IsInterestingCoverageFile(FileStr)) + return; + std::string FunctionStr = DescribePC("%F", VisualizePC); + if (FunctionStr.find("in ") == 0) + FunctionStr = FunctionStr.substr(3); + std::string LineStr = DescribePC("%l", VisualizePC); + size_t NumEdges = Last - First; + Vector<uintptr_t> UncoveredPCs; + for (auto TE = First; TE < Last; TE++) + if (!ObservedPCs.count(TE)) + UncoveredPCs.push_back(TE->PC); + Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter); + Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges); + Printf(" %s %s:%s\n", FunctionStr.c_str(), FileStr.c_str(), + LineStr.c_str()); + if (Counter) + for (auto PC : UncoveredPCs) + Printf(" UNCOVERED_PC: %s\n", + DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); + }; + + IterateCoveredFunctions(CoveredFunctionCallback); +} + +// Value profile. +// We keep track of various values that affect control flow. +// These values are inserted into a bit-set-based hash map. +// Every new bit in the map is treated as a new coverage. +// +// For memcmp/strcmp/etc the interesting value is the length of the common +// prefix of the parameters. +// For cmp instructions the interesting value is a XOR of the parameters. +// The interesting value is mixed up with the PC and is then added to the map. + +ATTRIBUTE_NO_SANITIZE_ALL +void TracePC::AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, + size_t n, bool StopAtZero) { + if (!n) return; + size_t Len = std::min(n, Word::GetMaxSize()); + const uint8_t *A1 = reinterpret_cast<const uint8_t *>(s1); + const uint8_t *A2 = reinterpret_cast<const uint8_t *>(s2); + uint8_t B1[Word::kMaxSize]; + uint8_t B2[Word::kMaxSize]; + // Copy the data into locals in this non-msan-instrumented function + // to avoid msan complaining further. + size_t Hash = 0; // Compute some simple hash of both strings. + for (size_t i = 0; i < Len; i++) { + B1[i] = A1[i]; + B2[i] = A2[i]; + size_t T = B1[i]; + Hash ^= (T << 8) | B2[i]; + } + size_t I = 0; + uint8_t HammingDistance = 0; + for (; I < Len; I++) { + if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) { + HammingDistance = Popcountll(B1[I] ^ B2[I]); + break; + } + } + size_t PC = reinterpret_cast<size_t>(caller_pc); + size_t Idx = (PC & 4095) | (I << 12); + Idx += HammingDistance; + ValueProfileMap.AddValue(Idx); + TORCW.Insert(Idx ^ Hash, Word(B1, Len), Word(B2, Len)); +} + +template <class T> +ATTRIBUTE_TARGET_POPCNT ALWAYS_INLINE +ATTRIBUTE_NO_SANITIZE_ALL +void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) { + uint64_t ArgXor = Arg1 ^ Arg2; + if (sizeof(T) == 4) + TORC4.Insert(ArgXor, Arg1, Arg2); + else if (sizeof(T) == 8) + TORC8.Insert(ArgXor, Arg1, Arg2); + uint64_t HammingDistance = Popcountll(ArgXor); // [0,64] + uint64_t AbsoluteDistance = (Arg1 == Arg2 ? 0 : Clzll(Arg1 - Arg2) + 1); + ValueProfileMap.AddValue(PC * 128 + HammingDistance); + ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance); +} + +static size_t InternalStrnlen(const char *S, size_t MaxLen) { + size_t Len = 0; + for (; Len < MaxLen && S[Len]; Len++) {} + return Len; +} + +// Finds min of (strlen(S1), strlen(S2)). +// Needed bacause one of these strings may actually be non-zero terminated. +static size_t InternalStrnlen2(const char *S1, const char *S2) { + size_t Len = 0; + for (; S1[Len] && S2[Len]; Len++) {} + return Len; +} + +void TracePC::ClearInlineCounters() { + IterateCounterRegions([](const Module::Region &R){ + if (R.Enabled) + memset(R.Start, 0, R.Stop - R.Start); + }); +} + +ATTRIBUTE_NO_SANITIZE_ALL +void TracePC::RecordInitialStack() { + int stack; + __sancov_lowest_stack = InitialStack = reinterpret_cast<uintptr_t>(&stack); +} + +uintptr_t TracePC::GetMaxStackOffset() const { + return InitialStack - __sancov_lowest_stack; // Stack grows down +} + +void WarnAboutDeprecatedInstrumentation(const char *flag) { + // Use RawPrint because Printf cannot be used on Windows before OutputFile is + // initialized. + RawPrint(flag); + RawPrint( + " is no longer supported by libFuzzer.\n" + "Please either migrate to a compiler that supports -fsanitize=fuzzer\n" + "or use an older version of libFuzzer\n"); + exit(1); +} + +} // namespace fuzzer + +extern "C" { +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); +} + +// Best-effort support for -fsanitize-coverage=trace-pc, which is available +// in both Clang and GCC. +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +void __sanitizer_cov_trace_pc() { + fuzzer::WarnAboutDeprecatedInstrumentation("-fsanitize-coverage=trace-pc"); +} + +ATTRIBUTE_INTERFACE +void __sanitizer_cov_trace_pc_guard_init(uint32_t *Start, uint32_t *Stop) { + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); +} + +ATTRIBUTE_INTERFACE +void __sanitizer_cov_8bit_counters_init(uint8_t *Start, uint8_t *Stop) { + fuzzer::TPC.HandleInline8bitCountersInit(Start, Stop); +} + +ATTRIBUTE_INTERFACE +void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + const uintptr_t *pcs_end) { + fuzzer::TPC.HandlePCsInit(pcs_beg, pcs_end); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +void __sanitizer_cov_trace_pc_indir(uintptr_t Callee) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCallerCallee(PC, Callee); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_cmp8(uint64_t Arg1, uint64_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +// Now the __sanitizer_cov_trace_const_cmp[1248] callbacks just mimic +// the behaviour of __sanitizer_cov_trace_cmp[1248] ones. This, however, +// should be changed later to make full use of instrumentation. +void __sanitizer_cov_trace_const_cmp8(uint64_t Arg1, uint64_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_cmp4(uint32_t Arg1, uint32_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_const_cmp4(uint32_t Arg1, uint32_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_cmp2(uint16_t Arg1, uint16_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_const_cmp2(uint16_t Arg1, uint16_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_cmp1(uint8_t Arg1, uint8_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_const_cmp1(uint8_t Arg1, uint8_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases) { + uint64_t N = Cases[0]; + uint64_t ValSizeInBits = Cases[1]; + uint64_t *Vals = Cases + 2; + // Skip the most common and the most boring case: all switch values are small. + // We may want to skip this at compile-time, but it will make the + // instrumentation less general. + if (Vals[N - 1] < 256) + return; + // Also skip small inputs values, they won't give good signal. + if (Val < 256) + return; + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + size_t i; + uint64_t Smaller = 0; + uint64_t Larger = ~(uint64_t)0; + // Find two switch values such that Smaller < Val < Larger. + // Use 0 and 0xfff..f as the defaults. + for (i = 0; i < N; i++) { + if (Val < Vals[i]) { + Larger = Vals[i]; + break; + } + if (Val > Vals[i]) Smaller = Vals[i]; + } + + // Apply HandleCmp to {Val,Smaller} and {Val, Larger}, + // use i as the PC modifier for HandleCmp. + if (ValSizeInBits == 16) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast<uint16_t>(Val), + (uint16_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast<uint16_t>(Val), + (uint16_t)(Larger)); + } else if (ValSizeInBits == 32) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast<uint32_t>(Val), + (uint32_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast<uint32_t>(Val), + (uint32_t)(Larger)); + } else { + fuzzer::TPC.HandleCmp(PC + 2*i, Val, Smaller); + fuzzer::TPC.HandleCmp(PC + 2*i + 1, Val, Larger); + } +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_div4(uint32_t Val) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Val, (uint32_t)0); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_div8(uint64_t Val) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Val, (uint64_t)0); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_gep(uintptr_t Idx) { + uintptr_t PC = reinterpret_cast<uintptr_t>(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Idx, (uintptr_t)0); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *s1, + const void *s2, size_t n, int result) { + if (!fuzzer::RunningUserCallback) return; + if (result == 0) return; // No reason to mutate. + if (n <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/false); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1, + const char *s2, size_t n, int result) { + if (!fuzzer::RunningUserCallback) return; + if (result == 0) return; // No reason to mutate. + size_t Len1 = fuzzer::InternalStrnlen(s1, n); + size_t Len2 = fuzzer::InternalStrnlen(s2, n); + n = std::min(n, Len1); + n = std::min(n, Len2); + if (n <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/true); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcmp(void *caller_pc, const char *s1, + const char *s2, int result) { + if (!fuzzer::RunningUserCallback) return; + if (result == 0) return; // No reason to mutate. + size_t N = fuzzer::InternalStrnlen2(s1, s2); + if (N <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, N, /*StopAtZero*/true); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result) { + if (!fuzzer::RunningUserCallback) return; + return __sanitizer_weak_hook_strncmp(called_pc, s1, s2, n, result); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, + const char *s2, int result) { + if (!fuzzer::RunningUserCallback) return; + return __sanitizer_weak_hook_strcmp(called_pc, s1, s2, result); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, + const char *s2, char *result) { + if (!fuzzer::RunningUserCallback) return; + fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, + const char *s2, char *result) { + if (!fuzzer::RunningUserCallback) return; + fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_memmem(void *called_pc, const void *s1, size_t len1, + const void *s2, size_t len2, void *result) { + if (!fuzzer::RunningUserCallback) return; + fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), len2); +} +} // extern "C" diff --git a/tools/fuzzing/libfuzzer/FuzzerTracePC.h b/tools/fuzzing/libfuzzer/FuzzerTracePC.h new file mode 100644 index 0000000000..b46ebb909d --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerTracePC.h @@ -0,0 +1,289 @@ +//===- FuzzerTracePC.h - Internal header for the Fuzzer ---------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::TracePC +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_TRACE_PC +#define LLVM_FUZZER_TRACE_PC + +#include "FuzzerDefs.h" +#include "FuzzerDictionary.h" +#include "FuzzerValueBitMap.h" + +#include <set> +#include <unordered_map> + +namespace fuzzer { + +// TableOfRecentCompares (TORC) remembers the most recently performed +// comparisons of type T. +// We record the arguments of CMP instructions in this table unconditionally +// because it seems cheaper this way than to compute some expensive +// conditions inside __sanitizer_cov_trace_cmp*. +// After the unit has been executed we may decide to use the contents of +// this table to populate a Dictionary. +template<class T, size_t kSizeT> +struct TableOfRecentCompares { + static const size_t kSize = kSizeT; + struct Pair { + T A, B; + }; + ATTRIBUTE_NO_SANITIZE_ALL + void Insert(size_t Idx, const T &Arg1, const T &Arg2) { + Idx = Idx % kSize; + Table[Idx].A = Arg1; + Table[Idx].B = Arg2; + } + + Pair Get(size_t I) { return Table[I % kSize]; } + + Pair Table[kSize]; +}; + +template <size_t kSizeT> +struct MemMemTable { + static const size_t kSize = kSizeT; + Word MemMemWords[kSize]; + Word EmptyWord; + + void Add(const uint8_t *Data, size_t Size) { + if (Size <= 2) return; + Size = std::min(Size, Word::GetMaxSize()); + size_t Idx = SimpleFastHash(Data, Size) % kSize; + MemMemWords[Idx].Set(Data, Size); + } + const Word &Get(size_t Idx) { + for (size_t i = 0; i < kSize; i++) { + const Word &W = MemMemWords[(Idx + i) % kSize]; + if (W.size()) return W; + } + EmptyWord.Set(nullptr, 0); + return EmptyWord; + } +}; + +class TracePC { + public: + void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop); + void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop); + void HandleCallerCallee(uintptr_t Caller, uintptr_t Callee); + template <class T> void HandleCmp(uintptr_t PC, T Arg1, T Arg2); + size_t GetTotalPCCoverage(); + void SetUseCounters(bool UC) { UseCounters = UC; } + void SetUseValueProfileMask(uint32_t VPMask) { UseValueProfileMask = VPMask; } + void SetPrintNewPCs(bool P) { DoPrintNewPCs = P; } + void SetPrintNewFuncs(size_t P) { NumPrintNewFuncs = P; } + void UpdateObservedPCs(); + template <class Callback> void CollectFeatures(Callback CB) const; + + void ResetMaps() { + ValueProfileMap.Reset(); + ClearExtraCounters(); + ClearInlineCounters(); + } + + void ClearInlineCounters(); + + void UpdateFeatureSet(size_t CurrentElementIdx, size_t CurrentElementSize); + void PrintFeatureSet(); + + void PrintModuleInfo(); + + void PrintCoverage(); + + template<class CallBack> + void IterateCoveredFunctions(CallBack CB); + + void AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, + size_t n, bool StopAtZero); + + TableOfRecentCompares<uint32_t, 32> TORC4; + TableOfRecentCompares<uint64_t, 32> TORC8; + TableOfRecentCompares<Word, 32> TORCW; + MemMemTable<1024> MMT; + + void RecordInitialStack(); + uintptr_t GetMaxStackOffset() const; + + template<class CallBack> + void ForEachObservedPC(CallBack CB) { + for (auto PC : ObservedPCs) + CB(PC); + } + + int SetFocusFunction(const std::string &FuncName); + bool ObservedFocusFunction(); + + struct PCTableEntry { + uintptr_t PC, PCFlags; + }; + + uintptr_t PCTableEntryIdx(const PCTableEntry *TE); + const PCTableEntry *PCTableEntryByIdx(uintptr_t Idx); + static uintptr_t GetNextInstructionPc(uintptr_t PC); + bool PcIsFuncEntry(const PCTableEntry *TE) { return TE->PCFlags & 1; } + +private: + bool UseCounters = false; + uint32_t UseValueProfileMask = false; + bool DoPrintNewPCs = false; + size_t NumPrintNewFuncs = 0; + + // Module represents the array of 8-bit counters split into regions + // such that every region, except maybe the first and the last one, is one + // full page. + struct Module { + struct Region { + uint8_t *Start, *Stop; + bool Enabled; + bool OneFullPage; + }; + Region *Regions; + size_t NumRegions; + uint8_t *Start() { return Regions[0].Start; } + uint8_t *Stop() { return Regions[NumRegions - 1].Stop; } + size_t Size() { return Stop() - Start(); } + size_t Idx(uint8_t *P) { + assert(P >= Start() && P < Stop()); + return P - Start(); + } + }; + + Module Modules[4096]; + size_t NumModules; // linker-initialized. + size_t NumInline8bitCounters; + + template <class Callback> + void IterateCounterRegions(Callback CB) { + for (size_t m = 0; m < NumModules; m++) + for (size_t r = 0; r < Modules[m].NumRegions; r++) + CB(Modules[m].Regions[r]); + } + + struct { const PCTableEntry *Start, *Stop; } ModulePCTable[4096]; + size_t NumPCTables; + size_t NumPCsInPCTables; + + Set<const PCTableEntry*> ObservedPCs; + std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter. + + uint8_t *FocusFunctionCounterPtr = nullptr; + + ValueBitMap ValueProfileMap; + uintptr_t InitialStack; +}; + +template <class Callback> +// void Callback(size_t FirstFeature, size_t Idx, uint8_t Value); +ATTRIBUTE_NO_SANITIZE_ALL +size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, + size_t FirstFeature, Callback Handle8bitCounter) { + typedef uintptr_t LargeType; + const size_t Step = sizeof(LargeType) / sizeof(uint8_t); + const size_t StepMask = Step - 1; + auto P = Begin; + // Iterate by 1 byte until either the alignment boundary or the end. + for (; reinterpret_cast<uintptr_t>(P) & StepMask && P < End; P++) + if (uint8_t V = *P) + Handle8bitCounter(FirstFeature, P - Begin, V); + + // Iterate by Step bytes at a time. + for (; P < End; P += Step) + if (LargeType Bundle = *reinterpret_cast<const LargeType *>(P)) + for (size_t I = 0; I < Step; I++, Bundle >>= 8) + if (uint8_t V = Bundle & 0xff) + Handle8bitCounter(FirstFeature, P - Begin + I, V); + + // Iterate by 1 byte until the end. + for (; P < End; P++) + if (uint8_t V = *P) + Handle8bitCounter(FirstFeature, P - Begin, V); + return End - Begin; +} + +// Given a non-zero Counter returns a number in the range [0,7]. +template<class T> +unsigned CounterToFeature(T Counter) { + // Returns a feature number by placing Counters into buckets as illustrated + // below. + // + // Counter bucket: [1] [2] [3] [4-7] [8-15] [16-31] [32-127] [128+] + // Feature number: 0 1 2 3 4 5 6 7 + // + // This is a heuristic taken from AFL (see + // http://lcamtuf.coredump.cx/afl/technical_details.txt). + // + // This implementation may change in the future so clients should + // not rely on it. + assert(Counter); + unsigned Bit = 0; + /**/ if (Counter >= 128) Bit = 7; + else if (Counter >= 32) Bit = 6; + else if (Counter >= 16) Bit = 5; + else if (Counter >= 8) Bit = 4; + else if (Counter >= 4) Bit = 3; + else if (Counter >= 3) Bit = 2; + else if (Counter >= 2) Bit = 1; + return Bit; +} + +template <class Callback> // void Callback(size_t Feature) +ATTRIBUTE_NO_SANITIZE_ADDRESS +ATTRIBUTE_NOINLINE +void TracePC::CollectFeatures(Callback HandleFeature) const { + auto Handle8bitCounter = [&](size_t FirstFeature, + size_t Idx, uint8_t Counter) { + if (UseCounters) + HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Counter)); + else + HandleFeature(FirstFeature + Idx); + }; + + size_t FirstFeature = 0; + + for (size_t i = 0; i < NumModules; i++) { + for (size_t r = 0; r < Modules[i].NumRegions; r++) { + if (!Modules[i].Regions[r].Enabled) continue; + FirstFeature += 8 * ForEachNonZeroByte(Modules[i].Regions[r].Start, + Modules[i].Regions[r].Stop, + FirstFeature, Handle8bitCounter); + } + } + + FirstFeature += + 8 * ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), + FirstFeature, Handle8bitCounter); + + if (UseValueProfileMask) { + ValueProfileMap.ForEach([&](size_t Idx) { + HandleFeature(FirstFeature + Idx); + }); + FirstFeature += ValueProfileMap.SizeInBits(); + } + + // Step function, grows similar to 8 * Log_2(A). + auto StackDepthStepFunction = [](uint32_t A) -> uint32_t { + if (!A) return A; + uint32_t Log2 = Log(A); + if (Log2 < 3) return A; + Log2 -= 3; + return (Log2 + 1) * 8 + ((A >> Log2) & 7); + }; + assert(StackDepthStepFunction(1024) == 64); + assert(StackDepthStepFunction(1024 * 4) == 80); + assert(StackDepthStepFunction(1024 * 1024) == 144); + + if (auto MaxStackOffset = GetMaxStackOffset()) + HandleFeature(FirstFeature + StackDepthStepFunction(MaxStackOffset / 8)); +} + +extern TracePC TPC; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_TRACE_PC diff --git a/tools/fuzzing/libfuzzer/FuzzerUtil.cpp b/tools/fuzzing/libfuzzer/FuzzerUtil.cpp new file mode 100644 index 0000000000..7eecb68d07 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtil.cpp @@ -0,0 +1,236 @@ +//===- FuzzerUtil.cpp - Misc utils ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils. +//===----------------------------------------------------------------------===// + +#include "FuzzerUtil.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include <cassert> +#include <chrono> +#include <cstring> +#include <errno.h> +#include <mutex> +#include <signal.h> +#include <sstream> +#include <stdio.h> +#include <sys/types.h> +#include <thread> + +namespace fuzzer { + +void PrintHexArray(const uint8_t *Data, size_t Size, + const char *PrintAfter) { + for (size_t i = 0; i < Size; i++) + Printf("0x%x,", (unsigned)Data[i]); + Printf("%s", PrintAfter); +} + +void Print(const Unit &v, const char *PrintAfter) { + PrintHexArray(v.data(), v.size(), PrintAfter); +} + +void PrintASCIIByte(uint8_t Byte) { + if (Byte == '\\') + Printf("\\\\"); + else if (Byte == '"') + Printf("\\\""); + else if (Byte >= 32 && Byte < 127) + Printf("%c", Byte); + else + Printf("\\x%02x", Byte); +} + +void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) { + for (size_t i = 0; i < Size; i++) + PrintASCIIByte(Data[i]); + Printf("%s", PrintAfter); +} + +void PrintASCII(const Unit &U, const char *PrintAfter) { + PrintASCII(U.data(), U.size(), PrintAfter); +} + +bool ToASCII(uint8_t *Data, size_t Size) { + bool Changed = false; + for (size_t i = 0; i < Size; i++) { + uint8_t &X = Data[i]; + auto NewX = X; + NewX &= 127; + if (!isspace(NewX) && !isprint(NewX)) + NewX = ' '; + Changed |= NewX != X; + X = NewX; + } + return Changed; +} + +bool IsASCII(const Unit &U) { return IsASCII(U.data(), U.size()); } + +bool IsASCII(const uint8_t *Data, size_t Size) { + for (size_t i = 0; i < Size; i++) + if (!(isprint(Data[i]) || isspace(Data[i]))) return false; + return true; +} + +bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) { + U->clear(); + if (Str.empty()) return false; + size_t L = 0, R = Str.size() - 1; // We are parsing the range [L,R]. + // Skip spaces from both sides. + while (L < R && isspace(Str[L])) L++; + while (R > L && isspace(Str[R])) R--; + if (R - L < 2) return false; + // Check the closing " + if (Str[R] != '"') return false; + R--; + // Find the opening " + while (L < R && Str[L] != '"') L++; + if (L >= R) return false; + assert(Str[L] == '\"'); + L++; + assert(L <= R); + for (size_t Pos = L; Pos <= R; Pos++) { + uint8_t V = (uint8_t)Str[Pos]; + if (!isprint(V) && !isspace(V)) return false; + if (V =='\\') { + // Handle '\\' + if (Pos + 1 <= R && (Str[Pos + 1] == '\\' || Str[Pos + 1] == '"')) { + U->push_back(Str[Pos + 1]); + Pos++; + continue; + } + // Handle '\xAB' + if (Pos + 3 <= R && Str[Pos + 1] == 'x' + && isxdigit(Str[Pos + 2]) && isxdigit(Str[Pos + 3])) { + char Hex[] = "0xAA"; + Hex[2] = Str[Pos + 2]; + Hex[3] = Str[Pos + 3]; + U->push_back(strtol(Hex, nullptr, 16)); + Pos += 3; + continue; + } + return false; // Invalid escape. + } else { + // Any other character. + U->push_back(V); + } + } + return true; +} + +bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units) { + if (Text.empty()) { + Printf("ParseDictionaryFile: file does not exist or is empty\n"); + return false; + } + std::istringstream ISS(Text); + Units->clear(); + Unit U; + int LineNo = 0; + std::string S; + while (std::getline(ISS, S, '\n')) { + LineNo++; + size_t Pos = 0; + while (Pos < S.size() && isspace(S[Pos])) Pos++; // Skip spaces. + if (Pos == S.size()) continue; // Empty line. + if (S[Pos] == '#') continue; // Comment line. + if (ParseOneDictionaryEntry(S, &U)) { + Units->push_back(U); + } else { + Printf("ParseDictionaryFile: error in line %d\n\t\t%s\n", LineNo, + S.c_str()); + return false; + } + } + return true; +} + +// Code duplicated (and tested) in llvm/include/llvm/Support/Base64.h +std::string Base64(const Unit &U) { + static const char Table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + std::string Buffer; + Buffer.resize(((U.size() + 2) / 3) * 4); + + size_t i = 0, j = 0; + for (size_t n = U.size() / 3 * 3; i < n; i += 3, j += 4) { + uint32_t x = ((unsigned char)U[i] << 16) | ((unsigned char)U[i + 1] << 8) | + (unsigned char)U[i + 2]; + Buffer[j + 0] = Table[(x >> 18) & 63]; + Buffer[j + 1] = Table[(x >> 12) & 63]; + Buffer[j + 2] = Table[(x >> 6) & 63]; + Buffer[j + 3] = Table[x & 63]; + } + if (i + 1 == U.size()) { + uint32_t x = ((unsigned char)U[i] << 16); + Buffer[j + 0] = Table[(x >> 18) & 63]; + Buffer[j + 1] = Table[(x >> 12) & 63]; + Buffer[j + 2] = '='; + Buffer[j + 3] = '='; + } else if (i + 2 == U.size()) { + uint32_t x = ((unsigned char)U[i] << 16) | ((unsigned char)U[i + 1] << 8); + Buffer[j + 0] = Table[(x >> 18) & 63]; + Buffer[j + 1] = Table[(x >> 12) & 63]; + Buffer[j + 2] = Table[(x >> 6) & 63]; + Buffer[j + 3] = '='; + } + return Buffer; +} + +static std::mutex SymbolizeMutex; + +std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC) { + std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock); + if (!EF->__sanitizer_symbolize_pc || !l.owns_lock()) + return "<can not symbolize>"; + char PcDescr[1024] = {}; + EF->__sanitizer_symbolize_pc(reinterpret_cast<void*>(PC), + SymbolizedFMT, PcDescr, sizeof(PcDescr)); + PcDescr[sizeof(PcDescr) - 1] = 0; // Just in case. + return PcDescr; +} + +void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC) { + if (EF->__sanitizer_symbolize_pc) + Printf("%s", DescribePC(SymbolizedFMT, PC).c_str()); + else + Printf(FallbackFMT, PC); +} + +void PrintStackTrace() { + std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock); + if (EF->__sanitizer_print_stack_trace && l.owns_lock()) + EF->__sanitizer_print_stack_trace(); +} + +void PrintMemoryProfile() { + std::unique_lock<std::mutex> l(SymbolizeMutex, std::try_to_lock); + if (EF->__sanitizer_print_memory_profile && l.owns_lock()) + EF->__sanitizer_print_memory_profile(95, 8); +} + +unsigned NumberOfCpuCores() { + unsigned N = std::thread::hardware_concurrency(); + if (!N) { + Printf("WARNING: std::thread::hardware_concurrency not well defined for " + "your platform. Assuming CPU count of 1.\n"); + N = 1; + } + return N; +} + +size_t SimpleFastHash(const uint8_t *Data, size_t Size) { + size_t Res = 0; + for (size_t i = 0; i < Size; i++) + Res = Res * 11 + Data[i]; + return Res; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerUtil.h b/tools/fuzzing/libfuzzer/FuzzerUtil.h new file mode 100644 index 0000000000..4ae3583830 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtil.h @@ -0,0 +1,111 @@ +//===- FuzzerUtil.h - Internal header for the Fuzzer Utils ------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Util functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_UTIL_H +#define LLVM_FUZZER_UTIL_H + +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" +#include "FuzzerCommand.h" +#include "FuzzerDefs.h" + +namespace fuzzer { + +void PrintHexArray(const Unit &U, const char *PrintAfter = ""); + +void PrintHexArray(const uint8_t *Data, size_t Size, + const char *PrintAfter = ""); + +void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter = ""); + +void PrintASCII(const Unit &U, const char *PrintAfter = ""); + +// Changes U to contain only ASCII (isprint+isspace) characters. +// Returns true iff U has been changed. +bool ToASCII(uint8_t *Data, size_t Size); + +bool IsASCII(const Unit &U); + +bool IsASCII(const uint8_t *Data, size_t Size); + +std::string Base64(const Unit &U); + +void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC); + +std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC); + +void PrintStackTrace(); + +void PrintMemoryProfile(); + +unsigned NumberOfCpuCores(); + +// Platform specific functions. +void SetSignalHandler(const FuzzingOptions& Options); + +void SleepSeconds(int Seconds); + +unsigned long GetPid(); + +size_t GetPeakRSSMb(); + +int ExecuteCommand(const Command &Cmd); +bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput); + +// Fuchsia does not have popen/pclose. +FILE *OpenProcessPipe(const char *Command, const char *Mode); +int CloseProcessPipe(FILE *F); + +const void *SearchMemory(const void *haystack, size_t haystacklen, + const void *needle, size_t needlelen); + +std::string CloneArgsWithoutX(const Vector<std::string> &Args, + const char *X1, const char *X2); + +inline std::string CloneArgsWithoutX(const Vector<std::string> &Args, + const char *X) { + return CloneArgsWithoutX(Args, X, X); +} + +inline std::pair<std::string, std::string> SplitBefore(std::string X, + std::string S) { + auto Pos = S.find(X); + if (Pos == std::string::npos) + return std::make_pair(S, ""); + return std::make_pair(S.substr(0, Pos), S.substr(Pos)); +} + +void DiscardOutput(int Fd); + +std::string DisassembleCmd(const std::string &FileName); + +std::string SearchRegexCmd(const std::string &Regex); + +size_t SimpleFastHash(const uint8_t *Data, size_t Size); + +inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } + +inline size_t PageSize() { return 4096; } +inline uint8_t *RoundUpByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast<uintptr_t>(P); + size_t Mask = PageSize() - 1; + X = (X + Mask) & ~Mask; + return reinterpret_cast<uint8_t *>(X); +} +inline uint8_t *RoundDownByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast<uintptr_t>(P); + size_t Mask = PageSize() - 1; + X = X & ~Mask; + return reinterpret_cast<uint8_t *>(X); +} + +} // namespace fuzzer + +#endif // LLVM_FUZZER_UTIL_H diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp new file mode 100644 index 0000000000..a5bed658a4 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp @@ -0,0 +1,170 @@ +//===- FuzzerUtilDarwin.cpp - Misc utils ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils for Darwin. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_APPLE +#include "FuzzerCommand.h" +#include "FuzzerIO.h" +#include <mutex> +#include <signal.h> +#include <spawn.h> +#include <stdlib.h> +#include <string.h> +#include <sys/wait.h> +#include <unistd.h> + +// There is no header for this on macOS so declare here +extern "C" char **environ; + +namespace fuzzer { + +static std::mutex SignalMutex; +// Global variables used to keep track of how signal handling should be +// restored. They should **not** be accessed without holding `SignalMutex`. +static int ActiveThreadCount = 0; +static struct sigaction OldSigIntAction; +static struct sigaction OldSigQuitAction; +static sigset_t OldBlockedSignalsSet; + +// This is a reimplementation of Libc's `system()`. On Darwin the Libc +// implementation contains a mutex which prevents it from being used +// concurrently. This implementation **can** be used concurrently. It sets the +// signal handlers when the first thread enters and restores them when the last +// thread finishes execution of the function and ensures this is not racey by +// using a mutex. +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + posix_spawnattr_t SpawnAttributes; + if (posix_spawnattr_init(&SpawnAttributes)) + return -1; + // Block and ignore signals of the current process when the first thread + // enters. + { + std::lock_guard<std::mutex> Lock(SignalMutex); + if (ActiveThreadCount == 0) { + static struct sigaction IgnoreSignalAction; + sigset_t BlockedSignalsSet; + memset(&IgnoreSignalAction, 0, sizeof(IgnoreSignalAction)); + IgnoreSignalAction.sa_handler = SIG_IGN; + + if (sigaction(SIGINT, &IgnoreSignalAction, &OldSigIntAction) == -1) { + Printf("Failed to ignore SIGINT\n"); + (void)posix_spawnattr_destroy(&SpawnAttributes); + return -1; + } + if (sigaction(SIGQUIT, &IgnoreSignalAction, &OldSigQuitAction) == -1) { + Printf("Failed to ignore SIGQUIT\n"); + // Try our best to restore the signal handlers. + (void)sigaction(SIGINT, &OldSigIntAction, NULL); + (void)posix_spawnattr_destroy(&SpawnAttributes); + return -1; + } + + (void)sigemptyset(&BlockedSignalsSet); + (void)sigaddset(&BlockedSignalsSet, SIGCHLD); + if (sigprocmask(SIG_BLOCK, &BlockedSignalsSet, &OldBlockedSignalsSet) == + -1) { + Printf("Failed to block SIGCHLD\n"); + // Try our best to restore the signal handlers. + (void)sigaction(SIGQUIT, &OldSigQuitAction, NULL); + (void)sigaction(SIGINT, &OldSigIntAction, NULL); + (void)posix_spawnattr_destroy(&SpawnAttributes); + return -1; + } + } + ++ActiveThreadCount; + } + + // NOTE: Do not introduce any new `return` statements past this + // point. It is important that `ActiveThreadCount` always be decremented + // when leaving this function. + + // Make sure the child process uses the default handlers for the + // following signals rather than inheriting what the parent has. + sigset_t DefaultSigSet; + (void)sigemptyset(&DefaultSigSet); + (void)sigaddset(&DefaultSigSet, SIGQUIT); + (void)sigaddset(&DefaultSigSet, SIGINT); + (void)posix_spawnattr_setsigdefault(&SpawnAttributes, &DefaultSigSet); + // Make sure the child process doesn't block SIGCHLD + (void)posix_spawnattr_setsigmask(&SpawnAttributes, &OldBlockedSignalsSet); + short SpawnFlags = POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK; + (void)posix_spawnattr_setflags(&SpawnAttributes, SpawnFlags); + + pid_t Pid; + char **Environ = environ; // Read from global + const char *CommandCStr = CmdLine.c_str(); + char *const Argv[] = { + strdup("sh"), + strdup("-c"), + strdup(CommandCStr), + NULL + }; + int ErrorCode = 0, ProcessStatus = 0; + // FIXME: We probably shouldn't hardcode the shell path. + ErrorCode = posix_spawn(&Pid, "/bin/sh", NULL, &SpawnAttributes, + Argv, Environ); + (void)posix_spawnattr_destroy(&SpawnAttributes); + if (!ErrorCode) { + pid_t SavedPid = Pid; + do { + // Repeat until call completes uninterrupted. + Pid = waitpid(SavedPid, &ProcessStatus, /*options=*/0); + } while (Pid == -1 && errno == EINTR); + if (Pid == -1) { + // Fail for some other reason. + ProcessStatus = -1; + } + } else if (ErrorCode == ENOMEM || ErrorCode == EAGAIN) { + // Fork failure. + ProcessStatus = -1; + } else { + // Shell execution failure. + ProcessStatus = W_EXITCODE(127, 0); + } + for (unsigned i = 0, n = sizeof(Argv) / sizeof(Argv[0]); i < n; ++i) + free(Argv[i]); + + // Restore the signal handlers of the current process when the last thread + // using this function finishes. + { + std::lock_guard<std::mutex> Lock(SignalMutex); + --ActiveThreadCount; + if (ActiveThreadCount == 0) { + bool FailedRestore = false; + if (sigaction(SIGINT, &OldSigIntAction, NULL) == -1) { + Printf("Failed to restore SIGINT handling\n"); + FailedRestore = true; + } + if (sigaction(SIGQUIT, &OldSigQuitAction, NULL) == -1) { + Printf("Failed to restore SIGQUIT handling\n"); + FailedRestore = true; + } + if (sigprocmask(SIG_BLOCK, &OldBlockedSignalsSet, NULL) == -1) { + Printf("Failed to unblock SIGCHLD\n"); + FailedRestore = true; + } + if (FailedRestore) + ProcessStatus = -1; + } + } + return ProcessStatus; +} + +void DiscardOutput(int Fd) { + FILE* Temp = fopen("/dev/null", "w"); + if (!Temp) + return; + dup2(fileno(Temp), Fd); + fclose(Temp); +} + +} // namespace fuzzer + +#endif // LIBFUZZER_APPLE diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilFuchsia.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilFuchsia.cpp new file mode 100644 index 0000000000..190fb78666 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilFuchsia.cpp @@ -0,0 +1,565 @@ +//===- FuzzerUtilFuchsia.cpp - Misc utils for Fuchsia. --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils implementation using Fuchsia/Zircon APIs. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" + +#if LIBFUZZER_FUCHSIA + +#include "FuzzerInternal.h" +#include "FuzzerUtil.h" +#include <cassert> +#include <cerrno> +#include <cinttypes> +#include <cstdint> +#include <fcntl.h> +#include <lib/fdio/fdio.h> +#include <lib/fdio/spawn.h> +#include <string> +#include <sys/select.h> +#include <thread> +#include <unistd.h> +#include <zircon/errors.h> +#include <zircon/process.h> +#include <zircon/sanitizer.h> +#include <zircon/status.h> +#include <zircon/syscalls.h> +#include <zircon/syscalls/debug.h> +#include <zircon/syscalls/exception.h> +#include <zircon/syscalls/object.h> +#include <zircon/types.h> + +#include <vector> + +namespace fuzzer { + +// Given that Fuchsia doesn't have the POSIX signals that libFuzzer was written +// around, the general approach is to spin up dedicated threads to watch for +// each requested condition (alarm, interrupt, crash). Of these, the crash +// handler is the most involved, as it requires resuming the crashed thread in +// order to invoke the sanitizers to get the needed state. + +// Forward declaration of assembly trampoline needed to resume crashed threads. +// This appears to have external linkage to C++, which is why it's not in the +// anonymous namespace. The assembly definition inside MakeTrampoline() +// actually defines the symbol with internal linkage only. +void CrashTrampolineAsm() __asm__("CrashTrampolineAsm"); + +namespace { + +// Helper function to handle Zircon syscall failures. +void ExitOnErr(zx_status_t Status, const char *Syscall) { + if (Status != ZX_OK) { + Printf("libFuzzer: %s failed: %s\n", Syscall, + _zx_status_get_string(Status)); + exit(1); + } +} + +void AlarmHandler(int Seconds) { + while (true) { + SleepSeconds(Seconds); + Fuzzer::StaticAlarmCallback(); + } +} + +void InterruptHandler() { + fd_set readfds; + // Ctrl-C sends ETX in Zircon. + do { + FD_ZERO(&readfds); + FD_SET(STDIN_FILENO, &readfds); + select(STDIN_FILENO + 1, &readfds, nullptr, nullptr, nullptr); + } while(!FD_ISSET(STDIN_FILENO, &readfds) || getchar() != 0x03); + Fuzzer::StaticInterruptCallback(); +} + +// CFAOffset is used to reference the stack pointer before entering the +// trampoline (Stack Pointer + CFAOffset = prev Stack Pointer). Before jumping +// to the trampoline we copy all the registers onto the stack. We need to make +// sure that the new stack has enough space to store all the registers. +// +// The trampoline holds CFI information regarding the registers stored in the +// stack, which is then used by the unwinder to restore them. +#if defined(__x86_64__) +// In x86_64 the crashing function might also be using the red zone (128 bytes +// on top of their rsp). +constexpr size_t CFAOffset = 128 + sizeof(zx_thread_state_general_regs_t); +#elif defined(__aarch64__) +// In aarch64 we need to always have the stack pointer aligned to 16 bytes, so we +// make sure that we are keeping that same alignment. +constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(uintptr_t)16; +#endif + +// For the crash handler, we need to call Fuzzer::StaticCrashSignalCallback +// without POSIX signal handlers. To achieve this, we use an assembly function +// to add the necessary CFI unwinding information and a C function to bridge +// from that back into C++. + +// FIXME: This works as a short-term solution, but this code really shouldn't be +// architecture dependent. A better long term solution is to implement remote +// unwinding and expose the necessary APIs through sanitizer_common and/or ASAN +// to allow the exception handling thread to gather the crash state directly. +// +// Alternatively, Fuchsia may in future actually implement basic signal +// handling for the machine trap signals. +#if defined(__x86_64__) +#define FOREACH_REGISTER(OP_REG, OP_NUM) \ + OP_REG(rax) \ + OP_REG(rbx) \ + OP_REG(rcx) \ + OP_REG(rdx) \ + OP_REG(rsi) \ + OP_REG(rdi) \ + OP_REG(rbp) \ + OP_REG(rsp) \ + OP_REG(r8) \ + OP_REG(r9) \ + OP_REG(r10) \ + OP_REG(r11) \ + OP_REG(r12) \ + OP_REG(r13) \ + OP_REG(r14) \ + OP_REG(r15) \ + OP_REG(rip) + +#elif defined(__aarch64__) +#define FOREACH_REGISTER(OP_REG, OP_NUM) \ + OP_NUM(0) \ + OP_NUM(1) \ + OP_NUM(2) \ + OP_NUM(3) \ + OP_NUM(4) \ + OP_NUM(5) \ + OP_NUM(6) \ + OP_NUM(7) \ + OP_NUM(8) \ + OP_NUM(9) \ + OP_NUM(10) \ + OP_NUM(11) \ + OP_NUM(12) \ + OP_NUM(13) \ + OP_NUM(14) \ + OP_NUM(15) \ + OP_NUM(16) \ + OP_NUM(17) \ + OP_NUM(18) \ + OP_NUM(19) \ + OP_NUM(20) \ + OP_NUM(21) \ + OP_NUM(22) \ + OP_NUM(23) \ + OP_NUM(24) \ + OP_NUM(25) \ + OP_NUM(26) \ + OP_NUM(27) \ + OP_NUM(28) \ + OP_NUM(29) \ + OP_REG(sp) + +#else +#error "Unsupported architecture for fuzzing on Fuchsia" +#endif + +// Produces a CFI directive for the named or numbered register. +// The value used refers to an assembler immediate operand with the same name +// as the register (see ASM_OPERAND_REG). +#define CFI_OFFSET_REG(reg) ".cfi_offset " #reg ", %c[" #reg "]\n" +#define CFI_OFFSET_NUM(num) CFI_OFFSET_REG(x##num) + +// Produces an assembler immediate operand for the named or numbered register. +// This operand contains the offset of the register relative to the CFA. +#define ASM_OPERAND_REG(reg) \ + [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg) - CFAOffset), +#define ASM_OPERAND_NUM(num) \ + [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num]) - CFAOffset), + +// Trampoline to bridge from the assembly below to the static C++ crash +// callback. +__attribute__((noreturn)) +static void StaticCrashHandler() { + Fuzzer::StaticCrashSignalCallback(); + for (;;) { + _Exit(1); + } +} + +// Creates the trampoline with the necessary CFI information to unwind through +// to the crashing call stack: +// * Defining the CFA so that it points to the stack pointer at the point +// of crash. +// * Storing all registers at the point of crash in the stack and refer to them +// via CFI information (relative to the CFA). +// * Setting the return column so the unwinder knows how to continue unwinding. +// * (x86_64) making sure rsp is aligned before calling StaticCrashHandler. +// * Calling StaticCrashHandler that will trigger the unwinder. +// +// The __attribute__((used)) is necessary because the function +// is never called; it's just a container around the assembly to allow it to +// use operands for compile-time computed constants. +__attribute__((used)) +void MakeTrampoline() { + __asm__(".cfi_endproc\n" + ".pushsection .text.CrashTrampolineAsm\n" + ".type CrashTrampolineAsm,STT_FUNC\n" +"CrashTrampolineAsm:\n" + ".cfi_startproc simple\n" + ".cfi_signal_frame\n" +#if defined(__x86_64__) + ".cfi_return_column rip\n" + ".cfi_def_cfa rsp, %c[CFAOffset]\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + "mov %%rsp, %%rbp\n" + ".cfi_def_cfa_register rbp\n" + "andq $-16, %%rsp\n" + "call %c[StaticCrashHandler]\n" + "ud2\n" +#elif defined(__aarch64__) + ".cfi_return_column 33\n" + ".cfi_def_cfa sp, %c[CFAOffset]\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + ".cfi_offset 33, %c[pc]\n" + ".cfi_offset 30, %c[lr]\n" + "bl %c[StaticCrashHandler]\n" + "brk 1\n" +#else +#error "Unsupported architecture for fuzzing on Fuchsia" +#endif + ".cfi_endproc\n" + ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n" + ".popsection\n" + ".cfi_startproc\n" + : // No outputs + : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM) +#if defined(__aarch64__) + ASM_OPERAND_REG(pc) + ASM_OPERAND_REG(lr) +#endif + [StaticCrashHandler] "i" (StaticCrashHandler), + [CFAOffset] "i" (CFAOffset)); +} + +void CrashHandler(zx_handle_t *Event) { + // This structure is used to ensure we close handles to objects we create in + // this handler. + struct ScopedHandle { + ~ScopedHandle() { _zx_handle_close(Handle); } + zx_handle_t Handle = ZX_HANDLE_INVALID; + }; + + // Create the exception channel. We need to claim to be a "debugger" so the + // kernel will allow us to modify and resume dying threads (see below). Once + // the channel is set, we can signal the main thread to continue and wait + // for the exception to arrive. + ScopedHandle Channel; + zx_handle_t Self = _zx_process_self(); + ExitOnErr(_zx_task_create_exception_channel( + Self, ZX_EXCEPTION_CHANNEL_DEBUGGER, &Channel.Handle), + "_zx_task_create_exception_channel"); + + ExitOnErr(_zx_object_signal(*Event, 0, ZX_USER_SIGNAL_0), + "_zx_object_signal"); + + // This thread lives as long as the process in order to keep handling + // crashes. In practice, the first crashed thread to reach the end of the + // StaticCrashHandler will end the process. + while (true) { + ExitOnErr(_zx_object_wait_one(Channel.Handle, ZX_CHANNEL_READABLE, + ZX_TIME_INFINITE, nullptr), + "_zx_object_wait_one"); + + zx_exception_info_t ExceptionInfo; + ScopedHandle Exception; + ExitOnErr(_zx_channel_read(Channel.Handle, 0, &ExceptionInfo, + &Exception.Handle, sizeof(ExceptionInfo), 1, + nullptr, nullptr), + "_zx_channel_read"); + + // Ignore informational synthetic exceptions. + if (ZX_EXCP_THREAD_STARTING == ExceptionInfo.type || + ZX_EXCP_THREAD_EXITING == ExceptionInfo.type || + ZX_EXCP_PROCESS_STARTING == ExceptionInfo.type) { + continue; + } + + // At this point, we want to get the state of the crashing thread, but + // libFuzzer and the sanitizers assume this will happen from that same + // thread via a POSIX signal handler. "Resurrecting" the thread in the + // middle of the appropriate callback is as simple as forcibly setting the + // instruction pointer/program counter, provided we NEVER EVER return from + // that function (since otherwise our stack will not be valid). + ScopedHandle Thread; + ExitOnErr(_zx_exception_get_thread(Exception.Handle, &Thread.Handle), + "_zx_exception_get_thread"); + + zx_thread_state_general_regs_t GeneralRegisters; + ExitOnErr(_zx_thread_read_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, + &GeneralRegisters, + sizeof(GeneralRegisters)), + "_zx_thread_read_state"); + + // To unwind properly, we need to push the crashing thread's register state + // onto the stack and jump into a trampoline with CFI instructions on how + // to restore it. +#if defined(__x86_64__) + uintptr_t StackPtr = GeneralRegisters.rsp - CFAOffset; + __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, + sizeof(GeneralRegisters)); + GeneralRegisters.rsp = StackPtr; + GeneralRegisters.rip = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm); + +#elif defined(__aarch64__) + uintptr_t StackPtr = GeneralRegisters.sp - CFAOffset; + __unsanitized_memcpy(reinterpret_cast<void *>(StackPtr), &GeneralRegisters, + sizeof(GeneralRegisters)); + GeneralRegisters.sp = StackPtr; + GeneralRegisters.pc = reinterpret_cast<zx_vaddr_t>(CrashTrampolineAsm); + +#else +#error "Unsupported architecture for fuzzing on Fuchsia" +#endif + + // Now force the crashing thread's state. + ExitOnErr( + _zx_thread_write_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, + &GeneralRegisters, sizeof(GeneralRegisters)), + "_zx_thread_write_state"); + + // Set the exception to HANDLED so it resumes the thread on close. + uint32_t ExceptionState = ZX_EXCEPTION_STATE_HANDLED; + ExitOnErr(_zx_object_set_property(Exception.Handle, ZX_PROP_EXCEPTION_STATE, + &ExceptionState, sizeof(ExceptionState)), + "zx_object_set_property"); + } +} + +} // namespace + +// Platform specific functions. +void SetSignalHandler(const FuzzingOptions &Options) { + // Make sure information from libFuzzer and the sanitizers are easy to + // reassemble. `__sanitizer_log_write` has the added benefit of ensuring the + // DSO map is always available for the symbolizer. + // A uint64_t fits in 20 chars, so 64 is plenty. + char Buf[64]; + memset(Buf, 0, sizeof(Buf)); + snprintf(Buf, sizeof(Buf), "==%lu== INFO: libFuzzer starting.\n", GetPid()); + if (EF->__sanitizer_log_write) + __sanitizer_log_write(Buf, sizeof(Buf)); + Printf("%s", Buf); + + // Set up alarm handler if needed. + if (Options.UnitTimeoutSec > 0) { + std::thread T(AlarmHandler, Options.UnitTimeoutSec / 2 + 1); + T.detach(); + } + + // Set up interrupt handler if needed. + if (Options.HandleInt || Options.HandleTerm) { + std::thread T(InterruptHandler); + T.detach(); + } + + // Early exit if no crash handler needed. + if (!Options.HandleSegv && !Options.HandleBus && !Options.HandleIll && + !Options.HandleFpe && !Options.HandleAbrt) + return; + + // Set up the crash handler and wait until it is ready before proceeding. + zx_handle_t Event; + ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create"); + + std::thread T(CrashHandler, &Event); + zx_status_t Status = + _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, ZX_TIME_INFINITE, nullptr); + _zx_handle_close(Event); + ExitOnErr(Status, "_zx_object_wait_one"); + + T.detach(); +} + +void SleepSeconds(int Seconds) { + _zx_nanosleep(_zx_deadline_after(ZX_SEC(Seconds))); +} + +unsigned long GetPid() { + zx_status_t rc; + zx_info_handle_basic_t Info; + if ((rc = _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &Info, + sizeof(Info), NULL, NULL)) != ZX_OK) { + Printf("libFuzzer: unable to get info about self: %s\n", + _zx_status_get_string(rc)); + exit(1); + } + return Info.koid; +} + +size_t GetPeakRSSMb() { + zx_status_t rc; + zx_info_task_stats_t Info; + if ((rc = _zx_object_get_info(_zx_process_self(), ZX_INFO_TASK_STATS, &Info, + sizeof(Info), NULL, NULL)) != ZX_OK) { + Printf("libFuzzer: unable to get info about self: %s\n", + _zx_status_get_string(rc)); + exit(1); + } + return (Info.mem_private_bytes + Info.mem_shared_bytes) >> 20; +} + +template <typename Fn> +class RunOnDestruction { + public: + explicit RunOnDestruction(Fn fn) : fn_(fn) {} + ~RunOnDestruction() { fn_(); } + + private: + Fn fn_; +}; + +template <typename Fn> +RunOnDestruction<Fn> at_scope_exit(Fn fn) { + return RunOnDestruction<Fn>(fn); +} + +static fdio_spawn_action_t clone_fd_action(int localFd, int targetFd) { + return { + .action = FDIO_SPAWN_ACTION_CLONE_FD, + .fd = + { + .local_fd = localFd, + .target_fd = targetFd, + }, + }; +} + +int ExecuteCommand(const Command &Cmd) { + zx_status_t rc; + + // Convert arguments to C array + auto Args = Cmd.getArguments(); + size_t Argc = Args.size(); + assert(Argc != 0); + std::unique_ptr<const char *[]> Argv(new const char *[Argc + 1]); + for (size_t i = 0; i < Argc; ++i) + Argv[i] = Args[i].c_str(); + Argv[Argc] = nullptr; + + // Determine output. On Fuchsia, the fuzzer is typically run as a component + // that lacks a mutable working directory. Fortunately, when this is the case + // a mutable output directory must be specified using "-artifact_prefix=...", + // so write the log file(s) there. + // However, we don't want to apply this logic for absolute paths. + int FdOut = STDOUT_FILENO; + bool discardStdout = false; + bool discardStderr = false; + + if (Cmd.hasOutputFile()) { + std::string Path = Cmd.getOutputFile(); + if (Path == getDevNull()) { + // On Fuchsia, there's no "/dev/null" like-file, so we + // just don't copy the FDs into the spawned process. + discardStdout = true; + } else { + bool IsAbsolutePath = Path.length() > 1 && Path[0] == '/'; + if (!IsAbsolutePath && Cmd.hasFlag("artifact_prefix")) + Path = Cmd.getFlagValue("artifact_prefix") + "/" + Path; + + FdOut = open(Path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); + if (FdOut == -1) { + Printf("libFuzzer: failed to open %s: %s\n", Path.c_str(), + strerror(errno)); + return ZX_ERR_IO; + } + } + } + auto CloseFdOut = at_scope_exit([FdOut]() { + if (FdOut != STDOUT_FILENO) + close(FdOut); + }); + + // Determine stderr + int FdErr = STDERR_FILENO; + if (Cmd.isOutAndErrCombined()) { + FdErr = FdOut; + if (discardStdout) + discardStderr = true; + } + + // Clone the file descriptors into the new process + std::vector<fdio_spawn_action_t> SpawnActions; + SpawnActions.push_back(clone_fd_action(STDIN_FILENO, STDIN_FILENO)); + + if (!discardStdout) + SpawnActions.push_back(clone_fd_action(FdOut, STDOUT_FILENO)); + if (!discardStderr) + SpawnActions.push_back(clone_fd_action(FdErr, STDERR_FILENO)); + + // Start the process. + char ErrorMsg[FDIO_SPAWN_ERR_MSG_MAX_LENGTH]; + zx_handle_t ProcessHandle = ZX_HANDLE_INVALID; + rc = fdio_spawn_etc(ZX_HANDLE_INVALID, + FDIO_SPAWN_CLONE_ALL & (~FDIO_SPAWN_CLONE_STDIO), Argv[0], + Argv.get(), nullptr, SpawnActions.size(), + SpawnActions.data(), &ProcessHandle, ErrorMsg); + + if (rc != ZX_OK) { + Printf("libFuzzer: failed to launch '%s': %s, %s\n", Argv[0], ErrorMsg, + _zx_status_get_string(rc)); + return rc; + } + auto CloseHandle = at_scope_exit([&]() { _zx_handle_close(ProcessHandle); }); + + // Now join the process and return the exit status. + if ((rc = _zx_object_wait_one(ProcessHandle, ZX_PROCESS_TERMINATED, + ZX_TIME_INFINITE, nullptr)) != ZX_OK) { + Printf("libFuzzer: failed to join '%s': %s\n", Argv[0], + _zx_status_get_string(rc)); + return rc; + } + + zx_info_process_t Info; + if ((rc = _zx_object_get_info(ProcessHandle, ZX_INFO_PROCESS, &Info, + sizeof(Info), nullptr, nullptr)) != ZX_OK) { + Printf("libFuzzer: unable to get return code from '%s': %s\n", Argv[0], + _zx_status_get_string(rc)); + return rc; + } + + return Info.return_code; +} + +bool ExecuteCommand(const Command &BaseCmd, std::string *CmdOutput) { + auto LogFilePath = TempPath("SimPopenOut", ".txt"); + Command Cmd(BaseCmd); + Cmd.setOutputFile(LogFilePath); + int Ret = ExecuteCommand(Cmd); + *CmdOutput = FileToString(LogFilePath); + RemoveFile(LogFilePath); + return Ret == 0; +} + +const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, + size_t PattLen) { + return memmem(Data, DataLen, Patt, PattLen); +} + +// In fuchsia, accessing /dev/null is not supported. There's nothing +// similar to a file that discards everything that is written to it. +// The way of doing something similar in fuchsia is by using +// fdio_null_create and binding that to a file descriptor. +void DiscardOutput(int Fd) { + fdio_t *fdio_null = fdio_null_create(); + if (fdio_null == nullptr) return; + int nullfd = fdio_bind_to_fd(fdio_null, -1, 0); + if (nullfd < 0) return; + dup2(nullfd, Fd); +} + +} // namespace fuzzer + +#endif // LIBFUZZER_FUCHSIA diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp new file mode 100644 index 0000000000..95490b992e --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp @@ -0,0 +1,41 @@ +//===- FuzzerUtilLinux.cpp - Misc utils for Linux. ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils for Linux. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ + LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN +#include "FuzzerCommand.h" + +#include <stdlib.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + + +namespace fuzzer { + +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + int exit_code = system(CmdLine.c_str()); + if (WIFEXITED(exit_code)) + return WEXITSTATUS(exit_code); + return exit_code; +} + +void DiscardOutput(int Fd) { + FILE* Temp = fopen("/dev/null", "w"); + if (!Temp) + return; + dup2(fileno(Temp), Fd); + fclose(Temp); +} + +} // namespace fuzzer + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp new file mode 100644 index 0000000000..fc57b724db --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp @@ -0,0 +1,185 @@ +//===- FuzzerUtilPosix.cpp - Misc utils for Posix. ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils implementation using Posix API. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_POSIX +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerTracePC.h" +#include <cassert> +#include <chrono> +#include <cstring> +#include <errno.h> +#include <iomanip> +#include <signal.h> +#include <stdio.h> +#include <sys/mman.h> +#include <sys/resource.h> +#include <sys/syscall.h> +#include <sys/time.h> +#include <sys/types.h> +#include <thread> +#include <unistd.h> + +namespace fuzzer { + +static void AlarmHandler(int, siginfo_t *, void *) { + Fuzzer::StaticAlarmCallback(); +} + +static void (*upstream_segv_handler)(int, siginfo_t *, void *); + +static void SegvHandler(int sig, siginfo_t *si, void *ucontext) { + assert(si->si_signo == SIGSEGV); + if (upstream_segv_handler) + return upstream_segv_handler(sig, si, ucontext); + Fuzzer::StaticCrashSignalCallback(); +} + +static void CrashHandler(int, siginfo_t *, void *) { + Fuzzer::StaticCrashSignalCallback(); +} + +static void InterruptHandler(int, siginfo_t *, void *) { + Fuzzer::StaticInterruptCallback(); +} + +static void GracefulExitHandler(int, siginfo_t *, void *) { + Fuzzer::StaticGracefulExitCallback(); +} + +static void FileSizeExceedHandler(int, siginfo_t *, void *) { + Fuzzer::StaticFileSizeExceedCallback(); +} + +static void SetSigaction(int signum, + void (*callback)(int, siginfo_t *, void *)) { + struct sigaction sigact = {}; + if (sigaction(signum, nullptr, &sigact)) { + Printf("libFuzzer: sigaction failed with %d\n", errno); + exit(1); + } + if (sigact.sa_flags & SA_SIGINFO) { + if (sigact.sa_sigaction) { + if (signum != SIGSEGV) + return; + upstream_segv_handler = sigact.sa_sigaction; + } + } else { + if (sigact.sa_handler != SIG_DFL && sigact.sa_handler != SIG_IGN && + sigact.sa_handler != SIG_ERR) + return; + } + + sigact = {}; + sigact.sa_flags = SA_SIGINFO; + sigact.sa_sigaction = callback; + if (sigaction(signum, &sigact, 0)) { + Printf("libFuzzer: sigaction failed with %d\n", errno); + exit(1); + } +} + +// Return true on success, false otherwise. +bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput) { + FILE *Pipe = popen(Cmd.toString().c_str(), "r"); + if (!Pipe) + return false; + + if (CmdOutput) { + char TmpBuffer[128]; + while (fgets(TmpBuffer, sizeof(TmpBuffer), Pipe)) + CmdOutput->append(TmpBuffer); + } + return pclose(Pipe) == 0; +} + +void SetTimer(int Seconds) { + struct itimerval T { + {Seconds, 0}, { Seconds, 0 } + }; + if (setitimer(ITIMER_REAL, &T, nullptr)) { + Printf("libFuzzer: setitimer failed with %d\n", errno); + exit(1); + } + SetSigaction(SIGALRM, AlarmHandler); +} + +void SetSignalHandler(const FuzzingOptions& Options) { + // setitimer is not implemented in emscripten. + if (Options.UnitTimeoutSec > 0 && !LIBFUZZER_EMSCRIPTEN) + SetTimer(Options.UnitTimeoutSec / 2 + 1); + if (Options.HandleInt) + SetSigaction(SIGINT, InterruptHandler); + if (Options.HandleTerm) + SetSigaction(SIGTERM, InterruptHandler); + if (Options.HandleSegv) + SetSigaction(SIGSEGV, SegvHandler); + if (Options.HandleBus) + SetSigaction(SIGBUS, CrashHandler); + if (Options.HandleAbrt) + SetSigaction(SIGABRT, CrashHandler); + if (Options.HandleIll) + SetSigaction(SIGILL, CrashHandler); + if (Options.HandleFpe) + SetSigaction(SIGFPE, CrashHandler); + if (Options.HandleXfsz) + SetSigaction(SIGXFSZ, FileSizeExceedHandler); + if (Options.HandleUsr1) + SetSigaction(SIGUSR1, GracefulExitHandler); + if (Options.HandleUsr2) + SetSigaction(SIGUSR2, GracefulExitHandler); +} + +void SleepSeconds(int Seconds) { + sleep(Seconds); // Use C API to avoid coverage from instrumented libc++. +} + +unsigned long GetPid() { return (unsigned long)getpid(); } + +size_t GetPeakRSSMb() { + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage)) + return 0; + if (LIBFUZZER_LINUX || LIBFUZZER_FREEBSD || LIBFUZZER_NETBSD || + LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN) { + // ru_maxrss is in KiB + return usage.ru_maxrss >> 10; + } else if (LIBFUZZER_APPLE) { + // ru_maxrss is in bytes + return usage.ru_maxrss >> 20; + } + assert(0 && "GetPeakRSSMb() is not implemented for your platform"); + return 0; +} + +FILE *OpenProcessPipe(const char *Command, const char *Mode) { + return popen(Command, Mode); +} + +int CloseProcessPipe(FILE *F) { + return pclose(F); +} + +const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, + size_t PattLen) { + return memmem(Data, DataLen, Patt, PattLen); +} + +std::string DisassembleCmd(const std::string &FileName) { + return "objdump -d " + FileName; +} + +std::string SearchRegexCmd(const std::string &Regex) { + return "grep '" + Regex + "'"; +} + +} // namespace fuzzer + +#endif // LIBFUZZER_POSIX diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp new file mode 100644 index 0000000000..6c693e3d7e --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp @@ -0,0 +1,221 @@ +//===- FuzzerUtilWindows.cpp - Misc utils for Windows. --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils implementation for Windows. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_WINDOWS +#include "FuzzerCommand.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include <cassert> +#include <chrono> +#include <cstring> +#include <errno.h> +#include <io.h> +#include <iomanip> +#include <signal.h> +#include <stdio.h> +#include <sys/types.h> +#include <windows.h> + +// This must be included after windows.h. +#include <psapi.h> + +namespace fuzzer { + +static const FuzzingOptions* HandlerOpt = nullptr; + +static LONG CALLBACK ExceptionHandler(PEXCEPTION_POINTERS ExceptionInfo) { + switch (ExceptionInfo->ExceptionRecord->ExceptionCode) { + case EXCEPTION_ACCESS_VIOLATION: + case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: + case EXCEPTION_STACK_OVERFLOW: + if (HandlerOpt->HandleSegv) + Fuzzer::StaticCrashSignalCallback(); + break; + case EXCEPTION_DATATYPE_MISALIGNMENT: + case EXCEPTION_IN_PAGE_ERROR: + if (HandlerOpt->HandleBus) + Fuzzer::StaticCrashSignalCallback(); + break; + case EXCEPTION_ILLEGAL_INSTRUCTION: + case EXCEPTION_PRIV_INSTRUCTION: + if (HandlerOpt->HandleIll) + Fuzzer::StaticCrashSignalCallback(); + break; + case EXCEPTION_FLT_DENORMAL_OPERAND: + case EXCEPTION_FLT_DIVIDE_BY_ZERO: + case EXCEPTION_FLT_INEXACT_RESULT: + case EXCEPTION_FLT_INVALID_OPERATION: + case EXCEPTION_FLT_OVERFLOW: + case EXCEPTION_FLT_STACK_CHECK: + case EXCEPTION_FLT_UNDERFLOW: + case EXCEPTION_INT_DIVIDE_BY_ZERO: + case EXCEPTION_INT_OVERFLOW: + if (HandlerOpt->HandleFpe) + Fuzzer::StaticCrashSignalCallback(); + break; + // TODO: handle (Options.HandleXfsz) + } + return EXCEPTION_CONTINUE_SEARCH; +} + +BOOL WINAPI CtrlHandler(DWORD dwCtrlType) { + switch (dwCtrlType) { + case CTRL_C_EVENT: + if (HandlerOpt->HandleInt) + Fuzzer::StaticInterruptCallback(); + return TRUE; + case CTRL_BREAK_EVENT: + if (HandlerOpt->HandleTerm) + Fuzzer::StaticInterruptCallback(); + return TRUE; + } + return FALSE; +} + +void CALLBACK AlarmHandler(PVOID, BOOLEAN) { + Fuzzer::StaticAlarmCallback(); +} + +class TimerQ { + HANDLE TimerQueue; + public: + TimerQ() : TimerQueue(NULL) {} + ~TimerQ() { + if (TimerQueue) + DeleteTimerQueueEx(TimerQueue, NULL); + } + void SetTimer(int Seconds) { + if (!TimerQueue) { + TimerQueue = CreateTimerQueue(); + if (!TimerQueue) { + Printf("libFuzzer: CreateTimerQueue failed.\n"); + exit(1); + } + } + HANDLE Timer; + if (!CreateTimerQueueTimer(&Timer, TimerQueue, AlarmHandler, NULL, + Seconds*1000, Seconds*1000, 0)) { + Printf("libFuzzer: CreateTimerQueueTimer failed.\n"); + exit(1); + } + } +}; + +static TimerQ Timer; + +static void CrashHandler(int) { Fuzzer::StaticCrashSignalCallback(); } + +void SetSignalHandler(const FuzzingOptions& Options) { + HandlerOpt = &Options; + + if (Options.UnitTimeoutSec > 0) + Timer.SetTimer(Options.UnitTimeoutSec / 2 + 1); + + if (Options.HandleInt || Options.HandleTerm) + if (!SetConsoleCtrlHandler(CtrlHandler, TRUE)) { + DWORD LastError = GetLastError(); + Printf("libFuzzer: SetConsoleCtrlHandler failed (Error code: %lu).\n", + LastError); + exit(1); + } + + if (Options.HandleSegv || Options.HandleBus || Options.HandleIll || + Options.HandleFpe) + SetUnhandledExceptionFilter(ExceptionHandler); + + if (Options.HandleAbrt) + if (SIG_ERR == signal(SIGABRT, CrashHandler)) { + Printf("libFuzzer: signal failed with %d\n", errno); + exit(1); + } +} + +void SleepSeconds(int Seconds) { Sleep(Seconds * 1000); } + +unsigned long GetPid() { return GetCurrentProcessId(); } + +size_t GetPeakRSSMb() { + PROCESS_MEMORY_COUNTERS info; + if (!GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info))) + return 0; + return info.PeakWorkingSetSize >> 20; +} + +FILE *OpenProcessPipe(const char *Command, const char *Mode) { + return _popen(Command, Mode); +} + +int CloseProcessPipe(FILE *F) { + return _pclose(F); +} + +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + return system(CmdLine.c_str()); +} + +bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput) { + FILE *Pipe = _popen(Cmd.toString().c_str(), "r"); + if (!Pipe) + return false; + + if (CmdOutput) { + char TmpBuffer[128]; + while (fgets(TmpBuffer, sizeof(TmpBuffer), Pipe)) + CmdOutput->append(TmpBuffer); + } + return _pclose(Pipe) == 0; +} + +const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, + size_t PattLen) { + // TODO: make this implementation more efficient. + const char *Cdata = (const char *)Data; + const char *Cpatt = (const char *)Patt; + + if (!Data || !Patt || DataLen == 0 || PattLen == 0 || DataLen < PattLen) + return NULL; + + if (PattLen == 1) + return memchr(Data, *Cpatt, DataLen); + + const char *End = Cdata + DataLen - PattLen + 1; + + for (const char *It = Cdata; It < End; ++It) + if (It[0] == Cpatt[0] && memcmp(It, Cpatt, PattLen) == 0) + return It; + + return NULL; +} + +std::string DisassembleCmd(const std::string &FileName) { + Vector<std::string> command_vector; + command_vector.push_back("dumpbin /summary > nul"); + if (ExecuteCommand(Command(command_vector)) == 0) + return "dumpbin /disasm " + FileName; + Printf("libFuzzer: couldn't find tool to disassemble (dumpbin)\n"); + exit(1); +} + +std::string SearchRegexCmd(const std::string &Regex) { + return "findstr /r \"" + Regex + "\""; +} + +void DiscardOutput(int Fd) { + FILE* Temp = fopen("nul", "w"); + if (!Temp) + return; + _dup2(_fileno(Temp), Fd); + fclose(Temp); +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerValueBitMap.h b/tools/fuzzing/libfuzzer/FuzzerValueBitMap.h new file mode 100644 index 0000000000..ddbfe200af --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerValueBitMap.h @@ -0,0 +1,73 @@ +//===- FuzzerValueBitMap.h - INTERNAL - Bit map -----------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// ValueBitMap. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_VALUE_BIT_MAP_H +#define LLVM_FUZZER_VALUE_BIT_MAP_H + +#include "FuzzerPlatform.h" +#include <cstdint> + +namespace fuzzer { + +// A bit map containing kMapSizeInWords bits. +struct ValueBitMap { + static const size_t kMapSizeInBits = 1 << 16; + static const size_t kMapPrimeMod = 65371; // Largest Prime < kMapSizeInBits; + static const size_t kBitsInWord = (sizeof(uintptr_t) * 8); + static const size_t kMapSizeInWords = kMapSizeInBits / kBitsInWord; + public: + + // Clears all bits. + void Reset() { memset(Map, 0, sizeof(Map)); } + + // Computes a hash function of Value and sets the corresponding bit. + // Returns true if the bit was changed from 0 to 1. + ATTRIBUTE_NO_SANITIZE_ALL + inline bool AddValue(uintptr_t Value) { + uintptr_t Idx = Value % kMapSizeInBits; + uintptr_t WordIdx = Idx / kBitsInWord; + uintptr_t BitIdx = Idx % kBitsInWord; + uintptr_t Old = Map[WordIdx]; + uintptr_t New = Old | (1ULL << BitIdx); + Map[WordIdx] = New; + return New != Old; + } + + ATTRIBUTE_NO_SANITIZE_ALL + inline bool AddValueModPrime(uintptr_t Value) { + return AddValue(Value % kMapPrimeMod); + } + + inline bool Get(uintptr_t Idx) { + assert(Idx < kMapSizeInBits); + uintptr_t WordIdx = Idx / kBitsInWord; + uintptr_t BitIdx = Idx % kBitsInWord; + return Map[WordIdx] & (1ULL << BitIdx); + } + + size_t SizeInBits() const { return kMapSizeInBits; } + + template <class Callback> + ATTRIBUTE_NO_SANITIZE_ALL + void ForEach(Callback CB) const { + for (size_t i = 0; i < kMapSizeInWords; i++) + if (uintptr_t M = Map[i]) + for (size_t j = 0; j < sizeof(M) * 8; j++) + if (M & ((uintptr_t)1 << j)) + CB(i * sizeof(M) * 8 + j); + } + + private: + ATTRIBUTE_ALIGNED(512) uintptr_t Map[kMapSizeInWords]; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_VALUE_BIT_MAP_H diff --git a/tools/fuzzing/libfuzzer/LICENSE.TXT b/tools/fuzzing/libfuzzer/LICENSE.TXT new file mode 100644 index 0000000000..fa6ac54000 --- /dev/null +++ b/tools/fuzzing/libfuzzer/LICENSE.TXT @@ -0,0 +1,279 @@ +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2003-2019 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + diff --git a/tools/fuzzing/libfuzzer/moz.build b/tools/fuzzing/libfuzzer/moz.build new file mode 100644 index 0000000000..6e77e29e09 --- /dev/null +++ b/tools/fuzzing/libfuzzer/moz.build @@ -0,0 +1,55 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library('fuzzer') + +EXPORTS += [ + 'FuzzerDefs.h', + 'FuzzerExtFunctions.def', + 'FuzzerExtFunctions.h', +] + +SOURCES += [ + 'FuzzerCrossOver.cpp', + 'FuzzerDataFlowTrace.cpp', + 'FuzzerDriver.cpp', + 'FuzzerExtFunctionsDlsym.cpp', + 'FuzzerExtFunctionsWeak.cpp', + 'FuzzerExtFunctionsWindows.cpp', + 'FuzzerExtraCounters.cpp', + 'FuzzerFork.cpp', + 'FuzzerIO.cpp', + 'FuzzerIOPosix.cpp', + 'FuzzerIOWindows.cpp', + 'FuzzerLoop.cpp', + 'FuzzerMerge.cpp', + 'FuzzerMutate.cpp', + 'FuzzerSHA1.cpp', + 'FuzzerTracePC.cpp', + 'FuzzerUtil.cpp', + 'FuzzerUtilDarwin.cpp', + 'FuzzerUtilFuchsia.cpp', + 'FuzzerUtilLinux.cpp', + 'FuzzerUtilPosix.cpp', + 'FuzzerUtilWindows.cpp', +] + +if CONFIG['CC_TYPE'] == 'clang': + CXXFLAGS += ['-Wno-unreachable-code-return'] + +# According to the LLVM docs, LibFuzzer isn't supposed to be built with any +# sanitizer flags and in fact, building it with ASan coverage currently causes +# Clang 3.9+ to crash, so we filter out all sanitizer-related flags here. +for flags_var in ('OS_CFLAGS', 'OS_CXXFLAGS'): + COMPILE_FLAGS[flags_var] = [ + f for f in COMPILE_FLAGS.get(flags_var, []) + if not f.startswith(('-fsanitize', '-fno-sanitize-')) + ] + +LINK_FLAGS['OS'] = [ + f for f in LINK_FLAGS.get('OS', []) + if not f.startswith(('-fsanitize', '-fno-sanitize-')) +] diff --git a/tools/fuzzing/libfuzzer/moz.yaml b/tools/fuzzing/libfuzzer/moz.yaml new file mode 100644 index 0000000000..f8074d07ed --- /dev/null +++ b/tools/fuzzing/libfuzzer/moz.yaml @@ -0,0 +1,52 @@ +schema: 1 + +bugzilla: + product: Core + component: General + +origin: + + name: fuzzer + + description: library for coverage-guided fuzz testing + + url: https://llvm.org/docs/LibFuzzer.html + + release: 76d07503f0c69f6632e6d8d4736e2a4cb4055a92 (2020-07-30T12:42:56Z). + + revision: 76d07503f0c69f6632e6d8d4736e2a4cb4055a92 + + license: Apache-2.0 + license-file: LICENSE.TXT + +vendoring: + url: https://github.com/llvm/llvm-project + source-hosting: github + + keep: + - LICENSE.TXT + + exclude: + - "**" + + include: + - "compiler-rt/lib/fuzzer/*.h" + - "compiler-rt/lib/fuzzer/*.cpp" + - "compiler-rt/lib/fuzzer/*.def" + + patches: + - patches/10-ef-runtime.patch + - patches/11-callback-rv.patch + - patches/12-custom-mutator-fail.patch + - patches/13-unused-write.patch + - patches/14-explicit-allocator.patch + - patches/15-return-to-exit.patch + + update-actions: + - action: move-dir + from: '{yaml_dir}/compiler-rt/lib/fuzzer' + to: '{yaml_dir}' + - action: delete-path + path: '{yaml_dir}/FuzzerMain*' + - action: delete-path + path: '{yaml_dir}/FuzzerInterceptors*' diff --git a/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch b/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch new file mode 100644 index 0000000000..a544f42bd7 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch @@ -0,0 +1,31 @@ +# HG changeset patch +# User Christian Holler <choller@mozilla.com> +# Date 1596126054 -7200 +# Thu Jul 30 18:20:54 2020 +0200 +# Node ID 8a2a26b33d516c43c366b2f24d731d27d9843349 +# Parent 997c4109edd112695097fd8c55cbacd976cab24a +[libFuzzer] Allow external functions to be defined at runtime + +diff --git a/FuzzerDriver.cpp b/FuzzerDriver.cpp +--- a/FuzzerDriver.cpp ++++ b/FuzzerDriver.cpp +@@ -608,17 +608,18 @@ static Vector<SizedFile> ReadCorpora(con + SizedFiles.push_back({File, Size}); + return SizedFiles; + } + + int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + using namespace fuzzer; + assert(argc && argv && "Argument pointers cannot be nullptr"); + std::string Argv0((*argv)[0]); +- EF = new ExternalFunctions(); ++ if (!EF) ++ EF = new ExternalFunctions(); + if (EF->LLVMFuzzerInitialize) + EF->LLVMFuzzerInitialize(argc, argv); + if (EF->__msan_scoped_disable_interceptor_checks) + EF->__msan_scoped_disable_interceptor_checks(); + const Vector<std::string> Args(*argv, *argv + *argc); + assert(!Args.empty()); + ProgName = new std::string(Args[0]); + if (Argv0 != *ProgName) { diff --git a/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch b/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch new file mode 100644 index 0000000000..3f9832b0a3 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch @@ -0,0 +1,132 @@ +# HG changeset patch +# User Christian Holler <choller@mozilla.com> +# Date 1596126448 -7200 +# Thu Jul 30 18:27:28 2020 +0200 +# Node ID ea198a0331a6db043cb5978512226977514104db +# Parent 8a2a26b33d516c43c366b2f24d731d27d9843349 +[libFuzzer] Change libFuzzer callback contract to allow positive return values + +diff --git a/FuzzerInternal.h b/FuzzerInternal.h +--- a/FuzzerInternal.h ++++ b/FuzzerInternal.h +@@ -60,17 +60,17 @@ public: + + static void StaticAlarmCallback(); + static void StaticCrashSignalCallback(); + static void StaticExitCallback(); + static void StaticInterruptCallback(); + static void StaticFileSizeExceedCallback(); + static void StaticGracefulExitCallback(); + +- void ExecuteCallback(const uint8_t *Data, size_t Size); ++ int ExecuteCallback(const uint8_t *Data, size_t Size); + bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, + InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); + + // Merge Corpora[1:] into Corpora[0]. + void Merge(const Vector<std::string> &Corpora); + void CrashResistantMergeInternalStep(const std::string &ControlFilePath); + MutationDispatcher &GetMD() { return MD; } + void PrintFinalStats(); +diff --git a/FuzzerLoop.cpp b/FuzzerLoop.cpp +--- a/FuzzerLoop.cpp ++++ b/FuzzerLoop.cpp +@@ -463,17 +463,19 @@ static void RenameFeatureSetFile(const s + DirPlusFile(FeaturesDir, NewFile)); + } + + bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, + InputInfo *II, bool *FoundUniqFeatures) { + if (!Size) + return false; + +- ExecuteCallback(Data, Size); ++ if (ExecuteCallback(Data, Size) > 0) { ++ return false; ++ } + + UniqFeatureSetTmp.clear(); + size_t FoundUniqFeaturesOfII = 0; + size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); + TPC.CollectFeatures([&](size_t Feature) { + if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + UniqFeatureSetTmp.push_back(Feature); + if (Options.Entropic) +@@ -530,48 +532,49 @@ static bool LooseMemeq(const uint8_t *A, + const size_t Limit = 64; + if (Size <= 64) + return !memcmp(A, B, Size); + // Compare first and last Limit/2 bytes. + return !memcmp(A, B, Limit / 2) && + !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); + } + +-void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { ++int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { + TPC.RecordInitialStack(); + TotalNumberOfRuns++; + assert(InFuzzingThread()); + // We copy the contents of Unit into a separate heap buffer + // so that we reliably find buffer overflows in it. + uint8_t *DataCopy = new uint8_t[Size]; + memcpy(DataCopy, Data, Size); + if (EF->__msan_unpoison) + EF->__msan_unpoison(DataCopy, Size); + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(2); + if (CurrentUnitData && CurrentUnitData != Data) + memcpy(CurrentUnitData, Data, Size); + CurrentUnitSize = Size; ++ int Res = 0; + { + ScopedEnableMsanInterceptorChecks S; + AllocTracer.Start(Options.TraceMalloc); + UnitStartTime = system_clock::now(); + TPC.ResetMaps(); + RunningUserCallback = true; +- int Res = CB(DataCopy, Size); ++ Res = CB(DataCopy, Size); + RunningUserCallback = false; + UnitStopTime = system_clock::now(); +- (void)Res; +- assert(Res == 0); ++ assert(Res >= 0); + HasMoreMallocsThanFrees = AllocTracer.Stop(); + } + if (!LooseMemeq(DataCopy, Data, Size)) + CrashOnOverwrittenData(); + CurrentUnitSize = 0; + delete[] DataCopy; ++ return Res; + } + + std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { + if (Options.OnlyASCII) + assert(IsASCII(U)); + if (Options.OutputCorpus.empty()) + return ""; + std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); +diff --git a/FuzzerMerge.cpp b/FuzzerMerge.cpp +--- a/FuzzerMerge.cpp ++++ b/FuzzerMerge.cpp +@@ -223,17 +223,19 @@ void Fuzzer::CrashResistantMergeInternal + U.shrink_to_fit(); + } + + // Write the pre-run marker. + OF << "STARTED " << i << " " << U.size() << "\n"; + OF.flush(); // Flush is important since Command::Execute may crash. + // Run. + TPC.ResetMaps(); +- ExecuteCallback(U.data(), U.size()); ++ if (ExecuteCallback(U.data(), U.size()) > 0) { ++ continue; ++ } + // Collect coverage. We are iterating over the files in this order: + // * First, files in the initial corpus ordered by size, smallest first. + // * Then, all other files, smallest first. + // So it makes no sense to record all features for all files, instead we + // only record features that were not seen before. + Set<size_t> UniqFeatures; + TPC.CollectFeatures([&](size_t Feature) { + if (AllFeatures.insert(Feature).second) diff --git a/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch b/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch new file mode 100644 index 0000000000..13bcedc872 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch @@ -0,0 +1,53 @@ +# HG changeset patch +# User Christian Holler <choller@mozilla.com> +# Date 1596126768 -7200 +# Thu Jul 30 18:32:48 2020 +0200 +# Node ID 64e7d096fa77a62b71a306b2c5383b8f75ac4945 +# Parent ea198a0331a6db043cb5978512226977514104db +[libFuzzer] Allow custom mutators to fail + +diff --git a/FuzzerLoop.cpp b/FuzzerLoop.cpp +--- a/FuzzerLoop.cpp ++++ b/FuzzerLoop.cpp +@@ -690,16 +690,20 @@ void Fuzzer::MutateAndTestOne() { + if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() && + Size <= CurrentMaxMutationLen) + NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size, + II.DataFlowTraceForFocusFunction); + + // If MutateWithMask either failed or wasn't called, call default Mutate. + if (!NewSize) + NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); ++ ++ if (!NewSize) ++ continue; ++ + assert(NewSize > 0 && "Mutator returned empty unit"); + assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); + Size = NewSize; + II.NumExecutedMutations++; + Corpus.IncrementNumExecutedMutations(); + + bool FoundUniqFeatures = false; + bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II, +@@ -850,17 +854,19 @@ void Fuzzer::Loop(Vector<SizedFile> &Cor + void Fuzzer::MinimizeCrashLoop(const Unit &U) { + if (U.size() <= 1) + return; + while (!TimedOut() && TotalNumberOfRuns < Options.MaxNumberOfRuns) { + MD.StartMutationSequence(); + memcpy(CurrentUnitData, U.data(), U.size()); + for (int i = 0; i < Options.MutateDepth; i++) { + size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen); +- assert(NewSize > 0 && NewSize <= MaxMutationLen); ++ assert(NewSize <= MaxMutationLen); ++ if (!NewSize) ++ continue; + ExecuteCallback(CurrentUnitData, NewSize); + PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); + TryDetectingAMemoryLeak(CurrentUnitData, NewSize, + /*DuringInitialCorpusExecution*/ false); + } + } + } + diff --git a/tools/fuzzing/libfuzzer/patches/13-unused-write.patch b/tools/fuzzing/libfuzzer/patches/13-unused-write.patch new file mode 100644 index 0000000000..7aaa8cf84f --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/13-unused-write.patch @@ -0,0 +1,88 @@ +# HG changeset patch +# User Christian Holler <choller@mozilla.com> +# Date 1596126946 -7200 +# Thu Jul 30 18:35:46 2020 +0200 +# Node ID 6c779ec81530b6784a714063af66085681ab7318 +# Parent 64e7d096fa77a62b71a306b2c5383b8f75ac4945 +[libFuzzer] Suppress warnings about unused return values + +diff --git a/FuzzerIO.cpp b/FuzzerIO.cpp +--- a/FuzzerIO.cpp ++++ b/FuzzerIO.cpp +@@ -3,16 +3,17 @@ + // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + // See https://llvm.org/LICENSE.txt for license information. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // + //===----------------------------------------------------------------------===// + // IO functions. + //===----------------------------------------------------------------------===// + ++#include "mozilla/Unused.h" + #include "FuzzerDefs.h" + #include "FuzzerExtFunctions.h" + #include "FuzzerIO.h" + #include "FuzzerUtil.h" + #include <algorithm> + #include <cstdarg> + #include <fstream> + #include <iterator> +@@ -68,17 +69,17 @@ void WriteToFile(const std::string &Data + WriteToFile(reinterpret_cast<const uint8_t *>(Data.c_str()), Data.size(), + Path); + } + + void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path) { + // Use raw C interface because this function may be called from a sig handler. + FILE *Out = fopen(Path.c_str(), "wb"); + if (!Out) return; +- fwrite(Data, sizeof(Data[0]), Size, Out); ++ mozilla::Unused << fwrite(Data, sizeof(Data[0]), Size, Out); + fclose(Out); + } + + void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, + long *Epoch, size_t MaxSize, bool ExitOnError) { + long E = Epoch ? *Epoch : 0; + Vector<std::string> Files; + ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); +diff --git a/FuzzerIOPosix.cpp b/FuzzerIOPosix.cpp +--- a/FuzzerIOPosix.cpp ++++ b/FuzzerIOPosix.cpp +@@ -2,16 +2,17 @@ + // + // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + // See https://llvm.org/LICENSE.txt for license information. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // + //===----------------------------------------------------------------------===// + // IO functions implementation using Posix API. + //===----------------------------------------------------------------------===// ++#include "mozilla/Unused.h" + #include "FuzzerPlatform.h" + #if LIBFUZZER_POSIX || LIBFUZZER_FUCHSIA + + #include "FuzzerExtFunctions.h" + #include "FuzzerIO.h" + #include <cstdarg> + #include <cstdio> + #include <dirent.h> +@@ -150,17 +151,17 @@ bool IsInterestingCoverageFile(const std + if (FileName.find("/usr/include/") != std::string::npos) + return false; + if (FileName == "<null>") + return false; + return true; + } + + void RawPrint(const char *Str) { +- write(2, Str, strlen(Str)); ++ mozilla::Unused << write(2, Str, strlen(Str)); + } + + void MkDir(const std::string &Path) { + mkdir(Path.c_str(), 0700); + } + + void RmDir(const std::string &Path) { + rmdir(Path.c_str()); diff --git a/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch b/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch new file mode 100644 index 0000000000..1781732286 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch @@ -0,0 +1,30 @@ +# HG changeset patch +# User Christian Holler <choller@mozilla.com> +# Date 1596126981 -7200 +# Thu Jul 30 18:36:21 2020 +0200 +# Node ID 069dfa3715b1d30905ff0ea1c0f66db88ce146f9 +# Parent 6c779ec81530b6784a714063af66085681ab7318 +[libFuzzer] Make fuzzer_allocator explicit + +diff --git a/FuzzerDefs.h b/FuzzerDefs.h +--- a/FuzzerDefs.h ++++ b/FuzzerDefs.h +@@ -41,17 +41,17 @@ extern ExternalFunctions *EF; + // We are using a custom allocator to give a different symbol name to STL + // containers in order to avoid ODR violations. + template<typename T> + class fuzzer_allocator: public std::allocator<T> { + public: + fuzzer_allocator() = default; + + template<class U> +- fuzzer_allocator(const fuzzer_allocator<U>&) {} ++ explicit fuzzer_allocator(const fuzzer_allocator<U>&) {} + + template<class Other> + struct rebind { typedef fuzzer_allocator<Other> other; }; + }; + + template<typename T> + using Vector = std::vector<T, fuzzer_allocator<T>>; + diff --git a/tools/fuzzing/libfuzzer/patches/15-return-to-exit.patch b/tools/fuzzing/libfuzzer/patches/15-return-to-exit.patch new file mode 100644 index 0000000000..14923f9363 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/15-return-to-exit.patch @@ -0,0 +1,968 @@ +commit f80733b3b1b5e05e7dfd7a071f60050fe20108c3 +Author: Jesse Schwartzentruber <truber@mozilla.com> +Date: Mon Mar 1 15:47:38 2021 -0500 + + [libfuzzer] In most cases, return instead of exit(). + +diff --git a/FuzzerDataFlowTrace.cpp b/FuzzerDataFlowTrace.cpp +index 0e9cdf7e66b1..06ea287a3cfe 100644 +--- a/FuzzerDataFlowTrace.cpp ++++ b/FuzzerDataFlowTrace.cpp +@@ -102,9 +102,11 @@ Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { + return Res; + } + +-void DataFlowTrace::ReadCoverage(const std::string &DirPath) { ++int DataFlowTrace::ReadCoverage(const std::string &DirPath) { + Vector<SizedFile> Files; +- GetSizedFilesFromDir(DirPath, &Files); ++ int Res = GetSizedFilesFromDir(DirPath, &Files); ++ if (Res != 0) ++ return Res; + for (auto &SF : Files) { + auto Name = Basename(SF.File); + if (Name == kFunctionsTxt) continue; +@@ -112,6 +114,7 @@ void DataFlowTrace::ReadCoverage(const std::string &DirPath) { + std::ifstream IF(SF.File); + Coverage.AppendCoverage(IF); + } ++ return 0; + } + + static void DFTStringAppendToVector(Vector<uint8_t> *DFT, +@@ -157,12 +160,14 @@ static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum, + return true; + } + +-bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, ++int DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + Vector<SizedFile> &CorporaFiles, Random &Rand) { +- if (DirPath.empty()) return false; ++ if (DirPath.empty()) return 0; + Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str()); + Vector<SizedFile> Files; +- GetSizedFilesFromDir(DirPath, &Files); ++ int Res = GetSizedFilesFromDir(DirPath, &Files); ++ if (Res != 0) ++ return Res; + std::string L; + size_t FocusFuncIdx = SIZE_MAX; + Vector<std::string> FunctionNames; +@@ -181,14 +186,16 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + FocusFuncIdx = NumFunctions - 1; + } + if (!NumFunctions) +- return false; ++ return 0; + + if (*FocusFunction == "auto") { + // AUTOFOCUS works like this: + // * reads the coverage data from the DFT files. + // * assigns weights to functions based on coverage. + // * chooses a random function according to the weights. +- ReadCoverage(DirPath); ++ Res = ReadCoverage(DirPath); ++ if (Res != 0) ++ return Res; + auto Weights = Coverage.FunctionWeights(NumFunctions); + Vector<double> Intervals(NumFunctions + 1); + std::iota(Intervals.begin(), Intervals.end(), 0); +@@ -209,7 +216,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + } + + if (!NumFunctions || FocusFuncIdx == SIZE_MAX || Files.size() <= 1) +- return false; ++ return 0; + + // Read traces. + size_t NumTraceFiles = 0; +@@ -228,8 +235,10 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + FunctionNum == FocusFuncIdx) { + NumTracesWithFocusFunction++; + +- if (FunctionNum >= NumFunctions) +- return ParseError("N is greater than the number of functions", L); ++ if (FunctionNum >= NumFunctions) { ++ ParseError("N is greater than the number of functions", L); ++ return 0; ++ } + Traces[Name] = DFTStringToVector(DFTString); + // Print just a few small traces. + if (NumTracesWithFocusFunction <= 3 && DFTString.size() <= 16) +@@ -241,7 +250,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + Printf("INFO: DataFlowTrace: %zd trace files, %zd functions, " + "%zd traces with focus function\n", + NumTraceFiles, NumFunctions, NumTracesWithFocusFunction); +- return NumTraceFiles > 0; ++ return 0; + } + + int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, +diff --git a/FuzzerDataFlowTrace.h b/FuzzerDataFlowTrace.h +index d6e3de30a4ef..767bad24f1d0 100644 +--- a/FuzzerDataFlowTrace.h ++++ b/FuzzerDataFlowTrace.h +@@ -113,8 +113,8 @@ class BlockCoverage { + + class DataFlowTrace { + public: +- void ReadCoverage(const std::string &DirPath); +- bool Init(const std::string &DirPath, std::string *FocusFunction, ++ int ReadCoverage(const std::string &DirPath); ++ int Init(const std::string &DirPath, std::string *FocusFunction, + Vector<SizedFile> &CorporaFiles, Random &Rand); + void Clear() { Traces.clear(); } + const Vector<uint8_t> *Get(const std::string &InputSha1) const { +diff --git a/FuzzerDriver.cpp b/FuzzerDriver.cpp +index cd720200848b..bedad16efa7b 100644 +--- a/FuzzerDriver.cpp ++++ b/FuzzerDriver.cpp +@@ -326,7 +326,7 @@ int CleanseCrashInput(const Vector<std::string> &Args, + if (Inputs->size() != 1 || !Flags.exact_artifact_path) { + Printf("ERROR: -cleanse_crash should be given one input file and" + " -exact_artifact_path\n"); +- exit(1); ++ return 1; + } + std::string InputFilePath = Inputs->at(0); + std::string OutputFilePath = Flags.exact_artifact_path; +@@ -380,7 +380,7 @@ int MinimizeCrashInput(const Vector<std::string> &Args, + const FuzzingOptions &Options) { + if (Inputs->size() != 1) { + Printf("ERROR: -minimize_crash should be given one input file\n"); +- exit(1); ++ return 1; + } + std::string InputFilePath = Inputs->at(0); + Command BaseCmd(Args); +@@ -411,7 +411,7 @@ int MinimizeCrashInput(const Vector<std::string> &Args, + bool Success = ExecuteCommand(Cmd, &CmdOutput); + if (Success) { + Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); +- exit(1); ++ return 1; + } + Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " + "it further\n", +@@ -466,42 +466,51 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { + Printf("INFO: Starting MinimizeCrashInputInternalStep: %zd\n", U.size()); + if (U.size() < 2) { + Printf("INFO: The input is small enough, exiting\n"); +- exit(0); ++ return 0; + } + F->SetMaxInputLen(U.size()); + F->SetMaxMutationLen(U.size() - 1); + F->MinimizeCrashLoop(U); + Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n"); +- exit(0); + return 0; + } + +-void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, ++int Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, + const Vector<std::string> &Corpora, const char *CFPathOrNull) { + if (Corpora.size() < 2) { + Printf("INFO: Merge requires two or more corpus dirs\n"); +- exit(0); ++ return 0; + } + + Vector<SizedFile> OldCorpus, NewCorpus; +- GetSizedFilesFromDir(Corpora[0], &OldCorpus); +- for (size_t i = 1; i < Corpora.size(); i++) +- GetSizedFilesFromDir(Corpora[i], &NewCorpus); ++ int Res = GetSizedFilesFromDir(Corpora[0], &OldCorpus); ++ if (Res != 0) ++ return Res; ++ for (size_t i = 1; i < Corpora.size(); i++) { ++ Res = GetSizedFilesFromDir(Corpora[i], &NewCorpus); ++ if (Res != 0) ++ return Res; ++ } + std::sort(OldCorpus.begin(), OldCorpus.end()); + std::sort(NewCorpus.begin(), NewCorpus.end()); + + std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt"); + Vector<std::string> NewFiles; + Set<uint32_t> NewFeatures, NewCov; +- CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, ++ Res = CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, + {}, &NewCov, CFPath, true); ++ if (Res != 0) ++ return Res; ++ ++ if (F->isGracefulExitRequested()) ++ return 0; + for (auto &Path : NewFiles) + F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); + // We are done, delete the control file if it was a temporary one. + if (!Flags.merge_control_file) + RemoveFile(CFPath); + +- exit(0); ++ return 0; + } + + int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, +@@ -570,10 +579,9 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, + return 0; + } + +-Vector<std::string> ParseSeedInuts(const char *seed_inputs) { ++int ParseSeedInuts(const char *seed_inputs, Vector<std::string> &Files) { + // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file +- Vector<std::string> Files; +- if (!seed_inputs) return Files; ++ if (!seed_inputs) return 0; + std::string SeedInputs; + if (Flags.seed_inputs[0] == '@') + SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. +@@ -581,7 +589,7 @@ Vector<std::string> ParseSeedInuts(const char *seed_inputs) { + SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. + if (SeedInputs.empty()) { + Printf("seed_inputs is empty or @file does not exist.\n"); +- exit(1); ++ return 1; + } + // Parse SeedInputs. + size_t comma_pos = 0; +@@ -590,7 +598,7 @@ Vector<std::string> ParseSeedInuts(const char *seed_inputs) { + SeedInputs = SeedInputs.substr(0, comma_pos); + } + Files.push_back(SeedInputs); +- return Files; ++ return 0; + } + + static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs, +@@ -624,7 +632,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + ProgName = new std::string(Args[0]); + if (Argv0 != *ProgName) { + Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n"); +- exit(1); ++ return 1; + } + ParseFlags(Args, EF); + if (Flags.help) { +@@ -723,7 +731,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + if (!Options.FocusFunction.empty()) { + Printf("ERROR: The parameters `--entropic` and `--focus_function` cannot " + "be used together.\n"); +- exit(1); ++ return 1; + } + Printf("INFO: Running with entropic power schedule (0x%X, %d).\n", + Options.EntropicFeatureFrequencyThreshold, +@@ -809,22 +817,21 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + "*** executed the target code on a fixed set of inputs.\n" + "***\n"); + F->PrintFinalStats(); +- exit(0); ++ return 0; + } + + if (Flags.fork) +- FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); ++ return FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + + if (Flags.merge) +- Merge(F, Options, Args, *Inputs, Flags.merge_control_file); ++ return Merge(F, Options, Args, *Inputs, Flags.merge_control_file); + + if (Flags.merge_inner) { + const size_t kDefaultMaxMergeLen = 1 << 20; + if (Options.MaxLen == 0) + F->SetMaxInputLen(kDefaultMaxMergeLen); + assert(Flags.merge_control_file); +- F->CrashResistantMergeInternalStep(Flags.merge_control_file); +- exit(0); ++ return F->CrashResistantMergeInternalStep(Flags.merge_control_file); + } + + if (Flags.analyze_dict) { +@@ -842,21 +849,31 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + } + if (AnalyzeDictionary(F, Dictionary, InitialCorpus)) { + Printf("Dictionary analysis failed\n"); +- exit(1); ++ return 1; + } + Printf("Dictionary analysis succeeded\n"); +- exit(0); ++ return 0; + } + +- auto CorporaFiles = ReadCorpora(*Inputs, ParseSeedInuts(Flags.seed_inputs)); +- F->Loop(CorporaFiles); ++ { ++ Vector<std::string> Files; ++ int Res = ParseSeedInuts(Flags.seed_inputs, Files); ++ if (Res != 0) ++ return Res; ++ auto CorporaFiles = ReadCorpora(*Inputs, Files); ++ Res = F->Loop(CorporaFiles); ++ if (Res != 0) ++ return Res; ++ if (F->isGracefulExitRequested()) ++ return 0; ++ } + + if (Flags.verbosity) + Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), + F->secondsSinceProcessStartUp()); + F->PrintFinalStats(); + +- exit(0); // Don't let F destroy itself. ++ return 0; // Don't let F destroy itself. + } + + extern "C" ATTRIBUTE_INTERFACE int +diff --git a/FuzzerFork.cpp b/FuzzerFork.cpp +index d9e6b79443e0..ee2a99a250c1 100644 +--- a/FuzzerFork.cpp ++++ b/FuzzerFork.cpp +@@ -177,14 +177,16 @@ struct GlobalEnv { + return Job; + } + +- void RunOneMergeJob(FuzzJob *Job) { ++ int RunOneMergeJob(FuzzJob *Job) { + auto Stats = ParseFinalStatsFromLog(Job->LogPath); + NumRuns += Stats.number_of_executed_units; + + Vector<SizedFile> TempFiles, MergeCandidates; + // Read all newly created inputs and their feature sets. + // Choose only those inputs that have new features. +- GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); ++ int Res = GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); ++ if (Res != 0) ++ return Res; + std::sort(TempFiles.begin(), TempFiles.end()); + for (auto &F : TempFiles) { + auto FeatureFile = F.File; +@@ -207,12 +209,14 @@ struct GlobalEnv { + Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, + secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds); + +- if (MergeCandidates.empty()) return; ++ if (MergeCandidates.empty()) return 0; + + Vector<std::string> FilesToAdd; + Set<uint32_t> NewFeatures, NewCov; + CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, + &NewFeatures, Cov, &NewCov, Job->CFPath, false); ++ if (Fuzzer::isGracefulExitRequested()) ++ return 0; + for (auto &Path : FilesToAdd) { + auto U = FileToVector(Path); + auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); +@@ -226,7 +230,7 @@ struct GlobalEnv { + if (TPC.PcIsFuncEntry(TE)) + PrintPC(" NEW_FUNC: %p %F %L\n", "", + TPC.GetNextInstructionPc(TE->PC)); +- ++ return 0; + } + + +@@ -280,7 +284,7 @@ void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { + } + + // This is just a skeleton of an experimental -fork=1 feature. +-void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, ++int FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector<std::string> &Args, + const Vector<std::string> &CorpusDirs, int NumJobs) { + Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); +@@ -294,8 +298,12 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + Env.DataFlowBinary = Options.CollectDataFlow; + + Vector<SizedFile> SeedFiles; +- for (auto &Dir : CorpusDirs) +- GetSizedFilesFromDir(Dir, &SeedFiles); ++ int Res; ++ for (auto &Dir : CorpusDirs) { ++ Res = GetSizedFilesFromDir(Dir, &SeedFiles); ++ if (Res != 0) ++ return Res; ++ } + std::sort(SeedFiles.begin(), SeedFiles.end()); + Env.TempDir = TempPath("FuzzWithFork", ".dir"); + Env.DFTDir = DirPlusFile(Env.TempDir, "DFT"); +@@ -310,9 +318,14 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); +- CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, ++ Res = CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, + {}, &Env.Cov, + CFPath, false); ++ if (Res != 0) ++ return Res; ++ if (Fuzzer::isGracefulExitRequested()) ++ return 0; ++ + RemoveFile(CFPath); + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, + Env.Files.size(), Env.TempDir.c_str()); +@@ -345,9 +358,14 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + StopJobs(); + break; + } +- Fuzzer::MaybeExitGracefully(); ++ if (Fuzzer::MaybeExitGracefully()) ++ return 0; + +- Env.RunOneMergeJob(Job.get()); ++ Res = Env.RunOneMergeJob(Job.get()); ++ if (Res != 0) ++ return Res; ++ if (Fuzzer::isGracefulExitRequested()) ++ return 0; + + // Continue if our crash is one of the ignorred ones. + if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) +@@ -403,7 +421,7 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + // Use the exit code from the last child process. + Printf("INFO: exiting: %d time: %zds\n", ExitCode, + Env.secondsSinceProcessStartUp()); +- exit(ExitCode); ++ return ExitCode; + } + + } // namespace fuzzer +diff --git a/FuzzerFork.h b/FuzzerFork.h +index b29a43e13fbc..1352171ad49d 100644 +--- a/FuzzerFork.h ++++ b/FuzzerFork.h +@@ -16,7 +16,7 @@ + #include <string> + + namespace fuzzer { +-void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, ++int FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector<std::string> &Args, + const Vector<std::string> &CorpusDirs, int NumJobs); + } // namespace fuzzer +diff --git a/FuzzerIO.cpp b/FuzzerIO.cpp +index 0053ef39f2b9..6be2be67c691 100644 +--- a/FuzzerIO.cpp ++++ b/FuzzerIO.cpp +@@ -82,7 +82,9 @@ void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, + long *Epoch, size_t MaxSize, bool ExitOnError) { + long E = Epoch ? *Epoch : 0; + Vector<std::string> Files; +- ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); ++ int Res = ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); ++ if (ExitOnError && Res != 0) ++ exit(Res); + size_t NumLoaded = 0; + for (size_t i = 0; i < Files.size(); i++) { + auto &X = Files[i]; +@@ -97,12 +99,15 @@ void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, + } + + +-void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) { ++int GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) { + Vector<std::string> Files; +- ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); ++ int Res = ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); ++ if (Res != 0) ++ return Res; + for (auto &File : Files) + if (size_t Size = FileSize(File)) + V->push_back({File, Size}); ++ return 0; + } + + std::string DirPlusFile(const std::string &DirPath, +diff --git a/FuzzerIO.h b/FuzzerIO.h +index 6e4368b971fa..6c90ba637322 100644 +--- a/FuzzerIO.h ++++ b/FuzzerIO.h +@@ -60,7 +60,7 @@ void RawPrint(const char *Str); + bool IsFile(const std::string &Path); + size_t FileSize(const std::string &Path); + +-void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, ++int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector<std::string> *V, bool TopDir); + + void RmDirRecursive(const std::string &Dir); +@@ -79,7 +79,7 @@ struct SizedFile { + bool operator<(const SizedFile &B) const { return Size < B.Size; } + }; + +-void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V); ++int GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V); + + char GetSeparator(); + // Similar to the basename utility: returns the file name w/o the dir prefix. +diff --git a/FuzzerIOPosix.cpp b/FuzzerIOPosix.cpp +index 4b453d286c80..1a50295c010f 100644 +--- a/FuzzerIOPosix.cpp ++++ b/FuzzerIOPosix.cpp +@@ -53,16 +53,16 @@ std::string Basename(const std::string &Path) { + return Path.substr(Pos + 1); + } + +-void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, ++int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector<std::string> *V, bool TopDir) { + auto E = GetEpoch(Dir); + if (Epoch) +- if (E && *Epoch >= E) return; ++ if (E && *Epoch >= E) return 0; + + DIR *D = opendir(Dir.c_str()); + if (!D) { + Printf("%s: %s; exiting\n", strerror(errno), Dir.c_str()); +- exit(1); ++ return 1; + } + while (auto E = readdir(D)) { + std::string Path = DirPlusFile(Dir, E->d_name); +@@ -71,12 +71,16 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + V->push_back(Path); + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && +- *E->d_name != '.') +- ListFilesInDirRecursive(Path, Epoch, V, false); ++ *E->d_name != '.') { ++ int Res = ListFilesInDirRecursive(Path, Epoch, V, false); ++ if (Res != 0) ++ return Res; ++ } + } + closedir(D); + if (Epoch && TopDir) + *Epoch = E; ++ return 0; + } + + +diff --git a/FuzzerIOWindows.cpp b/FuzzerIOWindows.cpp +index 651283a551cf..0e977bd02557 100644 +--- a/FuzzerIOWindows.cpp ++++ b/FuzzerIOWindows.cpp +@@ -98,11 +98,12 @@ size_t FileSize(const std::string &Path) { + return size.QuadPart; + } + +-void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, ++int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector<std::string> *V, bool TopDir) { ++ int Res; + auto E = GetEpoch(Dir); + if (Epoch) +- if (E && *Epoch >= E) return; ++ if (E && *Epoch >= E) return 0; + + std::string Path(Dir); + assert(!Path.empty()); +@@ -116,9 +117,9 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + if (FindHandle == INVALID_HANDLE_VALUE) + { + if (GetLastError() == ERROR_FILE_NOT_FOUND) +- return; ++ return 0; + Printf("No such file or directory: %s; exiting\n", Dir.c_str()); +- exit(1); ++ return 1; + } + + do { +@@ -131,7 +132,9 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + FindInfo.cFileName[1] == '.')) + continue; + +- ListFilesInDirRecursive(FileName, Epoch, V, false); ++ int Res = ListFilesInDirRecursive(FileName, Epoch, V, false); ++ if (Res != 0) ++ return Res; + } + else if (IsFile(FileName, FindInfo.dwFileAttributes)) + V->push_back(FileName); +@@ -145,6 +148,7 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + + if (Epoch && TopDir) + *Epoch = E; ++ return 0; + } + + +diff --git a/FuzzerInternal.h b/FuzzerInternal.h +index 1f7d671ed848..cc2650b58ef1 100644 +--- a/FuzzerInternal.h ++++ b/FuzzerInternal.h +@@ -35,8 +35,8 @@ public: + Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, + FuzzingOptions Options); + ~Fuzzer(); +- void Loop(Vector<SizedFile> &CorporaFiles); +- void ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles); ++ int Loop(Vector<SizedFile> &CorporaFiles); ++ int ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles); + void MinimizeCrashLoop(const Unit &U); + void RereadOutputCorpus(size_t MaxSize); + +@@ -65,13 +65,16 @@ public: + static void StaticFileSizeExceedCallback(); + static void StaticGracefulExitCallback(); + ++ static void GracefullyExit(); ++ static bool isGracefulExitRequested(); ++ + int ExecuteCallback(const uint8_t *Data, size_t Size); + bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, + InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); + + // Merge Corpora[1:] into Corpora[0]. + void Merge(const Vector<std::string> &Corpora); +- void CrashResistantMergeInternalStep(const std::string &ControlFilePath); ++ int CrashResistantMergeInternalStep(const std::string &ControlFilePath); + MutationDispatcher &GetMD() { return MD; } + void PrintFinalStats(); + void SetMaxInputLen(size_t MaxInputLen); +@@ -84,7 +87,7 @@ public: + bool DuringInitialCorpusExecution); + + void HandleMalloc(size_t Size); +- static void MaybeExitGracefully(); ++ static bool MaybeExitGracefully(); + std::string WriteToOutputCorpus(const Unit &U); + + private: +@@ -93,7 +96,7 @@ private: + void ExitCallback(); + void CrashOnOverwrittenData(); + void InterruptCallback(); +- void MutateAndTestOne(); ++ bool MutateAndTestOne(); + void PurgeAllocator(); + void ReportNewCoverage(InputInfo *II, const Unit &U); + void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size); +diff --git a/FuzzerLoop.cpp b/FuzzerLoop.cpp +index 4c4e8c271b1f..e7dfc187dbfe 100644 +--- a/FuzzerLoop.cpp ++++ b/FuzzerLoop.cpp +@@ -254,12 +254,20 @@ void Fuzzer::ExitCallback() { + _Exit(Options.ErrorExitCode); + } + +-void Fuzzer::MaybeExitGracefully() { +- if (!F->GracefulExitRequested) return; ++bool Fuzzer::MaybeExitGracefully() { ++ if (!F->GracefulExitRequested) return false; + Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid()); + RmDirRecursive(TempPath("FuzzWithFork", ".dir")); + F->PrintFinalStats(); +- _Exit(0); ++ return true; ++} ++ ++void Fuzzer::GracefullyExit() { ++ F->GracefulExitRequested = true; ++} ++ ++bool Fuzzer::isGracefulExitRequested() { ++ return F->GracefulExitRequested; + } + + void Fuzzer::InterruptCallback() { +@@ -663,7 +671,7 @@ void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size, + } + } + +-void Fuzzer::MutateAndTestOne() { ++bool Fuzzer::MutateAndTestOne() { + MD.StartMutationSequence(); + + auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); +@@ -685,7 +693,7 @@ void Fuzzer::MutateAndTestOne() { + for (int i = 0; i < Options.MutateDepth; i++) { + if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) + break; +- MaybeExitGracefully(); ++ if (MaybeExitGracefully()) return true; + size_t NewSize = 0; + if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() && + Size <= CurrentMaxMutationLen) +@@ -719,6 +727,7 @@ void Fuzzer::MutateAndTestOne() { + } + + II.NeedsEnergyUpdate = true; ++ return false; + } + + void Fuzzer::PurgeAllocator() { +@@ -736,7 +745,7 @@ void Fuzzer::PurgeAllocator() { + LastAllocatorPurgeAttemptTime = system_clock::now(); + } + +-void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { ++int Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { + const size_t kMaxSaneLen = 1 << 20; + const size_t kMinDefaultLen = 4096; + size_t MaxSize = 0; +@@ -795,16 +804,23 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { + if (Corpus.empty() && Options.MaxNumberOfRuns) { + Printf("ERROR: no interesting inputs were found. " + "Is the code instrumented for coverage? Exiting.\n"); +- exit(1); ++ return 1; + } ++ return 0; + } + +-void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) { ++int Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) { + auto FocusFunctionOrAuto = Options.FocusFunction; +- DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, ++ int Res = DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, + MD.GetRand()); +- TPC.SetFocusFunction(FocusFunctionOrAuto); +- ReadAndExecuteSeedCorpora(CorporaFiles); ++ if (Res != 0) ++ return Res; ++ Res = TPC.SetFocusFunction(FocusFunctionOrAuto); ++ if (Res != 0) ++ return Res; ++ Res = ReadAndExecuteSeedCorpora(CorporaFiles); ++ if (Res != 0) ++ return Res; + DFT.Clear(); // No need for DFT any more. + TPC.SetPrintNewPCs(Options.PrintNewCovPcs); + TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs); +@@ -842,13 +858,15 @@ void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) { + } + + // Perform several mutations and runs. +- MutateAndTestOne(); ++ if (MutateAndTestOne()) ++ return 0; + + PurgeAllocator(); + } + + PrintStats("DONE ", "\n"); + MD.PrintRecommendedDictionary(); ++ return 0; + } + + void Fuzzer::MinimizeCrashLoop(const Unit &U) { +diff --git a/FuzzerMerge.cpp b/FuzzerMerge.cpp +index 919eea848580..0a185c7325bb 100644 +--- a/FuzzerMerge.cpp ++++ b/FuzzerMerge.cpp +@@ -28,11 +28,12 @@ bool Merger::Parse(const std::string &Str, bool ParseCoverage) { + return Parse(SS, ParseCoverage); + } + +-void Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { ++int Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { + if (!Parse(IS, ParseCoverage)) { + Printf("MERGE: failed to parse the control file (unexpected error)\n"); +- exit(1); ++ return 1; + } ++ return 0; + } + + // The control file example: +@@ -194,11 +195,13 @@ Set<uint32_t> Merger::AllFeatures() const { + } + + // Inner process. May crash if the target crashes. +-void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { ++int Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { + Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); + Merger M; + std::ifstream IF(CFPath); +- M.ParseOrExit(IF, false); ++ int Res = M.ParseOrExit(IF, false); ++ if (Res != 0) ++ return Res; + IF.close(); + if (!M.LastFailure.empty()) + Printf("MERGE-INNER: '%s' caused a failure at the previous merge step\n", +@@ -216,7 +219,8 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { + }; + Set<const TracePC::PCTableEntry *> AllPCs; + for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { +- Fuzzer::MaybeExitGracefully(); ++ if (Fuzzer::MaybeExitGracefully()) ++ return 0; + auto U = FileToVector(M.Files[i].Name); + if (U.size() > MaxInputLen) { + U.resize(MaxInputLen); +@@ -261,12 +265,14 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { + OF.flush(); + } + PrintStatsWrapper("DONE "); ++ return 0; + } + +-static size_t WriteNewControlFile(const std::string &CFPath, ++static int WriteNewControlFile(const std::string &CFPath, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, +- const Vector<MergeFileInfo> &KnownFiles) { ++ const Vector<MergeFileInfo> &KnownFiles, ++ size_t &NumFiles) { + std::unordered_set<std::string> FilesToSkip; + for (auto &SF: KnownFiles) + FilesToSkip.insert(SF.Name); +@@ -292,14 +298,15 @@ static size_t WriteNewControlFile(const std::string &CFPath, + if (!ControlFile) { + Printf("MERGE-OUTER: failed to write to the control file: %s\n", + CFPath.c_str()); +- exit(1); ++ return 1; + } + +- return FilesToUse.size(); ++ NumFiles = FilesToUse.size(); ++ return 0; + } + + // Outer process. Does not call the target code and thus should not fail. +-void CrashResistantMerge(const Vector<std::string> &Args, ++int CrashResistantMerge(const Vector<std::string> &Args, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, + Vector<std::string> *NewFiles, +@@ -309,8 +316,9 @@ void CrashResistantMerge(const Vector<std::string> &Args, + Set<uint32_t> *NewCov, + const std::string &CFPath, + bool V /*Verbose*/) { +- if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. ++ if (NewCorpus.empty() && OldCorpus.empty()) return 0; // Nothing to merge. + size_t NumAttempts = 0; ++ int Res; + Vector<MergeFileInfo> KnownFiles; + if (FileSize(CFPath)) { + VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", +@@ -331,7 +339,8 @@ void CrashResistantMerge(const Vector<std::string> &Args, + VPrintf( + V, + "MERGE-OUTER: nothing to do, merge has been completed before\n"); +- exit(0); ++ Fuzzer::GracefullyExit(); ++ return 0; + } + + // Number of input files likely changed, start merge from scratch, but +@@ -356,7 +365,9 @@ void CrashResistantMerge(const Vector<std::string> &Args, + "%zd files, %zd in the initial corpus, %zd processed earlier\n", + OldCorpus.size() + NewCorpus.size(), OldCorpus.size(), + KnownFiles.size()); +- NumAttempts = WriteNewControlFile(CFPath, OldCorpus, NewCorpus, KnownFiles); ++ Res = WriteNewControlFile(CFPath, OldCorpus, NewCorpus, KnownFiles, NumAttempts); ++ if (Res != 0) ++ return Res; + } + + // Execute the inner process until it passes. +@@ -366,7 +377,8 @@ void CrashResistantMerge(const Vector<std::string> &Args, + BaseCmd.removeFlag("fork"); + BaseCmd.removeFlag("collect_data_flow"); + for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { +- Fuzzer::MaybeExitGracefully(); ++ if (Fuzzer::MaybeExitGracefully()) ++ return 0; + VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); + Command Cmd(BaseCmd); + Cmd.addFlag("merge_control_file", CFPath); +@@ -388,7 +400,9 @@ void CrashResistantMerge(const Vector<std::string> &Args, + VPrintf(V, "MERGE-OUTER: the control file has %zd bytes\n", + (size_t)IF.tellg()); + IF.seekg(0, IF.beg); +- M.ParseOrExit(IF, true); ++ Res = M.ParseOrExit(IF, true); ++ if (Res != 0) ++ return Res; + IF.close(); + VPrintf(V, + "MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", +@@ -399,6 +413,7 @@ void CrashResistantMerge(const Vector<std::string> &Args, + VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " + "%zd new coverage edges\n", + NewFiles->size(), NewFeatures->size(), NewCov->size()); ++ return 0; + } + + } // namespace fuzzer +diff --git a/FuzzerMerge.h b/FuzzerMerge.h +index e0c6bc539bdb..6dc1c4c45abf 100644 +--- a/FuzzerMerge.h ++++ b/FuzzerMerge.h +@@ -63,7 +63,7 @@ struct Merger { + + bool Parse(std::istream &IS, bool ParseCoverage); + bool Parse(const std::string &Str, bool ParseCoverage); +- void ParseOrExit(std::istream &IS, bool ParseCoverage); ++ int ParseOrExit(std::istream &IS, bool ParseCoverage); + size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures, + const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, + Vector<std::string> *NewFiles); +@@ -71,7 +71,7 @@ struct Merger { + Set<uint32_t> AllFeatures() const; + }; + +-void CrashResistantMerge(const Vector<std::string> &Args, ++int CrashResistantMerge(const Vector<std::string> &Args, + const Vector<SizedFile> &OldCorpus, + const Vector<SizedFile> &NewCorpus, + Vector<std::string> *NewFiles, +diff --git a/FuzzerTracePC.cpp b/FuzzerTracePC.cpp +index b2ca7693e540..fbceda39bc22 100644 +--- a/FuzzerTracePC.cpp ++++ b/FuzzerTracePC.cpp +@@ -238,13 +238,13 @@ void TracePC::IterateCoveredFunctions(CallBack CB) { + } + } + +-void TracePC::SetFocusFunction(const std::string &FuncName) { ++int TracePC::SetFocusFunction(const std::string &FuncName) { + // This function should be called once. + assert(!FocusFunctionCounterPtr); + // "auto" is not a valid function name. If this function is called with "auto" + // that means the auto focus functionality failed. + if (FuncName.empty() || FuncName == "auto") +- return; ++ return 0; + for (size_t M = 0; M < NumModules; M++) { + auto &PCTE = ModulePCTable[M]; + size_t N = PCTE.Stop - PCTE.Start; +@@ -256,13 +256,13 @@ void TracePC::SetFocusFunction(const std::string &FuncName) { + if (FuncName != Name) continue; + Printf("INFO: Focus function is set to '%s'\n", Name.c_str()); + FocusFunctionCounterPtr = Modules[M].Start() + I; +- return; ++ return 0; + } + } + + Printf("ERROR: Failed to set focus function. Make sure the function name is " + "valid (%s) and symbolization is enabled.\n", FuncName.c_str()); +- exit(1); ++ return 1; + } + + bool TracePC::ObservedFocusFunction() { +diff --git a/FuzzerTracePC.h b/FuzzerTracePC.h +index 501f3b544971..b46ebb909dbf 100644 +--- a/FuzzerTracePC.h ++++ b/FuzzerTracePC.h +@@ -116,7 +116,7 @@ class TracePC { + CB(PC); + } + +- void SetFocusFunction(const std::string &FuncName); ++ int SetFocusFunction(const std::string &FuncName); + bool ObservedFocusFunction(); + + struct PCTableEntry { diff --git a/tools/fuzzing/messagemanager/MessageManagerFuzzer.cpp b/tools/fuzzing/messagemanager/MessageManagerFuzzer.cpp new file mode 100644 index 0000000000..6a4475641f --- /dev/null +++ b/tools/fuzzing/messagemanager/MessageManagerFuzzer.cpp @@ -0,0 +1,327 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <climits> +#include <cmath> +#include "FuzzingTraits.h" +#include "jsapi.h" +#include "jsfriendapi.h" +#include "js/CharacterEncoding.h" +#include "js/Exception.h" +#include "js/PropertyAndElement.h" // JS_Enumerate, JS_GetProperty, JS_GetPropertyById, JS_SetProperty, JS_SetPropertyById +#include "prenv.h" +#include "MessageManagerFuzzer.h" +#include "mozilla/ErrorResult.h" +#include "nsComponentManagerUtils.h" +#include "nsDebug.h" +#include "nsError.h" +#include "nsFrameMessageManager.h" +#include "nsJSUtils.h" +#include "nsXULAppAPI.h" +#include "nsNetCID.h" +#include "nsString.h" +#include "nsUnicharUtils.h" +#include "nsIFile.h" +#include "nsIFileStreams.h" +#include "nsILineInputStream.h" +#include "nsLocalFile.h" +#include "nsTArray.h" + +#ifdef IsLoggingEnabled +// This is defined in the Windows SDK urlmon.h +# undef IsLoggingEnabled +#endif + +#define MESSAGEMANAGER_FUZZER_DEFAULT_MUTATION_PROBABILITY 2 +#define MSGMGR_FUZZER_LOG(fmt, args...) \ + if (MessageManagerFuzzer::IsLoggingEnabled()) { \ + printf_stderr("[MessageManagerFuzzer] " fmt "\n", ##args); \ + } + +namespace mozilla { +namespace dom { + +using namespace fuzzing; +using namespace ipc; + +/* static */ +void MessageManagerFuzzer::ReadFile(const char* path, + nsTArray<nsCString>& aArray) { + nsCOMPtr<nsIFile> file; + nsresult rv = + NS_NewLocalFile(NS_ConvertUTF8toUTF16(path), true, getter_AddRefs(file)); + NS_ENSURE_SUCCESS_VOID(rv); + + bool exists = false; + rv = file->Exists(&exists); + if (NS_FAILED(rv) || !exists) { + return; + } + + nsCOMPtr<nsIFileInputStream> fileStream( + do_CreateInstance(NS_LOCALFILEINPUTSTREAM_CONTRACTID, &rv)); + NS_ENSURE_SUCCESS_VOID(rv); + + rv = fileStream->Init(file, -1, -1, false); + NS_ENSURE_SUCCESS_VOID(rv); + + nsCOMPtr<nsILineInputStream> lineStream(do_QueryInterface(fileStream, &rv)); + NS_ENSURE_SUCCESS_VOID(rv); + + nsAutoCString line; + bool more = true; + do { + rv = lineStream->ReadLine(line, &more); + NS_ENSURE_SUCCESS_VOID(rv); + aArray.AppendElement(line); + } while (more); +} + +/* static */ +bool MessageManagerFuzzer::IsMessageNameBlacklisted( + const nsAString& aMessageName) { + static bool sFileLoaded = false; + static nsTArray<nsCString> valuesInFile; + + if (!sFileLoaded) { + ReadFile(PR_GetEnv("MESSAGEMANAGER_FUZZER_BLACKLIST"), valuesInFile); + sFileLoaded = true; + } + + if (valuesInFile.Length() == 0) { + return false; + } + + return valuesInFile.Contains(NS_ConvertUTF16toUTF8(aMessageName).get()); +} + +/* static */ +nsCString MessageManagerFuzzer::GetFuzzValueFromFile() { + static bool sFileLoaded = false; + static nsTArray<nsCString> valuesInFile; + + if (!sFileLoaded) { + ReadFile(PR_GetEnv("MESSAGEMANAGER_FUZZER_STRINGSFILE"), valuesInFile); + sFileLoaded = true; + } + + // If something goes wrong with importing the file we return an empty string. + if (valuesInFile.Length() == 0) { + return nsCString(); + } + + unsigned randIdx = RandomIntegerRange<unsigned>(0, valuesInFile.Length()); + return valuesInFile.ElementAt(randIdx); +} + +/* static */ +void MessageManagerFuzzer::MutateObject(JSContext* aCx, + JS::Handle<JS::Value> aValue, + unsigned short int aRecursionCounter) { + JS::Rooted<JSObject*> object(aCx, &aValue.toObject()); + JS::Rooted<JS::IdVector> ids(aCx, JS::IdVector(aCx)); + + if (!JS_Enumerate(aCx, object, &ids)) { + return; + } + + for (size_t i = 0, n = ids.length(); i < n; i++) { + // Retrieve Property name. + nsAutoJSString propName; + if (!propName.init(aCx, ids[i])) { + continue; + } + MSGMGR_FUZZER_LOG("%*s- Property: %s", aRecursionCounter * 4, "", + NS_ConvertUTF16toUTF8(propName).get()); + + // The likelihood when a value gets fuzzed of this object. + if (!FuzzingTraits::Sometimes(DefaultMutationProbability())) { + continue; + } + + // Retrieve Property value. + JS::Rooted<JS::Value> propertyValue(aCx); + JS_GetPropertyById(aCx, object, ids[i], &propertyValue); + + JS::Rooted<JS::Value> newPropValue(aCx); + MutateValue(aCx, propertyValue, &newPropValue, aRecursionCounter); + + JS_SetPropertyById(aCx, object, ids[i], newPropValue); + } +} + +/* static */ +bool MessageManagerFuzzer::MutateValue( + JSContext* aCx, JS::Handle<JS::Value> aValue, + JS::MutableHandle<JS::Value> aOutMutationValue, + unsigned short int aRecursionCounter) { + if (aValue.isInt32()) { + if (FuzzingTraits::Sometimes(DefaultMutationProbability() * 2)) { + aOutMutationValue.set(JS::Int32Value(RandomNumericLimit<int>())); + } else { + aOutMutationValue.set(JS::Int32Value(RandomInteger<int>())); + } + MSGMGR_FUZZER_LOG("%*s! Mutated value of type |int32|: '%d' to '%d'", + aRecursionCounter * 4, "", aValue.toInt32(), + aOutMutationValue.toInt32()); + return true; + } + + if (aValue.isDouble()) { + aOutMutationValue.set(JS::DoubleValue(RandomFloatingPoint<double>())); + MSGMGR_FUZZER_LOG("%*s! Mutated value of type |double|: '%f' to '%f'", + aRecursionCounter * 4, "", aValue.toDouble(), + aOutMutationValue.toDouble()); + return true; + } + + if (aValue.isBoolean()) { + aOutMutationValue.set(JS::BooleanValue(bool(RandomIntegerRange(0, 2)))); + MSGMGR_FUZZER_LOG("%*s! Mutated value of type |boolean|: '%d' to '%d'", + aRecursionCounter * 4, "", aValue.toBoolean(), + aOutMutationValue.toBoolean()); + return true; + } + + if (aValue.isString()) { + nsCString x = GetFuzzValueFromFile(); + if (x.IsEmpty()) { + return false; + } + JSString* str = JS_NewStringCopyZ(aCx, x.get()); + aOutMutationValue.set(JS::StringValue(str)); + JS::Rooted<JSString*> rootedValue(aCx, aValue.toString()); + JS::UniqueChars valueChars = JS_EncodeStringToUTF8(aCx, rootedValue); + MSGMGR_FUZZER_LOG("%*s! Mutated value of type |string|: '%s' to '%s'", + aRecursionCounter * 4, "", valueChars.get(), x.get()); + return true; + } + + if (aValue.isObject()) { + aRecursionCounter++; + MSGMGR_FUZZER_LOG("%*s<Enumerating found object>", aRecursionCounter * 4, + ""); + MutateObject(aCx, aValue, aRecursionCounter); + aOutMutationValue.set(aValue); + return true; + } + + return false; +} + +/* static */ +bool MessageManagerFuzzer::Mutate(JSContext* aCx, const nsAString& aMessageName, + ipc::StructuredCloneData* aData, + const JS::Value& aTransfer) { + MSGMGR_FUZZER_LOG("Message: %s in process: %d", + NS_ConvertUTF16toUTF8(aMessageName).get(), + XRE_GetProcessType()); + + unsigned short int aRecursionCounter = 0; + ErrorResult rv; + JS::Rooted<JS::Value> t(aCx, aTransfer); + + /* Read original StructuredCloneData. */ + JS::Rooted<JS::Value> scdContent(aCx); + aData->Read(aCx, &scdContent, rv); + if (NS_WARN_IF(rv.Failed())) { + rv.SuppressException(); + JS_ClearPendingException(aCx); + return false; + } + + JS::Rooted<JS::Value> scdMutationContent(aCx); + bool isMutated = + MutateValue(aCx, scdContent, &scdMutationContent, aRecursionCounter); + + /* Write mutated StructuredCloneData. */ + ipc::StructuredCloneData mutatedStructuredCloneData; + mutatedStructuredCloneData.Write(aCx, scdMutationContent, t, + JS::CloneDataPolicy(), rv); + if (NS_WARN_IF(rv.Failed())) { + rv.SuppressException(); + JS_ClearPendingException(aCx); + return false; + } + + // See: https://bugzilla.mozilla.org/show_bug.cgi?id=1346040 + aData->Copy(mutatedStructuredCloneData); + + /* Mutated and successfully written to StructuredCloneData object. */ + if (isMutated) { + JS::Rooted<JSString*> str(aCx, JS_ValueToSource(aCx, scdMutationContent)); + JS::UniqueChars strChars = JS_EncodeStringToUTF8(aCx, str); + MSGMGR_FUZZER_LOG("Mutated '%s' Message: %s", + NS_ConvertUTF16toUTF8(aMessageName).get(), + strChars.get()); + } + + return true; +} + +/* static */ +unsigned int MessageManagerFuzzer::DefaultMutationProbability() { + static unsigned long sPropValue = + MESSAGEMANAGER_FUZZER_DEFAULT_MUTATION_PROBABILITY; + static bool sInitialized = false; + + if (sInitialized) { + return sPropValue; + } + sInitialized = true; + + // Defines the likelihood of fuzzing a message. + const char* probability = + PR_GetEnv("MESSAGEMANAGER_FUZZER_MUTATION_PROBABILITY"); + if (probability) { + long n = std::strtol(probability, nullptr, 10); + if (n != 0) { + sPropValue = n; + return sPropValue; + } + } + + return sPropValue; +} + +/* static */ +bool MessageManagerFuzzer::IsLoggingEnabled() { + static bool sInitialized = false; + static bool sIsLoggingEnabled = false; + + if (!sInitialized) { + sIsLoggingEnabled = !!PR_GetEnv("MESSAGEMANAGER_FUZZER_ENABLE_LOGGING"); + sInitialized = true; + } + + return sIsLoggingEnabled; +} + +/* static */ +bool MessageManagerFuzzer::IsEnabled() { + return !!PR_GetEnv("MESSAGEMANAGER_FUZZER_ENABLE") && XRE_IsContentProcess(); +} + +/* static */ +void MessageManagerFuzzer::TryMutate(JSContext* aCx, + const nsAString& aMessageName, + ipc::StructuredCloneData* aData, + const JS::Value& aTransfer) { + if (!IsEnabled()) { + return; + } + + if (IsMessageNameBlacklisted(aMessageName)) { + MSGMGR_FUZZER_LOG("Blacklisted message: %s", + NS_ConvertUTF16toUTF8(aMessageName).get()); + return; + } + + Mutate(aCx, aMessageName, aData, aTransfer); +} + +} // namespace dom +} // namespace mozilla diff --git a/tools/fuzzing/messagemanager/MessageManagerFuzzer.h b/tools/fuzzing/messagemanager/MessageManagerFuzzer.h new file mode 100644 index 0000000000..fd7835055a --- /dev/null +++ b/tools/fuzzing/messagemanager/MessageManagerFuzzer.h @@ -0,0 +1,63 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_MessageManagerFuzzer_h__ +#define mozilla_dom_MessageManagerFuzzer_h__ + +#include "jspubtd.h" +#include "nsAString.h" +#include "nsTArray.h" + +namespace mozilla { +namespace dom { + +namespace ipc { +class StructuredCloneData; +} + +/* +Exposed environment variables: +MESSAGEMANAGER_FUZZER_ENABLE=1 +MESSAGEMANAGER_FUZZER_ENABLE_LOGGING=1 (optional) +MESSAGEMANAGER_FUZZER_MUTATION_PROBABILITY=2 (optional) +MESSAGEMANAGER_FUZZER_STRINGSFILE=<path> (optional) +MESSAGEMANAGER_FUZZER_BLACKLIST=<path> (optional) +*/ + +#ifdef IsLoggingEnabled +// This is defined in the Windows SDK urlmon.h +# undef IsLoggingEnabled +#endif + +class MessageManagerFuzzer { + public: + static void TryMutate(JSContext* aCx, const nsAString& aMessageName, + ipc::StructuredCloneData* aData, + const JS::Value& aTransfer); + + private: + static void ReadFile(const char* path, nsTArray<nsCString>& aArray); + static nsCString GetFuzzValueFromFile(); + static bool IsMessageNameBlacklisted(const nsAString& aMessageName); + static bool Mutate(JSContext* aCx, const nsAString& aMessageName, + ipc::StructuredCloneData* aData, + const JS::Value& aTransfer); + static void Mutate(JSContext* aCx, JS::Rooted<JS::Value>& aMutation); + static void MutateObject(JSContext* aCx, JS::Handle<JS::Value> aValue, + unsigned short int aRecursionCounter); + static bool MutateValue(JSContext* aCx, JS::Handle<JS::Value> aValue, + JS::MutableHandle<JS::Value> aOutMutationValue, + unsigned short int aRecursionCounter); + static unsigned int DefaultMutationProbability(); + static nsAutoString ReadJSON(JSContext* aCx, const JS::Value& aJSON); + static bool IsEnabled(); + static bool IsLoggingEnabled(); +}; + +} // namespace dom +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/messagemanager/moz.build b/tools/fuzzing/messagemanager/moz.build new file mode 100644 index 0000000000..9fc49d7b73 --- /dev/null +++ b/tools/fuzzing/messagemanager/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += ["MessageManagerFuzzer.cpp"] + +EXPORTS += ["MessageManagerFuzzer.h"] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/moz.build b/tools/fuzzing/moz.build new file mode 100644 index 0000000000..37dc85ddd1 --- /dev/null +++ b/tools/fuzzing/moz.build @@ -0,0 +1,33 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + "interface", + "registry", +] + +if not CONFIG["JS_STANDALONE"]: + DIRS += [ + "common", + "messagemanager", + "shmem", + ] + + if CONFIG["FUZZING_SNAPSHOT"]: + DIRS += [ + "ipc", + "nyx", + ] + + if CONFIG["LIBFUZZER"]: + DIRS += [ + "rust", + ] + +if CONFIG["LIBFUZZER"]: + DIRS += [ + "libfuzzer", + ] diff --git a/tools/fuzzing/nyx/Nyx.cpp b/tools/fuzzing/nyx/Nyx.cpp new file mode 100644 index 0000000000..3fbb520b11 --- /dev/null +++ b/tools/fuzzing/nyx/Nyx.cpp @@ -0,0 +1,230 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Assertions.h" +#include "mozilla/Unused.h" +#include "mozilla/Vector.h" +#include "mozilla/fuzzing/Nyx.h" +#include "prinrval.h" +#include "prthread.h" + +#include <algorithm> +#include <fstream> + +#include <unistd.h> + +namespace mozilla { +namespace fuzzing { + +Nyx::Nyx() { + char* testFilePtr = getenv("MOZ_FUZZ_TESTFILE"); + if (testFilePtr) { + mReplayMode = true; + } +} + +// static +Nyx& Nyx::instance() { + static Nyx nyx; + return nyx; +} + +extern "C" { +MOZ_EXPORT __attribute__((weak)) void nyx_start(void); +MOZ_EXPORT __attribute__((weak)) uint32_t nyx_get_next_fuzz_data(void*, + uint32_t); +MOZ_EXPORT __attribute__((weak)) uint32_t nyx_get_raw_fuzz_data(void*, + uint32_t); +MOZ_EXPORT __attribute__((weak)) void nyx_release(uint32_t); +MOZ_EXPORT __attribute__((weak)) void nyx_handle_event(const char*, const char*, + int, const char*); +MOZ_EXPORT __attribute__((weak)) void nyx_puts(const char*); +MOZ_EXPORT __attribute__((weak)) void nyx_dump_file(void* buffer, size_t len, + const char* filename); +} + +/* + * In this macro, we must avoid calling MOZ_CRASH and friends, as these + * calls will redirect to the Nyx event handler routines. If the library + * is not properly preloaded, we will crash in the process. Instead, emit + * a descriptive error and then force a crash that won't be redirected. + */ +#define NYX_CHECK_API(func) \ + if (!func) { \ + fprintf( \ + stderr, \ + "Error: Nyx library must be in LD_PRELOAD. Missing function \"%s\"\n", \ + #func); \ + MOZ_REALLY_CRASH(__LINE__); \ + } + +void Nyx::start(void) { + MOZ_RELEASE_ASSERT(!mInited); + mInited = true; + + // Check if we are in replay mode. + char* testFilePtr = getenv("MOZ_FUZZ_TESTFILE"); + if (testFilePtr) { + MOZ_FUZZING_NYX_PRINT("[Replay Mode] Reading data file...\n"); + + std::string testFile(testFilePtr); + std::ifstream is; + is.open(testFile, std::ios::binary); + + // The data chunks we receive through Nyx are stored in the data + // section of the testfile as chunks prefixed with a 16-bit data + // length that we mask down to 11-bit. We read all chunks and + // store them away to simulate how we originally received the data + // via Nyx. + + if (is.good()) { + mRawReplayBuffer = new Vector<uint8_t>(); + is.seekg(0, is.end); + int rawLength = is.tellg(); + mozilla::Unused << mRawReplayBuffer->initLengthUninitialized(rawLength); + is.seekg(0, is.beg); + is.read(reinterpret_cast<char*>(mRawReplayBuffer->begin()), rawLength); + is.seekg(0, is.beg); + } + + while (is.good()) { + uint16_t pktsize; + is.read(reinterpret_cast<char*>(&pktsize), sizeof(uint16_t)); + + pktsize &= 0x7ff; + + if (!is.good()) { + break; + } + + auto buffer = new Vector<uint8_t>(); + + mozilla::Unused << buffer->initLengthUninitialized(pktsize); + is.read(reinterpret_cast<char*>(buffer->begin()), buffer->length()); + + MOZ_FUZZING_NYX_PRINTF("[Replay Mode] Read data packet of size %zu\n", + buffer->length()); + + mReplayBuffers.push_back(buffer); + } + + if (!mReplayBuffers.size()) { + MOZ_FUZZING_NYX_PRINT("[Replay Mode] Error: No buffers read.\n"); + _exit(1); + } + + is.close(); + + if (!!getenv("MOZ_FUZZ_WAIT_BEFORE_REPLAY")) { + // This can be useful in some cases to reproduce intermittent issues. + PR_Sleep(PR_MillisecondsToInterval(5000)); + } + + return; + } + + NYX_CHECK_API(nyx_start); + NYX_CHECK_API(nyx_get_next_fuzz_data); + NYX_CHECK_API(nyx_get_raw_fuzz_data); + NYX_CHECK_API(nyx_release); + NYX_CHECK_API(nyx_handle_event); + NYX_CHECK_API(nyx_puts); + NYX_CHECK_API(nyx_dump_file); + + nyx_start(); +} + +bool Nyx::started(void) { return mInited; } + +bool Nyx::is_enabled(const char* identifier) { + static char* fuzzer = getenv("NYX_FUZZER"); + if (!fuzzer || strcmp(fuzzer, identifier)) { + return false; + } + return true; +} + +bool Nyx::is_replay() { return mReplayMode; } + +uint32_t Nyx::get_data(uint8_t* data, uint32_t size) { + MOZ_RELEASE_ASSERT(mInited); + + if (mReplayMode) { + if (!mReplayBuffers.size()) { + return 0xFFFFFFFF; + } + + Vector<uint8_t>* buffer = mReplayBuffers.front(); + mReplayBuffers.pop_front(); + + size = std::min(size, (uint32_t)buffer->length()); + memcpy(data, buffer->begin(), size); + + delete buffer; + + return size; + } + + return nyx_get_next_fuzz_data(data, size); +} + +uint32_t Nyx::get_raw_data(uint8_t* data, uint32_t size) { + MOZ_RELEASE_ASSERT(mInited); + + if (mReplayMode) { + size = std::min(size, (uint32_t)mRawReplayBuffer->length()); + memcpy(data, mRawReplayBuffer->begin(), size); + return size; + } + + return nyx_get_raw_fuzz_data(data, size); +} + +void Nyx::release(uint32_t iterations) { + MOZ_RELEASE_ASSERT(mInited); + + if (mReplayMode) { + MOZ_FUZZING_NYX_PRINT("[Replay Mode] Nyx::release() called.\n"); + + // If we reach this point in replay mode, we are essentially done. + // Let's wait a bit further for things to settle and then exit. + PR_Sleep(PR_MillisecondsToInterval(5000)); + _exit(1); + } + + MOZ_FUZZING_NYX_DEBUG("[DEBUG] Nyx::release() called.\n"); + + nyx_release(iterations); +} + +void Nyx::handle_event(const char* type, const char* file, int line, + const char* reason) { + if (mReplayMode) { + MOZ_FUZZING_NYX_PRINTF( + "[Replay Mode] Nyx::handle_event() called: %s at %s:%d : %s\n", type, + file, line, reason); + return; + } + + if (mInited) { + nyx_handle_event(type, file, line, reason); + } else { + // We can have events such as MOZ_CRASH even before we snapshot. + // Output some useful information to make it clear where it happened. + MOZ_FUZZING_NYX_PRINTF( + "[ERROR] PRE SNAPSHOT Nyx::handle_event() called: %s at %s:%d : %s\n", + type, file, line, reason); + } +} + +void Nyx::dump_file(void* buffer, size_t len, const char* filename) { + if (!mReplayMode) { + nyx_dump_file(buffer, len, filename); + } +} + +} // namespace fuzzing +} // namespace mozilla diff --git a/tools/fuzzing/nyx/Nyx.h b/tools/fuzzing/nyx/Nyx.h new file mode 100644 index 0000000000..0489b2088e --- /dev/null +++ b/tools/fuzzing/nyx/Nyx.h @@ -0,0 +1,57 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_fuzzing_Nyx_h +#define mozilla_fuzzing_Nyx_h + +#include <stdint.h> +#include <atomic> +#include <list> + +#ifndef NYX_DISALLOW_COPY_AND_ASSIGN +# define NYX_DISALLOW_COPY_AND_ASSIGN(T) \ + T(const T&); \ + void operator=(const T&) +#endif + +namespace mozilla { + +class MallocAllocPolicy; +template <class T, size_t MinInlineCapacity, class AllocPolicy> +class Vector; + +namespace fuzzing { + +class Nyx { + public: + static Nyx& instance(); + + void start(void); + bool started(void); + bool is_enabled(const char* identifier); + bool is_replay(); + uint32_t get_data(uint8_t* data, uint32_t size); + uint32_t get_raw_data(uint8_t* data, uint32_t size); + void release(uint32_t iterations = 1); + void handle_event(const char* type, const char* file, int line, + const char* reason); + void dump_file(void* buffer, size_t len, const char* filename); + + private: + std::atomic<bool> mInited; + + std::atomic<bool> mReplayMode; + std::list<Vector<uint8_t, 0, MallocAllocPolicy>*> mReplayBuffers; + Vector<uint8_t, 0, MallocAllocPolicy>* mRawReplayBuffer; + + Nyx(); + NYX_DISALLOW_COPY_AND_ASSIGN(Nyx); +}; + +} // namespace fuzzing +} // namespace mozilla + +#endif /* mozilla_fuzzing_Nyx_h */ diff --git a/tools/fuzzing/nyx/NyxWrapper.h b/tools/fuzzing/nyx/NyxWrapper.h new file mode 100644 index 0000000000..0bb6e0fc46 --- /dev/null +++ b/tools/fuzzing/nyx/NyxWrapper.h @@ -0,0 +1,33 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_fuzzing_NyxWrapper_h +#define mozilla_fuzzing_NyxWrapper_h + +#include "mozilla/Types.h" + +/* + * We need this event handler definition both in C and C++ to differentiate + * the various flavors of controlled aborts (e.g. MOZ_DIAGNOSTIC_ASSERT + * vs. MOZ_RELEASE_ASSERT). Hence we can't use the higher level C++ API + * for this and directly redirect to the Nyx event handler in the preloaded + * library. + */ + +#ifdef __cplusplus +extern "C" { +#endif +MOZ_EXPORT __attribute__((weak)) void nyx_handle_event(const char* type, + const char* file, + int line, + const char* reason); + +MOZ_EXPORT __attribute__((weak)) void nyx_puts(const char*); +#ifdef __cplusplus +} +#endif + +#endif /* mozilla_fuzzing_NyxWrapper_h */ diff --git a/tools/fuzzing/nyx/moz.build b/tools/fuzzing/nyx/moz.build new file mode 100644 index 0000000000..74c8ec8515 --- /dev/null +++ b/tools/fuzzing/nyx/moz.build @@ -0,0 +1,16 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += [ + "Nyx.cpp", +] + +EXPORTS.mozilla.fuzzing += [ + "Nyx.h", + "NyxWrapper.h", +] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/registry/FuzzerRegistry.cpp b/tools/fuzzing/registry/FuzzerRegistry.cpp new file mode 100644 index 0000000000..ed4b0c7fa8 --- /dev/null +++ b/tools/fuzzing/registry/FuzzerRegistry.cpp @@ -0,0 +1,26 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzerRegistry.h" + +namespace mozilla { + +FuzzerRegistry& FuzzerRegistry::getInstance() { + static FuzzerRegistry instance; + return instance; +} + +void FuzzerRegistry::registerModule(std::string moduleName, + FuzzerInitFunc initFunc, + FuzzerTestingFunc testingFunc) { + moduleMap.insert(std::pair<std::string, FuzzerFunctions>( + moduleName, FuzzerFunctions(initFunc, testingFunc))); +} + +FuzzerFunctions FuzzerRegistry::getModuleFunctions(std::string& moduleName) { + return moduleMap[moduleName]; +} + +} // namespace mozilla diff --git a/tools/fuzzing/registry/FuzzerRegistry.h b/tools/fuzzing/registry/FuzzerRegistry.h new file mode 100644 index 0000000000..5976ddc5b6 --- /dev/null +++ b/tools/fuzzing/registry/FuzzerRegistry.h @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _FuzzerRegistry_h__ +#define _FuzzerRegistry_h__ + +#include <cstdint> +#include <map> +#include <string> +#include <utility> + +#include "mozilla/Attributes.h" +#include "mozilla/Types.h" + +typedef int (*FuzzerInitFunc)(int*, char***); +typedef int (*FuzzerTestingFunc)(const uint8_t*, size_t); + +typedef int (*LibFuzzerDriver)(int*, char***, FuzzerTestingFunc); + +namespace mozilla { + +typedef std::pair<FuzzerInitFunc, FuzzerTestingFunc> FuzzerFunctions; + +class FuzzerRegistry { + public: + MOZ_EXPORT static FuzzerRegistry& getInstance(); + MOZ_EXPORT void registerModule(std::string moduleName, + FuzzerInitFunc initFunc, + FuzzerTestingFunc testingFunc); + MOZ_EXPORT FuzzerFunctions getModuleFunctions(std::string& moduleName); + + FuzzerRegistry(FuzzerRegistry const&) = delete; + void operator=(FuzzerRegistry const&) = delete; + + private: + FuzzerRegistry(){}; + std::map<std::string, FuzzerFunctions> moduleMap; +}; + +} // namespace mozilla + +#endif // _FuzzerRegistry_h__ diff --git a/tools/fuzzing/registry/moz.build b/tools/fuzzing/registry/moz.build new file mode 100644 index 0000000000..4aa005a56e --- /dev/null +++ b/tools/fuzzing/registry/moz.build @@ -0,0 +1,20 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-registry") + +SOURCES += [ + "FuzzerRegistry.cpp", +] + +EXPORTS += [ + "FuzzerRegistry.h", +] + +if CONFIG["JS_STANDALONE"]: + FINAL_LIBRARY = "js" +else: + FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/rust/Cargo.toml b/tools/fuzzing/rust/Cargo.toml new file mode 100644 index 0000000000..cceb319cfe --- /dev/null +++ b/tools/fuzzing/rust/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "gecko-fuzz-targets" +version = "0.1.0" +authors = ["fuzzing@mozilla.com"] + +[dependencies] +libc = "0.2" +tempfile = "3" +lazy_static = "1.4.0" +rkv = { version = "0.19", features = ["with-fuzzer-no-link"] } +lmdb-rkv = { version = "0.14", features = ["with-fuzzer-no-link"] } diff --git a/tools/fuzzing/rust/RustFuzzingTargets.cpp b/tools/fuzzing/rust/RustFuzzingTargets.cpp new file mode 100644 index 0000000000..7b4dd6c86c --- /dev/null +++ b/tools/fuzzing/rust/RustFuzzingTargets.cpp @@ -0,0 +1,15 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzingInterface.h" +#include "RustFuzzingTargets.h" + +int FuzzingInitDummy(int* argc, char*** argv) { return 0; } + +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_db_file, RkvDbFile); +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_db_name, RkvDbName); +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_key_write, RkvKeyWrite); +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_val_write, RkvValWrite); +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_calls, RkvCalls); diff --git a/tools/fuzzing/rust/RustFuzzingTargets.h b/tools/fuzzing/rust/RustFuzzingTargets.h new file mode 100644 index 0000000000..a9c1439598 --- /dev/null +++ b/tools/fuzzing/rust/RustFuzzingTargets.h @@ -0,0 +1,26 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface definitions for fuzzing rust modules + */ + +#ifndef RustFuzzingTargets_h__ +#define RustFuzzingTargets_h__ + +#include <stddef.h> +#include <stdint.h> + +extern "C" { + +int fuzz_rkv_db_file(const uint8_t* raw_data, size_t size); +int fuzz_rkv_db_name(const uint8_t* raw_data, size_t size); +int fuzz_rkv_key_write(const uint8_t* raw_data, size_t size); +int fuzz_rkv_val_write(const uint8_t* raw_data, size_t size); +int fuzz_rkv_calls(const uint8_t* raw_data, size_t size); + +} // extern "C" + +#endif // RustFuzzingTargets_h__ diff --git a/tools/fuzzing/rust/moz.build b/tools/fuzzing/rust/moz.build new file mode 100644 index 0000000000..aa6aaa29dc --- /dev/null +++ b/tools/fuzzing/rust/moz.build @@ -0,0 +1,16 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-rust-targets") + +SOURCES += [ + "RustFuzzingTargets.cpp", +] + +# Add libFuzzer configuration directives +include("/tools/fuzzing/libfuzzer-config.mozbuild") + +FINAL_LIBRARY = "xul-gtest" diff --git a/tools/fuzzing/rust/src/lib.rs b/tools/fuzzing/rust/src/lib.rs new file mode 100644 index 0000000000..25c9195fb8 --- /dev/null +++ b/tools/fuzzing/rust/src/lib.rs @@ -0,0 +1,341 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#[macro_use] +extern crate lazy_static; +extern crate libc; +extern crate lmdb; +extern crate rkv; +extern crate tempfile; + +use rkv::backend::{ + BackendEnvironmentBuilder, SafeMode, SafeModeDatabase, SafeModeEnvironment, + SafeModeRoTransaction, SafeModeRwTransaction, +}; +use std::fs; +use std::fs::File; +use std::io::Write; +use std::iter; +use std::path::Path; +use std::sync::Arc; +use std::thread; +use tempfile::Builder; + +type Rkv = rkv::Rkv<SafeModeEnvironment>; +type SingleStore = rkv::SingleStore<SafeModeDatabase>; + +fn eat_lmdb_err<T>(value: Result<T, rkv::StoreError>) -> Result<Option<T>, rkv::StoreError> { + match value { + Ok(value) => Ok(Some(value)), + Err(rkv::StoreError::LmdbError(_)) => Ok(None), + Err(err) => { + println!("Not a crash, but an error outside LMDB: {}", err); + println!("A refined fuzzing test, or changes to RKV, might be required."); + Err(err) + } + } +} + +fn panic_with_err(err: rkv::StoreError) { + println!("Got error: {}", err); + Result::Err(err).unwrap() +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_db_file(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + + // First 8192 bytes are for the lock file. + if data.len() < 8192 { + return 0; + } + let (lock, db) = data.split_at(8192); + let mut lock_file = File::create("data.lock").unwrap(); + lock_file.write_all(lock).unwrap(); + let mut db_file = File::create("data.mdb").unwrap(); + db_file.write_all(db).unwrap(); + + let env = Rkv::with_capacity::<SafeMode>(Path::new("."), 2).unwrap(); + let store = env + .open_single("test", rkv::StoreOptions::create()) + .unwrap(); + + let reader = env.read().unwrap(); + eat_lmdb_err(store.get(&reader, &[0])).unwrap(); + + 0 +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_db_name(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + + let root = Builder::new().prefix("fuzz_rkv_db_name").tempdir().unwrap(); + fs::create_dir_all(root.path()).unwrap(); + + let env = Rkv::new::<SafeMode>(root.path()).unwrap(); + let name = String::from_utf8_lossy(data); + println!("Checking string: '{:?}'", name); + // Some strings are invalid database names, and are handled as store errors. + // Ignore those errors, but not others. + let store = eat_lmdb_err(env.open_single(name.as_ref(), rkv::StoreOptions::create())).unwrap(); + + if let Some(store) = store { + let reader = env.read().unwrap(); + eat_lmdb_err(store.get(&reader, &[0])).unwrap(); + }; + + 0 +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_key_write(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + + let root = Builder::new() + .prefix("fuzz_rkv_key_write") + .tempdir() + .unwrap(); + fs::create_dir_all(root.path()).unwrap(); + + let env = Rkv::new::<SafeMode>(root.path()).unwrap(); + let store = env + .open_single("test", rkv::StoreOptions::create()) + .unwrap(); + + let mut writer = env.write().unwrap(); + // Some data are invalid values, and are handled as store errors. + // Ignore those errors, but not others. + eat_lmdb_err(store.put(&mut writer, data, &rkv::Value::Str("val"))).unwrap(); + + 0 +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_val_write(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + + let root = Builder::new() + .prefix("fuzz_rkv_val_write") + .tempdir() + .unwrap(); + fs::create_dir_all(root.path()).unwrap(); + + let env = Rkv::new::<SafeMode>(root.path()).unwrap(); + let store = env + .open_single("test", rkv::StoreOptions::create()) + .unwrap(); + + let mut writer = env.write().unwrap(); + let string = String::from_utf8_lossy(data); + let value = rkv::Value::Str(&string); + store.put(&mut writer, "key", &value).unwrap(); + + 0 +} + +lazy_static! { + static ref STATIC_DATA: Vec<String> = { + let sizes = vec![4, 16, 128, 512, 1024]; + let mut data = Vec::new(); + + for (i, s) in sizes.into_iter().enumerate() { + let bytes = iter::repeat('a' as u8 + i as u8).take(s).collect(); + data.push(String::from_utf8(bytes).unwrap()); + } + + data + }; +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_calls(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + let mut fuzz = data.iter().copied(); + + fn maybe_do<I: Iterator<Item = u8>>(fuzz: &mut I, f: impl FnOnce(&mut I) -> ()) -> Option<()> { + match fuzz.next().map(|byte| byte % 2) { + Some(0) => Some(f(fuzz)), + _ => None, + } + } + + fn maybe_abort<I: Iterator<Item = u8>>( + fuzz: &mut I, + read: rkv::Reader<SafeModeRoTransaction>, + ) -> Result<(), rkv::StoreError> { + match fuzz.next().map(|byte| byte % 2) { + Some(0) => Ok(read.abort()), + _ => Ok(()), + } + } + + fn maybe_commit<I: Iterator<Item = u8>>( + fuzz: &mut I, + write: rkv::Writer<SafeModeRwTransaction>, + ) -> Result<(), rkv::StoreError> { + match fuzz.next().map(|byte| byte % 3) { + Some(0) => write.commit(), + Some(1) => Ok(write.abort()), + _ => Ok(()), + } + } + + fn get_static_data<'a, I: Iterator<Item = u8>>(fuzz: &mut I) -> Option<&'a String> { + fuzz.next().map(|byte| { + let data = &*STATIC_DATA; + let n = byte as usize; + data.get(n % data.len()).unwrap() + }) + } + + fn get_fuzz_data<I: Iterator<Item = u8> + Clone>( + fuzz: &mut I, + max_len: usize, + ) -> Option<Vec<u8>> { + fuzz.next().map(|byte| { + let n = byte as usize; + fuzz.clone().take((n * n) % max_len).collect() + }) + } + + fn get_any_data<I: Iterator<Item = u8> + Clone>( + fuzz: &mut I, + max_len: usize, + ) -> Option<Vec<u8>> { + match fuzz.next().map(|byte| byte % 2) { + Some(0) => get_static_data(fuzz).map(|v| v.clone().into_bytes()), + Some(1) => get_fuzz_data(fuzz, max_len), + _ => None, + } + } + + fn store_put<I: Iterator<Item = u8> + Clone>(fuzz: &mut I, env: &Rkv, store: &SingleStore) { + let key = match get_any_data(fuzz, 1024) { + Some(key) => key, + None => return, + }; + let value = match get_any_data(fuzz, std::usize::MAX) { + Some(value) => value, + None => return, + }; + + let mut writer = env.write().unwrap(); + let mut full = false; + + match store.put(&mut writer, key, &rkv::Value::Blob(&value)) { + Ok(_) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::BadValSize)) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::MapFull)) => full = true, + Err(err) => panic_with_err(err), + }; + + if full { + writer.abort(); + store_resize(fuzz, env); + } else { + maybe_commit(fuzz, writer).unwrap(); + } + } + + fn store_get<I: Iterator<Item = u8> + Clone>(fuzz: &mut I, env: &Rkv, store: &SingleStore) { + let key = match get_any_data(fuzz, 1024) { + Some(key) => key, + None => return, + }; + + let mut reader = match env.read() { + Ok(reader) => reader, + Err(rkv::StoreError::LmdbError(lmdb::Error::ReadersFull)) => return, + Err(err) => return panic_with_err(err), + }; + + match store.get(&mut reader, key) { + Ok(_) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::BadValSize)) => {} + Err(err) => panic_with_err(err), + }; + + maybe_abort(fuzz, reader).unwrap(); + } + + fn store_delete<I: Iterator<Item = u8> + Clone>(fuzz: &mut I, env: &Rkv, store: &SingleStore) { + let key = match get_any_data(fuzz, 1024) { + Some(key) => key, + None => return, + }; + + let mut writer = env.write().unwrap(); + + match store.delete(&mut writer, key) { + Ok(_) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::BadValSize)) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::NotFound)) => {} + Err(err) => panic_with_err(err), + }; + + maybe_commit(fuzz, writer).unwrap(); + } + + fn store_resize<I: Iterator<Item = u8>>(fuzz: &mut I, env: &Rkv) { + let n = fuzz.next().unwrap_or(1) as usize; + env.set_map_size(1_048_576 * (n % 100)).unwrap() // 1,048,576 bytes, i.e. 1MiB. + } + + let root = Builder::new().prefix("fuzz_rkv_calls").tempdir().unwrap(); + fs::create_dir_all(root.path()).unwrap(); + + let mut builder: SafeMode = Rkv::environment_builder(); + builder.set_max_dbs(1); // need at least one db + + maybe_do(&mut fuzz, |fuzz| { + let n = fuzz.next().unwrap_or(126) as u32; // default + builder.set_max_readers(1 + n); + }); + maybe_do(&mut fuzz, |fuzz| { + let n = fuzz.next().unwrap_or(0) as u32; + builder.set_max_dbs(1 + n); + }); + maybe_do(&mut fuzz, |fuzz| { + let n = fuzz.next().unwrap_or(1) as usize; + builder.set_map_size(1_048_576 * (n % 100)); // 1,048,576 bytes, i.e. 1MiB. + }); + + let env = Rkv::from_builder(root.path(), builder).unwrap(); + let store = env + .open_single("test", rkv::StoreOptions::create()) + .unwrap(); + + let shared_env = Arc::new(env); + let shared_store = Arc::new(store); + + let use_threads = fuzz.next().map(|byte| byte % 10 == 0).unwrap_or(false); + let max_threads = if use_threads { 16 } else { 1 }; + let num_threads = fuzz.next().unwrap_or(0) as usize % max_threads + 1; + let chunk_size = fuzz.len() / num_threads; + + let threads = (0..num_threads).map(|_| { + let env = shared_env.clone(); + let store = shared_store.clone(); + + let chunk: Vec<_> = fuzz.by_ref().take(chunk_size).collect(); + let mut fuzz = chunk.into_iter(); + + thread::spawn(move || loop { + match fuzz.next().map(|byte| byte % 4) { + Some(0) => store_put(&mut fuzz, &env, &store), + Some(1) => store_get(&mut fuzz, &env, &store), + Some(2) => store_delete(&mut fuzz, &env, &store), + Some(3) => store_resize(&mut fuzz, &env), + _ => break, + } + }) + }); + + for handle in threads { + handle.join().unwrap() + } + + 0 +} diff --git a/tools/fuzzing/shmem/SharedMemoryFuzzer.cpp b/tools/fuzzing/shmem/SharedMemoryFuzzer.cpp new file mode 100644 index 0000000000..49a79fa975 --- /dev/null +++ b/tools/fuzzing/shmem/SharedMemoryFuzzer.cpp @@ -0,0 +1,122 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzingMutate.h" +#include "FuzzingTraits.h" +#include "nsDebug.h" +#include "prenv.h" +#include "SharedMemoryFuzzer.h" + +#define SHMEM_FUZZER_DEFAULT_MUTATION_PROBABILITY 2 +#define SHMEM_FUZZER_DEFAULT_MUTATION_FACTOR 500 +#define SHMEM_FUZZER_LOG(fmt, args...) \ + if (SharedMemoryFuzzer::IsLoggingEnabled()) { \ + printf_stderr("[SharedMemoryFuzzer] " fmt "\n", ##args); \ + } + +namespace mozilla { +namespace ipc { + +using namespace fuzzing; + +/* static */ +bool SharedMemoryFuzzer::IsLoggingEnabled() { + static bool sInitialized = false; + static bool sIsLoggingEnabled = false; + + if (!sInitialized) { + sIsLoggingEnabled = !!PR_GetEnv("SHMEM_FUZZER_ENABLE_LOGGING"); + sInitialized = true; + } + return sIsLoggingEnabled; +} + +/* static */ +bool SharedMemoryFuzzer::IsEnabled() { + static bool sInitialized = false; + static bool sIsFuzzerEnabled = false; + + if (!sInitialized) { + sIsFuzzerEnabled = !!PR_GetEnv("SHMEM_FUZZER_ENABLE"); + } + return sIsFuzzerEnabled; +} + +/* static */ +uint64_t SharedMemoryFuzzer::MutationProbability() { + static uint64_t sPropValue = SHMEM_FUZZER_DEFAULT_MUTATION_PROBABILITY; + static bool sInitialized = false; + + if (sInitialized) { + return sPropValue; + } + sInitialized = true; + + const char* probability = PR_GetEnv("SHMEM_FUZZER_MUTATION_PROBABILITY"); + if (probability) { + long n = std::strtol(probability, nullptr, 10); + if (n != 0) { + sPropValue = n; + return sPropValue; + } + } + return sPropValue; +} + +/* static */ +uint64_t SharedMemoryFuzzer::MutationFactor() { + static uint64_t sPropValue = SHMEM_FUZZER_DEFAULT_MUTATION_FACTOR; + static bool sInitialized = false; + + if (sInitialized) { + return sPropValue; + } + sInitialized = true; + + const char* factor = PR_GetEnv("SHMEM_FUZZER_MUTATION_FACTOR"); + if (factor) { + long n = strtol(factor, nullptr, 10); + if (n != 0) { + sPropValue = n; + return sPropValue; + } + } + return sPropValue; +} + +/* static */ +void* SharedMemoryFuzzer::MutateSharedMemory(void* aMemory, size_t aSize) { + if (!IsEnabled()) { + return aMemory; + } + + if (aSize == 0) { + /* Shmem opened from foreign handle. */ + SHMEM_FUZZER_LOG("shmem is of size 0."); + return aMemory; + } + + if (!aMemory) { + /* Memory space is not mapped. */ + SHMEM_FUZZER_LOG("shmem memory space is not mapped."); + return aMemory; + } + + // The likelihood when a value gets fuzzed of this object. + if (!FuzzingTraits::Sometimes(MutationProbability())) { + return aMemory; + } + + const size_t max = FuzzingTraits::Frequency(aSize, MutationFactor()); + SHMEM_FUZZER_LOG("shmem of size: %zu / mutations: %zu", aSize, max); + for (size_t i = 0; i < max; i++) { + FuzzingMutate::ChangeBit((uint8_t*)aMemory, aSize); + } + return aMemory; +} + +} // namespace ipc +} // namespace mozilla diff --git a/tools/fuzzing/shmem/SharedMemoryFuzzer.h b/tools/fuzzing/shmem/SharedMemoryFuzzer.h new file mode 100644 index 0000000000..bd862edf6a --- /dev/null +++ b/tools/fuzzing/shmem/SharedMemoryFuzzer.h @@ -0,0 +1,38 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_SharedMemoryFuzzer_h +#define mozilla_dom_SharedMemoryFuzzer_h + +#include <stddef.h> +#include <stdint.h> + +namespace mozilla { +namespace ipc { + +/* + * Exposed environment variables: + * SHMEM_FUZZER_ENABLE=1 + * SHMEM_FUZZER_ENABLE_LOGGING=1 (optional) + * SHMEM_FUZZER_MUTATION_PROBABILITY=2 (optional) + * SHMEM_FUZZER_MUTATION_FACTOR=500 (optional) + */ + +class SharedMemoryFuzzer { + public: + static void* MutateSharedMemory(void* aMemory, size_t aSize); + + private: + static uint64_t MutationProbability(); + static uint64_t MutationFactor(); + static bool IsEnabled(); + static bool IsLoggingEnabled(); +}; + +} // namespace ipc +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/shmem/moz.build b/tools/fuzzing/shmem/moz.build new file mode 100644 index 0000000000..ee9c549920 --- /dev/null +++ b/tools/fuzzing/shmem/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += ["SharedMemoryFuzzer.cpp"] + +EXPORTS.mozilla.ipc += ["SharedMemoryFuzzer.h"] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/smoke/grizzly_requirements.in b/tools/fuzzing/smoke/grizzly_requirements.in new file mode 100644 index 0000000000..96d49442a0 --- /dev/null +++ b/tools/fuzzing/smoke/grizzly_requirements.in @@ -0,0 +1,2 @@ +grizzly-framework==0.16.2 +-r ../../../build/psutil_requirements.txt diff --git a/tools/fuzzing/smoke/grizzly_requirements.txt b/tools/fuzzing/smoke/grizzly_requirements.txt new file mode 100644 index 0000000000..a6687e5426 --- /dev/null +++ b/tools/fuzzing/smoke/grizzly_requirements.txt @@ -0,0 +1,127 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --generate-hashes --output-file=tools/fuzzing/smoke/grizzly_requirements.txt tools/fuzzing/smoke/grizzly_requirements.in +# +certifi==2021.10.8 \ + --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ + --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 + # via requests +charset-normalizer==2.0.8 \ + --hash=sha256:735e240d9a8506778cd7a453d97e817e536bb1fc29f4f6961ce297b9c7a917b0 \ + --hash=sha256:83fcdeb225499d6344c8f7f34684c2981270beacc32ede2e669e94f7fa544405 + # via requests +cssbeautifier==1.14.0 \ + --hash=sha256:20be1f47f20762db32c78124ff44d351ba13894fa8e7cfe34014b672f9f6ecb2 + # via grizzly-framework +editorconfig==0.12.3 \ + --hash=sha256:6b0851425aa875b08b16789ee0eeadbd4ab59666e9ebe728e526314c4a2e52c1 + # via + # cssbeautifier + # jsbeautifier +fasteners==0.16.3 \ + --hash=sha256:8408e52656455977053871990bd25824d85803b9417aa348f10ba29ef0c751f7 + # via + # fuzzmanager + # grizzly-framework +ffpuppet==0.9.2 \ + --hash=sha256:340ec47fddc274c97e0b9d9e56d46885e96a07a45a265fd2fa5b21af7f655947 \ + --hash=sha256:41e93d2f3d8d3230822fd4962a85e66246a70927528dce76d765e0ffd859bf69 + # via grizzly-framework +fuzzmanager==0.4.1 \ + --hash=sha256:2bb17b5a725d8d6f03eb9979a75416a1137b7456c95a581a50c7b542fbf3d174 + # via grizzly-framework +grizzly-framework==0.16.2 \ + --hash=sha256:3270fd705a933c4d197784c6e403389c3edb791d09ce26b8a7d7c98934bbc87b \ + --hash=sha256:b01503d1a3a0f8a3fb65cfbc2397a1b1bd4d524389a24ce8eaf00613d374a91b + # via -r tools/fuzzing/smoke/grizzly_requirements.in +idna==3.3 \ + --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ + --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d + # via requests +jsbeautifier==1.14.0 \ + --hash=sha256:84fdb008d8af89619269a6aca702288b48f837a99427a0f529aa57ecfb36ee3c + # via + # cssbeautifier + # grizzly-framework +lithium-reducer==0.6.1 \ + --hash=sha256:ea2f77f496fc57bcb4d74209210c2ec84b1b327a7b707f98655f85575b6fcc16 + # via grizzly-framework +prefpicker==1.1.2 \ + --hash=sha256:1404cb0e7c07acca060a09fcc3d9203ae2d8784741b6fe97600a707b9b3ff75e \ + --hash=sha256:ea25b33e92a342a0bf2c38f5588dd4baf9d381f70faf7ae2145ca5cad49a2644 + # via grizzly-framework +psutil==5.9.4 \ + --hash=sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff \ + --hash=sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1 \ + --hash=sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62 \ + --hash=sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549 \ + --hash=sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08 \ + --hash=sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7 \ + --hash=sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e \ + --hash=sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe \ + --hash=sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24 \ + --hash=sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad \ + --hash=sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94 \ + --hash=sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8 \ + --hash=sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7 \ + --hash=sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4 + # via + # -r tools/fuzzing/smoke/../../../build/psutil_requirements.txt + # ffpuppet + # grizzly-framework +pyyaml==6.0 \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 + # via prefpicker +requests==2.26.0 \ + --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \ + --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7 + # via fuzzmanager +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # cssbeautifier + # fasteners + # fuzzmanager + # jsbeautifier +urllib3==1.26.7 \ + --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \ + --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844 + # via requests +xvfbwrapper==0.2.9 \ + --hash=sha256:bcf4ae571941b40254faf7a73432dfc119ad21ce688f1fdec533067037ecfc24 + # via ffpuppet diff --git a/tools/fuzzing/smoke/js.py b/tools/fuzzing/smoke/js.py new file mode 100755 index 0000000000..d6ad08eb6a --- /dev/null +++ b/tools/fuzzing/smoke/js.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Hello I am a fake jsshell for testing purpose. +Add more features! +""" +import argparse +import sys + + +def run(): + parser = argparse.ArgumentParser(description="Process some integers.") + parser.add_argument("-e", type=str, default=None) + + parser.add_argument("--fuzzing-safe", action="store_true", default=False) + + args = parser.parse_args() + + if args.e is not None: + if "crash()" in args.e: + sys.exit(1) + + +if __name__ == "__main__": + run() diff --git a/tools/fuzzing/smoke/python.toml b/tools/fuzzing/smoke/python.toml new file mode 100644 index 0000000000..24c6e56b55 --- /dev/null +++ b/tools/fuzzing/smoke/python.toml @@ -0,0 +1,5 @@ +[DEFAULT] +subsuite = "fuzzing" + +["test_grizzly.py"] +requirements = "tools/fuzzing/smoke/grizzly_requirements.txt" diff --git a/tools/fuzzing/smoke/smoke.py b/tools/fuzzing/smoke/smoke.py new file mode 100644 index 0000000000..bfc7cb56cd --- /dev/null +++ b/tools/fuzzing/smoke/smoke.py @@ -0,0 +1,71 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Smoke test script for Fuzzing + +This script can be used to perform simple calls using `jsshell` +or whatever other tools you may add. + +The call is done via `taskcluster/ci/fuzzing/kind.yml` and +files contained in the `target.jsshell.zip` and `target.fuzztest.tests.tar.gz` +build artifacts are downloaded to run things. + +Everything included in this directory will be added in +`target.fuzztest.tests.tar.gz` at build time, so you can add more scripts and +tools if you need. They will be located in `$MOZ_FETCHES_DIR` and follow the +same directory structure than the source tree. +""" +import os +import os.path +import shlex +import shutil +import subprocess +import sys + + +def run_jsshell(command, label=None): + """Invokes `jsshell` with command. + + This function will use the `JSSHELL` environment variable, + and fallback to a `js` executable if it finds one + """ + shell = os.environ.get("JSSHELL") + if shell is None: + shell = shutil.which("js") + if shell is None: + raise FileNotFoundError(shell) + else: + if not os.path.exists(shell) or not os.path.isfile(shell): + raise FileNotFoundError(shell) + + if label is None: + label = command + sys.stdout.write(label) + cmd = [shell] + shlex.split(command) + sys.stdout.flush() + try: + subprocess.check_call(cmd) + finally: + sys.stdout.write("\n") + sys.stdout.flush() + + +def smoke_test(): + # first, let's make sure it catches crashes so we don't have false + # positives. + try: + run_jsshell("-e 'crash();'", "Testing for crash\n") + except subprocess.CalledProcessError: + pass + else: + raise Exception("Could not get the process to crash") + + # now let's proceed with some tests + run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'", "Simple Fuzzing...") + + # add more smoke tests here + + +if __name__ == "__main__": + # if this calls raises an error, the job will turn red in the CI. + smoke_test() diff --git a/tools/fuzzing/smoke/test_grizzly.py b/tools/fuzzing/smoke/test_grizzly.py new file mode 100644 index 0000000000..2c8f406222 --- /dev/null +++ b/tools/fuzzing/smoke/test_grizzly.py @@ -0,0 +1,42 @@ +import os +import os.path +import sys +from subprocess import check_call + +import mozunit +import pytest +from moztest.selftest import fixtures + +MOZ_AUTOMATION = bool(os.getenv("MOZ_AUTOMATION", "0") == "1") + + +def test_grizzly_smoke(): + ffbin = fixtures.binary() + + if MOZ_AUTOMATION: + assert os.path.exists( + ffbin + ), "Missing Firefox build. Build it, or set GECKO_BINARY_PATH" + + elif not os.path.exists(ffbin): + pytest.skip("Missing Firefox build. Build it, or set GECKO_BINARY_PATH") + + check_call( + [ + sys.executable, + "-m", + "grizzly", + ffbin, + "no-op", + "--headless", + "--smoke-test", + "--limit", + "10", + "--relaunch", + "5", + ], + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/fuzzing/smoke/tests.py b/tools/fuzzing/smoke/tests.py new file mode 100644 index 0000000000..bc06e2427b --- /dev/null +++ b/tools/fuzzing/smoke/tests.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +from contextlib import contextmanager + +import pytest +import smoke + +JS = os.path.join(os.path.dirname(__file__), "js.py") + + +@contextmanager +def fake_js(): + os.environ["JSSHELL"] = JS + try: + yield + finally: + del os.environ["JSSHELL"] + + +def test_run_no_jsshell(): + with pytest.raises(FileNotFoundError): + smoke.run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'") + + +def test_run_jsshell_set(): + with fake_js(): + smoke.run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'") + + +def test_smoke_test(): + with fake_js(): + smoke.smoke_test() diff --git a/tools/github-sync/converter.py b/tools/github-sync/converter.py new file mode 100755 index 0000000000..104229e299 --- /dev/null +++ b/tools/github-sync/converter.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import subprocess +import sys + +import hglib +import pygit2 + +DEBUG = False + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def debugprint(*args, **kwargs): + if DEBUG: + eprint(*args, **kwargs) + + +class HgCommit: + def __init__(self, parent1, parent2): + self.parents = [] + if parent1 == NULL_PARENT_REV: + raise Exception( + "Encountered a hg changeset with no parents! We don't handle this...." + ) + self.parents.append(parent1) + if parent2 != NULL_PARENT_REV: + self.parents.append(parent2) + self.touches_sync_code = False + self.children = [] + + def add_child(self, rev): + self.children.append(rev) + + +class GitCommit: + def __init__(self, hg_rev, commit_obj): + self.hg_rev = hg_rev + self.commit_obj = commit_obj + + +def load_git_repository(): + commit_map = dict() + # First, scan the tags for "mozilla-xxx" that keep track of manually synchronized changes + sync_tags = filter( + lambda ref: ref.startswith("refs/tags/mozilla-"), + list(downstream_git_repo.references), + ) + for desc in sync_tags: + commit = downstream_git_repo.lookup_reference(desc).peel() + # cut out the revision hash from the output + hg_rev = desc[18:] + commit_map[hg_rev] = GitCommit(hg_rev, commit) + debugprint("Loaded pre-existing tag hg %s -> git %s" % (hg_rev, commit.oid)) + + # Next, scan the commits for a specific message format + re_commitmsg = re.compile( + r"^\[(ghsync|wrupdater)\] From https://hg.mozilla.org/mozilla-central/rev/([0-9a-fA-F]+)$", + re.MULTILINE, + ) + for commit in downstream_git_repo.walk(downstream_git_repo.head.target): + m = re_commitmsg.search(commit.message) + if not m: + continue + hg_rev = m.group(2) + commit_map[hg_rev] = GitCommit(hg_rev, commit) + debugprint("Loaded pre-existing commit hg %s -> git %s" % (hg_rev, commit.oid)) + return commit_map + + +def timeof(git_commit): + return git_commit.commit_obj.commit_time + git_commit.commit_obj.commit_time_offset + + +def find_newest_commit(commit_map): + newest_hg_rev = None + newest_commit_time = None + + for hg_rev, git_commit in commit_map.items(): + if newest_hg_rev is None or timeof(git_commit) > newest_commit_time: + newest_hg_rev = hg_rev + newest_commit_time = timeof(git_commit) + + return newest_hg_rev + + +def get_single_rev(revset): + output = subprocess.check_output( + ["hg", "log", "-r", revset, "--template", "{node}"] + ) + output = str(output, "ascii") + return output + + +def get_multiple_revs(revset, template): + output = subprocess.check_output( + ["hg", "log", "-r", revset, "--template", template + "\\n"] + ) + for line in output.splitlines(): + yield str(line, "ascii") + + +def get_base_hg_rev(commit_map): + base_hg_rev = find_newest_commit(commit_map) + eprint("Using %s as base hg revision" % base_hg_rev) + return base_hg_rev + + +def load_hg_commits(commits, query): + for cset in get_multiple_revs(query, "{node} {p1node} {p2node}"): + tokens = cset.split() + commits[tokens[0]] = HgCommit(tokens[1], tokens[2]) + return commits + + +def get_real_base_hg_rev(hg_data, commit_map): + # Some of the HG commits we want to port to github may have landed on codelines + # that branched off central prior to base_hg_rev. So when we create the git + # equivalents, they will have parents that are not the HEAD of the git repo, + # but instead will be descendants of older commits in the git repo. In order + # to do this correctly, we need to find the hg-equivalents of all of those + # possible git parents. So first we identify all the "tail" hg revisions in + # our hg_data set (think "tail" as in opposite of "head" which is the tipmost + # commit). The "tail" hg revisions are the ones for which we don't have their + # ancestors in hg_data. + tails = [] + for rev, cset in hg_data.items(): + for parent in cset.parents: + if parent not in hg_data: + tails.append(rev) + eprint("Found hg tail revisions %s" % tails) + # Then we find their common ancestor, which will be some ancestor of base_hg_rev + # from which those codelines. + if len(tails) == 0: + common_ancestor = get_single_rev(".") + else: + common_ancestor = get_single_rev("ancestor(" + ",".join(tails) + ")") + eprint("Found common ancestor of tail revisions: %s" % common_ancestor) + + # And then we find the newest git commit whose hg-equivalent is an ancestor of + # that common ancestor, to make sure we are starting from a known hg/git + # commit pair. + for git_commit in sorted(commit_map.values(), key=timeof, reverse=True): + new_base = get_single_rev( + "ancestor(" + common_ancestor + "," + git_commit.hg_rev + ")" + ) + if new_base == common_ancestor: + eprint( + "Pre-existing git commit %s from hg rev %s is descendant of common ancestor; %s" + % ( + git_commit.commit_obj.id, + git_commit.hg_rev, + "walking back further...", + ) + ) + continue + if new_base != git_commit.hg_rev: + eprint( + "Pre-existing git commit %s from hg rev %s is on sibling branch" + " of common ancestor; %s" + % ( + git_commit.commit_obj.id, + git_commit.hg_rev, + "walking back further...", + ) + ) + continue + eprint( + "Pre-existing git commit %s from hg rev %s is sufficiently old; stopping walk" + % (git_commit.commit_obj.id, git_commit.hg_rev) + ) + common_ancestor = new_base + break + + return common_ancestor + + +# Now we prune out all the uninteresting changesets from hg_commits. The +# uninteresting ones are ones that don't touch the target code, are not merges, +# and are not referenced by mozilla tags in the git repo. +# We do this by rewriting the parents to the "interesting" ancestor. +def prune_boring(rev): + while rev in hg_commits: + parent_pruned = False + for i in range(len(hg_commits[rev].parents)): + parent_rev = hg_commits[rev].parents[i] + if parent_rev not in hg_commits: + continue + if hg_commits[parent_rev].touches_sync_code: + continue + if len(hg_commits[parent_rev].parents) > 1: + continue + if parent_rev in hg_to_git_commit_map: + continue + + # If we get here, then `parent_rev` is a boring revision and we can + # prune it. Connect `rev` to its grandparent, and prune the parent + grandparent_rev = hg_commits[parent_rev].parents[0] + hg_commits[rev].parents[i] = grandparent_rev + # eprint("Pruned %s as boring parent of %s, using %s now" % + # (parent_rev, rev, grandparent_rev)) + parent_pruned = True + + if parent_pruned: + # If we pruned a parent, process `rev` again as we might want to + # prune more parents + continue + + # Collapse identical parents, because if the parents are identical + # we don't need to keep multiple copies of them. + hg_commits[rev].parents = list(dict.fromkeys(hg_commits[rev].parents)) + + # If we get here, all of `rev`s parents are interesting, so we can't + # prune them. Move up to the parent rev and start processing that, or + # if we have multiple parents then recurse on those nodes. + if len(hg_commits[rev].parents) == 1: + rev = hg_commits[rev].parents[0] + continue + + for parent_rev in hg_commits[rev].parents: + prune_boring(parent_rev) + return + + +class FakeCommit: + def __init__(self, oid): + self.oid = oid + + +def fake_commit(hg_rev, parent1, parent2): + if parent1 is None: + eprint("ERROR: Trying to build on None") + exit(1) + oid = "githash_%s" % hash(parent1) + eprint("Fake-built %s" % oid) + return FakeCommit(oid) + + +def build_tree(builder, treedata): + for name, value in treedata.items(): + if isinstance(value, dict): + subbuilder = downstream_git_repo.TreeBuilder() + build_tree(subbuilder, value) + builder.insert(name, subbuilder.write(), pygit2.GIT_FILEMODE_TREE) + else: + (filemode, contents) = value + blob_oid = downstream_git_repo.create_blob(contents) + builder.insert(name, blob_oid, filemode) + + +def author_to_signature(author): + pieces = author.strip().split("<") + if len(pieces) != 2 or pieces[1][-1] != ">": + # We could probably handle this better + return pygit2.Signature(author, "") + name = pieces[0].strip() + email = pieces[1][:-1].strip() + return pygit2.Signature(name, email) + + +def real_commit(hg_rev, parent1, parent2): + filetree = dict() + manifest = mozilla_hg_repo.manifest(rev=hg_rev) + for nodeid, permission, executable, symlink, filename in manifest: + if not filename.startswith(relative_path.encode("utf-8")): + continue + if symlink: + filemode = pygit2.GIT_FILEMODE_LINK + elif executable: + filemode = pygit2.GIT_FILEMODE_BLOB_EXECUTABLE + else: + filemode = pygit2.GIT_FILEMODE_BLOB + filecontent = mozilla_hg_repo.cat([filename], rev=hg_rev) + subtree = filetree + for component in filename.split(b"/")[2:-1]: + subtree = subtree.setdefault(component.decode("latin-1"), dict()) + filename = filename.split(b"/")[-1] + subtree[filename.decode("latin-1")] = (filemode, filecontent) + + builder = downstream_git_repo.TreeBuilder() + build_tree(builder, filetree) + tree_oid = builder.write() + + parent1_obj = downstream_git_repo.get(parent1) + if parent1_obj.tree_id == tree_oid: + eprint("Early-exit; tree matched that of parent git commit %s" % parent1) + return parent1_obj + + if parent2 is not None: + parent2_obj = downstream_git_repo.get(parent2) + if parent2_obj.tree_id == tree_oid: + eprint("Early-exit; tree matched that of parent git commit %s" % parent2) + return parent2_obj + + hg_rev_obj = mozilla_hg_repo.log(revrange=hg_rev, limit=1)[0] + commit_author = hg_rev_obj[4].decode("latin-1") + commit_message = hg_rev_obj[5].decode("latin-1") + commit_message += ( + "\n\n[ghsync] From https://hg.mozilla.org/mozilla-central/rev/%s" % hg_rev + + "\n" + ) + + parents = [parent1] + if parent2 is not None: + parents.append(parent2) + commit_oid = downstream_git_repo.create_commit( + None, + author_to_signature(commit_author), + author_to_signature(commit_author), + commit_message, + tree_oid, + parents, + ) + eprint("Built git commit %s" % commit_oid) + return downstream_git_repo.get(commit_oid) + + +def try_commit(hg_rev, parent1, parent2=None): + if False: + return fake_commit(hg_rev, parent1, parent2) + else: + return real_commit(hg_rev, parent1, parent2) + + +def build_git_commits(rev): + debugprint("build_git_commit(%s)..." % rev) + if rev in hg_to_git_commit_map: + debugprint(" maps to %s" % hg_to_git_commit_map[rev].commit_obj.oid) + return hg_to_git_commit_map[rev].commit_obj.oid + + if rev not in hg_commits: + debugprint(" not in hg_commits") + return None + + if len(hg_commits[rev].parents) == 1: + git_parent = build_git_commits(hg_commits[rev].parents[0]) + if not hg_commits[rev].touches_sync_code: + eprint( + "WARNING: Found rev %s that is non-merge and not related to the target" + % rev + ) + return git_parent + eprint("Building git equivalent for %s on top of %s" % (rev, git_parent)) + commit_obj = try_commit(rev, git_parent) + hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj) + debugprint(" built %s as %s" % (rev, commit_obj.oid)) + return commit_obj.oid + + git_parent_1 = build_git_commits(hg_commits[rev].parents[0]) + git_parent_2 = build_git_commits(hg_commits[rev].parents[1]) + if git_parent_1 is None or git_parent_2 is None or git_parent_1 == git_parent_2: + git_parent = git_parent_1 if git_parent_2 is None else git_parent_2 + if not hg_commits[rev].touches_sync_code: + debugprint( + " %s is merge with no parents or doesn't touch WR, returning %s" + % (rev, git_parent) + ) + return git_parent + + eprint( + "WARNING: Found merge rev %s whose parents have identical target code" + ", but modifies the target" % rev + ) + eprint("Building git equivalent for %s on top of %s" % (rev, git_parent)) + commit_obj = try_commit(rev, git_parent) + hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj) + debugprint(" built %s as %s" % (rev, commit_obj.oid)) + return commit_obj.oid + + # An actual merge + eprint( + "Building git equivalent for %s on top of %s, %s" + % (rev, git_parent_1, git_parent_2) + ) + commit_obj = try_commit(rev, git_parent_1, git_parent_2) + hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj) + debugprint(" built %s as %s" % (rev, commit_obj.oid)) + return commit_obj.oid + + +def pretty_print(rev, cset): + desc = " %s" % rev + desc += " parents: %s" % cset.parents + if rev in hg_to_git_commit_map: + desc += " git: %s" % hg_to_git_commit_map[rev].commit_obj.oid + if rev == hg_tip: + desc += " (tip)" + return desc + + +if len(sys.argv) < 3: + eprint("Usage: %s <local-checkout-path> <repo-relative-path>" % sys.argv[0]) + eprint("Current dir must be the mozilla hg repo") + exit(1) + +local_checkout_path = sys.argv[1] +relative_path = sys.argv[2] +mozilla_hg_path = os.getcwd() +NULL_PARENT_REV = "0000000000000000000000000000000000000000" + +downstream_git_repo = pygit2.Repository(pygit2.discover_repository(local_checkout_path)) +mozilla_hg_repo = hglib.open(mozilla_hg_path) +hg_to_git_commit_map = load_git_repository() +base_hg_rev = get_base_hg_rev(hg_to_git_commit_map) +if base_hg_rev is None: + eprint("Found no sync commits or 'mozilla-xxx' tags") + exit(1) + +hg_commits = load_hg_commits(dict(), "only(.," + base_hg_rev + ")") +eprint("Initial set has %s changesets" % len(hg_commits)) +base_hg_rev = get_real_base_hg_rev(hg_commits, hg_to_git_commit_map) +eprint("Using hg rev %s as common ancestor of all interesting changesets" % base_hg_rev) + +# Refresh hg_commits with our wider dataset +hg_tip = get_single_rev(".") +wider_range = "%s::%s" % (base_hg_rev, hg_tip) +hg_commits = load_hg_commits(hg_commits, wider_range) +eprint("Updated set has %s changesets" % len(hg_commits)) + +if DEBUG: + eprint("Graph of descendants of %s" % base_hg_rev) + output = subprocess.check_output( + [ + "hg", + "log", + "--graph", + "-r", + "descendants(" + base_hg_rev + ")", + "--template", + "{node} {desc|firstline}\\n", + ] + ) + for line in output.splitlines(): + eprint(line.decode("utf-8", "ignore")) + +# Also flag any changes that touch the project +query = "(" + wider_range + ') & file("glob:' + relative_path + '/**")' +for cset in get_multiple_revs(query, "{node}"): + debugprint("Changeset %s modifies %s" % (cset, relative_path)) + hg_commits[cset].touches_sync_code = True +eprint( + "Identified %s changesets that touch the target code" + % sum([1 if v.touches_sync_code else 0 for (k, v) in hg_commits.items()]) +) + +prune_boring(hg_tip) + +# hg_tip itself might be boring +if not hg_commits[hg_tip].touches_sync_code and len(hg_commits[hg_tip].parents) == 1: + new_tip = hg_commits[hg_tip].parents[0] + eprint("Pruned tip %s as boring, using %s now" % (hg_tip, new_tip)) + hg_tip = new_tip + +eprint("--- Interesting changesets ---") +for rev, cset in hg_commits.items(): + if cset.touches_sync_code or len(cset.parents) > 1 or rev in hg_to_git_commit_map: + eprint(pretty_print(rev, cset)) +if DEBUG: + eprint("--- Other changesets (not really interesting) ---") + for rev, cset in hg_commits.items(): + if not ( + cset.touches_sync_code + or len(cset.parents) > 1 + or rev in hg_to_git_commit_map + ): + eprint(pretty_print(rev, cset)) + +git_tip = build_git_commits(hg_tip) +if git_tip is None: + eprint("No new changesets generated, exiting.") +else: + downstream_git_repo.create_reference("refs/heads/github-sync", git_tip, force=True) + eprint("Updated github-sync branch to %s, done!" % git_tip) diff --git a/tools/github-sync/read-json.py b/tools/github-sync/read-json.py new file mode 100755 index 0000000000..87264d7df4 --- /dev/null +++ b/tools/github-sync/read-json.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import sys + +j = json.load(sys.stdin) +components = sys.argv[1].split("/") + + +def next_match(json_fragment, components): + if len(components) == 0: + yield json_fragment + else: + component = components[0] + if type(json_fragment) == list: + if component == "*": + for item in json_fragment: + yield from next_match(item, components[1:]) + else: + component = int(component) + if component >= len(j): + sys.exit(1) + yield from next_match(json_fragment[component], components[1:]) + elif type(json_fragment) == dict: + if component == "*": + for key in sorted(json_fragment.keys()): + yield from next_match(json_fragment[key], components[1:]) + elif component not in json_fragment: + sys.exit(1) + else: + yield from next_match(json_fragment[component], components[1:]) + + +for match in list(next_match(j, components)): + if type(match) == dict: + print(" ".join(match.keys())) + else: + print(match) diff --git a/tools/github-sync/readme.md b/tools/github-sync/readme.md new file mode 100644 index 0000000000..d691071336 --- /dev/null +++ b/tools/github-sync/readme.md @@ -0,0 +1,106 @@ +# Github synchronization scripts + +This tool aims to help synchronizing changes from mozilla-central to Github on pushes. +This is useful for Gecko sub-projects that have Github mirrors, like `gfx/wr` linking to `https://github.com/servo/webrender`. +Originally, the tools were developed in `https://github.com/staktrace/wrupdater`, +then got moved under `gfx/wr/ci-scripts/wrupdater`, +and finally migrated here while also abstracting away from WebRender specifically. + +The main entry point is the `sync-to-github.sh` script that is called with the following arguments: + 1. name of the project, matching the repository under `https://github.com/moz-gfx` user (e.g. `webrender`) + 2. relative folder in mozilla-central, which is the upstream for the changes (e.g. `gfx/wr`) + 3. downstream repository specified as "organization/project-name" (e.g. `servo/webrender`) + 4. name to call for auto-approving the pull request (e.g. `bors` or `@bors-servo`) + +It creates a staging directory at `~/.ghsync` if one doesn't already exist, +and clones the the downstream repo into it. +The script also requires the `GECKO_PATH` environment variable +to point to a mercurial clone of `mozilla-central`, and access to the +taskcluster secrets service to get a Github API token. + +The `sync-to-github.sh` script does some setup steps but the bulk of the actual work +is done by the `converter.py` script. This script scans the mercurial +repository for new changes to the relative folder in m-c, +and adds commits to the git repository corresponding to those changes. +There are some details in the implementation that make it more robust +than simply exporting patches and attempting to reapply them; +in particular it builds a commit tree structure that mirrors what is found in +the `mozilla-central` repository with respect to branches and merges. +So if conflicting changes land on autoland and inbound, and then get +merged, the git repository commits will have the same structure with +a fork/merge in the commit history. This was discovered to be +necessary after a previous version ran into multiple cases where +the simple patch approach didn't really work. + +One of the actions the `converter.py` takes is to find the last sync point +between Github and mozilla-central. This is done based on the following markers: + - commit message containing the string "[ghsync] From https://hg.mozilla.org/mozilla-central/rev/xxx" + - commit message containing the string "[wrupdater] From https://hg.mozilla.org/mozilla-central/rev/xxx" + - commit with tag "mozilla-xxx" +(where xxx is always a mozilla-central hg revision identifier). + +Once the converter is done converting, the `sync-to-github.sh` script +finishes the process by pushing the new commits to the `github-sync` branch +of the `https://github.com/moz-gfx/<project-name>` repository, +and generating a pull request against the downstream repository. It also +leaves a comment on the PR that triggers testing and automatic merge of the PR. +If there is already a pull request (perhaps from a previous run) the +pre-existing PR is force-updated instead. This allows for graceful +handling of scenarios where the PR failed to get merged (e.g. due to +CI failures on the Github side). + +The script is intended to by run by taskcluster for any changes that +touch the relative folder that land on `mozilla-central`. This may mean +that multiple instances of this script run concurrently, or even out +of order (i.e. the task for an older m-c push runs after the task for +a newer m-c push). The script was written with these possibilities in +mind and should be able to eventually recover from any such scenario +automatically (although it may take additional changes to mozilla-central +for such recovery to occur). That being said, the number of pathological +scenarios here is quite large and they were not really tested. + +## Ownership and access + +When this tool is run in Firefox CI, it needs to have push permissions to +the `moz-gfx` github user's account. It gets this permission via a secret token +stored in the Firefox CI taskcluster secrets service. If you need to update +the token, you need to find somebody who is a member of the +[webrender-ci access group](https://people.mozilla.org/a/webrender-ci/). The +Google Drive associated with that access group has additional documentation +on the `moz-gfx` github user and the secret token. + +## Debugging + +To debug the converter.py script, you need to have a hg checkout of +mozilla-central, let's assume it's at $MOZILLA. First create a virtualenv +with the right dependencies installed: + +``` +mkdir -p $HOME/.ghsync +virtualenv --python=python3 $HOME/.ghsync/venv +source $HOME/.ghsync/venv/bin/activate +pip3 install -r $MOZILLA/taskcluster/docker/github-sync/requirements.txt +``` + +Also create a checkout of the downstream github repo and set up a `github-sync` +branch to the point where you want port commits to. For example, for WebRender +you'd do: + +``` +cd $HOME/.ghsync +git clone https://github.com/servo/webrender +cd webrender +git checkout -b github-sync master +``` + +(You can set the github-sync branch to a past revision if you want to replicate +a failure that already got committed). + +Then run the converter from your hg checkout: + +``` +cd $MOZILLA +tools/github-sync/converter.py $HOME/.ghsync/webrender gfx/wr +``` + +You can set the DEBUG variable in the script to True to get more output. diff --git a/tools/github-sync/sync-to-github.sh b/tools/github-sync/sync-to-github.sh new file mode 100755 index 0000000000..d677649748 --- /dev/null +++ b/tools/github-sync/sync-to-github.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +# Do NOT set -x here, since that will expose a secret API token! +set -o errexit +set -o nounset +set -o pipefail + +if [[ "$(uname)" != "Linux" ]]; then + echo "Error: this script must be run on Linux due to readlink semantics" + exit 1 +fi + +# GECKO_PATH should definitely be set +if [[ -z "${GECKO_PATH}" ]]; then + echo "Error: GECKO_PATH must point to a hg clone of mozilla-central" + exit 1 +fi + +# Internal variables, don't fiddle with these +MYSELF=$(readlink -f ${0}) +MYDIR=$(dirname "${MYSELF}") +WORKDIR="${HOME}/.ghsync" +TMPDIR="${WORKDIR}/tmp" + +NAME="$1" +RELATIVE_PATH="$2" +DOWNSTREAM_REPO="$3" +BORS="$4" +BRANCH="github-sync" + +mkdir -p "${TMPDIR}" + +# Bring the project clone to a known good up-to-date state +if [[ ! -d "${WORKDIR}/${NAME}" ]]; then + echo "Setting up ${NAME} repo..." + git clone "https://github.com/${DOWNSTREAM_REPO}" "${WORKDIR}/${NAME}" + pushd "${WORKDIR}/${NAME}" + git remote add moz-gfx https://github.com/moz-gfx/${NAME} + popd +else + echo "Updating ${NAME} repo..." + pushd "${WORKDIR}/${NAME}" + git checkout master + git pull + popd +fi + +if [[ -n "${GITHUB_SECRET:-}" ]]; then + echo "Obtaining github API token..." + # Be careful, GITHUB_TOKEN is secret, so don't log it (or any variables + # built using it). + GITHUB_TOKEN=$( + curl -sSfL "$TASKCLUSTER_PROXY_URL/secrets/v1/secret/${GITHUB_SECRET}" | + ${MYDIR}/read-json.py "secret/token" + ) + AUTH="moz-gfx:${GITHUB_TOKEN}" + CURL_AUTH="Authorization: bearer ${GITHUB_TOKEN}" +fi + +echo "Pushing base ${BRANCH} branch..." +pushd "${WORKDIR}/${NAME}" +git fetch moz-gfx +git checkout -B ${BRANCH} moz-gfx/${BRANCH} || git checkout -B ${BRANCH} master + +if [[ -n "${GITHUB_SECRET:-}" ]]; then + # git may emit error messages that contain the URL, so let's sanitize them + # or we might leak the auth token to the task log. + git push "https://${AUTH}@github.com/moz-gfx/${NAME}" \ + "${BRANCH}:${BRANCH}" 2>&1 | sed -e "s/${AUTH}/_SANITIZED_/g" + # Re-fetch to update the remote moz-gfx/$BRANCH branch in the local repo; + # normally the push does this but we use a fully-qualified URL for + # pushing so it doesn't happen. + git fetch moz-gfx +fi +popd + +# Run the converter +echo "Running converter..." +pushd "${GECKO_PATH}" +"${MYDIR}/converter.py" "${WORKDIR}/${NAME}" "${RELATIVE_PATH}" +popd + +# Check to see if we have changes that need pushing +echo "Checking for new changes..." +pushd "${WORKDIR}/${NAME}" +PATCHCOUNT=$(git log --oneline moz-gfx/${BRANCH}..${BRANCH}| wc -l) +if [[ ${PATCHCOUNT} -eq 0 ]]; then + echo "No new patches found, aborting..." + exit 0 +fi + +# Log the new changes, just for logging purposes +echo "Here are the new changes:" +git log --graph --stat moz-gfx/${BRANCH}..${BRANCH} + +# Collect PR numbers of PRs opened on Github and merged to m-c +set +e +FIXES=$( + git log master..${BRANCH} | + grep "\[import_pr\] From https://github.com/${DOWNSTREAM_REPO}/pull" | + sed -e "s%.*pull/% Fixes #%" | + uniq | + tr '\n' ',' +) +echo "${FIXES}" +set -e + +if [[ -z "${GITHUB_SECRET:-}" ]]; then + echo "Running in try push, exiting now" + exit 0 +fi + +echo "Pushing new changes to moz-gfx..." +# git may emit error messages that contain the URL, so let's sanitize them +# or we might leak the auth token to the task log. +git push "https://${AUTH}@github.com/moz-gfx/${NAME}" +${BRANCH}:${BRANCH} \ + 2>&1 | sed -e "s/${AUTH}/_SANITIZED_/g" + +CURL_HEADER="Accept: application/vnd.github.v3+json" +CURL=(curl -sSfL -H "${CURL_HEADER}" -H "${CURL_AUTH}") +# URL extracted here mostly to make servo-tidy happy with line lengths +API_URL="https://api.github.com/repos/${DOWNSTREAM_REPO}" + +# Check if there's an existing PR open +echo "Listing pre-existing pull requests..." +"${CURL[@]}" "${API_URL}/pulls?head=moz-gfx:${BRANCH}" | + tee "${TMPDIR}/pr.get" +set +e +COMMENT_URL=$(cat "${TMPDIR}/pr.get" | ${MYDIR}/read-json.py "0/comments_url") +HAS_COMMENT_URL="${?}" +set -e + +if [[ ${HAS_COMMENT_URL} -ne 0 ]]; then + echo "Pull request not found, creating..." + # The PR doesn't exist yet, so let's create it + ( echo -n '{ "title": "Sync changes from mozilla-central '"${RELATIVE_PATH}"'"' + echo -n ', "body": "'"${FIXES}"'"' + echo -n ', "head": "moz-gfx:'"${BRANCH}"'"' + echo -n ', "base": "master" }' + ) > "${TMPDIR}/pr.create" + "${CURL[@]}" -d "@${TMPDIR}/pr.create" "${API_URL}/pulls" | + tee "${TMPDIR}/pr.response" + COMMENT_URL=$( + cat "${TMPDIR}/pr.response" | + ${MYDIR}/read-json.py "comments_url" + ) +fi + +# At this point COMMENTS_URL should be set, so leave a comment to tell bors +# to merge the PR. +echo "Posting r+ comment to ${COMMENT_URL}..." +echo '{ "body": "'"$BORS"' r=auto" }' > "${TMPDIR}/bors_rplus" +"${CURL[@]}" -d "@${TMPDIR}/bors_rplus" "${COMMENT_URL}" + +echo "All done!" diff --git a/tools/jprof/README.html b/tools/jprof/README.html new file mode 100644 index 0000000000..ac25acc5ef --- /dev/null +++ b/tools/jprof/README.html @@ -0,0 +1,330 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> +<head><title>The Jprof Profiler</title></head> + +<body bgcolor="#FFFFFF" text="#000000" + link="#0000EE" vlink="#551A8B" alink="#FF0000"> +<center> +<h1>The Jprof Profiler</h1> +<font size="-1"> +<a href="mailto:jim_nance%yahoo.com">jim_nance@yahoo.com</a><p> +Recent (4/2011) updates Randell Jesup (see bugzilla for contact info) +</font> +<hr> + +<a href="#introduction">Introduction</a> | <a href="#operation">Operation</a> | +<a href="#setup">Setup</a> | <a href="#usage">Usage</a> | +<a href="#interpretation">Interpretation</a> + +</center> +<hr> + +<h3><a name="introduction">Introduction</a></h3> + +Jprof is a profiling tool. I am writing it because I need to find out +where mozilla is spending its time, and there do not seem to be any +profilers for Linux that can handle threads and/or shared libraries. +This code is based heavily on Kipp Hickman's leaky. + +<h3><a name="operation">Operation</a></h3> + +Jprof operates by installing a timer which periodically interrupts mozilla. +When this timer goes off, the jprof code inside mozilla walks the function call +stack to determine which code was executing and saves the results into the +<code>jprof-log</code> and <code>jprof-map</code> files. By collecting a large +number of these call stacks, it is possible to deduce where mozilla is spending +its time. + +<h3><a name="setup">Setup</a></h3> + +<p>Configure your mozilla with jprof support by adding +<code>--enable-jprof</code> to your configure options (eg adding +<code>ac_add_options --enable-jprof</code> to your <code>.mozconfig</code>) and +making sure that you do <strong>not</strong> have the +<code>--enable-strip</code> configure option set -- jprof needs symbols to +operate. On many architectures with GCC, you'll need to add +<code>--enable-optimize="-O3 -fno-omit-frame-pointer"</code> or the +equivalent to ensure frame pointer generation in the compiler you're using.</p> + +<p>Finally, build mozilla with your new configuration. Now you can run jprof.</p> + +<h3><a name="usage">Usage</a></h3> +<pre> jprof [-v] [-t] [-e exclude] [-i include] [-s stackdepth] [--last] [--all] [--start n [--end m]] [--output-dir dir] prog log [log2 ...]</pre> +Options: +<ul> + <li><b>-s depth</b> : Limit depth looked at from captured stack + frames</li> + <li><b>-v</b> : Output some information about the symbols, memory map, etc.</li> + <li><b>-t or --threads</b> : Group output according to thread. May require external + LD_PRELOAD library to help force sampling of spawned threads; jprof + may capture the main thread only. See <a + href="http://sam.zoy.org/writings/programming/gprof.html">gprof-helper</a>; + it may need adaption for jprof.</li> + <li><b>--only-thread id</b> : Only output data for thread 'id'</li> + <li><b>-e exclusion</b> : Allows excluding specific stack frames</li> + <li><b>-i inclusion</b> : Allows including specific stack frames</li> + <li><b>--last</b> : Only process data from the last 'section' of sampling + (starting at the last PROF)</li> + <li><b>--start N</b> : Start processing data at 'section' N </li> + <li><b>--end N</b> : Stop processing data at 'section' N </li> + <li><b>--output-dir dir</b> : Store generated .html files in the given directory </li> +</ul> +The behavior of jprof is determined by the value of the JPROF_FLAGS environment +variable. This environment variable can be composed of several substrings +which have the following meanings: +<ul> + <li> <b>JP_START</b> : Install the signal handler, and start sending the + timer signals. + + <li> <b>JP_DEFER</b> : Install the signal handler, but don't start sending + the timer signals. The user must start the signals by sending the first + one (with <code>kill -PROF</code>, or with <code>kill -ALRM</code> if + JP_REALTIME is used, or with <code>kill -POLL</code> (also known as <code>kill -IO</code>) if JP_RTC_HZ is used). + + <li> <b>JP_FIRST=x</b> : Wait x seconds before starting the timer + + <li> <b>JP_PERIOD=y</b> : Set timer to interrupt every y seconds. Only + values of y greater than or equal to 0.001 are supported. Default is + 0.050 (50ms). + + <li> <b>JP_REALTIME</b> : Do the profiling in intervals of real time rather + than intervals of time used by the mozilla process (and the kernel + when doing work for mozilla). This could probably lead to weird + results (you'll see whatever runs when mozilla is waiting for events), + but is needed to see time spent in the X server. + + <li> <b>JP_RTC_HZ=freq</b> : This option, only available on Linux if the + kernel is built with RTC support, makes jprof use the RTC timer instead of + using its own timer. This option, like JP_REALTIME, uses intervals of real + time. This option overrides JP_PERIOD. <code>freq</code> is the frequency + at which the timer should fire, measured in Hz. It must be a power of 2. + The maximal frequency allowed by the kernel can be changed by writing to + <code>/proc/sys/dev/rtc/max-user-freq</code>; the maximum value it can be + set to is 8192. Note that <code>/dev/rtc</code> will need to be readable + by the Firefox process; making that file world-readable is a simple way to + accomplish that. + + <li> <b>JP_CIRCULAR=size</b> : This tells jprof to store samples in a + circular buffer of the given size, which then will be saved (appended) + to disk when SIGUSR1 is received or JProfStopProfiling is done. If the + buffer overflows, the oldest entries will be evicted until there's + space for the new entry.<p> + + SIGUSR2 will cause the circular buffer to be cleared. + + <li> <b>JP_FILENAME=basefilename</b> : This is the filename used for + saving the log files to; the default is "jprof-log". If Electrolysis + is used, each process after the first will have the process ID + added ("jprof-log-3212"); + +</ul> + +<h4>Starting and stopping jprof from JavaScript</h4> +<p> +A build with jprof enabled adds four functions to the Window object:<p> +<code>JProfStartProfiling()</code> and <code>JProfStopProfiling()</code>: When used with JP_DEFER, these +allow one to start and stop the timer just around whatever critical section is +being profiled.</p><p> +<code>JProfClearCircular()</code> and <code>JProfSaveCircular()</code>: +These clear the circular buffer and save the buffer (without stopping), respectively.</p> + +<h4>Examples of JPROF_FLAGS usage</h4> +<ul> + + <li>To make the timer start firing 3 seconds after the program is started and + fire every 25 milliseconds of program time use: + <pre> + setenv JPROF_FLAGS "JP_START JP_FIRST=3 JP_PERIOD=0.025" </pre> + + <li>To make the timer start on your signal and fire every 1 millisecond of + program time use: + <pre> + setenv JPROF_FLAGS "JP_DEFER JP_PERIOD=0.001" </pre> + + <li>To make the timer start on your signal and fire every 10 milliseconds of + wall-clock time use: + <pre> + setenv JPROF_FLAGS "JP_DEFER JP_PERIOD=0.010 JP_REALTIME" </pre> + + <li>To make the timer start on your signal and fire at 8192 Hz in wall-clock + time use: + <pre> + setenv JPROF_FLAGS "JP_DEFER JP_RTC_HZ=8192" </pre> + + <li>To make the timer start on JProfStartProfiling() and run continously + with a 1ms sample rate until told to stop, then save the last 1MB of + data: + <pre> + setenv JPROF_FLAGS "JP_DEFER JP_CIRCULAR=1048576 JP_PERIOD=0.001" </pre> + +</ul> + +<h4>Pausing profiles</h4> + +<P>jprof can be paused at any time by sending a SIGUSR1 to mozilla (<code>kill +-USR1</code>). This will cause the timer signals to stop and jprof-map to be +written, but it will not close jprof-log. Combining SIGUSR1 with the JP_DEFER +option allows profiling of one sequence of actions by starting the timer right +before starting the actions and stopping the timer right afterward. + +<P>After a SIGUSR1, sending another timer signal (SIGPROF, SIGALRM, or SIGPOLL (aka SIGIO), +depending on the mode) can be used to continue writing data to the same +output. + +<P>SIGUSR2 will cause the circular buffer to be cleared, if it's in use. +This is useful right before running a test when you're using a large, +continuous circular buffer, or programmatically at the start of an action +which might take too long (JProfClearCircular()). + +<h4>Looking at the results</h4> + +Now that we have <code>jprof-log</code> and <code>jprof-map</code> files, we +can use the jprof executable is used to turn them into readable output. To do +this jprof needs the name of the mozilla binary and the log file. It deduces +the name of the map file: + +<pre> + ./jprof /home/user/mozilla/objdir/dist/bin/firefox ./jprof-log > tmp.html +</pre> + +This will generate the file <code>tmp.html</code> which you should view in a +web browser. + +<pre> + ./jprof --output-dir=/tmp /home/user/mozilla/objdir/dist/bin/firefox ./jprof-log* +</pre> + +This will generate a set of files in /tmp for each process. + + +<h3><a name="interpretation">Interpretation</a></h3> + + +The Jprof output is split into a flat portion and a hierarchical portion. +There are links to each section at the top of the page. It is typically +easier to analyze the profile by starting with the flat output and following +the links contained in the flat output up to the hierarchical output. + +<h4><a name="flat">Flat output</a></h3> + +The flat portion of the profile indicates which functions were executing +when the timer was going off. It is displayed as a list of functions names +on the right and the number of times that function was interrupted on the +left. The list is sorted by decreasing interrupt count. For example: + +<blockquote> <pre> +Total hit count: 151603 +Count %Total Function Name + +<a href="#23081">8806 5.8 __libc_poll</a> +<a href="#40008">2254 1.5 __i686.get_pc_thunk.bx</a> +<a href="#21390">2053 1.4 _int_malloc</a> +<a href="#49013">1777 1.2 ComputedStyle::GetStyleData(nsStyleStructID)</a> +<a href="#21380">1600 1.1 __libc_malloc</a> +<a href="#603">1552 1.0 nsCOMPtr_base::~nsCOMPtr_base()</a> +</pre> </blockquote> + +This shows that of the 151603 times the timer fired, 1777 (1.2% of the total) were inside ComputedStyle::GetStyleData() and 1552 (1.0% of the total) were in the nsCOMPtr_base destructor. + +<p> +In general, the functions with the highest count are the functions which +are taking the most time. + +<P> +The function names are linked to the entry for that function in the +hierarchical profile, which is described in the next section. + +<h4><a name="hier">Hierarchical output</a></h4> + +The hierarchical output is divided up into sections, with each section +corresponding to one function. A typical section looks something like +this: + +<blockquote><pre> + index Count Hits Function Name + <A href="#72871"> 545 (46.4%) nsBlockFrame::ReflowInlineFrames(nsBlockReflowState&, nsLineList_iterator, int*)</A> + <A href="#72873"> 100 (8.5%) nsBlockFrame::ReflowDirtyLines(nsBlockReflowState&)</A> + 72870 4 (0.3%) <a name=72870> 645 (54.9%)</a> <b>nsBlockFrame::DoReflowInlineFrames(nsBlockReflowState&, nsLineLayout&, nsLineList_iterator, nsFlowAreaRect&, int&, nsFloatManager::SavedState*, int*, LineReflowStatus*, int)</b> + <A href="#72821"> 545 (46.4%) nsBlockFrame::ReflowInlineFrame(nsBlockReflowState&, nsLineLayout&, nsLineList_iterator, nsIFrame*, LineReflowStatus*)</A> + <A href="#72853"> 83 (7.1%) nsBlockFrame::PlaceLine(nsBlockReflowState&, nsLineLayout&, nsLineList_iterator, nsFloatManager::SavedState*, nsRect&, int&, int*)</A> + <A href="#74150"> 9 (0.8%) nsLineLayout::BeginLineReflow(int, int, int, int, int, int)</A> + <A href="#74897"> 1 (0.1%) nsTextFrame::GetType() const</A> + <A href="#74131"> 1 (0.1%) nsLineLayout::RelativePositionFrames(nsOverflowAreas&)</A> + <A href="#58320"> 1 (0.1%) __i686.get_pc_thunk.bx</A> + <A href="#53077"> 1 (0.1%) PL_ArenaAllocate</A> +</pre></blockquote> + +The information this block tells us is: + +<ul> +<li>There were 4 profiler hits <em>in</em> <code>nsBlockFrame::DoReflowInlineFrames</code> +<li>There were 645 profiler hits <em>in or under</em> <code>nsBlockFrame::DoReflowInlineFrames</code>. Of these: +<ul> + <li>545 were in or under <code>nsBlockFrame::ReflowInlineFrame</code> + <li>83 were in or under <code>nsBlockFrame::PlaceLine</code> + <li>9 were in or under <code>nsLineLayout::BeginLineReflow</code> + <li>1 was in or under <code>nsTextFrame::GetType</code> + <li>1 was in or under <code>nsLineLayout::RelativePositionFrames</code> + <li>1 was in or under <code>__i686.get_pc_thunk.bx</code> + <li>1 was in or under <code>PL_ArenaAllocate</code> +</ul> +<li>Of these 645 calls into <code>nsBlockFrame::DoReflowInlineFrames</code>: +<ul> + <li>545 came from <code>nsBlockFrame::ReflowInlineFrames</code> + <li>100 came from <code>nsBlockFrame::ReflowDirtyLines</code> +</ul> +</ul> + + +The rest of this section explains how to read this information off from the jprof output. + +<p>This block corresponds to the function <code>nsBlockFrame::DoReflowInlineFrames</code>, which is +therefore bolded and not a link. The name of this function is preceded by +five numbers which have the following meaning. The number on the left (72870) +is the index number, and is not important. The next number (4) and the +percentage following (0.3%) are the number +of times this function was interrupted by the timer and the percentage of +the total hits that is. The last number pair ("645 (54.9%)") +are the number of times this function was in the call stack when the timer went +off. That is, the timer went off while we were in code that was ultimately +called from <code>nsBlockFrame::DoReflowInlineFrames</code>. +<p>For our example we can see that our function was in the call stack for +645 interrupt ticks, but we were only the function that was running when +the interrupt arrived 4 times. +<P> +The functions listed above the line for <code>nsBlockFrame::DoReflowInlineFrames</code> are its +callers. The numbers to the left of these function names are the numbers of +times these functions were in the call stack as callers of +<code>nsBlockFrame::DoReflowInlineFrames</code>. In our example, we were called 545 times by +<code>nsBlockFrame::ReflowInlineFrames</code> and 100 times by +<code>nsBlockFrame::ReflowDirtyLines</code>. +<P> +The functions listed below the line for <code>nsBlockFrame::DoReflowInlineFrames</code> are its +callees. The numbers to the left of the function names are the numbers of +times these functions were in the callstack as callees of +<code>nsBlockFrame::DoReflowInlineFrames</code> and the corresponding percentages. In our example, of the 645 profiler hits under <code>nsBlockFrame::DoReflowInlineFrames</code> 545 were under <code>nsBlockFrame::ReflowInlineFrame</code>, 83 were under <code>nsBlockFrame::PlaceLine</code>, and so forth.<p> + +<b>NOTE:</b> If there are loops of execution or recursion, the numbers will +not add up and percentages can exceed 100%. If a function directly calls +itself "(self)" will be appended to the line, but indirect recursion will +not be marked. + +<h3>Bugs</h3> +The current build of Jprof has only been tested under Ubuntu 8.04 LTS, but +should work under any fairly modern linux distribution using GCC/GLIBC. +Please update this document with any known compatibilities/incompatibilities. +<p> +If you get an error:<p><code>Inconsistency detected by ld.so: dl-open.c: 260: dl_open_worker: Assertion `_dl_debug_initialize (0, args->nsid)->r_state == RT_CONSISTENT' failed! +</code><p>that means you've hit a timing hole in the version of glibc you're +running. See <a +href="http://sources.redhat.com/bugzilla/show_bug.cgi?id=4578">Redhat bug 4578</a>. +<!-- <h3>Update</h3> +<ul> +</ul> +--> + +</body> +</html> diff --git a/tools/jprof/bfd.cpp b/tools/jprof/bfd.cpp new file mode 100644 index 0000000000..6be8fde760 --- /dev/null +++ b/tools/jprof/bfd.cpp @@ -0,0 +1,221 @@ +// vim:ts=8:sw=2:et: +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "leaky.h" + +#ifdef USE_BFD +# include <stdio.h> +# include <string.h> +# include <fcntl.h> +# include <unistd.h> +# include <libgen.h> +# include <bfd.h> +# include <cxxabi.h> + +static bfd* try_debug_file(const char* filename, unsigned long crc32) { + int fd = open(filename, O_RDONLY); + if (fd < 0) return nullptr; + + unsigned char buf[4 * 1024]; + unsigned long crc = 0; + + while (1) { + ssize_t count = read(fd, buf, sizeof(buf)); + if (count <= 0) break; + + crc = bfd_calc_gnu_debuglink_crc32(crc, buf, count); + } + + close(fd); + + if (crc != crc32) return nullptr; + + bfd* object = bfd_openr(filename, nullptr); + if (!bfd_check_format(object, bfd_object)) { + bfd_close(object); + return nullptr; + } + + return object; +} + +static bfd* find_debug_file(bfd* lib, const char* aFileName) { + // check for a separate debug file with symbols + asection* sect = bfd_get_section_by_name(lib, ".gnu_debuglink"); + + if (!sect) return nullptr; + + bfd_size_type debuglinkSize = bfd_section_size(objfile->obfd, sect); + + char* debuglink = new char[debuglinkSize]; + bfd_get_section_contents(lib, sect, debuglink, 0, debuglinkSize); + + // crc checksum is aligned to 4 bytes, and after the NUL. + int crc_offset = (int(strlen(debuglink)) & ~3) + 4; + unsigned long crc32 = bfd_get_32(lib, debuglink + crc_offset); + + // directory component + char* dirbuf = strdup(aFileName); + const char* dir = dirname(dirbuf); + + static const char debug_subdir[] = ".debug"; + // This is gdb's default global debugging info directory, but gdb can + // be instructed to use a different directory. + static const char global_debug_dir[] = "/usr/lib/debug"; + + char* filename = + new char[strlen(global_debug_dir) + strlen(dir) + crc_offset + 3]; + + // /path/debuglink + sprintf(filename, "%s/%s", dir, debuglink); + bfd* debugFile = try_debug_file(filename, crc32); + if (!debugFile) { + // /path/.debug/debuglink + sprintf(filename, "%s/%s/%s", dir, debug_subdir, debuglink); + debugFile = try_debug_file(filename, crc32); + if (!debugFile) { + // /usr/lib/debug/path/debuglink + sprintf(filename, "%s/%s/%s", global_debug_dir, dir, debuglink); + debugFile = try_debug_file(filename, crc32); + } + } + + delete[] filename; + free(dirbuf); + delete[] debuglink; + + return debugFile; +} + +// Use an indirect array to avoid copying tons of objects +Symbol** leaky::ExtendSymbols(int num) { + long n = numExternalSymbols + num; + + externalSymbols = (Symbol**)realloc(externalSymbols, + (size_t)(sizeof(externalSymbols[0]) * n)); + Symbol* new_array = new Symbol[n]; + for (int i = 0; i < num; i++) { + externalSymbols[i + numExternalSymbols] = &new_array[i]; + } + lastSymbol = externalSymbols + n; + Symbol** sp = externalSymbols + numExternalSymbols; + numExternalSymbols = n; + return sp; +} + +# define NEXT_SYMBOL \ + do { \ + sp++; \ + if (sp >= lastSymbol) { \ + sp = ExtendSymbols(16384); \ + } \ + } while (0) + +void leaky::ReadSymbols(const char* aFileName, u_long aBaseAddress) { + int initialSymbols = usefulSymbols; + if (nullptr == externalSymbols) { + externalSymbols = (Symbol**)calloc(sizeof(Symbol*), 10000); + Symbol* new_array = new Symbol[10000]; + for (int i = 0; i < 10000; i++) { + externalSymbols[i] = &new_array[i]; + } + numExternalSymbols = 10000; + } + Symbol** sp = externalSymbols + usefulSymbols; + lastSymbol = externalSymbols + numExternalSymbols; + + // Create a dummy symbol for the library so, if it doesn't have any + // symbols, we show it by library. + (*sp)->Init(aFileName, aBaseAddress); + NEXT_SYMBOL; + + bfd_boolean kDynamic = (bfd_boolean) false; + + static int firstTime = 1; + if (firstTime) { + firstTime = 0; + bfd_init(); + } + + bfd* lib = bfd_openr(aFileName, nullptr); + if (nullptr == lib) { + return; + } + if (!bfd_check_format(lib, bfd_object)) { + bfd_close(lib); + return; + } + + bfd* symbolFile = find_debug_file(lib, aFileName); + + // read mini symbols + PTR minisyms; + unsigned int size; + long symcount = 0; + + if (symbolFile) { + symcount = bfd_read_minisymbols(symbolFile, kDynamic, &minisyms, &size); + if (symcount == 0) { + bfd_close(symbolFile); + } else { + bfd_close(lib); + } + } + if (symcount == 0) { + symcount = bfd_read_minisymbols(lib, kDynamic, &minisyms, &size); + if (symcount == 0) { + // symtab is empty; try dynamic symbols + kDynamic = (bfd_boolean) true; + symcount = bfd_read_minisymbols(lib, kDynamic, &minisyms, &size); + } + symbolFile = lib; + } + + asymbol* store; + store = bfd_make_empty_symbol(symbolFile); + + // Scan symbols + size_t demangle_buffer_size = 128; + char* demangle_buffer = (char*)malloc(demangle_buffer_size); + bfd_byte* from = (bfd_byte*)minisyms; + bfd_byte* fromend = from + symcount * size; + for (; from < fromend; from += size) { + asymbol* sym; + sym = + bfd_minisymbol_to_symbol(symbolFile, kDynamic, (const PTR)from, store); + + symbol_info syminfo; + bfd_get_symbol_info(symbolFile, sym, &syminfo); + + // if ((syminfo.type == 'T') || (syminfo.type == 't')) { + const char* nm = bfd_asymbol_name(sym); + if (nm && nm[0]) { + char* dnm = nullptr; + if (strncmp("__thunk", nm, 7)) { + dnm = + abi::__cxa_demangle(nm, demangle_buffer, &demangle_buffer_size, 0); + if (dnm) { + demangle_buffer = dnm; + } + } + (*sp)->Init(dnm ? dnm : nm, syminfo.value + aBaseAddress); + NEXT_SYMBOL; + } + // } + } + + free(demangle_buffer); + demangle_buffer = nullptr; + + bfd_close(symbolFile); + + int interesting = sp - externalSymbols; + if (!quiet) { + printf("%s provided %d symbols\n", aFileName, interesting - initialSymbols); + } + usefulSymbols = interesting; +} + +#endif /* USE_BFD */ diff --git a/tools/jprof/coff.cpp b/tools/jprof/coff.cpp new file mode 100644 index 0000000000..0efa83960c --- /dev/null +++ b/tools/jprof/coff.cpp @@ -0,0 +1,93 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "leaky.h" + +#ifdef USE_COFF + +# define LANGUAGE_C +# include <sym.h> +# include <cmplrs/stsupport.h> +# include <symconst.h> +# include <filehdr.h> +# include <ldfcn.h> +# include <string.h> +# include <stdlib.h> + +# ifdef IRIX4 +extern "C" { +extern char* demangle(char const* in); +}; +# else +# include <dem.h> +# endif + +static char* Demangle(char* rawName) { +# ifdef IRIX4 + return strdup(demangle(rawName)); +# else + char namebuf[4000]; + demangle(rawName, namebuf); + return strdup(namebuf); +# endif +} + +void leaky::readSymbols(const char* fileName) { + LDFILE* ldptr; + + ldptr = ldopen(fileName, nullptr); + if (!ldptr) { + fprintf(stderr, "%s: unable to open \"%s\"\n", applicationName, fileName); + exit(-1); + } + if (PSYMTAB(ldptr) == 0) { + fprintf(stderr, "%s: \"%s\": has no symbol table\n", applicationName, + fileName); + exit(-1); + } + + long isymMax = SYMHEADER(ldptr).isymMax; + long iextMax = SYMHEADER(ldptr).iextMax; + long iMax = isymMax + iextMax; + + long alloced = 10000; + Symbol* syms = (Symbol*)malloc(sizeof(Symbol) * 10000); + Symbol* sp = syms; + Symbol* last = syms + alloced; + SYMR symr; + + for (long isym = 0; isym < iMax; isym++) { + if (ldtbread(ldptr, isym, &symr) != SUCCESS) { + fprintf(stderr, "%s: can't read symbol #%d\n", applicationName, isym); + exit(-1); + } + if (isym < isymMax) { + if ((symr.st == stStaticProc) || + ((symr.st == stProc) && + ((symr.sc == scText) || (symr.sc == scAbs))) || + ((symr.st == stBlock) && (symr.sc == scText))) { + // Text symbol. Set name field to point to the symbol name + sp->name = Demangle(ldgetname(ldptr, &symr)); + sp->address = symr.value; + sp++; + if (sp >= last) { + long n = alloced + 10000; + syms = (Symbol*)realloc(syms, (size_t)(sizeof(Symbol) * n)); + last = syms + n; + sp = syms + alloced; + alloced = n; + } + } + } + } + + int interesting = sp - syms; + if (!quiet) { + printf("Total of %d symbols\n", interesting); + } + usefulSymbols = interesting; + externalSymbols = syms; +} + +#endif /* USE_COFF */ diff --git a/tools/jprof/elf.cpp b/tools/jprof/elf.cpp new file mode 100644 index 0000000000..c2e00f60da --- /dev/null +++ b/tools/jprof/elf.cpp @@ -0,0 +1,128 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "leaky.h" + +#ifdef USE_ELF + +# include "leaky.h" +# include <stdio.h> +# include <malloc.h> +# include <libelf/libelf.h> +# include <unistd.h> +# include <fcntl.h> +# include <string.h> + +void leaky::readSymbols(const char* fileName) { + int fd = ::open(fileName, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "%s: unable to open \"%s\"\n", applicationName, fileName); + exit(-1); + } + + elf_version(EV_CURRENT); + Elf* elf = elf_begin(fd, ELF_C_READ, 0); + if (!elf) { + fprintf(stderr, "%s: \"%s\": has no symbol table\n", applicationName, + fileName); + exit(-1); + } + + long alloced = 10000; + Symbol* syms = (Symbol*)malloc(sizeof(Symbol) * 10000); + Symbol* sp = syms; + Symbol* last = syms + alloced; + + // Get each of the relevant sections and add them to the list of + // symbols. + Elf32_Ehdr* ehdr = elf32_getehdr(elf); + if (!ehdr) { + fprintf(stderr, "%s: elf library lossage\n", applicationName); + exit(-1); + } +# if 0 + Elf32_Half ndx = ehdr->e_shstrndx; +# endif + + Elf_Scn* scn = 0; + int strtabndx = -1; + for (int i = 1; (scn = elf_nextscn(elf, scn)) != 0; i++) { + Elf32_Shdr* shdr = elf32_getshdr(scn); +# if 0 + char *name = elf_strptr(elf, ndx, (size_t) shdr->sh_name); + printf("Section %s (%d 0x%x)\n", name ? name : "(null)", + shdr->sh_type, shdr->sh_type); +# endif + if (shdr->sh_type == SHT_STRTAB) { + /* We assume here that string tables preceed symbol tables... */ + strtabndx = i; + continue; + } +# if 0 + if (shdr->sh_type == SHT_DYNAMIC) { + /* Dynamic */ + Elf_Data *data = elf_getdata(scn, 0); + if (!data || !data->d_size) { + printf("No data..."); + continue; + } + + Elf32_Dyn *dyn = (Elf32_Dyn*) data->d_buf; + Elf32_Dyn *lastdyn = + (Elf32_Dyn*) ((char*) data->d_buf + data->d_size); + for (; dyn < lastdyn; dyn++) { + printf("tag=%d value=0x%x\n", dyn->d_tag, dyn->d_un.d_val); + } + } else +# endif + if ((shdr->sh_type == SHT_SYMTAB) || (shdr->sh_type == SHT_DYNSYM)) { + /* Symbol table */ + Elf_Data* data = elf_getdata(scn, 0); + if (!data || !data->d_size) { + printf("No data..."); + continue; + } + + /* In theory we now have the symbols... */ + Elf32_Sym* esym = (Elf32_Sym*)data->d_buf; + Elf32_Sym* lastsym = (Elf32_Sym*)((char*)data->d_buf + data->d_size); + for (; esym < lastsym; esym++) { +# if 0 + char *nm = elf_strptr(elf, strtabndx, (size_t)esym->st_name); + printf("%20s 0x%08x %02x %02x\n", + nm, esym->st_value, ELF32_ST_BIND(esym->st_info), + ELF32_ST_TYPE(esym->st_info)); +# endif + if ((esym->st_value == 0) || + (ELF32_ST_BIND(esym->st_info) == STB_WEAK) || + (ELF32_ST_BIND(esym->st_info) == STB_NUM) || + (ELF32_ST_TYPE(esym->st_info) != STT_FUNC)) { + continue; + } +# if 1 + char* nm = elf_strptr(elf, strtabndx, (size_t)esym->st_name); +# endif + sp->name = nm ? strdup(nm) : "(no name)"; + sp->address = esym->st_value; + sp++; + if (sp >= last) { + long n = alloced + 10000; + syms = (Symbol*)realloc(syms, (size_t)(sizeof(Symbol) * n)); + last = syms + n; + sp = syms + alloced; + alloced = n; + } + } + } + } + + int interesting = sp - syms; + if (!quiet) { + printf("Total of %d symbols\n", interesting); + } + usefulSymbols = interesting; + externalSymbols = syms; +} + +#endif /* USE_ELF */ diff --git a/tools/jprof/intcnt.cpp b/tools/jprof/intcnt.cpp new file mode 100644 index 0000000000..a87b6ccf74 --- /dev/null +++ b/tools/jprof/intcnt.cpp @@ -0,0 +1,69 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "intcnt.h" + +IntCount::IntCount() : numInts(0), iPair(nullptr) {} +IntCount::~IntCount() { delete[] iPair; } +int IntCount::getSize() { return numInts; } +int IntCount::getCount(int pos) { return iPair[pos].cnt; } +int IntCount::getIndex(int pos) { return iPair[pos].idx; } + +void IntCount::clear() { + delete[] iPair; + iPair = new IntPair[0]; + numInts = 0; +} + +int IntCount::countAdd(int index, int increment) { + if (numInts) { + // Do a binary search to find the element + int divPoint = 0; + + if (index > iPair[numInts - 1].idx) { + divPoint = numInts; + } else if (index < iPair[0].idx) { + divPoint = 0; + } else { + int low = 0, high = numInts - 1; + int mid = (low + high) / 2; + while (1) { + mid = (low + high) / 2; + + if (index < iPair[mid].idx) { + high = mid; + } else if (index > iPair[mid].idx) { + if (mid < numInts - 1 && index < iPair[mid + 1].idx) { + divPoint = mid + 1; + break; + } else { + low = mid + 1; + } + } else if (index == iPair[mid].idx) { + return iPair[mid].cnt += increment; + } + } + } + + int i; + IntPair* tpair = new IntPair[numInts + 1]; + for (i = 0; i < divPoint; i++) { + tpair[i] = iPair[i]; + } + for (i = divPoint; i < numInts; i++) { + tpair[i + 1] = iPair[i]; + } + ++numInts; + delete[] iPair; + iPair = tpair; + iPair[divPoint].idx = index; + iPair[divPoint].cnt = increment; + return increment; + } else { + iPair = new IntPair[1]; + numInts = 1; + iPair[0].idx = index; + return iPair[0].cnt = increment; + } +} diff --git a/tools/jprof/intcnt.h b/tools/jprof/intcnt.h new file mode 100644 index 0000000000..2cf9ec1ff1 --- /dev/null +++ b/tools/jprof/intcnt.h @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef INTCNT_H +#define INTCNT_H + +class IntCount { + public: + IntCount(); + ~IntCount(); + void clear(); + int countAdd(int index, int increment = 1); + int countGet(int index); + int getSize(); + int getCount(int pos); + int getIndex(int pos); + + IntCount(const IntCount& old) { + numInts = old.numInts; + if (numInts > 0) { + iPair = new IntPair[numInts]; + for (int i = 0; i < numInts; i++) { + iPair[i] = old.iPair[i]; + } + } else { + iPair = nullptr; + } + } + + private: + int numInts; + struct IntPair { + int idx; + int cnt; + }* iPair; +}; + +#endif diff --git a/tools/jprof/jprofsig b/tools/jprof/jprofsig new file mode 100755 index 0000000000..02226fc4b6 --- /dev/null +++ b/tools/jprof/jprofsig @@ -0,0 +1,46 @@ +#!/bin/sh +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Find Mozilla PID and send it a signal, to be used +# with the jprof tool. +# + +jpsignal_usage() { + echo "Usage: jprofsig [start|stop]" + exit 1 +} + +if [ $# != 1 ]; then + echo "Wrong number of arguments." + jpsignal_usage +fi + +jpsignal_arg="$1" + +# Find & print mozilla PID +tmpmoz=`ps aux | grep mozilla-bin | head -1 | awk '{ print $2 }'` +echo "Mozilla PID = $tmpmoz" + +# See how we were called. +case "$jpsignal_arg" in + start) + if [ "$JP_REALTIME" = 1 ]; then + kill -ALRM $tmpmoz + else + # Normal, non-realtime mode. + kill -PROF $tmpmoz + fi + ;; + stop) + kill -USR1 $tmpmoz + ;; + *) + jpsignal_usage + exit 1 +esac + +exit 0 diff --git a/tools/jprof/leaky.cpp b/tools/jprof/leaky.cpp new file mode 100644 index 0000000000..91e9d8aa82 --- /dev/null +++ b/tools/jprof/leaky.cpp @@ -0,0 +1,827 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "leaky.h" +#include "intcnt.h" + +#include <sys/types.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <string.h> +#ifndef NTO +# include <getopt.h> +#endif +#include <assert.h> +#include <stdlib.h> +#include <stdio.h> + +#ifdef NTO +# include <mem.h> +#endif + +#ifndef FALSE +# define FALSE 0 +#endif +#ifndef TRUE +# define TRUE 1 +#endif + +static const u_int DefaultBuckets = 10007; // arbitrary, but prime +static const u_int MaxBuckets = 1000003; // arbitrary, but prime + +//---------------------------------------------------------------------- + +int main(int argc, char** argv) { + leaky* l = new leaky; + + l->initialize(argc, argv); + l->outputfd = stdout; + + for (int i = 0; i < l->numLogFiles; i++) { + if (l->output_dir || l->numLogFiles > 1) { + char name[2048]; // XXX fix + if (l->output_dir) + snprintf(name, sizeof(name), "%s/%s.html", l->output_dir, + argv[l->logFileIndex + i]); + else + snprintf(name, sizeof(name), "%s.html", argv[l->logFileIndex + i]); + + fprintf(stderr, "opening %s\n", name); + l->outputfd = fopen(name, "w"); + // if an error we won't process the file + } + if (l->outputfd) { // paranoia + l->open(argv[l->logFileIndex + i]); + + if (l->outputfd != stderr) { + fclose(l->outputfd); + l->outputfd = nullptr; + } + } + } + + return 0; +} + +char* htmlify(const char* in) { + const char* p = in; + char *out, *q; + int n = 0; + size_t newlen; + + // Count the number of '<' and '>' in the input. + while ((p = strpbrk(p, "<>"))) { + ++n; + ++p; + } + + // Knowing the number of '<' and '>', we can calculate the space + // needed for the output string. + newlen = strlen(in) + n * 3 + 1; + out = new char[newlen]; + + // Copy the input to the output, with substitutions. + p = in; + q = out; + do { + if (*p == '<') { + strcpy(q, "<"); + q += 4; + } else if (*p == '>') { + strcpy(q, ">"); + q += 4; + } else { + *q++ = *p; + } + p++; + } while (*p); + *q = '\0'; + + return out; +} + +leaky::leaky() { + applicationName = nullptr; + progFile = nullptr; + + quiet = true; + showAddress = false; + showThreads = false; + stackDepth = 100000; + onlyThread = 0; + cleo = false; + + mappedLogFile = -1; + firstLogEntry = lastLogEntry = 0; + + sfd = -1; + externalSymbols = 0; + usefulSymbols = 0; + numExternalSymbols = 0; + lowestSymbolAddr = 0; + highestSymbolAddr = 0; + + loadMap = nullptr; + + collect_last = false; + collect_start = -1; + collect_end = -1; +} + +leaky::~leaky() {} + +void leaky::usageError() { + fprintf(stderr, + "Usage: %s [-v] [-t] [-e exclude] [-i include] [-s stackdepth] " + "[--last] [--all] [--start n [--end m]] [--cleo] [--output-dir dir] " + "prog log [log2 ...]\n", + (char*)applicationName); + fprintf( + stderr, + "\t-v: verbose\n" + "\t-t | --threads: split threads\n" + "\t--only-thread n: only profile thread N\n" + "\t-i include-id: stack must include specified id\n" + "\t-e exclude-id: stack must NOT include specified id\n" + "\t-s stackdepth: Limit depth looked at from captured stack frames\n" + "\t--last: only profile the last capture section\n" + "\t--start n [--end m]: profile n to m (or end) capture sections\n" + "\t--cleo: format output for 'cleopatra' display\n" + "\t--output-dir dir: write output files to dir\n" + "\tIf there's one log, output goes to stdout unless --output-dir is set\n" + "\tIf there are more than one log, output files will be named with .html " + "added\n"); + exit(-1); +} + +static struct option longopts[] = { + {"threads", 0, nullptr, 't'}, {"only-thread", 1, nullptr, 'T'}, + {"last", 0, nullptr, 'l'}, {"start", 1, nullptr, 'x'}, + {"end", 1, nullptr, 'n'}, {"cleo", 0, nullptr, 'c'}, + {"output-dir", 1, nullptr, 'd'}, {nullptr, 0, nullptr, 0}, +}; + +void leaky::initialize(int argc, char** argv) { + applicationName = argv[0]; + applicationName = strrchr(applicationName, '/'); + if (!applicationName) { + applicationName = argv[0]; + } else { + applicationName++; + } + + int arg; + int errflg = 0; + int longindex = 0; + + onlyThread = 0; + output_dir = nullptr; + cleo = false; + + // XXX tons of cruft here left over from tracemalloc + // XXX The -- options shouldn't need short versions, or they should be + // documented + while (((arg = getopt_long(argc, argv, "adEe:gh:i:r:Rs:tT:qvx:ln:", longopts, + &longindex)) != -1)) { + switch (arg) { + case '?': + default: + fprintf(stderr, "error: unknown option %c\n", optopt); + errflg++; + break; + case 'a': + break; + case 'A': // not implemented + showAddress = true; + break; + case 'c': + cleo = true; + break; + case 'd': + output_dir = optarg; // reference to an argv pointer + break; + case 'R': + break; + case 'e': + exclusions.add(optarg); + break; + case 'g': + break; + case 'r': // not implemented + roots.add(optarg); + if (!includes.IsEmpty()) { + errflg++; + } + break; + case 'i': + includes.add(optarg); + if (!roots.IsEmpty()) { + errflg++; + } + break; + case 'h': + break; + case 's': + stackDepth = atoi(optarg); + if (stackDepth < 2) { + stackDepth = 2; + } + break; + case 'x': + // --start + collect_start = atoi(optarg); + break; + case 'n': + // --end + collect_end = atoi(optarg); + break; + case 'l': + // --last + collect_last = true; + break; + case 'q': + break; + case 'v': + quiet = !quiet; + break; + case 't': + showThreads = true; + break; + case 'T': + showThreads = true; + onlyThread = atoi(optarg); + break; + } + } + if (errflg || ((argc - optind) < 2)) { + usageError(); + } + progFile = argv[optind++]; + logFileIndex = optind; + numLogFiles = argc - optind; + if (!quiet) fprintf(stderr, "numlogfiles = %d\n", numLogFiles); +} + +static void* mapFile(int fd, u_int flags, off_t* sz) { + struct stat sb; + if (fstat(fd, &sb) < 0) { + perror("fstat"); + exit(-1); + } + void* base = mmap(0, (int)sb.st_size, flags, MAP_PRIVATE, fd, 0); + if (!base) { + perror("mmap"); + exit(-1); + } + *sz = sb.st_size; + return base; +} + +void leaky::LoadMap() { + malloc_map_entry mme; + char name[1000]; + + if (!loadMap) { + // all files use the same map + int fd = ::open(M_MAPFILE, O_RDONLY); + if (fd < 0) { + perror("open: " M_MAPFILE); + exit(-1); + } + for (;;) { + int nb = read(fd, &mme, sizeof(mme)); + if (nb != sizeof(mme)) break; + nb = read(fd, name, mme.nameLen); + if (nb != (int)mme.nameLen) break; + name[mme.nameLen] = 0; + if (!quiet) { + fprintf(stderr, "%s @ %lx\n", name, mme.address); + } + + LoadMapEntry* lme = new LoadMapEntry; + lme->address = mme.address; + lme->name = strdup(name); + lme->next = loadMap; + loadMap = lme; + } + close(fd); + } +} + +void leaky::open(char* logFile) { + int threadArray[100]; // should auto-expand + int last_thread = -1; + int numThreads = 0; + int section = -1; + bool collecting = false; + + LoadMap(); + + setupSymbols(progFile); + + // open up the log file + if (mappedLogFile) ::close(mappedLogFile); + + mappedLogFile = ::open(logFile, O_RDONLY); + if (mappedLogFile < 0) { + perror("open"); + exit(-1); + } + off_t size; + firstLogEntry = (malloc_log_entry*)mapFile(mappedLogFile, PROT_READ, &size); + lastLogEntry = (malloc_log_entry*)((char*)firstLogEntry + size); + + if (!collect_last || collect_start < 0) { + collecting = true; + } + + // First, restrict it to the capture sections specified (all, last, start/end) + // This loop walks through all the call stacks we recorded + for (malloc_log_entry* lep = firstLogEntry; lep < lastLogEntry; + lep = reinterpret_cast<malloc_log_entry*>(&lep->pcs[lep->numpcs])) { + if (lep->flags & JP_FIRST_AFTER_PAUSE) { + section++; + if (collect_last) { + firstLogEntry = lep; + numThreads = 0; + collecting = true; + } + if (collect_start == section) { + collecting = true; + firstLogEntry = lep; + } + if (collect_end == section) { + collecting = false; + lastLogEntry = lep; + } + if (!quiet) + fprintf(stderr, "New section %d: first=%p, last=%p, collecting=%d\n", + section, (void*)firstLogEntry, (void*)lastLogEntry, collecting); + } + + // Capture thread info at the same time + + // Find all the threads captured + + // pthread/linux docs say the signal can be delivered to any thread in + // the process. In practice, it appears in Linux that it's always + // delivered to the thread that called setitimer(), and each thread can + // have a separate itimer. There's a support library for gprof that + // overlays pthread_create() to set timers in any threads you spawn. + if (showThreads && collecting) { + if (lep->thread != last_thread) { + int i; + for (i = 0; i < numThreads; i++) { + if (lep->thread == threadArray[i]) break; + } + if (i == numThreads && + i < (int)(sizeof(threadArray) / sizeof(threadArray[0]))) { + threadArray[i] = lep->thread; + numThreads++; + if (!quiet) fprintf(stderr, "new thread %d\n", lep->thread); + } + } + } + } + if (!quiet) + fprintf(stderr, + "Done collecting: sections %d: first=%p, last=%p, numThreads=%d\n", + section, (void*)firstLogEntry, (void*)lastLogEntry, numThreads); + + if (!cleo) { + fprintf(outputfd, + "<html><head><title>Jprof Profile Report</title></head><body>\n"); + fprintf(outputfd, "<h1><center>Jprof Profile Report</center></h1>\n"); + } + + if (showThreads) { + fprintf(stderr, "Num threads %d\n", numThreads); + + if (!cleo) { + fprintf(outputfd, "<hr>Threads:<p><pre>\n"); + for (int i = 0; i < numThreads; i++) { + fprintf(outputfd, " <a href=\"#thread_%d\">%d</a> ", threadArray[i], + threadArray[i]); + if ((i + 1) % 10 == 0) fprintf(outputfd, "<br>\n"); + } + fprintf(outputfd, "</pre>"); + } + + for (int i = 0; i < numThreads; i++) { + if (!onlyThread || onlyThread == threadArray[i]) analyze(threadArray[i]); + } + } else { + analyze(0); + } + + if (!cleo) fprintf(outputfd, "</pre></body></html>\n"); +} + +//---------------------------------------------------------------------- + +static int symbolOrder(void const* a, void const* b) { + Symbol const** ap = (Symbol const**)a; + Symbol const** bp = (Symbol const**)b; + return (*ap)->address == (*bp)->address + ? 0 + : ((*ap)->address > (*bp)->address ? 1 : -1); +} + +void leaky::ReadSharedLibrarySymbols() { + LoadMapEntry* lme = loadMap; + while (nullptr != lme) { + ReadSymbols(lme->name, lme->address); + lme = lme->next; + } +} + +void leaky::setupSymbols(const char* fileName) { + if (usefulSymbols == 0) { + // only read once! + + // Read in symbols from the program + ReadSymbols(fileName, 0); + + // Read in symbols from the .so's + ReadSharedLibrarySymbols(); + + if (!quiet) { + fprintf(stderr, "A total of %d symbols were loaded\n", usefulSymbols); + } + + // Now sort them + qsort(externalSymbols, usefulSymbols, sizeof(Symbol*), symbolOrder); + lowestSymbolAddr = externalSymbols[0]->address; + highestSymbolAddr = externalSymbols[usefulSymbols - 1]->address; + } +} + +// Binary search the table, looking for a symbol that covers this +// address. +int leaky::findSymbolIndex(u_long addr) { + u_int base = 0; + u_int limit = usefulSymbols - 1; + Symbol** end = &externalSymbols[limit]; + while (base <= limit) { + u_int midPoint = (base + limit) >> 1; + Symbol** sp = &externalSymbols[midPoint]; + if (addr < (*sp)->address) { + if (midPoint == 0) { + return -1; + } + limit = midPoint - 1; + } else { + if (sp + 1 < end) { + if (addr < (*(sp + 1))->address) { + return midPoint; + } + } else { + return midPoint; + } + base = midPoint + 1; + } + } + return -1; +} + +Symbol* leaky::findSymbol(u_long addr) { + int idx = findSymbolIndex(addr); + + if (idx < 0) { + return nullptr; + } else { + return externalSymbols[idx]; + } +} + +//---------------------------------------------------------------------- + +bool leaky::excluded(malloc_log_entry* lep) { + if (exclusions.IsEmpty()) { + return false; + } + + char** pcp = &lep->pcs[0]; + u_int n = lep->numpcs; + for (u_int i = 0; i < n; i++, pcp++) { + Symbol* sp = findSymbol((u_long)*pcp); + if (sp && exclusions.contains(sp->name)) { + return true; + } + } + return false; +} + +bool leaky::included(malloc_log_entry* lep) { + if (includes.IsEmpty()) { + return true; + } + + char** pcp = &lep->pcs[0]; + u_int n = lep->numpcs; + for (u_int i = 0; i < n; i++, pcp++) { + Symbol* sp = findSymbol((u_long)*pcp); + if (sp && includes.contains(sp->name)) { + return true; + } + } + return false; +} + +//---------------------------------------------------------------------- + +void leaky::displayStackTrace(FILE* out, malloc_log_entry* lep) { + char** pcp = &lep->pcs[0]; + u_int n = (lep->numpcs < stackDepth) ? lep->numpcs : stackDepth; + for (u_int i = 0; i < n; i++, pcp++) { + u_long addr = (u_long)*pcp; + Symbol* sp = findSymbol(addr); + if (sp) { + fputs(sp->name, out); + if (showAddress) { + fprintf(out, "[%p]", (char*)addr); + } + } else { + fprintf(out, "<%p>", (char*)addr); + } + fputc(' ', out); + } + fputc('\n', out); +} + +void leaky::dumpEntryToLog(malloc_log_entry* lep) { + printf("%ld\t", lep->delTime); + printf(" --> "); + displayStackTrace(outputfd, lep); +} + +void leaky::generateReportHTML(FILE* fp, int* countArray, int count, + int thread) { + fprintf(fp, "<center>"); + if (showThreads) { + fprintf(fp, "<hr><A NAME=thread_%d><b>Thread: %d</b></A><p>", thread, + thread); + } + fprintf( + fp, + "<A href=#flat_%d>flat</A><b> | </b><A href=#hier_%d>hierarchical</A>", + thread, thread); + fprintf(fp, "</center><P><P><P>\n"); + + int totalTimerHits = count; + int* rankingTable = new int[usefulSymbols]; + + for (int cnt = usefulSymbols; --cnt >= 0; rankingTable[cnt] = cnt) + ; + + // Drat. I would use ::qsort() but I would need a global variable and my + // intro-pascal professor threatened to flunk anyone who used globals. + // She damaged me for life :-) (That was 1986. See how much influence + // she had. I don't remember her name but I always feel guilty about globals) + + // Shell Sort. 581130733 is the max 31 bit value of h = 3h+1 + int mx, i, h; + for (mx = usefulSymbols / 9, h = 581130733; h > 0; h /= 3) { + if (h < mx) { + for (i = h - 1; i < usefulSymbols; i++) { + int j, tmp = rankingTable[i], val = countArray[tmp]; + for (j = i; (j >= h) && (countArray[rankingTable[j - h]] < val); + j -= h) { + rankingTable[j] = rankingTable[j - h]; + } + rankingTable[j] = tmp; + } + } + } + + // Ok, We are sorted now. Let's go through the table until we get to + // functions that were never called. Right now we don't do much inside + // this loop. Later we can get callers and callees into it like gprof + // does + fprintf(fp, + "<h2><A NAME=hier_%d></A><center><a " + "href=\"http://searchfox.org/mozilla-central/source/tools/jprof/" + "README.html#hier\">Hierarchical Profile</a></center></h2><hr>\n", + thread); + fprintf(fp, "<pre>\n"); + fprintf(fp, "%6s %6s %4s %s\n", "index", "Count", "Hits", + "Function Name"); + + for (i = 0; i < usefulSymbols && countArray[rankingTable[i]] > 0; i++) { + Symbol** sp = &externalSymbols[rankingTable[i]]; + + (*sp)->cntP.printReport(fp, this, rankingTable[i], totalTimerHits); + + char* symname = htmlify((*sp)->name); + fprintf(fp, + "%6d %6d (%3.1f%%)%s <a name=%d>%8d (%3.1f%%)</a>%s <b>%s</b>\n", + rankingTable[i], (*sp)->timerHit, + ((*sp)->timerHit * 1000 / totalTimerHits) / 10.0, + ((*sp)->timerHit * 1000 / totalTimerHits) / 10.0 >= 10.0 ? "" : " ", + rankingTable[i], countArray[rankingTable[i]], + (countArray[rankingTable[i]] * 1000 / totalTimerHits) / 10.0, + (countArray[rankingTable[i]] * 1000 / totalTimerHits) / 10.0 >= 10.0 + ? "" + : " ", + symname); + delete[] symname; + + (*sp)->cntC.printReport(fp, this, rankingTable[i], totalTimerHits); + + fprintf(fp, "<hr>\n"); + } + fprintf(fp, "</pre>\n"); + + // OK, Now we want to print the flat profile. To do this we resort on + // the hit count. + + // Cut-N-Paste Shell sort from above. The Ranking Table has already been + // populated, so we do not have to reinitialize it. + for (mx = usefulSymbols / 9, h = 581130733; h > 0; h /= 3) { + if (h < mx) { + for (i = h - 1; i < usefulSymbols; i++) { + int j, tmp = rankingTable[i], val = externalSymbols[tmp]->timerHit; + for (j = i; + (j >= h) && (externalSymbols[rankingTable[j - h]]->timerHit < val); + j -= h) { + rankingTable[j] = rankingTable[j - h]; + } + rankingTable[j] = tmp; + } + } + } + + // Pre-count up total counter hits, to get a percentage. + // I wanted the total before walking the list, if this + // double-pass over externalSymbols gets slow we can + // do single-pass and print this out after the loop finishes. + totalTimerHits = 0; + for (i = 0; + i < usefulSymbols && externalSymbols[rankingTable[i]]->timerHit > 0; + i++) { + Symbol** sp = &externalSymbols[rankingTable[i]]; + totalTimerHits += (*sp)->timerHit; + } + if (totalTimerHits == 0) totalTimerHits = 1; + + if (totalTimerHits != count) + fprintf(stderr, "Hit count mismatch: count=%d; totalTimerHits=%d", count, + totalTimerHits); + + fprintf(fp, + "<h2><A NAME=flat_%d></A><center><a " + "href=\"http://searchfox.org/mozilla-central/source/tools/jprof/" + "README.html#flat\">Flat Profile</a></center></h2><br>\n", + thread); + fprintf(fp, "<pre>\n"); + + fprintf(fp, "Total hit count: %d\n", totalTimerHits); + fprintf(fp, "Count %%Total Function Name\n"); + // Now loop for as long as we have timer hits + for (i = 0; + i < usefulSymbols && externalSymbols[rankingTable[i]]->timerHit > 0; + i++) { + Symbol** sp = &externalSymbols[rankingTable[i]]; + + char* symname = htmlify((*sp)->name); + fprintf(fp, "<a href=\"#%d\">%3d %-2.1f %s</a>\n", rankingTable[i], + (*sp)->timerHit, + ((float)(*sp)->timerHit / (float)totalTimerHits) * 100.0, symname); + delete[] symname; + } +} + +void leaky::analyze(int thread) { + int* countArray = new int[usefulSymbols]; + int* flagArray = new int[usefulSymbols]; + + // Zero our function call counter + memset(countArray, 0, sizeof(countArray[0]) * usefulSymbols); + + // reset hit counts + for (int i = 0; i < usefulSymbols; i++) { + externalSymbols[i]->timerHit = 0; + externalSymbols[i]->regClear(); + } + + // The flag array is used to prevent counting symbols multiple times + // if functions are called recursively. In order to keep from having + // to zero it on each pass through the loop, we mark it with the value + // of stacks on each trip through the loop. This means we can determine + // if we have seen this symbol for this stack trace w/o having to reset + // from the prior stacktrace. + memset(flagArray, -1, sizeof(flagArray[0]) * usefulSymbols); + + if (cleo) fprintf(outputfd, "m-Start\n"); + + // This loop walks through all the call stacks we recorded + // --last, --start and --end can restrict it, as can excludes/includes + stacks = 0; + for (malloc_log_entry* lep = firstLogEntry; lep < lastLogEntry; + lep = reinterpret_cast<malloc_log_entry*>(&lep->pcs[lep->numpcs])) { + if ((thread != 0 && lep->thread != thread) || excluded(lep) || + !included(lep)) { + continue; + } + + ++stacks; // How many stack frames did we collect + + u_int n = (lep->numpcs < stackDepth) ? lep->numpcs : stackDepth; + char** pcp = &lep->pcs[n - 1]; + int idx = -1, parrentIdx = -1; // Init idx incase n==0 + if (cleo) { + // This loop walks through every symbol in the call stack. By walking it + // backwards we know who called the function when we get there. + char type = 's'; + for (int i = n - 1; i >= 0; --i, --pcp) { + idx = findSymbolIndex(reinterpret_cast<u_long>(*pcp)); + + if (idx >= 0) { + // Skip over bogus __restore_rt frames that realtime profiling + // can introduce. + if (i > 0 && !strcmp(externalSymbols[idx]->name, "__restore_rt")) { + --pcp; + --i; + idx = findSymbolIndex(reinterpret_cast<u_long>(*pcp)); + if (idx < 0) { + continue; + } + } + Symbol** sp = &externalSymbols[idx]; + char* symname = htmlify((*sp)->name); + fprintf(outputfd, "%c-%s\n", type, symname); + delete[] symname; + } + // else can't find symbol - ignore + type = 'c'; + } + } else { + // This loop walks through every symbol in the call stack. By walking it + // backwards we know who called the function when we get there. + for (int i = n - 1; i >= 0; --i, --pcp) { + idx = findSymbolIndex(reinterpret_cast<u_long>(*pcp)); + + if (idx >= 0) { + // Skip over bogus __restore_rt frames that realtime profiling + // can introduce. + if (i > 0 && !strcmp(externalSymbols[idx]->name, "__restore_rt")) { + --pcp; + --i; + idx = findSymbolIndex(reinterpret_cast<u_long>(*pcp)); + if (idx < 0) { + continue; + } + } + + // If we have not seen this symbol before count it and mark it as seen + if (flagArray[idx] != stacks && ((flagArray[idx] = stacks) || true)) { + ++countArray[idx]; + } + + // We know who we are and we know who our parrent is. Count this + if (parrentIdx >= 0) { + externalSymbols[parrentIdx]->regChild(idx); + externalSymbols[idx]->regParrent(parrentIdx); + } + // inside if() so an unknown in the middle of a stack won't break + // the link! + parrentIdx = idx; + } + } + + // idx should be the function that we were in when we received the signal. + if (idx >= 0) { + ++externalSymbols[idx]->timerHit; + } + } + } + if (!cleo) generateReportHTML(outputfd, countArray, stacks, thread); +} + +void FunctionCount::printReport(FILE* fp, leaky* lk, int parent, int total) { + const char* fmt = + " <A href=\"#%d\">%8d (%3.1f%%)%s %s</A>%s\n"; + + int nmax, tmax = ((~0U) >> 1); + + do { + nmax = 0; + for (int j = getSize(); --j >= 0;) { + int cnt = getCount(j); + if (cnt == tmax) { + int idx = getIndex(j); + char* symname = htmlify(lk->indexToName(idx)); + fprintf(fp, fmt, idx, getCount(j), getCount(j) * 100.0 / total, + getCount(j) * 100.0 / total >= 10.0 ? "" : " ", symname, + parent == idx ? " (self)" : ""); + delete[] symname; + } else if (cnt < tmax && cnt > nmax) { + nmax = cnt; + } + } + } while ((tmax = nmax) > 0); +} diff --git a/tools/jprof/leaky.h b/tools/jprof/leaky.h new file mode 100644 index 0000000000..6c9beb7b42 --- /dev/null +++ b/tools/jprof/leaky.h @@ -0,0 +1,124 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef __leaky_h_ +#define __leaky_h_ + +#include "config.h" +#include <stdio.h> +#include <string.h> +#include <sys/types.h> +#include "libmalloc.h" +#include "strset.h" +#include "intcnt.h" + +typedef unsigned int u_int; + +struct Symbol; +struct leaky; + +class FunctionCount : public IntCount { + public: + void printReport(FILE* fp, leaky* lk, int parent, int total); +}; + +struct Symbol { + char* name; + u_long address; + int timerHit; + FunctionCount cntP, cntC; + + int regChild(int id) { return cntC.countAdd(id, 1); } + int regParrent(int id) { return cntP.countAdd(id, 1); } + void regClear() { + cntC.clear(); + cntP.clear(); + } + + Symbol() : timerHit(0) {} + void Init(const char* aName, u_long aAddress) { + name = aName ? strdup(aName) : (char*)""; + address = aAddress; + } +}; + +struct LoadMapEntry { + char* name; // name of .so + u_long address; // base address where it was mapped in + LoadMapEntry* next; +}; + +struct leaky { + leaky(); + ~leaky(); + + void initialize(int argc, char** argv); + void open(char* arg); + + char* applicationName; + int logFileIndex; + int numLogFiles; + char* progFile; + FILE* outputfd; + + bool quiet; + bool showAddress; + bool showThreads; + bool cleo; + u_int stackDepth; + int onlyThread; + char* output_dir; + + int mappedLogFile; + malloc_log_entry* firstLogEntry; + malloc_log_entry* lastLogEntry; + + int stacks; + + int sfd; + Symbol** externalSymbols; + Symbol** lastSymbol; + int usefulSymbols; + int numExternalSymbols; + StrSet exclusions; + u_long lowestSymbolAddr; + u_long highestSymbolAddr; + + LoadMapEntry* loadMap; + + bool collect_last; + int collect_start; + int collect_end; + + StrSet roots; + StrSet includes; + + void usageError(); + + void LoadMap(); + + void analyze(int thread); + + void dumpEntryToLog(malloc_log_entry* lep); + + void insertAddress(u_long address, malloc_log_entry* lep); + void removeAddress(u_long address, malloc_log_entry* lep); + + void displayStackTrace(FILE* out, malloc_log_entry* lep); + + Symbol** ExtendSymbols(int num); + void ReadSymbols(const char* fileName, u_long aBaseAddress); + void ReadSharedLibrarySymbols(); + void setupSymbols(const char* fileName); + Symbol* findSymbol(u_long address); + bool excluded(malloc_log_entry* lep); + bool included(malloc_log_entry* lep); + const char* indexToName(int idx) { return externalSymbols[idx]->name; } + + private: + void generateReportHTML(FILE* fp, int* countArray, int count, int thread); + int findSymbolIndex(u_long address); +}; + +#endif /* __leaky_h_ */ diff --git a/tools/jprof/moz.build b/tools/jprof/moz.build new file mode 100644 index 0000000000..2aa4a8c294 --- /dev/null +++ b/tools/jprof/moz.build @@ -0,0 +1,28 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += ["stub"] + +Program("jprof") + +SOURCES += [ + "bfd.cpp", + "coff.cpp", + "elf.cpp", + "intcnt.cpp", + "leaky.cpp", + "strset.cpp", +] + +LOCAL_INCLUDES += [ + "stub", +] + +OS_LIBS += [ + "dl", + "bfd", + "iberty", +] diff --git a/tools/jprof/split-profile.py b/tools/jprof/split-profile.py new file mode 100755 index 0000000000..2e5fa89caf --- /dev/null +++ b/tools/jprof/split-profile.py @@ -0,0 +1,156 @@ +#!/usr/bin/python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This program splits up a jprof profile into multiple files based on a +# list of functions in a text file. First, a complete profile is +# generated. Then, for each line in the text file, a profile is +# generated containing only stacks that go through that line, and also +# excluding all stacks in earlier lines in the text file. This means +# that the text file, from start to end, is splitting out pieces of the +# profile in their own file. Finally, a final profile containing the +# remainder is produced. + +# The program takes four arguments: +# (1) The path to jprof. +# (2) The path to the text file describing the splits. The output +# will be placed in the same directory as this file. +# (3) The program that was profiled. +# (4) The jprof-log file generated by the profile, to be split up. +# (Really, all arguments from (3) and later are passed through to +# jprof, so additional arguments could be provided if you want to pass +# additional arguments to jprof.) + +# In slightly more detail: +# +# This script uses jprof's includes (-i) and excludes (-e) options to +# split profiles into segments. It takes as input a single text file, +# and from that text file creates a series of jprof profiles in the +# directory the text file is in. +# +# The input file format looks like the following: +# +# poll g_main_poll +# GetRuleCascade CSSRuleProcessor::GetRuleCascade(nsPresContext *, nsAtom *) +# RuleProcessorData RuleProcessorData::RuleProcessorData +# (nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *) +# +# +# From this input file, the script will construct a profile called +# jprof-0.html that contains the whole profile, a profile called +# jprof-1-poll.html that includes only stacks with g_main_poll, a +# profile called jprof-2-GetRuleCascade.html that includes only stacks +# that have GetRuleCascade and do not have g_main_poll, a profile called +# jprof-3-RuleProcessorData.html that includes only stacks that have the +# RuleProcessorData constructor and do not have GetRuleCascade or +# g_main_poll, and a profile called jprof-4.html that includes only +# stacks that do not have any of the three functions in them. +# +# This means that all of the segments of the profile, except +# jprof-0.html, are mutually exclusive. Thus clever ordering of the +# functions in the input file can lead to a logical splitting of the +# profile into segments. + +import os.path +import subprocess +import sys + +if len(sys.argv) < 5: + sys.stderr.write("Expected arguments: <jprof> <split-file> <program> <jprof-log>\n") + sys.exit(1) + +jprof = sys.argv[1] +splitfile = sys.argv[2] +passthrough = sys.argv[3:] + +for f in [jprof, splitfile]: + if not os.path.isfile(f): + sys.stderr.write("could not find file: {0}\n".format(f)) + sys.exit(1) + + +def read_splits(splitfile): + """ + Read splitfile (each line of which contains a name, a space, and + then a function name to split on), and return a list of pairs + representing exactly that. (Note that the name cannot contain + spaces, but the function name can, and often does.) + """ + + def line_to_split(line): + line = line.strip("\r\n") + idx = line.index(" ") + return (line[0:idx], line[idx + 1 :]) + + io = open(splitfile, "r") + result = [line_to_split(line) for line in io] + io.close() + return result + + +splits = read_splits(splitfile) + + +def generate_profile(options, destfile): + """ + Run jprof to generate one split of the profile. + """ + args = [jprof] + options + passthrough + print("Generating {}".format(destfile)) + destio = open(destfile, "w") + # jprof expects the "jprof-map" file to be in its current working directory + cwd = None + for option in passthrough: + if option.find("jprof-log"): + cwd = os.path.dirname(option) + if cwd is None: + raise Exception("no jprof-log option given") + process = subprocess.Popen(args, stdout=destio, cwd=cwd) + process.wait() + destio.close() + if process.returncode != 0: + os.remove(destfile) + sys.stderr.write( + "Error {0} from command:\n {1}\n".format( + process.returncode, " ".join(args) + ) + ) + sys.exit(process.returncode) + + +def output_filename(number, splitname): + """ + Return the filename (absolute path) we should use to output the + profile segment with the given number and splitname. Splitname + should be None for the complete profile and the remainder. + """ + + def pad_count(i): + result = str(i) + # 0-pad to the same length + result = "0" * (len(str(len(splits) + 1)) - len(result)) + result + return result + + name = pad_count(number) + if splitname is not None: + name += "-" + splitname + + return os.path.join(os.path.dirname(splitfile), "jprof-{0}.html".format(name)) + + +# generate the complete profile +generate_profile([], output_filename(0, None)) + +# generate the listed splits +count = 1 +excludes = [] +for splitname, splitfunction in splits: + generate_profile( + excludes + ["-i" + splitfunction], output_filename(count, splitname) + ) + excludes += ["-e" + splitfunction] + count = count + 1 + +# generate the remainder after the splits +generate_profile(excludes, output_filename(count, None)) diff --git a/tools/jprof/strset.cpp b/tools/jprof/strset.cpp new file mode 100644 index 0000000000..514b8c03e0 --- /dev/null +++ b/tools/jprof/strset.cpp @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "strset.h" +#include <malloc.h> +#include <string.h> + +StrSet::StrSet() { + strings = 0; + numstrings = 0; +} + +void StrSet::add(const char* s) { + if (strings) { + strings = (char**)realloc(strings, (numstrings + 1) * sizeof(char*)); + } else { + strings = (char**)malloc(sizeof(char*)); + } + strings[numstrings] = strdup(s); + numstrings++; +} + +int StrSet::contains(const char* s) { + char** sp = strings; + int i = numstrings; + + while (--i >= 0) { + char* ss = *sp++; + if (ss[0] == s[0]) { + if (strcmp(ss, s) == 0) { + return 1; + } + } + } + return 0; +} diff --git a/tools/jprof/strset.h b/tools/jprof/strset.h new file mode 100644 index 0000000000..681ed22a25 --- /dev/null +++ b/tools/jprof/strset.h @@ -0,0 +1,19 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef __strset_h_ +#define __strset_h_ + +struct StrSet { + StrSet(); + + void add(const char* string); + int contains(const char* string); + bool IsEmpty() const { return 0 == numstrings; } + + char** strings; + int numstrings; +}; + +#endif /* __strset_h_ */ diff --git a/tools/jprof/stub/Makefile.in b/tools/jprof/stub/Makefile.in new file mode 100644 index 0000000000..8e6b6b8f8d --- /dev/null +++ b/tools/jprof/stub/Makefile.in @@ -0,0 +1,8 @@ +#! gmake +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# override optimization +MOZ_OPTIMIZE_FLAGS = -fno-omit-frame-pointer diff --git a/tools/jprof/stub/config.h b/tools/jprof/stub/config.h new file mode 100644 index 0000000000..6e5789452a --- /dev/null +++ b/tools/jprof/stub/config.h @@ -0,0 +1,18 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef config_h___ +#define config_h___ + +#define MAX_STACK_CRAWL 500 +#define M_LOGFILE "jprof-log" +#define M_MAPFILE "jprof-map" + +#if defined(linux) || defined(NTO) +# define USE_BFD +# undef NEED_WRAPPERS + +#endif /* linux */ + +#endif /* config_h___ */ diff --git a/tools/jprof/stub/jprof.h b/tools/jprof/stub/jprof.h new file mode 100644 index 0000000000..c118760750 --- /dev/null +++ b/tools/jprof/stub/jprof.h @@ -0,0 +1,17 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jprof_h___ +#define jprof_h___ +#include "nscore.h" + +#ifdef _IMPL_JPPROF_API +# define JPROF_API(type) NS_EXPORT_(type) +#else +# define JPROF_API(type) NS_IMPORT_(type) +#endif + +JPROF_API(void) setupProfilingStuff(void); + +#endif /* jprof_h___ */ diff --git a/tools/jprof/stub/libmalloc.cpp b/tools/jprof/stub/libmalloc.cpp new file mode 100644 index 0000000000..3003543a93 --- /dev/null +++ b/tools/jprof/stub/libmalloc.cpp @@ -0,0 +1,740 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +// vim:cindent:sw=4:et:ts=8: +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// The linux glibc hides part of sigaction if _POSIX_SOURCE is defined +#if defined(linux) +# undef _POSIX_SOURCE +# undef _SVID_SOURCE +# ifndef _GNU_SOURCE +# define _GNU_SOURCE +# endif +#endif + +#include <errno.h> +#if defined(linux) +# include <linux/rtc.h> +# include <pthread.h> +#endif +#include <unistd.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <signal.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/ioctl.h> +#include <sys/stat.h> +#include <sys/syscall.h> +#include <ucontext.h> +#include <execinfo.h> + +#include "libmalloc.h" +#include "jprof.h" +#include <string.h> +#include <errno.h> +#include <dlfcn.h> + +#ifdef NTO +# include <sys/link.h> +extern r_debug _r_debug; +#else +# include <link.h> +#endif + +#define USE_GLIBC_BACKTRACE 1 +// To debug, use #define JPROF_STATIC +#define JPROF_STATIC static + +static int gLogFD = -1; +static pthread_t main_thread; + +static bool gIsChild = false; +static int gFilenamePID; + +static void startSignalCounter(unsigned long millisec); +static int enableRTCSignals(bool enable); + +//---------------------------------------------------------------------- +// replace use of atexit() + +static void DumpAddressMap(); + +struct JprofShutdown { + JprofShutdown() {} + ~JprofShutdown() { DumpAddressMap(); } +}; + +static void RegisterJprofShutdown() { + // This instanciates the dummy class above, and will trigger the class + // destructor when libxul is unloaded. This is equivalent to atexit(), + // but gracefully handles dlclose(). + static JprofShutdown t; +} + +#if defined(i386) || defined(_i386) || defined(__x86_64__) +JPROF_STATIC void CrawlStack(malloc_log_entry* me, void* stack_top, + void* top_instr_ptr) { +# if USE_GLIBC_BACKTRACE + // This probably works on more than x86! But we need a way to get the + // top instruction pointer, which is kindof arch-specific + void* array[500]; + int cnt, i; + u_long numpcs = 0; + + // This is from glibc. A more generic version might use + // libunwind and/or CaptureStackBackTrace() on Windows + cnt = backtrace(&array[0], sizeof(array) / sizeof(array[0])); + + // StackHook->JprofLog->CrawlStack + // Then we have sigaction, which replaced top_instr_ptr + array[3] = top_instr_ptr; + for (i = 3; i < cnt; i++) { + me->pcs[numpcs++] = (char*)array[i]; + } + me->numpcs = numpcs; + +# else + // original code - this breaks on many platforms + void** bp; +# if defined(__i386) + __asm__("movl %%ebp, %0" : "=g"(bp)); +# elif defined(__x86_64__) + __asm__("movq %%rbp, %0" : "=g"(bp)); +# else + // It would be nice if this worked uniformly, but at least on i386 and + // x86_64, it stopped working with gcc 4.1, because it points to the + // end of the saved registers instead of the start. + bp = __builtin_frame_address(0); +# endif + u_long numpcs = 0; + bool tracing = false; + + me->pcs[numpcs++] = (char*)top_instr_ptr; + + while (numpcs < MAX_STACK_CRAWL) { + void** nextbp = (void**)*bp++; + void* pc = *bp; + if (nextbp < bp) { + break; + } + if (tracing) { + // Skip the signal handling. + me->pcs[numpcs++] = (char*)pc; + } else if (pc == top_instr_ptr) { + tracing = true; + } + bp = nextbp; + } + me->numpcs = numpcs; +# endif +} +#endif + +//---------------------------------------------------------------------- + +static int rtcHz; +static int rtcFD = -1; +static bool circular = false; + +#if defined(linux) || defined(NTO) +static void DumpAddressMap() { + // Turn off the timer so we don't get interrupts during shutdown +# if defined(linux) + if (rtcHz) { + enableRTCSignals(false); + } else +# endif + { + startSignalCounter(0); + } + + char filename[2048]; + if (gIsChild) + snprintf(filename, sizeof(filename), "%s-%d", M_MAPFILE, gFilenamePID); + else + snprintf(filename, sizeof(filename), "%s", M_MAPFILE); + + int mfd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666); + if (mfd >= 0) { + malloc_map_entry mme; + link_map* map = _r_debug.r_map; + while (nullptr != map) { + if (map->l_name && *map->l_name) { + mme.nameLen = strlen(map->l_name); + mme.address = map->l_addr; + write(mfd, &mme, sizeof(mme)); + write(mfd, map->l_name, mme.nameLen); +# if 0 + write(1, map->l_name, mme.nameLen); + write(1, "\n", 1); +# endif + } + map = map->l_next; + } + close(mfd); + } +} +#endif + +static bool was_paused = true; + +JPROF_STATIC void JprofBufferDump(); +JPROF_STATIC void JprofBufferClear(); + +static void ClearProfilingHook(int signum) { + if (circular) { + JprofBufferClear(); + puts("Jprof: cleared circular buffer."); + } +} + +static void EndProfilingHook(int signum) { + if (circular) JprofBufferDump(); + + DumpAddressMap(); + was_paused = true; + puts("Jprof: profiling paused."); +} + +//---------------------------------------------------------------------- +// proper usage would be a template, including the function to find the +// size of an entry, or include a size header explicitly to each entry. +#if defined(linux) +# define DUMB_LOCK() pthread_mutex_lock(&mutex); +# define DUMB_UNLOCK() pthread_mutex_unlock(&mutex); +#else +# define DUMB_LOCK() FIXME() +# define DUMB_UNLOCK() FIXME() +#endif + +class DumbCircularBuffer { + public: + DumbCircularBuffer(size_t init_buffer_size) { + used = 0; + buffer_size = init_buffer_size; + buffer = (unsigned char*)malloc(buffer_size); + head = tail = buffer; + +#if defined(linux) + pthread_mutexattr_t mAttr; + pthread_mutexattr_settype(&mAttr, PTHREAD_MUTEX_RECURSIVE_NP); + pthread_mutex_init(&mutex, &mAttr); + pthread_mutexattr_destroy(&mAttr); +#endif + } + ~DumbCircularBuffer() { + free(buffer); +#if defined(linux) + pthread_mutex_destroy(&mutex); +#endif + } + + void clear() { + DUMB_LOCK(); + head = tail; + used = 0; + DUMB_UNLOCK(); + } + + bool empty() { return head == tail; } + + size_t space_available() { + size_t result; + DUMB_LOCK(); + if (tail > head) + result = buffer_size - (tail - head) - 1; + else + result = head - tail - 1; + DUMB_UNLOCK(); + return result; + } + + void drop(size_t size) { + // assumes correctness! + DUMB_LOCK(); + head += size; + if (head >= &buffer[buffer_size]) head -= buffer_size; + used--; + DUMB_UNLOCK(); + } + + bool insert(void* data, size_t size) { + // can fail if not enough space in the entire buffer + DUMB_LOCK(); + if (space_available() < size) return false; + + size_t max_without_wrap = &buffer[buffer_size] - tail; + size_t initial = size > max_without_wrap ? max_without_wrap : size; +#if DEBUG_CIRCULAR + fprintf(stderr, "insert(%d): max_without_wrap %d, size %d, initial %d\n", + used, max_without_wrap, size, initial); +#endif + memcpy(tail, data, initial); + tail += initial; + data = ((char*)data) + initial; + size -= initial; + if (size != 0) { +#if DEBUG_CIRCULAR + fprintf(stderr, "wrapping by %d bytes\n", size); +#endif + memcpy(buffer, data, size); + tail = &(((unsigned char*)buffer)[size]); + } + + used++; + DUMB_UNLOCK(); + + return true; + } + + // for external access to the buffer (saving) + void lock() { DUMB_LOCK(); } + + void unlock() { DUMB_UNLOCK(); } + + // XXX These really shouldn't be public... + unsigned char* head; + unsigned char* tail; + unsigned int used; + unsigned char* buffer; + size_t buffer_size; + + private: + pthread_mutex_t mutex; +}; + +class DumbCircularBuffer* JprofBuffer; + +JPROF_STATIC void JprofBufferInit(size_t size) { + JprofBuffer = new DumbCircularBuffer(size); +} + +JPROF_STATIC void JprofBufferClear() { + fprintf(stderr, "Told to clear JPROF circular buffer\n"); + JprofBuffer->clear(); +} + +JPROF_STATIC size_t JprofEntrySizeof(malloc_log_entry* me) { + return offsetof(malloc_log_entry, pcs) + me->numpcs * sizeof(char*); +} + +JPROF_STATIC void JprofBufferAppend(malloc_log_entry* me) { + size_t size = JprofEntrySizeof(me); + + do { + while (JprofBuffer->space_available() < size && JprofBuffer->used > 0) { +#if DEBUG_CIRCULAR + fprintf( + stderr, + "dropping entry: %d in use, %d free, need %d, size_to_free = %d\n", + JprofBuffer->used, JprofBuffer->space_available(), size, + JprofEntrySizeof((malloc_log_entry*)JprofBuffer->head)); +#endif + JprofBuffer->drop(JprofEntrySizeof((malloc_log_entry*)JprofBuffer->head)); + } + if (JprofBuffer->space_available() < size) return; + + } while (!JprofBuffer->insert(me, size)); +} + +JPROF_STATIC void JprofBufferDump() { + JprofBuffer->lock(); +#if DEBUG_CIRCULAR + fprintf( + stderr, "dumping JP_CIRCULAR buffer, %d of %d bytes\n", + JprofBuffer->tail > JprofBuffer->head + ? JprofBuffer->tail - JprofBuffer->head + : JprofBuffer->buffer_size + JprofBuffer->tail - JprofBuffer->head, + JprofBuffer->buffer_size); +#endif + if (JprofBuffer->tail >= JprofBuffer->head) { + write(gLogFD, JprofBuffer->head, JprofBuffer->tail - JprofBuffer->head); + } else { + write(gLogFD, JprofBuffer->head, + &(JprofBuffer->buffer[JprofBuffer->buffer_size]) - JprofBuffer->head); + write(gLogFD, JprofBuffer->buffer, JprofBuffer->tail - JprofBuffer->buffer); + } + JprofBuffer->clear(); + JprofBuffer->unlock(); +} + +//---------------------------------------------------------------------- + +JPROF_STATIC void JprofLog(u_long aTime, void* stack_top, void* top_instr_ptr) { + // Static is simply to make debugging tolerable + static malloc_log_entry me; + + me.delTime = aTime; + me.thread = syscall(SYS_gettid); // gettid(); + if (was_paused) { + me.flags = JP_FIRST_AFTER_PAUSE; + was_paused = 0; + } else { + me.flags = 0; + } + + CrawlStack(&me, stack_top, top_instr_ptr); + +#ifndef NTO + if (circular) { + JprofBufferAppend(&me); + } else { + write(gLogFD, &me, JprofEntrySizeof(&me)); + } +#else + printf("Neutrino is missing the pcs member of malloc_log_entry!! \n"); +#endif +} + +static int realTime; + +/* Lets interrupt at 10 Hz. This is so my log files don't get too large. + * This can be changed to a faster value latter. This timer is not + * programmed to reset, even though it is capable of doing so. This is + * to keep from getting interrupts from inside of the handler. + */ +static void startSignalCounter(unsigned long millisec) { + struct itimerval tvalue; + + tvalue.it_interval.tv_sec = 0; + tvalue.it_interval.tv_usec = 0; + tvalue.it_value.tv_sec = millisec / 1000; + tvalue.it_value.tv_usec = (millisec % 1000) * 1000; + + if (realTime) { + setitimer(ITIMER_REAL, &tvalue, nullptr); + } else { + setitimer(ITIMER_PROF, &tvalue, nullptr); + } +} + +static long timerMilliSec = 50; + +#if defined(linux) +static int setupRTCSignals(int hz, struct sigaction* sap) { + /* global */ rtcFD = open("/dev/rtc", O_RDONLY); + if (rtcFD < 0) { + perror("JPROF_RTC setup: open(\"/dev/rtc\", O_RDONLY)"); + return 0; + } + + if (sigaction(SIGIO, sap, nullptr) == -1) { + perror("JPROF_RTC setup: sigaction(SIGIO)"); + return 0; + } + + if (ioctl(rtcFD, RTC_IRQP_SET, hz) == -1) { + perror("JPROF_RTC setup: ioctl(/dev/rtc, RTC_IRQP_SET, $JPROF_RTC_HZ)"); + return 0; + } + + if (ioctl(rtcFD, RTC_PIE_ON, 0) == -1) { + perror("JPROF_RTC setup: ioctl(/dev/rtc, RTC_PIE_ON)"); + return 0; + } + + if (fcntl(rtcFD, F_SETSIG, 0) == -1) { + perror("JPROF_RTC setup: fcntl(/dev/rtc, F_SETSIG, 0)"); + return 0; + } + + if (fcntl(rtcFD, F_SETOWN, getpid()) == -1) { + perror("JPROF_RTC setup: fcntl(/dev/rtc, F_SETOWN, getpid())"); + return 0; + } + + return 1; +} + +static int enableRTCSignals(bool enable) { + static bool enabled = false; + if (enabled == enable) { + return 0; + } + enabled = enable; + + int flags = fcntl(rtcFD, F_GETFL); + if (flags < 0) { + perror("JPROF_RTC setup: fcntl(/dev/rtc, F_GETFL)"); + return 0; + } + + if (enable) { + flags |= FASYNC; + } else { + flags &= ~FASYNC; + } + + if (fcntl(rtcFD, F_SETFL, flags) == -1) { + if (enable) { + perror("JPROF_RTC setup: fcntl(/dev/rtc, F_SETFL, flags | FASYNC)"); + } else { + perror("JPROF_RTC setup: fcntl(/dev/rtc, F_SETFL, flags & ~FASYNC)"); + } + return 0; + } + + return 1; +} +#endif + +JPROF_STATIC void StackHook(int signum, siginfo_t* info, void* ucontext) { + static struct timeval tFirst; + static int first = 1; + size_t millisec = 0; + +#if defined(linux) + if (rtcHz && pthread_self() != main_thread) { + // Only collect stack data on the main thread, for now. + return; + } +#endif + + if (first && !(first = 0)) { + puts("Jprof: received first signal"); +#if defined(linux) + if (rtcHz) { + enableRTCSignals(true); + } else +#endif + { + gettimeofday(&tFirst, 0); + millisec = 0; + } + } else { +#if defined(linux) + if (rtcHz) { + enableRTCSignals(true); + } else +#endif + { + struct timeval tNow; + gettimeofday(&tNow, 0); + double usec = 1e6 * (tNow.tv_sec - tFirst.tv_sec); + usec += (tNow.tv_usec - tFirst.tv_usec); + millisec = static_cast<size_t>(usec * 1e-3); + } + } + + gregset_t& gregs = ((ucontext_t*)ucontext)->uc_mcontext.gregs; +#ifdef __x86_64__ + JprofLog(millisec, (void*)gregs[REG_RSP], (void*)gregs[REG_RIP]); +#else + JprofLog(millisec, (void*)gregs[REG_ESP], (void*)gregs[REG_EIP]); +#endif + + if (!rtcHz) startSignalCounter(timerMilliSec); +} + +NS_EXPORT_(void) setupProfilingStuff(void) { + static int gFirstTime = 1; + char filename[2048]; // XXX fix + + if (gFirstTime && !(gFirstTime = 0)) { + int startTimer = 1; + int doNotStart = 1; + int firstDelay = 0; + int append = O_TRUNC; + char* tst = getenv("JPROF_FLAGS"); + + /* Options from JPROF_FLAGS environment variable: + * JP_DEFER -> Wait for a SIGPROF (or SIGALRM, if JP_REALTIME + * is set) from userland before starting + * to generate them internally + * JP_START -> Install the signal handler + * JP_PERIOD -> Time between profiler ticks + * JP_FIRST -> Extra delay before starting + * JP_REALTIME -> Take stack traces in intervals of real time + * rather than time used by the process (and the + * system for the process). This is useful for + * finding time spent by the X server. + * JP_APPEND -> Append to jprof-log rather than overwriting it. + * This is somewhat risky since it depends on the + * address map staying constant across multiple runs. + * JP_FILENAME -> base filename to use when saving logs. Note that + * this does not affect the mapfile. + * JP_CIRCULAR -> use a circular buffer of size N, write/clear on SIGUSR1 + * + * JPROF_ISCHILD is set if this is not the first process. + */ + + circular = false; + + if (tst) { + if (strstr(tst, "JP_DEFER")) { + doNotStart = 0; + startTimer = 0; + } + if (strstr(tst, "JP_START")) doNotStart = 0; + if (strstr(tst, "JP_REALTIME")) realTime = 1; + if (strstr(tst, "JP_APPEND")) append = O_APPEND; + + char* delay = strstr(tst, "JP_PERIOD="); + if (delay) { + double tmp = strtod(delay + strlen("JP_PERIOD="), nullptr); + if (tmp >= 1e-3) { + timerMilliSec = static_cast<unsigned long>(1000 * tmp); + } else { + fprintf(stderr, "JP_PERIOD of %g less than 0.001 (1ms), using 1ms\n", + tmp); + timerMilliSec = 1; + } + } + + char* circular_op = strstr(tst, "JP_CIRCULAR="); + if (circular_op) { + size_t size = atol(circular_op + strlen("JP_CIRCULAR=")); + if (size < 1000) { + fprintf(stderr, "JP_CIRCULAR of %lu less than 1000, using 10000\n", + (unsigned long)size); + size = 10000; + } + JprofBufferInit(size); + fprintf(stderr, "JP_CIRCULAR buffer of %lu bytes\n", + (unsigned long)size); + circular = true; + } + + char* first = strstr(tst, "JP_FIRST="); + if (first) { + firstDelay = atol(first + strlen("JP_FIRST=")); + } + + char* rtc = strstr(tst, "JP_RTC_HZ="); + if (rtc) { +#if defined(linux) + rtcHz = atol(rtc + strlen("JP_RTC_HZ=")); + timerMilliSec = 0; /* This makes JP_FIRST work right. */ + realTime = 1; /* It's the _R_TC and all. ;) */ + +# define IS_POWER_OF_TWO(x) (((x) & ((x)-1)) == 0) + + if (!IS_POWER_OF_TWO(rtcHz) || rtcHz < 2) { + fprintf(stderr, + "JP_RTC_HZ must be power of two and >= 2, " + "but %d was provided; using default of 2048\n", + rtcHz); + rtcHz = 2048; + } +#else + fputs( + "JP_RTC_HZ found, but RTC profiling only supported on " + "Linux!\n", + stderr); + +#endif + } + const char* f = strstr(tst, "JP_FILENAME="); + if (f) + f = f + strlen("JP_FILENAME="); + else + f = M_LOGFILE; + + char* is_child = getenv("JPROF_ISCHILD"); + if (!is_child) setenv("JPROF_ISCHILD", "", 0); + gIsChild = !!is_child; + + gFilenamePID = syscall(SYS_gettid); // gettid(); + if (is_child) + snprintf(filename, sizeof(filename), "%s-%d", f, gFilenamePID); + else + snprintf(filename, sizeof(filename), "%s", f); + + // XXX FIX! inherit current capture state! + } + + if (!doNotStart) { + if (gLogFD < 0) { + gLogFD = open(filename, O_CREAT | O_WRONLY | append, 0666); + if (gLogFD < 0) { + fprintf(stderr, "Unable to create " M_LOGFILE); + perror(":"); + } else { + struct sigaction action; + sigset_t mset; + + // Dump out the address map when we terminate + RegisterJprofShutdown(); + + main_thread = pthread_self(); + // fprintf(stderr,"jprof: main_thread = %u\n", + // (unsigned int)main_thread); + + // FIX! probably should block these against each other + // Very unlikely. + sigemptyset(&mset); + action.sa_handler = nullptr; + action.sa_sigaction = StackHook; + action.sa_mask = mset; + action.sa_flags = SA_RESTART | SA_SIGINFO; +#if defined(linux) + if (rtcHz) { + if (!setupRTCSignals(rtcHz, &action)) { + fputs( + "jprof: Error initializing RTC, NOT " + "profiling\n", + stderr); + return; + } + } + + if (!rtcHz || firstDelay != 0) +#endif + { + if (realTime) { + sigaction(SIGALRM, &action, nullptr); + } + } + // enable PROF in all cases to simplify JP_DEFER/pause/restart + sigaction(SIGPROF, &action, nullptr); + + // make it so a SIGUSR1 will stop the profiling + // Note: It currently does not close the logfile. + // This could be configurable (so that it could + // later be reopened). + + struct sigaction stop_action; + stop_action.sa_handler = EndProfilingHook; + stop_action.sa_mask = mset; + stop_action.sa_flags = SA_RESTART; + sigaction(SIGUSR1, &stop_action, nullptr); + + // make it so a SIGUSR2 will clear the circular buffer + + stop_action.sa_handler = ClearProfilingHook; + stop_action.sa_mask = mset; + stop_action.sa_flags = SA_RESTART; + sigaction(SIGUSR2, &stop_action, nullptr); + + printf( + "Jprof: Initialized signal handler and set " + "timer for %lu %s, %d s " + "initial delay\n", + rtcHz ? rtcHz : timerMilliSec, rtcHz ? "Hz" : "ms", firstDelay); + + if (startTimer) { +#if defined(linux) + /* If we have an initial delay we can just use + startSignalCounter to set up a timer to fire the + first stackHook after that delay. When that happens + we'll go and switch to RTC profiling. */ + if (rtcHz && firstDelay == 0) { + puts("Jprof: enabled RTC signals"); + enableRTCSignals(true); + } else +#endif + { + puts("Jprof: started timer"); + startSignalCounter(firstDelay * 1000 + timerMilliSec); + } + } + } + } + } + } else { + printf("setupProfilingStuff() called multiple times\n"); + } +} diff --git a/tools/jprof/stub/libmalloc.h b/tools/jprof/stub/libmalloc.h new file mode 100644 index 0000000000..a78b35ade8 --- /dev/null +++ b/tools/jprof/stub/libmalloc.h @@ -0,0 +1,45 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef libmalloc_h___ +#define libmalloc_h___ + +#include <sys/types.h> +#include <malloc.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#include "config.h" + +typedef unsigned long u_long; + +// For me->flags +#define JP_FIRST_AFTER_PAUSE 1 + +// Format of a jprof log entry. This is what's written out to the +// "jprof-log" file. +// It's called malloc_log_entry because the history of jprof is that +// it's a modified version of tracemalloc. +struct malloc_log_entry { + u_long delTime; + u_long numpcs; + unsigned int flags; + int thread; + char* pcs[MAX_STACK_CRAWL]; +}; + +// Format of a malloc map entry; after this struct is nameLen+1 bytes of +// name data. +struct malloc_map_entry { + u_long nameLen; + u_long address; // base address +}; + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif + +#endif /* libmalloc_h___ */ diff --git a/tools/jprof/stub/moz.build b/tools/jprof/stub/moz.build new file mode 100644 index 0000000000..692c6ea37f --- /dev/null +++ b/tools/jprof/stub/moz.build @@ -0,0 +1,17 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS += [ + "jprof.h", +] + +SOURCES += [ + "libmalloc.cpp", +] + +SharedLibrary("jprof") + +DEFINES["_IMPL_JPROF_API"] = True diff --git a/tools/leak-gauge/leak-gauge.html b/tools/leak-gauge/leak-gauge.html new file mode 100644 index 0000000000..7eb2de10c1 --- /dev/null +++ b/tools/leak-gauge/leak-gauge.html @@ -0,0 +1,320 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"> +<!-- + vim:sw=4:ts=4:et: + This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. +--> +<html lang="en-US"> + <head> + <meta charset="UTF-8" /> + <title>Leak Gauge</title> + + <style type="text/css"> + pre { + margin: 0; + } + pre.output { + border: medium solid; + padding: 1em; + margin: 1em; + } + </style> + <script> + function runfile(file) { + var result = "Results of processing log " + file.fileName + " :\n"; + + var fileReader = new FileReader(); + fileReader.onload = function (e) { + runContents(result, e.target.result); + }; + fileReader.readAsText(file, "iso-8859-1"); + } + + function runContents(result, contents) { + // A hash of objects (keyed by the first word of the line in the log) + // that have two public methods, handle_line and dump (to be called using + // call, above), along with any private data they need. + var handlers = { + DOMWINDOW: { + count: 0, + windows: {}, + handle_line(line) { + var match = line.match(/^([0-9a-f]*) (\S*)(.*)/); + if (match) { + var addr = match[1]; + var verb = match[2]; + var rest = match[3]; + if (verb == "created") { + let m = rest.match(/ outer=([0-9a-f]*)$/); + if (!m) throw new Error("outer expected"); + this.windows[addr] = { outer: m[1] }; + ++this.count; + } else if (verb == "destroyed") { + delete this.windows[addr]; + } else if (verb == "SetNewDocument") { + let m = rest.match(/^ (.*)$/); + if (!m) throw new Error("URI expected"); + this.windows[addr][m[1]] = true; + } + } + }, + dump() { + for (var addr in this.windows) { + var winobj = this.windows[addr]; + var outer = winobj.outer; + delete winobj.outer; + result += + "Leaked " + + (outer == "0" ? "outer" : "inner") + + " window " + + addr + + " " + + (outer == "0" ? "" : "(outer " + outer + ") ") + + "at address " + + addr + + ".\n"; + for (var uri in winobj) { + result += ' ... with URI "' + uri + '".\n'; + } + } + }, + summary() { + result += + "Leaked " + + Object.keys(this.windows).length + + " out of " + + this.count + + " DOM Windows\n"; + }, + }, + DOCUMENT: { + count: 0, + docs: {}, + handle_line(line) { + var match = line.match(/^([0-9a-f]*) (\S*)(.*)/); + if (match) { + var addr = match[1]; + var verb = match[2]; + var rest = match[3]; + if (verb == "created") { + this.docs[addr] = {}; + ++this.count; + } else if (verb == "destroyed") { + delete this.docs[addr]; + } else if ( + verb == "ResetToURI" || + verb == "StartDocumentLoad" + ) { + var m = rest.match(/^ (.*)$/); + if (!m) throw new Error("URI expected"); + var uri = m[1]; + var doc_info = this.docs[addr]; + doc_info[uri] = true; + if ("nim" in doc_info) { + doc_info.nim[uri] = true; + } + } + } + }, + dump() { + for (var addr in this.docs) { + var doc = this.docs[addr]; + result += "Leaked document at address " + addr + ".\n"; + for (var uri in doc) { + if (uri != "nim") { + result += ' ... with URI "' + uri + '".\n'; + } + } + } + }, + summary() { + result += + "Leaked " + + Object.keys(this.docs).length + + " out of " + + this.count + + " documents\n"; + }, + }, + DOCSHELL: { + count: 0, + shells: {}, + handle_line(line) { + var match = line.match(/^([0-9a-f]*) (\S*)(.*)/); + if (match) { + var addr = match[1]; + var verb = match[2]; + var rest = match[3]; + if (verb == "created") { + this.shells[addr] = {}; + ++this.count; + } else if (verb == "destroyed") { + delete this.shells[addr]; + } else if (verb == "InternalLoad" || verb == "SetCurrentURI") { + var m = rest.match(/^ (.*)$/); + if (!m) throw new Error("URI expected"); + this.shells[addr][m[1]] = true; + } + } + }, + dump() { + for (var addr in this.shells) { + var doc = this.shells[addr]; + result += "Leaked docshell at address " + addr + ".\n"; + for (var uri in doc) { + result += ' ... which loaded URI "' + uri + '".\n'; + } + } + }, + summary() { + result += + "Leaked " + + Object.keys(this.shells).length + + " out of " + + this.count + + " docshells\n"; + }, + }, + NODEINFOMANAGER: { + count: 0, + nims: {}, + handle_line(line) { + var match = line.match(/^([0-9a-f]*) (\S*)(.*)/); + if (match) { + var addr = match[1]; + var verb = match[2]; + var rest = match[3]; + if (verb == "created") { + this.nims[addr] = {}; + ++this.count; + } else if (verb == "destroyed") { + delete this.nims[addr]; + } else if (verb == "Init") { + var m = rest.match(/^ document=(.*)$/); + if (!m) throw new Error("document pointer expected"); + var nim_info = this.nims[addr]; + var doc = m[1]; + if (doc != "0") { + var doc_info = handlers.DOCUMENT.docs[doc]; + for (var uri in doc_info) { + nim_info[uri] = true; + } + doc_info.nim = nim_info; + } + } + } + }, + dump() { + for (var addr in this.nims) { + var nim = this.nims[addr]; + result += + "Leaked content nodes associated with node info manager at address " + + addr + + ".\n"; + for (var uri in nim) { + result += ' ... with document URI "' + uri + '".\n'; + } + } + }, + summary() { + result += + "Leaked content nodes in " + + Object.keys(this.nims).length + + " out of " + + this.count + + " documents\n"; + }, + }, + }; + + var lines = contents.split(/[\r\n]+/); + for (var j in lines) { + var line = lines[j]; + // strip off initial "-", thread id, and thread pointer; separate + // first word and rest + var matches = line.match(/^\-?[0-9]*\[[0-9a-f]*\]: (\S*) (.*)$/); + if (matches) { + let handler = matches[1]; + var data = matches[2]; + if (typeof handlers[handler] != "undefined") { + handlers[handler].handle_line(data); + } + } + } + + for (let handler in handlers) handlers[handler].dump(); + if (result.length) result += "\n"; + result += "Summary:\n"; + for (let handler in handlers) handlers[handler].summary(); + result += "\n"; + + var out = document.createElement("pre"); + out.className = "output"; + out.appendChild(document.createTextNode(result)); + document.body.appendChild(out); + } + + function run() { + var input = document.getElementById("fileinput"); + var files = input.files; + for (var i = 0; i < files.length; ++i) runfile(files[i]); + // So the user can process the same filename again (after + // overwriting the log), clear the value on the form input so we + // will always get an onchange event. + input.value = ""; + } + </script> + </head> + <body> + <h1>Leak Gauge</h1> + + <pre> +$Id: leak-gauge.html,v 1.8 2008/02/08 19:55:34 dbaron%dbaron.org Exp $</pre + > + + <p> + This script is designed to help testers isolate and simplify testcases for + many classes of leaks (those that involve large graphs of core data + structures) in Mozilla-based browsers. It is designed to print information + about what has leaked by processing a log taken while running the browser. + Such a log can be taken over a long session of normal browsing and then + the log can be processed to find sites that leak. Once a site is known to + leak, the logging can then be repeated to figure out under what conditions + the leak occurs. + </p> + + <p>The way to create this log is to set the environment variables:</p> + <pre> MOZ_LOG=DOMLeak:5,DocumentLeak:5,nsDocShellLeak:5,NodeInfoManagerLeak:5 + MOZ_LOG_FILE=nspr.log <i>(or any other filename of your choice)</i></pre> + <p>in your shell and then run the program.</p> + <ul> + <li> + In a Windows command prompt, set environment variables with + <pre> set VAR=value</pre> + </li> + <li> + In an sh-based shell such as bash, set environment variables with + <pre> export VAR=value</pre> + </li> + <li> + In a csh-based shell such as tcsh, set environment variables with + <pre> setenv VAR value</pre> + </li> + </ul> + + <p> + Once you have this log from a complete run of the browser (you have to + exit; otherwise it will look like everything leaked), you can load this + page (be careful not to overwrite the log when starting the browser to + load this page) and enter the filename of the log: + </p> + + <p><input type="file" id="fileinput" onchange="run()" /></p> + + <p> + Then you'll see the output below, which will tell you which of certain + core objects leaked and the URLs associated with those objects. + </p> + </body> +</html> diff --git a/tools/leak-gauge/leak-gauge.pl b/tools/leak-gauge/leak-gauge.pl new file mode 100755 index 0000000000..76ac597df1 --- /dev/null +++ b/tools/leak-gauge/leak-gauge.pl @@ -0,0 +1,239 @@ +#!/usr/bin/perl -w +# vim:sw=4:ts=4:et: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# $Id: leak-gauge.pl,v 1.8 2008/02/08 19:55:03 dbaron%dbaron.org Exp $ +# This script is designed to help testers isolate and simplify testcases +# for many classes of leaks (those that involve large graphs of core +# data structures) in Mozilla-based browsers. It is designed to print +# information about what has leaked by processing a log taken while +# running the browser. Such a log can be taken over a long session of +# normal browsing and then the log can be processed to find sites that +# leak. Once a site is known to leak, the logging can then be repeated +# to figure out under what conditions the leak occurs. +# +# The way to create this log is to set the environment variables: +# MOZ_LOG=DOMLeak:5,DocumentLeak:5,nsDocShellLeak:5,NodeInfoManagerLeak:5 +# MOZ_LOG_FILE=nspr.log (or any other filename of your choice) +# in your shell and then run the program. +# * In a Windows command prompt, set environment variables with +# set VAR=value +# * In an sh-based shell such as bash, set environment variables with +# export VAR=value +# * In a csh-based shell such as tcsh, set environment variables with +# setenv VAR value +# +# Then, after you have exited the browser, run this perl script over the +# log. Either of the following commands should work: +# perl ./path/to/leak-gauge.pl nspr.log +# cat nspr.log | perl ./path/to/leak-gauge.pl +# and it will tell you which of certain core objects leaked and the URLs +# associated with those objects. + + +# Nobody said I'm not allowed to write my own object system in perl. No +# classes here. Just objects and methods. +sub call { + my $func = shift; + my $obj = shift; + my $funcref = ${$obj}{$func}; + &$funcref($obj, @_); +} + +# A hash of objects (keyed by the first word of the line in the log) +# that have two public methods, handle_line and dump (to be called using +# call, above), along with any private data they need. +my $handlers = { + "DOMWINDOW" => { + count => 0, + windows => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + my $windows = ${$self}{windows}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + $rest =~ / outer=([0-9a-f]*)$/ || die "outer expected"; + my $outer = $1; + ${$windows}{$addr} = { outer => $1 }; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$windows}{$addr}; + } elsif ($verb eq "SetNewDocument") { + $rest =~ /^ (.*)$/ || die "URI expected"; + my $uri = ($1); + ${${$windows}{$addr}}{$uri} = 1; + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $windows = ${$self}{windows}; + foreach my $addr (keys(%{$windows})) { + my $winobj = ${$windows}{$addr}; + my $outer = delete ${$winobj}{outer}; + print "Leaked " . ($outer eq "0" ? "outer" : "inner") . + " window $addr " . + ($outer eq "0" ? "" : "(outer $outer) ") . + "at address $addr.\n"; + foreach my $uri (keys(%{$winobj})) { + print " ... with URI \"$uri\".\n"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $windows = ${$self}{windows}; + print 'Leaked ' . keys(%{$windows}) . ' out of ' . + ${$self}{count} . " DOM Windows\n"; + } + }, + "DOCUMENT" => { + count => 0, + docs => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + # This doesn't work; I don't have time to figure out why not. + # my $docs = ${$self}{docs}; + my $docs = ${$handlers}{"DOCUMENT"}{docs}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + ${$docs}{$addr} = {}; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$docs}{$addr}; + } elsif ($verb eq "ResetToURI" || + $verb eq "StartDocumentLoad") { + $rest =~ /^ (.*)$/ || die "URI expected"; + my $uri = $1; + my $doc_info = ${$docs}{$addr}; + ${$doc_info}{$uri} = 1; + if (exists(${$doc_info}{"nim"})) { + ${$doc_info}{"nim"}{$uri} = 1; + } + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $docs = ${$self}{docs}; + foreach my $addr (keys(%{$docs})) { + print "Leaked document at address $addr.\n"; + foreach my $uri (keys(%{${$docs}{$addr}})) { + print " ... with URI \"$uri\".\n" unless $uri eq "nim"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $docs = ${$self}{docs}; + print 'Leaked ' . keys(%{$docs}) . ' out of ' . + ${$self}{count} . " documents\n"; + } + }, + "DOCSHELL" => { + count => 0, + shells => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + my $shells = ${$self}{shells}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + ${$shells}{$addr} = {}; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$shells}{$addr}; + } elsif ($verb eq "InternalLoad" || + $verb eq "SetCurrentURI") { + $rest =~ /^ (.*)$/ || die "URI expected"; + my $uri = $1; + ${${$shells}{$addr}}{$uri} = 1; + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $shells = ${$self}{shells}; + foreach my $addr (keys(%{$shells})) { + print "Leaked docshell at address $addr.\n"; + foreach my $uri (keys(%{${$shells}{$addr}})) { + print " ... which loaded URI \"$uri\".\n"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $shells = ${$self}{shells}; + print 'Leaked ' . keys(%{$shells}) . ' out of ' . + ${$self}{count} . " docshells\n"; + } + }, + "NODEINFOMANAGER" => { + count => 0, + nims => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + my $nims = ${$self}{nims}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + ${$nims}{$addr} = {}; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$nims}{$addr}; + } elsif ($verb eq "Init") { + $rest =~ /^ document=(.*)$/ || + die "document pointer expected"; + my $doc = $1; + if ($doc ne "0") { + my $nim_info = ${$nims}{$addr}; + my $doc_info = ${$handlers}{"DOCUMENT"}{docs}{$doc}; + foreach my $uri (keys(%{$doc_info})) { + ${$nim_info}{$uri} = 1; + } + ${$doc_info}{"nim"} = $nim_info; + } + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $nims = ${$self}{nims}; + foreach my $addr (keys(%{$nims})) { + print "Leaked content nodes associated with node info manager at address $addr.\n"; + foreach my $uri (keys(%{${$nims}{$addr}})) { + print " ... with document URI \"$uri\".\n"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $nims = ${$self}{nims}; + print 'Leaked content nodes within ' . keys(%{$nims}) . ' out of ' . + ${$self}{count} . " documents\n"; + } + } +}; + +while (<>) { + # strip off initial "-", thread id, and thread pointer; separate + # first word and rest + if (/^\-?[0-9]*\[[0-9a-f]*\]: (\S*) ([^\n\r]*)[\n\r]*$/) { + my ($handler, $data) = ($1, $2); + if (defined(${$handlers}{$handler})) { + call("handle_line", ${$handlers}{$handler}, $data); + } + } +} + +foreach my $key (keys(%{$handlers})) { + call("dump", ${$handlers}{$key}); +} +print "Summary:\n"; +foreach my $key (keys(%{$handlers})) { + call("summary", ${$handlers}{$key}); +} diff --git a/tools/lint/android-api-lint.yml b/tools/lint/android-api-lint.yml new file mode 100644 index 0000000000..0b81e79a83 --- /dev/null +++ b/tools/lint/android-api-lint.yml @@ -0,0 +1,15 @@ +--- +android-api-lint: + description: Android api-lint + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:api_lint + setup: android.lints:setup diff --git a/tools/lint/android-checkstyle.yml b/tools/lint/android-checkstyle.yml new file mode 100644 index 0000000000..abd4974e6a --- /dev/null +++ b/tools/lint/android-checkstyle.yml @@ -0,0 +1,15 @@ +--- +android-checkstyle: + description: Android checkstyle + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:checkstyle + setup: android.lints:setup diff --git a/tools/lint/android-format.yml b/tools/lint/android-format.yml new file mode 100644 index 0000000000..cacf3ff2d7 --- /dev/null +++ b/tools/lint/android-format.yml @@ -0,0 +1,15 @@ +--- +android-format: + description: Android formatting lint + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:format + setup: android.lints:setup diff --git a/tools/lint/android-javadoc.yml b/tools/lint/android-javadoc.yml new file mode 100644 index 0000000000..a0811a08cd --- /dev/null +++ b/tools/lint/android-javadoc.yml @@ -0,0 +1,15 @@ +--- +android-javadoc: + description: Android javadoc + include: ['mobile/android/geckoview'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:javadoc + setup: android.lints:setup diff --git a/tools/lint/android-lint.yml b/tools/lint/android-lint.yml new file mode 100644 index 0000000000..6a1dbe7b74 --- /dev/null +++ b/tools/lint/android-lint.yml @@ -0,0 +1,15 @@ +--- +android-lint: + description: Android lint + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:lint + setup: android.lints:setup diff --git a/tools/lint/android-test.yml b/tools/lint/android-test.yml new file mode 100644 index 0000000000..65739295dd --- /dev/null +++ b/tools/lint/android-test.yml @@ -0,0 +1,15 @@ +--- +android-test: + description: Android test + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:test + setup: android.lints:setup diff --git a/tools/lint/android/__init__.py b/tools/lint/android/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/lint/android/__init__.py diff --git a/tools/lint/android/lints.py b/tools/lint/android/lints.py new file mode 100644 index 0000000000..53c8626013 --- /dev/null +++ b/tools/lint/android/lints.py @@ -0,0 +1,420 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import glob +import itertools +import json +import os +import re +import shlex +import subprocess +import sys +import xml.etree.ElementTree as ET + +import mozpack.path as mozpath +from mozlint import result +from mozpack.files import FileFinder + +# The Gradle target invocations are serialized with a simple locking file scheme. It's fine for +# them to take a while, since the first will compile all the Java, etc, and then perform +# potentially expensive static analyses. +GRADLE_LOCK_MAX_WAIT_SECONDS = 20 * 60 + + +def setup(root, **setupargs): + if setupargs.get("substs", {}).get("MOZ_BUILD_APP") != "mobile/android": + return 1 + + if "topobjdir" not in setupargs: + print( + "Skipping {}: a configured Android build is required!".format( + setupargs["name"] + ) + ) + return 1 + + return 0 + + +def gradle(log, topsrcdir=None, topobjdir=None, tasks=[], extra_args=[], verbose=True): + sys.path.insert(0, os.path.join(topsrcdir, "mobile", "android")) + from gradle import gradle_lock + + with gradle_lock(topobjdir, max_wait_seconds=GRADLE_LOCK_MAX_WAIT_SECONDS): + # The android-lint parameter can be used by gradle tasks to run special + # logic when they are run for a lint using + # project.hasProperty('android-lint') + cmd_args = ( + [ + sys.executable, + os.path.join(topsrcdir, "mach"), + "gradle", + "--verbose", + "-Pandroid-lint", + "--", + ] + + tasks + + extra_args + ) + + cmd = " ".join(shlex.quote(arg) for arg in cmd_args) + log.debug(cmd) + + # Gradle and mozprocess do not get along well, so we use subprocess + # directly. + proc = subprocess.Popen(cmd_args, cwd=topsrcdir) + status = None + # Leave it to the subprocess to handle Ctrl+C. If it terminates as a result + # of Ctrl+C, proc.wait() will return a status code, and, we get out of the + # loop. If it doesn't, like e.g. gdb, we continue waiting. + while status is None: + try: + status = proc.wait() + except KeyboardInterrupt: + pass + + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + raise + + return proc.returncode + + +def format(config, fix=None, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + if fix: + tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FIX_TASKS"] + else: + tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_CHECK_TASKS"] + + ret = gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=tasks, + extra_args=lintargs.get("extra_args") or [], + ) + + results = [] + for path in lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FOLDERS"]: + folder = os.path.join( + topobjdir, "gradle", "build", path, "spotless", "spotlessJava" + ) + for filename in glob.iglob(folder + "/**/*.java", recursive=True): + err = { + "rule": "spotless-java", + "path": os.path.join( + topsrcdir, path, mozpath.relpath(filename, folder) + ), + "lineno": 0, + "column": 0, + "message": "Formatting error, please run ./mach lint -l android-format --fix", + "level": "error", + } + results.append(result.from_config(config, **err)) + folder = os.path.join( + topobjdir, "gradle", "build", path, "spotless", "spotlessKotlin" + ) + for filename in glob.iglob(folder + "/**/*.kt", recursive=True): + err = { + "rule": "spotless-kt", + "path": os.path.join( + topsrcdir, path, mozpath.relpath(filename, folder) + ), + "lineno": 0, + "column": 0, + "message": "Formatting error, please run ./mach lint -l android-format --fix", + "level": "error", + } + results.append(result.from_config(config, **err)) + + if len(results) == 0 and ret != 0: + # spotless seems to hit unfixed error. + err = { + "rule": "spotless", + "path": "", + "lineno": 0, + "column": 0, + "message": "Unexpected error", + "level": "error", + } + results.append(result.from_config(config, **err)) + + # If --fix was passed, we just report the number of files that were changed + if fix: + return {"results": [], "fixed": len(results)} + return results + + +def api_lint(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + folder = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_APILINT_FOLDER"] + + results = [] + + with open(os.path.join(topobjdir, folder, "apilint-result.json")) as f: + issues = json.load(f) + + for rule in ("compat_failures", "failures"): + for r in issues[rule]: + err = { + "rule": r["rule"] if rule == "failures" else "compat_failures", + "path": r["file"], + "lineno": int(r["line"]), + "column": int(r.get("column") or 0), + "message": r["msg"], + "level": "error" if r["error"] else "warning", + } + results.append(result.from_config(config, **err)) + + for r in issues["api_changes"]: + err = { + "rule": "api_changes", + "path": r["file"], + "lineno": int(r["line"]), + "column": int(r.get("column") or 0), + "message": "Unexpected api change. Please run ./mach gradle {} for more " + "information".format( + " ".join(lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"]) + ), + } + results.append(result.from_config(config, **err)) + + return results + + +def javadoc(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + output_files = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_OUTPUT_FILES"] + + results = [] + + for output_file in output_files: + with open(os.path.join(topobjdir, output_file)) as f: + # Like: '[{"path":"/absolute/path/to/topsrcdir/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java","lineno":"462","level":"warning","message":"no @return"}]'. # NOQA: E501 + issues = json.load(f) + + for issue in issues: + # We want warnings to be errors for linting purposes. + # TODO: Bug 1316188 - resolve missing javadoc comments + issue["level"] = ( + "error" if issue["message"] != ": no comment" else "warning" + ) + results.append(result.from_config(config, **issue)) + + return results + + +def lint(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_LINT_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + # It's surprising that this is the App variant name, but this is "withoutGeckoBinariesDebug" + # right now and the GeckoView variant name is "withGeckoBinariesDebug". This will be addressed + # as we unify variants. + path = os.path.join( + lintargs["topobjdir"], + "gradle/build/mobile/android/geckoview/reports", + "lint-results-{}.xml".format( + lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"] + ), + ) + tree = ET.parse(open(path, "rt")) + root = tree.getroot() + + results = [] + + for issue in root.findall("issue"): + location = issue[0] + if "third_party" in location.get("file") or "thirdparty" in location.get( + "file" + ): + continue + err = { + "level": issue.get("severity").lower(), + "rule": issue.get("id"), + "message": issue.get("message"), + "path": location.get("file"), + "lineno": int(location.get("line") or 0), + } + results.append(result.from_config(config, **err)) + + return results + + +def _parse_checkstyle_output(config, topsrcdir=None, report_path=None): + tree = ET.parse(open(report_path, "rt")) + root = tree.getroot() + + for file in root.findall("file"): + for error in file.findall("error"): + # Like <error column="42" line="22" message="Name 'mPorts' must match pattern 'xm[A-Z][A-Za-z]*$'." severity="error" source="com.puppycrawl.tools.checkstyle.checks.naming.MemberNameCheck" />. # NOQA: E501 + err = { + "level": "error", + "rule": error.get("source"), + "message": error.get("message"), + "path": file.get("name"), + "lineno": int(error.get("line") or 0), + "column": int(error.get("column") or 0), + } + yield result.from_config(config, **err) + + +def checkstyle(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + results = [] + + for relative_path in lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_OUTPUT_FILES"]: + report_path = os.path.join(lintargs["topobjdir"], relative_path) + results.extend( + _parse_checkstyle_output( + config, topsrcdir=lintargs["root"], report_path=report_path + ) + ) + + return results + + +def _parse_android_test_results(config, topsrcdir=None, report_dir=None): + # A brute force way to turn a Java FQN into a path on disk. Assumes Java + # and Kotlin sources are in mobile/android for performance and simplicity. + sourcepath_finder = FileFinder(os.path.join(topsrcdir, "mobile", "android")) + + finder = FileFinder(report_dir) + reports = list(finder.find("TEST-*.xml")) + if not reports: + raise RuntimeError("No reports found under {}".format(report_dir)) + + for report, _ in reports: + tree = ET.parse(open(os.path.join(finder.base, report), "rt")) + root = tree.getroot() + + class_name = root.get( + "name" + ) # Like 'org.mozilla.gecko.permissions.TestPermissions'. + path = ( + "**/" + class_name.replace(".", "/") + ".*" + ) # Like '**/org/mozilla/gecko/permissions/TestPermissions.*'. # NOQA: E501 + + for testcase in root.findall("testcase"): + function_name = testcase.get("name") + + # Schema cribbed from http://llg.cubic.org/docs/junit/. + for unexpected in itertools.chain( + testcase.findall("error"), testcase.findall("failure") + ): + sourcepaths = list(sourcepath_finder.find(path)) + if not sourcepaths: + raise RuntimeError( + "No sourcepath found for class {class_name}".format( + class_name=class_name + ) + ) + + for sourcepath, _ in sourcepaths: + lineno = 0 + message = unexpected.get("message") + # Turn '... at org.mozilla.gecko.permissions.TestPermissions.testMultipleRequestsAreQueuedAndDispatchedSequentially(TestPermissions.java:118)' into 118. # NOQA: E501 + pattern = r"at {class_name}\.{function_name}\(.*:(\d+)\)" + pattern = pattern.format( + class_name=class_name, function_name=function_name + ) + match = re.search(pattern, message) + if match: + lineno = int(match.group(1)) + else: + msg = "No source line found for {class_name}.{function_name}".format( + class_name=class_name, function_name=function_name + ) + raise RuntimeError(msg) + + err = { + "level": "error", + "rule": unexpected.get("type"), + "message": message, + "path": os.path.join( + topsrcdir, "mobile", "android", sourcepath + ), + "lineno": lineno, + } + yield result.from_config(config, **err) + + +def test(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_TEST_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + results = [] + + def capitalize(s): + # Can't use str.capitalize because it lower cases trailing letters. + return (s[0].upper() + s[1:]) if s else "" + + pairs = [("geckoview", lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"])] + for project, variant in pairs: + report_dir = os.path.join( + lintargs["topobjdir"], + "gradle/build/mobile/android/{}/test-results/test{}UnitTest".format( + project, capitalize(variant) + ), + ) + results.extend( + _parse_android_test_results( + config, topsrcdir=lintargs["root"], report_dir=report_dir + ) + ) + + return results diff --git a/tools/lint/black.yml b/tools/lint/black.yml new file mode 100644 index 0000000000..c182f9939b --- /dev/null +++ b/tools/lint/black.yml @@ -0,0 +1,19 @@ +--- +black: + description: Reformat python + exclude: + - gfx/harfbuzz/src/meson.build + - '**/*.mako.py' + - python/mozbuild/mozbuild/test/frontend/data/reader-error-syntax/moz.build + - testing/mozharness/configs/test/test_malformed.py + - testing/web-platform/tests + extensions: + - build + - configure + - mozbuild + - py + support-files: + - 'tools/lint/python/**' + type: external + payload: python.black:lint + setup: python.black:setup diff --git a/tools/lint/clang-format.yml b/tools/lint/clang-format.yml new file mode 100644 index 0000000000..bcdf9d8591 --- /dev/null +++ b/tools/lint/clang-format.yml @@ -0,0 +1,12 @@ +--- +clang-format: + description: Reformat C/C++ + include: + - '.' + extensions: ['cpp', 'c', 'cc', 'h', 'm', 'mm'] + support-files: + - 'tools/lint/clang-format/**' + type: external + payload: clang-format:lint + code_review_warnings: false + setup: clang-format:setup diff --git a/tools/lint/clang-format/__init__.py b/tools/lint/clang-format/__init__.py new file mode 100644 index 0000000000..243ea51b39 --- /dev/null +++ b/tools/lint/clang-format/__init__.py @@ -0,0 +1,229 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import signal +import subprocess +import sys +import xml.etree.ElementTree as ET + +from mozboot.util import get_tools_dir +from mozlint import result +from mozlint.pathutils import expand_exclusions + +CLANG_FORMAT_NOT_FOUND = """ +Could not find clang-format! It should've been installed automatically - \ +please report a bug here: +https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=Lint%20and%20Formatting +""".strip() + + +def setup(root, mach_command_context, **lintargs): + if get_clang_format_binary(): + return 0 + + from mozbuild.code_analysis.mach_commands import get_clang_tools + + rc, _ = get_clang_tools(mach_command_context) + if rc: + return 1 + + +def run_process(config, cmd): + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) + signal.signal(signal.SIGINT, orig) + try: + output, _ = proc.communicate() + proc.wait() + except KeyboardInterrupt: + proc.kill() + + return output + + +def get_clang_format_binary(): + """ + Returns the path of the first clang-format binary available + if not found returns None + """ + binary = os.environ.get("CLANG_FORMAT") + if binary: + return binary + + clang_tools_path = os.path.join(get_tools_dir(), "clang-tools") + bin_path = os.path.join(clang_tools_path, "clang-tidy", "bin") + binary = os.path.join(bin_path, "clang-format") + + if sys.platform.startswith("win"): + binary += ".exe" + + if not os.path.isfile(binary): + return None + + return binary + + +def is_ignored_path(ignored_dir_re, topsrcdir, f): + # Remove up to topsrcdir in pathname and match + if f.startswith(topsrcdir + "/"): + match_f = f[len(topsrcdir + "/") :] + else: + match_f = f + return re.match(ignored_dir_re, match_f) + + +def remove_ignored_path(paths, topsrcdir, log): + path_to_third_party = os.path.join(topsrcdir, ".clang-format-ignore") + + ignored_dir = [] + with open(path_to_third_party, "r") as fh: + for l in fh: + # In case it starts with a space + line = l.strip() + # Remove comments and empty lines + if line.startswith("#") or len(line) == 0: + continue + # The regexp is to make sure we are managing relative paths + ignored_dir.append(r"^[\./]*" + line.rstrip()) + + # Generates the list of regexp + ignored_dir_re = "(%s)" % "|".join(ignored_dir) + + path_list = [] + for f in paths: + if is_ignored_path(ignored_dir_re, topsrcdir, f): + # Early exit if we have provided an ignored directory + log.debug("Ignored third party code '{0}'".format(f)) + continue + path_list.append(f) + + return path_list + + +def lint(paths, config, fix=None, **lintargs): + log = lintargs["log"] + paths = list(expand_exclusions(paths, config, lintargs["root"])) + + # We ignored some specific files for a bunch of reasons. + # Not using excluding to avoid duplication + if lintargs.get("use_filters", True): + paths = remove_ignored_path(paths, lintargs["root"], log) + + # An empty path array can occur when the user passes in `-n`. If we don't + # return early in this case, rustfmt will attempt to read stdin and hang. + if not paths: + return [] + + binary = get_clang_format_binary() + + if not binary: + print(CLANG_FORMAT_NOT_FOUND) + if "MOZ_AUTOMATION" in os.environ: + return 1 + return [] + + cmd_args = [binary] + + base_command = cmd_args + ["--version"] + version = run_process(config, base_command).rstrip("\r\n") + log.debug("Version: {}".format(version)) + + cmd_args.append("--output-replacements-xml") + base_command = cmd_args + paths + log.debug("Command: {}".format(" ".join(cmd_args))) + output = run_process(config, base_command) + + def replacement(parser): + for end, e in parser.read_events(): + assert end == "end" + if e.tag == "replacement": + item = {k: int(v) for k, v in e.items()} + assert sorted(item.keys()) == ["length", "offset"] + item["with"] = (e.text or "").encode("utf-8") + yield item + + # When given multiple paths as input, --output-replacements-xml + # will output one xml per path, in the order they are given, but + # XML parsers don't know how to handle that, so do it manually. + parser = None + replacements = [] + for l in output.split("\n"): + line = l.rstrip("\r\n") + if line.startswith("<?xml "): + if parser: + replacements.append(list(replacement(parser))) + parser = ET.XMLPullParser(["end"]) + parser.feed(line) + replacements.append(list(replacement(parser))) + + results = [] + fixed = 0 + for path, replacement in zip(paths, replacements): + if not replacement: + continue + with open(path, "rb") as fh: + data = fh.read() + + linenos = [] + patched_data = b"" + last_offset = 0 + lineno_before = 1 + lineno_after = 1 + + for item in replacement: + offset = item["offset"] + length = item["length"] + replace_with = item["with"] + since_last_offset = data[last_offset:offset] + replaced = data[offset : offset + length] + + lines_since_last_offset = since_last_offset.count(b"\n") + lineno_before += lines_since_last_offset + lineno_after += lines_since_last_offset + start_lineno = (lineno_before, lineno_after) + + lineno_before += replaced.count(b"\n") + lineno_after += replace_with.count(b"\n") + end_lineno = (lineno_before, lineno_after) + + if linenos and start_lineno[0] <= linenos[-1][1][0]: + linenos[-1] = (linenos[-1][0], end_lineno) + else: + linenos.append((start_lineno, end_lineno)) + + patched_data += since_last_offset + replace_with + last_offset = offset + len(replaced) + patched_data += data[last_offset:] + + lines_before = data.decode("utf-8", "replace").splitlines() + lines_after = patched_data.decode("utf-8", "replace").splitlines() + for (start_before, start_after), (end_before, end_after) in linenos: + diff = "".join( + "-" + l + "\n" for l in lines_before[start_before - 1 : end_before] + ) + diff += "".join( + "+" + l + "\n" for l in lines_after[start_after - 1 : end_after] + ) + + results.append( + result.from_config( + config, + path=path, + diff=diff, + level="warning", + lineno=start_before, + column=0, + ) + ) + + if fix: + with open(path, "wb") as fh: + fh.write(patched_data) + fixed += len(linenos) + + return {"results": results, "fixed": fixed} diff --git a/tools/lint/clippy.yml b/tools/lint/clippy.yml new file mode 100644 index 0000000000..7b8d196e21 --- /dev/null +++ b/tools/lint/clippy.yml @@ -0,0 +1,110 @@ +--- +clippy: + description: Lint rust + include: + - build/workspace-hack/ + - dom/midi/midir_impl/ + - dom/media/gtest/ + - dom/webauthn/libudev-sys/ + - gfx/webrender_bindings/ + - gfx/wr/peek-poke/ + - gfx/wr/peek-poke/peek-poke-derive/ + - gfx/wr/webrender_build/ + - gfx/wr/wr_malloc_size_of/ + - js/src/frontend/smoosh/ + - js/src/rust/shared/ + - modules/libpref/init/static_prefs/ + - mozglue/static/rust/ + - netwerk/base/mozurl/ + - security/manager/ssl/data_storage/ + - servo/components/derive_common/ + - servo/components/selectors/ + - servo/components/servo_arc/ + - servo/components/style/ + - servo/components/style_derive/ + - servo/components/style_traits/ + - servo/components/to_shmem/ + - servo/components/to_shmem_derive/ + - servo/tests/unit/style/ + - testing/geckodriver/ + - testing/mozbase/rust/mozdevice/ + - testing/mozbase/rust/mozprofile/ + - testing/mozbase/rust/mozrunner/ + - testing/mozbase/rust/mozversion/ + - testing/webdriver/ + - third_party/rust/mp4parse/ + - third_party/rust/mp4parse_capi/ + - toolkit/crashreporter/mozannotation_client/ + - toolkit/crashreporter/mozannotation_server/ + - toolkit/components/kvstore/ + - toolkit/components/glean/ + - toolkit/library/rust/ + - tools/fuzzing/rust/ + - tools/profiler/rust-api/ + - xpcom/rust/gtest/bench-collections/ + - xpcom/rust/xpcom/xpcom_macros/ + exclude: + # Many are failing for the same reasons: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1606073 + # https://bugzilla.mozilla.org/show_bug.cgi?id=1606077 + - Cargo.toml + # nsstring + # derive_hash_xor_eq + - gfx/wr/ + - gfx/wr/webrender/ + - gfx/wr/examples/ + # windows-only + - gfx/wr/example-compositor/compositor-windows/ + - gfx/wr/webrender_api/ + - gfx/wr/wrench/ + - gfx/wgpu_bindings/ + # not_unsafe_ptr_arg_deref + - modules/libpref/parser/ + - tools/profiler/rust-helper/ + - toolkit/library/rust/shared/ + - toolkit/library/gtest/rust/ + # not_unsafe_ptr_arg_deref + - remote/ + - dom/media/webrtc/sdp/rsdparsa_capi/ + - intl/encoding_glue/ + # not_unsafe_ptr_arg_deref + - storage/rust/ + - storage/variant/ + # nsstring + - servo/ports/geckolib/tests/ + - xpcom/rust/xpcom/ + - xpcom/rust/nsstring/ + - xpcom/rust/gtest/xpcom/ + - xpcom/rust/gtest/nsstring/ + - security/manager/ssl/cert_storage/ + - intl/locale/rust/fluent-langneg-ffi/ + - intl/locale/rust/unic-langid-ffi/ + - toolkit/components/places/bookmark_sync/ + - xpcom/rust/nserror/ + - xpcom/rust/moz_task/ + - xpcom/rust/gkrust_utils/ + - netwerk/socket/neqo_glue/ + - dom/media/webrtc/transport/mdns_service/ + - tools/lint/test/files/clippy/ + - servo/ports/geckolib/ + - servo/ports/geckolib/tests/ + - servo/tests/unit/malloc_size_of/ + - servo/components/malloc_size_of/ + - dom/media/webrtc/sdp/rsdparsa_capi/ + - testing/geckodriver/marionette/ + - toolkit/components/bitsdownload/bits_client/ + - gfx/wr/example-compositor/compositor/ + - toolkit/components/bitsdownload/bits_client/bits/ + # mac and windows only + - security/manager/ssl/osclientcerts/ + extensions: + - rs + support-files: + - 'tools/lint/clippy/**' + # the version of cargo-clippy is: + # clippy 0.1.65 (2019147 2022-09-19) + # we use the date instead to facilitate the check + # replacing - by . because Python packaging.version.Version expects this + min_clippy_version: 2022.09.19 + type: external + payload: clippy:lint diff --git a/tools/lint/clippy/__init__.py b/tools/lint/clippy/__init__.py new file mode 100644 index 0000000000..1facb246fc --- /dev/null +++ b/tools/lint/clippy/__init__.py @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import bisect +import json +import os +import subprocess +import sys + +from mozlint import result +from mozlint.pathutils import expand_exclusions + + +def in_sorted_list(l, x): + i = bisect.bisect_left(l, x) + return i < len(l) and l[i] == x + + +def handle_clippy_msg(config, line, log, base_path, files): + try: + detail = json.loads(line) + if "message" in detail: + p = detail["target"]["src_path"] + detail = detail["message"] + if "level" in detail: + if ( + detail["level"] == "error" or detail["level"] == "failure-note" + ) and not detail["code"]: + log.debug( + "Error outside of clippy." + "This means that the build failed. Therefore, skipping this" + ) + log.debug("File = {} / Detail = {}".format(p, detail)) + return + # We are in a clippy warning + if len(detail["spans"]) == 0: + # For some reason, at the end of the summary, we can + # get the following line + # {'rendered': 'warning: 5 warnings emitted\n\n', 'children': + # [], 'code': None, 'level': 'warning', 'message': + # '5 warnings emitted', 'spans': []} + # if this is the case, skip it + log.debug( + "Skipping the summary line {} for file {}".format(detail, p) + ) + return + + l = detail["spans"][0] + if not in_sorted_list(files, p): + return + p = os.path.join(base_path, l["file_name"]) + res = { + "path": p, + "level": detail["level"], + "lineno": l["line_start"], + "column": l["column_start"], + "message": detail["message"], + "hint": detail["rendered"], + "rule": detail["code"]["code"], + "lineoffset": l["line_end"] - l["line_start"], + } + return result.from_config(config, **res) + + except json.decoder.JSONDecodeError: + log.debug("Could not parse the output:") + log.debug("clippy output: {}".format(line)) + return + + +def lint(paths, config, fix=None, **lintargs): + files = list(expand_exclusions(paths, config, lintargs["root"])) + files.sort() + log = lintargs["log"] + results = [] + mach_path = lintargs["root"] + "/mach" + march_cargo_process = subprocess.Popen( + [ + sys.executable, + mach_path, + "--log-no-times", + "cargo", + "clippy", + "--", + "--message-format=json", + ], + stdout=subprocess.PIPE, + text=True, + ) + for l in march_cargo_process.stdout: + r = handle_clippy_msg(config, l, log, lintargs["root"], files) + if r is not None: + results.append(r) + march_cargo_process.wait() + + if fix: + log.error("Rust linting in mach does not support --fix") + + return results diff --git a/tools/lint/codespell.yml b/tools/lint/codespell.yml new file mode 100644 index 0000000000..9f821f6296 --- /dev/null +++ b/tools/lint/codespell.yml @@ -0,0 +1,98 @@ +--- +codespell: + description: Check code for common misspellings + include: + - browser/base/content/docs/ + - browser/branding/ + - browser/components/asrouter/docs/ + - browser/components/newtab/docs/ + - browser/components/places/docs/ + - browser/components/search/docs/ + - browser/components/search/schema/ + - browser/components/touchbar/docs/ + - browser/components/urlbar/docs/ + - browser/extensions/formautofill/locales/en-US/ + - browser/extensions/report-site-issue/locales/en-US/ + - browser/installer/windows/docs/ + - browser/locales/en-US/ + - build/docs/ + - devtools/client/locales/en-US/ + - devtools/docs/ + - devtools/shared/locales/en-US/ + - devtools/startup/locales/en-US/ + - docs/ + - dom/docs/ + - dom/locales/en-US/ + - gfx/docs/ + - intl/docs/ + - intl/locales/en-US/ + - ipc/docs/ + - js/src/doc/ + - layout/tools/layout-debug/ui/content/layoutdebug.ftl + - mobile/android/branding/ + - mobile/android/docs/ + - mobile/android/locales/en-US/ + - netwerk/locales/en-US/ + - netwerk/docs/ + - python/docs/ + - python/mach/docs/ + - python/mozlint/ + - python/mozperftest/perfdocs/ + - remote/doc/ + - security/manager/locales/en-US/ + - services/settings/docs/ + - taskcluster/docs/ + - testing/docs/xpcshell/ + - testing/geckodriver/doc/ + - testing/mozbase/docs/ + - testing/raptor/raptor/perfdocs/ + - toolkit/components/extensions/docs/ + - toolkit/components/normandy/docs/ + - toolkit/components/search/docs/ + - toolkit/components/search/schema/ + - toolkit/components/telemetry/docs/ + - toolkit/crashreporter/docs/ + - toolkit/docs/ + - toolkit/locales/en-US/ + - toolkit/modules/docs/ + - tools/code-coverage/docs/ + - tools/fuzzing/docs/ + - tools/lint/ + - tools/moztreedocs/ + - tools/profiler/docs/ + - tools/sanitizer/docs/ + - tools/tryselect/ + - uriloader/docs/ + - xpcom/docs/ + exclude: + - devtools/docs/contributor/tools/storage/ + - tools/lint/cpp/mingw-headers.txt + - tools/lint/test/test_codespell.py + - "**/package-lock.json" + # List of extensions coming from: + # tools/lint/{flake8,eslint}.yml + # tools/mach_commands.py (clang-format) + # + documentation + # + localization files + extensions: + - js + - jsm + - jxs + - mjs + - xml + - html + - xhtml + - cpp + - c + - h + - configure + - py + - properties + - rst + - md + - ftl + support-files: + - 'tools/lint/spell/**' + type: external + setup: spell:setup + payload: spell:lint diff --git a/tools/lint/condprof-addons.yml b/tools/lint/condprof-addons.yml new file mode 100644 index 0000000000..a62c1fb6b9 --- /dev/null +++ b/tools/lint/condprof-addons.yml @@ -0,0 +1,10 @@ +--- +condprof-addons: + description: Lint condprof customizations json files sideloading addons + include: + - 'testing/condprofile/condprof/customization' + exclude: [] + extensions: ['json'] + support-files: ['taskcluster/ci/fetch/browsertime.yml'] + type: structured_log + payload: condprof-addons:lint diff --git a/tools/lint/condprof-addons/__init__.py b/tools/lint/condprof-addons/__init__.py new file mode 100644 index 0000000000..f17ab26f3f --- /dev/null +++ b/tools/lint/condprof-addons/__init__.py @@ -0,0 +1,217 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib +import json +import os +import tarfile +import tempfile +from pathlib import Path + +import requests +import yaml +from mozlint.pathutils import expand_exclusions + +BROWSERTIME_FETCHES_PATH = Path("taskcluster/ci/fetch/browsertime.yml") +CUSTOMIZATIONS_PATH = Path("testing/condprofile/condprof/customization/") +DOWNLOAD_TIMEOUT = 30 +ERR_FETCH_TASK_MISSING = "firefox-addons taskcluster fetch config section not found" +ERR_FETCH_TASK_ADDPREFIX = "firefox-addons taskcluster config 'add-prefix' attribute should be set to 'firefox-addons/'" +ERR_FETCH_TASK_ARCHIVE = ( + "Error downloading or opening archive from firefox-addons taskcluster fetch url" +) +LINTER_NAME = "condprof-addons" +MOZ_FETCHES_DIR = os.environ.get("MOZ_FETCHES_DIR") +RULE_DESC = "condprof addons all listed in firefox-addons.tar fetched archive" +MOZ_AUTOMATION = "MOZ_AUTOMATION" in os.environ + +tempdir = tempfile.gettempdir() + + +def lint(paths, config, logger, fix=None, **lintargs): + filepaths = [Path(p) for p in expand_exclusions(paths, config, lintargs["root"])] + + if len(filepaths) == 0: + return + + linter = CondprofAddonsLinter(topsrcdir=lintargs["root"], logger=logger) + + for filepath in filepaths: + linter.lint(filepath) + + +class CondprofAddonsLinter: + def __init__(self, topsrcdir, logger): + self.topsrcdir = topsrcdir + self.logger = logger + self.BROWSERTIME_FETCHES_FULLPATH = Path( + self.topsrcdir, BROWSERTIME_FETCHES_PATH + ) + self.CUSTOMIZATIONS_FULLPATH = Path(self.topsrcdir, CUSTOMIZATIONS_PATH) + self.tar_xpi_filenames = self.get_firefox_addons_tar_names() + + def lint(self, filepath): + data = self.read_json(filepath) + + if "addons" not in data: + return + + for addon_key in data["addons"]: + xpi_url = data["addons"][addon_key] + xpi_filename = xpi_url.split("/")[-1] + self.logger.info(f"Found addon {xpi_filename}") + if xpi_filename not in self.tar_xpi_filenames: + self.logger.lint_error( + self.get_missing_xpi_msg(xpi_filename), + lineno=0, + column=None, + path=str(filepath), + linter=LINTER_NAME, + rule=RULE_DESC, + ) + + def get_missing_xpi_msg(self, xpi_filename): + return f"{xpi_filename} is missing from the firefox-addons.tar archive" + + def read_json(self, filepath): + with filepath.open("r") as f: + return json.load(f) + + def read_yaml(self, filepath): + with filepath.open("r") as f: + return yaml.safe_load(f) + + def download_firefox_addons_tar(self, firefox_addons_tar_url, tar_tmp_path): + self.logger.info(f"Downloading {firefox_addons_tar_url} to {tar_tmp_path}") + res = requests.get( + firefox_addons_tar_url, stream=True, timeout=DOWNLOAD_TIMEOUT + ) + res.raise_for_status() + with tar_tmp_path.open("wb") as f: + for chunk in res.iter_content(chunk_size=1024): + if chunk is not None: + f.write(chunk) + f.flush() + + def get_firefox_addons_tar_names(self): + # Get firefox-addons fetch task config. + browsertime_fetches = self.read_yaml(self.BROWSERTIME_FETCHES_FULLPATH) + + if not ( + "firefox-addons" in browsertime_fetches + and "fetch" in browsertime_fetches["firefox-addons"] + ): + self.logger.lint_error( + ERR_FETCH_TASK_MISSING, + lineno=0, + column=None, + path=BROWSERTIME_FETCHES_PATH, + linter=LINTER_NAME, + rule=RULE_DESC, + ) + return [] + + fetch_config = browsertime_fetches["firefox-addons"]["fetch"] + + if not ( + "add-prefix" in fetch_config + and fetch_config["add-prefix"] == "firefox-addons/" + ): + self.logger.lint_error( + ERR_FETCH_TASK_ADDPREFIX, + lineno=0, + column=None, + path=BROWSERTIME_FETCHES_PATH, + linter=LINTER_NAME, + rule=RULE_DESC, + ) + return [] + + firefox_addons_tar_url = fetch_config["url"] + firefox_addons_tar_sha256 = fetch_config["sha256"] + + tar_xpi_files = list() + + # When running on the CI, try to retrieve the list of xpi files from the target MOZ_FETCHES_DIR + # subdirectory instead of downloading the archive from the fetch url. + if MOZ_AUTOMATION: + fetches_path = ( + Path(MOZ_FETCHES_DIR) if MOZ_FETCHES_DIR is not None else None + ) + if fetches_path is not None and fetches_path.exists(): + self.logger.info( + "Detected MOZ_FETCHES_DIR, look for pre-downloaded firefox-addons fetch results" + ) + # add-prefix presence and value has been enforced at the start of this method. + fetches_addons = Path(fetches_path, "firefox-addons/") + if fetches_addons.exists(): + self.logger.info( + f"Retrieve list of xpi files from firefox-addons fetch result at {str(fetches_addons)}" + ) + for xpi_path in fetches_addons.iterdir(): + if xpi_path.suffix == ".xpi": + tar_xpi_files.append(xpi_path.name) + return tar_xpi_files + else: + self.logger.warning( + "No 'firefox-addons/' subdir found in MOZ_FETCHES_DIR" + ) + + # Fallback to download the tar archive and retrieve the list of xpi file from it + # (e.g. when linting the local changes on the developers environment). + tar_tmp_path = Path(tempdir, "firefox-addons.tar") + tar_tmp_ready = False + + # If the firefox-addons.tar file is found in the tempdir, check if the + # file hash matches, if it does then don't download it again. + if tar_tmp_path.exists(): + tar_tmp_hash = hashlib.sha256() + with tar_tmp_path.open("rb") as f: + while chunk := f.read(1024): + tar_tmp_hash.update(chunk) + if tar_tmp_hash.hexdigest() == firefox_addons_tar_sha256: + self.logger.info( + f"Pre-downloaded file for {tar_tmp_path} found and sha256 matching" + ) + tar_tmp_ready = True + else: + self.logger.info( + f"{tar_tmp_path} sha256 does not match the fetch config" + ) + + # If the file is not found or the hash doesn't match, download it from the fetch task url. + if not tar_tmp_ready: + try: + self.download_firefox_addons_tar(firefox_addons_tar_url, tar_tmp_path) + except requests.exceptions.HTTPError as http_err: + self.logger.lint_error( + f"{ERR_FETCH_TASK_ARCHIVE}, {str(http_err)}", + lineno=0, + column=None, + path=BROWSERTIME_FETCHES_PATH, + linter=LINTER_NAME, + rule=RULE_DESC, + ) + return [] + + # Retrieve and return the list of xpi file names. + try: + with tarfile.open(tar_tmp_path, "r") as tf: + names = tf.getnames() + for name in names: + file_path = Path(name) + if file_path.suffix == ".xpi": + tar_xpi_files.append(file_path.name) + except tarfile.ReadError as read_err: + self.logger.lint_error( + f"{ERR_FETCH_TASK_ARCHIVE}, {str(read_err)}", + lineno=0, + column=None, + path=BROWSERTIME_FETCHES_PATH, + linter=LINTER_NAME, + rule=RULE_DESC, + ) + return [] + + return tar_xpi_files diff --git a/tools/lint/cpp/__init__.py b/tools/lint/cpp/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/lint/cpp/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/lint/cpp/mingw-capitalization.py b/tools/lint/cpp/mingw-capitalization.py new file mode 100644 index 0000000000..b5a4b07c6a --- /dev/null +++ b/tools/lint/cpp/mingw-capitalization.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re + +from mozlint.types import LineType + +here = os.path.abspath(os.path.dirname(__file__)) +HEADERS_FILE = os.path.join(here, "mingw-headers.txt") +# generated by cd mingw-w64/mingw-w64-headers && +# find . -name "*.h" | xargs -I bob -- basename bob | sort | uniq) + + +class MinGWCapitalization(LineType): + def __init__(self, *args, **kwargs): + super(MinGWCapitalization, self).__init__(*args, **kwargs) + with open(HEADERS_FILE, "r") as fh: + self.headers = fh.read().strip().splitlines() + self.regex = re.compile("^#include\s*<(" + "|".join(self.headers) + ")>") + + def condition(self, payload, line, config): + if not line.startswith("#include"): + return False + + if self.regex.search(line, re.I): + return not self.regex.search(line) + + +def lint(paths, config, **lintargs): + results = [] + + m = MinGWCapitalization() + for path in paths: + results.extend(m._lint(path, config, **lintargs)) + return results diff --git a/tools/lint/cpp/mingw-headers.txt b/tools/lint/cpp/mingw-headers.txt new file mode 100644 index 0000000000..5ee737c393 --- /dev/null +++ b/tools/lint/cpp/mingw-headers.txt @@ -0,0 +1,1452 @@ +accctrl.h +aclapi.h +aclui.h +acpiioct.h +activation.h +activaut.h +activdbg100.h +activdbg.h +activecf.h +activeds.h +activprof.h +activscp.h +adc.h +adhoc.h +admex.h +adoctint.h +adodef.h +adogpool_backcompat.h +adogpool.h +adoguids.h +adoid.h +adoint_backcompat.h +adoint.h +adojet.h +adomd.h +adptif.h +adsdb.h +adserr.h +adshlp.h +adsiid.h +adsnms.h +adsprop.h +adssts.h +adtgen.h +advpub.h +afilter.h +af_irda.h +afxres.h +agtctl.h +agterr.h +agtsvr.h +alg.h +alink.h +amaudio.h +amstream.h +amtvuids.h +amvideo.h +apdevpkey.h +apisetcconv.h +apiset.h +appmgmt.h +aqadmtyp.h +asptlb.h +assert.h +atacct.h +atalkwsh.h +atm.h +atsmedia.h +audevcod.h +audioapotypes.h +audioclient.h +audioendpoints.h +audioengineendpoint.h +audiopolicy.h +audiosessiontypes.h +austream.h +authif.h +authz.h +aux_ulib.h +avifmt.h +aviriff.h +avrfsdk.h +avrt.h +axextendenums.h +azroles.h +basetsd.h +basetyps.h +batclass.h +bcrypt.h +bdaiface_enums.h +bdaiface.h +bdamedia.h +bdasup.h +bdatypes.h +bemapiset.h +bh.h +bidispl.h +bits1_5.h +bits2_0.h +bitscfg.h +bits.h +bitsmsg.h +blberr.h +bluetoothapis.h +_bsd_types.h +bthdef.h +bthsdpdef.h +bugcodes.h +callobj.h +cardmod.h +casetup.h +cchannel.h +cdefs.h +cderr.h +cdoexerr.h +cdoex.h +cdoexm.h +cdoexstr.h +cdonts.h +cdosyserr.h +cdosys.h +cdosysstr.h +celib.h +certadm.h +certbase.h +certbcli.h +certcli.h +certenc.h +certenroll.h +certexit.h +certif.h +certmod.h +certpol.h +certreqd.h +certsrv.h +certview.h +cfg.h +cfgmgr32.h +cguid.h +chanmgr.h +cierror.h +classpnp.h +clfs.h +clfsmgmt.h +clfsmgmtw32.h +clfsw32.h +client.h +cluadmex.h +clusapi.h +cluscfgguids.h +cluscfgserver.h +cluscfgwizard.h +cmdtree.h +cmnquery.h +codecapi.h +colordlg.h +comadmin.h +combaseapi.h +comcat.h +comdef.h +comdefsp.h +comip.h +comlite.h +commapi.h +commctrl.h +commdlg.h +commoncontrols.h +complex.h +compobj.h +compressapi.h +compstui.h +comsvcs.h +comutil.h +confpriv.h +conio.h +conio_s.h +control.h +corecrt_startup.h +corerror.h +corewrappers.h +cor.h +corhdr.h +correg.h +cplext.h +cpl.h +credssp.h +crtdbg.h +crtdbg_s.h +crtdefs.h +cryptuiapi.h +cryptxml.h +cscapi.h +cscobj.h +csq.h +ctfutb.h +ctxtcall.h +ctype.h +custcntl.h +_cygwin.h +d2d1_1.h +d2d1_1helper.h +d2d1effectauthor.h +d2d1effecthelpers.h +d2d1effects.h +d2d1.h +d2d1helper.h +d2dbasetypes.h +d2derr.h +d3d10_1.h +d3d10_1shader.h +d3d10effect.h +d3d10.h +d3d10misc.h +d3d10sdklayers.h +d3d10shader.h +d3d11_1.h +d3d11_2.h +d3d11_3.h +d3d11_4.h +d3d11.h +d3d11sdklayers.h +d3d11shader.h +d3d8caps.h +d3d8.h +d3d8types.h +d3d9caps.h +d3d9.h +d3d9types.h +d3dcaps.h +d3dcommon.h +d3dcompiler.h +d3d.h +d3dhalex.h +d3dhal.h +d3dnthal.h +d3drmdef.h +d3drm.h +d3drmobj.h +d3dtypes.h +d3dx9anim.h +d3dx9core.h +d3dx9effect.h +d3dx9.h +d3dx9math.h +d3dx9mesh.h +d3dx9shader.h +d3dx9shape.h +d3dx9tex.h +d3dx9xof.h +d4drvif.h +d4iface.h +daogetrw.h +datapath.h +datetimeapi.h +davclnt.h +dbdaoerr.h +_dbdao.h +dbdaoid.h +dbdaoint.h +dbgautoattach.h +_dbg_common.h +dbgeng.h +dbghelp.h +_dbg_LOAD_IMAGE.h +dbgprop.h +dbt.h +dciddi.h +dciman.h +dcommon.h +dcompanimation.h +dcomp.h +dcomptypes.h +dde.h +ddeml.h +dderror.h +ddkernel.h +ddkmapi.h +ddrawgdi.h +ddraw.h +ddrawi.h +ddrawint.h +ddstream.h +debugapi.h +delayimp.h +devguid.h +devicetopology.h +devioctl.h +devpkey.h +devpropdef.h +dhcpcsdk.h +dhcpsapi.h +dhcpssdk.h +dhcpv6csdk.h +dhtmldid.h +dhtmled.h +dhtmliid.h +digitalv.h +dimm.h +dinput.h +direct.h +dirent.h +dir.h +diskguid.h +dispatch.h +dispdib.h +dispex.h +dlcapi.h +dlgs.h +dls1.h +dls2.h +dmdls.h +dmemmgr.h +dmerror.h +dmksctrl.h +dmodshow.h +dmo.h +dmoreg.h +dmort.h +dmplugin.h +dmusbuff.h +dmusicc.h +dmusicf.h +dmusici.h +dmusicks.h +dmusics.h +docobjectservice.h +docobj.h +documenttarget.h +domdid.h +dos.h +downloadmgr.h +dpaddr.h +dpapi.h +dpfilter.h +dplay8.h +dplay.h +dplobby8.h +dplobby.h +dpnathlp.h +driverspecs.h +drivinit.h +drmexternals.h +drmk.h +dsadmin.h +dsclient.h +dsconf.h +dsdriver.h +dsgetdc.h +dshow.h +dskquota.h +dsound.h +dsquery.h +dsrole.h +dssec.h +dtchelp.h +dvbsiparser.h +dvdevcod.h +dvdmedia.h +dvec.h +dvobj.h +dvp.h +dwmapi.h +dwrite_1.h +dwrite_2.h +dwrite_3.h +dwrite.h +dxapi.h +dxdiag.h +dxerr8.h +dxerr9.h +dxfile.h +dxgi1_2.h +dxgi1_3.h +dxgi1_4.h +dxgi1_5.h +dxgicommon.h +dxgiformat.h +dxgi.h +dxgitype.h +dxtmpl.h +dxva2api.h +dxva.h +dxvahd.h +eapauthenticatoractiondefine.h +eapauthenticatortypes.h +eaphosterror.h +eaphostpeerconfigapis.h +eaphostpeertypes.h +eapmethodauthenticatorapis.h +eapmethodpeerapis.h +eapmethodtypes.h +eappapis.h +eaptypes.h +edevdefs.h +eh.h +ehstorapi.h +elscore.h +emostore.h +emptyvc.h +endpointvolume.h +errhandlingapi.h +errno.h +error.h +errorrep.h +errors.h +esent.h +evcode.h +evcoll.h +eventsys.h +evntcons.h +evntprov.h +evntrace.h +evr9.h +evr.h +exchform.h +excpt.h +exdisp.h +exdispid.h +fci.h +fcntl.h +fdi.h +_fd_types.h +fenv.h +fibersapi.h +fileapi.h +fileextd.h +file.h +filehc.h +filter.h +filterr.h +float.h +fltdefs.h +fltsafe.h +fltuser.h +fltuserstructures.h +fltwinerror.h +fpieee.h +fsrmenums.h +fsrmerr.h +fsrm.h +fsrmpipeline.h +fsrmquota.h +fsrmreports.h +fsrmscreen.h +ftsiface.h +ftw.h +functiondiscoveryapi.h +functiondiscoverycategories.h +functiondiscoveryconstraints.h +functiondiscoverykeys_devpkey.h +functiondiscoverykeys.h +functiondiscoverynotification.h +fusion.h +fvec.h +fwpmtypes.h +fwpmu.h +fwptypes.h +gb18030.h +gdiplusbase.h +gdiplusbrush.h +gdipluscolor.h +gdipluscolormatrix.h +gdipluseffects.h +gdiplusenums.h +gdiplusflat.h +gdiplusgpstubs.h +gdiplusgraphics.h +gdiplus.h +gdiplusheaders.h +gdiplusimageattributes.h +gdiplusimagecodec.h +gdiplusimaging.h +gdiplusimpl.h +gdiplusinit.h +gdipluslinecaps.h +gdiplusmatrix.h +gdiplusmem.h +gdiplusmetafile.h +gdiplusmetaheader.h +gdipluspath.h +gdipluspen.h +gdipluspixelformats.h +gdiplusstringformat.h +gdiplustypes.h +getopt.h +glaux.h +glcorearb.h +glext.h +gl.h +glu.h +glxext.h +gpedit.h +gpio.h +gpmgmt.h +guiddef.h +h323priv.h +handleapi.h +heapapi.h +hidclass.h +hidpi.h +hidsdi.h +hidusage.h +highlevelmonitorconfigurationapi.h +hlguids.h +hliface.h +hlink.h +hostinfo.h +hstring.h +htiface.h +htiframe.h +htmlguid.h +htmlhelp.h +httpext.h +httpfilt.h +http.h +httprequestid.h +hubbusif.h +ia64reg.h +iaccess.h +iadmext.h +iadmw.h +iads.h +icftypes.h +icm.h +icmpapi.h +icodecapi.h +icrsint.h +i_cryptasn1tls.h +ide.h +identitycommon.h +identitystore.h +idf.h +idispids.h +iedial.h +ieeefp.h +ieverp.h +ifdef.h +iiisext.h +iiis.h +iimgctx.h +iiscnfg.h +iisrsta.h +iketypes.h +imagehlp.h +ime.h +imessage.h +imm.h +in6addr.h +inaddr.h +indexsrv.h +inetreg.h +inetsdk.h +infstr.h +initguid.h +initoid.h +inputscope.h +inspectable.h +interlockedapi.h +internal.h +intrin.h +intrin-impl.h +intsafe.h +intshcut.h +inttypes.h +invkprxy.h +ioaccess.h +ioapiset.h +ioevent.h +io.h +ipexport.h +iphlpapi.h +ipifcons.h +ipinfoid.h +ipmib.h +_ip_mreq1.h +ipmsp.h +iprtrmib.h +ipsectypes.h +_ip_types.h +iptypes.h +ipxconst.h +ipxrip.h +ipxrtdef.h +ipxsap.h +ipxtfflt.h +iscsidsc.h +isguids.h +issper16.h +issperr.h +isysmon.h +ivec.h +iwamreg.h +jobapi.h +kbdmou.h +kcom.h +knownfolders.h +ksdebug.h +ksguid.h +ks.h +ksmedia.h +ksproxy.h +ksuuids.h +ktmtypes.h +ktmw32.h +kxia64.h +l2cmn.h +libgen.h +libloaderapi.h +limits.h +lmaccess.h +lmalert.h +lmapibuf.h +lmat.h +lmaudit.h +lmconfig.h +lmcons.h +lmdfs.h +lmerr.h +lmerrlog.h +lm.h +lmjoin.h +lmmsg.h +lmon.h +lmremutl.h +lmrepl.h +lmserver.h +lmshare.h +lmsname.h +lmstats.h +lmsvc.h +lmuseflg.h +lmuse.h +lmwksta.h +loadperf.h +locale.h +locationapi.h +locking.h +lpmapi.h +lzexpand.h +madcapcl.h +magnification.h +mailmsgprops.h +malloc.h +manipulations.h +mapicode.h +mapidbg.h +mapidefs.h +mapiform.h +mapiguid.h +mapi.h +mapihook.h +mapinls.h +mapioid.h +mapispi.h +mapitags.h +mapiutil.h +mapival.h +mapiwin.h +mapiwz.h +mapix.h +math.h +mbctype.h +mbstring.h +mbstring_s.h +mcd.h +mce.h +mciavi.h +mcx.h +mdcommsg.h +mddefw.h +mdhcp.h +mdmsg.h +mediaerr.h +mediaobj.h +medparam.h +mem.h +memoryapi.h +memory.h +mergemod.h +mfapi.h +mferror.h +mfidl.h +mfmp2dlna.h +mfobjects.h +mfplay.h +mfreadwrite.h +mftransform.h +mgm.h +mgmtapi.h +midles.h +mimedisp.h +mimeinfo.h +_mingw_ddk.h +_mingw_directx.h +_mingw_dxhelper.h +_mingw_mac.h +_mingw_off_t.h +_mingw_print_pop.h +_mingw_print_push.h +_mingw_secapi.h +_mingw_stat64.h +_mingw_stdarg.h +_mingw_unicode.h +miniport.h +minitape.h +minmax.h +minwinbase.h +minwindef.h +mlang.h +mmc.h +mmcobj.h +mmdeviceapi.h +mmreg.h +mmstream.h +mmsystem.h +mobsync.h +module.h +moniker.h +mountdev.h +mountmgr.h +mpeg2bits.h +mpeg2data.h +mpeg2psiparser.h +mpeg2structs.h +mprapi.h +mprerror.h +mq.h +mqmail.h +mqoai.h +msacmdlg.h +msacm.h +msado15.h +msasn1.h +msber.h +mscat.h +mschapp.h +msclus.h +mscoree.h +msctf.h +msctfmonitorapi.h +msdadc.h +msdaguid.h +msdaipper.h +msdaipp.h +msdaora.h +msdaosp.h +msdasc.h +msdasql.h +msdatsrc.h +msdrmdefs.h +msdrm.h +msdshape.h +msfs.h +mshtmcid.h +mshtmdid.h +mshtmhst.h +mshtmlc.h +mshtml.h +msidefs.h +msi.h +msimcntl.h +msimcsdk.h +msinkaut.h +msiquery.h +msoav.h +msopc.h +mspab.h +mspaddr.h +mspbase.h +mspcall.h +mspcoll.h +mspenum.h +msp.h +msplog.h +msports.h +mspst.h +mspstrm.h +mspterm.h +mspthrd.h +msptrmac.h +msptrmar.h +msptrmvc.h +msputils.h +msrdc.h +msremote.h +mssip.h +msstkppg.h +mstask.h +mstcpip.h +msterr.h +mswsock.h +msxml2did.h +msxml2.h +msxmldid.h +msxml.h +mtsadmin.h +mtsevents.h +mtsgrp.h +mtxadmin.h +mtxattr.h +mtxdm.h +mtx.h +muiload.h +multimon.h +multinfo.h +mxdc.h +namedpipeapi.h +namespaceapi.h +napcertrelyingparty.h +napcommon.h +napenforcementclient.h +napmanagement.h +napmicrosoftvendorids.h +napprotocol.h +napservermanagement.h +napsystemhealthagent.h +napsystemhealthvalidator.h +naptypes.h +naputil.h +nb30.h +ncrypt.h +ndattrib.h +ndfapi.h +ndhelper.h +ndisguid.h +ndis.h +ndistapi.h +ndiswan.h +ndkinfo.h +ndr64types.h +ndrtypes.h +netcon.h +neterr.h +netevent.h +netfw.h +netioapi.h +netlistmgr.h +netmon.h +netpnp.h +netprov.h +nettypes.h +newapis.h +newdev.h +new.h +nldef.h +nmsupp.h +npapi.h +nsemail.h +nspapi.h +ntagp.h +ntdd1394.h +ntdd8042.h +ntddbeep.h +ntddcdrm.h +ntddcdvd.h +ntddchgr.h +ntdddisk.h +ntddft.h +ntddkbd.h +ntddk.h +ntddmmc.h +ntddmodm.h +ntddmou.h +ntddndis.h +ntddpar.h +ntddpcm.h +ntddpsch.h +ntddscsi.h +ntddser.h +ntddsnd.h +ntddstor.h +ntddtape.h +ntddtdi.h +ntddvdeo.h +ntddvol.h +ntdef.h +ntdsapi.h +ntdsbcli.h +ntdsbmsg.h +ntgdi.h +ntifs.h +ntimage.h +ntiologc.h +ntldap.h +ntmsapi.h +ntmsmli.h +ntnls.h +ntpoapi.h +ntquery.h +ntsdexts.h +ntsecapi.h +ntsecpkg.h +ntstatus.h +ntstrsafe.h +ntverp.h +oaidl.h +objbase.h +objectarray.h +objerror.h +objidlbase.h +objidl.h +objsafe.h +objsel.h +ocidl.h +ocmm.h +odbcinst.h +odbcss.h +ole2.h +ole2ver.h +oleacc.h +oleauto.h +olectl.h +olectlid.h +oledbdep.h +oledberr.h +oledbguid.h +oledb.h +oledlg.h +ole.h +oleidl.h +oletx2xa.h +opmapi.h +oprghdlr.h +optary.h +p2p.h +packoff.h +packon.h +parallel.h +param.h +parser.h +patchapi.h +patchwiz.h +pathcch.h +pbt.h +pchannel.h +pciprop.h +pcrt32.h +pdh.h +pdhmsg.h +penwin.h +perflib.h +perhist.h +persist.h +pfhook.h +pgobootrun.h +physicalmonitorenumerationapi.h +pla.h +pnrpdef.h +pnrpns.h +poclass.h +polarity.h +_pop_BOOL.h +poppack.h +portabledeviceconnectapi.h +portabledevicetypes.h +portcls.h +powrprof.h +prnasnot.h +prntfont.h +processenv.h +process.h +processthreadsapi.h +processtopologyapi.h +profileapi.h +profile.h +profinfo.h +propidl.h +propkeydef.h +propkey.h +propsys.h +propvarutil.h +prsht.h +psapi.h +pshpack1.h +pshpack2.h +pshpack4.h +pshpack8.h +pshpck16.h +pstore.h +pthread_signal.h +pthread_time.h +pthread_unistd.h +punknown.h +_push_BOOL.h +qedit.h +qmgr.h +qnetwork.h +qos2.h +qos.h +qosname.h +qospol.h +qossp.h +rasdlg.h +raseapif.h +raserror.h +ras.h +rassapi.h +rasshost.h +ratings.h +rdpencomapi.h +realtimeapiset.h +reason.h +recguids.h +reconcil.h +regbag.h +regstr.h +rend.h +resapi.h +restartmanager.h +richedit.h +richole.h +rkeysvcc.h +rnderr.h +roapi.h +routprot.h +rpcasync.h +rpcdce.h +rpcdcep.h +rpc.h +rpcndr.h +rpcnsi.h +rpcnsip.h +rpcnterr.h +rpcproxy.h +rpcsal.h +rpcssl.h +rrascfg.h +rtcapi.h +rtccore.h +rtcerr.h +rtinfo.h +rtm.h +rtmv2.h +rtutils.h +sal.h +sapi51.h +sapi53.h +sapi54.h +sapi.h +sas.h +sbe.h +scarddat.h +scarderr.h +scardmgr.h +scardsrv.h +scardssp.h +scesvc.h +schannel.h +schedule.h +schemadef.h +schnlsp.h +scode.h +scrnsave.h +scrptids.h +scsi.h +scsiscan.h +scsiwmi.h +sddl.h +sdkddkver.h +sdoias.h +sdpblb.h +sdperr.h +search.h +search_s.h +secext.h +securityappcontainer.h +securitybaseapi.h +security.h +sehmap.h +sensapi.h +sensevts.h +sens.h +sensorsapi.h +sensors.h +servprov.h +setjmpex.h +setjmp.h +setupapi.h +sfc.h +shappmgr.h +share.h +shdeprecated.h +shdispid.h +shellapi.h +sherrors.h +shfolder.h +shldisp.h +shlguid.h +shlobj.h +shlwapi.h +shobjidl.h +shtypes.h +signal.h +simpdata.h +simpdc.h +sipbase.h +sisbkup.h +slerror.h +slpublic.h +smbus.h +smpab.h +smpms.h +smpxp.h +smtpguid.h +smx.h +snmp.h +_socket_types.h +softpub.h +specstrings.h +sperror.h +sphelper.h +sporder.h +sql_1.h +sqlext.h +sql.h +sqloledb.h +sqltypes.h +sqlucode.h +srb.h +srrestoreptapi.h +srv.h +sspguid.h +sspi.h +sspserr.h +sspsidl.h +stat.h +stdarg.h +stddef.h +stdexcpt.h +stdint.h +stdio.h +stdio_s.h +stdlib.h +stdlib_s.h +stdunk.h +stierr.h +sti.h +stireg.h +stllock.h +stm.h +storage.h +storduid.h +storport.h +storprop.h +stralign.h +stralign_s.h +stringapiset.h +string.h +string_s.h +strings.h +strmif.h +strmini.h +strsafe.h +structuredquerycondition.h +subauth.h +subsmgr.h +svcguid.h +svrapi.h +swenum.h +synchapi.h +sysinfoapi.h +syslimits.h +systemtopologyapi.h +t2embapi.h +tabflicks.h +tapi3cc.h +tapi3ds.h +tapi3err.h +tapi3.h +tapi3if.h +tapi.h +taskschd.h +tbs.h +tcerror.h +tcguid.h +tchar.h +tchar_s.h +tcpestats.h +tcpmib.h +tdh.h +tdi.h +tdiinfo.h +tdikrnl.h +tdistat.h +termmgr.h +textserv.h +textstor.h +threadpoolapiset.h +threadpoollegacyapiset.h +timeb.h +timeb_s.h +time.h +timeprov.h +_timeval.h +timezoneapi.h +tlbref.h +tlhelp32.h +tlogstg.h +tmschema.h +tnef.h +tom.h +tpcshrd.h +traffic.h +transact.h +triedcid.h +triediid.h +triedit.h +tsattrs.h +tspi.h +tssbx.h +tsuserex.h +tuner.h +tvout.h +txcoord.h +txctx.h +txdtc.h +txfw32.h +typeinfo.h +types.h +uastrfnc.h +uchar.h +udpmib.h +uianimation.h +uiautomationclient.h +uiautomationcoreapi.h +uiautomationcore.h +uiautomation.h +uiviewsettingsinterop.h +umx.h +unistd.h +unknown.h +unknwnbase.h +unknwn.h +upssvc.h +urlhist.h +urlmon.h +usb100.h +usb200.h +usbbusif.h +usbcamdi.h +usbdi.h +usbdlib.h +usbdrivr.h +usb.h +usbioctl.h +usbiodef.h +usbkern.h +usbprint.h +usbprotocoldefs.h +usbrpmif.h +usbscan.h +usbspec.h +usbstorioctl.h +usbuser.h +userenv.h +usp10.h +utilapiset.h +utime.h +uuids.h +uxtheme.h +vadefs.h +varargs.h +_varenum.h +vcr.h +vdmdbg.h +vds.h +vdslun.h +versionhelpers.h +vfw.h +vfwmsgs.h +videoagp.h +video.h +virtdisk.h +vmr9.h +vsadmin.h +vsbackup.h +vsmgmt.h +vsprov.h +vss.h +vsstyle.h +vssym32.h +vswriter.h +w32api.h +wabapi.h +wabcode.h +wabdefs.h +wab.h +wabiab.h +wabmem.h +wabnot.h +wabtags.h +wabutil.h +wbemads.h +wbemcli.h +wbemdisp.h +wbemidl.h +wbemprov.h +wbemtran.h +wchar.h +wchar_s.h +wcmconfig.h +wcsplugin.h +wct.h +wctype.h +wdmguid.h +wdm.h +wdsbp.h +wdsclientapi.h +wdspxe.h +wdstci.h +wdstpdi.h +wdstptmgmt.h +werapi.h +wfext.h +wglext.h +wiadef.h +wiadevd.h +wia.h +wiavideo.h +winable.h +winapifamily.h +winbase.h +winber.h +wincodec.h +wincon.h +wincred.h +wincrypt.h +winddi.h +winddiui.h +windef.h +windns.h +windot11.h +windows.foundation.h +windows.h +windows.security.cryptography.h +windows.storage.h +windows.storage.streams.h +windows.system.threading.h +windowsx.h +winefs.h +winerror.h +winevt.h +wingdi.h +winhttp.h +wininet.h +winineti.h +winioctl.h +winldap.h +winnetwk.h +winnls32.h +winnls.h +winnt.h +winperf.h +winreg.h +winresrc.h +winsafer.h +winsatcominterfacei.h +winscard.h +winsdkver.h +winsmcrd.h +winsnmp.h +winsock2.h +winsock.h +winsplp.h +winspool.h +winstring.h +winsvc.h +winsxs.h +winsync.h +winternl.h +wintrust.h +winusb.h +winusbio.h +winuser.h +winver.h +winwlx.h +wlanapi.h +wlanihvtypes.h +wlantypes.h +wmcodecdsp.h +wmcontainer.h +wmdrmsdk.h +wmiatlprov.h +wmidata.h +wmilib.h +wmistr.h +wmiutils.h +wmsbuffer.h +wmsdkidl.h +wnnc.h +wow64apiset.h +wownt16.h +wownt32.h +wpapi.h +wpapimsg.h +wpcapi.h +wpcevent.h +wpcrsmsg.h +wpftpmsg.h +wppstmsg.h +wpspihlp.h +wptypes.h +wpwizmsg.h +wrl.h +_ws1_undef.h +ws2atm.h +ws2bth.h +ws2def.h +ws2dnet.h +ws2ipdef.h +ws2san.h +ws2spi.h +ws2tcpip.h +_wsadata.h +_wsa_errnos.h +wsdapi.h +wsdattachment.h +wsdbase.h +wsdclient.h +wsddisco.h +wsdhost.h +wsdtypes.h +wsdutil.h +wsdxmldom.h +wsdxml.h +wshisotp.h +wsipv6ok.h +wsipx.h +wsmandisp.h +wsman.h +wsnetbs.h +wsnwlink.h +wspiapi.h +wsrm.h +wsvns.h +wtsapi32.h +wtypesbase.h +wtypes.h +xa.h +xcmcext.h +xcmc.h +xcmcmsx2.h +xcmcmsxt.h +xenroll.h +xfilter.h +xinput.h +xlocinfo.h +xmath.h +_xmitfile.h +xmldomdid.h +xmldsodid.h +xmllite.h +xmltrnsf.h +xolehlp.h +xpsdigitalsignature.h +xpsobjectmodel_1.h +xpsobjectmodel.h +xpsprint.h +xpsrassvc.h +ymath.h +yvals.h +zmouse.h diff --git a/tools/lint/eslint.yml b/tools/lint/eslint.yml new file mode 100644 index 0000000000..b54d328e05 --- /dev/null +++ b/tools/lint/eslint.yml @@ -0,0 +1,31 @@ +--- +eslint: + description: JavaScript linter + # ESLint infra handles its own path filtering, so just include cwd + include: ['.'] + exclude: [] + # When adding to this list, consider updating hooks_js_format.py as well. + extensions: ['mjs', 'js', 'jsm', 'json', 'jsx', 'html', 'sjs', 'xhtml'] + support-files: + - '**/.eslintrc.js' + - '.eslintrc-test-paths.js' + - '.eslintignore' + - 'tools/lint/eslint/**' + # Files that can influence global variables + - 'browser/base/content/nsContextMenu.js' + - 'browser/base/content/utilityOverlay.js' + - 'browser/components/customizableui/content/panelUI.js' + - 'browser/components/downloads/content/downloads.js' + - 'browser/components/downloads/content/indicator.js' + - 'testing/mochitest/tests/SimpleTest/EventUtils.js' + - 'testing/mochitest/tests/SimpleTest/MockObjects.js' + - 'testing/mochitest/tests/SimpleTest/SimpleTest.js' + - 'testing/mochitest/tests/SimpleTest/WindowSnapshot.js' + - 'toolkit/components/printing/content/printUtils.js' + - 'toolkit/components/viewsource/content/viewSourceUtils.js' + - 'toolkit/content/contentAreaUtils.js' + - 'toolkit/content/editMenuOverlay.js' + - 'toolkit/content/globalOverlay.js' + type: external + payload: eslint:lint + setup: eslint:setup diff --git a/tools/lint/eslint/.eslintrc.js b/tools/lint/eslint/.eslintrc.js new file mode 100644 index 0000000000..762ddba02c --- /dev/null +++ b/tools/lint/eslint/.eslintrc.js @@ -0,0 +1,31 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + plugins: ["eslint-plugin"], + extends: ["plugin:eslint-plugin/recommended"], + // eslint-plugin-mozilla runs under node, so we need a more restrictive + // environment / parser setup here than the rest of mozilla-central. + env: { + browser: false, + node: true, + }, + parser: "espree", + parserOptions: { + // This should match with the minimum node version that the ESLint CI + // process uses (check the linux64-node toolchain). + ecmaVersion: 12, + }, + + rules: { + camelcase: ["error", { properties: "never" }], + "handle-callback-err": ["error", "er"], + "no-shadow": "error", + "no-undef-init": "error", + "one-var": ["error", "never"], + strict: ["error", "global"], + }, +}; diff --git a/tools/lint/eslint/__init__.py b/tools/lint/eslint/__init__.py new file mode 100644 index 0000000000..fdd7504c34 --- /dev/null +++ b/tools/lint/eslint/__init__.py @@ -0,0 +1,293 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import signal +import subprocess +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "eslint")) +from mozbuild.nodeutil import find_node_executable +from mozlint import result + +from eslint import setup_helper + +ESLINT_ERROR_MESSAGE = """ +An error occurred running eslint. Please check the following error messages: + +{} +""".strip() + +ESLINT_NOT_FOUND_MESSAGE = """ +Could not find eslint! We looked at the --binary option, at the ESLINT +environment variable, and then at your local node_modules path. Please Install +eslint and needed plugins with: + +mach eslint --setup + +and try again. +""".strip() + +PRETTIER_ERROR_MESSAGE = """ +An error occurred running prettier. Please check the following error messages: + +{} +""".strip() + +PRETTIER_FORMATTING_MESSAGE = ( + "This file needs formatting with Prettier (use 'mach lint --fix <path>')." +) + + +def setup(root, **lintargs): + setup_helper.set_project_root(root) + + if not setup_helper.check_node_executables_valid(): + return 1 + + return setup_helper.eslint_maybe_setup() + + +def lint(paths, config, binary=None, fix=None, rules=[], setup=None, **lintargs): + """Run eslint.""" + log = lintargs["log"] + setup_helper.set_project_root(lintargs["root"]) + module_path = setup_helper.get_project_root() + + # Valid binaries are: + # - Any provided by the binary argument. + # - Any pointed at by the ESLINT environmental variable. + # - Those provided by |mach lint --setup|. + + if not binary: + binary, _ = find_node_executable() + + if not binary: + print(ESLINT_NOT_FOUND_MESSAGE) + return 1 + + extra_args = lintargs.get("extra_args") or [] + exclude_args = [] + for path in config.get("exclude", []): + exclude_args.extend( + ["--ignore-pattern", os.path.relpath(path, lintargs["root"])] + ) + + for rule in rules: + extra_args.extend(["--rule", rule]) + + # First run ESLint + cmd_args = ( + [ + binary, + os.path.join(module_path, "node_modules", "eslint", "bin", "eslint.js"), + # This keeps ext as a single argument. + "--ext", + "[{}]".format(",".join(config["extensions"])), + "--format", + "json", + "--no-error-on-unmatched-pattern", + ] + + rules + + extra_args + + exclude_args + + paths + ) + + if fix: + # eslint requires that --fix be set before the --ext argument. + cmd_args.insert(2, "--fix") + + log.debug("ESLint command: {}".format(" ".join(cmd_args))) + + result = run(cmd_args, config) + if result == 1: + return result + + # Then run Prettier + cmd_args = ( + [ + binary, + os.path.join(module_path, "node_modules", "prettier", "bin-prettier.js"), + "--list-different", + "--no-error-on-unmatched-pattern", + ] + + extra_args + # Prettier does not support exclude arguments. + # + exclude_args + + paths + ) + log.debug("Prettier command: {}".format(" ".join(cmd_args))) + + if fix: + cmd_args.append("--write") + + prettier_result = run_prettier(cmd_args, config, fix) + if prettier_result == 1: + return prettier_result + + result["results"].extend(prettier_result["results"]) + result["fixed"] = result["fixed"] + prettier_result["fixed"] + return result + + +def run(cmd_args, config): + shell = False + if ( + os.environ.get("MSYSTEM") in ("MINGW32", "MINGW64") + or "MOZILLABUILD" in os.environ + ): + # The eslint binary needs to be run from a shell with msys + shell = True + encoding = "utf-8" + + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = subprocess.Popen( + cmd_args, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + signal.signal(signal.SIGINT, orig) + + try: + output, errors = proc.communicate() + except KeyboardInterrupt: + proc.kill() + return {"results": [], "fixed": 0} + + if errors: + errors = errors.decode(encoding, "replace") + print(ESLINT_ERROR_MESSAGE.format(errors)) + + if proc.returncode >= 2: + return 1 + + if not output: + return {"results": [], "fixed": 0} # no output means success + output = output.decode(encoding, "replace") + try: + jsonresult = json.loads(output) + except ValueError: + print(ESLINT_ERROR_MESSAGE.format(output)) + return 1 + + results = [] + fixed = 0 + for obj in jsonresult: + errors = obj["messages"] + # This will return a count of files fixed, rather than issues fixed, as + # that is the only count we have. + if "output" in obj: + fixed = fixed + 1 + + for err in errors: + err.update( + { + "hint": err.get("fix"), + "level": "error" if err["severity"] == 2 else "warning", + "lineno": err.get("line") or 0, + "path": obj["filePath"], + "rule": err.get("ruleId"), + } + ) + results.append(result.from_config(config, **err)) + + return {"results": results, "fixed": fixed} + + +def run_prettier(cmd_args, config, fix): + shell = False + if is_windows(): + # The eslint binary needs to be run from a shell with msys + shell = True + encoding = "utf-8" + + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = subprocess.Popen( + cmd_args, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + signal.signal(signal.SIGINT, orig) + + try: + output, errors = proc.communicate() + except KeyboardInterrupt: + proc.kill() + return {"results": [], "fixed": 0} + + results = [] + + if errors: + errors = errors.decode(encoding, "replace").strip().split("\n") + errors = [ + error + for error in errors + # Unknown options are not an issue for Prettier, this avoids + # errors during tests. + if not ("Ignored unknown option" in error) + ] + if len(errors): + results.append( + result.from_config( + config, + **{ + "name": "eslint", + "path": os.path.abspath("."), + "message": PRETTIER_ERROR_MESSAGE.format("\n".join(errors)), + "level": "error", + "rule": "prettier", + "lineno": 0, + "column": 0, + } + ) + ) + + if not output: + # If we have errors, but no output, we assume something really bad happened. + if errors and len(errors): + return {"results": results, "fixed": 0} + + return {"results": [], "fixed": 0} # no output means success + + output = output.decode(encoding, "replace").splitlines() + + fixed = 0 + + if fix: + # When Prettier is running in fix mode, it outputs the list of files + # that have been fixed, so sum them up here. + # If it can't fix files, it will throw an error, which will be handled + # above. + fixed = len(output) + else: + # When in "check" mode, Prettier will output the list of files that + # need changing, so we'll wrap them in our result structure here. + for file in output: + if not file: + continue + + file = os.path.abspath(file) + results.append( + result.from_config( + config, + **{ + "name": "eslint", + "path": file, + "message": PRETTIER_FORMATTING_MESSAGE, + "level": "error", + "rule": "prettier", + "lineno": 0, + "column": 0, + } + ) + ) + + return {"results": results, "fixed": fixed} + + +def is_windows(): + return ( + os.environ.get("MSYSTEM") in ("MINGW32", "MINGW64") + or "MOZILLABUILD" in os.environ + ) diff --git a/tools/lint/eslint/eslint-plugin-mozilla/.npmignore b/tools/lint/eslint/eslint-plugin-mozilla/.npmignore new file mode 100644 index 0000000000..3713448c7a --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/.npmignore @@ -0,0 +1,8 @@ +.eslintrc.js +.npmignore +node_modules +reporters +scripts +tests +package-lock.json +update.sh diff --git a/tools/lint/eslint/eslint-plugin-mozilla/LICENSE b/tools/lint/eslint/eslint-plugin-mozilla/LICENSE new file mode 100644 index 0000000000..e87a115e46 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/tools/lint/eslint/eslint-plugin-mozilla/README.md b/tools/lint/eslint/eslint-plugin-mozilla/README.md new file mode 100644 index 0000000000..650507754e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/README.md @@ -0,0 +1,56 @@ +# eslint-plugin-mozilla + +A collection of rules that help enforce JavaScript coding standard in the Mozilla project. + +These are primarily developed and used within the Firefox build system ([mozilla-central](https://hg.mozilla.org/mozilla-central/)), but are made available for other +related projects to use as well. + +## Installation + +### Within mozilla-central: + +``` +$ ./mach eslint --setup +``` + +### Outside mozilla-central: + +Install ESLint [ESLint](http://eslint.org): + +``` +$ npm i eslint --save-dev +``` + +Next, install `eslint-plugin-mozilla`: + +``` +$ npm install eslint-plugin-mozilla --save-dev +``` + +## Documentation + +For details about the rules, please see the [firefox documentation page](http://firefox-source-docs.mozilla.org/tools/lint/linters/eslint-plugin-mozilla.html). + +## Source Code + +The sources can be found at: + +* Code: https://searchfox.org/mozilla-central/source/tools/lint/eslint/eslint-plugin-mozilla +* Documentation: https://searchfox.org/mozilla-central/source/docs/code-quality/lint/linters + +## Bugs + +Please file bugs in Bugzilla in the Lint component of the Testing product. + +* [Existing bugs](https://bugzilla.mozilla.org/buglist.cgi?resolution=---&query_format=advanced&component=Lint&product=Testing) +* [New bugs](https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=Lint) + +## Tests + +The tests can only be run from within mozilla-central. To run the tests: + +``` +./mach eslint --setup +cd tools/lint/eslint/eslint-plugin-mozilla +npm run test +``` diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/.eslintrc.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/.eslintrc.js new file mode 100644 index 0000000000..76df4134f5 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/.eslintrc.js @@ -0,0 +1,8 @@ +"use strict"; + +module.exports = { + rules: { + // Require object keys to be sorted. + "sort-keys": "error", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/browser-test.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/browser-test.js new file mode 100644 index 0000000000..08747a3f88 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/browser-test.js @@ -0,0 +1,95 @@ +// Parent config file for all browser-chrome files. +"use strict"; + +module.exports = { + env: { + browser: true, + "mozilla/browser-window": true, + "mozilla/simpletest": true, + // "node": true + }, + + // All globals made available in the test environment. + globals: { + // `$` is defined in SimpleTest.js + $: false, + Assert: false, + BrowserTestUtils: false, + ContentTask: false, + ContentTaskUtils: false, + EventUtils: false, + IOUtils: false, + PathUtils: false, + PromiseDebugging: false, + SpecialPowers: false, + TestUtils: false, + addLoadEvent: false, + add_setup: false, + add_task: false, + content: false, + executeSoon: false, + expectUncaughtException: false, + export_assertions: false, + extractJarToTmp: false, + finish: false, + gTestPath: false, + getChromeDir: false, + getJar: false, + getResolvedURI: false, + getRootDirectory: false, + getTestFilePath: false, + ignoreAllUncaughtExceptions: false, + info: false, + is: false, + isnot: false, + ok: false, + record: false, + registerCleanupFunction: false, + requestLongerTimeout: false, + setExpectedFailuresForSelfTest: false, + stringContains: false, + stringMatches: false, + todo: false, + todo_is: false, + todo_isnot: false, + waitForClipboard: false, + waitForExplicitFinish: false, + waitForFocus: false, + }, + + plugins: ["mozilla", "@microsoft/sdl"], + + rules: { + // No using of insecure url, so no http urls + "@microsoft/sdl/no-insecure-url": [ + "error", + { + exceptions: [ + "^http:\\/\\/mochi\\.test?.*", + "^http:\\/\\/localhost?.*", + "^http:\\/\\/127\\.0\\.0\\.1?.*", + // Exempt xmlns urls + "^http:\\/\\/www\\.w3\\.org?.*", + "^http:\\/\\/www\\.mozilla\\.org\\/keymaster\\/gatekeeper?.*", + // Exempt urls that start with ftp or ws. + "^ws:?.*", + "^ftp:?.*", + ], + varExceptions: ["insecure?.*"], + }, + ], + "mozilla/import-content-task-globals": "error", + "mozilla/import-headjs-globals": "error", + "mozilla/mark-test-function-used": "error", + "mozilla/no-addtask-setup": "error", + "mozilla/no-arbitrary-setTimeout": "error", + "mozilla/no-redeclare-with-import-autofix": [ + "error", + { errorForNonImports: false }, + ], + // Turn off no-unsanitized for tests, as we do want to be able to use + // these for testing. + "no-unsanitized/method": "off", + "no-unsanitized/property": "off", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/chrome-test.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/chrome-test.js new file mode 100644 index 0000000000..3b5bbc06e2 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/chrome-test.js @@ -0,0 +1,65 @@ +// Parent config file for all mochitest files. +"use strict"; + +module.exports = { + env: { + browser: true, + "mozilla/browser-window": true, + }, + + // All globals made available in the test environment. + globals: { + // SpecialPowers is injected into the window object via SimpleTest.js + SpecialPowers: false, + extractJarToTmp: false, + getChromeDir: false, + getJar: false, + getResolvedURI: false, + getRootDirectory: false, + }, + + overrides: [ + { + env: { + // Ideally we wouldn't be using the simpletest env here, but our uses of + // js files mean we pick up everything from the global scope, which could + // be any one of a number of html files. So we just allow the basics... + "mozilla/simpletest": true, + }, + files: ["*.js"], + }, + ], + + plugins: ["mozilla", "@microsoft/sdl"], + + rules: { + // No using of insecure url, so no http urls + "@microsoft/sdl/no-insecure-url": [ + "error", + { + exceptions: [ + "^http:\\/\\/mochi\\.test?.*", + "^http:\\/\\/localhost?.*", + "^http:\\/\\/127\\.0\\.0\\.1?.*", + // Exempt xmlns urls + "^http:\\/\\/www\\.w3\\.org?.*", + "^http:\\/\\/www\\.mozilla\\.org\\/keymaster\\/gatekeeper?.*", + // Exempt urls that start with ftp or ws. + "^ws:?.*", + "^ftp:?.*", + ], + varExceptions: ["insecure?.*"], + }, + ], + "mozilla/import-content-task-globals": "error", + "mozilla/import-headjs-globals": "error", + "mozilla/mark-test-function-used": "error", + // We mis-predict globals for HTML test files in directories shared + // with browser tests. + "mozilla/no-redeclare-with-import-autofix": "off", + // Turn off no-unsanitized for tests, as we do want to be able to use + // these for testing. + "no-unsanitized/method": "off", + "no-unsanitized/property": "off", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/mochitest-test.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/mochitest-test.js new file mode 100644 index 0000000000..ceca2beec4 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/mochitest-test.js @@ -0,0 +1,66 @@ +// Parent config file for all mochitest files. +"use strict"; + +module.exports = { + env: { + browser: true, + }, + + // All globals made available in the test environment. + globals: { + // SpecialPowers is injected into the window object via SimpleTest.js + SpecialPowers: false, + }, + + overrides: [ + { + env: { + // Ideally we wouldn't be using the simpletest env here, but our uses of + // js files mean we pick up everything from the global scope, which could + // be any one of a number of html files. So we just allow the basics... + "mozilla/simpletest": true, + }, + files: ["*.js"], + }, + ], + plugins: ["mozilla", "@microsoft/sdl"], + + rules: { + // No using of insecure url, so no http urls + "@microsoft/sdl/no-insecure-url": [ + "error", + { + exceptions: [ + "^http:\\/\\/mochi\\.test?.*", + "^http:\\/\\/mochi\\.xorigin-test?.*", + "^http:\\/\\/localhost?.*", + "^http:\\/\\/127\\.0\\.0\\.1?.*", + // Exempt xmlns urls + "^http:\\/\\/www\\.w3\\.org?.*", + "^http:\\/\\/www\\.mozilla\\.org\\/keymaster\\/gatekeeper?.*", + // Exempt urls that start with ftp or ws. + "^ws:?.*", + "^ftp:?.*", + ], + varExceptions: ["insecure?.*"], + }, + ], + "mozilla/import-content-task-globals": "error", + "mozilla/import-headjs-globals": "error", + "mozilla/mark-test-function-used": "error", + // Turn off no-define-cc-etc for mochitests as these don't have Cc etc defined in the + // global scope. + "mozilla/no-define-cc-etc": "off", + // We mis-predict globals for HTML test files in directories shared + // with browser tests, so don't try to "fix" imports that are needed. + "mozilla/no-redeclare-with-import-autofix": "off", + // Turn off use-chromeutils-generateqi as these tests don't have ChromeUtils + // available. + "mozilla/use-chromeutils-generateqi": "off", + "no-shadow": "error", + // Turn off no-unsanitized for tests, as we do want to be able to use + // these for testing. + "no-unsanitized/method": "off", + "no-unsanitized/property": "off", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js new file mode 100644 index 0000000000..db7a0dc731 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js @@ -0,0 +1,351 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +/** + * The configuration is based on eslint:recommended config. The details for all + * the ESLint rules, and which ones are in the recommended configuration can + * be found here: + * + * https://eslint.org/docs/rules/ + * + * Rules that we've explicitly decided not to enable: + * + * require-await - bug 1381030. + * no-prototype-builtins - bug 1551829. + * require-atomic-updates - bug 1551829. + * - This generates too many false positives that are not easy to work + * around, and false positives seem to be inherent in the rule. + */ +module.exports = { + env: { + browser: true, + es2022: true, + "mozilla/privileged": true, + "mozilla/specific": true, + }, + + // The prettier configuration here comes from eslint-config-prettier and + // turns off all of ESLint's rules related to formatting. + extends: [ + "eslint:recommended", + "prettier", + "plugin:json/recommended-with-comments", + ], + + overrides: [ + { + // System mjs files and jsm files are not loaded in the browser scope, + // so we turn that off for those. Though we do have our own special + // environment for them. + env: { + browser: false, + "mozilla/jsm": true, + }, + files: ["**/*.sys.mjs", "**/*.jsm"], + rules: { + "mozilla/lazy-getter-object-name": "error", + "mozilla/reject-eager-module-in-lazy-getter": "error", + "mozilla/reject-global-this": "error", + "mozilla/reject-globalThis-modification": "error", + // For all system modules, we expect no properties to need importing, + // hence reject everything. + "mozilla/reject-importGlobalProperties": ["error", "everything"], + "mozilla/reject-mixing-eager-and-lazy": "error", + "mozilla/reject-top-level-await": "error", + // TODO: Bug 1575506 turn `builtinGlobals` on here. + // We can enable builtinGlobals for jsms due to their scopes. + "no-redeclare": ["error", { builtinGlobals: false }], + }, + }, + { + files: ["**/*.mjs", "**/*.jsx", "**/*.jsm", "**/?(*.)worker.?(m)js"], + rules: { + // Modules and workers are far easier to check for no-unused-vars on a + // global scope, than our content files. Hence we turn that on here. + "no-unused-vars": [ + "error", + { + args: "none", + vars: "all", + }, + ], + }, + }, + { + excludedFiles: ["**/*.sys.mjs"], + files: ["**/*.mjs"], + rules: { + "mozilla/reject-import-system-module-from-non-system": "error", + "mozilla/reject-lazy-imports-into-globals": "error", + "no-shadow": ["error", { allow: ["event"], builtinGlobals: true }], + }, + }, + { + files: ["**/*.mjs", "**/*.jsx"], + parserOptions: { + sourceType: "module", + }, + rules: { + "mozilla/use-static-import": "error", + // This rule defaults to not allowing "use strict" in module files since + // they are always loaded in strict mode. + strict: "error", + }, + }, + { + files: ["**/*.jsm"], + rules: { + "mozilla/mark-exported-symbols-as-used": "error", + }, + }, + { + env: { + browser: false, + "mozilla/privileged": false, + "mozilla/sjs": true, + "mozilla/specific": false, + }, + files: ["**/*.sjs"], + rules: { + // For sjs files, reject everything as we should update the sandbox + // to include the globals we need, as these are test-only files. + "mozilla/reject-importGlobalProperties": ["error", "everything"], + }, + }, + { + env: { + browser: false, + worker: true, + }, + files: [ + // Most files should use the `.worker.` format to be consistent with + // other items like `.sys.mjs`, but we allow simply calling the file + // "worker" as well. + "**/?(*.)worker.?(m)js", + ], + }, + ], + + parserOptions: { + ecmaVersion: "latest", + }, + + // When adding items to this file please check for effects on sub-directories. + plugins: ["fetch-options", "html", "json", "no-unsanitized"], + + // When adding items to this file please check for effects on all of toolkit + // and browser + rules: { + // This may conflict with prettier, so we turn it off. + "arrow-body-style": "off", + + // Warn about cyclomatic complexity in functions. + // XXX Get this down to 20? + complexity: ["error", 34], + + // Functions must always return something or nothing + "consistent-return": "error", + + // XXX This rule line should be removed to enable it. See bug 1487642. + // Require super() calls in constructors + "constructor-super": "off", + + // Require braces around blocks that start a new line + curly: ["error", "all"], + + // Encourage the use of dot notation whenever possible. + "dot-notation": "error", + + // XXX This rule should be enabled, see Bug 1557040 + // No credentials submitted with fetch calls + "fetch-options/no-fetch-credentials": "off", + + // XXX This rule line should be removed to enable it. See bug 1487642. + // Enforce return statements in getters + "getter-return": "off", + + // Don't enforce the maximum depth that blocks can be nested. The complexity + // rule is a better rule to check this. + "max-depth": "off", + + // Maximum depth callbacks can be nested. + "max-nested-callbacks": ["error", 10], + + "mozilla/avoid-removeChild": "error", + "mozilla/consistent-if-bracing": "error", + "mozilla/import-browser-window-globals": "error", + "mozilla/import-globals": "error", + "mozilla/no-compare-against-boolean-literals": "error", + "mozilla/no-cu-reportError": "error", + "mozilla/no-define-cc-etc": "error", + "mozilla/no-throw-cr-literal": "error", + "mozilla/no-useless-parameters": "error", + "mozilla/no-useless-removeEventListener": "error", + "mozilla/prefer-boolean-length-check": "error", + "mozilla/prefer-formatValues": "error", + "mozilla/reject-addtask-only": "error", + "mozilla/reject-chromeutils-import-params": "error", + "mozilla/reject-importGlobalProperties": ["error", "allownonwebidl"], + "mozilla/reject-multiple-getters-calls": "error", + "mozilla/reject-scriptableunicodeconverter": "warn", + "mozilla/rejects-requires-await": "error", + "mozilla/use-cc-etc": "error", + "mozilla/use-chromeutils-definelazygetter": "error", + "mozilla/use-chromeutils-generateqi": "error", + "mozilla/use-chromeutils-import": "error", + "mozilla/use-console-createInstance": "error", + "mozilla/use-default-preference-values": "error", + "mozilla/use-includes-instead-of-indexOf": "error", + "mozilla/use-isInstance": "error", + "mozilla/use-ownerGlobal": "error", + "mozilla/use-returnValue": "error", + "mozilla/use-services": "error", + "mozilla/valid-lazy": "error", + "mozilla/valid-services": "error", + + // Use [] instead of Array() + "no-array-constructor": "error", + + // Disallow use of arguments.caller or arguments.callee. + "no-caller": "error", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Disallow lexical declarations in case clauses + "no-case-declarations": "off", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Disallow the use of console + "no-console": "off", + + // Disallows expressions where the operation doesn't affect the value. + // TODO: This is enabled by default in ESLint's v9 recommended configuration. + "no-constant-binary-expression": "error", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Disallow constant expressions in conditions + "no-constant-condition": "off", + + // If an if block ends with a return no need for an else block + "no-else-return": "error", + + // No empty statements + "no-empty": ["error", { allowEmptyCatch: true }], + + // Disallow eval and setInteral/setTimeout with strings + "no-eval": "error", + + // Disallow unnecessary calls to .bind() + "no-extra-bind": "error", + + // Disallow fallthrough of case statements + "no-fallthrough": [ + "error", + { + // The eslint rule doesn't allow for case-insensitive regex option. + // The following pattern allows for a dash between "fall through" as + // well as alternate spelling of "fall thru". The pattern also allows + // for an optional "s" at the end of "fall" ("falls through"). + commentPattern: + "[Ff][Aa][Ll][Ll][Ss]?[\\s-]?([Tt][Hh][Rr][Oo][Uu][Gg][Hh]|[Tt][Hh][Rr][Uu])", + }, + ], + + // Disallow eval and setInteral/setTimeout with strings + "no-implied-eval": "error", + + // This has been superseded since we're using ES6. + // Disallow variable or function declarations in nested blocks + "no-inner-declarations": "off", + + // Disallow the use of the __iterator__ property + "no-iterator": "error", + + // No labels + "no-labels": "error", + + // Disallow unnecessary nested blocks + "no-lone-blocks": "error", + + // No single if block inside an else block + "no-lonely-if": "error", + + // Nested ternary statements are confusing + "no-nested-ternary": "error", + + // Disallow use of new wrappers + "no-new-wrappers": "error", + + // Use {} instead of new Object(), unless arguments are passed. + "no-object-constructor": "error", + + // We don't want this, see bug 1551829 + "no-prototype-builtins": "off", + + // Disable builtinGlobals for no-redeclare as this conflicts with our + // globals declarations especially for browser window. + "no-redeclare": ["error", { builtinGlobals: false }], + + // Disallow use of event global. + "no-restricted-globals": ["error", "event"], + + // No unnecessary comparisons + "no-self-compare": "error", + + // No comma sequenced statements + "no-sequences": "error", + + // No declaring variables from an outer scope + // "no-shadow": "error", + + // Disallow throwing literals (eg. throw "error" instead of + // throw new Error("error")). + "no-throw-literal": "error", + + // Disallow the use of Boolean literals in conditional expressions. + "no-unneeded-ternary": "error", + + // No unsanitized use of innerHTML=, document.write() etc. + // cf. https://github.com/mozilla/eslint-plugin-no-unsanitized#rule-details + "no-unsanitized/method": "error", + "no-unsanitized/property": "error", + + // No declaring variables that are never used + "no-unused-vars": [ + "error", + { + args: "none", + vars: "local", + }, + ], + + // No using variables before defined + // "no-use-before-define": ["error", "nofunc"], + + // Disallow unnecessary .call() and .apply() + "no-useless-call": "error", + + // Don't concatenate string literals together (unless they span multiple + // lines) + "no-useless-concat": "error", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Disallow unnecessary escape characters + "no-useless-escape": "off", + + // Disallow redundant return statements + "no-useless-return": "error", + + // Require object-literal shorthand with ES6 method syntax + "object-shorthand": ["error", "always", { avoidQuotes: true }], + + // This may conflict with prettier, so turn it off. + "prefer-arrow-callback": "off", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Require generator functions to contain yield + "require-yield": "off", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/require-jsdoc.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/require-jsdoc.js new file mode 100644 index 0000000000..086fc8b1d3 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/require-jsdoc.js @@ -0,0 +1,41 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + plugins: ["jsdoc"], + + rules: { + "jsdoc/require-jsdoc": [ + "error", + { + require: { + ClassDeclaration: true, + FunctionDeclaration: false, + }, + }, + ], + "jsdoc/require-param": "error", + "jsdoc/require-param-description": "error", + "jsdoc/require-param-name": "error", + "jsdoc/require-property": "error", + "jsdoc/require-property-description": "error", + "jsdoc/require-property-name": "error", + "jsdoc/require-property-type": "error", + "jsdoc/require-returns": "error", + "jsdoc/require-returns-check": "error", + "jsdoc/require-yields": "error", + "jsdoc/require-yields-check": "error", + }, + settings: { + jsdoc: { + // This changes what's allowed in JSDocs, enabling more type-inference + // friendly types. This is the default in eslint-plugin-jsdoc versions + // since May 2023, but we're still on 39.9 and need opt-in for now. + // https://github.com/gajus/eslint-plugin-jsdoc/issues/834 + mode: "typescript", + }, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/valid-jsdoc.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/valid-jsdoc.js new file mode 100644 index 0000000000..65fb760fe0 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/valid-jsdoc.js @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + plugins: ["jsdoc"], + + rules: { + "jsdoc/check-access": "error", + // Handled by prettier + // "jsdoc/check-alignment": "error", + "jsdoc/check-param-names": "error", + "jsdoc/check-property-names": "error", + "jsdoc/check-tag-names": "error", + "jsdoc/check-types": "error", + "jsdoc/empty-tags": "error", + "jsdoc/newline-after-description": "error", + "jsdoc/no-multi-asterisks": "error", + "jsdoc/require-param-type": "error", + "jsdoc/require-returns-type": "error", + "jsdoc/valid-types": "error", + }, + settings: { + jsdoc: { + // This changes what's allowed in JSDocs, enabling more type-inference + // friendly types. This is the default in eslint-plugin-jsdoc versions + // since May 2023, but we're still on 39.9 and need opt-in for now. + // https://github.com/gajus/eslint-plugin-jsdoc/issues/834 + mode: "typescript", + }, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/xpcshell-test.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/xpcshell-test.js new file mode 100644 index 0000000000..6a4d572911 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/xpcshell-test.js @@ -0,0 +1,54 @@ +// Parent config file for all xpcshell files. +"use strict"; + +module.exports = { + env: { + browser: false, + "mozilla/privileged": true, + "mozilla/xpcshell": true, + }, + + overrides: [ + { + // If it is a head file, we turn off global unused variable checks, as it + // would require searching the other test files to know if they are used or not. + // This would be expensive and slow, and it isn't worth it for head files. + // We could get developers to declare as exported, but that doesn't seem worth it. + files: "head*.js", + rules: { + "no-unused-vars": [ + "error", + { + args: "none", + vars: "local", + }, + ], + }, + }, + { + // No declaring variables that are never used + files: "test*.js", + rules: { + "no-unused-vars": [ + "error", + { + args: "none", + vars: "all", + }, + ], + }, + }, + ], + + rules: { + "mozilla/import-headjs-globals": "error", + "mozilla/mark-test-function-used": "error", + "mozilla/no-arbitrary-setTimeout": "error", + "mozilla/no-useless-run-test": "error", + "no-shadow": "error", + // Turn off no-unsanitized for tests, as we do want to be able to use + // these for testing. + "no-unsanitized/method": "off", + "no-unsanitized/property": "off", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/browser-window.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/browser-window.js new file mode 100644 index 0000000000..241299e2d3 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/browser-window.js @@ -0,0 +1,121 @@ +/** + * @fileoverview Defines the environment when in the browser.xhtml window. + * Imports many globals from various files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var fs = require("fs"); +var helpers = require("../helpers"); +var { getScriptGlobals } = require("./utils"); + +// When updating EXTRA_SCRIPTS or MAPPINGS, be sure to also update the +// 'support-files' config in `tools/lint/eslint.yml`. + +// These are scripts not loaded from browser.xhtml or global-scripts.inc +// but via other includes. +const EXTRA_SCRIPTS = [ + "browser/base/content/nsContextMenu.js", + "browser/components/downloads/content/downloads.js", + "browser/components/downloads/content/indicator.js", + "toolkit/content/customElements.js", + "toolkit/content/editMenuOverlay.js", +]; + +const extraDefinitions = [ + // Via Components.utils, defineModuleGetter, defineLazyModuleGetters or + // defineLazyScriptGetter (and map to + // single) variable. + { name: "XPCOMUtils", writable: false }, + { name: "Task", writable: false }, + { name: "windowGlobalChild", writable: false }, + // structuredClone is a new global that would be defined for the `browser` + // environment in ESLint, but only Firefox has implemented it currently and so + // it isn't in ESLint's globals yet. + // https://developer.mozilla.org/docs/Web/API/structuredClone + { name: "structuredClone", writable: false }, +]; + +// Some files in global-scripts.inc need mapping to specific locations. +const MAPPINGS = { + "printUtils.js": "toolkit/components/printing/content/printUtils.js", + "panelUI.js": "browser/components/customizableui/content/panelUI.js", + "viewSourceUtils.js": + "toolkit/components/viewsource/content/viewSourceUtils.js", + "browserPlacesViews.js": + "browser/components/places/content/browserPlacesViews.js", + "places-tree.js": "browser/components/places/content/places-tree.js", + "places-menupopup.js": + "browser/components/places/content/places-menupopup.js", + "shopping-sidebar.js": + "browser/components/shopping/content/shopping-sidebar.js", +}; + +const globalScriptsRegExp = + /^\s*Services.scriptloader.loadSubScript\(\"(.*?)\", this\);$/; + +function getGlobalScriptIncludes(scriptPath) { + let fileData; + try { + fileData = fs.readFileSync(scriptPath, { encoding: "utf8" }); + } catch (ex) { + // The file isn't present, so this isn't an m-c repository. + return null; + } + + fileData = fileData.split("\n"); + + let result = []; + + for (let line of fileData) { + let match = line.match(globalScriptsRegExp); + if (match) { + let sourceFile = match[1] + .replace( + "chrome://browser/content/search/", + "browser/components/search/content/" + ) + .replace( + "chrome://browser/content/screenshots/", + "browser/components/screenshots/content/" + ) + .replace("chrome://browser/content/", "browser/base/content/") + .replace("chrome://global/content/", "toolkit/content/"); + + for (let mapping of Object.getOwnPropertyNames(MAPPINGS)) { + if (sourceFile.includes(mapping)) { + sourceFile = MAPPINGS[mapping]; + } + } + + result.push(sourceFile); + } + } + + return result; +} + +function getGlobalScripts() { + let results = []; + for (let scriptPath of helpers.globalScriptPaths) { + results = results.concat(getGlobalScriptIncludes(scriptPath)); + } + return results; +} + +module.exports = getScriptGlobals( + "browser-window", + getGlobalScripts().concat(EXTRA_SCRIPTS), + extraDefinitions, + { + browserjsScripts: getGlobalScripts().concat(EXTRA_SCRIPTS), + } +); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-script.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-script.js new file mode 100644 index 0000000000..9b0ae54a2e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-script.js @@ -0,0 +1,28 @@ +/** + * @fileoverview Defines the environment for SpecialPowers chrome script. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var { globals } = require("./special-powers-sandbox"); +var util = require("util"); + +module.exports = { + globals: util._extend( + { + // testing/specialpowers/content/SpecialPowersParent.sys.mjs + + // SPLoadChromeScript block + createWindowlessBrowser: false, + sendAsyncMessage: false, + addMessageListener: false, + removeMessageListener: false, + actorParent: false, + }, + globals + ), +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-worker.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-worker.js new file mode 100644 index 0000000000..db5759b26c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-worker.js @@ -0,0 +1,25 @@ +/** + * @fileoverview Defines the environment for chrome workers. This differs + * from normal workers by the fact that `ctypes` can be accessed + * as well. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var globals = require("globals"); +var util = require("util"); + +var workerGlobals = util._extend( + { + ctypes: false, + }, + globals.worker +); + +module.exports = { + globals: workerGlobals, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/frame-script.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/frame-script.js new file mode 100644 index 0000000000..7ac5c941cf --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/frame-script.js @@ -0,0 +1,39 @@ +/** + * @fileoverview Defines the environment for frame scripts. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // dom/chrome-webidl/MessageManager.webidl + + // MessageManagerGlobal + dump: false, + atob: false, + btoa: false, + + // MessageListenerManagerMixin + addMessageListener: false, + removeMessageListener: false, + addWeakMessageListener: false, + removeWeakMessageListener: false, + + // MessageSenderMixin + sendAsyncMessage: false, + processMessageManager: false, + remoteType: false, + + // SyncMessageSenderMixin + sendSyncMessage: false, + + // ContentFrameMessageManager + content: false, + docShell: false, + tabEventTarget: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/jsm.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/jsm.js new file mode 100644 index 0000000000..30d8e0eb9c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/jsm.js @@ -0,0 +1,25 @@ +/** + * @fileoverview Defines the environment for jsm files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // These globals are hard-coded and available in .jsm scopes. + // https://searchfox.org/mozilla-central/rev/dcb0cfb66e4ed3b9c7fbef1e80572426ff5f3c3a/js/xpconnect/loader/mozJSModuleLoader.cpp#222-223 + // Although `debug` is allowed for jsm files, this is non-standard and something + // we don't want to allow in mjs files. Hence it is not included here. + atob: false, + btoa: false, + dump: false, + // The WebAssembly global is available in most (if not all) contexts where + // JS can run. It's definitely available in JSMs. So even if this is not + // the perfect place to add it, it's not wrong, and we can move it later. + WebAssembly: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/privileged.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/privileged.js new file mode 100644 index 0000000000..c517de6209 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/privileged.js @@ -0,0 +1,819 @@ +/** + * @fileoverview Defines the environment for privileges JS files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // Intl and WebAssembly are available everywhere but are not webIDL definitions. + Intl: false, + WebAssembly: false, + // This list of items is currently obtained manually from the list of + // mozilla::dom::constructor::id::ID enumerations in an object directory + // generated dom/bindings/RegisterBindings.cpp + APZHitResultFlags: false, + AbortController: false, + AbortSignal: false, + AccessibleNode: false, + Addon: false, + AddonEvent: false, + AddonInstall: false, + AddonManager: true, + AddonManagerPermissions: false, + AnalyserNode: false, + Animation: false, + AnimationEffect: false, + AnimationEvent: false, + AnimationPlaybackEvent: false, + AnimationTimeline: false, + AnonymousContent: false, + Attr: false, + AudioBuffer: false, + AudioBufferSourceNode: false, + AudioContext: false, + AudioDecoder: false, + AudioDestinationNode: false, + AudioData: false, + AudioEncoder: false, + AudioListener: false, + AudioNode: false, + AudioParam: false, + AudioParamMap: false, + AudioProcessingEvent: false, + AudioScheduledSourceNode: false, + AudioTrack: false, + AudioTrackList: false, + AudioWorklet: false, + AudioWorkletNode: false, + AuthenticatorAssertionResponse: false, + AuthenticatorAttestationResponse: false, + AuthenticatorResponse: false, + BarProp: false, + BaseAudioContext: false, + BatteryManager: false, + BeforeUnloadEvent: false, + BiquadFilterNode: false, + Blob: false, + BlobEvent: false, + BoxObject: false, + BroadcastChannel: false, + BrowsingContext: false, + ByteLengthQueuingStrategy: false, + CanonicalBrowsingContext: false, + CDATASection: false, + CSS: false, + CSS2Properties: false, + CSSAnimation: false, + CSSConditionRule: false, + CSSCounterStyleRule: false, + CSSFontFaceRule: false, + CSSFontFeatureValuesRule: false, + CSSGroupingRule: false, + CSSImportRule: false, + CSSKeyframeRule: false, + CSSKeyframesRule: false, + CSSMediaRule: false, + CSSMozDocumentRule: false, + CSSNamespaceRule: false, + CSSPageRule: false, + CSSPseudoElement: false, + CSSRule: false, + CSSRuleList: false, + CSSStyleDeclaration: false, + CSSStyleRule: false, + CSSStyleSheet: false, + CSSSupportsRule: false, + CSSTransition: false, + Cache: false, + CacheStorage: false, + CanvasCaptureMediaStream: false, + CanvasGradient: false, + CanvasPattern: false, + CanvasRenderingContext2D: false, + CaretPosition: false, + CaretStateChangedEvent: false, + ChannelMergerNode: false, + ChannelSplitterNode: false, + ChannelWrapper: false, + CharacterData: false, + CheckerboardReportService: false, + ChildProcessMessageManager: false, + ChildSHistory: false, + ChromeMessageBroadcaster: false, + ChromeMessageSender: false, + ChromeNodeList: false, + ChromeUtils: false, + ChromeWorker: false, + Clipboard: false, + ClipboardEvent: false, + ClonedErrorHolder: false, + CloseEvent: false, + CommandEvent: false, + Comment: false, + CompositionEvent: false, + ConsoleInstance: false, + ConstantSourceNode: false, + ContentFrameMessageManager: false, + ContentProcessMessageManager: false, + ConvolverNode: false, + CountQueuingStrategy: false, + CreateOfferRequest: false, + Credential: false, + CredentialsContainer: false, + Crypto: false, + CryptoKey: false, + CustomElementRegistry: false, + CustomEvent: false, + DOMError: false, + DOMException: false, + DOMImplementation: false, + DOMLocalization: false, + DOMMatrix: false, + DOMMatrixReadOnly: false, + DOMParser: false, + DOMPoint: false, + DOMPointReadOnly: false, + DOMQuad: false, + DOMRect: false, + DOMRectList: false, + DOMRectReadOnly: false, + DOMRequest: false, + DOMStringList: false, + DOMStringMap: false, + DOMTokenList: false, + DataTransfer: false, + DataTransferItem: false, + DataTransferItemList: false, + DebuggerNotificationObserver: false, + DelayNode: false, + DeprecationReportBody: false, + DeviceLightEvent: false, + DeviceMotionEvent: false, + DeviceOrientationEvent: false, + DeviceProximityEvent: false, + Directory: false, + Document: false, + DocumentFragment: false, + DocumentTimeline: false, + DocumentType: false, + DominatorTree: false, + DragEvent: false, + DynamicsCompressorNode: false, + Element: false, + EncodedAudioChunk: false, + EncodedVideoChunk: false, + ErrorEvent: false, + Event: false, + EventSource: false, + EventTarget: false, + FeaturePolicyViolationReportBody: false, + FetchObserver: false, + File: false, + FileList: false, + FileReader: false, + FileSystem: false, + FileSystemDirectoryEntry: false, + FileSystemDirectoryReader: false, + FileSystemEntry: false, + FileSystemFileEntry: false, + Flex: false, + FlexItemValues: false, + FlexLineValues: false, + FluentBundle: false, + FluentResource: false, + FocusEvent: false, + FontFace: false, + FontFaceSet: false, + FontFaceSetLoadEvent: false, + FormData: false, + FrameCrashedEvent: false, + FrameLoader: false, + GainNode: false, + Gamepad: false, + GamepadAxisMoveEvent: false, + GamepadButton: false, + GamepadButtonEvent: false, + GamepadEvent: false, + GamepadHapticActuator: false, + GamepadPose: false, + GamepadServiceTest: false, + Glean: false, + GleanPings: false, + Grid: false, + GridArea: false, + GridDimension: false, + GridLine: false, + GridLines: false, + GridTrack: false, + GridTracks: false, + HTMLAllCollection: false, + HTMLAnchorElement: false, + HTMLAreaElement: false, + HTMLAudioElement: false, + Audio: false, + HTMLBRElement: false, + HTMLBaseElement: false, + HTMLBodyElement: false, + HTMLButtonElement: false, + HTMLCanvasElement: false, + HTMLCollection: false, + HTMLDListElement: false, + HTMLDataElement: false, + HTMLDataListElement: false, + HTMLDetailsElement: false, + HTMLDialogElement: false, + HTMLDirectoryElement: false, + HTMLDivElement: false, + HTMLDocument: false, + HTMLElement: false, + HTMLEmbedElement: false, + HTMLFieldSetElement: false, + HTMLFontElement: false, + HTMLFormControlsCollection: false, + HTMLFormElement: false, + HTMLFrameElement: false, + HTMLFrameSetElement: false, + HTMLHRElement: false, + HTMLHeadElement: false, + HTMLHeadingElement: false, + HTMLHtmlElement: false, + HTMLIFrameElement: false, + HTMLImageElement: false, + Image: false, + HTMLInputElement: false, + HTMLLIElement: false, + HTMLLabelElement: false, + HTMLLegendElement: false, + HTMLLinkElement: false, + HTMLMapElement: false, + HTMLMarqueeElement: false, + HTMLMediaElement: false, + HTMLMenuElement: false, + HTMLMenuItemElement: false, + HTMLMetaElement: false, + HTMLMeterElement: false, + HTMLModElement: false, + HTMLOListElement: false, + HTMLObjectElement: false, + HTMLOptGroupElement: false, + HTMLOptionElement: false, + Option: false, + HTMLOptionsCollection: false, + HTMLOutputElement: false, + HTMLParagraphElement: false, + HTMLParamElement: false, + HTMLPictureElement: false, + HTMLPreElement: false, + HTMLProgressElement: false, + HTMLQuoteElement: false, + HTMLScriptElement: false, + HTMLSelectElement: false, + HTMLSlotElement: false, + HTMLSourceElement: false, + HTMLSpanElement: false, + HTMLStyleElement: false, + HTMLTableCaptionElement: false, + HTMLTableCellElement: false, + HTMLTableColElement: false, + HTMLTableElement: false, + HTMLTableRowElement: false, + HTMLTableSectionElement: false, + HTMLTemplateElement: false, + HTMLTextAreaElement: false, + HTMLTimeElement: false, + HTMLTitleElement: false, + HTMLTrackElement: false, + HTMLUListElement: false, + HTMLUnknownElement: false, + HTMLVideoElement: false, + HashChangeEvent: false, + Headers: false, + HeapSnapshot: false, + History: false, + IDBCursor: false, + IDBCursorWithValue: false, + IDBDatabase: false, + IDBFactory: false, + IDBFileHandle: false, + IDBFileRequest: false, + IDBIndex: false, + IDBKeyRange: false, + IDBMutableFile: false, + IDBObjectStore: false, + IDBOpenDBRequest: false, + IDBRequest: false, + IDBTransaction: false, + IDBVersionChangeEvent: false, + IIRFilterNode: false, + IdleDeadline: false, + ImageBitmap: false, + ImageBitmapRenderingContext: false, + ImageCapture: false, + ImageCaptureErrorEvent: false, + ImageData: false, + ImageDocument: false, + InputEvent: false, + InspectorFontFace: false, + InspectorUtils: false, + InstallTriggerImpl: false, + IntersectionObserver: false, + IntersectionObserverEntry: false, + IOUtils: false, + JSProcessActorChild: false, + JSProcessActorParent: false, + JSWindowActorChild: false, + JSWindowActorParent: false, + KeyEvent: false, + KeyboardEvent: false, + KeyframeEffect: false, + L10nFileSource: false, + L10nRegistry: false, + Localization: false, + Location: false, + MIDIAccess: false, + MIDIConnectionEvent: false, + MIDIInput: false, + MIDIInputMap: false, + MIDIMessageEvent: false, + MIDIOutput: false, + MIDIOutputMap: false, + MIDIPort: false, + MatchGlob: false, + MatchPattern: false, + MatchPatternSet: false, + MediaCapabilities: false, + MediaCapabilitiesInfo: false, + MediaControlService: false, + MediaDeviceInfo: false, + MediaDevices: false, + MediaElementAudioSourceNode: false, + MediaEncryptedEvent: false, + MediaError: false, + MediaKeyError: false, + MediaKeyMessageEvent: false, + MediaKeySession: false, + MediaKeyStatusMap: false, + MediaKeySystemAccess: false, + MediaKeys: false, + MediaList: false, + MediaQueryList: false, + MediaQueryListEvent: false, + MediaRecorder: false, + MediaRecorderErrorEvent: false, + MediaSource: false, + MediaStream: false, + MediaStreamAudioDestinationNode: false, + MediaStreamAudioSourceNode: false, + MediaStreamEvent: false, + MediaStreamTrack: false, + MediaStreamTrackAudioSourceNode: false, + MediaStreamTrackEvent: false, + MerchantValidationEvent: false, + MessageBroadcaster: false, + MessageChannel: false, + MessageEvent: false, + MessageListenerManager: false, + MessagePort: false, + MessageSender: false, + MimeType: false, + MimeTypeArray: false, + MouseEvent: false, + MouseScrollEvent: false, + MozCanvasPrintState: false, + MozDocumentMatcher: false, + MozDocumentObserver: false, + MozQueryInterface: false, + MozSharedMap: false, + MozSharedMapChangeEvent: false, + MozStorageAsyncStatementParams: false, + MozStorageStatementParams: false, + MozStorageStatementRow: false, + MozWritableSharedMap: false, + MutationEvent: false, + MutationObserver: false, + MutationRecord: false, + NamedNodeMap: false, + Navigator: false, + NetworkInformation: false, + Node: false, + NodeFilter: false, + NodeIterator: false, + NodeList: false, + Notification: false, + NotifyPaintEvent: false, + OfflineAudioCompletionEvent: false, + OfflineAudioContext: false, + OfflineResourceList: false, + OffscreenCanvas: false, + OscillatorNode: false, + PageTransitionEvent: false, + PaintRequest: false, + PaintRequestList: false, + PannerNode: false, + ParentProcessMessageManager: false, + Path2D: false, + PathUtils: false, + PaymentAddress: false, + PaymentMethodChangeEvent: false, + PaymentRequest: false, + PaymentRequestUpdateEvent: false, + PaymentResponse: false, + PeerConnectionImpl: false, + PeerConnectionObserver: false, + Performance: false, + PerformanceEntry: false, + PerformanceEntryEvent: false, + PerformanceMark: false, + PerformanceMeasure: false, + PerformanceNavigation: false, + PerformanceNavigationTiming: false, + PerformanceObserver: false, + PerformanceObserverEntryList: false, + PerformanceResourceTiming: false, + PerformanceServerTiming: false, + PerformanceTiming: false, + PeriodicWave: false, + PermissionStatus: false, + Permissions: false, + PlacesBookmark: false, + PlacesBookmarkAddition: false, + PlacesBookmarkGuid: false, + PlacesBookmarkKeyword: false, + PlacesBookmarkMoved: false, + PlacesBookmarkRemoved: false, + PlacesBookmarkTags: false, + PlacesBookmarkTime: false, + PlacesBookmarkTitle: false, + PlacesBookmarkUrl: false, + PlacesEvent: false, + PlacesHistoryCleared: false, + PlacesObservers: false, + PlacesPurgeCaches: false, + PlacesRanking: false, + PlacesVisit: false, + PlacesVisitRemoved: false, + PlacesVisitTitle: false, + PlacesWeakCallbackWrapper: false, + Plugin: false, + PluginArray: false, + PluginCrashedEvent: false, + PointerEvent: false, + PopStateEvent: false, + PopupBlockedEvent: false, + PrecompiledScript: false, + Presentation: false, + PresentationAvailability: false, + PresentationConnection: false, + PresentationConnectionAvailableEvent: false, + PresentationConnectionCloseEvent: false, + PresentationConnectionList: false, + PresentationReceiver: false, + PresentationRequest: false, + PrioEncoder: false, + ProcessMessageManager: false, + ProcessingInstruction: false, + ProgressEvent: false, + PromiseDebugging: false, + PromiseRejectionEvent: false, + PublicKeyCredential: false, + PushManager: false, + PushManagerImpl: false, + PushSubscription: false, + PushSubscriptionOptions: false, + RTCCertificate: false, + RTCDTMFSender: false, + RTCDTMFToneChangeEvent: false, + RTCDataChannel: false, + RTCDataChannelEvent: false, + RTCIceCandidate: false, + RTCPeerConnection: false, + RTCPeerConnectionIceEvent: false, + RTCPeerConnectionStatic: false, + RTCRtpReceiver: false, + RTCRtpSender: false, + RTCRtpTransceiver: false, + RTCSessionDescription: false, + RTCStatsReport: false, + RTCTrackEvent: false, + RadioNodeList: false, + Range: false, + ReadableStreamBYOBReader: false, + ReadableStreamBYOBRequest: false, + ReadableByteStreamController: false, + ReadableStream: false, + ReadableStreamDefaultController: false, + ReadableStreamDefaultReader: false, + Report: false, + ReportBody: false, + ReportingObserver: false, + Request: false, + Response: false, + SessionStoreUtils: false, + SVGAElement: false, + SVGAngle: false, + SVGAnimateElement: false, + SVGAnimateMotionElement: false, + SVGAnimateTransformElement: false, + SVGAnimatedAngle: false, + SVGAnimatedBoolean: false, + SVGAnimatedEnumeration: false, + SVGAnimatedInteger: false, + SVGAnimatedLength: false, + SVGAnimatedLengthList: false, + SVGAnimatedNumber: false, + SVGAnimatedNumberList: false, + SVGAnimatedPreserveAspectRatio: false, + SVGAnimatedRect: false, + SVGAnimatedString: false, + SVGAnimatedTransformList: false, + SVGAnimationElement: false, + SVGCircleElement: false, + SVGClipPathElement: false, + SVGComponentTransferFunctionElement: false, + SVGDefsElement: false, + SVGDescElement: false, + SVGElement: false, + SVGEllipseElement: false, + SVGFEBlendElement: false, + SVGFEColorMatrixElement: false, + SVGFEComponentTransferElement: false, + SVGFECompositeElement: false, + SVGFEConvolveMatrixElement: false, + SVGFEDiffuseLightingElement: false, + SVGFEDisplacementMapElement: false, + SVGFEDistantLightElement: false, + SVGFEDropShadowElement: false, + SVGFEFloodElement: false, + SVGFEFuncAElement: false, + SVGFEFuncBElement: false, + SVGFEFuncGElement: false, + SVGFEFuncRElement: false, + SVGFEGaussianBlurElement: false, + SVGFEImageElement: false, + SVGFEMergeElement: false, + SVGFEMergeNodeElement: false, + SVGFEMorphologyElement: false, + SVGFEOffsetElement: false, + SVGFEPointLightElement: false, + SVGFESpecularLightingElement: false, + SVGFESpotLightElement: false, + SVGFETileElement: false, + SVGFETurbulenceElement: false, + SVGFilterElement: false, + SVGForeignObjectElement: false, + SVGGElement: false, + SVGGeometryElement: false, + SVGGradientElement: false, + SVGGraphicsElement: false, + SVGImageElement: false, + SVGLength: false, + SVGLengthList: false, + SVGLineElement: false, + SVGLinearGradientElement: false, + SVGMPathElement: false, + SVGMarkerElement: false, + SVGMaskElement: false, + SVGMatrix: false, + SVGMetadataElement: false, + SVGNumber: false, + SVGNumberList: false, + SVGPathElement: false, + SVGPathSegList: false, + SVGPatternElement: false, + SVGPoint: false, + SVGPointList: false, + SVGPolygonElement: false, + SVGPolylineElement: false, + SVGPreserveAspectRatio: false, + SVGRadialGradientElement: false, + SVGRect: false, + SVGRectElement: false, + SVGSVGElement: false, + SVGScriptElement: false, + SVGSetElement: false, + SVGStopElement: false, + SVGStringList: false, + SVGStyleElement: false, + SVGSwitchElement: false, + SVGSymbolElement: false, + SVGTSpanElement: false, + SVGTextContentElement: false, + SVGTextElement: false, + SVGTextPathElement: false, + SVGTextPositioningElement: false, + SVGTitleElement: false, + SVGTransform: false, + SVGTransformList: false, + SVGUnitTypes: false, + SVGUseElement: false, + SVGViewElement: false, + SVGZoomAndPan: false, + Screen: false, + ScreenLuminance: false, + ScreenOrientation: false, + ScriptProcessorNode: false, + ScrollAreaEvent: false, + ScrollViewChangeEvent: false, + SecurityPolicyViolationEvent: false, + Selection: false, + ServiceWorker: false, + ServiceWorkerContainer: false, + ServiceWorkerRegistration: false, + ShadowRoot: false, + SharedWorker: false, + SimpleGestureEvent: false, + SourceBuffer: false, + SourceBufferList: false, + SpeechGrammar: false, + SpeechGrammarList: false, + SpeechRecognition: false, + SpeechRecognitionAlternative: false, + SpeechRecognitionError: false, + SpeechRecognitionEvent: false, + SpeechRecognitionResult: false, + SpeechRecognitionResultList: false, + SpeechSynthesis: false, + SpeechSynthesisErrorEvent: false, + SpeechSynthesisEvent: false, + SpeechSynthesisUtterance: false, + SpeechSynthesisVoice: false, + StereoPannerNode: false, + Storage: false, + StorageEvent: false, + StorageManager: false, + StreamFilter: false, + StreamFilterDataEvent: false, + StructuredCloneHolder: false, + StructuredCloneTester: false, + StyleSheet: false, + StyleSheetApplicableStateChangeEvent: false, + StyleSheetList: false, + StyleSheetRemovedEvent: false, + SubtleCrypto: false, + SyncMessageSender: false, + TCPServerSocket: false, + TCPServerSocketEvent: false, + TCPSocket: false, + TCPSocketErrorEvent: false, + TCPSocketEvent: false, + TelemetryStopwatch: false, + TestingDeprecatedInterface: false, + Text: false, + TextClause: false, + TextDecoder: false, + TextEncoder: false, + TextMetrics: false, + TextTrack: false, + TextTrackCue: false, + TextTrackCueList: false, + TextTrackList: false, + TimeEvent: false, + TimeRanges: false, + Touch: false, + TouchEvent: false, + TouchList: false, + TrackEvent: false, + TransceiverImpl: false, + TransformStream: false, + TransformStreamDefaultController: false, + TransitionEvent: false, + TreeColumn: false, + TreeColumns: false, + TreeContentView: false, + TreeWalker: false, + U2F: false, + UDPMessageEvent: false, + UDPSocket: false, + UIEvent: false, + URL: false, + URLSearchParams: false, + UserInteraction: false, + UserProximityEvent: false, + VRDisplay: false, + VRDisplayCapabilities: false, + VRDisplayEvent: false, + VREyeParameters: false, + VRFieldOfView: false, + VRFrameData: false, + VRMockController: false, + VRMockDisplay: false, + VRPose: false, + VRServiceTest: false, + VRStageParameters: false, + VRSubmitFrameResult: false, + VTTCue: false, + VTTRegion: false, + ValidityState: false, + VideoColorSpace: false, + VideoDecoder: false, + VideoEncoder: false, + VideoFrame: false, + VideoPlaybackQuality: false, + VideoTrack: false, + VideoTrackList: false, + VisualViewport: false, + WaveShaperNode: false, + WebExtensionContentScript: false, + WebExtensionPolicy: false, + WebGL2RenderingContext: false, + WebGLActiveInfo: false, + WebGLBuffer: false, + WebGLContextEvent: false, + WebGLFramebuffer: false, + WebGLProgram: false, + WebGLQuery: false, + WebGLRenderbuffer: false, + WebGLRenderingContext: false, + WebGLSampler: false, + WebGLShader: false, + WebGLShaderPrecisionFormat: false, + WebGLSync: false, + WebGLTexture: false, + WebGLTransformFeedback: false, + WebGLUniformLocation: false, + WebGLVertexArrayObject: false, + WebGPU: false, + WebGPUAdapter: false, + WebGPUAttachmentState: false, + WebGPUBindGroup: false, + WebGPUBindGroupLayout: false, + WebGPUBindingType: false, + WebGPUBlendFactor: false, + WebGPUBlendOperation: false, + WebGPUBlendState: false, + WebGPUBuffer: false, + WebGPUBufferUsage: false, + WebGPUColorWriteBits: false, + WebGPUCommandBuffer: false, + WebGPUCommandEncoder: false, + WebGPUCompareFunction: false, + WebGPUComputePipeline: false, + WebGPUDepthStencilState: false, + WebGPUDevice: false, + WebGPUFence: false, + WebGPUFilterMode: false, + WebGPUIndexFormat: false, + WebGPUInputState: false, + WebGPUInputStepMode: false, + WebGPULoadOp: false, + WebGPULogEntry: false, + WebGPUPipelineLayout: false, + WebGPUPrimitiveTopology: false, + WebGPUQueue: false, + WebGPURenderPipeline: false, + WebGPUSampler: false, + WebGPUShaderModule: false, + WebGPUShaderStage: false, + WebGPUShaderStageBit: false, + WebGPUStencilOperation: false, + WebGPUStoreOp: false, + WebGPUSwapChain: false, + WebGPUTexture: false, + WebGPUTextureDimension: false, + WebGPUTextureFormat: false, + WebGPUTextureUsage: false, + WebGPUTextureView: false, + WebGPUVertexFormat: false, + WebKitCSSMatrix: false, + WebSocket: false, + WebrtcGlobalInformation: false, + WheelEvent: false, + Window: false, + WindowGlobalChild: false, + WindowGlobalParent: false, + WindowRoot: false, + Worker: false, + Worklet: false, + WritableStream: false, + WritableStreamDefaultController: false, + WritableStreamDefaultWriter: false, + XMLDocument: false, + XMLHttpRequest: false, + XMLHttpRequestEventTarget: false, + XMLHttpRequestUpload: false, + XMLSerializer: false, + XPathEvaluator: false, + XPathExpression: false, + XPathResult: false, + XSLTProcessor: false, + XULCommandEvent: false, + XULElement: false, + XULFrameElement: false, + XULMenuElement: false, + XULPopupElement: false, + XULScrollElement: false, + XULTextElement: false, + console: false, + // These are hard-coded and available in privileged scopes. + // See BackstagePass::Resolve. + fetch: false, + crypto: false, + indexedDB: false, + structuredClone: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/process-script.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/process-script.js new file mode 100644 index 0000000000..f329a6650b --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/process-script.js @@ -0,0 +1,38 @@ +/** + * @fileoverview Defines the environment for process scripts. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // dom/chrome-webidl/MessageManager.webidl + + // MessageManagerGlobal + dump: false, + atob: false, + btoa: false, + + // MessageListenerManagerMixin + addMessageListener: false, + removeMessageListener: false, + addWeakMessageListener: false, + removeWeakMessageListener: false, + + // MessageSenderMixin + sendAsyncMessage: false, + processMessageManager: false, + remoteType: false, + + // SyncMessageSenderMixin + sendSyncMessage: false, + + // ContentProcessMessageManager + initialProcessData: false, + sharedData: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/remote-page.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/remote-page.js new file mode 100644 index 0000000000..74055457fe --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/remote-page.js @@ -0,0 +1,43 @@ +/** + * @fileoverview Defines the environment for remote page. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + atob: false, + btoa: false, + RPMAddTRRExcludedDomain: false, + RPMGetAppBuildID: false, + RPMGetInnerMostURI: false, + RPMGetIntPref: false, + RPMGetStringPref: false, + RPMGetBoolPref: false, + RPMSetPref: false, + RPMGetFormatURLPref: false, + RPMIsTRROnlyFailure: false, + RPMIsFirefox: false, + RPMIsNativeFallbackFailure: false, + RPMIsWindowPrivate: false, + RPMSendAsyncMessage: false, + RPMSendQuery: false, + RPMAddMessageListener: false, + RPMRecordTelemetryEvent: false, + RPMCheckAlternateHostAvailable: false, + RPMAddToHistogram: false, + RPMRemoveMessageListener: false, + RPMGetHttpResponseHeader: false, + RPMTryPingSecureWWWLink: false, + RPMOpenSecureWWWLink: false, + RPMOpenPreferences: false, + RPMGetTRRSkipReason: false, + RPMGetTRRDomain: false, + RPMIsSiteSpecificTRRError: false, + RPMSetTRRDisabledLoadFlags: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/simpletest.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/simpletest.js new file mode 100644 index 0000000000..2f5dd5c33e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/simpletest.js @@ -0,0 +1,35 @@ +/** + * @fileoverview Defines the environment for scripts that use the SimpleTest + * mochitest harness. Imports the globals from the relevant files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var path = require("path"); +var { getScriptGlobals } = require("./utils"); + +// When updating this list, be sure to also update the 'support-files' config +// in `tools/lint/eslint.yml`. +const simpleTestFiles = [ + "AccessibilityUtils.js", + "ExtensionTestUtils.js", + "EventUtils.js", + "MockObjects.js", + "SimpleTest.js", + "WindowSnapshot.js", + "paint_listener.js", +]; +const simpleTestPath = "testing/mochitest/tests/SimpleTest"; + +module.exports = getScriptGlobals( + "simpletest", + simpleTestFiles.map(file => path.join(simpleTestPath, file)) +); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/sjs.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/sjs.js new file mode 100644 index 0000000000..4f10641c09 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/sjs.js @@ -0,0 +1,41 @@ +/** + * @fileoverview Defines the environment for sjs files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // All these variables are hard-coded to be available for sjs scopes only. + // https://searchfox.org/mozilla-central/rev/26a1b0fce12e6dd495a954c542bb1e7bd6e0d548/netwerk/test/httpserver/httpd.js#2879 + atob: false, + btoa: false, + Cc: false, + ChromeUtils: false, + Ci: false, + Components: false, + Cr: false, + Cu: false, + dump: false, + IOUtils: false, + PathUtils: false, + TextDecoder: false, + TextEncoder: false, + URLSearchParams: false, + URL: false, + getState: false, + setState: false, + getSharedState: false, + setSharedState: false, + getObjectState: false, + setObjectState: false, + registerPathHandler: false, + Services: false, + // importScripts is also available. + importScripts: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/special-powers-sandbox.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/special-powers-sandbox.js new file mode 100644 index 0000000000..5a28c91883 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/special-powers-sandbox.js @@ -0,0 +1,46 @@ +/** + * @fileoverview Defines the environment for SpecialPowers sandbox. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // wantComponents defaults to true, + Components: false, + Ci: false, + Cr: false, + Cc: false, + Cu: false, + Services: false, + + // testing/specialpowers/content/SpecialPowersSandbox.sys.mjs + + // SANDBOX_GLOBALS + Blob: false, + ChromeUtils: false, + FileReader: false, + TextDecoder: false, + TextEncoder: false, + URL: false, + + // EXTRA_IMPORTS + EventUtils: false, + + // SpecialPowersSandbox constructor + assert: false, + Assert: false, + BrowsingContext: false, + InspectorUtils: false, + ok: false, + is: false, + isnot: false, + todo: false, + todo_is: false, + info: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/specific.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/specific.js new file mode 100644 index 0000000000..23ebcb5bb1 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/specific.js @@ -0,0 +1,31 @@ +/** + * @fileoverview Defines the environment for the Firefox browser. Allows global + * variables which are non-standard and specific to Firefox. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + Cc: false, + ChromeUtils: false, + Ci: false, + Components: false, + Cr: false, + Cu: false, + Debugger: false, + InstallTrigger: false, + // https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/InternalError + InternalError: true, + Services: false, + // https://developer.mozilla.org/docs/Web/API/Window/dump + dump: true, + openDialog: false, + // https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/uneval + uneval: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/testharness.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/testharness.js new file mode 100644 index 0000000000..cea4088a4c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/testharness.js @@ -0,0 +1,61 @@ +/** + * @fileoverview Defines the environment for testharness.js files. This + * is automatically included in (x)html files including + * /resources/testharness.js. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// These globals are taken from dom/imptests/testharness.js, via the expose +// function. + +module.exports = { + globals: { + EventWatcher: false, + test: false, + async_test: false, + promise_test: false, + promise_rejects: false, + generate_tests: false, + setup: false, + done: false, + on_event: false, + step_timeout: false, + format_value: false, + assert_true: false, + assert_false: false, + assert_equals: false, + assert_not_equals: false, + assert_in_array: false, + assert_object_equals: false, + assert_array_equals: false, + assert_approx_equals: false, + assert_less_than: false, + assert_greater_than: false, + assert_between_exclusive: false, + assert_less_than_equal: false, + assert_greater_than_equal: false, + assert_between_inclusive: false, + assert_regexp_match: false, + assert_class_string: false, + assert_exists: false, + assert_own_property: false, + assert_not_exists: false, + assert_inherits: false, + assert_idl_attribute: false, + assert_readonly: false, + assert_throws: false, + assert_unreaded: false, + assert_any: false, + fetch_tests_from_worker: false, + timeout: false, + add_start_callback: false, + add_test_state_callback: false, + add_result_callback: false, + add_completion_callback: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/utils.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/utils.js new file mode 100644 index 0000000000..aeda690ba5 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/utils.js @@ -0,0 +1,62 @@ +/** + * @fileoverview Provides utilities for setting up environments. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var path = require("path"); +var helpers = require("../helpers"); +var globals = require("../globals"); + +/** + * Obtains the globals for a list of files. + * + * @param {Array.<String>} files + * The array of files to get globals for. The paths are relative to the topsrcdir. + * @returns {Object} + * Returns an object with keys of the global names and values of if they are + * writable or not. + */ +function getGlobalsForScripts(environmentName, files, extraDefinitions) { + let fileGlobals = extraDefinitions; + const root = helpers.rootDir; + for (const file of files) { + const fileName = path.join(root, file); + try { + fileGlobals = fileGlobals.concat(globals.getGlobalsForFile(fileName)); + } catch (e) { + console.error(`Could not load globals from file ${fileName}: ${e}`); + console.error( + `You may need to update the mappings for the ${environmentName} environment` + ); + throw new Error(`Could not load globals from file ${fileName}: ${e}`); + } + } + + var globalObjects = {}; + for (let global of fileGlobals) { + globalObjects[global.name] = global.writable; + } + return globalObjects; +} + +module.exports = { + getScriptGlobals( + environmentName, + files, + extraDefinitions = [], + extraEnv = {} + ) { + if (helpers.isMozillaCentralBased()) { + return { + globals: getGlobalsForScripts(environmentName, files, extraDefinitions), + ...extraEnv, + }; + } + return helpers.getSavedEnvironmentItems(environmentName); + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/xpcshell.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/xpcshell.js new file mode 100644 index 0000000000..408bc2e277 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/xpcshell.js @@ -0,0 +1,59 @@ +/** + * @fileoverview Defines the environment for xpcshell test files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var { getScriptGlobals } = require("./utils"); + +const extraGlobals = [ + // Defined in XPCShellImpl.cpp + "print", + "readline", + "load", + "quit", + "dumpXPC", + "dump", + "gc", + "gczeal", + "options", + "sendCommand", + "atob", + "btoa", + "setInterruptCallback", + "simulateNoScriptActivity", + "registerXPCTestComponents", + + // Assert.sys.mjs globals. + "setReporter", + "report", + "ok", + "equal", + "notEqual", + "deepEqual", + "notDeepEqual", + "strictEqual", + "notStrictEqual", + "throws", + "rejects", + "greater", + "greaterOrEqual", + "less", + "lessOrEqual", + // TestingFunctions.cpp globals + "allocationMarker", + "byteSize", + "saveStack", +]; + +module.exports = getScriptGlobals( + "xpcshell", + ["testing/xpcshell/head.js"], + extraGlobals.map(g => { + return { name: g, writable: false }; + }) +); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js new file mode 100644 index 0000000000..25c149fa9f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js @@ -0,0 +1,668 @@ +/** + * @fileoverview functions for scanning an AST for globals including + * traversing referenced scripts. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const path = require("path"); +const fs = require("fs"); +const helpers = require("./helpers"); +const htmlparser = require("htmlparser2"); +const testharnessEnvironment = require("./environments/testharness.js"); + +const callExpressionDefinitions = [ + /^loader\.lazyGetter\((?:globalThis|this), "(\w+)"/, + /^loader\.lazyServiceGetter\((?:globalThis|this), "(\w+)"/, + /^loader\.lazyRequireGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyGetter\((?:globalThis|this), "(\w+)"/, + /^ChromeUtils\.defineLazyGetter\((?:globalThis|this), "(\w+)"/, + /^ChromeUtils\.defineModuleGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyPreferenceGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyScriptGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyServiceGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineConstant\((?:globalThis|this), "(\w+)"/, + /^DevToolsUtils\.defineLazyGetter\((?:globalThis|this), "(\w+)"/, + /^Object\.defineProperty\((?:globalThis|this), "(\w+)"/, + /^Reflect\.defineProperty\((?:globalThis|this), "(\w+)"/, + /^this\.__defineGetter__\("(\w+)"/, +]; + +const callExpressionMultiDefinitions = [ + "XPCOMUtils.defineLazyGlobalGetters(this,", + "XPCOMUtils.defineLazyGlobalGetters(globalThis,", + "XPCOMUtils.defineLazyModuleGetters(this,", + "XPCOMUtils.defineLazyModuleGetters(globalThis,", + "XPCOMUtils.defineLazyServiceGetters(this,", + "XPCOMUtils.defineLazyServiceGetters(globalThis,", + "ChromeUtils.defineESModuleGetters(this,", + "ChromeUtils.defineESModuleGetters(globalThis,", + "loader.lazyRequireGetter(this,", + "loader.lazyRequireGetter(globalThis,", +]; + +const subScriptMatches = [ + /Services\.scriptloader\.loadSubScript\("(.*?)", this\)/, +]; + +const workerImportFilenameMatch = /(.*\/)*((.*?)\.jsm?)/; + +/** + * Parses a list of "name:boolean_value" or/and "name" options divided by comma + * or whitespace. + * + * This function was copied from eslint.js + * + * @param {string} string The string to parse. + * @param {Comment} comment The comment node which has the string. + * @returns {Object} Result map object of names and boolean values + */ +function parseBooleanConfig(string, comment) { + let items = {}; + + // Collapse whitespace around : to make parsing easier + string = string.replace(/\s*:\s*/g, ":"); + // Collapse whitespace around , + string = string.replace(/\s*,\s*/g, ","); + + string.split(/\s|,+/).forEach(function (name) { + if (!name) { + return; + } + + let pos = name.indexOf(":"); + let value; + if (pos !== -1) { + value = name.substring(pos + 1, name.length); + name = name.substring(0, pos); + } + + items[name] = { + value: value === "true", + comment, + }; + }); + + return items; +} + +/** + * Global discovery can require parsing many files. This map of + * {String} => {Object} caches what globals were discovered for a file path. + */ +const globalCache = new Map(); + +/** + * Global discovery can occasionally meet circular dependencies due to the way + * js files are included via html/xhtml files etc. This set is used to avoid + * getting into loops whilst the discovery is in progress. + */ +var globalDiscoveryInProgressForFiles = new Set(); + +/** + * When looking for globals in HTML files, it can be common to have more than + * one script tag with inline javascript. These will normally be called together, + * so we store the globals for just the last HTML file processed. + */ +var lastHTMLGlobals = {}; + +/** + * Attempts to convert an CallExpressions that look like module imports + * into global variable definitions. + * + * @param {Object} node + * The AST node to convert. + * @param {boolean} isGlobal + * True if the current node is in the global scope. + * + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ +function convertCallExpressionToGlobals(node, isGlobal) { + let express = node.expression; + if ( + express.type === "CallExpression" && + express.callee.type === "MemberExpression" && + express.callee.object && + express.callee.object.type === "Identifier" && + express.arguments.length === 1 && + express.arguments[0].type === "ArrayExpression" && + express.callee.property.type === "Identifier" && + express.callee.property.name === "importGlobalProperties" + ) { + return express.arguments[0].elements.map(literal => { + return { + explicit: true, + name: literal.value, + writable: false, + }; + }); + } + + let source; + try { + source = helpers.getASTSource(node); + } catch (e) { + return []; + } + + // The definition matches below must be in the global scope for us to define + // a global, so bail out early if we're not a global. + if (!isGlobal) { + return []; + } + + for (let reg of subScriptMatches) { + let match = source.match(reg); + if (match) { + return getGlobalsForScript(match[1], "script").map(g => { + // We don't want any loadSubScript globals to be explicit, as this + // could trigger no-unused-vars when importing multiple variables + // from a script and not using all of them. + g.explicit = false; + return g; + }); + } + } + + for (let reg of callExpressionDefinitions) { + let match = source.match(reg); + if (match) { + return [{ name: match[1], writable: true, explicit: true }]; + } + } + + if ( + callExpressionMultiDefinitions.some(expr => source.startsWith(expr)) && + node.expression.arguments[1] + ) { + let arg = node.expression.arguments[1]; + if (arg.type === "ObjectExpression") { + return arg.properties + .map(p => ({ + name: p.type === "Property" && p.key.name, + writable: true, + explicit: true, + })) + .filter(g => g.name); + } + if (arg.type === "ArrayExpression") { + return arg.elements + .map(p => ({ + name: p.type === "Literal" && p.value, + writable: true, + explicit: true, + })) + .filter(g => typeof g.name == "string"); + } + } + + if ( + node.expression.callee.type == "MemberExpression" && + node.expression.callee.property.type == "Identifier" && + node.expression.callee.property.name == "defineLazyScriptGetter" + ) { + // The case where we have a single symbol as a string has already been + // handled by the regexp, so we have an array of symbols here. + return node.expression.arguments[1].elements.map(n => ({ + name: n.value, + writable: true, + explicit: true, + })); + } + + return []; +} + +/** + * Attempts to convert an AssignmentExpression into a global variable + * definition if it applies to `this` in the global scope. + * + * @param {Object} node + * The AST node to convert. + * @param {boolean} isGlobal + * True if the current node is in the global scope. + * + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ +function convertThisAssignmentExpressionToGlobals(node, isGlobal) { + if ( + isGlobal && + node.expression.left && + node.expression.left.object && + node.expression.left.object.type === "ThisExpression" && + node.expression.left.property && + node.expression.left.property.type === "Identifier" + ) { + return [{ name: node.expression.left.property.name, writable: true }]; + } + return []; +} + +/** + * Attempts to convert an ExpressionStatement to likely global variable + * definitions. + * + * @param {Object} node + * The AST node to convert. + * @param {boolean} isGlobal + * True if the current node is in the global scope. + * + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ +function convertWorkerExpressionToGlobals(node, isGlobal, dirname) { + let results = []; + let expr = node.expression; + + if ( + node.expression.type === "CallExpression" && + expr.callee && + expr.callee.type === "Identifier" && + expr.callee.name === "importScripts" + ) { + for (var arg of expr.arguments) { + var match = arg.value && arg.value.match(workerImportFilenameMatch); + if (match) { + if (!match[1]) { + let filePath = path.resolve(dirname, match[2]); + if (fs.existsSync(filePath)) { + let additionalGlobals = module.exports.getGlobalsForFile(filePath); + results = results.concat(additionalGlobals); + } + } + // Import with relative/absolute path should explicitly use + // `import-globals-from` comment. + } + } + } + + return results; +} + +/** + * Attempts to load the globals for a given script. + * + * @param {string} src + * The source path or url of the script to look for. + * @param {string} type + * The type of the current file (script/module). + * @param {string} [dir] + * The directory of the current file. + * @returns {object[]} + * An array of objects with details of the globals in them. + */ +function getGlobalsForScript(src, type, dir) { + let scriptName; + if (src.includes("http:")) { + // We don't handle this currently as the paths are complex to match. + } else if (src.startsWith("chrome://mochikit/content/")) { + // Various ways referencing test files. + src = src.replace("chrome://mochikit/content/", "/"); + scriptName = path.join(helpers.rootDir, "testing", "mochitest", src); + } else if (src.startsWith("chrome://mochitests/content/browser")) { + src = src.replace("chrome://mochitests/content/browser", ""); + scriptName = path.join(helpers.rootDir, src); + } else if (src.includes("SimpleTest")) { + // This is another way of referencing test files... + scriptName = path.join(helpers.rootDir, "testing", "mochitest", src); + } else if (src.startsWith("/tests/")) { + scriptName = path.join(helpers.rootDir, src.substring(7)); + } else if (src.startsWith("/resources/testharness.js")) { + return Object.keys(testharnessEnvironment.globals).map(name => ({ + name, + writable: true, + })); + } else if (dir) { + // Fallback to hoping this is a relative path. + scriptName = path.join(dir, src); + } + if (scriptName && fs.existsSync(scriptName)) { + return module.exports.getGlobalsForFile(scriptName, { + ecmaVersion: helpers.getECMAVersion(), + sourceType: type, + }); + } + return []; +} + +/** + * An object that returns found globals for given AST node types. Each prototype + * property should be named for a node type and accepts a node parameter and a + * parents parameter which is a list of the parent nodes of the current node. + * Each returns an array of globals found. + * + * @param {String} filePath + * The absolute path of the file being parsed. + */ +function GlobalsForNode(filePath, context) { + this.path = filePath; + this.context = context; + + if (this.path) { + this.dirname = path.dirname(this.path); + } else { + this.dirname = null; + } +} + +GlobalsForNode.prototype = { + Program(node) { + let globals = []; + for (let comment of node.comments) { + if (comment.type !== "Block") { + continue; + } + let value = comment.value.trim(); + value = value.replace(/\n/g, ""); + + // We have to discover any globals that ESLint would have defined through + // comment directives. + let match = /^globals?\s+(.+)/.exec(value); + if (match) { + let values = parseBooleanConfig(match[1].trim(), node); + for (let name of Object.keys(values)) { + globals.push({ + name, + writable: values[name].value, + }); + } + // We matched globals, so we won't match import-globals-from. + continue; + } + + match = /^import-globals-from\s+(.+)$/.exec(value); + if (!match) { + continue; + } + + if (!this.dirname) { + // If this is testing context without path, ignore import. + return globals; + } + + let filePath = match[1].trim(); + + if (filePath.endsWith(".mjs")) { + if (this.context) { + this.context.report( + comment, + "import-globals-from does not support module files - use a direct import instead" + ); + } else { + // Fall back to throwing an error, as we do not have a context in all situations, + // e.g. when loading the environment. + throw new Error( + "import-globals-from does not support module files - use a direct import instead" + ); + } + continue; + } + + if (!path.isAbsolute(filePath)) { + filePath = path.resolve(this.dirname, filePath); + } else { + filePath = path.join(helpers.rootDir, filePath); + } + globals = globals.concat(module.exports.getGlobalsForFile(filePath)); + } + + return globals; + }, + + ExpressionStatement(node, parents, globalScope) { + let isGlobal = helpers.getIsGlobalThis(parents); + let globals = []; + + // Note: We check the expression types here and only call the necessary + // functions to aid performance. + if (node.expression.type === "AssignmentExpression") { + globals = convertThisAssignmentExpressionToGlobals(node, isGlobal); + } else if (node.expression.type === "CallExpression") { + globals = convertCallExpressionToGlobals(node, isGlobal); + } + + // Here we assume that if importScripts is set in the global scope, then + // this is a worker. It would be nice if eslint gave us a way of getting + // the environment directly. + // + // If this is testing context without path, ignore import. + if (globalScope && globalScope.set.get("importScripts") && this.dirname) { + let workerDetails = convertWorkerExpressionToGlobals( + node, + isGlobal, + this.dirname + ); + globals = globals.concat(workerDetails); + } + + return globals; + }, +}; + +module.exports = { + /** + * Returns all globals for a given file. Recursively searches through + * import-globals-from directives and also includes globals defined by + * standard eslint directives. + * + * @param {String} filePath + * The absolute path of the file to be parsed. + * @param {Object} astOptions + * Extra options to pass to the parser. + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ + getGlobalsForFile(filePath, astOptions = {}) { + if (globalCache.has(filePath)) { + return globalCache.get(filePath); + } + + if (globalDiscoveryInProgressForFiles.has(filePath)) { + // We're already processing this file, so return an empty set for now - + // the initial processing will pick up on the globals for this file. + return []; + } + globalDiscoveryInProgressForFiles.add(filePath); + + let content = fs.readFileSync(filePath, "utf8"); + + // Parse the content into an AST + let { ast, scopeManager, visitorKeys } = helpers.parseCode( + content, + astOptions + ); + + // Discover global declarations + let globalScope = scopeManager.acquire(ast); + + let globals = Object.keys(globalScope.variables).map(v => ({ + name: globalScope.variables[v].name, + writable: true, + })); + + // Walk over the AST to find any of our custom globals + let handler = new GlobalsForNode(filePath); + + helpers.walkAST(ast, visitorKeys, (type, node, parents) => { + if (type in handler) { + let newGlobals = handler[type](node, parents, globalScope); + globals.push.apply(globals, newGlobals); + } + }); + + globalCache.set(filePath, globals); + + globalDiscoveryInProgressForFiles.delete(filePath); + return globals; + }, + + /** + * Returns all globals for a code. + * This is only for testing. + * + * @param {String} code + * The JS code + * @param {Object} astOptions + * Extra options to pass to the parser. + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ + getGlobalsForCode(code, astOptions = {}) { + // Parse the content into an AST + let { ast, scopeManager, visitorKeys } = helpers.parseCode( + code, + astOptions, + { useBabel: false } + ); + + // Discover global declarations + let globalScope = scopeManager.acquire(ast); + + let globals = Object.keys(globalScope.variables).map(v => ({ + name: globalScope.variables[v].name, + writable: true, + })); + + // Walk over the AST to find any of our custom globals + let handler = new GlobalsForNode(null); + + helpers.walkAST(ast, visitorKeys, (type, node, parents) => { + if (type in handler) { + let newGlobals = handler[type](node, parents, globalScope); + globals.push.apply(globals, newGlobals); + } + }); + + return globals; + }, + + /** + * Returns all the globals for an html file that are defined by imported + * scripts (i.e. <script src="foo.js">). + * + * This function will cache results for one html file only - we expect + * this to be called sequentially for each chunk of a HTML file, rather + * than chucks of different files in random order. + * + * @param {String} filePath + * The absolute path of the file to be parsed. + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ + getImportedGlobalsForHTMLFile(filePath) { + if (lastHTMLGlobals.filename === filePath) { + return lastHTMLGlobals.globals; + } + + let dir = path.dirname(filePath); + let globals = []; + + let content = fs.readFileSync(filePath, "utf8"); + let scriptSrcs = []; + + // We use htmlparser as this ensures we find the script tags correctly. + let parser = new htmlparser.Parser( + { + onopentag(name, attribs) { + if (name === "script" && "src" in attribs) { + scriptSrcs.push({ + src: attribs.src, + type: + "type" in attribs && attribs.type == "module" + ? "module" + : "script", + }); + } + }, + }, + { + xmlMode: filePath.endsWith("xhtml"), + } + ); + + parser.parseComplete(content); + + for (let script of scriptSrcs) { + // Ensure that the script src isn't just "". + if (!script.src) { + continue; + } + globals.push(...getGlobalsForScript(script.src, script.type, dir)); + } + + lastHTMLGlobals.filePath = filePath; + return (lastHTMLGlobals.globals = globals); + }, + + /** + * Intended to be used as-is for an ESLint rule that parses for globals in + * the current file and recurses through import-globals-from directives. + * + * @param {Object} context + * The ESLint parsing context. + */ + getESLintGlobalParser(context) { + let globalScope; + + let parser = { + Program(node) { + globalScope = context.getScope(); + }, + }; + let filename = context.getFilename(); + + let extraHTMLGlobals = []; + if (filename.endsWith(".html") || filename.endsWith(".xhtml")) { + extraHTMLGlobals = module.exports.getImportedGlobalsForHTMLFile(filename); + } + + // Install thin wrappers around GlobalsForNode + let handler = new GlobalsForNode(helpers.getAbsoluteFilePath(context)); + + for (let type of Object.keys(GlobalsForNode.prototype)) { + parser[type] = function (node) { + if (type === "Program") { + globalScope = context.getScope(); + helpers.addGlobals(extraHTMLGlobals, globalScope); + } + let globals = handler[type](node, context.getAncestors(), globalScope); + helpers.addGlobals( + globals, + globalScope, + node.type !== "Program" && node + ); + }; + } + + return parser; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/helpers.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/helpers.js new file mode 100644 index 0000000000..dc4106631a --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/helpers.js @@ -0,0 +1,797 @@ +/** + * @fileoverview A collection of helper functions. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +"use strict"; + +const parser = require("espree"); +const { analyze } = require("eslint-scope"); +const { KEYS: defaultVisitorKeys } = require("eslint-visitor-keys"); +const estraverse = require("estraverse"); +const path = require("path"); +const fs = require("fs"); +const toml = require("toml-eslint-parser"); +const recommendedConfig = require("./configs/recommended"); + +var gRootDir = null; +var directoryManifests = new Map(); + +let xpidlData; + +module.exports = { + get servicesData() { + return require("./services.json"); + }, + + /** + * Obtains xpidl data from the object directory specified in the + * environment. + * + * @returns {Map<string, object>} + * A map of interface names to the interface details. + */ + get xpidlData() { + let xpidlDir; + + if (process.env.TASK_ID && !process.env.MOZ_XPT_ARTIFACTS_DIR) { + throw new Error( + "MOZ_XPT_ARTIFACTS_DIR must be set for this rule in automation" + ); + } + xpidlDir = process.env.MOZ_XPT_ARTIFACTS_DIR; + + if (!xpidlDir && process.env.MOZ_OBJDIR) { + xpidlDir = `${process.env.MOZ_OBJDIR}/dist/xpt_artifacts/`; + if (!fs.existsSync(xpidlDir)) { + xpidlDir = `${process.env.MOZ_OBJDIR}/config/makefiles/xpidl/`; + } + } + if (!xpidlDir) { + throw new Error( + "MOZ_OBJDIR must be defined in the environment for this rule, i.e. MOZ_OBJDIR=objdir-ff ./mach ..." + ); + } + if (xpidlData) { + return xpidlData; + } + let files = fs.readdirSync(`${xpidlDir}`); + // `Makefile` is an expected file in the directory. + if (files.length <= 1) { + throw new Error("Missing xpidl data files, maybe you need to build?"); + } + xpidlData = new Map(); + for (let file of files) { + if (!file.endsWith(".xpt")) { + continue; + } + let data = JSON.parse(fs.readFileSync(path.join(`${xpidlDir}`, file))); + for (let details of data) { + xpidlData.set(details.name, details); + } + } + return xpidlData; + }, + + /** + * Gets the abstract syntax tree (AST) of the JavaScript source code contained + * in sourceText. This matches the results for an eslint parser, see + * https://eslint.org/docs/developer-guide/working-with-custom-parsers. + * + * @param {String} sourceText + * Text containing valid JavaScript. + * @param {Object} astOptions + * Extra configuration to pass to the espree parser, these will override + * the configuration from getPermissiveConfig(). + * @param {Object} configOptions + * Extra options for getPermissiveConfig(). + * + * @return {Object} + * Returns an object containing `ast`, `scopeManager` and + * `visitorKeys` + */ + parseCode(sourceText, astOptions = {}, configOptions = {}) { + // Use a permissive config file to allow parsing of anything that Espree + // can parse. + let config = { ...this.getPermissiveConfig(configOptions), ...astOptions }; + + let parseResult = parser.parse(sourceText, config); + + let visitorKeys = parseResult.visitorKeys || defaultVisitorKeys; + + // eslint-scope doesn't support "latest" as a version, so we pass a really + // big number to ensure this always reads as the latest. + // xref https://github.com/eslint/eslint-scope/issues/74 + config.ecmaVersion = + config.ecmaVersion == "latest" ? 1e8 : config.ecmaVersion; + + return { + ast: parseResult, + scopeManager: parseResult.scopeManager || analyze(parseResult, config), + visitorKeys, + }; + }, + + /** + * A simplistic conversion of some AST nodes to a standard string form. + * + * @param {Object} node + * The AST node to convert. + * + * @return {String} + * The JS source for the node. + */ + getASTSource(node, context) { + switch (node.type) { + case "MemberExpression": + if (node.computed) { + let filename = context && context.getFilename(); + throw new Error( + `getASTSource unsupported computed MemberExpression in ${filename}` + ); + } + return ( + this.getASTSource(node.object) + + "." + + this.getASTSource(node.property) + ); + case "ThisExpression": + return "this"; + case "Identifier": + return node.name; + case "Literal": + return JSON.stringify(node.value); + case "CallExpression": + var args = node.arguments.map(a => this.getASTSource(a)).join(", "); + return this.getASTSource(node.callee) + "(" + args + ")"; + case "ObjectExpression": + return "{}"; + case "ExpressionStatement": + return this.getASTSource(node.expression) + ";"; + case "FunctionExpression": + return "function() {}"; + case "ArrayExpression": + return "[" + node.elements.map(this.getASTSource, this).join(",") + "]"; + case "ArrowFunctionExpression": + return "() => {}"; + case "AssignmentExpression": + return ( + this.getASTSource(node.left) + " = " + this.getASTSource(node.right) + ); + case "BinaryExpression": + return ( + this.getASTSource(node.left) + + " " + + node.operator + + " " + + this.getASTSource(node.right) + ); + case "UnaryExpression": + return node.operator + " " + this.getASTSource(node.argument); + default: + throw new Error("getASTSource unsupported node type: " + node.type); + } + }, + + /** + * This walks an AST in a manner similar to ESLint passing node events to the + * listener. The listener is expected to be a simple function + * which accepts node type, node and parents arguments. + * + * @param {Object} ast + * The AST to walk. + * @param {Array} visitorKeys + * The visitor keys to use for the AST. + * @param {Function} listener + * A callback function to call for the nodes. Passed three arguments, + * event type, node and an array of parent nodes for the current node. + */ + walkAST(ast, visitorKeys, listener) { + let parents = []; + + estraverse.traverse(ast, { + enter(node, parent) { + listener(node.type, node, parents); + + parents.push(node); + }, + + leave(node, parent) { + if (!parents.length) { + throw new Error("Left more nodes than entered."); + } + parents.pop(); + }, + + keys: visitorKeys, + }); + if (parents.length) { + throw new Error("Entered more nodes than left."); + } + }, + + /** + * Add a variable to the current scope. + * HACK: This relies on eslint internals so it could break at any time. + * + * @param {String} name + * The variable name to add to the scope. + * @param {ASTScope} scope + * The scope to add to. + * @param {boolean} writable + * Whether the global can be overwritten. + * @param {Object} [node] + * The AST node that defined the globals. + */ + addVarToScope(name, scope, writable, node) { + scope.__defineGeneric(name, scope.set, scope.variables, null, null); + + let variable = scope.set.get(name); + variable.eslintExplicitGlobal = false; + variable.writeable = writable; + if (node) { + variable.defs.push({ + type: "Variable", + node, + name: { name, parent: node.parent }, + }); + variable.identifiers.push(node); + } + + // Walk to the global scope which holds all undeclared variables. + while (scope.type != "global") { + scope = scope.upper; + } + + // "through" contains all references with no found definition. + scope.through = scope.through.filter(function (reference) { + if (reference.identifier.name != name) { + return true; + } + + // Links the variable and the reference. + // And this reference is removed from `Scope#through`. + reference.resolved = variable; + variable.references.push(reference); + return false; + }); + }, + + /** + * Adds a set of globals to a scope. + * + * @param {Array} globalVars + * An array of global variable names. + * @param {ASTScope} scope + * The scope. + * @param {Object} [node] + * The AST node that defined the globals. + */ + addGlobals(globalVars, scope, node) { + globalVars.forEach(v => + this.addVarToScope(v.name, scope, v.writable, v.explicit && node) + ); + }, + + /** + * To allow espree to parse almost any JavaScript we need as many features as + * possible turned on. This method returns that config. + * + * @param {Object} options + * { + * useBabel: {boolean} whether to set babelOptions. + * } + * @return {Object} + * Espree compatible permissive config. + */ + getPermissiveConfig({ useBabel = true } = {}) { + return { + range: true, + loc: true, + comment: true, + attachComment: true, + ecmaVersion: this.getECMAVersion(), + sourceType: "script", + }; + }, + + /** + * Returns the ECMA version of the recommended config. + * + * @return {Number} The ECMA version of the recommended config. + */ + getECMAVersion() { + return recommendedConfig.parserOptions.ecmaVersion; + }, + + /** + * Check whether it's inside top-level script. + * + * @param {Array} ancestors + * The parents of the current node. + * + * @return {Boolean} + * True or false + */ + getIsTopLevelScript(ancestors) { + for (let parent of ancestors) { + switch (parent.type) { + case "ArrowFunctionExpression": + case "FunctionDeclaration": + case "FunctionExpression": + case "PropertyDefinition": + case "StaticBlock": + return false; + } + } + return true; + }, + + isTopLevel(ancestors) { + for (let parent of ancestors) { + switch (parent.type) { + case "ArrowFunctionExpression": + case "FunctionDeclaration": + case "FunctionExpression": + case "PropertyDefinition": + case "StaticBlock": + case "BlockStatement": + return false; + } + } + return true; + }, + + /** + * Check whether `this` expression points the global this. + * + * @param {Array} ancestors + * The parents of the current node. + * + * @return {Boolean} + * True or false + */ + getIsGlobalThis(ancestors) { + for (let parent of ancestors) { + switch (parent.type) { + case "FunctionDeclaration": + case "FunctionExpression": + case "PropertyDefinition": + case "StaticBlock": + return false; + } + } + return true; + }, + + /** + * Check whether the node is evaluated at top-level script unconditionally. + * + * @param {Array} ancestors + * The parents of the current node. + * + * @return {Boolean} + * True or false + */ + getIsTopLevelAndUnconditionallyExecuted(ancestors) { + for (let parent of ancestors) { + switch (parent.type) { + // Control flow + case "IfStatement": + case "SwitchStatement": + case "TryStatement": + case "WhileStatement": + case "DoWhileStatement": + case "ForStatement": + case "ForInStatement": + case "ForOfStatement": + return false; + + // Function + case "FunctionDeclaration": + case "FunctionExpression": + case "ArrowFunctionExpression": + case "ClassBody": + return false; + + // Branch + case "LogicalExpression": + case "ConditionalExpression": + case "ChainExpression": + return false; + + case "AssignmentExpression": + switch (parent.operator) { + // Branch + case "||=": + case "&&=": + case "??=": + return false; + } + break; + + // Implicit branch (default value) + case "ObjectPattern": + case "ArrayPattern": + return false; + } + } + return true; + }, + + /** + * Check whether we might be in a test head file. + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsHeadFile(context) + * + * @return {Boolean} + * True or false + */ + getIsHeadFile(scope) { + var pathAndFilename = this.cleanUpPath(scope.getFilename()); + + return /.*[\\/]head(_.+)?\.js$/.test(pathAndFilename); + }, + + /** + * Gets the head files for a potential test file + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsHeadFile(context) + * + * @return {String[]} + * Paths to head files to load for the test + */ + getTestHeadFiles(scope) { + if (!this.getIsTest(scope)) { + return []; + } + + let filepath = this.cleanUpPath(scope.getFilename()); + let dir = path.dirname(filepath); + + let names = fs + .readdirSync(dir) + .filter( + name => + (name.startsWith("head") || name.startsWith("xpcshell-head")) && + name.endsWith(".js") + ) + .map(name => path.join(dir, name)); + return names; + }, + + /** + * Gets all the test manifest data for a directory + * + * @param {String} dir + * The directory + * + * @return {Array} + * An array of objects with file and manifest properties + */ + getManifestsForDirectory(dir) { + if (directoryManifests.has(dir)) { + return directoryManifests.get(dir); + } + + let manifests = []; + let names = []; + try { + names = fs.readdirSync(dir); + } catch (err) { + // Ignore directory not found, it might be faked by a test + if (err.code !== "ENOENT") { + throw err; + } + } + + for (let name of names) { + if (name.endsWith(".toml")) { + try { + const ast = toml.parseTOML( + fs.readFileSync(path.join(dir, name), "utf8") + ); + var manifest = {}; + ast.body.forEach(top => { + if (top.type == "TOMLTopLevelTable") { + top.body.forEach(obj => { + if (obj.type == "TOMLTable") { + manifest[obj.resolvedKey] = {}; + } + }); + } + }); + manifests.push({ + file: path.join(dir, name), + manifest, + }); + } catch (e) { + console.log( + "TOML ERROR: " + + e.message + + " @line: " + + e.lineNumber + + ", column: " + + e.column + ); + } + } + } + + directoryManifests.set(dir, manifests); + return manifests; + }, + + /** + * Gets the manifest file a test is listed in + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsHeadFile(context) + * + * @return {String} + * The path to the test manifest file + */ + getTestManifest(scope) { + let filepath = this.cleanUpPath(scope.getFilename()); + + let dir = path.dirname(filepath); + let filename = path.basename(filepath); + + for (let manifest of this.getManifestsForDirectory(dir)) { + if (filename in manifest.manifest) { + return manifest.file; + } + } + + return null; + }, + + /** + * Check whether we are in a test of some kind. + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsTest(context) + * + * @return {Boolean} + * True or false + */ + getIsTest(scope) { + // Regardless of the manifest name being in a manifest means we're a test. + let manifest = this.getTestManifest(scope); + if (manifest) { + return true; + } + + return !!this.getTestType(scope); + }, + + /* + * Check if this is an .sjs file. + */ + getIsSjs(scope) { + let filepath = this.cleanUpPath(scope.getFilename()); + + return path.extname(filepath) == ".sjs"; + }, + + /** + * Gets the type of test or null if this isn't a test. + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsHeadFile(context) + * + * @return {String or null} + * Test type: xpcshell, browser, chrome, mochitest + */ + getTestType(scope) { + let testTypes = ["browser", "xpcshell", "chrome", "mochitest", "a11y"]; + let manifest = this.getTestManifest(scope); + if (manifest) { + let name = path.basename(manifest); + for (let testType of testTypes) { + if (name.startsWith(testType)) { + return testType; + } + } + } + + let filepath = this.cleanUpPath(scope.getFilename()); + let filename = path.basename(filepath); + + if (filename.startsWith("browser_")) { + return "browser"; + } + + if (filename.startsWith("test_")) { + let parent = path.basename(path.dirname(filepath)); + for (let testType of testTypes) { + if (parent.startsWith(testType)) { + return testType; + } + } + + // It likely is a test, we're just not sure what kind. + return "unknown"; + } + + // Likely not a test + return null; + }, + + getIsWorker(filePath) { + let filename = path.basename(this.cleanUpPath(filePath)).toLowerCase(); + + return filename.includes("worker"); + }, + + /** + * Gets the root directory of the repository by walking up directories from + * this file until a .eslintignore file is found. If this fails, the same + * procedure will be attempted from the current working dir. + * @return {String} The absolute path of the repository directory + */ + get rootDir() { + if (!gRootDir) { + function searchUpForIgnore(dirName, filename) { + let parsed = path.parse(dirName); + while (parsed.root !== dirName) { + if (fs.existsSync(path.join(dirName, filename))) { + return dirName; + } + // Move up a level + dirName = parsed.dir; + parsed = path.parse(dirName); + } + return null; + } + + let possibleRoot = searchUpForIgnore( + path.dirname(module.filename), + ".eslintignore" + ); + if (!possibleRoot) { + possibleRoot = searchUpForIgnore(path.resolve(), ".eslintignore"); + } + if (!possibleRoot) { + possibleRoot = searchUpForIgnore(path.resolve(), "package.json"); + } + if (!possibleRoot) { + // We've couldn't find a root from the module or CWD, so lets just go + // for the CWD. We really don't want to throw if possible, as that + // tends to give confusing results when used with ESLint. + possibleRoot = process.cwd(); + } + + gRootDir = possibleRoot; + } + + return gRootDir; + }, + + /** + * ESLint may be executed from various places: from mach, at the root of the + * repository, or from a directory in the repository when, for instance, + * executed by a text editor's plugin. + * The value returned by context.getFileName() varies because of this. + * This helper function makes sure to return an absolute file path for the + * current context, by looking at process.cwd(). + * @param {Context} context + * @return {String} The absolute path + */ + getAbsoluteFilePath(context) { + var fileName = this.cleanUpPath(context.getFilename()); + var cwd = process.cwd(); + + if (path.isAbsolute(fileName)) { + // Case 2: executed from the repo's root with mach: + // fileName: /path/to/mozilla/repo/a/b/c/d.js + // cwd: /path/to/mozilla/repo + return fileName; + } else if (path.basename(fileName) == fileName) { + // Case 1b: executed from a nested directory, fileName is the base name + // without any path info (happens in Atom with linter-eslint) + return path.join(cwd, fileName); + } + // Case 1: executed form in a nested directory, e.g. from a text editor: + // fileName: a/b/c/d.js + // cwd: /path/to/mozilla/repo/a/b/c + var dirName = path.dirname(fileName); + return cwd.slice(0, cwd.length - dirName.length) + fileName; + }, + + /** + * When ESLint is run from SublimeText, paths retrieved from + * context.getFileName contain leading and trailing double-quote characters. + * These characters need to be removed. + */ + cleanUpPath(pathName) { + return pathName.replace(/^"/, "").replace(/"$/, ""); + }, + + get globalScriptPaths() { + return [ + path.join(this.rootDir, "browser", "base", "content", "browser.xhtml"), + path.join( + this.rootDir, + "browser", + "base", + "content", + "global-scripts.inc" + ), + ]; + }, + + isMozillaCentralBased() { + return fs.existsSync(this.globalScriptPaths[0]); + }, + + getSavedEnvironmentItems(environment) { + return require("./environments/saved-globals.json").environments[ + environment + ]; + }, + + getSavedRuleData(rule) { + return require("./rules/saved-rules-data.json").rulesData[rule]; + }, + + getBuildEnvironment() { + var { execFileSync } = require("child_process"); + var output = execFileSync( + path.join(this.rootDir, "mach"), + ["environment", "--format=json"], + { silent: true } + ); + return JSON.parse(output); + }, + + /** + * Extract the path of require (and require-like) helpers used in DevTools. + */ + getDevToolsRequirePath(node) { + if ( + node.callee.type == "Identifier" && + node.callee.name == "require" && + node.arguments.length == 1 && + node.arguments[0].type == "Literal" + ) { + return node.arguments[0].value; + } else if ( + node.callee.type == "MemberExpression" && + node.callee.property.type == "Identifier" && + node.callee.property.name == "lazyRequireGetter" && + node.arguments.length >= 3 && + node.arguments[2].type == "Literal" + ) { + return node.arguments[2].value; + } + return null; + }, + + /** + * Returns property name from MemberExpression. Also accepts Identifier for consistency. + * @param {import("estree").MemberExpression | import("estree").Identifier} node + * @returns {string | null} + * + * @example `foo` gives "foo" + * @example `foo.bar` gives "bar" + * @example `foo.bar.baz` gives "baz" + */ + maybeGetMemberPropertyName(node) { + if (node.type === "MemberExpression") { + return node.property.name; + } + if (node.type === "Identifier") { + return node.name; + } + return null; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/index.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/index.js new file mode 100644 index 0000000000..0801958597 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/index.js @@ -0,0 +1,102 @@ +/** + * @fileoverview A collection of rules that help enforce JavaScript coding + * standard and avoid common errors in the Mozilla project. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Plugin Definition +// ------------------------------------------------------------------------------ +module.exports = { + configs: { + "browser-test": require("../lib/configs/browser-test"), + "chrome-test": require("../lib/configs/chrome-test"), + "mochitest-test": require("../lib/configs/mochitest-test"), + recommended: require("../lib/configs/recommended"), + "require-jsdoc": require("../lib/configs/require-jsdoc"), + "valid-jsdoc": require("../lib/configs/valid-jsdoc"), + "xpcshell-test": require("../lib/configs/xpcshell-test"), + }, + environments: { + "browser-window": require("../lib/environments/browser-window.js"), + "chrome-script": require("../lib/environments/chrome-script.js"), + "chrome-worker": require("../lib/environments/chrome-worker.js"), + "frame-script": require("../lib/environments/frame-script.js"), + jsm: require("../lib/environments/jsm.js"), + privileged: require("../lib/environments/privileged.js"), + "process-script": require("../lib/environments/process-script.js"), + "remote-page": require("../lib/environments/remote-page.js"), + simpletest: require("../lib/environments/simpletest.js"), + sjs: require("../lib/environments/sjs.js"), + "special-powers-sandbox": require("../lib/environments/special-powers-sandbox.js"), + specific: require("../lib/environments/specific"), + testharness: require("../lib/environments/testharness.js"), + xpcshell: require("../lib/environments/xpcshell.js"), + }, + rules: { + "avoid-Date-timing": require("../lib/rules/avoid-Date-timing"), + "avoid-removeChild": require("../lib/rules/avoid-removeChild"), + "balanced-listeners": require("../lib/rules/balanced-listeners"), + "balanced-observers": require("../lib/rules/balanced-observers"), + "consistent-if-bracing": require("../lib/rules/consistent-if-bracing"), + "import-browser-window-globals": require("../lib/rules/import-browser-window-globals"), + "import-content-task-globals": require("../lib/rules/import-content-task-globals"), + "import-globals": require("../lib/rules/import-globals"), + "import-headjs-globals": require("../lib/rules/import-headjs-globals"), + "lazy-getter-object-name": require("../lib/rules/lazy-getter-object-name"), + "mark-exported-symbols-as-used": require("../lib/rules/mark-exported-symbols-as-used"), + "mark-test-function-used": require("../lib/rules/mark-test-function-used"), + "no-aArgs": require("../lib/rules/no-aArgs"), + "no-addtask-setup": require("../lib/rules/no-addtask-setup"), + "no-arbitrary-setTimeout": require("../lib/rules/no-arbitrary-setTimeout"), + "no-browser-refs-in-toolkit": require("../lib/rules/no-browser-refs-in-toolkit"), + "no-compare-against-boolean-literals": require("../lib/rules/no-compare-against-boolean-literals"), + "no-comparison-or-assignment-inside-ok": require("../lib/rules/no-comparison-or-assignment-inside-ok"), + "no-cu-reportError": require("../lib/rules/no-cu-reportError"), + "no-define-cc-etc": require("../lib/rules/no-define-cc-etc"), + "no-redeclare-with-import-autofix": require("../lib/rules/no-redeclare-with-import-autofix"), + "no-throw-cr-literal": require("../lib/rules/no-throw-cr-literal"), + "no-useless-parameters": require("../lib/rules/no-useless-parameters"), + "no-useless-removeEventListener": require("../lib/rules/no-useless-removeEventListener"), + "no-useless-run-test": require("../lib/rules/no-useless-run-test"), + "prefer-boolean-length-check": require("../lib/rules/prefer-boolean-length-check"), + "prefer-formatValues": require("../lib/rules/prefer-formatValues"), + "reject-addtask-only": require("../lib/rules/reject-addtask-only"), + "reject-chromeutils-import": require("../lib/rules/reject-chromeutils-import"), + "reject-chromeutils-import-params": require("../lib/rules/reject-chromeutils-import-params"), + "reject-eager-module-in-lazy-getter": require("../lib/rules/reject-eager-module-in-lazy-getter"), + "reject-global-this": require("../lib/rules/reject-global-this"), + "reject-globalThis-modification": require("../lib/rules/reject-globalThis-modification"), + "reject-import-system-module-from-non-system": require("../lib/rules/reject-import-system-module-from-non-system"), + "reject-importGlobalProperties": require("../lib/rules/reject-importGlobalProperties"), + "reject-lazy-imports-into-globals": require("../lib/rules/reject-lazy-imports-into-globals"), + "reject-mixing-eager-and-lazy": require("../lib/rules/reject-mixing-eager-and-lazy"), + "reject-multiple-getters-calls": require("../lib/rules/reject-multiple-getters-calls"), + "reject-scriptableunicodeconverter": require("../lib/rules/reject-scriptableunicodeconverter"), + "reject-relative-requires": require("../lib/rules/reject-relative-requires"), + "reject-some-requires": require("../lib/rules/reject-some-requires"), + "reject-top-level-await": require("../lib/rules/reject-top-level-await"), + "rejects-requires-await": require("../lib/rules/rejects-requires-await"), + "use-cc-etc": require("../lib/rules/use-cc-etc"), + "use-chromeutils-definelazygetter": require("../lib/rules/use-chromeutils-definelazygetter"), + "use-chromeutils-generateqi": require("../lib/rules/use-chromeutils-generateqi"), + "use-chromeutils-import": require("../lib/rules/use-chromeutils-import"), + "use-console-createInstance": require("../lib/rules/use-console-createInstance"), + "use-default-preference-values": require("../lib/rules/use-default-preference-values"), + "use-ownerGlobal": require("../lib/rules/use-ownerGlobal"), + "use-includes-instead-of-indexOf": require("../lib/rules/use-includes-instead-of-indexOf"), + "use-isInstance": require("./rules/use-isInstance"), + "use-returnValue": require("../lib/rules/use-returnValue"), + "use-services": require("../lib/rules/use-services"), + "use-static-import": require("../lib/rules/use-static-import"), + "valid-ci-uses": require("../lib/rules/valid-ci-uses"), + "valid-lazy": require("../lib/rules/valid-lazy"), + "valid-services": require("../lib/rules/valid-services"), + "valid-services-property": require("../lib/rules/valid-services-property"), + "var-only-at-top-level": require("../lib/rules/var-only-at-top-level"), + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/avoid-Date-timing.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/avoid-Date-timing.js new file mode 100644 index 0000000000..437c53e244 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/avoid-Date-timing.js @@ -0,0 +1,61 @@ +/** + * @fileoverview Disallow using Date for timing in performance sensitive code + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/avoid-Date-timing.html", + }, + messages: { + usePerfNow: + "use performance.now() instead of Date.now() for timing measurements", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + CallExpression(node) { + let callee = node.callee; + if ( + callee.type !== "MemberExpression" || + callee.object.type !== "Identifier" || + callee.object.name !== "Date" || + callee.property.type !== "Identifier" || + callee.property.name !== "now" + ) { + return; + } + + context.report({ + node, + messageId: "usePerfNow", + }); + }, + + NewExpression(node) { + let callee = node.callee; + if ( + callee.type !== "Identifier" || + callee.name !== "Date" || + node.arguments.length + ) { + return; + } + + context.report({ + node, + messageId: "usePerfNow", + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/avoid-removeChild.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/avoid-removeChild.js new file mode 100644 index 0000000000..6c74d8aa59 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/avoid-removeChild.js @@ -0,0 +1,70 @@ +/** + * @fileoverview Reject using element.parentNode.removeChild(element) when + * element.remove() can be used instead. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var helpers = require("../helpers"); + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/avoid-removeChild.html", + }, + messages: { + useRemove: + "use element.remove() instead of element.parentNode.removeChild(element)", + useFirstChildRemove: + "use element.firstChild.remove() instead of element.removeChild(element.firstChild)", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + CallExpression(node) { + let callee = node.callee; + if ( + callee.type !== "MemberExpression" || + callee.property.type !== "Identifier" || + callee.property.name != "removeChild" || + node.arguments.length != 1 + ) { + return; + } + + if ( + callee.object.type == "MemberExpression" && + callee.object.property.type == "Identifier" && + callee.object.property.name == "parentNode" && + helpers.getASTSource(callee.object.object, context) == + helpers.getASTSource(node.arguments[0]) + ) { + context.report({ + node, + messageId: "useRemove", + }); + } + + if ( + node.arguments[0].type == "MemberExpression" && + node.arguments[0].property.type == "Identifier" && + node.arguments[0].property.name == "firstChild" && + helpers.getASTSource(callee.object, context) == + helpers.getASTSource(node.arguments[0].object) + ) { + context.report({ + node, + messageId: "useFirstChildRemove", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-listeners.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-listeners.js new file mode 100644 index 0000000000..f1c98a01bc --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-listeners.js @@ -0,0 +1,149 @@ +/** + * @fileoverview Check that there's a removeEventListener for each + * addEventListener and an off for each on. + * Note that for now, this rule is rather simple in that it only checks that + * for each event name there is both an add and remove listener. It doesn't + * check that these are called on the right objects or with the same callback. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/balanced-listeners.html", + }, + messages: { + noCorresponding: + "No corresponding '{{functionName}}({{type}})' was found.", + }, + schema: [], + type: "problem", + }, + + create(context) { + var DICTIONARY = { + addEventListener: "removeEventListener", + on: "off", + }; + // Invert this dictionary to make it easy later. + var INVERTED_DICTIONARY = {}; + for (var i in DICTIONARY) { + INVERTED_DICTIONARY[DICTIONARY[i]] = i; + } + + // Collect the add/remove listeners in these 2 arrays. + var addedListeners = []; + var removedListeners = []; + + function addAddedListener(node) { + var capture = false; + let options = node.arguments[2]; + if (options) { + if (options.type == "ObjectExpression") { + if ( + options.properties.some( + p => p.key.name == "once" && p.value.value === true + ) + ) { + // No point in adding listeners using the 'once' option. + return; + } + capture = options.properties.some( + p => p.key.name == "capture" && p.value.value === true + ); + } else { + capture = options.value; + } + } + addedListeners.push({ + functionName: node.callee.property.name, + type: node.arguments[0].value, + node: node.callee.property, + useCapture: capture, + }); + } + + function addRemovedListener(node) { + var capture = false; + let options = node.arguments[2]; + if (options) { + if (options.type == "ObjectExpression") { + capture = options.properties.some( + p => p.key.name == "capture" && p.value.value === true + ); + } else { + capture = options.value; + } + } + removedListeners.push({ + functionName: node.callee.property.name, + type: node.arguments[0].value, + useCapture: capture, + }); + } + + function getUnbalancedListeners() { + var unbalanced = []; + + for (var j = 0; j < addedListeners.length; j++) { + if (!hasRemovedListener(addedListeners[j])) { + unbalanced.push(addedListeners[j]); + } + } + addedListeners = removedListeners = []; + + return unbalanced; + } + + function hasRemovedListener(addedListener) { + for (var k = 0; k < removedListeners.length; k++) { + var listener = removedListeners[k]; + if ( + DICTIONARY[addedListener.functionName] === listener.functionName && + addedListener.type === listener.type && + addedListener.useCapture === listener.useCapture + ) { + return true; + } + } + + return false; + } + + return { + CallExpression(node) { + if (node.arguments.length === 0) { + return; + } + + if (node.callee.type === "MemberExpression") { + var listenerMethodName = node.callee.property.name; + + if (DICTIONARY.hasOwnProperty(listenerMethodName)) { + addAddedListener(node); + } else if (INVERTED_DICTIONARY.hasOwnProperty(listenerMethodName)) { + addRemovedListener(node); + } + } + }, + + "Program:exit": function () { + getUnbalancedListeners().forEach(function (listener) { + context.report({ + node: listener.node, + messageId: "noCorresponding", + data: { + functionName: DICTIONARY[listener.functionName], + type: listener.type, + }, + }); + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-observers.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-observers.js new file mode 100644 index 0000000000..854fbc9a63 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-observers.js @@ -0,0 +1,121 @@ +/** + * @fileoverview Check that there's a Services.(prefs|obs).removeObserver for + * each addObserver. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/balanced-observers.html", + }, + messages: { + noCorresponding: + "No corresponding 'removeObserver(\"{{observable}}\")' was found.", + }, + schema: [], + type: "problem", + }, + + create(context) { + var addedObservers = []; + var removedObservers = []; + + function getObserverAPI(node) { + const object = node.callee.object; + if ( + object.type == "MemberExpression" && + object.property.type == "Identifier" + ) { + return object.property.name; + } + return null; + } + + function isServicesObserver(api) { + return api == "obs" || api == "prefs"; + } + + function getObservableName(node, api) { + if (api === "obs") { + return node.arguments[1].value; + } + return node.arguments[0].value; + } + + function addAddedObserver(node) { + const api = getObserverAPI(node); + if (!isServicesObserver(api)) { + return; + } + + addedObservers.push({ + functionName: node.callee.property.name, + observable: getObservableName(node, api), + node: node.callee.property, + }); + } + + function addRemovedObserver(node) { + const api = getObserverAPI(node); + if (!isServicesObserver(api)) { + return; + } + + removedObservers.push({ + functionName: node.callee.property.name, + observable: getObservableName(node, api), + }); + } + + function getUnbalancedObservers() { + const unbalanced = addedObservers.filter( + observer => !hasRemovedObserver(observer) + ); + addedObservers = removedObservers = []; + + return unbalanced; + } + + function hasRemovedObserver(addedObserver) { + return removedObservers.some( + observer => addedObserver.observable === observer.observable + ); + } + + return { + CallExpression(node) { + if (node.arguments.length === 0) { + return; + } + + if (node.callee.type === "MemberExpression") { + var methodName = node.callee.property.name; + + if (methodName === "addObserver") { + addAddedObserver(node); + } else if (methodName === "removeObserver") { + addRemovedObserver(node); + } + } + }, + + "Program:exit": function () { + getUnbalancedObservers().forEach(function (observer) { + context.report({ + node: observer.node, + messageId: "noCorresponding", + data: { + observable: observer.observable, + }, + }); + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/consistent-if-bracing.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/consistent-if-bracing.js new file mode 100644 index 0000000000..0c9c9a342f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/consistent-if-bracing.js @@ -0,0 +1,54 @@ +/** + * @fileoverview checks if/else if/else bracing is consistent + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/consistent-if-bracing.html", + }, + messages: { + consistentIfBracing: "Bracing of if..else bodies should be consistent.", + }, + schema: [], + type: "layout", + }, + + create(context) { + return { + IfStatement(node) { + if (node.parent.type !== "IfStatement") { + let types = new Set(); + for ( + let currentNode = node; + currentNode; + currentNode = currentNode.alternate + ) { + let type = currentNode.consequent.type; + types.add(type == "BlockStatement" ? "Block" : "NotBlock"); + if ( + currentNode.alternate && + currentNode.alternate.type !== "IfStatement" + ) { + type = currentNode.alternate.type; + types.add(type == "BlockStatement" ? "Block" : "NotBlock"); + break; + } + } + if (types.size > 1) { + context.report({ + node, + messageId: "consistentIfBracing", + }); + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-browser-window-globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-browser-window-globals.js new file mode 100644 index 0000000000..7a099ba340 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-browser-window-globals.js @@ -0,0 +1,50 @@ +/** + * @fileoverview For scripts included in browser-window, this will automatically + * inject the browser-window global scopes into the file. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var path = require("path"); +var helpers = require("../helpers"); +var browserWindowEnv = require("../environments/browser-window"); + +module.exports = { + // This rule currently has no messages. + // eslint-disable-next-line eslint-plugin/prefer-message-ids + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/import-browser-window-globals.html", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + Program(node) { + let filePath = helpers.getAbsoluteFilePath(context); + let relativePath = path.relative(helpers.rootDir, filePath); + // We need to translate the path on Windows, due to the change + // from \ to /, and browserjsScripts assumes Posix. + if (path.win32) { + relativePath = relativePath.split(path.sep).join("/"); + } + + if (browserWindowEnv.browserjsScripts?.includes(relativePath)) { + for (let global in browserWindowEnv.globals) { + helpers.addVarToScope( + global, + context.getScope(), + browserWindowEnv.globals[global] + ); + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-content-task-globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-content-task-globals.js new file mode 100644 index 0000000000..e2b66ce8b0 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-content-task-globals.js @@ -0,0 +1,73 @@ +/** + * @fileoverview For ContentTask.spawn, this will automatically declare the + * frame script variables in the global scope. + * Note: due to the way ESLint works, it appears it is only + * easy to declare these variables on a file-global scope, rather + * than function global. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var helpers = require("../helpers"); +var frameScriptEnv = require("../environments/frame-script"); +var sandboxEnv = require("../environments/special-powers-sandbox"); + +module.exports = { + // eslint-disable-next-line eslint-plugin/prefer-message-ids + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/import-content-task-globals.html", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + "CallExpression[callee.object.name='ContentTask'][callee.property.name='spawn']": + function (node) { + // testing/mochitest/BrowserTestUtils/content/content-task.js + // This script is loaded as a sub script into a frame script. + for (let [name, value] of Object.entries(frameScriptEnv.globals)) { + helpers.addVarToScope(name, context.getScope(), value); + } + }, + "CallExpression[callee.object.name='SpecialPowers'][callee.property.name='spawn']": + function (node) { + for (let [name, value] of Object.entries(sandboxEnv.globals)) { + helpers.addVarToScope(name, context.getScope(), value); + } + let globals = [ + // testing/specialpowers/content/SpecialPowersChild.sys.mjs + // SpecialPowersChild._spawnTask + "SpecialPowers", + "ContentTaskUtils", + "content", + "docShell", + ]; + for (let global of globals) { + helpers.addVarToScope(global, context.getScope(), false); + } + }, + "CallExpression[callee.object.name='SpecialPowers'][callee.property.name='spawnChrome']": + function (node) { + for (let [name, value] of Object.entries(sandboxEnv.globals)) { + helpers.addVarToScope(name, context.getScope(), value); + } + let globals = [ + // testing/specialpowers/content/SpecialPowersParent.sys.mjs + // SpecialPowersParent._spawnChrome + "windowGlobalParent", + "browsingContext", + ]; + for (let global of globals) { + helpers.addVarToScope(global, context.getScope(), false); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-globals.js new file mode 100644 index 0000000000..abbab511ff --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-globals.js @@ -0,0 +1,21 @@ +/** + * @fileoverview Discovers all globals for the current file. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/import-globals.html", + }, + schema: [], + type: "problem", + }, + + create: require("../globals").getESLintGlobalParser, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-headjs-globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-headjs-globals.js new file mode 100644 index 0000000000..d4fa484b99 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-headjs-globals.js @@ -0,0 +1,51 @@ +/** + * @fileoverview Import globals from head.js and from any files that were + * imported by head.js (as far as we can correctly resolve the path). + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var fs = require("fs"); +var helpers = require("../helpers"); +var globals = require("../globals"); + +function importHead(context, path, node) { + try { + let stats = fs.statSync(path); + if (!stats.isFile()) { + return; + } + } catch (e) { + return; + } + + let newGlobals = globals.getGlobalsForFile(path); + helpers.addGlobals(newGlobals, context.getScope()); +} + +module.exports = { + // This rule currently has no messages. + // eslint-disable-next-line eslint-plugin/prefer-message-ids + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/import-headjs-globals.html", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + Program(node) { + let heads = helpers.getTestHeadFiles(context); + for (let head of heads) { + importHead(context, head, node); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/lazy-getter-object-name.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/lazy-getter-object-name.js new file mode 100644 index 0000000000..b18cbc3725 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/lazy-getter-object-name.js @@ -0,0 +1,48 @@ +/** + * @fileoverview Enforce the standard object name for + * ChromeUtils.defineESModuleGetters + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isIdentifier(node, id) { + return node.type === "Identifier" && node.name === id; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/lazy-getter-object-name.html", + }, + messages: { + mustUseLazy: + "The variable name of the object passed to ChromeUtils.defineESModuleGetters must be `lazy`", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + CallExpression(node) { + let { callee } = node; + if ( + callee.type === "MemberExpression" && + isIdentifier(callee.object, "ChromeUtils") && + isIdentifier(callee.property, "defineESModuleGetters") && + node.arguments.length >= 1 && + !isIdentifier(node.arguments[0], "lazy") + ) { + context.report({ + node, + messageId: "mustUseLazy", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-exported-symbols-as-used.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-exported-symbols-as-used.js new file mode 100644 index 0000000000..5d0e57e4c8 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-exported-symbols-as-used.js @@ -0,0 +1,90 @@ +/** + * @fileoverview Simply marks exported symbols as used. Designed for use in + * .jsm files only. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function markArrayElementsAsUsed(context, node, expression) { + if (expression.type != "ArrayExpression") { + context.report({ + node, + messageId: "nonArrayAssignedToImported", + }); + return; + } + + for (let element of expression.elements) { + context.markVariableAsUsed(element.value); + } + // Also mark EXPORTED_SYMBOLS as used. + context.markVariableAsUsed("EXPORTED_SYMBOLS"); +} + +// Ignore assignments not in the global scope, e.g. where special module +// definitions are required due to having different ways of importing files, +// e.g. osfile. +function isGlobalScope(context) { + return !context.getScope().upper; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/mark-exported-symbols-as-used.html", + }, + messages: { + useLetForExported: + "EXPORTED_SYMBOLS cannot be declared via `let`. Use `var` or `this.EXPORTED_SYMBOLS =`", + nonArrayAssignedToImported: + "Unexpected assignment of non-Array to EXPORTED_SYMBOLS", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + AssignmentExpression(node, parents) { + if ( + node.operator === "=" && + node.left.type === "MemberExpression" && + node.left.object.type === "ThisExpression" && + node.left.property.name === "EXPORTED_SYMBOLS" && + isGlobalScope(context) + ) { + markArrayElementsAsUsed(context, node, node.right); + } + }, + + VariableDeclaration(node, parents) { + if (!isGlobalScope(context)) { + return; + } + + for (let item of node.declarations) { + if ( + item.id && + item.id.type == "Identifier" && + item.id.name === "EXPORTED_SYMBOLS" + ) { + if (node.kind === "let") { + // The use of 'let' isn't allowed as the lexical scope may die after + // the script executes. + context.report({ + node, + messageId: "useLetForExported", + }); + } + + markArrayElementsAsUsed(context, node, item.init); + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-test-function-used.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-test-function-used.js new file mode 100644 index 0000000000..4afe8a70ac --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-test-function-used.js @@ -0,0 +1,44 @@ +/** + * @fileoverview Simply marks `test` (the test method) or `run_test` as used + * when in mochitests or xpcshell tests respectively. This avoids ESLint telling + * us that the function is never called. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var helpers = require("../helpers"); + +module.exports = { + // This rule currently has no messages. + // eslint-disable-next-line eslint-plugin/prefer-message-ids + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/mark-test-function-used.html", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + Program() { + let testType = helpers.getTestType(context); + if (testType == "browser") { + context.markVariableAsUsed("test"); + } + + if (testType == "xpcshell") { + context.markVariableAsUsed("run_test"); + } + + if (helpers.getIsSjs(context)) { + context.markVariableAsUsed("handleRequest"); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-aArgs.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-aArgs.js new file mode 100644 index 0000000000..7135890761 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-aArgs.js @@ -0,0 +1,57 @@ +/** + * @fileoverview warns against using hungarian notation in function arguments + * (i.e. aArg). + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isPrefixed(name) { + return name.length >= 2 && /^a[A-Z]/.test(name); +} + +function deHungarianize(name) { + return name.substring(1, 2).toLowerCase() + name.substring(2, name.length); +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-aArgs.html", + }, + messages: { + dontUseHungarian: + "Parameter '{{name}}' uses Hungarian Notation, consider using '{{suggestion}}' instead.", + }, + schema: [], + type: "layout", + }, + + create(context) { + function checkFunction(node) { + for (var i = 0; i < node.params.length; i++) { + var param = node.params[i]; + if (param.name && isPrefixed(param.name)) { + var errorObj = { + name: param.name, + suggestion: deHungarianize(param.name), + }; + context.report({ + node: param, + messageId: "dontUseHungarian", + data: errorObj, + }); + } + } + } + + return { + FunctionDeclaration: checkFunction, + ArrowFunctionExpression: checkFunction, + FunctionExpression: checkFunction, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-addtask-setup.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-addtask-setup.js new file mode 100644 index 0000000000..e711252e09 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-addtask-setup.js @@ -0,0 +1,57 @@ +/** + * @fileoverview Reject `add_task(async function setup` or similar patterns in + * favour of add_setup. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isNamedLikeSetup(name) { + return /^(init|setup)$/i.test(name); +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-addtask-setup.html", + }, + fixable: "code", + messages: { + useAddSetup: "Do not use add_task() for setup, use add_setup() instead.", + }, + schema: [], + type: "suggestion", + }, + create(context) { + return { + "Program > ExpressionStatement > CallExpression": function (node) { + let callee = node.callee; + if (callee.type === "Identifier" && callee.name === "add_task") { + let arg = node.arguments[0]; + if ( + arg.type !== "FunctionExpression" || + !arg.id || + !isNamedLikeSetup(arg.id.name) + ) { + return; + } + context.report({ + node, + messageId: "useAddSetup", + fix: fixer => { + let range = [node.callee.range[0], arg.id.range[1]]; + let asyncOrNot = arg.async ? "async " : ""; + return fixer.replaceTextRange( + range, + `add_setup(${asyncOrNot}function` + ); + }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-arbitrary-setTimeout.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-arbitrary-setTimeout.js new file mode 100644 index 0000000000..d0e891292d --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-arbitrary-setTimeout.js @@ -0,0 +1,65 @@ +/** + * @fileoverview Reject use of non-zero values in setTimeout + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var helpers = require("../helpers"); +var testTypes = new Set(["browser", "xpcshell"]); + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-arbitrary-setTimeout.html", + }, + messages: { + listenForEvents: + "listen for events instead of setTimeout() with arbitrary delay", + }, + schema: [], + type: "problem", + }, + + create(context) { + // We don't want to run this on mochitest plain as it already + // prevents flaky setTimeout at runtime. This check is built-in + // to the rule itself as sometimes other tests can live alongside + // plain mochitests and so it can't be configured via eslintrc. + if (!testTypes.has(helpers.getTestType(context))) { + return {}; + } + + return { + CallExpression(node) { + let callee = node.callee; + if (callee.type === "MemberExpression") { + if ( + callee.property.name !== "setTimeout" || + callee.object.name !== "window" || + node.arguments.length < 2 + ) { + return; + } + } else if (callee.type === "Identifier") { + if (callee.name !== "setTimeout" || node.arguments.length < 2) { + return; + } + } else { + return; + } + + let timeout = node.arguments[1]; + if (timeout.type !== "Literal" || timeout.value > 0) { + context.report({ + node, + messageId: "listenForEvents", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-browser-refs-in-toolkit.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-browser-refs-in-toolkit.js new file mode 100644 index 0000000000..fea94d364e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-browser-refs-in-toolkit.js @@ -0,0 +1,48 @@ +/** + * @fileoverview Reject use of browser/-based references from code in + * directories like toolkit/ that ought not to depend on + * running inside desktop Firefox. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-browser-refs-in-toolkit.html", + }, + messages: { + noBrowserChrome: + "> {{url}} is part of Desktop Firefox and cannot be unconditionally " + + "used by this code (which has to also work elsewhere).", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + Literal(node) { + if (typeof node.value != "string") { + return; + } + if ( + node.value.startsWith("chrome://browser") || + node.value.startsWith("resource:///") || + node.value.startsWith("resource://app/") || + (node.value.startsWith("browser/") && node.value.endsWith(".ftl")) + ) { + context.report({ + node, + messageId: "noBrowserChrome", + data: { url: node.value }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-compare-against-boolean-literals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-compare-against-boolean-literals.js new file mode 100644 index 0000000000..cf52b2ad21 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-compare-against-boolean-literals.js @@ -0,0 +1,40 @@ +/** + * @fileoverview Restrict comparing against `true` or `false`. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-compare-against-boolean-literals.html", + }, + messages: { + noCompareBoolean: + "Don't compare for inexact equality against boolean literals", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + BinaryExpression(node) { + if ( + ["==", "!="].includes(node.operator) && + (["true", "false"].includes(node.left.raw) || + ["true", "false"].includes(node.right.raw)) + ) { + context.report({ + node, + messageId: "noCompareBoolean", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-comparison-or-assignment-inside-ok.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-comparison-or-assignment-inside-ok.js new file mode 100644 index 0000000000..9bab06b000 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-comparison-or-assignment-inside-ok.js @@ -0,0 +1,80 @@ +/** + * @fileoverview Don't allow accidental assignments inside `ok()`, + * and encourage people to use appropriate alternatives + * when using comparisons between 2 values. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const operatorToAssertionMap = { + "==": "Assert.equal", + "===": "Assert.strictEqual", + "!=": "Assert.notEqual", + "!==": "Assert.notStrictEqual", + ">": "Assert.greater", + "<": "Assert.less", + "<=": "Assert.lessOrEqual", + ">=": "Assert.greaterOrEqual", +}; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-comparison-or-assignment-inside-ok.html", + }, + fixable: "code", + messages: { + assignment: + "Assigning to a variable inside ok() is odd - did you mean to compare the two?", + comparison: + "Use dedicated assertion methods rather than ok(a {{operator}} b).", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + const exprs = new Set(["BinaryExpression", "AssignmentExpression"]); + return { + CallExpression(node) { + if (node.callee.type != "Identifier" || node.callee.name != "ok") { + return; + } + let firstArg = node.arguments[0]; + if (!exprs.has(firstArg.type)) { + return; + } + if (firstArg.type == "AssignmentExpression") { + context.report({ + node: firstArg, + messageId: "assignment", + }); + } else if ( + firstArg.type == "BinaryExpression" && + operatorToAssertionMap.hasOwnProperty(firstArg.operator) + ) { + context.report({ + node, + messageId: "comparison", + data: { operator: firstArg.operator }, + fix: fixer => { + let left = context.sourceCode.getText(firstArg.left); + let right = context.sourceCode.getText(firstArg.right); + return [ + fixer.replaceText(firstArg, left + ", " + right), + fixer.replaceText( + node.callee, + operatorToAssertionMap[firstArg.operator] + ), + ]; + }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-cu-reportError.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-cu-reportError.js new file mode 100644 index 0000000000..85daa8823e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-cu-reportError.js @@ -0,0 +1,130 @@ +/** + * @fileoverview Reject common XPCOM methods called with useless optional + * parameters, or non-existent parameters. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isCuReportError(node) { + return ( + node.type == "MemberExpression" && + node.object.type == "Identifier" && + node.object.name == "Cu" && + node.property.type == "Identifier" && + node.property.name == "reportError" + ); +} + +function isConcatenation(node) { + return node.type == "BinaryExpression" && node.operator == "+"; +} + +function isIdentOrMember(node) { + return node.type == "MemberExpression" || node.type == "Identifier"; +} + +function isLiteralOrConcat(node) { + return node.type == "Literal" || isConcatenation(node); +} + +function replaceConcatWithComma(fixer, node) { + let fixes = []; + let didFixTrailingIdentifier = false; + let recursiveFixes; + let trailingIdentifier; + // Deal with recursion first: + if (isConcatenation(node.right)) { + // Uh oh. If the RHS is a concatenation, there are parens involved, + // e.g.: + // console.error("literal" + (b + "literal")); + // It's pretty much impossible to guess what to do here so bail out: + return { fixes: [], trailingIdentifier: false }; + } + if (isConcatenation(node.left)) { + ({ fixes: recursiveFixes, trailingIdentifier } = replaceConcatWithComma( + fixer, + node.left + )); + fixes.push(...recursiveFixes); + } + // If the left is an identifier or memberexpression, and the right is a + // literal or concatenation - or vice versa - replace a + with a comma: + if ( + (isIdentOrMember(node.left) && isLiteralOrConcat(node.right)) || + (isIdentOrMember(node.right) && isLiteralOrConcat(node.left)) || + // Or if the rhs is a literal/concatenation, while the right-most part of + // the lhs is also an identifier (need 2 commas either side!) + (trailingIdentifier && isLiteralOrConcat(node.right)) + ) { + fixes.push( + fixer.replaceTextRange([node.left.range[1], node.right.range[0]], ", ") + ); + didFixTrailingIdentifier = isIdentOrMember(node.right); + } + return { fixes, trailingIdentifier: didFixTrailingIdentifier }; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-cu-reportError.html", + }, + fixable: "code", + messages: { + useConsoleError: "Please use console.error instead of Cu.reportError", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + CallExpression(node) { + let checkNodes = []; + if (isCuReportError(node.callee)) { + // Handles cases of `Cu.reportError()`. + if (node.arguments.length > 1) { + // TODO: Bug 1802347 For initial landing, we allow the two + // argument form of Cu.reportError as the second argument is a stack + // argument which is more complicated to deal with. + return; + } + checkNodes = [node.callee]; + } else if (node.arguments.length >= 1) { + // Handles cases of `.foo(Cu.reportError)`. + checkNodes = node.arguments.filter(n => isCuReportError(n)); + } + + for (let checkNode of checkNodes) { + context.report({ + node, + fix: fixer => { + let fixes = [ + fixer.replaceText(checkNode.object, "console"), + fixer.replaceText(checkNode.property, "error"), + ]; + // If we're adding stuff together as an argument, split + // into multiple arguments instead: + if ( + checkNode == node.callee && + isConcatenation(node.arguments[0]) + ) { + let { fixes: recursiveFixes } = replaceConcatWithComma( + fixer, + node.arguments[0] + ); + fixes.push(...recursiveFixes); + } + return fixes; + }, + messageId: "useConsoleError", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-define-cc-etc.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-define-cc-etc.js new file mode 100644 index 0000000000..05e7648632 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-define-cc-etc.js @@ -0,0 +1,57 @@ +/** + * @fileoverview Reject defining Cc/Ci/Cr/Cu. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const componentsBlacklist = ["Cc", "Ci", "Cr", "Cu"]; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-define-cc-etc.html", + }, + messages: { + noSeparateDefinition: + "{{name}} is now defined in global scope, a separate definition is no longer necessary.", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + VariableDeclarator(node) { + if ( + node.id.type == "Identifier" && + componentsBlacklist.includes(node.id.name) + ) { + context.report({ + node, + messageId: "noSeparateDefinition", + data: { name: node.id.name }, + }); + } + + if (node.id.type == "ObjectPattern") { + for (let property of node.id.properties) { + if ( + property.type == "Property" && + componentsBlacklist.includes(property.value.name) + ) { + context.report({ + node, + messageId: "noSeparateDefinition", + data: { name: property.value.name }, + }); + } + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-redeclare-with-import-autofix.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-redeclare-with-import-autofix.js new file mode 100644 index 0000000000..d914e003d3 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-redeclare-with-import-autofix.js @@ -0,0 +1,160 @@ +/** + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const { dirname, join } = require("path"); + +const eslintBasePath = dirname(require.resolve("eslint")); + +const noredeclarePath = join(eslintBasePath, "rules/no-redeclare.js"); +const baseRule = require(noredeclarePath); +const astUtils = require(join(eslintBasePath, "rules/utils/ast-utils.js")); + +// Hack alert: our eslint env is pretty confused about `require` and +// `loader` for devtools modules - so ignore it for now. +// See bug 1812547 +const gIgnoredImports = new Set(["loader", "require"]); + +/** + * Create a trap for a call to `report` that the original rule is + * trying to make on `context`. + * + * Returns a function that forwards to `report` but provides a fixer + * for redeclared imports that just removes those imports. + * + * @return {function} + */ +function trapReport(context) { + return function (obj) { + let declarator = obj.node.parent; + while ( + declarator && + declarator.parent && + declarator.type != "VariableDeclarator" + ) { + declarator = declarator.parent; + } + if ( + declarator && + declarator.type == "VariableDeclarator" && + declarator.id.type == "ObjectPattern" && + declarator.init.type == "CallExpression" + ) { + let initialization = declarator.init; + if ( + astUtils.isSpecificMemberAccess( + initialization.callee, + "ChromeUtils", + /^import(ESModule|)$/ + ) + ) { + // Hack alert: our eslint env is pretty confused about `require` and + // `loader` for devtools modules - so ignore it for now. + // See bug 1812547 + if (gIgnoredImports.has(obj.node.name)) { + return; + } + // OK, we've got something we can fix. But we should be careful in case + // there are multiple imports being destructured. + // Do the easy (and common) case first - just one property: + if (declarator.id.properties.length == 1) { + context.report({ + node: declarator.parent, + messageId: "duplicateImport", + data: { + name: declarator.id.properties[0].key.name, + }, + fix(fixer) { + return fixer.remove(declarator.parent); + }, + }); + return; + } + + // OK, figure out which import is duplicated here: + let node = obj.node.parent; + // Then remove a comma after it, or a comma before + // if there's no comma after it. + let sourceCode = context.getSourceCode(); + let rangeToRemove = node.range; + let tokenAfter = sourceCode.getTokenAfter(node); + let tokenBefore = sourceCode.getTokenBefore(node); + if (astUtils.isCommaToken(tokenAfter)) { + rangeToRemove[1] = tokenAfter.range[1]; + } else if (astUtils.isCommaToken(tokenBefore)) { + rangeToRemove[0] = tokenBefore.range[0]; + } + context.report({ + node, + messageId: "duplicateImport", + data: { + name: node.key.name, + }, + fix(fixer) { + return fixer.removeRange(rangeToRemove); + }, + }); + return; + } + } + if (context.options[0]?.errorForNonImports) { + // Report the result from no-redeclare - we can't autofix it. + // This can happen for other redeclaration issues, e.g. naming + // variables in a way that conflicts with builtins like "URL" or + // "escape". + context.report(obj); + } + }; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-redeclare-with-import-autofix.html", + }, + messages: { + ...baseRule.meta.messages, + duplicateImport: + "The import of '{{ name }}' is redundant with one set up earlier (e.g. head.js or the browser window environment). It should be removed.", + }, + schema: [ + { + type: "object", + properties: { + errorForNonImports: { + type: "boolean", + default: true, + }, + }, + additionalProperties: false, + }, + ], + type: "suggestion", + fixable: "code", + }, + + create(context) { + // Test modules get the browser env applied wrongly in some cases, + // don't try and remove imports there. This works out of the box + // for sys.mjs modules because eslint won't check builtinGlobals + // for the no-redeclare rule. + if (context.getFilename().endsWith(".jsm")) { + return {}; + } + let newOptions = [{ builtinGlobals: true }]; + const contextForBaseRule = Object.create(context, { + report: { + value: trapReport(context), + writable: false, + }, + options: { + value: newOptions, + }, + }); + return baseRule.create(contextForBaseRule); + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-throw-cr-literal.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-throw-cr-literal.js new file mode 100644 index 0000000000..5ff6bfd7c9 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-throw-cr-literal.js @@ -0,0 +1,101 @@ +/** + * @fileoverview Rule to prevent throwing bare Cr.ERRORs. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isCr(object) { + return object.type === "Identifier" && object.name === "Cr"; +} + +function isComponentsResults(object) { + return ( + object.type === "MemberExpression" && + object.object.type === "Identifier" && + object.object.name === "Components" && + object.property.type === "Identifier" && + object.property.name === "results" + ); +} + +function isNewError(argument) { + return ( + argument.type === "NewExpression" && + argument.callee.type === "Identifier" && + argument.callee.name === "Error" && + argument.arguments.length === 1 + ); +} + +function fixT(context, node, argument, fixer) { + const sourceText = context.getSourceCode().getText(argument); + return fixer.replaceText(node, `Components.Exception("", ${sourceText})`); +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-throw-cr-literal.html", + }, + fixable: "code", + messages: { + bareCR: "Do not throw bare Cr.ERRORs, use Components.Exception instead", + bareComponentsResults: + "Do not throw bare Components.results.ERRORs, use Components.Exception instead", + newErrorCR: + "Do not pass Cr.ERRORs to new Error(), use Components.Exception instead", + newErrorComponentsResults: + "Do not pass Components.results.ERRORs to new Error(), use Components.Exception instead", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + ThrowStatement(node) { + if (node.argument.type === "MemberExpression") { + const fix = fixT.bind(null, context, node.argument, node.argument); + + if (isCr(node.argument.object)) { + context.report({ + node, + messageId: "bareCR", + fix, + }); + } else if (isComponentsResults(node.argument.object)) { + context.report({ + node, + messageId: "bareComponentsResults", + fix, + }); + } + } else if (isNewError(node.argument)) { + const argument = node.argument.arguments[0]; + + if (argument.type === "MemberExpression") { + const fix = fixT.bind(null, context, node.argument, argument); + + if (isCr(argument.object)) { + context.report({ + node, + messageId: "newErrorCR", + fix, + }); + } else if (isComponentsResults(argument.object)) { + context.report({ + node, + messageId: "newErrorComponentsResults", + fix, + }); + } + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-parameters.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-parameters.js new file mode 100644 index 0000000000..ac1cc334e6 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-parameters.js @@ -0,0 +1,156 @@ +/** + * @fileoverview Reject common XPCOM methods called with useless optional + * parameters, or non-existent parameters. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-useless-parameters.html", + }, + fixable: "code", + messages: { + newURIParams: "newURI's last parameters are optional.", + obmittedWhenFalse: + "{{fnName}}'s {{index}} parameter can be omitted when it's false.", + onlyTakes: "{{fnName}} only takes {{params}}", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + function getRangeAfterArgToEnd(argNumber, args) { + let sourceCode = context.getSourceCode(); + return [ + sourceCode.getTokenAfter(args[argNumber]).range[0], + args[args.length - 1].range[1], + ]; + } + + return { + CallExpression(node) { + let callee = node.callee; + if ( + callee.type !== "MemberExpression" || + callee.property.type !== "Identifier" + ) { + return; + } + + let isFalse = arg => arg.type === "Literal" && arg.value === false; + let isFalsy = arg => arg.type === "Literal" && !arg.value; + let isBool = arg => + arg.type === "Literal" && (arg.value === false || arg.value === true); + let name = callee.property.name; + let args = node.arguments; + + if ( + ["addEventListener", "removeEventListener", "addObserver"].includes( + name + ) && + args.length === 3 && + isFalse(args[2]) + ) { + context.report({ + node, + fix: fixer => { + return fixer.removeRange(getRangeAfterArgToEnd(1, args)); + }, + messageId: "obmittedWhenFalse", + data: { fnName: name, index: "third" }, + }); + } + + if (name === "clearUserPref" && args.length > 1) { + context.report({ + node, + fix: fixer => { + return fixer.removeRange(getRangeAfterArgToEnd(0, args)); + }, + messageId: "onlyTakes", + data: { fnName: name, params: "1 parameter" }, + }); + } + + if (name === "removeObserver" && args.length === 3 && isBool(args[2])) { + context.report({ + node, + fix: fixer => { + return fixer.removeRange(getRangeAfterArgToEnd(1, args)); + }, + messageId: "onlyTakes", + data: { fnName: name, params: "2 parameters" }, + }); + } + + if (name === "appendElement" && args.length === 2 && isFalse(args[1])) { + context.report({ + node, + fix: fixer => { + return fixer.removeRange(getRangeAfterArgToEnd(0, args)); + }, + messageId: "obmittedWhenFalse", + data: { fnName: name, index: "second" }, + }); + } + + if ( + name === "notifyObservers" && + args.length === 3 && + isFalsy(args[2]) + ) { + context.report({ + node, + fix: fixer => { + return fixer.removeRange(getRangeAfterArgToEnd(1, args)); + }, + messageId: "obmittedWhenFalse", + data: { fnName: name, index: "third" }, + }); + } + + if ( + name === "getComputedStyle" && + args.length === 2 && + isFalsy(args[1]) + ) { + context.report({ + node, + fix: fixer => { + return fixer.removeRange(getRangeAfterArgToEnd(0, args)); + }, + messageId: "obmittedWhenFalse", + data: { fnName: "getComputedStyle", index: "second" }, + }); + } + + if ( + name === "newURI" && + args.length > 1 && + isFalsy(args[args.length - 1]) + ) { + context.report({ + node, + fix: fixer => { + if (args.length > 2 && isFalsy(args[args.length - 2])) { + return fixer.removeRange(getRangeAfterArgToEnd(0, args)); + } + + return fixer.removeRange( + getRangeAfterArgToEnd(args.length - 2, args) + ); + }, + messageId: "newURIParams", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-removeEventListener.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-removeEventListener.js new file mode 100644 index 0000000000..d5f19ab717 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-removeEventListener.js @@ -0,0 +1,69 @@ +/** + * @fileoverview Reject calls to removeEventListenter where {once: true} could + * be used instead. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-useless-removeEventListener.html", + }, + messages: { + useOnce: + "use {once: true} instead of removeEventListener as the first instruction of the listener", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + CallExpression(node) { + let callee = node.callee; + if ( + callee.type !== "MemberExpression" || + callee.property.type !== "Identifier" || + callee.property.name !== "addEventListener" || + node.arguments.length == 4 + ) { + return; + } + + let listener = node.arguments[1]; + if ( + !listener || + listener.type != "FunctionExpression" || + !listener.body || + listener.body.type != "BlockStatement" || + !listener.body.body.length || + listener.body.body[0].type != "ExpressionStatement" || + listener.body.body[0].expression.type != "CallExpression" + ) { + return; + } + + let call = listener.body.body[0].expression; + if ( + call.callee.type == "MemberExpression" && + call.callee.property.type == "Identifier" && + call.callee.property.name == "removeEventListener" && + ((call.arguments[0].type == "Literal" && + call.arguments[0].value == node.arguments[0].value) || + (call.arguments[0].type == "Identifier" && + call.arguments[0].name == node.arguments[0].name)) + ) { + context.report({ + node: call, + messageId: "useOnce", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-run-test.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-run-test.js new file mode 100644 index 0000000000..ddfbea05e3 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-run-test.js @@ -0,0 +1,76 @@ +/** + * @fileoverview Reject run_test() definitions where they aren't necessary. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/no-useless-run-test.html", + }, + fixable: "code", + messages: { + noUselessRunTest: + "Useless run_test function - only contains run_next_test; whole function can be removed", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + "Program > FunctionDeclaration": function (node) { + if ( + node.id.name === "run_test" && + node.body.type === "BlockStatement" && + node.body.body.length === 1 && + node.body.body[0].type === "ExpressionStatement" && + node.body.body[0].expression.type === "CallExpression" && + node.body.body[0].expression.callee.name === "run_next_test" + ) { + context.report({ + node, + fix: fixer => { + let sourceCode = context.getSourceCode(); + let startNode; + if (sourceCode.getCommentsBefore) { + // ESLint 4 has getCommentsBefore. + startNode = sourceCode.getCommentsBefore(node); + } else if (node && node.body && node.leadingComments) { + // This is for ESLint 3. + startNode = node.leadingComments; + } + + // If we have comments, we want the start node to be the comments, + // rather than the token before the comments, so that we don't + // remove the comments - for run_test, these are likely to be useful + // information about the test. + if (startNode?.length) { + startNode = startNode[startNode.length - 1]; + } else { + startNode = sourceCode.getTokenBefore(node); + } + + return fixer.removeRange([ + // If there's no startNode, we fall back to zero, i.e. start of + // file. + startNode ? startNode.range[1] + 1 : 0, + // We know the function is a block and it'll end with }. Normally + // there's a new line after that, so just advance past it. This + // may be slightly not dodgy in some cases, but covers the existing + // cases. + node.range[1] + 1, + ]); + }, + messageId: "noUselessRunTest", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/prefer-boolean-length-check.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/prefer-boolean-length-check.js new file mode 100644 index 0000000000..41c0aa1d30 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/prefer-boolean-length-check.js @@ -0,0 +1,129 @@ +/** + * @fileoverview Prefer boolean length check + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function funcForBooleanLength(context, node, conditionCheck) { + let newText = ""; + const sourceCode = context.getSourceCode(); + switch (node.operator) { + case ">": + if (node.right.value == 0) { + if (conditionCheck) { + newText = sourceCode.getText(node.left); + } else { + newText = "!!" + sourceCode.getText(node.left); + } + } else { + newText = "!" + sourceCode.getText(node.right); + } + break; + case "<": + if (node.right.value == 0) { + newText = "!" + sourceCode.getText(node.left); + } else if (conditionCheck) { + newText = sourceCode.getText(node.right); + } else { + newText = "!!" + sourceCode.getText(node.right); + } + break; + case "==": + if (node.right.value == 0) { + newText = "!" + sourceCode.getText(node.left); + } else { + newText = "!" + sourceCode.getText(node.right); + } + break; + case "!=": + if (node.right.value == 0) { + if (conditionCheck) { + newText = sourceCode.getText(node.left); + } else { + newText = "!!" + sourceCode.getText(node.left); + } + } else if (conditionCheck) { + newText = sourceCode.getText(node.right); + } else { + newText = "!!" + sourceCode.getText(node.right); + } + break; + } + return newText; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/prefer-boolean-length-check.html", + }, + fixable: "code", + messages: { + preferBooleanCheck: "Prefer boolean length check", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + const conditionStatement = [ + "IfStatement", + "WhileStatement", + "DoWhileStatement", + "ForStatement", + "ForInStatement", + "ConditionalExpression", + ]; + + return { + BinaryExpression(node) { + if ( + ["==", "!=", ">", "<"].includes(node.operator) && + ((node.right.type == "Literal" && + node.right.value == 0 && + node.left.property?.name == "length") || + (node.left.type == "Literal" && + node.left.value == 0 && + node.right.property?.name == "length")) + ) { + if ( + conditionStatement.includes(node.parent.type) || + (node.parent.type == "LogicalExpression" && + conditionStatement.includes(node.parent.parent.type)) + ) { + context.report({ + node, + fix: fixer => { + let generateExpression = funcForBooleanLength( + context, + node, + true + ); + + return fixer.replaceText(node, generateExpression); + }, + messageId: "preferBooleanCheck", + }); + } else { + context.report({ + node, + fix: fixer => { + let generateExpression = funcForBooleanLength( + context, + node, + false + ); + return fixer.replaceText(node, generateExpression); + }, + messageId: "preferBooleanCheck", + }); + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/prefer-formatValues.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/prefer-formatValues.js new file mode 100644 index 0000000000..4807cf1f1f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/prefer-formatValues.js @@ -0,0 +1,97 @@ +/** + * @fileoverview Reject multiple calls to document.l10n.formatValue in the same + * code block. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isIdentifier(node, id) { + return node && node.type === "Identifier" && node.name === id; +} + +/** + * As we enter blocks new sets are pushed onto this stack and then popped when + * we exit the block. + */ +const BlockStack = []; + +module.exports = { + meta: { + docs: { + description: "disallow multiple document.l10n.formatValue calls", + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/prefer-formatValues.html", + }, + messages: { + outsideCallBlock: "call expression found outside of known block", + useSingleCall: + "prefer to use a single document.l10n.formatValues call instead " + + "of multiple calls to document.l10n.formatValue or document.l10n.formatValues", + }, + schema: [], + type: "problem", + }, + + create(context) { + function enterBlock() { + BlockStack.push(new Set()); + } + + function exitBlock() { + let calls = BlockStack.pop(); + if (calls.size > 1) { + for (let callNode of calls) { + context.report({ + node: callNode, + messageId: "useSingleCall", + }); + } + } + } + + return { + Program: enterBlock, + "Program:exit": exitBlock, + BlockStatement: enterBlock, + "BlockStatement:exit": exitBlock, + + CallExpression(node) { + if (!BlockStack.length) { + context.report({ + node, + messageId: "outsideCallBlock", + }); + } + + let callee = node.callee; + if (callee.type !== "MemberExpression") { + return; + } + + if ( + !isIdentifier(callee.property, "formatValue") && + !isIdentifier(callee.property, "formatValues") + ) { + return; + } + + if (callee.object.type !== "MemberExpression") { + return; + } + + if ( + !isIdentifier(callee.object.object, "document") || + !isIdentifier(callee.object.property, "l10n") + ) { + return; + } + + let calls = BlockStack[BlockStack.length - 1]; + calls.add(node); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-addtask-only.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-addtask-only.js new file mode 100644 index 0000000000..b1a67cad7d --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-addtask-only.js @@ -0,0 +1,53 @@ +/** + * @fileoverview Don't allow only() in tests + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-addtask-only.html", + }, + hasSuggestions: true, + messages: { + addTaskNotAllowed: + "add_task(...).only() not allowed - add an exception if this is intentional", + addTaskNotAllowedSuggestion: "Remove only() call from task", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + CallExpression(node) { + if ( + ["add_task", "decorate_task"].includes( + node.callee.object?.callee?.name + ) && + node.callee.property?.name == "only" + ) { + context.report({ + node, + messageId: "addTaskNotAllowed", + suggest: [ + { + messageId: "addTaskNotAllowedSuggestion", + fix: fixer => + fixer.replaceTextRange( + [node.callee.object.range[1], node.range[1]], + "" + ), + }, + ], + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-chromeutils-import-params.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-chromeutils-import-params.js new file mode 100644 index 0000000000..ccfb0a1cb0 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-chromeutils-import-params.js @@ -0,0 +1,66 @@ +/** + * @fileoverview Reject calls to ChromeUtils.import(..., null). This allows to + * retrieve the global object for the JSM, instead we should rely on explicitly + * exported symbols. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isIdentifier(node, id) { + return node && node.type === "Identifier" && node.name === id; +} + +function getRangeAfterArgToEnd(context, argNumber, args) { + let sourceCode = context.getSourceCode(); + return [ + sourceCode.getTokenAfter(args[argNumber]).range[0], + args[args.length - 1].range[1], + ]; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-chromeutils-import-params.html", + }, + hasSuggestions: true, + messages: { + importOnlyOneArg: "ChromeUtils.import only takes one argument.", + importOnlyOneArgSuggestion: "Remove the unnecessary parameters.", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + CallExpression(node) { + let { callee } = node; + if ( + isIdentifier(callee.object, "ChromeUtils") && + isIdentifier(callee.property, "import") && + node.arguments.length >= 2 + ) { + context.report({ + node, + messageId: "importOnlyOneArg", + suggest: [ + { + messageId: "importOnlyOneArgSuggestion", + fix: fixer => { + return fixer.removeRange( + getRangeAfterArgToEnd(context, 0, node.arguments) + ); + }, + }, + ], + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-chromeutils-import.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-chromeutils-import.js new file mode 100644 index 0000000000..1f746db730 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-chromeutils-import.js @@ -0,0 +1,80 @@ +/** + * @fileoverview Reject use of Cu.import and ChromeUtils.import + * in favor of ChromeUtils.importESModule. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isIdentifier(node, id) { + return node && node.type === "Identifier" && node.name === id; +} + +function isMemberExpression(node, object, member) { + return ( + node.type === "MemberExpression" && + isIdentifier(node.object, object) && + isIdentifier(node.property, member) + ); +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-chromeutils-import.html", + }, + messages: { + useImportESModule: + "Please use ChromeUtils.importESModule instead of " + + "ChromeUtils.import unless the module is not yet ESMified", + useImportESModuleLazy: + "Please use ChromeUtils.defineESModuleGetters instead of " + + "ChromeUtils.defineModuleGetter " + + "unless the module is not yet ESMified", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + CallExpression(node) { + if (node.callee.type !== "MemberExpression") { + return; + } + + let { callee } = node; + + if ( + (isIdentifier(callee.object, "ChromeUtils") || + isMemberExpression( + callee.object, + "SpecialPowers", + "ChromeUtils" + )) && + isIdentifier(callee.property, "import") + ) { + context.report({ + node, + messageId: "useImportESModule", + }); + } + + if ( + (isMemberExpression(callee.object, "SpecialPowers", "ChromeUtils") && + isIdentifier(callee.property, "defineModuleGetter")) || + isMemberExpression(callee, "ChromeUtils", "defineModuleGetter") || + isMemberExpression(callee, "XPCOMUtils", "defineLazyModuleGetters") + ) { + context.report({ + node, + messageId: "useImportESModuleLazy", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-eager-module-in-lazy-getter.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-eager-module-in-lazy-getter.js new file mode 100644 index 0000000000..133dd6d71f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-eager-module-in-lazy-getter.js @@ -0,0 +1,99 @@ +/** + * @fileoverview Reject use of lazy getters for modules that's loaded early in + * the startup process and not necessarily be lazy. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; +const helpers = require("../helpers"); + +function isString(node) { + return node.type === "Literal" && typeof node.value === "string"; +} + +function isEagerModule(resourceURI) { + return [ + "resource://gre/modules/XPCOMUtils", + "resource://gre/modules/AppConstants", + ].includes(resourceURI.replace(/(\.jsm|\.jsm\.js|\.js|\.sys\.mjs)$/, "")); +} + +function checkEagerModule(context, node, resourceURI) { + if (!isEagerModule(resourceURI)) { + return; + } + context.report({ + node, + messageId: "eagerModule", + data: { uri: resourceURI }, + }); +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-eager-module-in-lazy-getter.html", + }, + messages: { + eagerModule: + 'Module "{{uri}}" is known to be loaded early in the startup process, and should be loaded eagerly, instead of defining a lazy getter.', + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + CallExpression(node) { + if (node.callee.type !== "MemberExpression") { + return; + } + + let callerSource; + try { + callerSource = helpers.getASTSource(node.callee); + } catch (e) { + return; + } + + if (callerSource === "ChromeUtils.defineModuleGetter") { + if (node.arguments.length < 3) { + return; + } + const resourceURINode = node.arguments[2]; + if (!isString(resourceURINode)) { + return; + } + checkEagerModule(context, node, resourceURINode.value); + } else if ( + callerSource === "XPCOMUtils.defineLazyModuleGetters" || + callerSource === "ChromeUtils.defineESModuleGetters" + ) { + if (node.arguments.length < 2) { + return; + } + const obj = node.arguments[1]; + if (obj.type !== "ObjectExpression") { + return; + } + for (let prop of obj.properties) { + if (prop.type !== "Property") { + continue; + } + if (prop.kind !== "init") { + continue; + } + const resourceURINode = prop.value; + if (!isString(resourceURINode)) { + continue; + } + checkEagerModule(context, node, resourceURINode.value); + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-global-this.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-global-this.js new file mode 100644 index 0000000000..ec4b5fd43d --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-global-this.js @@ -0,0 +1,43 @@ +/** + * @fileoverview Reject attempts to use the global object in jsms. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const helpers = require("../helpers"); + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-global-this.html", + }, + messages: { + avoidGlobalThis: "JSM should not use the global this", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + ThisExpression(node) { + if (!helpers.getIsGlobalThis(context.getAncestors())) { + return; + } + + context.report({ + node, + messageId: "avoidGlobalThis", + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-globalThis-modification.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-globalThis-modification.js new file mode 100644 index 0000000000..13052db80c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-globalThis-modification.js @@ -0,0 +1,74 @@ +/** + * @fileoverview Enforce the standard object name for + * ChromeUtils.defineESMGetters + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isIdentifier(node, id) { + return node.type === "Identifier" && node.name === id; +} + +function calleeToString(node) { + if (node.type === "Identifier") { + return node.name; + } + + if (node.type === "MemberExpression" && !node.computed) { + return calleeToString(node.object) + "." + node.property.name; + } + + return "???"; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-globalThis-modification.html", + }, + messages: { + rejectModifyGlobalThis: + "`globalThis` shouldn't be modified. `globalThis` is the shared global inside the system module, and properties defined on it is visible from all modules.", + rejectPassingGlobalThis: + "`globalThis` shouldn't be passed to function that can modify it. `globalThis` is the shared global inside the system module, and properties defined on it is visible from all modules.", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + AssignmentExpression(node, parents) { + let target = node.left; + while (target.type === "MemberExpression") { + target = target.object; + } + if (isIdentifier(target, "globalThis")) { + context.report({ + node, + messageId: "rejectModifyGlobalThis", + }); + } + }, + CallExpression(node) { + const calleeStr = calleeToString(node.callee); + if (calleeStr.endsWith(".deserialize")) { + return; + } + + for (const arg of node.arguments) { + if (isIdentifier(arg, "globalThis")) { + context.report({ + node, + messageId: "rejectPassingGlobalThis", + }); + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-import-system-module-from-non-system.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-import-system-module-from-non-system.js new file mode 100644 index 0000000000..2cbc4e7652 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-import-system-module-from-non-system.js @@ -0,0 +1,36 @@ +/** + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-import-system-module-from-non-system.html", + }, + messages: { + rejectStaticImportSystemModuleFromNonSystem: + "System modules (*.sys.mjs) can be imported with static import declaration only from system modules.", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + ImportDeclaration(node) { + if (!node.source.value.endsWith(".sys.mjs")) { + return; + } + + context.report({ + node, + messageId: "rejectStaticImportSystemModuleFromNonSystem", + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-importGlobalProperties.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-importGlobalProperties.js new file mode 100644 index 0000000000..b2f0aad1ae --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-importGlobalProperties.js @@ -0,0 +1,97 @@ +/** + * @fileoverview Reject use of Cu.importGlobalProperties + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const path = require("path"); + +const privilegedGlobals = Object.keys( + require("../environments/privileged.js").globals +); + +function getMessageId(context) { + return path.extname(context.getFilename()) == ".sjs" + ? "unexpectedCallSjs" + : "unexpectedCall"; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-importGlobalProperties.html", + }, + messages: { + unexpectedCall: "Unexpected call to Cu.importGlobalProperties", + unexpectedCallCuWebIdl: + "Unnecessary call to Cu.importGlobalProperties for {{name}} (webidl names are automatically imported)", + unexpectedCallSjs: + "Do not call Cu.importGlobalProperties in sjs files, expand the global instead (see rule docs).", + unexpectedCallXPCOMWebIdl: + "Unnecessary call to XPCOMUtils.defineLazyGlobalGetters for {{name}} (webidl names are automatically imported)", + }, + schema: [ + { + enum: ["everything", "allownonwebidl"], + }, + ], + type: "problem", + }, + + create(context) { + return { + CallExpression(node) { + if (node.callee.type !== "MemberExpression") { + return; + } + let memexp = node.callee; + if ( + memexp.object.type === "Identifier" && + // Only Cu, not Components.utils as `use-cc-etc` handles this for us. + memexp.object.name === "Cu" && + memexp.property.type === "Identifier" && + memexp.property.name === "importGlobalProperties" + ) { + if (context.options.includes("allownonwebidl")) { + for (let element of node.arguments[0].elements) { + if (privilegedGlobals.includes(element.value)) { + context.report({ + node, + messageId: "unexpectedCallCuWebIdl", + data: { name: element.value }, + }); + } + } + } else { + context.report({ node, messageId: getMessageId(context) }); + } + } + if ( + memexp.object.type === "Identifier" && + memexp.object.name === "XPCOMUtils" && + memexp.property.type === "Identifier" && + memexp.property.name === "defineLazyGlobalGetters" && + node.arguments.length >= 2 + ) { + if (context.options.includes("allownonwebidl")) { + for (let element of node.arguments[1].elements) { + if (privilegedGlobals.includes(element.value)) { + context.report({ + node, + messageId: "unexpectedCallXPCOMWebIdl", + data: { name: element.value }, + }); + } + } + } else { + context.report({ node, messageId: getMessageId(context) }); + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-lazy-imports-into-globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-lazy-imports-into-globals.js new file mode 100644 index 0000000000..492a1e3bd7 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-lazy-imports-into-globals.js @@ -0,0 +1,72 @@ +/** + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const helpers = require("../helpers"); + +const callExpressionDefinitions = [ + /^loader\.lazyGetter\((?:globalThis|window), "(\w+)"/, + /^loader\.lazyServiceGetter\((?:globalThis|window), "(\w+)"/, + /^loader\.lazyRequireGetter\((?:globalThis|window), "(\w+)"/, + /^ChromeUtils\.defineLazyGetter\((?:globalThis|window), "(\w+)"/, + /^ChromeUtils\.defineModuleGetter\((?:globalThis|window), "(\w+)"/, + /^XPCOMUtils\.defineLazyPreferenceGetter\((?:globalThis|window), "(\w+)"/, + /^XPCOMUtils\.defineLazyScriptGetter\((?:globalThis|window), "(\w+)"/, + /^XPCOMUtils\.defineLazyServiceGetter\((?:globalThis|window), "(\w+)"/, + /^XPCOMUtils\.defineConstant\((?:globalThis|window), "(\w+)"/, + /^DevToolsUtils\.defineLazyGetter\((?:globalThis|window), "(\w+)"/, + /^Object\.defineProperty\((?:globalThis|window), "(\w+)"/, + /^Reflect\.defineProperty\((?:globalThis|window), "(\w+)"/, + /^this\.__defineGetter__\("(\w+)"/, +]; + +const callExpressionMultiDefinitions = [ + "XPCOMUtils.defineLazyGlobalGetters(window,", + "XPCOMUtils.defineLazyGlobalGetters(globalThis,", + "XPCOMUtils.defineLazyModuleGetters(window,", + "XPCOMUtils.defineLazyModuleGetters(globalThis,", + "XPCOMUtils.defineLazyServiceGetters(window,", + "XPCOMUtils.defineLazyServiceGetters(globalThis,", + "ChromeUtils.defineESModuleGetters(window,", + "ChromeUtils.defineESModuleGetters(globalThis,", + "loader.lazyRequireGetter(window,", + "loader.lazyRequireGetter(globalThis,", +]; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-lazy-imports-into-globals.html", + }, + messages: { + rejectLazyImportsIntoGlobals: + "Non-system modules should not import into globalThis nor window. Prefer a lazy object holder", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + CallExpression(node) { + let source; + try { + source = helpers.getASTSource(node); + } catch (e) { + return; + } + + if ( + callExpressionDefinitions.some(expr => source.match(expr)) || + callExpressionMultiDefinitions.some(expr => source.startsWith(expr)) + ) { + context.report({ node, messageId: "rejectLazyImportsIntoGlobals" }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-mixing-eager-and-lazy.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-mixing-eager-and-lazy.js new file mode 100644 index 0000000000..5779a90afd --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-mixing-eager-and-lazy.js @@ -0,0 +1,150 @@ +/** + * @fileoverview Reject use of lazy getters for modules that's loaded early in + * the startup process and not necessarily be lazy. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; +const helpers = require("../helpers"); + +function isIdentifier(node, id) { + return node.type === "Identifier" && node.name === id; +} + +function isString(node) { + return node.type === "Literal" && typeof node.value === "string"; +} + +function checkMixed(loadedModules, context, node, type, resourceURI) { + if (!loadedModules.has(resourceURI)) { + loadedModules.set(resourceURI, type); + } + + if (loadedModules.get(resourceURI) === type) { + return; + } + + context.report({ + node, + messageId: "mixedEagerAndLazy", + data: { uri: resourceURI }, + }); +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-mixed-eager-and-lazy.html", + }, + messages: { + mixedEagerAndLazy: + 'Module "{{uri}}" is loaded eagerly, and should not be used for lazy getter.', + }, + schema: [], + type: "problem", + }, + + create(context) { + const loadedModules = new Map(); + + return { + ImportDeclaration(node) { + const resourceURI = node.source.value; + checkMixed(loadedModules, context, node, "eager", resourceURI); + }, + CallExpression(node) { + if (node.callee.type !== "MemberExpression") { + return; + } + + let callerSource; + try { + callerSource = helpers.getASTSource(node.callee); + } catch (e) { + return; + } + + if ( + (callerSource === "ChromeUtils.import" || + callerSource === "ChromeUtils.importESModule") && + helpers.getIsTopLevelAndUnconditionallyExecuted( + context.getAncestors() + ) + ) { + if (node.arguments.length < 1) { + return; + } + const resourceURINode = node.arguments[0]; + if (!isString(resourceURINode)) { + return; + } + checkMixed( + loadedModules, + context, + node, + "eager", + resourceURINode.value + ); + } + + if (callerSource === "ChromeUtils.defineModuleGetter") { + if (node.arguments.length < 3) { + return; + } + if (!isIdentifier(node.arguments[0], "lazy")) { + return; + } + + const resourceURINode = node.arguments[2]; + if (!isString(resourceURINode)) { + return; + } + checkMixed( + loadedModules, + context, + node, + "lazy", + resourceURINode.value + ); + } else if ( + callerSource === "XPCOMUtils.defineLazyModuleGetters" || + callerSource === "ChromeUtils.defineESModuleGetters" + ) { + if (node.arguments.length < 2) { + return; + } + if (!isIdentifier(node.arguments[0], "lazy")) { + return; + } + + const obj = node.arguments[1]; + if (obj.type !== "ObjectExpression") { + return; + } + for (let prop of obj.properties) { + if (prop.type !== "Property") { + continue; + } + if (prop.kind !== "init") { + continue; + } + const resourceURINode = prop.value; + if (!isString(resourceURINode)) { + continue; + } + checkMixed( + loadedModules, + context, + node, + "lazy", + resourceURINode.value + ); + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-multiple-getters-calls.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-multiple-getters-calls.js new file mode 100644 index 0000000000..e6e37ad035 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-multiple-getters-calls.js @@ -0,0 +1,81 @@ +/** + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const helpers = require("../helpers"); + +function findStatement(node) { + while (node && node.type !== "ExpressionStatement") { + node = node.parent; + } + + return node; +} + +function isIdentifier(node, id) { + return node && node.type === "Identifier" && node.name === id; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-multiple-getters-calls.html", + }, + messages: { + rejectMultipleCalls: + "ChromeUtils.defineESModuleGetters is already called for {{target}} in the same context. Please merge those calls", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + const parentToTargets = new Map(); + + return { + CallExpression(node) { + let callee = node.callee; + if ( + callee.type === "MemberExpression" && + isIdentifier(callee.object, "ChromeUtils") && + isIdentifier(callee.property, "defineESModuleGetters") + ) { + const stmt = findStatement(node); + if (!stmt) { + return; + } + + let target; + try { + target = helpers.getASTSource(node.arguments[0]); + } catch (e) { + return; + } + + const parent = stmt.parent; + let targets; + if (parentToTargets.has(parent)) { + targets = parentToTargets.get(parent); + } else { + targets = new Set(); + parentToTargets.set(parent, targets); + } + + if (targets.has(target)) { + context.report({ + node, + messageId: "rejectMultipleCalls", + data: { target }, + }); + } + + targets.add(target); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-relative-requires.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-relative-requires.js new file mode 100644 index 0000000000..34e4b6bd5e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-relative-requires.js @@ -0,0 +1,42 @@ +/** + * @fileoverview Reject some uses of require. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var helpers = require("../helpers"); + +const isRelativePath = function (path) { + return path.startsWith("./") || path.startsWith("../"); +}; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-relative-requires.html", + }, + messages: { + rejectRelativeRequires: "relative paths are not allowed with require()", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + CallExpression(node) { + const path = helpers.getDevToolsRequirePath(node); + if (path && isRelativePath(path)) { + context.report({ + node, + messageId: "rejectRelativeRequires", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-scriptableunicodeconverter.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-scriptableunicodeconverter.js new file mode 100644 index 0000000000..e29a60089c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-scriptableunicodeconverter.js @@ -0,0 +1,44 @@ +/** + * @fileoverview Reject calls into Ci.nsIScriptableUnicodeConverter. We're phasing this out in + * favour of TextEncoder or TextDecoder. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isIdentifier(node, id) { + return node && node.type === "Identifier" && node.name === id; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-scriptableunicodeconverter.html", + }, + messages: { + rejectScriptableUnicodeConverter: + "Ci.nsIScriptableUnicodeConverter is deprecated. You should use TextEncoder or TextDecoder instead.", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + MemberExpression(node) { + if ( + isIdentifier(node.object, "Ci") && + isIdentifier(node.property, "nsIScriptableUnicodeConverter") + ) { + context.report({ + node, + messageId: "rejectScriptableUnicodeConverter", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-some-requires.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-some-requires.js new file mode 100644 index 0000000000..5a4c6b4df7 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-some-requires.js @@ -0,0 +1,44 @@ +/** + * @fileoverview Reject some uses of require. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var helpers = require("../helpers"); + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-some-requires.html", + }, + messages: { + rejectRequire: `require({{path}}) is not allowed`, + }, + schema: [ + { + type: "string", + }, + ], + type: "problem", + }, + + create(context) { + if (typeof context.options[0] !== "string") { + throw new Error("reject-some-requires expects a regexp"); + } + const RX = new RegExp(context.options[0]); + + return { + CallExpression(node) { + const path = helpers.getDevToolsRequirePath(node); + if (path && RX.test(path)) { + context.report({ node, messageId: "rejectRequire", data: { path } }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-top-level-await.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-top-level-await.js new file mode 100644 index 0000000000..dff7db0f9a --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-top-level-await.js @@ -0,0 +1,45 @@ +/** + * @fileoverview Don't allow only() in tests + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var helpers = require("../helpers"); + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-top-level-await.html", + }, + messages: { + rejectTopLevelAwait: + "Top-level await is not currently supported in component files.", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + AwaitExpression(node) { + if (!helpers.getIsTopLevelScript(context.getAncestors())) { + return; + } + context.report({ node, messageId: "rejectTopLevelAwait" }); + }, + ForOfStatement(node) { + if ( + !node.await || + !helpers.getIsTopLevelScript(context.getAncestors()) + ) { + return; + } + context.report({ node, messageId: "rejectTopLevelAwait" }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/rejects-requires-await.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/rejects-requires-await.js new file mode 100644 index 0000000000..a7e7d9d7e2 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/rejects-requires-await.js @@ -0,0 +1,47 @@ +/** + * @fileoverview Ensure Assert.rejects is preceded by await. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/reject-requires-await.html", + }, + messages: { + rejectRequiresAwait: "Assert.rejects needs to be preceded by await.", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + CallExpression(node) { + if (node.callee.type === "MemberExpression") { + let memexp = node.callee; + if ( + memexp.object.type === "Identifier" && + memexp.object.name === "Assert" && + memexp.property.type === "Identifier" && + memexp.property.name === "rejects" + ) { + // We have ourselves an Assert.rejects. + + if (node.parent.type !== "AwaitExpression") { + context.report({ + node, + messageId: "rejectRequiresAwait", + }); + } + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-cc-etc.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-cc-etc.js new file mode 100644 index 0000000000..f47f03f0d2 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-cc-etc.js @@ -0,0 +1,57 @@ +/** + * @fileoverview Reject use of Components.classes etc, prefer the shorthand instead. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const componentsMap = { + classes: "Cc", + interfaces: "Ci", + results: "Cr", + utils: "Cu", +}; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-cc-etc.html", + }, + fixable: "code", + messages: { + useCcEtc: "Use {{ shortName }} rather than Components.{{ oldName }}", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + MemberExpression(node) { + if ( + node.object.type === "Identifier" && + node.object.name === "Components" && + node.property.type === "Identifier" && + Object.getOwnPropertyNames(componentsMap).includes(node.property.name) + ) { + context.report({ + node, + messageId: "useCcEtc", + data: { + shortName: componentsMap[node.property.name], + oldName: node.property.name, + }, + fix: fixer => + fixer.replaceTextRange( + [node.range[0], node.range[1]], + componentsMap[node.property.name] + ), + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-definelazygetter.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-definelazygetter.js new file mode 100644 index 0000000000..a9f43a945a --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-definelazygetter.js @@ -0,0 +1,58 @@ +/** + * @fileoverview Reject use of XPCOMUtils.defineLazyGetter in favor of ChromeUtils.defineLazyGetter. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isIdentifier(node, id) { + return node && node.type === "Identifier" && node.name === id; +} + +function isMemberExpression(node, object, member) { + return ( + node.type === "MemberExpression" && + isIdentifier(node.object, object) && + isIdentifier(node.property, member) + ); +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-chromeutils-definelazygetter.html", + }, + fixable: "code", + messages: { + useChromeUtilsDefineLazyGetter: + "Please use ChromeUtils.defineLazyGetter instead of XPCOMUtils.defineLazyGetter", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + CallExpression(node) { + if (node.callee.type !== "MemberExpression") { + return; + } + + let { callee } = node; + + if (isMemberExpression(callee, "XPCOMUtils", "defineLazyGetter")) { + context.report({ + node, + messageId: "useChromeUtilsDefineLazyGetter", + fix(fixer) { + return fixer.replaceText(callee, "ChromeUtils.defineLazyGetter"); + }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-generateqi.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-generateqi.js new file mode 100644 index 0000000000..d654b0410c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-generateqi.js @@ -0,0 +1,105 @@ +/** + * @fileoverview Reject use of XPCOMUtils.generateQI and JS-implemented + * QueryInterface methods in favor of ChromeUtils. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isIdentifier(node, id) { + return node && node.type === "Identifier" && node.name === id; +} + +function isMemberExpression(node, object, member) { + return ( + node.type === "MemberExpression" && + isIdentifier(node.object, object) && + isIdentifier(node.property, member) + ); +} + +function funcToGenerateQI(context, node) { + const sourceCode = context.getSourceCode(); + const text = sourceCode.getText(node); + + let interfaces = []; + let match; + let re = /\bCi\.([a-zA-Z0-9]+)\b|\b(nsI[A-Z][a-zA-Z0-9]+)\b/g; + while ((match = re.exec(text))) { + interfaces.push(match[1] || match[2]); + } + + let ifaces = interfaces + .filter(iface => iface != "nsISupports") + .map(iface => JSON.stringify(iface)) + .join(", "); + + return `ChromeUtils.generateQI([${ifaces}])`; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-chromeutils-generateqi.html", + }, + fixable: "code", + messages: { + noJSQueryInterface: + "Please use ChromeUtils.generateQI rather than " + + "manually creating JavaScript QueryInterface functions", + noXpcomUtilsGenerateQI: + "Please use ChromeUtils.generateQI instead of XPCOMUtils.generateQI", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + CallExpression(node) { + let { callee } = node; + if (isMemberExpression(callee, "XPCOMUtils", "generateQI")) { + context.report({ + node, + messageId: "noXpcomUtilsGenerateQI", + fix(fixer) { + return fixer.replaceText(callee, "ChromeUtils.generateQI"); + }, + }); + } + }, + + "AssignmentExpression > MemberExpression[property.name='QueryInterface']": + function (node) { + const { right } = node.parent; + if (right.type === "FunctionExpression") { + context.report({ + node: node.parent, + messageId: "noJSQueryInterface", + fix(fixer) { + return fixer.replaceText( + right, + funcToGenerateQI(context, right) + ); + }, + }); + } + }, + + "Property[key.name='QueryInterface'][value.type='FunctionExpression']": + function (node) { + context.report({ + node, + messageId: "noJSQueryInterface", + fix(fixer) { + let generateQI = funcToGenerateQI(context, node.value); + return fixer.replaceText(node, `QueryInterface: ${generateQI}`); + }, + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-import.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-import.js new file mode 100644 index 0000000000..925b4800bc --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-import.js @@ -0,0 +1,63 @@ +/** + * @fileoverview Reject use of Cu.import in favor of ChromeUtils. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +function isIdentifier(node, id) { + return node && node.type === "Identifier" && node.name === id; +} + +function isMemberExpression(node, object, member) { + return ( + node.type === "MemberExpression" && + isIdentifier(node.object, object) && + isIdentifier(node.property, member) + ); +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-chromeutils-import.html", + }, + fixable: "code", + messages: { + useChromeUtilsImport: + "Please use ChromeUtils.import instead of Cu.import", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + CallExpression(node) { + if (node.callee.type !== "MemberExpression") { + return; + } + + let { callee } = node; + + // Is the expression starting with `Cu` or `Components.utils`? + if ( + (isIdentifier(callee.object, "Cu") || + isMemberExpression(callee.object, "Components", "utils")) && + isIdentifier(callee.property, "import") + ) { + context.report({ + node, + messageId: "useChromeUtilsImport", + fix(fixer) { + return fixer.replaceText(callee, "ChromeUtils.import"); + }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-console-createInstance.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-console-createInstance.js new file mode 100644 index 0000000000..72add0ab24 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-console-createInstance.js @@ -0,0 +1,44 @@ +/** + * @fileoverview Reject use of Console.sys.mjs and Log.sys.mjs. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-console-createInstance.html", + }, + messages: { + useConsoleRatherThanModule: + "Use console.createInstance rather than {{module}}", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + Literal(node) { + if (typeof node.value != "string") { + return; + } + /* eslint-disable mozilla/use-console-createInstance */ + if ( + node.value == "resource://gre/modules/Console.sys.mjs" || + node.value == "resource://gre/modules/Log.sys.mjs" + ) { + context.report({ + node, + messageId: "useConsoleRatherThanModule", + data: { module: node.value.split("/").at(-1) }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-default-preference-values.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-default-preference-values.js new file mode 100644 index 0000000000..edc1e28405 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-default-preference-values.js @@ -0,0 +1,56 @@ +/** + * @fileoverview Require providing a second parameter to get*Pref + * methods instead of using a try/catch block. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-default-preference-values.html", + }, + messages: { + provideDefaultValue: + "provide a default value instead of using a try/catch block", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + TryStatement(node) { + let types = ["Bool", "Char", "Float", "Int"]; + let methods = types.map(type => "get" + type + "Pref"); + if ( + node.block.type != "BlockStatement" || + node.block.body.length != 1 + ) { + return; + } + + let firstStm = node.block.body[0]; + if ( + firstStm.type != "ExpressionStatement" || + firstStm.expression.type != "AssignmentExpression" || + firstStm.expression.right.type != "CallExpression" || + firstStm.expression.right.callee.type != "MemberExpression" || + firstStm.expression.right.callee.property.type != "Identifier" || + !methods.includes(firstStm.expression.right.callee.property.name) + ) { + return; + } + + context.report({ + node, + messageId: "provideDefaultValue", + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-includes-instead-of-indexOf.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-includes-instead-of-indexOf.js new file mode 100644 index 0000000000..245c89a095 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-includes-instead-of-indexOf.js @@ -0,0 +1,53 @@ +/** + * @fileoverview Use .includes instead of .indexOf + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-includes-instead-of-indexOf.html", + }, + messages: { + useIncludes: "use .includes instead of .indexOf", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + BinaryExpression(node) { + if ( + node.left.type != "CallExpression" || + node.left.callee.type != "MemberExpression" || + node.left.callee.property.type != "Identifier" || + node.left.callee.property.name != "indexOf" + ) { + return; + } + + if ( + (["!=", "!==", "==", "==="].includes(node.operator) && + node.right.type == "UnaryExpression" && + node.right.operator == "-" && + node.right.argument.type == "Literal" && + node.right.argument.value == 1) || + ([">=", "<"].includes(node.operator) && + node.right.type == "Literal" && + node.right.value == 0) + ) { + context.report({ + node, + messageId: "useIncludes", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-isInstance.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-isInstance.js new file mode 100644 index 0000000000..ffd9bc9566 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-isInstance.js @@ -0,0 +1,155 @@ +/** + * @fileoverview Reject use of instanceof against DOM interfaces + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const fs = require("fs"); + +const { maybeGetMemberPropertyName } = require("../helpers"); + +const privilegedGlobals = Object.keys( + require("../environments/privileged.js").globals +); + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +/** + * Whether an identifier is defined by eslint configuration. + * `env: { browser: true }` or `globals: []` for example. + * @param {import("eslint-scope").Scope} currentScope + * @param {import("estree").Identifier} id + */ +function refersToEnvironmentGlobals(currentScope, id) { + const reference = currentScope.references.find(ref => ref.identifier === id); + const { resolved } = reference || {}; + if (!resolved) { + return false; + } + + // No definition in script files; defined via .eslintrc + return resolved.scope.type === "global" && resolved.defs.length === 0; +} + +/** + * Whether a node points to a DOM interface. + * Includes direct references to interfaces objects and also indirect references + * via property access. + * OS.File and lazy.(Foo) are explicitly excluded. + * + * @example HTMLElement + * @example win.HTMLElement + * @example iframe.contentWindow.HTMLElement + * @example foo.HTMLElement + * + * @param {import("eslint-scope").Scope} currentScope + * @param {import("estree").Node} node + */ +function pointsToDOMInterface(currentScope, node) { + if (node.type === "MemberExpression") { + const objectName = maybeGetMemberPropertyName(node.object); + if (objectName === "lazy") { + // lazy.Foo is probably a non-IDL import. + return false; + } + if (objectName === "OS" && node.property.name === "File") { + // OS.File is an exception that is not a Web IDL interface + return false; + } + // For `win.Foo`, `iframe.contentWindow.Foo`, or such. + return privilegedGlobals.includes(node.property.name); + } + + if ( + node.type === "Identifier" && + refersToEnvironmentGlobals(currentScope, node) + ) { + return privilegedGlobals.includes(node.name); + } + + return false; +} + +/** + * @param {import("eslint").Rule.RuleContext} context + */ +function isChromeContext(context) { + const filename = context.getFilename(); + const isChromeFileName = + filename.endsWith(".sys.mjs") || filename.endsWith(".jsm"); + if (isChromeFileName) { + return true; + } + + if (filename.endsWith(".xhtml")) { + // Treat scripts in XUL files as chrome scripts + // Note: readFile is needed as getSourceCode() only gives JS blocks + return fs.readFileSync(filename).includes("there.is.only.xul"); + } + + // Treat scripts as chrome privileged when using: + // 1. ChromeUtils, but not SpecialPowers.ChromeUtils + // 2. BrowserTestUtils, PlacesUtils + // 3. document.createXULElement + // 4. loader.lazyRequireGetter + // 5. Services.foo, but not SpecialPowers.Services.foo + // 6. evalInSandbox + const source = context.getSourceCode().text; + return !!source.match( + /(^|\s)ChromeUtils|BrowserTestUtils|PlacesUtils|createXULElement|lazyRequireGetter|(^|\s)Services\.|evalInSandbox/ + ); +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-isInstance.html", + }, + fixable: "code", + messages: { + preferIsInstance: + "Please prefer .isInstance() in chrome scripts over the standard instanceof operator for DOM interfaces, " + + "since the latter will return false when the object is created from a different context.", + }, + schema: [], + type: "problem", + }, + /** + * @param {import("eslint").Rule.RuleContext} context + */ + create(context) { + if (!isChromeContext(context)) { + return {}; + } + + return { + BinaryExpression(node) { + const { operator, right } = node; + if ( + operator === "instanceof" && + pointsToDOMInterface(context.getScope(), right) + ) { + context.report({ + node, + messageId: "preferIsInstance", + fix(fixer) { + const sourceCode = context.getSourceCode(); + return fixer.replaceText( + node, + `${sourceCode.getText(right)}.isInstance(${sourceCode.getText( + node.left + )})` + ); + }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-ownerGlobal.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-ownerGlobal.js new file mode 100644 index 0000000000..1d71e82b8f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-ownerGlobal.js @@ -0,0 +1,43 @@ +/** + * @fileoverview Require .ownerGlobal instead of .ownerDocument.defaultView. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-ownerGlobal.html", + }, + messages: { + useOwnerGlobal: "use .ownerGlobal instead of .ownerDocument.defaultView", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + MemberExpression(node) { + if ( + node.property.type != "Identifier" || + node.property.name != "defaultView" || + node.object.type != "MemberExpression" || + node.object.property.type != "Identifier" || + node.object.property.name != "ownerDocument" + ) { + return; + } + + context.report({ + node, + messageId: "useOwnerGlobal", + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-returnValue.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-returnValue.js new file mode 100644 index 0000000000..23bbc040b9 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-returnValue.js @@ -0,0 +1,48 @@ +/** + * @fileoverview Warn when idempotent methods are called and their return value is unused. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-returnValue.html", + }, + messages: { + useReturnValue: + "{Array/String}.{{ property }} doesn't modify the instance in-place", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + ExpressionStatement(node) { + if ( + node.expression?.type != "CallExpression" || + node.expression.callee?.type != "MemberExpression" || + node.expression.callee.property?.type != "Identifier" || + !["concat", "join", "slice"].includes( + node.expression.callee.property?.name + ) + ) { + return; + } + + context.report({ + node, + messageId: "useReturnValue", + data: { + property: node.expression.callee.property.name, + }, + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-services.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-services.js new file mode 100644 index 0000000000..3a7cc34633 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-services.js @@ -0,0 +1,120 @@ +/** + * @fileoverview Require use of Services.* rather than getService. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; +const helpers = require("../helpers"); + +let servicesInterfaceMap = helpers.servicesData; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-services.html", + }, + // fixable: "code", + messages: { + useServices: + "Use Services.{{ serviceName }} rather than {{ getterName }}.", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + CallExpression(node) { + if (!node.callee || !node.callee.property) { + return; + } + + if ( + node.callee.property.type == "Identifier" && + node.callee.property.name == "defineLazyServiceGetter" && + node.arguments.length == 4 && + node.arguments[3].type == "Literal" && + node.arguments[3].value in servicesInterfaceMap + ) { + let serviceName = servicesInterfaceMap[node.arguments[3].value]; + + context.report({ + node, + messageId: "useServices", + data: { + serviceName, + getterName: "defineLazyServiceGetter", + }, + }); + return; + } + + if ( + node.callee.property.type == "Identifier" && + node.callee.property.name == "defineLazyServiceGetters" && + node.arguments.length == 2 && + node.arguments[1].type == "ObjectExpression" + ) { + for (let property of node.arguments[1].properties) { + if ( + property.value.type == "ArrayExpression" && + property.value.elements.length == 2 && + property.value.elements[1].value in servicesInterfaceMap + ) { + let serviceName = + servicesInterfaceMap[property.value.elements[1].value]; + + context.report({ + node: property.value, + messageId: "useServices", + data: { + serviceName, + getterName: "defineLazyServiceGetters", + }, + }); + } + } + return; + } + + if ( + node.callee.property.type != "Identifier" || + node.callee.property.name != "getService" || + node.arguments.length != 1 || + !node.arguments[0].property || + node.arguments[0].property.type != "Identifier" || + !node.arguments[0].property.name || + !(node.arguments[0].property.name in servicesInterfaceMap) + ) { + return; + } + + let serviceName = servicesInterfaceMap[node.arguments[0].property.name]; + context.report({ + node, + messageId: "useServices", + data: { + serviceName, + getterName: "getService()", + }, + // This is not enabled by default as for mochitest plain tests we + // would need to replace with `SpecialPowers.Services.${serviceName}`. + // At the moment we do not have an easy way to detect that. + // fix(fixer) { + // let sourceCode = context.getSourceCode(); + // return fixer.replaceTextRange( + // [ + // sourceCode.getFirstToken(node.callee).range[0], + // sourceCode.getLastToken(node).range[1], + // ], + // `Services.${serviceName}` + // ); + // }, + }); + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-static-import.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-static-import.js new file mode 100644 index 0000000000..100b5682de --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-static-import.js @@ -0,0 +1,87 @@ +/** + * @fileoverview Require use of static imports where possible. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const helpers = require("../helpers"); + +function isIdentifier(node, id) { + return node && node.type === "Identifier" && node.name === id; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/use-static-import.html", + }, + fixable: "code", + messages: { + useStaticImport: + "Please use static import instead of ChromeUtils.importESModule", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + VariableDeclarator(node) { + if ( + node.init?.type != "CallExpression" || + node.init?.callee?.type != "MemberExpression" || + !context.getFilename().endsWith(".sys.mjs") || + !helpers.isTopLevel(context.getAncestors()) + ) { + return; + } + + let callee = node.init.callee; + + if ( + isIdentifier(callee.object, "ChromeUtils") && + isIdentifier(callee.property, "importESModule") && + callee.parent.arguments.length == 1 + ) { + let sourceCode = context.getSourceCode(); + let importItemSource; + if (node.id.type != "ObjectPattern") { + importItemSource = sourceCode.getText(node.id); + } else { + importItemSource = "{ "; + let initial = true; + for (let property of node.id.properties) { + if (!initial) { + importItemSource += ", "; + } + initial = false; + if (property.key.name == property.value.name) { + importItemSource += property.key.name; + } else { + importItemSource += `${property.key.name} as ${property.value.name}`; + } + } + importItemSource += " }"; + } + + context.report({ + node: node.parent, + messageId: "useStaticImport", + fix(fixer) { + return fixer.replaceText( + node.parent, + `import ${importItemSource} from ${sourceCode.getText( + callee.parent.arguments[0] + )}` + ); + }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-ci-uses.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-ci-uses.js new file mode 100644 index 0000000000..4036a72928 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-ci-uses.js @@ -0,0 +1,172 @@ +/** + * @fileoverview Reject uses of unknown interfaces on Ci and properties of those + * interfaces. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const os = require("os"); +const helpers = require("../helpers"); + +// These interfaces are all platform specific, so may be not present +// on all platforms. +const platformSpecificInterfaces = new Map([ + ["nsIAboutThirdParty", "windows"], + ["nsIAboutWindowsMessages", "windows"], + ["nsIDefaultAgent", "windows"], + ["nsIJumpListBuilder", "windows"], + ["nsILegacyJumpListBuilder", "windows"], + ["nsILegacyJumpListItem", "windows"], + ["nsILegacyJumpListLink", "windows"], + ["nsILegacyJumpListSeparator", "windows"], + ["nsILegacyJumpListShortcut", "windows"], + ["nsITaskbarWindowPreview", "windows"], + ["nsIWindowsAlertsService", "windows"], + ["nsIWindowsAlertNotification", "windows"], + ["nsIWindowsMutexFactory", "windows"], + ["nsIWinAppHelper", "windows"], + ["nsIWinTaskbar", "windows"], + ["nsIWinTaskSchedulerService", "windows"], + ["nsIWindowsRegKey", "windows"], + ["nsIWindowsPackageManager", "windows"], + ["nsIWindowsShellService", "windows"], + ["nsIAccessibleMacEvent", "darwin"], + ["nsIAccessibleMacInterface", "darwin"], + ["nsILocalFileMac", "darwin"], + ["nsIAccessibleMacEvent", "darwin"], + ["nsIMacAttributionService", "darwin"], + ["nsIMacShellService", "darwin"], + ["nsIMacDockSupport", "darwin"], + ["nsIMacFinderProgress", "darwin"], + ["nsIMacPreferencesReader", "darwin"], + ["nsIMacSharingService", "darwin"], + ["nsIMacUserActivityUpdater", "darwin"], + ["nsIMacWebAppUtils", "darwin"], + ["nsIStandaloneNativeMenu", "darwin"], + ["nsITouchBarHelper", "darwin"], + ["nsITouchBarInput", "darwin"], + ["nsITouchBarUpdater", "darwin"], + ["mozISandboxReporter", "linux"], + ["nsIApplicationChooser", "linux"], + ["nsIGNOMEShellService", "linux"], + ["nsIGtkTaskbarProgress", "linux"], + + // These are used in the ESLint test code. + ["amIFoo", "any"], + ["nsIMeh", "any"], + // Can't easily detect android builds from ESLint at the moment. + ["nsIAndroidBridge", "any"], + ["nsIAndroidView", "any"], + // Code coverage is enabled only for certain builds (MOZ_CODE_COVERAGE). + ["nsICodeCoverage", "any"], + // Layout debugging is enabled only for certain builds (MOZ_LAYOUT_DEBUGGER). + ["nsILayoutDebuggingTools", "any"], + // Sandbox test is only enabled for certain configurations (MOZ_SANDBOX, + // MOZ_DEBUG, ENABLE_TESTS). + ["mozISandboxTest", "any"], +]); + +function interfaceHasProperty(interfaceName, propertyName) { + // `Ci.nsIFoo.number` is valid, it returns the iid. + if (propertyName == "number") { + return true; + } + + let interfaceInfo = helpers.xpidlData.get(interfaceName); + + if (!interfaceInfo) { + return true; + } + + // If the property is not in the lists of consts for this interface, check + // any parents as well. + if (!interfaceInfo.consts.find(e => e.name === propertyName)) { + if (interfaceInfo.parent && interfaceInfo.parent != "nsISupports") { + return interfaceHasProperty(interfaceName.parent, propertyName); + } + return false; + } + return true; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/valid-ci-uses.html", + }, + messages: { + missingInterface: + "{{ interface }} is defined in this rule's platform specific list, but is not available", + unknownInterface: "Use of unknown interface Ci.{{ interface}}", + unknownProperty: + "Use of unknown property Ci.{{ interface }}.{{ property }}", + }, + schema: [], + type: "problem", + }, + + create(context) { + return { + MemberExpression(node) { + if ( + node.computed === false && + node.type === "MemberExpression" && + node.object.type === "Identifier" && + node.object.name === "Ci" && + node.property.type === "Identifier" && + node.property.name.includes("I") + ) { + if (!helpers.xpidlData.get(node.property.name)) { + let platformSpecific = platformSpecificInterfaces.get( + node.property.name + ); + if (!platformSpecific) { + context.report({ + node, + messageId: "unknownInterface", + data: { + interface: node.property.name, + }, + }); + } else if (platformSpecific == os.platform) { + context.report({ + node, + messageId: "missingInterface", + data: { + interface: node.property.name, + }, + }); + } + } + } + + if ( + node.computed === false && + node.object.type === "MemberExpression" && + node.object.object.type === "Identifier" && + node.object.object.name === "Ci" && + node.object.property.type === "Identifier" && + node.object.property.name.includes("I") && + node.property.type === "Identifier" + ) { + if ( + !interfaceHasProperty(node.object.property.name, node.property.name) + ) { + context.report({ + node, + messageId: "unknownProperty", + data: { + interface: node.object.property.name, + property: node.property.name, + }, + }); + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-lazy.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-lazy.js new file mode 100644 index 0000000000..048ed17e3e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-lazy.js @@ -0,0 +1,276 @@ +/** + * @fileoverview Ensures that definitions and uses of properties on the + * ``lazy`` object are valid. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; +const helpers = require("../helpers"); + +const items = [ + "loader", + "XPCOMUtils", + "Integration", + "ChromeUtils", + "DevToolsUtils", + "Object", + "Reflect", +]; + +const callExpressionDefinitions = [ + /^loader\.lazyGetter\(lazy, "(\w+)"/, + /^loader\.lazyServiceGetter\(lazy, "(\w+)"/, + /^loader\.lazyRequireGetter\(lazy, "(\w+)"/, + /^XPCOMUtils\.defineLazyGetter\(lazy, "(\w+)"/, + /^Integration\.downloads\.defineESModuleGetter\(lazy, "(\w+)"/, + /^ChromeUtils\.defineLazyGetter\(lazy, "(\w+)"/, + /^ChromeUtils\.defineModuleGetter\(lazy, "(\w+)"/, + /^XPCOMUtils\.defineLazyPreferenceGetter\(lazy, "(\w+)"/, + /^XPCOMUtils\.defineLazyScriptGetter\(lazy, "(\w+)"/, + /^XPCOMUtils\.defineLazyServiceGetter\(lazy, "(\w+)"/, + /^XPCOMUtils\.defineConstant\(lazy, "(\w+)"/, + /^DevToolsUtils\.defineLazyGetter\(lazy, "(\w+)"/, + /^Object\.defineProperty\(lazy, "(\w+)"/, + /^Reflect\.defineProperty\(lazy, "(\w+)"/, +]; + +const callExpressionMultiDefinitions = [ + "ChromeUtils.defineESModuleGetters(lazy,", + "XPCOMUtils.defineLazyModuleGetters(lazy,", + "XPCOMUtils.defineLazyServiceGetters(lazy,", + "Object.defineProperties(lazy,", + "loader.lazyRequireGetter(lazy,", +]; + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/valid-lazy.html", + }, + messages: { + duplicateSymbol: "Duplicate symbol {{name}} being added to lazy.", + incorrectType: "Unexpected literal for property name {{name}}", + unknownProperty: "Unknown lazy member property {{name}}", + unusedProperty: "Unused lazy property {{name}}", + topLevelAndUnconditional: + "Lazy property {{name}} is used at top-level unconditionally. It should be non-lazy.", + }, + schema: [], + type: "problem", + }, + + create(context) { + let lazyProperties = new Map(); + let unknownProperties = []; + let isLazyExported = false; + + function getAncestorNodes(node) { + const ancestors = []; + node = node.parent; + while (node) { + ancestors.unshift(node); + node = node.parent; + } + return ancestors; + } + + // Returns true if lazy getter definitions in prevNode and currNode are + // duplicate. + // This returns false if prevNode and currNode have the same IfStatement as + // ancestor and they're in different branches. + function isDuplicate(prevNode, currNode) { + const prevAncestors = getAncestorNodes(prevNode); + const currAncestors = getAncestorNodes(currNode); + + for ( + let i = 0; + i < prevAncestors.length && i < currAncestors.length; + i++ + ) { + const prev = prevAncestors[i]; + const curr = currAncestors[i]; + if (prev === curr && prev.type === "IfStatement") { + if (prevAncestors[i + 1] !== currAncestors[i + 1]) { + return false; + } + } + } + + return true; + } + + function addProp(callNode, propNode, name) { + if ( + lazyProperties.has(name) && + isDuplicate(lazyProperties.get(name).callNode, callNode) + ) { + context.report({ + node: propNode, + messageId: "duplicateSymbol", + data: { name }, + }); + return; + } + lazyProperties.set(name, { used: false, callNode, propNode }); + } + + function setPropertiesFromArgument(callNode, arg) { + if (arg.type === "ObjectExpression") { + for (let propNode of arg.properties) { + if (propNode.key.type == "Literal") { + context.report({ + node: propNode, + messageId: "incorrectType", + data: { name: propNode.key.value }, + }); + continue; + } + addProp(callNode, propNode, propNode.key.name); + } + } else if (arg.type === "ArrayExpression") { + for (let propNode of arg.elements) { + if (propNode.type != "Literal") { + continue; + } + addProp(callNode, propNode, propNode.value); + } + } + } + + return { + VariableDeclarator(node) { + if ( + node.id.type === "Identifier" && + node.id.name == "lazy" && + node.init.type == "CallExpression" && + node.init.callee.name == "createLazyLoaders" + ) { + setPropertiesFromArgument(node.init, node.init.arguments[0]); + } + }, + + CallExpression(node) { + if ( + node.callee.type != "MemberExpression" || + (node.callee.object.type == "MemberExpression" && + !items.includes(node.callee.object.object.name)) || + (node.callee.object.type != "MemberExpression" && + !items.includes(node.callee.object.name)) + ) { + return; + } + + let source; + try { + source = helpers.getASTSource(node); + } catch (e) { + return; + } + + for (let reg of callExpressionDefinitions) { + let match = source.match(reg); + if (match) { + if ( + lazyProperties.has(match[1]) && + isDuplicate(lazyProperties.get(match[1]).callNode, node) + ) { + context.report({ + node, + messageId: "duplicateSymbol", + data: { name: match[1] }, + }); + return; + } + lazyProperties.set(match[1], { + used: false, + callNode: node, + propNode: node, + }); + break; + } + } + + if ( + callExpressionMultiDefinitions.some(expr => + source.startsWith(expr) + ) && + node.arguments[1] + ) { + setPropertiesFromArgument(node, node.arguments[1]); + } + }, + + MemberExpression(node) { + if (node.computed || node.object.type !== "Identifier") { + return; + } + + let name; + if (node.object.name == "lazy") { + name = node.property.name; + } else { + return; + } + let property = lazyProperties.get(name); + if (!property) { + // These will be reported on Program:exit - some definitions may + // be after first use, so we need to wait until we've processed + // the whole file before reporting. + unknownProperties.push({ name, node }); + } else { + property.used = true; + } + if ( + helpers.getIsTopLevelAndUnconditionallyExecuted( + context.getAncestors() + ) + ) { + context.report({ + node, + messageId: "topLevelAndUnconditional", + data: { name }, + }); + } + }, + + ExportNamedDeclaration(node) { + for (const spec of node.specifiers) { + if (spec.local.name === "lazy") { + // If the lazy object is exported, do not check unused property. + isLazyExported = true; + break; + } + } + }, + + "Program:exit": function () { + for (let { name, node } of unknownProperties) { + let property = lazyProperties.get(name); + if (!property) { + context.report({ + node, + messageId: "unknownProperty", + data: { name }, + }); + } else { + property.used = true; + } + } + if (!isLazyExported) { + for (let [name, property] of lazyProperties.entries()) { + if (!property.used) { + context.report({ + node: property.propNode, + messageId: "unusedProperty", + data: { name }, + }); + } + } + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-services-property.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-services-property.js new file mode 100644 index 0000000000..8f665d6d8a --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-services-property.js @@ -0,0 +1,126 @@ +/** + * @fileoverview Ensures that property accesses on Services.<alias> are valid. + * Although this largely duplicates the valid-services rule, the checks here + * require an objdir and a manual run. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const helpers = require("../helpers"); + +function findInterfaceNames(name) { + let interfaces = []; + for (let [key, value] of Object.entries(helpers.servicesData)) { + if (value == name) { + interfaces.push(key); + } + } + return interfaces; +} + +function isInInterface(interfaceName, name) { + let interfaceDetails = helpers.xpidlData.get(interfaceName); + + // TODO: Bug 1790261 - check only methods if the expression is callable. + if (interfaceDetails.methods.some(m => m.name == name)) { + return true; + } + + if (interfaceDetails.consts.some(c => c.name == name)) { + return true; + } + + if (interfaceDetails.parent) { + return isInInterface(interfaceDetails.parent, name); + } + return false; +} + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/valid-services-property.html", + }, + messages: { + unknownProperty: + "Unknown property access Services.{{ alias }}.{{ propertyName }}, Interfaces: {{ checkedInterfaces }}", + }, + schema: [], + type: "problem", + }, + + create(context) { + let servicesInterfaceMap = helpers.servicesData; + let serviceAliases = new Set([ + ...Object.values(servicesInterfaceMap), + // This is defined only for Android, so most builds won't pick it up. + "androidBridge", + // These are defined without interfaces and hence are not in the services map. + "cpmm", + "crashmanager", + "mm", + "ppmm", + // The new xulStore also does not have an interface. + "xulStore", + ]); + return { + MemberExpression(node) { + if (node.computed || node.object.type !== "Identifier") { + return; + } + + let mainNode; + if (node.object.name == "Services") { + mainNode = node; + } else if ( + node.property.name == "Services" && + node.parent.type == "MemberExpression" + ) { + mainNode = node.parent; + } else { + return; + } + + let alias = mainNode.property.name; + if (!serviceAliases.has(alias)) { + return; + } + + if ( + mainNode.parent.type == "MemberExpression" && + !mainNode.parent.computed + ) { + let propertyName = mainNode.parent.property.name; + if (propertyName == "wrappedJSObject") { + return; + } + let interfaces = findInterfaceNames(alias); + if (!interfaces.length) { + return; + } + + let checkedInterfaces = []; + for (let item of interfaces) { + if (isInInterface(item, propertyName)) { + return; + } + checkedInterfaces.push(item); + } + context.report({ + node, + messageId: "unknownProperty", + data: { + alias, + propertyName, + checkedInterfaces, + }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-services.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-services.js new file mode 100644 index 0000000000..7380fda491 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-services.js @@ -0,0 +1,68 @@ +/** + * @fileoverview Ensures that Services uses have valid property names. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; +const helpers = require("../helpers"); + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/valid-services.html", + }, + messages: { + unknownProperty: "Unknown Services member property {{ alias }}", + }, + schema: [], + type: "problem", + }, + + create(context) { + let servicesInterfaceMap = helpers.servicesData; + let serviceAliases = new Set([ + ...Object.values(servicesInterfaceMap), + // This is defined only for Android, so most builds won't pick it up. + "androidBridge", + // These are defined without interfaces and hence are not in the services map. + "cpmm", + "crashmanager", + "mm", + "ppmm", + // The new xulStore also does not have an interface. + "xulStore", + ]); + return { + MemberExpression(node) { + if (node.computed || node.object.type !== "Identifier") { + return; + } + + let alias; + if (node.object.name == "Services") { + alias = node.property.name; + } else if ( + node.property.name == "Services" && + node.parent.type == "MemberExpression" + ) { + alias = node.parent.property.name; + } else { + return; + } + + if (!serviceAliases.has(alias)) { + context.report({ + node, + messageId: "unknownProperty", + data: { + alias, + }, + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/var-only-at-top-level.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/var-only-at-top-level.js new file mode 100644 index 0000000000..5da799c643 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/var-only-at-top-level.js @@ -0,0 +1,42 @@ +/** + * @fileoverview Marks all var declarations that are not at the top level + * invalid. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var helpers = require("../helpers"); + +module.exports = { + meta: { + docs: { + url: "https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint-plugin-mozilla/rules/var-only-at-top-level.html", + }, + messages: { + unexpectedVar: "Unexpected var, use let or const instead.", + }, + schema: [], + type: "suggestion", + }, + + create(context) { + return { + VariableDeclaration(node) { + if (node.kind === "var") { + if (helpers.getIsTopLevelScript(context.getAncestors())) { + return; + } + + context.report({ + node, + messageId: "unexpectedVar", + }); + } + }, + }; + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/services.json b/tools/lint/eslint/eslint-plugin-mozilla/lib/services.json new file mode 100644 index 0000000000..476c6bf784 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/services.json @@ -0,0 +1,63 @@ +{ + "mozIJSSubScriptLoader": "scriptloader", + "mozILocaleService": "locale", + "mozIMozIntl": "intl", + "mozIStorageService": "storage", + "nsIAppShellService": "appShell", + "nsIAppStartup": "startup", + "nsIBlocklistService": "blocklist", + "nsICacheStorageService": "cache2", + "nsICategoryManager": "catMan", + "nsIClearDataService": "clearData", + "nsIClipboard": "clipboard", + "nsIConsoleService": "console", + "nsICookieBannerService": "cookieBanners", + "nsICookieManager": "cookies", + "nsICookieService": "cookies", + "nsICrashReporter": "appinfo", + "nsIDAPTelemetry": "DAPTelemetry", + "nsIDOMRequestService": "DOMRequest", + "nsIDOMStorageManager": "domStorageManager", + "nsIDNSService": "dns", + "nsIDirectoryService": "dirsvc", + "nsIDroppedLinkHandler": "droppedLinkHandler", + "nsIEffectiveTLDService": "eTLD", + "nsIEnterprisePolicies": "policies", + "nsIEnvironment": "env", + "nsIEventListenerService": "els", + "nsIFOG": "fog", + "nsIFocusManager": "focus", + "nsIIOService": "io", + "nsILoadContextInfoFactory": "loadContextInfo", + "nsILocalStorageManager": "domStorageManager", + "nsILoginManager": "logins", + "nsINetUtil": "io", + "nsIObserverService": "obs", + "nsIPermissionManager": "perms", + "nsIPrefBranch": "prefs", + "nsIPrefService": "prefs", + "nsIProfiler": "profiler", + "nsIPromptService": "prompt", + "nsIProperties": "dirsvc", + "nsIPropertyBag2": "sysinfo", + "nsIQuotaManagerService": "qms", + "nsIRFPService": "rfp", + "nsIScriptSecurityManager": "scriptSecurityManager", + "nsISearchService": "search", + "nsISessionStorageService": "sessionStorage", + "nsISpeculativeConnect": "io", + "nsIStringBundleService": "strings", + "nsISystemInfo": "sysinfo", + "nsITelemetry": "telemetry", + "nsITextToSubURI": "textToSubURI", + "nsIThreadManager": "tm", + "nsIURIFixup": "uriFixup", + "nsIURLFormatter": "urlFormatter", + "nsIUUIDGenerator": "uuid", + "nsIVersionComparator": "vc", + "nsIWindowMediator": "wm", + "nsIWindowWatcher": "ww", + "nsIXULAppInfo": "appinfo", + "nsIXULRuntime": "appinfo", + "nsIXULStore": "xulStore" +} diff --git a/tools/lint/eslint/eslint-plugin-mozilla/manifest.tt b/tools/lint/eslint/eslint-plugin-mozilla/manifest.tt new file mode 100644 index 0000000000..47ca98109b --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/manifest.tt @@ -0,0 +1,10 @@ +[ + { + "filename": "eslint-plugin-mozilla.tar.gz", + "size": 5546844, + "algorithm": "sha512", + "digest": "7df05ad96a7e892f04079570e7847cef71048e3d96e7a338f7e5aa431d5bf34225196200fffc42a630678f2bb074b806dd24d76da25827fbfc16d0ae5e8e6cd5", + "unpack": true, + "visibility": "public" + } +]
\ No newline at end of file diff --git a/tools/lint/eslint/eslint-plugin-mozilla/package-lock.json b/tools/lint/eslint/eslint-plugin-mozilla/package-lock.json new file mode 100644 index 0000000000..8265c41a94 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/package-lock.json @@ -0,0 +1,5249 @@ +{ + "name": "eslint-plugin-mozilla", + "version": "3.7.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "eslint-plugin-mozilla", + "version": "3.7.0", + "license": "MPL-2.0", + "dependencies": { + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "estraverse": "^5.3.0", + "htmlparser2": "^8.0.1", + "toml-eslint-parser": "0.9.3" + }, + "devDependencies": { + "eslint": "8.56.0", + "mocha": "10.2.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "@microsoft/eslint-plugin-sdl": "^0.2.2", + "eslint": "^7.23.0 || ^8.0.0", + "eslint-config-prettier": "^8.0.0 || ^9.0.0", + "eslint-plugin-fetch-options": "^0.0.5", + "eslint-plugin-html": "^7.0.0", + "eslint-plugin-json": "^3.1.0", + "eslint-plugin-no-unsanitized": "^4.0.0" + } + }, + "node_modules/@aashutoshrathi/word-wrap": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", + "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", + "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "dependencies": { + "eslint-visitor-keys": "^3.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", + "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.56.0.tgz", + "integrity": "sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.11.14", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", + "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "dependencies": { + "@humanwhocodes/object-schema": "^2.0.2", + "debug": "^4.3.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz", + "integrity": "sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==" + }, + "node_modules/@microsoft/eslint-plugin-sdl": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@microsoft/eslint-plugin-sdl/-/eslint-plugin-sdl-0.2.2.tgz", + "integrity": "sha512-TiBepeQMSxHpvIbKA03TbO9nZqRrKR1th47wGdjY1sH2SSer+JgKlSF3S8GURGA8/zp2T/HwSiAJelclJ3hEvg==", + "peer": true, + "dependencies": { + "eslint-plugin-node": "11.1.0", + "eslint-plugin-react": "7.33.0", + "eslint-plugin-security": "1.4.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "eslint": "^4.19.1 || ^5 || ^6 || ^7 || ^8" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==" + }, + "node_modules/acorn": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", + "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/array-buffer-byte-length": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz", + "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "is-array-buffer": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array-includes": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.7.tgz", + "integrity": "sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", + "is-string": "^1.0.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array.prototype.flat": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz", + "integrity": "sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "es-shim-unscopables": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array.prototype.flatmap": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz", + "integrity": "sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "es-shim-unscopables": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/array.prototype.tosorted": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/array.prototype.tosorted/-/array.prototype.tosorted-1.1.2.tgz", + "integrity": "sha512-HuQCHOlk1Weat5jzStICBCd83NxiIMwqDg/dHEsoefabn/hJRj5pVdWcPUSpRrwhwxZOsQassMpgN/xRYFBMIg==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "es-shim-unscopables": "^1.0.0", + "get-intrinsic": "^1.2.1" + } + }, + "node_modules/arraybuffer.prototype.slice": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.2.tgz", + "integrity": "sha512-yMBKppFur/fbHu9/6USUe03bZ4knMYiwFBcyiaXB8Go0qNehwX6inYPzK9U0NeQvGxKthcmHcaR8P5MStSRBAw==", + "peer": true, + "dependencies": { + "array-buffer-byte-length": "^1.0.0", + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", + "is-array-buffer": "^3.0.2", + "is-shared-array-buffer": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/available-typed-arrays": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.6.tgz", + "integrity": "sha512-j1QzY8iPNPG4o4xmO3ptzpRxTciqD3MgEHtifP/YnJpIo58Xu+ne4BejlbkuaLfXn/nz6HFiw29bLpj2PNMdGg==", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browser-stdout": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", + "dev": true + }, + "node_modules/call-bind": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", + "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", + "peer": true, + "dependencies": { + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.1", + "set-function-length": "^1.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decamelize": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" + }, + "node_modules/define-data-property": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", + "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "peer": true, + "dependencies": { + "get-intrinsic": "^1.2.1", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "peer": true, + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/diff": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-abstract": { + "version": "1.22.3", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.22.3.tgz", + "integrity": "sha512-eiiY8HQeYfYH2Con2berK+To6GrK2RxbPawDkGq4UiCQQfZHb6wX9qQqkbpPqaxQFcl8d9QzZqo0tGE0VcrdwA==", + "peer": true, + "dependencies": { + "array-buffer-byte-length": "^1.0.0", + "arraybuffer.prototype.slice": "^1.0.2", + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.5", + "es-set-tostringtag": "^2.0.1", + "es-to-primitive": "^1.2.1", + "function.prototype.name": "^1.1.6", + "get-intrinsic": "^1.2.2", + "get-symbol-description": "^1.0.0", + "globalthis": "^1.0.3", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0", + "internal-slot": "^1.0.5", + "is-array-buffer": "^3.0.2", + "is-callable": "^1.2.7", + "is-negative-zero": "^2.0.2", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.2", + "is-string": "^1.0.7", + "is-typed-array": "^1.1.12", + "is-weakref": "^1.0.2", + "object-inspect": "^1.13.1", + "object-keys": "^1.1.1", + "object.assign": "^4.1.4", + "regexp.prototype.flags": "^1.5.1", + "safe-array-concat": "^1.0.1", + "safe-regex-test": "^1.0.0", + "string.prototype.trim": "^1.2.8", + "string.prototype.trimend": "^1.0.7", + "string.prototype.trimstart": "^1.0.7", + "typed-array-buffer": "^1.0.0", + "typed-array-byte-length": "^1.0.0", + "typed-array-byte-offset": "^1.0.0", + "typed-array-length": "^1.0.4", + "unbox-primitive": "^1.0.2", + "which-typed-array": "^1.1.13" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.2.tgz", + "integrity": "sha512-BuDyupZt65P9D2D2vA/zqcI3G5xRsklm5N3xCwuiy+/vKy8i0ifdsQP1sLgO4tZDSCaQUSnmC48khknGMV3D2Q==", + "peer": true, + "dependencies": { + "get-intrinsic": "^1.2.2", + "has-tostringtag": "^1.0.0", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-shim-unscopables": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz", + "integrity": "sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==", + "peer": true, + "dependencies": { + "hasown": "^2.0.0" + } + }, + "node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "peer": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.56.0.tgz", + "integrity": "sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==", + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.56.0", + "@humanwhocodes/config-array": "^0.11.13", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-config-prettier": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz", + "integrity": "sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==", + "peer": true, + "bin": { + "eslint-config-prettier": "bin/cli.js" + }, + "peerDependencies": { + "eslint": ">=7.0.0" + } + }, + "node_modules/eslint-plugin-es": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-3.0.1.tgz", + "integrity": "sha512-GUmAsJaN4Fc7Gbtl8uOBlayo2DqhwWvEzykMHSCZHU3XdJ+NSzzZcVhXh3VxX5icqQ+oQdIEawXX8xkR3mIFmQ==", + "peer": true, + "dependencies": { + "eslint-utils": "^2.0.0", + "regexpp": "^3.0.0" + }, + "engines": { + "node": ">=8.10.0" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + }, + "peerDependencies": { + "eslint": ">=4.19.1" + } + }, + "node_modules/eslint-plugin-fetch-options": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/eslint-plugin-fetch-options/-/eslint-plugin-fetch-options-0.0.5.tgz", + "integrity": "sha512-ZMxrccsOAZ7uMQ4nMvPJLqLg6oyLF96YOEwTKWAIbDHpwWUp1raXALZom8ikKucaEnhqWSRuBWI8jBXveFwkJg==", + "peer": true, + "engines": { + "node": ">=0.9.0" + } + }, + "node_modules/eslint-plugin-html": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-html/-/eslint-plugin-html-7.1.0.tgz", + "integrity": "sha512-fNLRraV/e6j8e3XYOC9xgND4j+U7b1Rq+OygMlLcMg+wI/IpVbF+ubQa3R78EjKB9njT6TQOlcK5rFKBVVtdfg==", + "peer": true, + "dependencies": { + "htmlparser2": "^8.0.1" + } + }, + "node_modules/eslint-plugin-json": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-json/-/eslint-plugin-json-3.1.0.tgz", + "integrity": "sha512-MrlG2ynFEHe7wDGwbUuFPsaT2b1uhuEFhJ+W1f1u+1C2EkXmTYJp4B1aAdQQ8M+CC3t//N/oRKiIVw14L2HR1g==", + "peer": true, + "dependencies": { + "lodash": "^4.17.21", + "vscode-json-languageservice": "^4.1.6" + }, + "engines": { + "node": ">=12.0" + } + }, + "node_modules/eslint-plugin-no-unsanitized": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-no-unsanitized/-/eslint-plugin-no-unsanitized-4.0.2.tgz", + "integrity": "sha512-Pry0S9YmHoz8NCEMRQh7N0Yexh2MYCNPIlrV52hTmS7qXnTghWsjXouF08bgsrrZqaW9tt1ZiK3j5NEmPE+EjQ==", + "peer": true, + "peerDependencies": { + "eslint": "^6 || ^7 || ^8" + } + }, + "node_modules/eslint-plugin-node": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-11.1.0.tgz", + "integrity": "sha512-oUwtPJ1W0SKD0Tr+wqu92c5xuCeQqB3hSCHasn/ZgjFdA9iDGNkNf2Zi9ztY7X+hNuMib23LNGRm6+uN+KLE3g==", + "peer": true, + "dependencies": { + "eslint-plugin-es": "^3.0.0", + "eslint-utils": "^2.0.0", + "ignore": "^5.1.1", + "minimatch": "^3.0.4", + "resolve": "^1.10.1", + "semver": "^6.1.0" + }, + "engines": { + "node": ">=8.10.0" + }, + "peerDependencies": { + "eslint": ">=5.16.0" + } + }, + "node_modules/eslint-plugin-react": { + "version": "7.33.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.33.0.tgz", + "integrity": "sha512-qewL/8P34WkY8jAqdQxsiL82pDUeT7nhs8IsuXgfgnsEloKCT4miAV9N9kGtx7/KM9NH/NCGUE7Edt9iGxLXFw==", + "peer": true, + "dependencies": { + "array-includes": "^3.1.6", + "array.prototype.flatmap": "^1.3.1", + "array.prototype.tosorted": "^1.1.1", + "doctrine": "^2.1.0", + "estraverse": "^5.3.0", + "jsx-ast-utils": "^2.4.1 || ^3.0.0", + "minimatch": "^3.1.2", + "object.entries": "^1.1.6", + "object.fromentries": "^2.0.6", + "object.hasown": "^1.1.2", + "object.values": "^1.1.6", + "prop-types": "^15.8.1", + "resolve": "^2.0.0-next.4", + "semver": "^6.3.1", + "string.prototype.matchall": "^4.0.8" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8" + } + }, + "node_modules/eslint-plugin-react/node_modules/doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "peer": true, + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/eslint-plugin-react/node_modules/resolve": { + "version": "2.0.0-next.5", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz", + "integrity": "sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA==", + "peer": true, + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/eslint-plugin-security": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-security/-/eslint-plugin-security-1.4.0.tgz", + "integrity": "sha512-xlS7P2PLMXeqfhyf3NpqbvbnW04kN8M9NtmhpR3XGyOvt/vNKS7XPXT5EDbwKW9vCjWH4PpfQvgD/+JgN0VJKA==", + "peer": true, + "dependencies": { + "safe-regex": "^1.1.0" + } + }, + "node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "peer": true, + "dependencies": { + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + } + }, + "node_modules/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "dependencies": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" + }, + "node_modules/fastq": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.0.tgz", + "integrity": "sha512-zGygtijUMT7jnk3h26kUms3BkSDp4IfIKjmnqI2tvx6nuBfiF1UqOxbnLfzdv+apBy+53oaImsKtMw/xYbW+1w==", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true, + "bin": { + "flat": "cli.js" + } + }, + "node_modules/flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.2.9", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.9.tgz", + "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==" + }, + "node_modules/for-each": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", + "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "peer": true, + "dependencies": { + "is-callable": "^1.1.3" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/function.prototype.name": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.6.tgz", + "integrity": "sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "functions-have-names": "^1.2.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/functions-have-names": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", + "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", + "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", + "peer": true, + "dependencies": { + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/globalthis": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz", + "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==", + "peer": true, + "dependencies": { + "define-properties": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "peer": true, + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==" + }, + "node_modules/has-bigints": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", + "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", + "peer": true, + "dependencies": { + "get-intrinsic": "^1.2.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "peer": true, + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "peer": true, + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "bin": { + "he": "bin/he" + } + }, + "node_modules/htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "node_modules/ignore": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", + "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/internal-slot": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.6.tgz", + "integrity": "sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==", + "peer": true, + "dependencies": { + "get-intrinsic": "^1.2.2", + "hasown": "^2.0.0", + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-array-buffer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", + "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.2.0", + "is-typed-array": "^1.1.10" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "peer": true, + "dependencies": { + "has-bigints": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-core-module": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "peer": true, + "dependencies": { + "hasown": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-date-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "peer": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-negative-zero": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-number-object": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "peer": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-regex": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-shared-array-buffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "peer": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "peer": true, + "dependencies": { + "has-symbols": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-typed-array": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", + "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", + "peer": true, + "dependencies": { + "which-typed-array": "^1.1.11" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-weakref": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", + "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "peer": true + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "peer": true + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==" + }, + "node_modules/jsonc-parser": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.1.tgz", + "integrity": "sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA==", + "peer": true + }, + "node_modules/jsx-ast-utils": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz", + "integrity": "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==", + "peer": true, + "dependencies": { + "array-includes": "^3.1.6", + "array.prototype.flat": "^1.3.1", + "object.assign": "^4.1.4", + "object.values": "^1.1.6" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "peer": true + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + }, + "node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "peer": true, + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mocha": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", + "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", + "dev": true, + "dependencies": { + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.2.0", + "he": "1.2.0", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", + "ms": "2.1.3", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "workerpool": "6.2.1", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" + }, + "bin": { + "_mocha": "bin/_mocha", + "mocha": "bin/mocha.js" + }, + "engines": { + "node": ">= 14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mochajs" + } + }, + "node_modules/mocha/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/mocha/node_modules/minimatch": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mocha/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/mocha/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/nanoid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", + "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.assign": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", + "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.5", + "define-properties": "^1.2.1", + "has-symbols": "^1.0.3", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.entries": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.7.tgz", + "integrity": "sha512-jCBs/0plmPsOnrKAfFQXRG2NFjlhZgjjcBLSmTnEhU8U6vVTsVe8ANeQJCHTl3gSsI4J+0emOoCgoKlmQPMgmA==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.fromentries": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.7.tgz", + "integrity": "sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.hasown": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.3.tgz", + "integrity": "sha512-fFI4VcYpRHvSLXxP7yiZOMAd331cPfd2p7PFDVbgUsYOfCT3tICVqXWngbjr4m49OvsBwUBQ6O2uQoJvy3RexA==", + "peer": true, + "dependencies": { + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.values": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.7.tgz", + "integrity": "sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", + "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", + "dependencies": { + "@aashutoshrathi/word-wrap": "^1.2.3", + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "peer": true + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "peer": true, + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "peer": true + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/regexp.prototype.flags": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", + "integrity": "sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "set-function-name": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/regexpp": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", + "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", + "peer": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "peer": true, + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "engines": { + "node": ">=4" + } + }, + "node_modules/ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "peer": true, + "engines": { + "node": ">=0.12" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safe-array-concat": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.0.tgz", + "integrity": "sha512-ZdQ0Jeb9Ofti4hbt5lX3T2JcAamT9hfzYU1MNB+z/jaEbB6wfFfPIR/zEORmZqobkCCJhSjodobH6WHNmJ97dg==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.5", + "get-intrinsic": "^1.2.2", + "has-symbols": "^1.0.3", + "isarray": "^2.0.5" + }, + "engines": { + "node": ">=0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safe-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha512-aJXcif4xnaNUzvUuC5gcb46oTS7zvg4jpMTnuqtrEPlR3vFr4pxtdTwaF1Qs3Enjn9HK+ZlwQui+a7z0SywIzg==", + "peer": true, + "dependencies": { + "ret": "~0.1.10" + } + }, + "node_modules/safe-regex-test": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.2.tgz", + "integrity": "sha512-83S9w6eFq12BBIJYvjMux6/dkirb8+4zJRA9cxNBVb7Wq5fJBW+Xze48WqR8pxua7bDuAaaAxtVVd4Idjp1dBQ==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.5", + "get-intrinsic": "^1.2.2", + "is-regex": "^1.1.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "peer": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/serialize-javascript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "dev": true, + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/set-function-length": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz", + "integrity": "sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==", + "peer": true, + "dependencies": { + "define-data-property": "^1.1.1", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.2", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/set-function-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz", + "integrity": "sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==", + "peer": true, + "dependencies": { + "define-data-property": "^1.0.1", + "functions-have-names": "^1.2.3", + "has-property-descriptors": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string.prototype.matchall": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.10.tgz", + "integrity": "sha512-rGXbGmOEosIQi6Qva94HUjgPs9vKW+dkG7Y8Q5O2OYkWL6wFaTRZO8zM4mhP94uX55wgyrXzfS2aGtGzUL7EJQ==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.5", + "regexp.prototype.flags": "^1.5.0", + "set-function-name": "^2.0.0", + "side-channel": "^1.0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/string.prototype.trim": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz", + "integrity": "sha512-lfjY4HcixfQXOfaqCvcBuOIapyaroTXhbkfJN3gcB1OtyupngWK4sEET9Knd0cXd28kTUqu/kHoV4HKSJdnjiQ==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/string.prototype.trimend": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.7.tgz", + "integrity": "sha512-Ni79DqeB72ZFq1uH/L6zJ+DKZTkOtPIHovb3YZHQViE+HDouuU4mBrLOLDn5Dde3RF8qw5qVETEjhu9locMLvA==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/string.prototype.trimstart": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.7.tgz", + "integrity": "sha512-NGhtDFu3jCEm7B4Fy0DpLewdJQOZcQ0rGbwQ/+stjnrp2i+rlKeCvos9hOIeCmqwratM47OBxY7uFZzjxHXmrg==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toml-eslint-parser": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/toml-eslint-parser/-/toml-eslint-parser-0.9.3.tgz", + "integrity": "sha512-moYoCvkNUAPCxSW9jmHmRElhm4tVJpHL8ItC/+uYD0EpPSFXbck7yREz9tNdJVTSpHVod8+HoipcpbQ0oE6gsw==", + "dependencies": { + "eslint-visitor-keys": "^3.0.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ota-meshi" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typed-array-buffer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.0.tgz", + "integrity": "sha512-Y8KTSIglk9OZEr8zywiIHG/kmQ7KWyjseXs1CbSo8vC42w7hg2HgYTxSWwP0+is7bWDc1H+Fo026CpHFwm8tkw==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.2.1", + "is-typed-array": "^1.1.10" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/typed-array-byte-length": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.0.tgz", + "integrity": "sha512-Or/+kvLxNpeQ9DtSydonMxCx+9ZXOswtwJn17SNLvhptaXYDJvkFFP5zbfU/uLmvnBJlI4yrnXRxpdWH/M5tNA==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "has-proto": "^1.0.1", + "is-typed-array": "^1.1.10" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/typed-array-byte-offset": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.0.tgz", + "integrity": "sha512-RD97prjEt9EL8YgAgpOkf3O4IF9lhJFr9g0htQkm0rchFp/Vx7LW5Q8fSXXub7BXAODyUQohRMyOc3faCPd0hg==", + "peer": true, + "dependencies": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "has-proto": "^1.0.1", + "is-typed-array": "^1.1.10" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/typed-array-length": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.4.tgz", + "integrity": "sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "is-typed-array": "^1.1.9" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/unbox-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", + "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", + "peer": true, + "dependencies": { + "call-bind": "^1.0.2", + "has-bigints": "^1.0.2", + "has-symbols": "^1.0.3", + "which-boxed-primitive": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/vscode-json-languageservice": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", + "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", + "peer": true, + "dependencies": { + "jsonc-parser": "^3.0.0", + "vscode-languageserver-textdocument": "^1.0.3", + "vscode-languageserver-types": "^3.16.0", + "vscode-nls": "^5.0.0", + "vscode-uri": "^3.0.3" + } + }, + "node_modules/vscode-languageserver-textdocument": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.11.tgz", + "integrity": "sha512-X+8T3GoiwTVlJbicx/sIAF+yuJAqz8VvwJyoMVhwEMoEKE/fkDmrqUgDMyBECcM2A2frVZIUj5HI/ErRXCfOeA==", + "peer": true + }, + "node_modules/vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", + "peer": true + }, + "node_modules/vscode-nls": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.2.0.tgz", + "integrity": "sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==", + "peer": true + }, + "node_modules/vscode-uri": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.8.tgz", + "integrity": "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==", + "peer": true + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "peer": true, + "dependencies": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/which-typed-array": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz", + "integrity": "sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==", + "peer": true, + "dependencies": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.4", + "for-each": "^0.3.3", + "gopd": "^1.0.1", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/workerpool": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", + "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", + "dev": true + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-unparser": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", + "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", + "dev": true, + "dependencies": { + "camelcase": "^6.0.0", + "decamelize": "^4.0.0", + "flat": "^5.0.2", + "is-plain-obj": "^2.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + }, + "dependencies": { + "@aashutoshrathi/word-wrap": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz", + "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==" + }, + "@eslint-community/eslint-utils": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", + "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", + "requires": { + "eslint-visitor-keys": "^3.3.0" + } + }, + "@eslint-community/regexpp": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.0.tgz", + "integrity": "sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA==" + }, + "@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "requires": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + } + }, + "@eslint/js": { + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.56.0.tgz", + "integrity": "sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==" + }, + "@humanwhocodes/config-array": { + "version": "0.11.14", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", + "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "requires": { + "@humanwhocodes/object-schema": "^2.0.2", + "debug": "^4.3.1", + "minimatch": "^3.0.5" + } + }, + "@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==" + }, + "@humanwhocodes/object-schema": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz", + "integrity": "sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==" + }, + "@microsoft/eslint-plugin-sdl": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@microsoft/eslint-plugin-sdl/-/eslint-plugin-sdl-0.2.2.tgz", + "integrity": "sha512-TiBepeQMSxHpvIbKA03TbO9nZqRrKR1th47wGdjY1sH2SSer+JgKlSF3S8GURGA8/zp2T/HwSiAJelclJ3hEvg==", + "peer": true, + "requires": { + "eslint-plugin-node": "11.1.0", + "eslint-plugin-react": "7.33.0", + "eslint-plugin-security": "1.4.0" + } + }, + "@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "requires": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==" + }, + "@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "requires": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + } + }, + "@ungap/structured-clone": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==" + }, + "acorn": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", + "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==" + }, + "acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "requires": {} + }, + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "dev": true + }, + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "array-buffer-byte-length": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz", + "integrity": "sha512-LPuwb2P+NrQw3XhxGc36+XSvuBPopovXYTR9Ew++Du9Yb/bx5AzBfrIsBoj0EZUifjQU+sHL21sseZ3jerWO/A==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "is-array-buffer": "^3.0.1" + } + }, + "array-includes": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.7.tgz", + "integrity": "sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", + "is-string": "^1.0.7" + } + }, + "array.prototype.flat": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz", + "integrity": "sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "es-shim-unscopables": "^1.0.0" + } + }, + "array.prototype.flatmap": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz", + "integrity": "sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "es-shim-unscopables": "^1.0.0" + } + }, + "array.prototype.tosorted": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/array.prototype.tosorted/-/array.prototype.tosorted-1.1.2.tgz", + "integrity": "sha512-HuQCHOlk1Weat5jzStICBCd83NxiIMwqDg/dHEsoefabn/hJRj5pVdWcPUSpRrwhwxZOsQassMpgN/xRYFBMIg==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "es-shim-unscopables": "^1.0.0", + "get-intrinsic": "^1.2.1" + } + }, + "arraybuffer.prototype.slice": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.2.tgz", + "integrity": "sha512-yMBKppFur/fbHu9/6USUe03bZ4knMYiwFBcyiaXB8Go0qNehwX6inYPzK9U0NeQvGxKthcmHcaR8P5MStSRBAw==", + "peer": true, + "requires": { + "array-buffer-byte-length": "^1.0.0", + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", + "is-array-buffer": "^3.0.2", + "is-shared-array-buffer": "^1.0.2" + } + }, + "available-typed-arrays": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.6.tgz", + "integrity": "sha512-j1QzY8iPNPG4o4xmO3ptzpRxTciqD3MgEHtifP/YnJpIo58Xu+ne4BejlbkuaLfXn/nz6HFiw29bLpj2PNMdGg==", + "peer": true + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "browser-stdout": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", + "dev": true + }, + "call-bind": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", + "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", + "peer": true, + "requires": { + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.1", + "set-function-length": "^1.1.1" + } + }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==" + }, + "camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "requires": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "dependencies": { + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + } + } + }, + "cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "decamelize": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", + "integrity": "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==", + "dev": true + }, + "deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" + }, + "define-data-property": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", + "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "peer": true, + "requires": { + "get-intrinsic": "^1.2.1", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0" + } + }, + "define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "peer": true, + "requires": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + } + }, + "diff": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", + "dev": true + }, + "doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "requires": { + "esutils": "^2.0.2" + } + }, + "dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + } + }, + "domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==" + }, + "domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "requires": { + "domelementtype": "^2.3.0" + } + }, + "domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "requires": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + } + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==" + }, + "es-abstract": { + "version": "1.22.3", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.22.3.tgz", + "integrity": "sha512-eiiY8HQeYfYH2Con2berK+To6GrK2RxbPawDkGq4UiCQQfZHb6wX9qQqkbpPqaxQFcl8d9QzZqo0tGE0VcrdwA==", + "peer": true, + "requires": { + "array-buffer-byte-length": "^1.0.0", + "arraybuffer.prototype.slice": "^1.0.2", + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.5", + "es-set-tostringtag": "^2.0.1", + "es-to-primitive": "^1.2.1", + "function.prototype.name": "^1.1.6", + "get-intrinsic": "^1.2.2", + "get-symbol-description": "^1.0.0", + "globalthis": "^1.0.3", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0", + "internal-slot": "^1.0.5", + "is-array-buffer": "^3.0.2", + "is-callable": "^1.2.7", + "is-negative-zero": "^2.0.2", + "is-regex": "^1.1.4", + "is-shared-array-buffer": "^1.0.2", + "is-string": "^1.0.7", + "is-typed-array": "^1.1.12", + "is-weakref": "^1.0.2", + "object-inspect": "^1.13.1", + "object-keys": "^1.1.1", + "object.assign": "^4.1.4", + "regexp.prototype.flags": "^1.5.1", + "safe-array-concat": "^1.0.1", + "safe-regex-test": "^1.0.0", + "string.prototype.trim": "^1.2.8", + "string.prototype.trimend": "^1.0.7", + "string.prototype.trimstart": "^1.0.7", + "typed-array-buffer": "^1.0.0", + "typed-array-byte-length": "^1.0.0", + "typed-array-byte-offset": "^1.0.0", + "typed-array-length": "^1.0.4", + "unbox-primitive": "^1.0.2", + "which-typed-array": "^1.1.13" + } + }, + "es-set-tostringtag": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.2.tgz", + "integrity": "sha512-BuDyupZt65P9D2D2vA/zqcI3G5xRsklm5N3xCwuiy+/vKy8i0ifdsQP1sLgO4tZDSCaQUSnmC48khknGMV3D2Q==", + "peer": true, + "requires": { + "get-intrinsic": "^1.2.2", + "has-tostringtag": "^1.0.0", + "hasown": "^2.0.0" + } + }, + "es-shim-unscopables": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/es-shim-unscopables/-/es-shim-unscopables-1.0.2.tgz", + "integrity": "sha512-J3yBRXCzDu4ULnQwxyToo/OjdMx6akgVC7K6few0a7F/0wLtmKKN7I73AH5T2836UuXRqN7Qg+IIUw/+YJksRw==", + "peer": true, + "requires": { + "hasown": "^2.0.0" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "peer": true, + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true + }, + "escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==" + }, + "eslint": { + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.56.0.tgz", + "integrity": "sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==", + "requires": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.56.0", + "@humanwhocodes/config-array": "^0.11.13", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + } + }, + "eslint-config-prettier": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz", + "integrity": "sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==", + "peer": true, + "requires": {} + }, + "eslint-plugin-es": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-3.0.1.tgz", + "integrity": "sha512-GUmAsJaN4Fc7Gbtl8uOBlayo2DqhwWvEzykMHSCZHU3XdJ+NSzzZcVhXh3VxX5icqQ+oQdIEawXX8xkR3mIFmQ==", + "peer": true, + "requires": { + "eslint-utils": "^2.0.0", + "regexpp": "^3.0.0" + } + }, + "eslint-plugin-fetch-options": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/eslint-plugin-fetch-options/-/eslint-plugin-fetch-options-0.0.5.tgz", + "integrity": "sha512-ZMxrccsOAZ7uMQ4nMvPJLqLg6oyLF96YOEwTKWAIbDHpwWUp1raXALZom8ikKucaEnhqWSRuBWI8jBXveFwkJg==", + "peer": true + }, + "eslint-plugin-html": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-html/-/eslint-plugin-html-7.1.0.tgz", + "integrity": "sha512-fNLRraV/e6j8e3XYOC9xgND4j+U7b1Rq+OygMlLcMg+wI/IpVbF+ubQa3R78EjKB9njT6TQOlcK5rFKBVVtdfg==", + "peer": true, + "requires": { + "htmlparser2": "^8.0.1" + } + }, + "eslint-plugin-json": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-json/-/eslint-plugin-json-3.1.0.tgz", + "integrity": "sha512-MrlG2ynFEHe7wDGwbUuFPsaT2b1uhuEFhJ+W1f1u+1C2EkXmTYJp4B1aAdQQ8M+CC3t//N/oRKiIVw14L2HR1g==", + "peer": true, + "requires": { + "lodash": "^4.17.21", + "vscode-json-languageservice": "^4.1.6" + } + }, + "eslint-plugin-no-unsanitized": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-no-unsanitized/-/eslint-plugin-no-unsanitized-4.0.2.tgz", + "integrity": "sha512-Pry0S9YmHoz8NCEMRQh7N0Yexh2MYCNPIlrV52hTmS7qXnTghWsjXouF08bgsrrZqaW9tt1ZiK3j5NEmPE+EjQ==", + "peer": true, + "requires": {} + }, + "eslint-plugin-node": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-11.1.0.tgz", + "integrity": "sha512-oUwtPJ1W0SKD0Tr+wqu92c5xuCeQqB3hSCHasn/ZgjFdA9iDGNkNf2Zi9ztY7X+hNuMib23LNGRm6+uN+KLE3g==", + "peer": true, + "requires": { + "eslint-plugin-es": "^3.0.0", + "eslint-utils": "^2.0.0", + "ignore": "^5.1.1", + "minimatch": "^3.0.4", + "resolve": "^1.10.1", + "semver": "^6.1.0" + } + }, + "eslint-plugin-react": { + "version": "7.33.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.33.0.tgz", + "integrity": "sha512-qewL/8P34WkY8jAqdQxsiL82pDUeT7nhs8IsuXgfgnsEloKCT4miAV9N9kGtx7/KM9NH/NCGUE7Edt9iGxLXFw==", + "peer": true, + "requires": { + "array-includes": "^3.1.6", + "array.prototype.flatmap": "^1.3.1", + "array.prototype.tosorted": "^1.1.1", + "doctrine": "^2.1.0", + "estraverse": "^5.3.0", + "jsx-ast-utils": "^2.4.1 || ^3.0.0", + "minimatch": "^3.1.2", + "object.entries": "^1.1.6", + "object.fromentries": "^2.0.6", + "object.hasown": "^1.1.2", + "object.values": "^1.1.6", + "prop-types": "^15.8.1", + "resolve": "^2.0.0-next.4", + "semver": "^6.3.1", + "string.prototype.matchall": "^4.0.8" + }, + "dependencies": { + "doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "peer": true, + "requires": { + "esutils": "^2.0.2" + } + }, + "resolve": { + "version": "2.0.0-next.5", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz", + "integrity": "sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA==", + "peer": true, + "requires": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + } + } + }, + "eslint-plugin-security": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-security/-/eslint-plugin-security-1.4.0.tgz", + "integrity": "sha512-xlS7P2PLMXeqfhyf3NpqbvbnW04kN8M9NtmhpR3XGyOvt/vNKS7XPXT5EDbwKW9vCjWH4PpfQvgD/+JgN0VJKA==", + "peer": true, + "requires": { + "safe-regex": "^1.1.0" + } + }, + "eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + } + }, + "eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "peer": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "peer": true + } + } + }, + "eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==" + }, + "espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "requires": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + } + }, + "esquery": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", + "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", + "requires": { + "estraverse": "^5.1.0" + } + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "requires": { + "estraverse": "^5.2.0" + } + }, + "estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" + }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" + }, + "fastq": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.0.tgz", + "integrity": "sha512-zGygtijUMT7jnk3h26kUms3BkSDp4IfIKjmnqI2tvx6nuBfiF1UqOxbnLfzdv+apBy+53oaImsKtMw/xYbW+1w==", + "requires": { + "reusify": "^1.0.4" + } + }, + "file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "requires": { + "flat-cache": "^3.0.4" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "requires": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + } + }, + "flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true + }, + "flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "requires": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + } + }, + "flatted": { + "version": "3.2.9", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.9.tgz", + "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==" + }, + "for-each": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", + "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", + "peer": true, + "requires": { + "is-callable": "^1.1.3" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "optional": true + }, + "function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "peer": true + }, + "function.prototype.name": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.6.tgz", + "integrity": "sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "functions-have-names": "^1.2.3" + } + }, + "functions-have-names": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", + "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "peer": true + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "get-intrinsic": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", + "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", + "peer": true, + "requires": { + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + } + }, + "get-symbol-description": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.0.tgz", + "integrity": "sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + } + }, + "glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "requires": { + "is-glob": "^4.0.3" + } + }, + "globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "requires": { + "type-fest": "^0.20.2" + } + }, + "globalthis": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz", + "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==", + "peer": true, + "requires": { + "define-properties": "^1.1.3" + } + }, + "gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "peer": true, + "requires": { + "get-intrinsic": "^1.1.3" + } + }, + "graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==" + }, + "has-bigints": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", + "peer": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "has-property-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", + "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", + "peer": true, + "requires": { + "get-intrinsic": "^1.2.2" + } + }, + "has-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "peer": true + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "peer": true + }, + "has-tostringtag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", + "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "peer": true, + "requires": { + "has-symbols": "^1.0.2" + } + }, + "hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "peer": true, + "requires": { + "function-bind": "^1.1.2" + } + }, + "he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true + }, + "htmlparser2": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", + "integrity": "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==", + "requires": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "entities": "^4.4.0" + } + }, + "ignore": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", + "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==" + }, + "import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==" + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "internal-slot": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.6.tgz", + "integrity": "sha512-Xj6dv+PsbtwyPpEflsejS+oIZxmMlV44zAhG479uYu89MsjcYOhCFnNyKrkJrihbsiasQyY0afoCl/9BLR65bg==", + "peer": true, + "requires": { + "get-intrinsic": "^1.2.2", + "hasown": "^2.0.0", + "side-channel": "^1.0.4" + } + }, + "is-array-buffer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.2.tgz", + "integrity": "sha512-y+FyyR/w8vfIRq4eQcM1EYgSTnmHXPqaF+IgzgraytCFq5Xh8lllDVmAZolPJiZttZLeFSINPYMaEJ7/vWUa1w==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.2.0", + "is-typed-array": "^1.1.10" + } + }, + "is-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "peer": true, + "requires": { + "has-bigints": "^1.0.1" + } + }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, + "is-boolean-object": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "peer": true + }, + "is-core-module": { + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "peer": true, + "requires": { + "hasown": "^2.0.0" + } + }, + "is-date-object": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", + "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", + "peer": true, + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-negative-zero": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==", + "peer": true + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "is-number-object": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "peer": true, + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==" + }, + "is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "dev": true + }, + "is-regex": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", + "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + } + }, + "is-shared-array-buffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "peer": true, + "requires": { + "call-bind": "^1.0.2" + } + }, + "is-string": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", + "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", + "peer": true, + "requires": { + "has-tostringtag": "^1.0.0" + } + }, + "is-symbol": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", + "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", + "peer": true, + "requires": { + "has-symbols": "^1.0.2" + } + }, + "is-typed-array": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", + "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", + "peer": true, + "requires": { + "which-typed-array": "^1.1.11" + } + }, + "is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "dev": true + }, + "is-weakref": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", + "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", + "peer": true, + "requires": { + "call-bind": "^1.0.2" + } + }, + "isarray": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", + "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", + "peer": true + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "peer": true + }, + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "requires": { + "argparse": "^2.0.1" + } + }, + "json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==" + }, + "jsonc-parser": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.1.tgz", + "integrity": "sha512-AilxAyFOAcK5wA1+LeaySVBrHsGQvUFCDWXKpZjzaL0PqW+xfBOttn8GNtWKFWqneyMZj41MWF9Kl6iPWLwgOA==", + "peer": true + }, + "jsx-ast-utils": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz", + "integrity": "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==", + "peer": true, + "requires": { + "array-includes": "^3.1.6", + "array.prototype.flat": "^1.3.1", + "object.assign": "^4.1.4", + "object.values": "^1.1.6" + } + }, + "keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "requires": { + "json-buffer": "3.0.1" + } + }, + "levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "requires": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + } + }, + "locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "requires": { + "p-locate": "^5.0.0" + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "peer": true + }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + }, + "log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "requires": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + } + }, + "loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "peer": true, + "requires": { + "js-tokens": "^3.0.0 || ^4.0.0" + } + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "mocha": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", + "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", + "dev": true, + "requires": { + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.2.0", + "he": "1.2.0", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", + "ms": "2.1.3", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "workerpool": "6.2.1", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + }, + "minimatch": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "dev": true, + "requires": { + "brace-expansion": "^2.0.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "nanoid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", + "dev": true + }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==" + }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "peer": true + }, + "object-inspect": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", + "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", + "peer": true + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "peer": true + }, + "object.assign": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", + "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", + "peer": true, + "requires": { + "call-bind": "^1.0.5", + "define-properties": "^1.2.1", + "has-symbols": "^1.0.3", + "object-keys": "^1.1.1" + } + }, + "object.entries": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.7.tgz", + "integrity": "sha512-jCBs/0plmPsOnrKAfFQXRG2NFjlhZgjjcBLSmTnEhU8U6vVTsVe8ANeQJCHTl3gSsI4J+0emOoCgoKlmQPMgmA==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + } + }, + "object.fromentries": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.7.tgz", + "integrity": "sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + } + }, + "object.hasown": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.3.tgz", + "integrity": "sha512-fFI4VcYpRHvSLXxP7yiZOMAd331cPfd2p7PFDVbgUsYOfCT3tICVqXWngbjr4m49OvsBwUBQ6O2uQoJvy3RexA==", + "peer": true, + "requires": { + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + } + }, + "object.values": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.7.tgz", + "integrity": "sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" + } + }, + "optionator": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", + "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==", + "requires": { + "@aashutoshrathi/word-wrap": "^1.2.3", + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0" + } + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "requires": { + "p-limit": "^3.0.2" + } + }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "requires": { + "callsites": "^3.0.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "peer": true + }, + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true + }, + "prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==" + }, + "prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "peer": true, + "requires": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==" + }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" + }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "requires": { + "safe-buffer": "^5.1.0" + } + }, + "react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "peer": true + }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "requires": { + "picomatch": "^2.2.1" + } + }, + "regexp.prototype.flags": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", + "integrity": "sha512-sy6TXMN+hnP/wMy+ISxg3krXx7BAtWVO4UouuCN/ziM9UEne0euamVNafDfvC83bRNr95y0V5iijeDQFUNpvrg==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "set-function-name": "^2.0.0" + } + }, + "regexpp": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.2.0.tgz", + "integrity": "sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg==", + "peer": true + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true + }, + "resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "peer": true, + "requires": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==" + }, + "ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "peer": true + }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==" + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "requires": { + "glob": "^7.1.3" + } + }, + "run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "requires": { + "queue-microtask": "^1.2.2" + } + }, + "safe-array-concat": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.0.tgz", + "integrity": "sha512-ZdQ0Jeb9Ofti4hbt5lX3T2JcAamT9hfzYU1MNB+z/jaEbB6wfFfPIR/zEORmZqobkCCJhSjodobH6WHNmJ97dg==", + "peer": true, + "requires": { + "call-bind": "^1.0.5", + "get-intrinsic": "^1.2.2", + "has-symbols": "^1.0.3", + "isarray": "^2.0.5" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "safe-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha512-aJXcif4xnaNUzvUuC5gcb46oTS7zvg4jpMTnuqtrEPlR3vFr4pxtdTwaF1Qs3Enjn9HK+ZlwQui+a7z0SywIzg==", + "peer": true, + "requires": { + "ret": "~0.1.10" + } + }, + "safe-regex-test": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.2.tgz", + "integrity": "sha512-83S9w6eFq12BBIJYvjMux6/dkirb8+4zJRA9cxNBVb7Wq5fJBW+Xze48WqR8pxua7bDuAaaAxtVVd4Idjp1dBQ==", + "peer": true, + "requires": { + "call-bind": "^1.0.5", + "get-intrinsic": "^1.2.2", + "is-regex": "^1.1.4" + } + }, + "semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "peer": true + }, + "serialize-javascript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "dev": true, + "requires": { + "randombytes": "^2.1.0" + } + }, + "set-function-length": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz", + "integrity": "sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==", + "peer": true, + "requires": { + "define-data-property": "^1.1.1", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.2", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.1" + } + }, + "set-function-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz", + "integrity": "sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==", + "peer": true, + "requires": { + "define-data-property": "^1.0.1", + "functions-have-names": "^1.2.3", + "has-property-descriptors": "^1.0.0" + } + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "peer": true, + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "string.prototype.matchall": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.10.tgz", + "integrity": "sha512-rGXbGmOEosIQi6Qva94HUjgPs9vKW+dkG7Y8Q5O2OYkWL6wFaTRZO8zM4mhP94uX55wgyrXzfS2aGtGzUL7EJQ==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.5", + "regexp.prototype.flags": "^1.5.0", + "set-function-name": "^2.0.0", + "side-channel": "^1.0.4" + } + }, + "string.prototype.trim": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.8.tgz", + "integrity": "sha512-lfjY4HcixfQXOfaqCvcBuOIapyaroTXhbkfJN3gcB1OtyupngWK4sEET9Knd0cXd28kTUqu/kHoV4HKSJdnjiQ==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + } + }, + "string.prototype.trimend": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.7.tgz", + "integrity": "sha512-Ni79DqeB72ZFq1uH/L6zJ+DKZTkOtPIHovb3YZHQViE+HDouuU4mBrLOLDn5Dde3RF8qw5qVETEjhu9locMLvA==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + } + }, + "string.prototype.trimstart": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.7.tgz", + "integrity": "sha512-NGhtDFu3jCEm7B4Fy0DpLewdJQOZcQ0rGbwQ/+stjnrp2i+rlKeCvos9hOIeCmqwratM47OBxY7uFZzjxHXmrg==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } + }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "peer": true + }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==" + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + }, + "toml-eslint-parser": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/toml-eslint-parser/-/toml-eslint-parser-0.9.3.tgz", + "integrity": "sha512-moYoCvkNUAPCxSW9jmHmRElhm4tVJpHL8ItC/+uYD0EpPSFXbck7yREz9tNdJVTSpHVod8+HoipcpbQ0oE6gsw==", + "requires": { + "eslint-visitor-keys": "^3.0.0" + } + }, + "type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "requires": { + "prelude-ls": "^1.2.1" + } + }, + "type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==" + }, + "typed-array-buffer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.0.tgz", + "integrity": "sha512-Y8KTSIglk9OZEr8zywiIHG/kmQ7KWyjseXs1CbSo8vC42w7hg2HgYTxSWwP0+is7bWDc1H+Fo026CpHFwm8tkw==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.2.1", + "is-typed-array": "^1.1.10" + } + }, + "typed-array-byte-length": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.0.tgz", + "integrity": "sha512-Or/+kvLxNpeQ9DtSydonMxCx+9ZXOswtwJn17SNLvhptaXYDJvkFFP5zbfU/uLmvnBJlI4yrnXRxpdWH/M5tNA==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "has-proto": "^1.0.1", + "is-typed-array": "^1.1.10" + } + }, + "typed-array-byte-offset": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.0.tgz", + "integrity": "sha512-RD97prjEt9EL8YgAgpOkf3O4IF9lhJFr9g0htQkm0rchFp/Vx7LW5Q8fSXXub7BXAODyUQohRMyOc3faCPd0hg==", + "peer": true, + "requires": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "has-proto": "^1.0.1", + "is-typed-array": "^1.1.10" + } + }, + "typed-array-length": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.4.tgz", + "integrity": "sha512-KjZypGq+I/H7HI5HlOoGHkWUUGq+Q0TPhQurLbyrVrvnKTBgzLhIJ7j6J/XTQOi0d1RjyZ0wdas8bKs2p0x3Ng==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "is-typed-array": "^1.1.9" + } + }, + "unbox-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", + "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", + "peer": true, + "requires": { + "call-bind": "^1.0.2", + "has-bigints": "^1.0.2", + "has-symbols": "^1.0.3", + "which-boxed-primitive": "^1.0.2" + } + }, + "uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "requires": { + "punycode": "^2.1.0" + } + }, + "vscode-json-languageservice": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", + "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", + "peer": true, + "requires": { + "jsonc-parser": "^3.0.0", + "vscode-languageserver-textdocument": "^1.0.3", + "vscode-languageserver-types": "^3.16.0", + "vscode-nls": "^5.0.0", + "vscode-uri": "^3.0.3" + } + }, + "vscode-languageserver-textdocument": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.11.tgz", + "integrity": "sha512-X+8T3GoiwTVlJbicx/sIAF+yuJAqz8VvwJyoMVhwEMoEKE/fkDmrqUgDMyBECcM2A2frVZIUj5HI/ErRXCfOeA==", + "peer": true + }, + "vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", + "peer": true + }, + "vscode-nls": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.2.0.tgz", + "integrity": "sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==", + "peer": true + }, + "vscode-uri": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.8.tgz", + "integrity": "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==", + "peer": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "requires": { + "isexe": "^2.0.0" + } + }, + "which-boxed-primitive": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", + "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", + "peer": true, + "requires": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + } + }, + "which-typed-array": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.13.tgz", + "integrity": "sha512-P5Nra0qjSncduVPEAr7xhoF5guty49ArDTwzJ/yNuPIbZppyRxFQsRCWrocxIY+CnMVG+qfbU2FmDKyvSGClow==", + "peer": true, + "requires": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.4", + "for-each": "^0.3.3", + "gopd": "^1.0.1", + "has-tostringtag": "^1.0.0" + } + }, + "workerpool": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", + "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", + "dev": true + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true + }, + "yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "requires": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + } + }, + "yargs-parser": { + "version": "20.2.4", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", + "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "dev": true + }, + "yargs-unparser": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/yargs-unparser/-/yargs-unparser-2.0.0.tgz", + "integrity": "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==", + "dev": true, + "requires": { + "camelcase": "^6.0.0", + "decamelize": "^4.0.0", + "flat": "^5.0.2", + "is-plain-obj": "^2.1.0" + } + }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" + } + } +} diff --git a/tools/lint/eslint/eslint-plugin-mozilla/package.json b/tools/lint/eslint/eslint-plugin-mozilla/package.json new file mode 100644 index 0000000000..9d4a111dd2 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/package.json @@ -0,0 +1,53 @@ +{ + "name": "eslint-plugin-mozilla", + "version": "3.7.0", + "description": "A collection of rules that help enforce JavaScript coding standard in the Mozilla project.", + "keywords": [ + "eslint", + "eslintplugin", + "eslint-plugin", + "mozilla", + "firefox" + ], + "bugs": { + "url": "https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=Lint" + }, + "homepage": "http://firefox-source-docs.mozilla.org/tools/lint/linters/eslint-plugin-mozilla.html", + "repository": { + "type": "hg", + "url": "https://hg.mozilla.org/mozilla-central/" + }, + "author": "Mike Ratcliffe", + "main": "lib/index.js", + "dependencies": { + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "estraverse": "^5.3.0", + "htmlparser2": "^8.0.1", + "toml-eslint-parser": "0.9.3" + }, + "devDependencies": { + "eslint": "8.56.0", + "mocha": "10.2.0" + }, + "peerDependencies": { + "@microsoft/eslint-plugin-sdl": "^0.2.2", + "eslint": "^7.23.0 || ^8.0.0", + "eslint-config-prettier": "^8.0.0 || ^9.0.0", + "eslint-plugin-fetch-options": "^0.0.5", + "eslint-plugin-html": "^7.0.0", + "eslint-plugin-json": "^3.1.0", + "eslint-plugin-no-unsanitized": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "scripts": { + "prepack": "node scripts/createExports.js", + "test": "mocha --reporter 'reporters/mozilla-format.js' tests", + "postpublish": "rm -f lib/environments/saved-globals.json lib/rules/saved-rules-data.json", + "update-tooltool": "./update.sh" + }, + "license": "MPL-2.0" +} diff --git a/tools/lint/eslint/eslint-plugin-mozilla/reporters/mozilla-format.js b/tools/lint/eslint/eslint-plugin-mozilla/reporters/mozilla-format.js new file mode 100644 index 0000000000..a3f96e7bef --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/reporters/mozilla-format.js @@ -0,0 +1,57 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * This file outputs the format that treeherder requires. If we integrate + * these tests with ./mach, then we may replace this with a json handler within + * mach itself. + */ + +"use strict"; + +var mocha = require("mocha"); +var path = require("path"); +module.exports = MozillaFormatter; + +function MozillaFormatter(runner) { + mocha.reporters.Base.call(this, runner); + var passes = 0; + var failures = []; + + runner.on("start", () => { + console.log("SUITE-START | eslint-plugin-mozilla"); + }); + + runner.on("pass", function (test) { + passes++; + let title = test.title.replace(/\n/g, "|"); + console.log(`TEST-PASS | ${path.basename(test.file)} | ${title}`); + }); + + runner.on("fail", function (test, err) { + failures.push(test); + // Replace any newlines in the title. + let title = test.title.replace(/\n/g, "|"); + console.log( + `TEST-UNEXPECTED-FAIL | ${path.basename(test.file)} | ${title} | ${ + err.message + }` + ); + mocha.reporters.Base.list([test]); + }); + + runner.on("end", function () { + // Space the results out visually with an additional blank line. + console.log(""); + console.log("INFO | Result summary:"); + console.log(`INFO | Passed: ${passes}`); + console.log(`INFO | Failed: ${failures.length}`); + console.log("SUITE-END"); + // Space the failures out visually with an additional blank line. + console.log(""); + console.log("Failure summary:"); + mocha.reporters.Base.list(failures); + process.exit(failures.length); + }); +} diff --git a/tools/lint/eslint/eslint-plugin-mozilla/scripts/createExports.js b/tools/lint/eslint/eslint-plugin-mozilla/scripts/createExports.js new file mode 100644 index 0000000000..7f839edfba --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/scripts/createExports.js @@ -0,0 +1,77 @@ +/** + * @fileoverview A script to export the known globals to a file. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +"use strict"; + +var fs = require("fs"); +var path = require("path"); +var helpers = require("../lib/helpers"); + +const eslintDir = path.join(helpers.rootDir, "tools", "lint", "eslint"); + +const globalsFile = path.join( + eslintDir, + "eslint-plugin-mozilla", + "lib", + "environments", + "saved-globals.json" +); +const rulesFile = path.join( + eslintDir, + "eslint-plugin-mozilla", + "lib", + "rules", + "saved-rules-data.json" +); + +console.log("Copying services.json"); + +const env = helpers.getBuildEnvironment(); + +const servicesFile = path.join( + env.topobjdir, + "xpcom", + "components", + "services.json" +); +const shipServicesFile = path.join( + eslintDir, + "eslint-plugin-mozilla", + "lib", + "services.json" +); + +fs.writeFileSync(shipServicesFile, fs.readFileSync(servicesFile)); + +console.log("Generating globals file"); + +// Export the environments. +let environmentGlobals = require("../lib/index.js").environments; + +return fs.writeFile( + globalsFile, + JSON.stringify({ environments: environmentGlobals }), + err => { + if (err) { + console.error(err); + process.exit(1); + } + + console.log("Globals file generation complete"); + + console.log("Creating rules data file"); + let rulesData = {}; + + return fs.writeFile(rulesFile, JSON.stringify({ rulesData }), err1 => { + if (err1) { + console.error(err1); + process.exit(1); + } + + console.log("Globals file generation complete"); + }); + } +); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/avoid-Date-timing.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/avoid-Date-timing.js new file mode 100644 index 0000000000..37fc424617 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/avoid-Date-timing.js @@ -0,0 +1,34 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/avoid-Date-timing"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, type, message) { + return { code, errors: [{ messageId: "usePerfNow", type }] }; +} + +ruleTester.run("avoid-Date-timing", rule, { + valid: [ + "new Date('2017-07-11');", + "new Date(1499790192440);", + "new Date(2017, 7, 11);", + "Date.UTC(2017, 7);", + ], + invalid: [ + invalidCode("Date.now();", "CallExpression"), + invalidCode("new Date();", "NewExpression"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/avoid-removeChild.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/avoid-removeChild.js new file mode 100644 index 0000000000..3736f25853 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/avoid-removeChild.js @@ -0,0 +1,37 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/avoid-removeChild"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, messageId = "useRemove") { + return { code, errors: [{ messageId, type: "CallExpression" }] }; +} + +ruleTester.run("avoid-removeChild", rule, { + valid: [ + "elt.remove();", + "elt.parentNode.parentNode.removeChild(elt2.parentNode);", + "elt.parentNode.removeChild(elt2);", + "elt.removeChild(elt2);", + ], + invalid: [ + invalidCode("elt.parentNode.removeChild(elt);"), + invalidCode("elt.parentNode.parentNode.removeChild(elt.parentNode);"), + invalidCode("$(e).parentNode.removeChild($(e));"), + invalidCode("$('e').parentNode.removeChild($('e'));"), + invalidCode("elt.removeChild(elt.firstChild);", "useFirstChildRemove"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/balanced-listeners.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/balanced-listeners.js new file mode 100644 index 0000000000..b7cd4e2c2f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/balanced-listeners.js @@ -0,0 +1,100 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/balanced-listeners"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function error(code, functionName, type) { + return { + code, + errors: [ + { + messageId: "noCorresponding", + type: "Identifier", + data: { functionName, type }, + }, + ], + }; +} + +ruleTester.run("balanced-listeners", rule, { + valid: [ + "elt.addEventListener('event', handler);" + + "elt.removeEventListener('event', handler);", + + "elt.addEventListener('event', handler, true);" + + "elt.removeEventListener('event', handler, true);", + + "elt.addEventListener('event', handler, false);" + + "elt.removeEventListener('event', handler, false);", + + "elt.addEventListener('event', handler);" + + "elt.removeEventListener('event', handler, false);", + + "elt.addEventListener('event', handler, false);" + + "elt.removeEventListener('event', handler);", + + "elt.addEventListener('event', handler, {capture: false});" + + "elt.removeEventListener('event', handler);", + + "elt.addEventListener('event', handler);" + + "elt.removeEventListener('event', handler, {capture: false});", + + "elt.addEventListener('event', handler, {capture: true});" + + "elt.removeEventListener('event', handler, true);", + + "elt.addEventListener('event', handler, true);" + + "elt.removeEventListener('event', handler, {capture: true});", + + "elt.addEventListener('event', handler, {once: true});", + + "elt.addEventListener('event', handler, {once: true, capture: true});", + ], + invalid: [ + error( + "elt.addEventListener('click', handler, false);", + "removeEventListener", + "click" + ), + + error( + "elt.addEventListener('click', handler, false);" + + "elt.removeEventListener('click', handler, true);", + "removeEventListener", + "click" + ), + + error( + "elt.addEventListener('click', handler, {capture: false});" + + "elt.removeEventListener('click', handler, true);", + "removeEventListener", + "click" + ), + + error( + "elt.addEventListener('click', handler, {capture: true});" + + "elt.removeEventListener('click', handler);", + "removeEventListener", + "click" + ), + + error( + "elt.addEventListener('click', handler, true);" + + "elt.removeEventListener('click', handler);", + "removeEventListener", + "click" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/balanced-observers.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/balanced-observers.js new file mode 100644 index 0000000000..97e578a1f4 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/balanced-observers.js @@ -0,0 +1,80 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/balanced-observers"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function error(code, observable) { + return { + code, + errors: [ + { + messageId: "noCorresponding", + type: "Identifier", + data: { observable }, + }, + ], + }; +} + +ruleTester.run("balanced-observers", rule, { + valid: [ + "Services.obs.addObserver(observer, 'observable');" + + "Services.obs.removeObserver(observer, 'observable');", + "Services.prefs.addObserver('preference.name', otherObserver);" + + "Services.prefs.removeObserver('preference.name', otherObserver);", + ], + invalid: [ + error( + // missing Services.obs.removeObserver + "Services.obs.addObserver(observer, 'observable');", + "observable" + ), + + error( + // wrong observable name for Services.obs.removeObserver + "Services.obs.addObserver(observer, 'observable');" + + "Services.obs.removeObserver(observer, 'different-observable');", + "observable" + ), + + error( + // missing Services.prefs.removeObserver + "Services.prefs.addObserver('preference.name', otherObserver);", + "preference.name" + ), + + error( + // wrong observable name for Services.prefs.removeObserver + "Services.prefs.addObserver('preference.name', otherObserver);" + + "Services.prefs.removeObserver('other.preference', otherObserver);", + "preference.name" + ), + + error( + // mismatch Services.prefs vs Services.obs + "Services.obs.addObserver(observer, 'observable');" + + "Services.prefs.removeObserver(observer, 'observable');", + "observable" + ), + + error( + "Services.prefs.addObserver('preference.name', otherObserver);" + + // mismatch Services.prefs vs Services.obs + "Services.obs.removeObserver('preference.name', otherObserver);", + "preference.name" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/consistent-if-bracing.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/consistent-if-bracing.js new file mode 100644 index 0000000000..9712fd9a78 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/consistent-if-bracing.js @@ -0,0 +1,39 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/consistent-if-bracing"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code) { + return { code, errors: [{ messageId: "consistentIfBracing" }] }; +} + +ruleTester.run("consistent-if-bracing", rule, { + valid: [ + "if (true) {1} else {0}", + "if (false) 1; else 0", + "if (true) {1} else if (true) {2} else {0}", + "if (true) {1} else if (true) {2} else if (true) {3} else {0}", + ], + invalid: [ + invalidCode(`if (true) {1} else 0`), + invalidCode("if (true) 1; else {0}"), + invalidCode("if (true) {1} else if (true) 2; else {0}"), + invalidCode("if (true) 1; else if (true) {2} else {0}"), + invalidCode("if (true) {1} else if (true) {2} else 0"), + invalidCode("if (true) {1} else if (true) {2} else if (true) 3; else {0}"), + invalidCode("if (true) {if (true) 1; else {0}} else {0}"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/globals.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/globals.js new file mode 100644 index 0000000000..1098cb8a34 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/globals.js @@ -0,0 +1,161 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +var { getGlobalsForCode } = require("../lib/globals"); +var assert = require("assert"); + +/* global describe, it */ + +describe("globals", function () { + it("should reflect top-level this property assignment", function () { + const globals = getGlobalsForCode(` +this.foo = 10; +`); + assert.deepEqual(globals, [{ name: "foo", writable: true }]); + }); + + it("should reflect this property assignment inside block", function () { + const globals = getGlobalsForCode(` +{ + this.foo = 10; +} +`); + assert.deepEqual(globals, [{ name: "foo", writable: true }]); + }); + + it("should ignore this property assignment inside function declaration", function () { + const globals = getGlobalsForCode(` +function f() { + this.foo = 10; +} +`); + assert.deepEqual(globals, [{ name: "f", writable: true }]); + }); + + it("should ignore this property assignment inside function expression", function () { + const globals = getGlobalsForCode(` +(function f() { + this.foo = 10; +}); +`); + assert.deepEqual(globals, []); + }); + + it("should ignore this property assignment inside method", function () { + const globals = getGlobalsForCode(` +({ + method() { + this.foo = 10; + } +}); +`); + assert.deepEqual(globals, []); + }); + + it("should ignore this property assignment inside accessor", function () { + const globals = getGlobalsForCode(` +({ + get x() { + this.foo = 10; + }, + set x(v) { + this.bar = 10; + } +}); +`); + assert.deepEqual(globals, []); + }); + + it("should reflect this property assignment inside arrow function", function () { + const globals = getGlobalsForCode(` +() => { + this.foo = 10; +}; +`); + assert.deepEqual(globals, [{ name: "foo", writable: true }]); + }); + + it("should ignore this property assignment inside arrow function inside function expression", function () { + const globals = getGlobalsForCode(` +(function f() { + () => { + this.foo = 10; + }; +}); +`); + assert.deepEqual(globals, []); + }); + + it("should ignore this property assignment inside class static", function () { + const globals = getGlobalsForCode(` +class A { + static { + this.foo = 10; + (() => { + this.bar = 10; + })(); + } +} +`); + assert.deepEqual(globals, [{ name: "A", writable: true }]); + }); + + it("should ignore this property assignment inside class property", function () { + const globals = getGlobalsForCode(` +class A { + a = this.foo = 10; + b = (() => { + this.bar = 10; + })(); +} +`); + assert.deepEqual(globals, [{ name: "A", writable: true }]); + }); + + it("should ignore this property assignment inside class static property", function () { + const globals = getGlobalsForCode(` +class A { + static a = this.foo = 10; + static b = (() => { + this.bar = 10; + })(); +} +`); + assert.deepEqual(globals, [{ name: "A", writable: true }]); + }); + + it("should ignore this property assignment inside class private property", function () { + const globals = getGlobalsForCode(` +class A { + #a = this.foo = 10; + #b = (() => { + this.bar = 10; + })(); +} +`); + assert.deepEqual(globals, [{ name: "A", writable: true }]); + }); + + it("should ignore this property assignment inside class static private property", function () { + const globals = getGlobalsForCode(` +class A { + static #a = this.foo = 10; + static #b = (() => { + this.bar = 10; + })(); +} +`); + assert.deepEqual(globals, [{ name: "A", writable: true }]); + }); + + it("should reflect lazy getters", function () { + const globals = getGlobalsForCode(` +ChromeUtils.defineESModuleGetters(this, { + A: "B", +}); +`); + assert.deepEqual(globals, [{ name: "A", writable: true, explicit: true }]); + }); +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/lazy-getter-object-name.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/lazy-getter-object-name.js new file mode 100644 index 0000000000..33a2a6c130 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/lazy-getter-object-name.js @@ -0,0 +1,51 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/lazy-getter-object-name"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code) { + return { + code, + errors: [{ messageId: "mustUseLazy", type: "CallExpression" }], + }; +} + +ruleTester.run("lazy-getter-object-name", rule, { + valid: [ + ` + ChromeUtils.defineESModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); +`, + ], + invalid: [ + invalidCode(` + ChromeUtils.defineESModuleGetters(obj, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); +`), + invalidCode(` + ChromeUtils.defineESModuleGetters(this, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); +`), + invalidCode(` + ChromeUtils.defineESModuleGetters(window, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); +`), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/mark-exported-symbols-as-used.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/mark-exported-symbols-as-used.js new file mode 100644 index 0000000000..e738825fab --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/mark-exported-symbols-as-used.js @@ -0,0 +1,45 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/mark-exported-symbols-as-used"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, type, messageId) { + return { code, errors: [{ messageId, type }] }; +} + +ruleTester.run("mark-exported-symbols-as-used", rule, { + valid: [ + "var EXPORTED_SYMBOLS = ['foo'];", + "this.EXPORTED_SYMBOLS = ['foo'];", + ], + invalid: [ + invalidCode( + "let EXPORTED_SYMBOLS = ['foo'];", + "VariableDeclaration", + "useLetForExported" + ), + invalidCode( + "var EXPORTED_SYMBOLS = 'foo';", + "VariableDeclaration", + "nonArrayAssignedToImported" + ), + invalidCode( + "this.EXPORTED_SYMBOLS = 'foo';", + "AssignmentExpression", + "nonArrayAssignedToImported" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-addtask-setup.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-addtask-setup.js new file mode 100644 index 0000000000..cd036d7d91 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-addtask-setup.js @@ -0,0 +1,69 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-addtask-setup"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function callError() { + return [{ messageId: "useAddSetup", type: "CallExpression" }]; +} + +ruleTester.run("no-addtask-setup", rule, { + valid: [ + "add_task(function() {});", + "add_task(function () {});", + "add_task(function foo() {});", + "add_task(async function() {});", + "add_task(async function () {});", + "add_task(async function foo() {});", + "something(function setup() {});", + "something(async function setup() {});", + "add_task(setup);", + "add_task(setup => {});", + "add_task(async setup => {});", + ], + invalid: [ + { + code: "add_task(function setup() {});", + output: "add_setup(function() {});", + errors: callError(), + }, + { + code: "add_task(function setup () {});", + output: "add_setup(function () {});", + errors: callError(), + }, + { + code: "add_task(async function setup() {});", + output: "add_setup(async function() {});", + errors: callError(), + }, + { + code: "add_task(async function setup () {});", + output: "add_setup(async function () {});", + errors: callError(), + }, + { + code: "add_task(async function setUp() {});", + output: "add_setup(async function() {});", + errors: callError(), + }, + { + code: "add_task(async function init() {});", + output: "add_setup(async function() {});", + errors: callError(), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-arbitrary-setTimeout.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-arbitrary-setTimeout.js new file mode 100644 index 0000000000..ea194af6e3 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-arbitrary-setTimeout.js @@ -0,0 +1,39 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-arbitrary-setTimeout"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function wrapCode(code, filename = "xpcshell/test_foo.js") { + return { code, filename }; +} + +function invalidCode(code) { + let obj = wrapCode(code); + obj.errors = [{ messageId: "listenForEvents", type: "CallExpression" }]; + return obj; +} + +ruleTester.run("no-arbitrary-setTimeout", rule, { + valid: [ + wrapCode("setTimeout(function() {}, 0);"), + wrapCode("setTimeout(function() {});"), + wrapCode("setTimeout(function() {}, 10);", "test_foo.js"), + ], + invalid: [ + invalidCode("setTimeout(function() {}, 10);"), + invalidCode("setTimeout(function() {}, timeout);"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-compare-against-boolean-literals.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-compare-against-boolean-literals.js new file mode 100644 index 0000000000..983ff0583e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-compare-against-boolean-literals.js @@ -0,0 +1,59 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-compare-against-boolean-literals"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function callError() { + return [{ messageId: "noCompareBoolean", type: "BinaryExpression" }]; +} + +ruleTester.run("no-compare-against-boolean-literals", rule, { + valid: [`if (!foo) {}`, `if (!!foo) {}`], + invalid: [ + { + code: `if (foo == true) {}`, + errors: callError(), + }, + { + code: `if (foo != true) {}`, + errors: callError(), + }, + { + code: `if (foo == false) {}`, + errors: callError(), + }, + { + code: `if (foo != false) {}`, + errors: callError(), + }, + { + code: `if (true == foo) {}`, + errors: callError(), + }, + { + code: `if (true != foo) {}`, + errors: callError(), + }, + { + code: `if (false == foo) {}`, + errors: callError(), + }, + { + code: `if (false != foo) {}`, + errors: callError(), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-comparison-or-assignment-inside-ok.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-comparison-or-assignment-inside-ok.js new file mode 100644 index 0000000000..3c43e5879f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-comparison-or-assignment-inside-ok.js @@ -0,0 +1,139 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-comparison-or-assignment-inside-ok"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, output, messageId, options = []) { + let rv = { + code, + errors: [{ messageId }], + }; + if (output) { + rv.output = output; + } + return rv; +} + +ruleTester.run("no-comparison-or-assignment-inside-ok", rule, { + valid: [ + "ok(foo)", + "ok(!bar)", + "ok(!foo, 'Message')", + "ok(bar, 'Message')", + "ok(!!foo, 'Message')", + ], + invalid: [ + // Assignment + invalidCode("ok(foo = bar)", null, "assignment"), + invalidCode("ok(foo = bar, 'msg')", null, "assignment"), + + // Comparisons: + invalidCode("ok(foo == bar)", "Assert.equal(foo, bar)", "comparison", { + operator: "==", + }), + invalidCode("ok(foo != bar)", "Assert.notEqual(foo, bar)", "comparison", { + operator: "!=", + }), + invalidCode("ok(foo < bar)", "Assert.less(foo, bar)", "comparison", { + operator: "<", + }), + invalidCode("ok(foo > bar)", "Assert.greater(foo, bar)", "comparison", { + operator: ">", + }), + invalidCode( + "ok(foo <= bar)", + "Assert.lessOrEqual(foo, bar)", + "comparison", + { operator: "<=" } + ), + invalidCode( + "ok(foo >= bar)", + "Assert.greaterOrEqual(foo, bar)", + "comparison", + { operator: ">=" } + ), + invalidCode( + "ok(foo === bar)", + "Assert.strictEqual(foo, bar)", + "comparison", + { operator: "===" } + ), + invalidCode( + "ok(foo !== bar)", + "Assert.notStrictEqual(foo, bar)", + "comparison", + { operator: "!==" } + ), + + // Comparisons with messages: + invalidCode( + "ok(foo == bar, 'hi')", + "Assert.equal(foo, bar, 'hi')", + "comparison", + { operator: "==" } + ), + invalidCode( + "ok(foo != bar, 'hi')", + "Assert.notEqual(foo, bar, 'hi')", + "comparison", + { operator: "!=" } + ), + invalidCode( + "ok(foo < bar, 'hi')", + "Assert.less(foo, bar, 'hi')", + "comparison", + { operator: "<" } + ), + invalidCode( + "ok(foo > bar, 'hi')", + "Assert.greater(foo, bar, 'hi')", + "comparison", + { operator: ">" } + ), + invalidCode( + "ok(foo <= bar, 'hi')", + "Assert.lessOrEqual(foo, bar, 'hi')", + "comparison", + { operator: "<=" } + ), + invalidCode( + "ok(foo >= bar, 'hi')", + "Assert.greaterOrEqual(foo, bar, 'hi')", + "comparison", + { operator: ">=" } + ), + invalidCode( + "ok(foo === bar, 'hi')", + "Assert.strictEqual(foo, bar, 'hi')", + "comparison", + { operator: "===" } + ), + invalidCode( + "ok(foo !== bar, 'hi')", + "Assert.notStrictEqual(foo, bar, 'hi')", + "comparison", + { operator: "!==" } + ), + + // Confusing bits that break fixup: + invalidCode( + "async () => ok((await foo) === bar, 'Oh no')", + "async () => Assert.strictEqual(await foo, bar, 'Oh no')", + "comparison", + { operator: "===" } + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-cu-reportError.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-cu-reportError.js new file mode 100644 index 0000000000..6650c8bf4e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-cu-reportError.js @@ -0,0 +1,100 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-cu-reportError"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function callError() { + return [{ messageId: "useConsoleError", type: "CallExpression" }]; +} + +ruleTester.run("no-cu-reportError", rule, { + valid: [ + "console.error('foo')", + "Cu.cloneInto({}, {})", + "foo().catch(console.error)", + // TODO: Bug 1802347 - this should be treated as an error as well. + "Cu.reportError('foo', stack)", + ], + invalid: [ + { + code: "Cu.reportError('foo')", + output: "console.error('foo')", + errors: callError(), + }, + { + code: "Cu.reportError(bar)", + output: "console.error(bar)", + errors: callError(), + }, + { + code: "Cu.reportError(bar.stack)", + output: "console.error(bar.stack)", + errors: callError(), + }, + { + code: "foo().catch(Cu.reportError)", + output: "foo().catch(console.error)", + errors: callError(), + }, + { + code: "foo().then(bar, Cu.reportError)", + output: "foo().then(bar, console.error)", + errors: callError(), + }, + // When referencing identifiers/members, try to reference them rather + // than stringifying: + { + code: "Cu.reportError('foo' + e)", + output: "console.error('foo', e)", + errors: callError(), + }, + { + code: "Cu.reportError('foo' + msg.data)", + output: "console.error('foo', msg.data)", + errors: callError(), + }, + // Don't touch existing concatenation of literals (usually done for + // wrapping reasons) + { + code: "Cu.reportError('foo' + 'bar' + 'baz')", + output: "console.error('foo' + 'bar' + 'baz')", + errors: callError(), + }, + // Ensure things work when nested: + { + code: "Cu.reportError('foo' + e + 'baz')", + output: "console.error('foo', e, 'baz')", + errors: callError(), + }, + // Ensure things work when nested in different places: + { + code: "Cu.reportError('foo' + e + 'quux' + 'baz')", + output: "console.error('foo', e, 'quux' + 'baz')", + errors: callError(), + }, + { + code: "Cu.reportError('foo' + 'quux' + e + 'baz')", + output: "console.error('foo' + 'quux', e, 'baz')", + errors: callError(), + }, + // Ensure things work with parens changing order of operations: + { + code: "Cu.reportError('foo' + 'quux' + (e + 'baz'))", + output: "console.error('foo' + 'quux' + (e + 'baz'))", + errors: callError(), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-define-cc-etc.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-define-cc-etc.js new file mode 100644 index 0000000000..ed71f6087e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-define-cc-etc.js @@ -0,0 +1,63 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-define-cc-etc"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, varNames) { + if (!Array.isArray(varNames)) { + varNames = [varNames]; + } + return { + code, + errors: varNames.map(name => { + return { + messageId: "noSeparateDefinition", + data: { name }, + type: "VariableDeclarator", + }; + }), + }; +} + +ruleTester.run("no-define-cc-etc", rule, { + valid: [ + "var Cm = Components.manager;", + "const CC = Components.Constructor;", + "var {Constructor: CC, manager: Cm} = Components;", + "const {Constructor: CC, manager: Cm} = Components;", + "foo.Cc.test();", + "const {bar, ...foo} = obj;", + ], + invalid: [ + invalidCode("var Cc;", "Cc"), + invalidCode("let Cc;", "Cc"), + invalidCode("let Ci;", "Ci"), + invalidCode("let Cr;", "Cr"), + invalidCode("let Cu;", "Cu"), + invalidCode("var Cc = Components.classes;", "Cc"), + invalidCode("const {classes: Cc} = Components;", "Cc"), + invalidCode("let {classes: Cc, manager: Cm} = Components", "Cc"), + invalidCode("const Cu = Components.utils;", "Cu"), + invalidCode("var Ci = Components.interfaces, Cc = Components.classes;", [ + "Ci", + "Cc", + ]), + invalidCode("var {'interfaces': Ci, 'classes': Cc} = Components;", [ + "Ci", + "Cc", + ]), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-redeclare-with-import-autofix.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-redeclare-with-import-autofix.js new file mode 100644 index 0000000000..36842000d5 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-redeclare-with-import-autofix.js @@ -0,0 +1,77 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-redeclare-with-import-autofix"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, output, messageId, options = []) { + let rv = { + code, + errors: [{ messageId }], + options, + }; + if (output) { + rv.output = output; + } + return rv; +} + +ruleTester.run("no-redeclare-with-import-autofix", rule, { + valid: [ + 'const foo = "whatever";', + 'let foo = "whatever";', + 'const {foo} = {foo: "whatever"};', + 'const {foo} = ChromeUtils.import("foo.jsm")', + 'let {foo} = ChromeUtils.import("foo.jsm")', + 'const {foo} = ChromeUtils.importESModule("foo.sys.mjs")', + 'let {foo} = ChromeUtils.importESModule("foo.sys.mjs")', + ], + invalid: [ + invalidCode( + `var {foo} = ChromeUtils.importESModule("foo.sys.mjs"); +var {foo} = ChromeUtils.importESModule("foo.sys.mjs");`, + 'var {foo} = ChromeUtils.importESModule("foo.sys.mjs");\n', + "duplicateImport" + ), + invalidCode( + `var {foo} = ChromeUtils.import("foo.jsm"); +var {foo} = ChromeUtils.import("foo.jsm");`, + 'var {foo} = ChromeUtils.import("foo.jsm");\n', + "duplicateImport" + ), + + invalidCode( + `var {foo} = ChromeUtils.import("foo.jsm"); +var {foo, bar} = ChromeUtils.import("foo.jsm");`, + `var {foo} = ChromeUtils.import("foo.jsm"); +var { bar} = ChromeUtils.import("foo.jsm");`, + "duplicateImport" + ), + + invalidCode( + `var {foo} = ChromeUtils.import("foo.jsm"); +var {bar, foo} = ChromeUtils.import("foo.jsm");`, + `var {foo} = ChromeUtils.import("foo.jsm"); +var {bar} = ChromeUtils.import("foo.jsm");`, + "duplicateImport" + ), + + invalidCode(`var foo = 5; var foo = 10;`, "", "redeclared", [ + { + errorForNonImports: true, + }, + ]), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-throw-cr-literal.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-throw-cr-literal.js new file mode 100644 index 0000000000..aecc5cd971 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-throw-cr-literal.js @@ -0,0 +1,62 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-throw-cr-literal"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, output, messageId) { + return { + code, + output, + errors: [{ messageId, type: "ThrowStatement" }], + }; +} + +ruleTester.run("no-throw-cr-literal", rule, { + valid: [ + 'throw Components.Exception("", Cr.NS_ERROR_NOT_IMPLEMENTED);', + 'throw Components.Exception("", Components.results.NS_ERROR_UNEXPECTED);', + 'function t() { throw Components.Exception("", Cr.NS_ERROR_NO_CONTENT); }', + // We don't handle combined values, regular no-throw-literal catches them + 'throw Components.results.NS_ERROR_UNEXPECTED + "whoops";', + ], + invalid: [ + invalidCode( + "throw Cr.NS_ERROR_NO_INTERFACE;", + 'throw Components.Exception("", Cr.NS_ERROR_NO_INTERFACE);', + "bareCR" + ), + invalidCode( + "throw Components.results.NS_ERROR_ABORT;", + 'throw Components.Exception("", Components.results.NS_ERROR_ABORT);', + "bareComponentsResults" + ), + invalidCode( + "function t() { throw Cr.NS_ERROR_NULL_POINTER; }", + 'function t() { throw Components.Exception("", Cr.NS_ERROR_NULL_POINTER); }', + "bareCR" + ), + invalidCode( + "throw new Error(Cr.NS_ERROR_ABORT);", + 'throw Components.Exception("", Cr.NS_ERROR_ABORT);', + "newErrorCR" + ), + invalidCode( + "throw new Error(Components.results.NS_ERROR_ABORT);", + 'throw Components.Exception("", Components.results.NS_ERROR_ABORT);', + "newErrorComponentsResults" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-parameters.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-parameters.js new file mode 100644 index 0000000000..28d4d0d151 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-parameters.js @@ -0,0 +1,175 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-useless-parameters"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function callError(messageId, data = {}) { + return [{ messageId, data, type: "CallExpression" }]; +} + +ruleTester.run("no-useless-parameters", rule, { + valid: [ + "Services.prefs.clearUserPref('browser.search.custom');", + "Services.removeObserver('notification name', {});", + "Services.io.newURI('http://example.com');", + "Services.io.newURI('http://example.com', 'utf8');", + "elt.addEventListener('click', handler);", + "elt.addEventListener('click', handler, true);", + "elt.addEventListener('click', handler, {once: true});", + "elt.removeEventListener('click', handler);", + "elt.removeEventListener('click', handler, true);", + "Services.obs.addObserver(this, 'topic', true);", + "Services.obs.addObserver(this, 'topic');", + "Services.prefs.addObserver('branch', this, true);", + "Services.prefs.addObserver('branch', this);", + "array.appendElement(elt);", + "Services.obs.notifyObservers(obj, 'topic', 'data');", + "Services.obs.notifyObservers(obj, 'topic');", + "window.getComputedStyle(elt);", + "window.getComputedStyle(elt, ':before');", + ], + invalid: [ + { + code: "Services.prefs.clearUserPref('browser.search.custom', false);", + output: "Services.prefs.clearUserPref('browser.search.custom');", + errors: callError("onlyTakes", { + fnName: "clearUserPref", + params: "1 parameter", + }), + }, + { + code: "Services.prefs.clearUserPref('browser.search.custom',\n false);", + output: "Services.prefs.clearUserPref('browser.search.custom');", + errors: callError("onlyTakes", { + fnName: "clearUserPref", + params: "1 parameter", + }), + }, + { + code: "Services.removeObserver('notification name', {}, false);", + output: "Services.removeObserver('notification name', {});", + errors: callError("onlyTakes", { + fnName: "removeObserver", + params: "2 parameters", + }), + }, + { + code: "Services.removeObserver('notification name', {}, true);", + output: "Services.removeObserver('notification name', {});", + errors: callError("onlyTakes", { + fnName: "removeObserver", + params: "2 parameters", + }), + }, + { + code: "Services.io.newURI('http://example.com', null, null);", + output: "Services.io.newURI('http://example.com');", + errors: callError("newURIParams"), + }, + { + code: "Services.io.newURI('http://example.com', 'utf8', null);", + output: "Services.io.newURI('http://example.com', 'utf8');", + errors: callError("newURIParams"), + }, + { + code: "Services.io.newURI('http://example.com', null);", + output: "Services.io.newURI('http://example.com');", + errors: callError("newURIParams"), + }, + { + code: "Services.io.newURI('http://example.com', '', '');", + output: "Services.io.newURI('http://example.com');", + errors: callError("newURIParams"), + }, + { + code: "Services.io.newURI('http://example.com', '');", + output: "Services.io.newURI('http://example.com');", + errors: callError("newURIParams"), + }, + { + code: "elt.addEventListener('click', handler, false);", + output: "elt.addEventListener('click', handler);", + errors: callError("obmittedWhenFalse", { + fnName: "addEventListener", + index: "third", + }), + }, + { + code: "elt.removeEventListener('click', handler, false);", + output: "elt.removeEventListener('click', handler);", + errors: callError("obmittedWhenFalse", { + fnName: "removeEventListener", + index: "third", + }), + }, + { + code: "Services.obs.addObserver(this, 'topic', false);", + output: "Services.obs.addObserver(this, 'topic');", + errors: callError("obmittedWhenFalse", { + fnName: "addObserver", + index: "third", + }), + }, + { + code: "Services.prefs.addObserver('branch', this, false);", + output: "Services.prefs.addObserver('branch', this);", + errors: callError("obmittedWhenFalse", { + fnName: "addObserver", + index: "third", + }), + }, + { + code: "array.appendElement(elt, false);", + output: "array.appendElement(elt);", + errors: callError("obmittedWhenFalse", { + fnName: "appendElement", + index: "second", + }), + }, + { + code: "Services.obs.notifyObservers(obj, 'topic', null);", + output: "Services.obs.notifyObservers(obj, 'topic');", + errors: callError("obmittedWhenFalse", { + fnName: "notifyObservers", + index: "third", + }), + }, + { + code: "Services.obs.notifyObservers(obj, 'topic', '');", + output: "Services.obs.notifyObservers(obj, 'topic');", + errors: callError("obmittedWhenFalse", { + fnName: "notifyObservers", + index: "third", + }), + }, + { + code: "window.getComputedStyle(elt, null);", + output: "window.getComputedStyle(elt);", + errors: callError("obmittedWhenFalse", { + fnName: "getComputedStyle", + index: "second", + }), + }, + { + code: "window.getComputedStyle(elt, '');", + output: "window.getComputedStyle(elt);", + errors: callError("obmittedWhenFalse", { + fnName: "getComputedStyle", + index: "second", + }), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-removeEventListener.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-removeEventListener.js new file mode 100644 index 0000000000..f5fc60158d --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-removeEventListener.js @@ -0,0 +1,96 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-useless-removeEventListener"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code) { + return { code, errors: [{ messageId: "useOnce", type: "CallExpression" }] }; +} + +ruleTester.run("no-useless-removeEventListener", rule, { + valid: [ + // Listeners that aren't a function are always valid. + "elt.addEventListener('click', handler);", + "elt.addEventListener('click', handler, true);", + "elt.addEventListener('click', handler, {once: true});", + + // Should not fail on empty functions. + "elt.addEventListener('click', function() {});", + + // Should not reject when removing a listener for another event. + "elt.addEventListener('click', function listener() {" + + " elt.removeEventListener('keypress', listener);" + + "});", + + // Should not reject when there's another instruction before + // removeEventListener. + "elt.addEventListener('click', function listener() {" + + " elt.focus();" + + " elt.removeEventListener('click', listener);" + + "});", + + // Should not reject when wantsUntrusted is true. + "elt.addEventListener('click', function listener() {" + + " elt.removeEventListener('click', listener);" + + "}, false, true);", + + // Should not reject when there's a literal and a variable + "elt.addEventListener('click', function listener() {" + + " elt.removeEventListener(eventName, listener);" + + "});", + + // Should not reject when there's 2 different variables + "elt.addEventListener(event1, function listener() {" + + " elt.removeEventListener(event2, listener);" + + "});", + + // Should not fail if this is a different type of event listener function. + "myfunc.addEventListener(listener);", + ], + invalid: [ + invalidCode( + "elt.addEventListener('click', function listener() {" + + " elt.removeEventListener('click', listener);" + + "});" + ), + invalidCode( + "elt.addEventListener('click', function listener() {" + + " elt.removeEventListener('click', listener, true);" + + "}, true);" + ), + invalidCode( + "elt.addEventListener('click', function listener() {" + + " elt.removeEventListener('click', listener);" + + "}, {once: true});" + ), + invalidCode( + "elt.addEventListener('click', function listener() {" + + " /* Comment */" + + " elt.removeEventListener('click', listener);" + + "});" + ), + invalidCode( + "elt.addEventListener('click', function() {" + + " elt.removeEventListener('click', arguments.callee);" + + "});" + ), + invalidCode( + "elt.addEventListener(eventName, function listener() {" + + " elt.removeEventListener(eventName, listener);" + + "});" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-run-test.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-run-test.js new file mode 100644 index 0000000000..5ee6e0e575 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-run-test.js @@ -0,0 +1,126 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/no-useless-run-test"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, output) { + return { + code, + output, + errors: [{ messageId: "noUselessRunTest", type: "FunctionDeclaration" }], + }; +} + +ruleTester.run("no-useless-run-test", rule, { + valid: [ + "function run_test() {}", + "function run_test() {let args = 1; run_next_test();}", + "function run_test() {fakeCall(); run_next_test();}", + ], + invalid: [ + // Single-line case. + invalidCode("function run_test() { run_next_test(); }", ""), + // Multiple-line cases + invalidCode( + ` +function run_test() { + run_next_test(); +}`, + `` + ), + invalidCode( + ` +foo(); +function run_test() { + run_next_test(); +} +`, + ` +foo(); +` + ), + invalidCode( + ` +foo(); +function run_test() { + run_next_test(); +} +bar(); +`, + ` +foo(); +bar(); +` + ), + invalidCode( + ` +foo(); + +function run_test() { + run_next_test(); +} + +bar();`, + ` +foo(); + +bar();` + ), + invalidCode( + ` +foo(); + +function run_test() { + run_next_test(); +} + +// A comment +bar(); +`, + ` +foo(); + +// A comment +bar(); +` + ), + invalidCode( + ` +foo(); + +/** + * A useful comment. + */ +function run_test() { + run_next_test(); +} + +// A comment +bar(); +`, + ` +foo(); + +/** + * A useful comment. + */ + +// A comment +bar(); +` + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/prefer-boolean-length-check.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/prefer-boolean-length-check.js new file mode 100644 index 0000000000..102112a3f5 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/prefer-boolean-length-check.js @@ -0,0 +1,96 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/prefer-boolean-length-check"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidError() { + return [{ messageId: "preferBooleanCheck", type: "BinaryExpression" }]; +} + +ruleTester.run("check-length", rule, { + valid: [ + "if (foo.length && foo.length) {}", + "if (!foo.length) {}", + "if (foo.value == 0) {}", + "if (foo.value > 0) {}", + "if (0 == foo.value) {}", + "if (0 < foo.value) {}", + "var a = !!foo.length", + "function bar() { return !!foo.length }", + ], + invalid: [ + { + code: "if (foo.length == 0) {}", + output: "if (!foo.length) {}", + errors: invalidError(), + }, + { + code: "if (foo.length > 0) {}", + output: "if (foo.length) {}", + errors: invalidError(), + }, + { + code: "if (0 < foo.length) {}", + output: "if (foo.length) {}", + errors: invalidError(), + }, + { + code: "if (0 == foo.length) {}", + output: "if (!foo.length) {}", + errors: invalidError(), + }, + { + code: "if (foo && foo.length == 0) {}", + output: "if (foo && !foo.length) {}", + errors: invalidError(), + }, + { + code: "if (foo.bar.length == 0) {}", + output: "if (!foo.bar.length) {}", + errors: invalidError(), + }, + { + code: "var a = foo.length>0", + output: "var a = !!foo.length", + errors: invalidError(), + }, + { + code: "function bar() { return foo.length>0 }", + output: "function bar() { return !!foo.length }", + errors: invalidError(), + }, + { + code: "if (foo && bar.length>0) {}", + output: "if (foo && bar.length) {}", + errors: invalidError(), + }, + { + code: "while (foo && bar.length>0) {}", + output: "while (foo && bar.length) {}", + errors: invalidError(), + }, + { + code: "x = y && bar.length > 0", + output: "x = y && !!bar.length", + errors: invalidError(), + }, + { + code: "function bar() { return x && foo.length > 0}", + output: "function bar() { return x && !!foo.length}", + errors: invalidError(), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/prefer-formatValues.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/prefer-formatValues.js new file mode 100644 index 0000000000..4aa5cd1d3c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/prefer-formatValues.js @@ -0,0 +1,70 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/prefer-formatValues"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function error(line, column = undefined) { + return { + messageId: "useSingleCall", + type: "CallExpression", + line, + column, + }; +} + +ruleTester.run("check-length", rule, { + valid: [ + "document.l10n.formatValue('foobar');", + "document.l10n.formatValues(['foobar', 'foobaz']);", + `if (foo) { + document.l10n.formatValue('foobar'); + } else { + document.l10n.formatValue('foobaz'); + }`, + `document.l10n.formatValue('foobaz'); + if (foo) { + document.l10n.formatValue('foobar'); + }`, + `if (foo) { + document.l10n.formatValue('foobar'); + } + document.l10n.formatValue('foobaz');`, + `if (foo) { + document.l10n.formatValue('foobar'); + } + document.l10n.formatValues(['foobaz']);`, + ], + invalid: [ + { + code: `document.l10n.formatValue('foobar'); + document.l10n.formatValue('foobaz');`, + errors: [error(1, 1), error(2, 14)], + }, + { + code: `document.l10n.formatValue('foobar'); + if (foo) { + document.l10n.formatValue('foobiz'); + } + document.l10n.formatValue('foobaz');`, + errors: [error(1, 1), error(5, 14)], + }, + { + code: `document.l10n.formatValues('foobar'); + document.l10n.formatValue('foobaz');`, + errors: [error(1, 1), error(2, 14)], + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-addtask-only.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-addtask-only.js new file mode 100644 index 0000000000..f29726d9fc --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-addtask-only.js @@ -0,0 +1,54 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-addtask-only"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidError(output) { + return [ + { + messageId: "addTaskNotAllowed", + type: "CallExpression", + suggestions: [{ messageId: "addTaskNotAllowedSuggestion", output }], + }, + ]; +} + +ruleTester.run("reject-addtask-only", rule, { + valid: [ + "add_task(foo())", + "add_task(foo()).skip()", + "add_task(function() {})", + "add_task(function() {}).skip()", + ], + invalid: [ + { + code: "add_task(foo()).only()", + errors: invalidError("add_task(foo())"), + }, + { + code: "add_task(foo()).only(bar())", + errors: invalidError("add_task(foo())"), + }, + { + code: "add_task(function() {}).only()", + errors: invalidError("add_task(function() {})"), + }, + { + code: "add_task(function() {}).only(bar())", + errors: invalidError("add_task(function() {})"), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-chromeutils-import-params.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-chromeutils-import-params.js new file mode 100644 index 0000000000..c2b5fd9349 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-chromeutils-import-params.js @@ -0,0 +1,67 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-chromeutils-import-params"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidError(suggested) { + return [ + { + messageId: "importOnlyOneArg", + type: "CallExpression", + suggestions: [ + { + messageId: "importOnlyOneArgSuggestion", + output: suggested, + }, + ], + }, + ]; +} + +ruleTester.run("reject-chromeutils-import-params", rule, { + valid: ['ChromeUtils.import("resource://some/path/to/My.jsm")'], + invalid: [ + { + code: 'ChromeUtils.import("resource://some/path/to/My.jsm", null)', + errors: invalidError( + `ChromeUtils.import("resource://some/path/to/My.jsm")` + ), + }, + { + code: ` +ChromeUtils.import( + "resource://some/path/to/My.jsm", + null +);`, + errors: invalidError(` +ChromeUtils.import( + "resource://some/path/to/My.jsm" +);`), + }, + { + code: 'ChromeUtils.import("resource://some/path/to/My.jsm", this)', + errors: invalidError( + `ChromeUtils.import("resource://some/path/to/My.jsm")` + ), + }, + { + code: 'ChromeUtils.import("resource://some/path/to/My.jsm", foo, bar)', + errors: invalidError( + `ChromeUtils.import("resource://some/path/to/My.jsm")` + ), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-chromeutils-import.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-chromeutils-import.js new file mode 100644 index 0000000000..ced46f4a0f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-chromeutils-import.js @@ -0,0 +1,60 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-chromeutils-import"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +const invalidError = [ + { + messageId: "useImportESModule", + type: "CallExpression", + }, +]; + +const invalidErrorLazy = [ + { + messageId: "useImportESModuleLazy", + type: "CallExpression", + }, +]; + +ruleTester.run("reject-chromeutils-import", rule, { + valid: [ + 'ChromeUtils.importESModule("resource://some/path/to/My.sys.mjs")', + 'ChromeUtils.defineESModuleGetters(obj, { My: "resource://some/path/to/My.sys.mjs" })', + ], + invalid: [ + { + code: 'ChromeUtils.import("resource://some/path/to/My.jsm")', + errors: invalidError, + }, + { + code: 'SpecialPowers.ChromeUtils.import("resource://some/path/to/My.jsm")', + errors: invalidError, + }, + { + code: 'ChromeUtils.defineModuleGetter(obj, "My", "resource://some/path/to/My.jsm")', + errors: invalidErrorLazy, + }, + { + code: 'SpecialPowers.ChromeUtils.defineModuleGetter(obj, "My", "resource://some/path/to/My.jsm")', + errors: invalidErrorLazy, + }, + { + code: 'XPCOMUtils.defineLazyModuleGetters(obj, { My: "resource://some/path/to/My.jsm" })', + errors: invalidErrorLazy, + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-eager-module-in-lazy-getter.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-eager-module-in-lazy-getter.js new file mode 100644 index 0000000000..490346b94a --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-eager-module-in-lazy-getter.js @@ -0,0 +1,67 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-eager-module-in-lazy-getter"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, uri) { + return { code, errors: [{ messageId: "eagerModule", data: { uri } }] }; +} + +ruleTester.run("reject-eager-module-in-lazy-getter", rule, { + valid: [ + ` + ChromeUtils.defineModuleGetter( + lazy, "Integration", "resource://gre/modules/Integration.jsm" + ); +`, + ` + XPCOMUtils.defineLazyModuleGetters(lazy, { + Integration: "resource://gre/modules/Integration.jsm", + }); +`, + ` + ChromeUtils.defineESModuleGetters(lazy, { + Integration: "resource://gre/modules/Integration.sys.mjs", + }); +`, + ], + invalid: [ + invalidCode( + ` + ChromeUtils.defineModuleGetter( + lazy, "XPCOMUtils", "resource://gre/modules/XPCOMUtils.jsm" + ); +`, + "resource://gre/modules/XPCOMUtils.jsm" + ), + invalidCode( + ` + XPCOMUtils.defineLazyModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.jsm", + }); +`, + "resource://gre/modules/AppConstants.jsm" + ), + invalidCode( + ` + ChromeUtils.defineESModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); +`, + "resource://gre/modules/AppConstants.sys.mjs" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-global-this.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-global-this.js new file mode 100644 index 0000000000..ec9ba711dc --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-global-this.js @@ -0,0 +1,51 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-global-this"); +var RuleTester = require("eslint").RuleTester; + +// class static block is available from ES2022 = 13. +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code) { + return { + code, + errors: [{ messageId: "avoidGlobalThis", type: "ThisExpression" }], + }; +} + +ruleTester.run("reject-top-level-await", rule, { + valid: [ + "function f() { this; }", + "(function f() { this; });", + "({ foo() { this; } });", + "({ get foo() { this; } })", + "({ set foo(x) { this; } })", + "class X { foo() { this; } }", + "class X { get foo() { this; } }", + "class X { set foo(x) { this; } }", + "class X { static foo() { this; } }", + "class X { static get foo() { this; } }", + "class X { static set foo(x) { this; } }", + "class X { P = this; }", + "class X { #P = this; }", + "class X { static { this; } }", + ], + invalid: [ + invalidCode("this;"), + invalidCode("() => this;"), + + invalidCode("this.foo = 10;"), + invalidCode("ChromeUtils.defineModuleGetter(this, {});"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-globalThis-modification.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-globalThis-modification.js new file mode 100644 index 0000000000..c0244eaeab --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-globalThis-modification.js @@ -0,0 +1,83 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-globalThis-modification"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCall(code) { + return { + code, + errors: [ + { + messageId: "rejectPassingGlobalThis", + type: "CallExpression", + }, + ], + }; +} + +function invalidAssignment(code) { + return { + code, + errors: [ + { + messageId: "rejectModifyGlobalThis", + type: "AssignmentExpression", + }, + ], + }; +} + +ruleTester.run("reject-globalThis-modification", rule, { + valid: [ + `var x = globalThis.Array;`, + `Array in globalThis;`, + `result.deserialize(globalThis)`, + ], + invalid: [ + invalidAssignment(` + globalThis.foo = 10; +`), + invalidCall(` + Object.defineProperty(globalThis, "foo", { value: 10 }); +`), + invalidCall(` + Object.defineProperties(globalThis, { + foo: { value: 10 }, + }); +`), + invalidCall(` + Object.assign(globalThis, { foo: 10 }); +`), + invalidCall(` + ChromeUtils.defineModuleGetter( + globalThis, "AppConstants", "resource://gre/modules/AppConstants.jsm" + ); +`), + invalidCall(` + ChromeUtils.defineESMGetters(globalThis, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); +`), + invalidCall(` + XPCOMUtils.defineLazyModuleGetters(globalThis, { + AppConstants: "resource://gre/modules/AppConstants.jsm", + }); +`), + invalidCall(` + someFunction(1, globalThis); +`), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-import-system-module-from-non-system.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-import-system-module-from-non-system.js new file mode 100644 index 0000000000..5b09007626 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-import-system-module-from-non-system.js @@ -0,0 +1,33 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-import-system-module-from-non-system"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ + parserOptions: { ecmaVersion: "latest", sourceType: "module" }, +}); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +ruleTester.run("reject-import-system-module-from-non-system", rule, { + valid: [ + { + code: `const { AppConstants } = ChromeUtils.importESM("resource://gre/modules/AppConstants.sys.mjs");`, + }, + ], + invalid: [ + { + code: `import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs";`, + errors: [{ messageId: "rejectStaticImportSystemModuleFromNonSystem" }], + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-importGlobalProperties.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-importGlobalProperties.js new file mode 100644 index 0000000000..7f796abae7 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-importGlobalProperties.js @@ -0,0 +1,89 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-importGlobalProperties"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +ruleTester.run("reject-importGlobalProperties", rule, { + valid: [ + { + code: "Cu.something();", + }, + { + options: ["allownonwebidl"], + code: "Cu.importGlobalProperties(['caches'])", + }, + { + options: ["allownonwebidl"], + code: "XPCOMUtils.defineLazyGlobalGetters(this, ['caches'])", + }, + ], + invalid: [ + { + code: "Cu.importGlobalProperties(['fetch'])", + options: ["everything"], + errors: [{ messageId: "unexpectedCall" }], + }, + { + code: "XPCOMUtils.defineLazyGlobalGetters(this, ['fetch'])", + options: ["everything"], + errors: [{ messageId: "unexpectedCall" }], + }, + { + code: "Cu.importGlobalProperties(['TextEncoder'])", + options: ["everything"], + errors: [{ messageId: "unexpectedCall" }], + }, + { + options: ["everything"], + code: "XPCOMUtils.defineLazyGlobalGetters(this, ['TextEncoder'])", + errors: [{ messageId: "unexpectedCallSjs" }], + filename: "foo.sjs", + }, + { + code: "XPCOMUtils.defineLazyGlobalGetters(this, ['TextEncoder'])", + options: ["everything"], + errors: [{ messageId: "unexpectedCall" }], + }, + { + code: "Cu.importGlobalProperties(['TextEncoder'])", + options: ["allownonwebidl"], + errors: [{ messageId: "unexpectedCallCuWebIdl" }], + }, + { + code: "XPCOMUtils.defineLazyGlobalGetters(this, ['TextEncoder'])", + options: ["allownonwebidl"], + errors: [{ messageId: "unexpectedCallXPCOMWebIdl" }], + }, + { + options: ["allownonwebidl"], + code: "Cu.importGlobalProperties(['TextEncoder'])", + errors: [{ messageId: "unexpectedCallCuWebIdl" }], + filename: "foo.js", + }, + { + options: ["allownonwebidl"], + code: "XPCOMUtils.defineLazyGlobalGetters(this, ['TextEncoder'])", + errors: [{ messageId: "unexpectedCallXPCOMWebIdl" }], + filename: "foo.js", + }, + { + options: ["allownonwebidl"], + code: "Cu.importGlobalProperties(['TextEncoder'])", + errors: [{ messageId: "unexpectedCallCuWebIdl" }], + filename: "foo.sjs", + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-lazy-imports-into-globals.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-lazy-imports-into-globals.js new file mode 100644 index 0000000000..2c50c5524d --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-lazy-imports-into-globals.js @@ -0,0 +1,46 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-lazy-imports-into-globals"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ + parserOptions: { ecmaVersion: "latest", sourceType: "module" }, +}); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code) { + return { code, errors: [{ messageId: "rejectLazyImportsIntoGlobals" }] }; +} + +ruleTester.run("reject-lazy-imports-into-globals", rule, { + valid: [ + ` + const lazy = {}; + ChromeUtils.defineLazyGetter(lazy, "foo", () => {}); + `, + ], + invalid: [ + invalidCode(`ChromeUtils.defineLazyGetter(globalThis, "foo", () => {});`), + invalidCode(`ChromeUtils.defineLazyGetter(window, "foo", () => {});`), + invalidCode( + `XPCOMUtils.defineLazyPreferenceGetter(globalThis, "foo", "foo.bar");` + ), + invalidCode( + `XPCOMUtils.defineLazyServiceGetter(globalThis, "foo", "@foo", "nsIFoo");` + ), + invalidCode(`XPCOMUtils.defineLazyGlobalGetters(globalThis, {});`), + invalidCode(`XPCOMUtils.defineLazyGlobalGetters(window, {});`), + invalidCode(`XPCOMUtils.defineLazyModuleGetters(globalThis, {});`), + invalidCode(`ChromeUtils.defineESModuleGetters(window, {});`), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-mixing-eager-and-lazy.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-mixing-eager-and-lazy.js new file mode 100644 index 0000000000..af0dd99c22 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-mixing-eager-and-lazy.js @@ -0,0 +1,104 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-mixing-eager-and-lazy"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ + parserOptions: { ecmaVersion: "latest", sourceType: "module" }, +}); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, uri) { + return { code, errors: [{ messageId: "mixedEagerAndLazy", data: { uri } }] }; +} + +ruleTester.run("reject-mixing-eager-and-lazy", rule, { + valid: [ + ` + ChromeUtils.import("resource://gre/modules/AppConstants.jsm"); +`, + ` + ChromeUtils.importESModule("resource://gre/modules/AppConstants.sys.mjs"); +`, + ` + import{ AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; +`, + ` + ChromeUtils.defineModuleGetter( + lazy, "AppConstants", "resource://gre/modules/AppConstants.jsm" + ); +`, + ` + XPCOMUtils.defineLazyModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.jsm", + }); +`, + ` + ChromeUtils.defineESModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); +`, + ` + if (some_condition) { + ChromeUtils.import("resource://gre/modules/AppConstants.jsm"); + } + XPCOMUtils.defineLazyModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.jsm" + }); +`, + ` + ChromeUtils.import("resource://gre/modules/AppConstants.jsm"); + XPCOMUtils.defineLazyModuleGetters(sandbox, { + AppConstants: "resource://gre/modules/AppConstants.jsm", + }); +`, + ], + invalid: [ + invalidCode( + ` + ChromeUtils.import("resource://gre/modules/AppConstants.jsm"); + ChromeUtils.defineModuleGetter( + lazy, "AppConstants", "resource://gre/modules/AppConstants.jsm" + ); +`, + "resource://gre/modules/AppConstants.jsm" + ), + invalidCode( + ` + ChromeUtils.import("resource://gre/modules/AppConstants.jsm"); + XPCOMUtils.defineLazyModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.jsm", + }); +`, + "resource://gre/modules/AppConstants.jsm" + ), + invalidCode( + ` + ChromeUtils.importESModule("resource://gre/modules/AppConstants.sys.mjs"); + ChromeUtils.defineESModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); +`, + "resource://gre/modules/AppConstants.sys.mjs" + ), + invalidCode( + ` + import { AppConstants } from "resource://gre/modules/AppConstants.sys.mjs"; + ChromeUtils.defineESModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); +`, + "resource://gre/modules/AppConstants.sys.mjs" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-multiple-getters-calls.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-multiple-getters-calls.js new file mode 100644 index 0000000000..a2b88a8652 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-multiple-getters-calls.js @@ -0,0 +1,60 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-multiple-getters-calls"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code) { + return { code, errors: [{ messageId: "rejectMultipleCalls" }] }; +} + +ruleTester.run("reject-multiple-getters-calls", rule, { + valid: [ + ` + ChromeUtils.defineESModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + }); + `, + ` + ChromeUtils.defineESModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); + ChromeUtils.defineESModuleGetters(window, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + }); + `, + ` + ChromeUtils.defineESModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); + if (cond) { + ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + }); + } + `, + ], + invalid: [ + invalidCode(` + ChromeUtils.defineESModuleGetters(lazy, { + AppConstants: "resource://gre/modules/AppConstants.sys.mjs", + }); + ChromeUtils.defineESModuleGetters(lazy, { + PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs", + }); + `), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-relative-requires.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-relative-requires.js new file mode 100644 index 0000000000..a7bad6f992 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-relative-requires.js @@ -0,0 +1,56 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-relative-requires"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidError() { + return [{ messageId: "rejectRelativeRequires", type: "CallExpression" }]; +} + +ruleTester.run("reject-relative-requires", rule, { + valid: [ + 'require("devtools/absolute/path")', + 'require("resource://gre/modules/SomeModule.jsm")', + 'loader.lazyRequireGetter(this, "path", "devtools/absolute/path", true)', + 'loader.lazyRequireGetter(this, "Path", "devtools/absolute/path")', + ], + invalid: [ + { + code: 'require("./relative/path")', + errors: invalidError(), + }, + { + code: 'require("../parent/folder/path")', + errors: invalidError(), + }, + { + code: 'loader.lazyRequireGetter(this, "path", "./relative/path", true)', + errors: invalidError(), + }, + { + code: 'loader.lazyRequireGetter(this, "path", "../parent/folder/path", true)', + errors: invalidError(), + }, + { + code: 'loader.lazyRequireGetter(this, "path", "./relative/path")', + errors: invalidError(), + }, + { + code: 'loader.lazyRequireGetter(this, "path", "../parent/folder/path")', + errors: invalidError(), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-scriptableunicodeconverter.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-scriptableunicodeconverter.js new file mode 100644 index 0000000000..22123bb99c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-scriptableunicodeconverter.js @@ -0,0 +1,33 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-scriptableunicodeconverter"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidError() { + return [ + { messageId: "rejectScriptableUnicodeConverter", type: "MemberExpression" }, + ]; +} + +ruleTester.run("reject-scriptableunicodeconverter", rule, { + valid: ["TextEncoder", "TextDecoder"], + invalid: [ + { + code: "Ci.nsIScriptableUnicodeConverter", + errors: invalidError(), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-some-requires.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-some-requires.js new file mode 100644 index 0000000000..3415946b07 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-some-requires.js @@ -0,0 +1,50 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-some-requires"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function requirePathError(path) { + return [ + { messageId: "rejectRequire", data: { path }, type: "CallExpression" }, + ]; +} + +const DEVTOOLS_FORBIDDEN_PATH = "^(resource://)?devtools/forbidden"; + +ruleTester.run("reject-some-requires", rule, { + valid: [ + { + code: 'require("devtools/not-forbidden/path")', + options: [DEVTOOLS_FORBIDDEN_PATH], + }, + { + code: 'require("resource://devtools/not-forbidden/path")', + options: [DEVTOOLS_FORBIDDEN_PATH], + }, + ], + invalid: [ + { + code: 'require("devtools/forbidden/path")', + errors: requirePathError("devtools/forbidden/path"), + options: [DEVTOOLS_FORBIDDEN_PATH], + }, + { + code: 'require("resource://devtools/forbidden/path")', + errors: requirePathError("resource://devtools/forbidden/path"), + options: [DEVTOOLS_FORBIDDEN_PATH], + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-top-level-await.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-top-level-await.js new file mode 100644 index 0000000000..4416b42abb --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/reject-top-level-await.js @@ -0,0 +1,36 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/reject-top-level-await"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ + parserOptions: { ecmaVersion: "latest", sourceType: "module" }, +}); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, messageId) { + return { code, errors: [{ messageId: "rejectTopLevelAwait" }] }; +} + +ruleTester.run("reject-top-level-await", rule, { + valid: [ + "async() => { await bar() }", + "async() => { for await (let x of []) {} }", + ], + invalid: [ + invalidCode("await foo"), + invalidCode("{ await foo }"), + invalidCode("(await foo)"), + invalidCode("for await (let x of []) {}"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/rejects-requires-await.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/rejects-requires-await.js new file mode 100644 index 0000000000..a7e95769c1 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/rejects-requires-await.js @@ -0,0 +1,32 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/rejects-requires-await"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, messageId) { + return { code, errors: [{ messageId: "rejectRequiresAwait" }] }; +} + +ruleTester.run("reject-requires-await", rule, { + valid: [ + "async() => { await Assert.rejects(foo, /assertion/) }", + "async() => { await Assert.rejects(foo, /assertion/, 'msg') }", + ], + invalid: [ + invalidCode("Assert.rejects(foo)"), + invalidCode("Assert.rejects(foo, 'msg')"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-cc-etc.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-cc-etc.js new file mode 100644 index 0000000000..d588fabc84 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-cc-etc.js @@ -0,0 +1,64 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-cc-etc"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, originalName, newName, output) { + return { + code, + output, + errors: [ + { + messageId: "useCcEtc", + data: { + shortName: newName, + oldName: originalName, + }, + type: "MemberExpression", + }, + ], + }; +} + +ruleTester.run("use-cc-etc", rule, { + valid: ["Components.Constructor();", "let x = Components.foo;"], + invalid: [ + invalidCode( + "let foo = Components.classes['bar'];", + "classes", + "Cc", + "let foo = Cc['bar'];" + ), + invalidCode( + "let bar = Components.interfaces.bar;", + "interfaces", + "Ci", + "let bar = Ci.bar;" + ), + invalidCode( + "Components.results.NS_ERROR_ILLEGAL_INPUT;", + "results", + "Cr", + "Cr.NS_ERROR_ILLEGAL_INPUT;" + ), + invalidCode( + "Components.utils.reportError('fake');", + "utils", + "Cu", + "Cu.reportError('fake');" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-definelazygetter.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-definelazygetter.js new file mode 100644 index 0000000000..89948f12b7 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-definelazygetter.js @@ -0,0 +1,34 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-chromeutils-definelazygetter"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function callError(messageId) { + return [{ messageId, type: "CallExpression" }]; +} + +ruleTester.run("use-chromeutils-definelazygetter", rule, { + valid: [ + `ChromeUtils.defineLazyGetter(lazy, "textEncoder", function () { return new TextEncoder(); });`, + ], + invalid: [ + { + code: `XPCOMUtils.defineLazyGetter(lazy, "textEncoder", function () { return new TextEncoder(); });`, + output: `ChromeUtils.defineLazyGetter(lazy, "textEncoder", function () { return new TextEncoder(); });`, + errors: callError("useChromeUtilsDefineLazyGetter"), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-generateqi.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-generateqi.js new file mode 100644 index 0000000000..ca32d2ac26 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-generateqi.js @@ -0,0 +1,71 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-chromeutils-generateqi"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function error(messageId, type) { + return [{ messageId, type }]; +} + +/* globals nsIFlug */ +function QueryInterface(iid) { + if ( + iid.equals(Ci.nsISupports) || + iid.equals(Ci.nsIMeh) || + iid.equals(nsIFlug) || + iid.equals(Ci.amIFoo) + ) { + return this; + } + throw Components.Exception("", Cr.NS_ERROR_NO_INTERFACE); +} + +ruleTester.run("use-chromeutils-generateqi", rule, { + valid: [ + `X.prototype.QueryInterface = ChromeUtils.generateQI(["nsIMeh"]);`, + `X.prototype = { QueryInterface: ChromeUtils.generateQI(["nsIMeh"]) }`, + ], + invalid: [ + { + code: `X.prototype.QueryInterface = XPCOMUtils.generateQI(["nsIMeh"]);`, + output: `X.prototype.QueryInterface = ChromeUtils.generateQI(["nsIMeh"]);`, + errors: error("noXpcomUtilsGenerateQI", "CallExpression"), + }, + { + code: `X.prototype = { QueryInterface: XPCOMUtils.generateQI(["nsIMeh"]) };`, + output: `X.prototype = { QueryInterface: ChromeUtils.generateQI(["nsIMeh"]) };`, + errors: error("noXpcomUtilsGenerateQI", "CallExpression"), + }, + { + code: `X.prototype = { QueryInterface: ${QueryInterface} };`, + output: `X.prototype = { QueryInterface: ChromeUtils.generateQI(["nsIMeh", "nsIFlug", "amIFoo"]) };`, + errors: error("noJSQueryInterface", "Property"), + }, + { + code: `X.prototype = { ${String(QueryInterface).replace( + /^function /, + "" + )} };`, + output: `X.prototype = { QueryInterface: ChromeUtils.generateQI(["nsIMeh", "nsIFlug", "amIFoo"]) };`, + errors: error("noJSQueryInterface", "Property"), + }, + { + code: `X.prototype.QueryInterface = ${QueryInterface};`, + output: `X.prototype.QueryInterface = ChromeUtils.generateQI(["nsIMeh", "nsIFlug", "amIFoo"]);`, + errors: error("noJSQueryInterface", "AssignmentExpression"), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-import.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-import.js new file mode 100644 index 0000000000..37976d252e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-import.js @@ -0,0 +1,47 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-chromeutils-import"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function callError(messageId) { + return [{ messageId, type: "CallExpression" }]; +} + +ruleTester.run("use-chromeutils-import", rule, { + valid: [ + `ChromeUtils.import("resource://gre/modules/AppConstants.jsm");`, + `ChromeUtils.import("resource://gre/modules/AppConstants.jsm", this);`, + `ChromeUtils.defineModuleGetter(this, "AppConstants", + "resource://gre/modules/AppConstants.jsm");`, + ], + invalid: [ + { + code: `Cu.import("resource://gre/modules/AppConstants.jsm");`, + output: `ChromeUtils.import("resource://gre/modules/AppConstants.jsm");`, + errors: callError("useChromeUtilsImport"), + }, + { + code: `Cu.import("resource://gre/modules/AppConstants.jsm", this);`, + output: `ChromeUtils.import("resource://gre/modules/AppConstants.jsm", this);`, + errors: callError("useChromeUtilsImport"), + }, + { + code: `Components.utils.import("resource://gre/modules/AppConstants.jsm");`, + output: `ChromeUtils.import("resource://gre/modules/AppConstants.jsm");`, + errors: callError("useChromeUtilsImport"), + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-console-createInstance.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-console-createInstance.js new file mode 100644 index 0000000000..bd7fe3a507 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-console-createInstance.js @@ -0,0 +1,41 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-console-createInstance"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +ruleTester.run("use-console-createInstance", rule, { + valid: ['"resource://gre/modules/Foo.sys.mjs"'], + invalid: [ + { + code: '"resource://gre/modules/Console.sys.mjs"', + errors: [ + { + messageId: "useConsoleRatherThanModule", + data: { module: "Console.sys.mjs" }, + }, + ], + }, + { + code: '"resource://gre/modules/Log.sys.mjs"', + errors: [ + { + messageId: "useConsoleRatherThanModule", + data: { module: "Log.sys.mjs" }, + }, + ], + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-default-preference-values.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-default-preference-values.js new file mode 100644 index 0000000000..a8353ec200 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-default-preference-values.js @@ -0,0 +1,43 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-default-preference-values"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code) { + return { + code, + errors: [{ messageId: "provideDefaultValue", type: "TryStatement" }], + }; +} + +let types = ["Bool", "Char", "Float", "Int"]; +let methods = types.map(type => "get" + type + "Pref"); + +ruleTester.run("use-default-preference-values", rule, { + valid: [].concat( + methods.map(m => "blah = branch." + m + "('blah', true);"), + methods.map(m => "blah = branch." + m + "('blah');"), + methods.map( + m => "try { canThrow(); blah = branch." + m + "('blah'); } catch(e) {}" + ) + ), + + invalid: [].concat( + methods.map(m => + invalidCode("try { blah = branch." + m + "('blah'); } catch(e) {}") + ) + ), +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-includes-instead-of-indexOf.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-includes-instead-of-indexOf.js new file mode 100644 index 0000000000..fff826874f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-includes-instead-of-indexOf.js @@ -0,0 +1,40 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-includes-instead-of-indexOf"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code) { + return { + code, + errors: [{ messageId: "useIncludes", type: "BinaryExpression" }], + }; +} + +ruleTester.run("use-includes-instead-of-indexOf", rule, { + valid: [ + "let a = foo.includes(bar);", + "let a = foo.indexOf(bar) > 0;", + "let a = foo.indexOf(bar) != 0;", + ], + invalid: [ + invalidCode("let a = foo.indexOf(bar) >= 0;"), + invalidCode("let a = foo.indexOf(bar) != -1;"), + invalidCode("let a = foo.indexOf(bar) !== -1;"), + invalidCode("let a = foo.indexOf(bar) == -1;"), + invalidCode("let a = foo.indexOf(bar) === -1;"), + invalidCode("let a = foo.indexOf(bar) < 0;"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-isInstance.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-isInstance.js new file mode 100644 index 0000000000..0de5e4577c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-isInstance.js @@ -0,0 +1,128 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-isInstance"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +const errors = [ + { + messageId: "preferIsInstance", + type: "BinaryExpression", + }, +]; + +const env = { browser: true }; + +/** + * A test case boilerplate simulating chrome privileged script. + * @param {string} code + */ +function mockChromeScript(code) { + return { + code, + filename: "foo.sys.mjs", + env, + }; +} + +/** + * A test case boilerplate simulating content script. + * @param {string} code + */ +function mockContentScript(code) { + return { + code, + filename: "foo.js", + env, + }; +} + +ruleTester.run("use-isInstance", rule, { + valid: [ + mockChromeScript("(() => {}) instanceof Function;"), + mockChromeScript("({}) instanceof Object;"), + mockChromeScript("Node instanceof Object;"), + mockChromeScript("node instanceof lazy.Node;"), + mockChromeScript("var Node;node instanceof Node;"), + mockChromeScript("file instanceof lazy.File;"), + mockChromeScript("file instanceof OS.File;"), + mockChromeScript("file instanceof OS.File.Error;"), + mockChromeScript("file instanceof lazy.OS.File;"), + mockChromeScript("file instanceof lazy.OS.File.Error;"), + mockChromeScript("file instanceof lazy.lazy.OS.File;"), + mockChromeScript("var File;file instanceof File;"), + mockChromeScript("foo instanceof RandomGlobalThing;"), + mockChromeScript("foo instanceof lazy.RandomGlobalThing;"), + mockContentScript("node instanceof Node;"), + mockContentScript("file instanceof File;"), + mockContentScript( + "SpecialPowers.ChromeUtils.import(''); file instanceof File;" + ), + ], + invalid: [ + { + code: "node instanceof Node", + output: "Node.isInstance(node)", + env, + errors, + filename: "foo.sys.mjs", + }, + { + code: "text instanceof win.Text", + output: "win.Text.isInstance(text)", + errors, + filename: "foo.sys.mjs", + }, + { + code: "target instanceof this.contentWindow.HTMLAudioElement", + output: "this.contentWindow.HTMLAudioElement.isInstance(target)", + errors, + filename: "foo.sys.mjs", + }, + { + code: "target instanceof File", + output: "File.isInstance(target)", + env, + errors, + filename: "foo.sys.mjs", + }, + { + code: "target instanceof win.File", + output: "win.File.isInstance(target)", + errors, + filename: "foo.sys.mjs", + }, + { + code: "window.arguments[0] instanceof window.XULElement", + output: "window.XULElement.isInstance(window.arguments[0])", + errors, + filename: "foo.sys.mjs", + }, + { + code: "ChromeUtils.import(''); node instanceof Node", + output: "ChromeUtils.import(''); Node.isInstance(node)", + env, + errors, + filename: "foo.js", + }, + { + code: "ChromeUtils.import(''); file instanceof File", + output: "ChromeUtils.import(''); File.isInstance(file)", + env, + errors, + filename: "foo.js", + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-ownerGlobal.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-ownerGlobal.js new file mode 100644 index 0000000000..150f0fac63 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-ownerGlobal.js @@ -0,0 +1,37 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-ownerGlobal"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code) { + return { + code, + errors: [{ messageId: "useOwnerGlobal", type: "MemberExpression" }], + }; +} + +ruleTester.run("use-ownerGlobal", rule, { + valid: [ + "aEvent.target.ownerGlobal;", + "this.DOMPointNode.ownerGlobal.getSelection();", + "windowToMessageManager(node.ownerGlobal);", + ], + invalid: [ + invalidCode("aEvent.target.ownerDocument.defaultView;"), + invalidCode("this.DOMPointNode.ownerDocument.defaultView.getSelection();"), + invalidCode("windowToMessageManager(node.ownerDocument.defaultView);"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-returnValue.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-returnValue.js new file mode 100644 index 0000000000..2c15349139 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-returnValue.js @@ -0,0 +1,49 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-returnValue"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, methodName) { + return { + code, + errors: [ + { + messageId: "useReturnValue", + data: { + property: methodName, + }, + type: "ExpressionStatement", + }, + ], + }; +} + +ruleTester.run("use-returnValue", rule, { + valid: [ + "a = foo.concat(bar)", + "b = bar.concat([1,3,4])", + "c = baz.concat()", + "d = qux.join(' ')", + "e = quux.slice(1)", + ], + invalid: [ + invalidCode("foo.concat(bar)", "concat"), + invalidCode("bar.concat([1,3,4])", "concat"), + invalidCode("baz.concat()", "concat"), + invalidCode("qux.join(' ')", "join"), + invalidCode("quux.slice(1)", "slice"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-services.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-services.js new file mode 100644 index 0000000000..fc4af03ac1 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-services.js @@ -0,0 +1,64 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-services"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, serviceName, getterName, type = "CallExpression") { + return { + code, + errors: [ + { messageId: "useServices", data: { serviceName, getterName }, type }, + ], + }; +} + +ruleTester.run("use-services", rule, { + valid: [ + 'Cc["@mozilla.org/fakeservice;1"].getService(Ci.nsIFake)', + 'Components.classes["@mozilla.org/fakeservice;1"].getService(Components.interfaces.nsIFake)', + "Services.wm.addListener()", + ], + invalid: [ + invalidCode( + 'Cc["@mozilla.org/appshell/window-mediator;1"].getService(Ci.nsIWindowMediator);', + "wm", + "getService()" + ), + invalidCode( + 'Components.classes["@mozilla.org/toolkit/app-startup;1"].getService(Components.interfaces.nsIAppStartup);', + "startup", + "getService()" + ), + invalidCode( + `XPCOMUtils.defineLazyServiceGetters(this, { + uuidGen: ["@mozilla.org/uuid-generator;1", "nsIUUIDGenerator"], + });`, + "uuid", + "defineLazyServiceGetters", + "ArrayExpression" + ), + invalidCode( + `XPCOMUtils.defineLazyServiceGetter( + this, + "gELS", + "@mozilla.org/eventlistenerservice;1", + "nsIEventListenerService" + );`, + "els", + "defineLazyServiceGetter" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/use-static-import.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-static-import.js new file mode 100644 index 0000000000..656e83e0e9 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/use-static-import.js @@ -0,0 +1,80 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/use-static-import"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ + parserOptions: { ecmaVersion: "latest", sourceType: "module" }, +}); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function callError() { + return [{ messageId: "useStaticImport", type: "VariableDeclaration" }]; +} + +ruleTester.run("use-static-import", rule, { + valid: [ + { + // Already converted, no issues. + code: 'import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";', + filename: "test.sys.mjs", + }, + { + // Inside an if statement. + code: 'if (foo) { const { XPCOMUtils } = ChromeUtils.importESModule("resource://gre/modules/XPCOMUtils.sys.mjs") }', + filename: "test.sys.mjs", + }, + { + // Inside a function. + code: 'function foo() { const { XPCOMUtils } = ChromeUtils.importESModule("resource://gre/modules/XPCOMUtils.sys.mjs") }', + filename: "test.sys.mjs", + }, + { + // importESModule with two args cannot be converted. + code: 'const { f } = ChromeUtils.importESModule("some/module.sys.mjs", { loadInDevToolsLoader : true });', + filename: "test.sys.mjs", + }, + { + // A non-system file attempting to import a system file should not be + // converted. + code: 'const { XPCOMUtils } = ChromeUtils.importESModule("resource://gre/modules/XPCOMUtils.sys.mjs")', + filename: "test.mjs", + }, + ], + invalid: [ + { + // Simple import in system module should be converted. + code: 'const { XPCOMUtils } = ChromeUtils.importESModule("resource://gre/modules/XPCOMUtils.sys.mjs")', + errors: callError(), + filename: "test.sys.mjs", + output: + 'import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"', + }, + { + // Should handle rewritten variables as well. + code: 'const { XPCOMUtils: foo } = ChromeUtils.importESModule("resource://gre/modules/XPCOMUtils.sys.mjs")', + errors: callError(), + filename: "test.sys.mjs", + output: + 'import { XPCOMUtils as foo } from "resource://gre/modules/XPCOMUtils.sys.mjs"', + }, + { + // Should handle multiple variables. + code: 'const { foo, XPCOMUtils } = ChromeUtils.importESModule("resource://gre/modules/XPCOMUtils.sys.mjs")', + errors: callError(), + filename: "test.sys.mjs", + output: + 'import { foo, XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs"', + }, + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-ci-uses.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-ci-uses.js new file mode 100644 index 0000000000..0f5fe2eadf --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-ci-uses.js @@ -0,0 +1,58 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var os = require("os"); +var rule = require("../lib/rules/valid-ci-uses"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, messageId, data) { + return { code, errors: [{ messageId, data }] }; +} + +process.env.MOZ_XPT_ARTIFACTS_DIR = `${__dirname}/xpidl`; + +const tests = { + valid: ["Ci.nsIURIFixup", "Ci.nsIURIFixup.FIXUP_FLAG_NONE"], + invalid: [ + invalidCode("Ci.nsIURIFixup.UNKNOWN_CONSTANT", "unknownProperty", { + interface: "nsIURIFixup", + property: "UNKNOWN_CONSTANT", + }), + invalidCode("Ci.nsIFoo", "unknownInterface", { + interface: "nsIFoo", + }), + ], +}; + +// For ESLint tests, we only have a couple of xpt examples in the xpidl directory. +// Therefore we can pretend that these interfaces no longer exist. +switch (os.platform) { + case "windows": + tests.invalid.push( + invalidCode("Ci.nsIJumpListShortcut", "missingInterface") + ); + break; + case "darwin": + tests.invalid.push( + invalidCode("Ci.nsIMacShellService", "missingInterface") + ); + break; + case "linux": + tests.invalid.push( + invalidCode("Ci.mozISandboxReporter", "missingInterface") + ); +} + +ruleTester.run("valid-ci-uses", rule, tests); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-lazy.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-lazy.js new file mode 100644 index 0000000000..ba0a8dafab --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-lazy.js @@ -0,0 +1,160 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/valid-lazy"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ + parserOptions: { ecmaVersion: "latest", sourceType: "module" }, +}); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, name, messageId) { + return { code, errors: [{ messageId, data: { name } }] }; +} + +ruleTester.run("valid-lazy", rule, { + // Note: these tests build on top of one another, although lazy gets + // re-declared, it + valid: [ + ` + const lazy = {}; + ChromeUtils.defineLazyGetter(lazy, "foo", () => {}); + if (x) { lazy.foo.bar(); } + `, + ` + const lazy = {}; + ChromeUtils.defineESModuleGetters(lazy, { + foo: "foo.mjs", + }); + if (x) { lazy.foo.bar(); } + `, + ` + const lazy = {}; + Integration.downloads.defineESModuleGetter(lazy, "foo", "foo.sys.mjs"); + if (x) { lazy.foo.bar(); } + `, + ` + const lazy = createLazyLoaders({ foo: () => {}}); + if (x) { lazy.foo.bar(); } + `, + ` + const lazy = {}; + loader.lazyRequireGetter( + lazy, + ["foo1", "foo2"], + "bar", + true + ); + if (x) { + lazy.foo1.bar(); + lazy.foo2.bar(); + } + `, + // Test for top-level unconditional. + ` + const lazy = {}; + ChromeUtils.defineLazyGetter(lazy, "foo", () => {}); + if (x) { lazy.foo.bar(); } + for (;;) { lazy.foo.bar(); } + for (var x in y) { lazy.foo.bar(); } + for (var x of y) { lazy.foo.bar(); } + while (true) { lazy.foo.bar(); } + do { lazy.foo.bar(); } while (true); + switch (x) { case 1: lazy.foo.bar(); } + try { lazy.foo.bar(); } catch (e) {} + function f() { lazy.foo.bar(); } + (function f() { lazy.foo.bar(); }); + () => { lazy.foo.bar(); }; + class C { + constructor() { lazy.foo.bar(); } + foo() { lazy.foo.bar(); } + get x() { lazy.foo.bar(); } + set x(v) { lazy.foo.bar(); } + a = lazy.foo.bar(); + #b = lazy.foo.bar(); + static { + lazy.foo.bar(); + } + } + a && lazy.foo.bar(); + a || lazy.foo.bar(); + a ?? lazy.foo.bar(); + a ? lazy.foo.bar() : b; + a?.b[lazy.foo.bar()]; + a ||= lazy.foo.bar(); + a &&= lazy.foo.bar(); + a ??= lazy.foo.bar(); + var { x = lazy.foo.bar() } = {}; + var [ y = lazy.foo.bar() ] = []; + `, + ` + const lazy = {}; + ChromeUtils.defineLazyGetter(lazy, "foo", () => {}); + export { lazy as Foo }; + `, + ` + const lazy = {}; + if (cond) { + ChromeUtils.defineLazyGetter(lazy, "foo", () => { return 1; }); + } else { + ChromeUtils.defineLazyGetter(lazy, "foo", () => { return 2; }); + } + if (x) { lazy.foo; } + `, + ], + invalid: [ + invalidCode("if (x) { lazy.bar; }", "bar", "unknownProperty"), + invalidCode( + ` + const lazy = {}; + ChromeUtils.defineLazyGetter(lazy, "foo", "foo.jsm"); + ChromeUtils.defineLazyGetter(lazy, "foo", "foo1.jsm"); + if (x) { lazy.foo.bar(); } + `, + "foo", + "duplicateSymbol" + ), + invalidCode( + ` + const lazy = {}; + XPCOMUtils.defineLazyModuleGetters(lazy, { + "foo-bar": "foo.jsm", + }); + if (x) { lazy["foo-bar"].bar(); } + `, + "foo-bar", + "incorrectType" + ), + invalidCode( + `const lazy = {}; + ChromeUtils.defineLazyGetter(lazy, "foo", "foo.jsm"); + `, + "foo", + "unusedProperty" + ), + invalidCode( + `const lazy = {}; + ChromeUtils.defineLazyGetter(lazy, "foo1", () => {}); + lazy.foo1.bar();`, + "foo1", + "topLevelAndUnconditional" + ), + invalidCode( + `const lazy = {}; + ChromeUtils.defineLazyGetter(lazy, "foo1", () => {}); + { x = -f(1 + lazy.foo1.bar()); }`, + "foo1", + "topLevelAndUnconditional" + ), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-services-property.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-services-property.js new file mode 100644 index 0000000000..a9806ccc72 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-services-property.js @@ -0,0 +1,42 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/valid-services-property"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, messageId, data) { + return { code, errors: [{ messageId, data }] }; +} + +process.env.MOZ_XPT_ARTIFACTS_DIR = `${__dirname}/xpidl`; + +ruleTester.run("valid-services-property", rule, { + valid: [ + "Services.uriFixup.keywordToURI()", + "Services.uriFixup.FIXUP_FLAG_NONE", + ], + invalid: [ + invalidCode("Services.uriFixup.UNKNOWN_CONSTANT", "unknownProperty", { + alias: "uriFixup", + propertyName: "UNKNOWN_CONSTANT", + checkedInterfaces: ["nsIURIFixup"], + }), + invalidCode("Services.uriFixup.foo()", "unknownProperty", { + alias: "uriFixup", + propertyName: "foo", + checkedInterfaces: ["nsIURIFixup"], + }), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-services.js b/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-services.js new file mode 100644 index 0000000000..1453e95b7b --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/valid-services.js @@ -0,0 +1,32 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Requirements +// ------------------------------------------------------------------------------ + +var rule = require("../lib/rules/valid-services"); +var RuleTester = require("eslint").RuleTester; + +const ruleTester = new RuleTester({ parserOptions: { ecmaVersion: "latest" } }); + +// ------------------------------------------------------------------------------ +// Tests +// ------------------------------------------------------------------------------ + +function invalidCode(code, alias) { + return { code, errors: [{ messageId: "unknownProperty", data: { alias } }] }; +} + +ruleTester.run("valid-services", rule, { + valid: ["Services.crashmanager", "lazy.Services.crashmanager"], + invalid: [ + invalidCode("Services.foo", "foo"), + invalidCode("Services.foo()", "foo"), + invalidCode("lazy.Services.foo", "foo"), + invalidCode("Services.foo.bar()", "foo"), + invalidCode("lazy.Services.foo.bar()", "foo"), + ], +}); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/xpidl/docshell.xpt b/tools/lint/eslint/eslint-plugin-mozilla/tests/xpidl/docshell.xpt new file mode 100644 index 0000000000..fc8a45216b --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/xpidl/docshell.xpt @@ -0,0 +1,5940 @@ +[ + { + "consts": [ + { + "name": "ePrompt", + "type": { + "tag": "TD_UINT8" + }, + "value": 0 + }, + { + "name": "eDontPromptAndDontUnload", + "type": { + "tag": "TD_UINT8" + }, + "value": 1 + }, + { + "name": "eDontPromptAndUnload", + "type": { + "tag": "TD_UINT8" + }, + "value": 2 + }, + { + "name": "eAllowNavigation", + "type": { + "tag": "TD_UINT8" + }, + "value": 0 + }, + { + "name": "eRequestBlockNavigation", + "type": { + "tag": "TD_UINT8" + }, + "value": 1 + }, + { + "name": "eDelayResize", + "type": { + "tag": "TD_UINT32" + }, + "value": 1 + } + ], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [ + "hidden" + ], + "name": "init", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "container", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocShell", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "container", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocShell", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "loadStart", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "headerFile": "mozilla/dom/Document.h", + "name": "Document", + "native": "mozilla::dom::Document", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [], + "name": "loadComplete", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hidden", + "hasretval" + ], + "name": "loadCompleted", + "params": [] + }, + { + "flags": [ + "getter", + "hidden", + "hasretval" + ], + "name": "isStopped", + "params": [] + }, + { + "flags": [ + "hasretval" + ], + "name": "permitUnload", + "params": [ + { + "flags": [ + "in", + "optional" + ], + "type": { + "tag": "TD_UINT8" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "inPermitUnload", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "dispatchBeforeUnload", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "beforeUnloadFiring", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "pageHide", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "close", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsISHEntry", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "destroy", + "params": [] + }, + { + "flags": [], + "name": "stop", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "DOMDocument", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "headerFile": "mozilla/dom/Document.h", + "name": "Document", + "native": "mozilla::dom::Document", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getDocument", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "setDocument", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "headerFile": "mozilla/dom/Document.h", + "name": "Document", + "native": "mozilla::dom::Document", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getBounds", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "setBounds", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "setBoundsWithFlags", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hidden", + "hasretval" + ], + "name": "previousViewer", + "params": [] + }, + { + "flags": [ + "setter", + "hidden" + ], + "name": "previousViewer", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocumentViewer", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "move", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "show", + "params": [] + }, + { + "flags": [], + "name": "hide", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "sticky", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "sticky", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "open", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISHEntry", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "clearHistoryEntry", + "params": [] + }, + { + "flags": [], + "name": "setPageModeForTesting", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIPrintSettings", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "setPrintSettingsForSubdocument", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIPrintSettings", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "historyEntry", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsISHEntry", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isTabModalPromptAllowed", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isHidden", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "isHidden", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hidden", + "hasretval" + ], + "name": "presShell", + "params": [] + }, + { + "flags": [ + "getter", + "hidden", + "hasretval" + ], + "name": "presContext", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "setDocumentInternal", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "headerFile": "mozilla/dom/Document.h", + "name": "Document", + "native": "mozilla::dom::Document", + "tag": "TD_DOMOBJECT" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "findContainerView", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "setNavigationTiming", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "deviceFullZoomForTest", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_FLOAT" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "authorStyleDisabled", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "authorStyleDisabled", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "getContentSize", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "getContentSizeConstrained", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getReloadEncodingAndSource", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "setReloadEncodingAndSource", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "forgetReloadEncoding", + "params": [] + } + ], + "name": "nsIDocumentViewer", + "parent": "nsISupports", + "uuid": "48118355-e9a5-4452-ab18-59cc426fb817" + }, + { + "consts": [ + { + "name": "COPY_IMAGE_TEXT", + "type": { + "tag": "TD_INT32" + }, + "value": 1 + }, + { + "name": "COPY_IMAGE_HTML", + "type": { + "tag": "TD_INT32" + }, + "value": 2 + }, + { + "name": "COPY_IMAGE_DATA", + "type": { + "tag": "TD_INT32" + }, + "value": 4 + }, + { + "name": "COPY_IMAGE_ALL", + "type": { + "tag": "TD_INT32" + }, + "value": -1 + } + ], + "flags": [], + "methods": [ + { + "flags": [], + "name": "clearSelection", + "params": [] + }, + { + "flags": [], + "name": "selectAll", + "params": [] + }, + { + "flags": [], + "name": "copySelection", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "copyable", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "copyLinkLocation", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "inLink", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "copyImage", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "inImage", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "getContents", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "canGetContents", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "setCommandNode", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "headerFile": "nsIContent.h", + "name": "Node", + "native": "nsINode", + "tag": "TD_DOMOBJECT" + } + } + ] + } + ], + "name": "nsIDocumentViewerEdit", + "parent": "nsISupports", + "uuid": "35be2d7e-f29b-48ec-bf7e-80a30a724de3" + }, + { + "consts": [ + { + "name": "ENUMERATE_FORWARDS", + "type": { + "tag": "TD_UINT8" + }, + "value": 0 + }, + { + "name": "ENUMERATE_BACKWARDS", + "type": { + "tag": "TD_UINT8" + }, + "value": 1 + }, + { + "name": "APP_TYPE_UNKNOWN", + "type": { + "tag": "TD_UINT8" + }, + "value": 0 + }, + { + "name": "APP_TYPE_MAIL", + "type": { + "tag": "TD_UINT8" + }, + "value": 1 + }, + { + "name": "APP_TYPE_EDITOR", + "type": { + "tag": "TD_UINT8" + }, + "value": 2 + }, + { + "name": "BUSY_FLAGS_NONE", + "type": { + "tag": "TD_UINT8" + }, + "value": 0 + }, + { + "name": "BUSY_FLAGS_BUSY", + "type": { + "tag": "TD_UINT8" + }, + "value": 1 + }, + { + "name": "BUSY_FLAGS_BEFORE_PAGE_LOAD", + "type": { + "tag": "TD_UINT8" + }, + "value": 2 + }, + { + "name": "BUSY_FLAGS_PAGE_LOADING", + "type": { + "tag": "TD_UINT8" + }, + "value": 4 + }, + { + "name": "LOAD_CMD_NORMAL", + "type": { + "tag": "TD_UINT8" + }, + "value": 1 + }, + { + "name": "LOAD_CMD_RELOAD", + "type": { + "tag": "TD_UINT8" + }, + "value": 2 + }, + { + "name": "LOAD_CMD_HISTORY", + "type": { + "tag": "TD_UINT8" + }, + "value": 4 + }, + { + "name": "LOAD_CMD_PUSHSTATE", + "type": { + "tag": "TD_UINT8" + }, + "value": 8 + }, + { + "name": "META_VIEWPORT_OVERRIDE_DISABLED", + "type": { + "tag": "TD_UINT8" + }, + "value": 0 + }, + { + "name": "META_VIEWPORT_OVERRIDE_ENABLED", + "type": { + "tag": "TD_UINT8" + }, + "value": 1 + }, + { + "name": "META_VIEWPORT_OVERRIDE_NONE", + "type": { + "tag": "TD_UINT8" + }, + "value": 2 + } + ], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [], + "name": "setCancelContentJSEpoch", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "loadURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "jscontext" + ], + "name": "addState", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_JSVAL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "prepareForNewContentModel", + "params": [] + }, + { + "flags": [], + "name": "setCurrentURIForSessionStore", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "firePageHideNotification", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hidden", + "hasretval" + ], + "name": "presContext", + "params": [] + }, + { + "flags": [ + "getter", + "hidden", + "hasretval" + ], + "name": "presShell", + "params": [] + }, + { + "flags": [ + "getter", + "hidden", + "hasretval" + ], + "name": "eldestPresShell", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "docViewer", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocumentViewer", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "outerWindowID", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "chromeEventHandler", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "headerFile": "mozilla/dom/EventTarget.h", + "name": "EventTarget", + "native": "mozilla::dom::EventTarget", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "chromeEventHandler", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "headerFile": "mozilla/dom/EventTarget.h", + "name": "EventTarget", + "native": "mozilla::dom::EventTarget", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "customUserAgent", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "customUserAgent", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "cssErrorReportingEnabled", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "cssErrorReportingEnabled", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowPlugins", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowPlugins", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowMetaRedirects", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowMetaRedirects", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowSubframes", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowSubframes", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowImages", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowImages", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowMedia", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowMedia", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowDNSPrefetch", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowDNSPrefetch", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowWindowControl", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowWindowControl", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowContentRetargeting", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowContentRetargeting", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowContentRetargetingOnChildren", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowContentRetargetingOnChildren", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "getAllDocShellsInSubtree", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT8" + } + }, + { + "flags": [ + "out" + ], + "type": { + "element": { + "name": "nsIDocShell", + "tag": "TD_INTERFACE_TYPE" + }, + "tag": "TD_ARRAY" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "appType", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT8" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "appType", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT8" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "allowAuth", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "allowAuth", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "zoom", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_FLOAT" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "zoom", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_FLOAT" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "tabToTreeOwner", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "busyFlags", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT8" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "loadType", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "loadType", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "defaultLoadFlags", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "defaultLoadFlags", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "isBeingDestroyed", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isExecutingOnLoadHandler", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "layoutHistoryState", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsILayoutHistoryState", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "layoutHistoryState", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsILayoutHistoryState", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "loadURIDelegate", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsILoadURIDelegate", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "suspendRefreshURIs", + "params": [] + }, + { + "flags": [], + "name": "resumeRefreshURIs", + "params": [] + }, + { + "flags": [], + "name": "beginRestore", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocumentViewer", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "finishRestore", + "params": [] + }, + { + "flags": [], + "name": "clearCachedUserAgent", + "params": [] + }, + { + "flags": [], + "name": "clearCachedPlatform", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "restoringDocument", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "useErrorPages", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "useErrorPages", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "displayLoadError", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PWSTRING" + } + }, + { + "flags": [ + "in", + "optional" + ], + "type": { + "name": "nsIChannel", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "failedChannel", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIChannel", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "previousEntryIndex", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "loadedEntryIndex", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "historyPurged", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "currentDocumentChannel", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIChannel", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isInUnload", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "DetachEditorFromWindow", + "params": [] + }, + { + "flags": [], + "name": "exitPrintPreview", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "historyID", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_NSID" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "HistoryID", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isAppTab", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "isAppTab", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "createAboutBlankDocumentViewer", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIPrincipal", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIPrincipal", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in", + "optional" + ], + "type": { + "name": "nsIContentSecurityPolicy", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "charset", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_CSTRING" + } + } + ] + }, + { + "flags": [], + "name": "forceEncodingDetection", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "setParentCharset", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIPrincipal", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getParentCharset", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "out" + ], + "type": { + "name": "nsIPrincipal", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "now", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_DOUBLE" + } + } + ] + }, + { + "flags": [], + "name": "addWeakPrivacyTransitionObserver", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIPrivacyTransitionObserver", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "addWeakReflowObserver", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIReflowObserver", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "removeWeakReflowObserver", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIReflowObserver", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "notifyReflowObservers", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_DOUBLE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_DOUBLE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "addWeakScrollObserver", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIScrollObserver", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "removeWeakScrollObserver", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIScrollObserver", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "notifyScrollObservers", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isTopLevelContentDocShell", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "asyncPanZoomEnabled", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "pluginsAllowedInCurrentDoc", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "mayEnableCharacterEncodingMenu", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "editor", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIEditor", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "editor", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIEditor", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "editable", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "hasEditingSession", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "makeEditable", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "getCurrentSHEntry", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsISHEntry", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "isCommandEnabled", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "doCommand", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + } + ] + }, + { + "flags": [], + "name": "doCommandWithParams", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsICommandParams", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "IsInvisible", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "SetInvisible", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "GetScriptGlobalObject", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "getExtantDocument", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "deviceSizeIsPageSize", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "deviceSizeIsPageSize", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "hasLoadedNonBlankURI", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "windowDraggingAllowed", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "windowDraggingAllowed", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "currentScrollRestorationIsManual", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "currentScrollRestorationIsManual", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "jscontext", + "hasretval" + ], + "name": "getOriginAttributes", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_JSVAL" + } + } + ] + }, + { + "flags": [ + "jscontext" + ], + "name": "setOriginAttributes", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_JSVAL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "editingSession", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIEditingSession", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "browserChild", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIBrowserChild", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "GetBrowserChild", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "GetCommandManager", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "metaViewportOverride", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT8" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "metaViewportOverride", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT8" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "useTrackingProtection", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "useTrackingProtection", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "dispatchLocationChangeEvent", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "startDelayedAutoplayMediaComponents", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "TakeInitialClientSource", + "params": [] + }, + { + "flags": [], + "name": "setColorMatrix", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "element": { + "tag": "TD_FLOAT" + }, + "tag": "TD_ARRAY" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isForceReloading", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "getColorMatrix", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "element": { + "tag": "TD_FLOAT" + }, + "tag": "TD_ARRAY" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "messageManager", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "headerFile": "mozilla/dom/ContentFrameMessageManager.h", + "name": "ContentFrameMessageManager", + "native": "mozilla::dom::ContentFrameMessageManager", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "getHasTrackingContentBlocked", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_PROMISE" + } + } + ] + }, + { + "flags": [ + "getter", + "hidden", + "hasretval" + ], + "name": "isAttemptingToNavigate", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isNavigating", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "synchronizeLayoutHistoryState", + "params": [] + }, + { + "flags": [], + "name": "persistLayoutHistoryState", + "params": [] + } + ], + "name": "nsIDocShell", + "parent": "nsIDocShellTreeItem", + "uuid": "049234fe-da10-478b-bc5d-bc6f9a1ba63d" + }, + { + "consts": [ + { + "name": "typeChrome", + "type": { + "tag": "TD_INT32" + }, + "value": 0 + }, + { + "name": "typeContent", + "type": { + "tag": "TD_INT32" + }, + "value": 1 + }, + { + "name": "typeContentWrapper", + "type": { + "tag": "TD_INT32" + }, + "value": 2 + }, + { + "name": "typeChromeWrapper", + "type": { + "tag": "TD_INT32" + }, + "value": 3 + }, + { + "name": "typeAll", + "type": { + "tag": "TD_INT32" + }, + "value": 2147483647 + } + ], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [ + "getter", + "hasretval" + ], + "name": "name", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "name", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "nameEquals", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "itemType", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "ItemType", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "parent", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "sameTypeParent", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "rootTreeItem", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "sameTypeRootTreeItem", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "treeOwner", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocShellTreeOwner", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "setTreeOwner", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocShellTreeOwner", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "childCount", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "addChild", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "removeChild", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "getChildAt", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "browsingContext", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "headerFile": "mozilla/dom/BrowsingContext.h", + "name": "BrowsingContext", + "native": "mozilla::dom::BrowsingContext", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getBrowsingContext", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "domWindow", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "mozIDOMWindowProxy", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getDocument", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "getWindow", + "params": [] + } + ], + "name": "nsIDocShellTreeItem", + "parent": "nsISupports", + "uuid": "9b7c586f-9214-480c-a2c4-49b526fff1a6" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "contentShellAdded", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "contentShellRemoved", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "primaryContentShell", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "remoteTabAdded", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIRemoteTab", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "remoteTabRemoved", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIRemoteTab", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "primaryRemoteTab", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIRemoteTab", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "primaryContentBrowsingContext", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "headerFile": "mozilla/dom/BrowsingContext.h", + "name": "BrowsingContext", + "native": "mozilla::dom::BrowsingContext", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [], + "name": "sizeShellTo", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocShellTreeItem", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "getPrimaryContentSize", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "setPrimaryContentSize", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "getRootShellSize", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "setRootShellSize", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "setPersistence", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "getPersistence", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "tabCount", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "hasPrimaryContent", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + } + ], + "name": "nsIDocShellTreeOwner", + "parent": "nsISupports", + "uuid": "0e3dc4b1-4cea-4a37-af71-79f0afd07574" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "createInstance", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIChannel", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsILoadGroup", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocShell", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "out" + ], + "type": { + "name": "nsIStreamListener", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocumentViewer", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "createInstanceForDocument", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "headerFile": "mozilla/dom/Document.h", + "name": "Document", + "native": "mozilla::dom::Document", + "tag": "TD_DOMOBJECT" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "out" + ], + "type": { + "name": "nsIDocumentViewer", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsIDocumentLoaderFactory", + "parent": "nsISupports", + "uuid": "e795239e-9d3c-47c4-b063-9e600fb3b287" + }, + { + "consts": [], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [ + "getter", + "hasretval" + ], + "name": "associatedWindow", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "mozIDOMWindowProxy", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "topWindow", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "mozIDOMWindowProxy", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "topFrameElement", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "headerFile": "mozilla/dom/Element.h", + "name": "Element", + "native": "mozilla::dom::Element", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isContent", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "usePrivateBrowsing", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "usePrivateBrowsing", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "useRemoteTabs", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "useRemoteSubframes", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "useTrackingProtection", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "useTrackingProtection", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "SetPrivateBrowsing", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "SetRemoteTabs", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "SetRemoteSubframes", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "originAttributes", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_JSVAL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "GetOriginAttributes", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + } + ], + "name": "nsILoadContext", + "parent": "nsISupports", + "uuid": "2813a7a3-d084-4d00-acd0-f76620315c02" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "loadURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT16" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIPrincipal", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "handleLoadError", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT16" + } + }, + { + "flags": [ + "out" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsILoadURIDelegate", + "parent": "nsISupports", + "uuid": "78e42d37-a34c-4d96-b901-25385669aba4" + }, + { + "consts": [], + "flags": [ + "function" + ], + "methods": [ + { + "flags": [], + "name": "privateModeChanged", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + } + ], + "name": "nsIPrivacyTransitionObserver", + "parent": "nsISupports", + "uuid": "b4b1449d-0ef0-47f5-b62e-adc57fd49702" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "reflow", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_DOUBLE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_DOUBLE" + } + } + ] + }, + { + "flags": [], + "name": "reflowInterruptible", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_DOUBLE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_DOUBLE" + } + } + ] + } + ], + "name": "nsIReflowObserver", + "parent": "nsISupports", + "uuid": "832e692c-c4a6-11e2-8fd1-dce678957a39" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "refreshURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIPrincipal", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [], + "name": "forceRefreshURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIPrincipal", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [], + "name": "cancelRefreshURITimers", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "refreshPending", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + } + ], + "name": "nsIRefreshURI", + "parent": "nsISupports", + "uuid": "a5e61a3c-51bd-45be-ac0c-e87b71860656" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "onShowTooltip", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [], + "name": "onHideTooltip", + "params": [] + } + ], + "name": "nsITooltipListener", + "parent": "nsISupports", + "uuid": "44b78386-1dd2-11b2-9ad2-e4eee2ca1916" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "getNodeText", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "headerFile": "nsIContent.h", + "name": "Node", + "native": "nsINode", + "tag": "TD_DOMOBJECT" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_PWSTRING" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_PWSTRING" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + } + ], + "name": "nsITooltipTextProvider", + "parent": "nsISupports", + "uuid": "b128a1e6-44f3-4331-8fbe-5af360ff21ee" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "getter", + "hasretval" + ], + "name": "consumer", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "headerFile": "mozilla/dom/BrowsingContext.h", + "name": "BrowsingContext", + "native": "mozilla::dom::BrowsingContext", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "consumer", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "headerFile": "mozilla/dom/BrowsingContext.h", + "name": "BrowsingContext", + "native": "mozilla::dom::BrowsingContext", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "preferredURI", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "preferredURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "fixedURI", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "fixedURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "keywordProviderName", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "keywordProviderName", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "keywordAsSent", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "keywordAsSent", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "fixupChangedProtocol", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "fixupChangedProtocol", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "fixupCreatedAlternateURI", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "fixupCreatedAlternateURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "originalInput", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UTF8STRING" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "originalInput", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UTF8STRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "postData", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIInputStream", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "postData", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIInputStream", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsIURIFixupInfo", + "parent": "nsISupports", + "uuid": "4819f183-b532-4932-ac09-b309cd853be7" + }, + { + "consts": [ + { + "name": "FIXUP_FLAG_NONE", + "type": { + "tag": "TD_UINT32" + }, + "value": 0 + }, + { + "name": "FIXUP_FLAG_ALLOW_KEYWORD_LOOKUP", + "type": { + "tag": "TD_UINT32" + }, + "value": 1 + }, + { + "name": "FIXUP_FLAGS_MAKE_ALTERNATE_URI", + "type": { + "tag": "TD_UINT32" + }, + "value": 2 + }, + { + "name": "FIXUP_FLAG_PRIVATE_CONTEXT", + "type": { + "tag": "TD_UINT32" + }, + "value": 4 + }, + { + "name": "FIXUP_FLAG_FIX_SCHEME_TYPOS", + "type": { + "tag": "TD_UINT32" + }, + "value": 8 + } + ], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "getFixupURIInfo", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UTF8STRING" + } + }, + { + "flags": [ + "in", + "optional" + ], + "type": { + "tag": "TD_UINT32" + } + }, + { + "flags": [ + "out" + ], + "type": { + "name": "nsIURIFixupInfo", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "webNavigationFlagsToFixupFlags", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UTF8STRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "keywordToURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UTF8STRING" + } + }, + { + "flags": [ + "in", + "optional" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "out" + ], + "type": { + "name": "nsIURIFixupInfo", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "forceHttpFixup", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UTF8STRING" + } + }, + { + "flags": [ + "out" + ], + "type": { + "name": "nsIURIFixupInfo", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "checkHost", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDNSListener", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in", + "optional" + ], + "type": { + "tag": "TD_JSVAL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "isDomainKnown", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UTF8STRING" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + } + ], + "name": "nsIURIFixup", + "parent": "nsISupports", + "uuid": "1da7e9d4-620b-4949-849a-1cd6077b1b2d" + }, + { + "consts": [ + { + "name": "LOAD_FLAGS_MASK", + "type": { + "tag": "TD_UINT32" + }, + "value": 65535 + }, + { + "name": "LOAD_FLAGS_NONE", + "type": { + "tag": "TD_UINT32" + }, + "value": 0 + }, + { + "name": "LOAD_FLAGS_IS_REFRESH", + "type": { + "tag": "TD_UINT32" + }, + "value": 16 + }, + { + "name": "LOAD_FLAGS_IS_LINK", + "type": { + "tag": "TD_UINT32" + }, + "value": 32 + }, + { + "name": "LOAD_FLAGS_BYPASS_HISTORY", + "type": { + "tag": "TD_UINT32" + }, + "value": 64 + }, + { + "name": "LOAD_FLAGS_REPLACE_HISTORY", + "type": { + "tag": "TD_UINT32" + }, + "value": 128 + }, + { + "name": "LOAD_FLAGS_BYPASS_CACHE", + "type": { + "tag": "TD_UINT32" + }, + "value": 256 + }, + { + "name": "LOAD_FLAGS_BYPASS_PROXY", + "type": { + "tag": "TD_UINT32" + }, + "value": 512 + }, + { + "name": "LOAD_FLAGS_CHARSET_CHANGE", + "type": { + "tag": "TD_UINT32" + }, + "value": 1024 + }, + { + "name": "LOAD_FLAGS_STOP_CONTENT", + "type": { + "tag": "TD_UINT32" + }, + "value": 2048 + }, + { + "name": "LOAD_FLAGS_FROM_EXTERNAL", + "type": { + "tag": "TD_UINT32" + }, + "value": 4096 + }, + { + "name": "LOAD_FLAGS_FIRST_LOAD", + "type": { + "tag": "TD_UINT32" + }, + "value": 16384 + }, + { + "name": "LOAD_FLAGS_ALLOW_POPUPS", + "type": { + "tag": "TD_UINT32" + }, + "value": 32768 + }, + { + "name": "LOAD_FLAGS_BYPASS_CLASSIFIER", + "type": { + "tag": "TD_UINT32" + }, + "value": 65536 + }, + { + "name": "LOAD_FLAGS_FORCE_ALLOW_COOKIES", + "type": { + "tag": "TD_UINT32" + }, + "value": 131072 + }, + { + "name": "LOAD_FLAGS_DISALLOW_INHERIT_PRINCIPAL", + "type": { + "tag": "TD_UINT32" + }, + "value": 262144 + }, + { + "name": "LOAD_FLAGS_ERROR_LOAD_CHANGES_RV", + "type": { + "tag": "TD_UINT32" + }, + "value": 524288 + }, + { + "name": "LOAD_FLAGS_ALLOW_THIRD_PARTY_FIXUP", + "type": { + "tag": "TD_UINT32" + }, + "value": 1048576 + }, + { + "name": "LOAD_FLAGS_FIXUP_SCHEME_TYPOS", + "type": { + "tag": "TD_UINT32" + }, + "value": 2097152 + }, + { + "name": "LOAD_FLAGS_FORCE_ALLOW_DATA_URI", + "type": { + "tag": "TD_UINT32" + }, + "value": 4194304 + }, + { + "name": "LOAD_FLAGS_IS_REDIRECT", + "type": { + "tag": "TD_UINT32" + }, + "value": 8388608 + }, + { + "name": "LOAD_FLAGS_DISABLE_TRR", + "type": { + "tag": "TD_UINT32" + }, + "value": 16777216 + }, + { + "name": "LOAD_FLAGS_FORCE_TRR", + "type": { + "tag": "TD_UINT32" + }, + "value": 33554432 + }, + { + "name": "LOAD_FLAGS_BYPASS_LOAD_URI_DELEGATE", + "type": { + "tag": "TD_UINT32" + }, + "value": 67108864 + }, + { + "name": "LOAD_FLAGS_USER_ACTIVATION", + "type": { + "tag": "TD_UINT32" + }, + "value": 134217728 + }, + { + "name": "STOP_NETWORK", + "type": { + "tag": "TD_UINT32" + }, + "value": 1 + }, + { + "name": "STOP_CONTENT", + "type": { + "tag": "TD_UINT32" + }, + "value": 2 + }, + { + "name": "STOP_ALL", + "type": { + "tag": "TD_UINT32" + }, + "value": 3 + } + ], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [ + "getter", + "hasretval" + ], + "name": "canGoBack", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "canGoForward", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "goBack", + "params": [ + { + "flags": [ + "in", + "optional" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in", + "optional" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "goForward", + "params": [ + { + "flags": [ + "in", + "optional" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in", + "optional" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "gotoIndex", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in", + "optional" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "jscontext" + ], + "name": "loadURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_JSVAL" + } + } + ] + }, + { + "flags": [], + "name": "binaryLoadURI", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [], + "name": "reload", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [], + "name": "stop", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "document", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "headerFile": "mozilla/dom/Document.h", + "name": "Document", + "native": "mozilla::dom::Document", + "tag": "TD_DOMOBJECT" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "currentURI", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIURI", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "sessionHistory", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "resumeRedirectedLoad", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT64" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + } + ], + "name": "nsIWebNavigation", + "parent": "nsISupports", + "uuid": "3ade79d4-8cb9-4952-b18d-4f9b63ca0d31" + }, + { + "consts": [ + { + "name": "UNSUPPORTED", + "type": { + "tag": "TD_UINT32" + }, + "value": 0 + }, + { + "name": "IMAGE", + "type": { + "tag": "TD_UINT32" + }, + "value": 1 + }, + { + "name": "FALLBACK", + "type": { + "tag": "TD_UINT32" + }, + "value": 2 + }, + { + "name": "OTHER", + "type": { + "tag": "TD_UINT32" + }, + "value": 32768 + } + ], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "isTypeSupported", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + } + ], + "name": "nsIWebNavigationInfo", + "parent": "nsISupports", + "uuid": "62a93afb-93a1-465c-84c8-0432264229de" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "loadPageAsViewSource", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDocShell", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "currentDescriptor", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsIWebPageDescriptor", + "parent": "nsISupports", + "uuid": "6f30b676-3710-4c2c-80b1-0395fb26516e" + } +]
\ No newline at end of file diff --git a/tools/lint/eslint/eslint-plugin-mozilla/tests/xpidl/xpcom_base.xpt b/tools/lint/eslint/eslint-plugin-mozilla/tests/xpidl/xpcom_base.xpt new file mode 100644 index 0000000000..8b3db4bdeb --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/tests/xpidl/xpcom_base.xpt @@ -0,0 +1,3165 @@ +[ + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "unloadTabAsync", + "params": [] + } + ], + "name": "nsITabUnloader", + "parent": "nsISupports", + "uuid": "2e530956-6054-464f-9f4c-0ae6f8de5523" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "registerTabUnloader", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsITabUnloader", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "onUnloadAttemptCompleted", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + } + ], + "name": "nsIAvailableMemoryWatcherBase", + "parent": "nsISupports", + "uuid": "b0b5701e-239d-49db-9009-37e89f86441c" + }, + { + "consts": [], + "flags": [ + "function" + ], + "methods": [ + { + "flags": [], + "name": "observe", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIConsoleMessage", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsIConsoleListener", + "parent": "nsISupports", + "uuid": "35c400a4-5792-438c-b915-65e30d58d557" + }, + { + "consts": [ + { + "name": "debug", + "type": { + "tag": "TD_UINT32" + }, + "value": 0 + }, + { + "name": "info", + "type": { + "tag": "TD_UINT32" + }, + "value": 1 + }, + { + "name": "warn", + "type": { + "tag": "TD_UINT32" + }, + "value": 2 + }, + { + "name": "error", + "type": { + "tag": "TD_UINT32" + }, + "value": 3 + } + ], + "flags": [], + "methods": [ + { + "flags": [ + "getter", + "hasretval" + ], + "name": "logLevel", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "timeStamp", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "microSecondTimeStamp", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "message", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isForwardedFromContentProcess", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "isForwardedFromContentProcess", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "toString", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UTF8STRING" + } + } + ] + } + ], + "name": "nsIConsoleMessage", + "parent": "nsISupports", + "uuid": "3aba9617-10e2-4839-83ae-2e6fc4df428b" + }, + { + "consts": [ + { + "name": "SuppressLog", + "type": { + "tag": "TD_UINT8" + }, + "value": 0 + }, + { + "name": "OutputToLog", + "type": { + "tag": "TD_UINT8" + }, + "value": 1 + } + ], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [], + "name": "logMessage", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIConsoleMessage", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "logMessageWithMode", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIConsoleMessage", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT8" + } + } + ] + }, + { + "flags": [], + "name": "logStringMessage", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PWSTRING" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "getMessageArray", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "element": { + "name": "nsIConsoleMessage", + "tag": "TD_INTERFACE_TYPE" + }, + "tag": "TD_ARRAY" + } + } + ] + }, + { + "flags": [], + "name": "registerListener", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIConsoleListener", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "unregisterListener", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIConsoleListener", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "reset", + "params": [] + }, + { + "flags": [], + "name": "resetWindow", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT64" + } + } + ] + } + ], + "name": "nsIConsoleService", + "parent": "nsISupports", + "uuid": "0eb81d20-c37e-42d4-82a8-ca9ae96bdf52" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "noteRefCountedObject", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + } + ] + }, + { + "flags": [], + "name": "noteGCedObject", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + } + ] + }, + { + "flags": [], + "name": "noteEdge", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + } + ] + }, + { + "flags": [], + "name": "describeRoot", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UINT32" + } + } + ] + }, + { + "flags": [], + "name": "describeGarbage", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + } + ] + } + ], + "name": "nsICycleCollectorHandler", + "parent": "nsISupports", + "uuid": "7f093367-1492-4b89-87af-c01dbc831246" + }, + { + "consts": [], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [ + "hidden" + ], + "name": "open", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [], + "name": "closeGCLog", + "params": [] + }, + { + "flags": [], + "name": "closeCCLog", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "filenameIdentifier", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "filenameIdentifier", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "processIdentifier", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "processIdentifier", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "gcLog", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIFile", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "ccLog", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIFile", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsICycleCollectorLogSink", + "parent": "nsISupports", + "uuid": "3ad9875f-d0e4-4ac2-87e3-f127f6c02ce1" + }, + { + "consts": [], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "allTraces", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsICycleCollectorListener", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "wantAllTraces", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "disableLog", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "disableLog", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "logSink", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsICycleCollectorLogSink", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "logSink", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsICycleCollectorLogSink", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "wantAfterProcessing", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "setter" + ], + "name": "wantAfterProcessing", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "processNext", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsICycleCollectorHandler", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden", + "hasretval" + ], + "name": "asLogger", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + } + ], + "name": "nsICycleCollectorListener", + "parent": "nsISupports", + "uuid": "703b53b6-24f6-40c6-9ea9-aeb2dc53d170" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isDebugBuild", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "assertionCount", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isDebuggerAttached", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "assertion", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "warning", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "break", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "abort", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [], + "name": "rustPanic", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + } + ] + }, + { + "flags": [], + "name": "rustLog", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_PSTRING" + } + } + ] + }, + { + "flags": [], + "name": "crashWithOOM", + "params": [] + } + ], + "name": "nsIDebug2", + "parent": "nsISupports", + "uuid": "9641dc15-10fb-42e3-a285-18be90a5c10b" + }, + { + "consts": [], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "filename", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "name", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "sourceId", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "lineNumber", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "columnNumber", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "sourceLine", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UTF8STRING" + } + } + ] + }, + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "asyncCause", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "asyncCaller", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIStackFrame", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "caller", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIStackFrame", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "getter", + "jscontext", + "hasretval" + ], + "name": "formattedStack", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "nativeSavedFrame", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_JSVAL" + } + } + ] + }, + { + "flags": [ + "jscontext", + "hasretval" + ], + "name": "toString", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UTF8STRING" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getFilename", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getName", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getSourceId", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getLineNumber", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getColumnNumber", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getAsyncCause", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getAsyncCaller", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getCaller", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getFormattedStack", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "toStringInfallible", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_UTF8STRING" + } + } + ] + } + ], + "name": "nsIStackFrame", + "parent": "nsISupports", + "uuid": "28bfb2a2-5ea6-4738-918b-049dc4d51f0b" + }, + { + "consts": [], + "flags": [ + "builtinclass" + ], + "methods": [], + "name": "nsIException", + "parent": "nsISupports", + "uuid": "4371b5bf-6845-487f-8d9d-3f1e4a9badd2" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "init", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIFile", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "initANSIFileDesc", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + }, + { + "flags": [], + "name": "write", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UTF8STRING" + } + } + ] + }, + { + "flags": [], + "name": "finish", + "params": [] + } + ], + "name": "nsIGZFileWriter", + "parent": "nsISupports", + "uuid": "6bd5642c-1b90-4499-ba4b-199f27efaba5" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "getInterface", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_NSID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "iid_is": 0, + "tag": "TD_INTERFACE_IS_TYPE" + } + } + ] + } + ], + "name": "nsIInterfaceRequestor", + "parent": "nsISupports", + "uuid": "033a1470-8b2a-11d3-af88-00a024ffc08c" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "policiesEnabled", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "jscontext", + "hasretval" + ], + "name": "readPreferences", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_JSVAL" + } + } + ] + } + ], + "name": "nsIMacPreferencesReader", + "parent": "nsISupports", + "uuid": "b0f20595-88ce-4738-a1a4-24de78eb8051" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "getter", + "hasretval" + ], + "name": "architecturesInBinary", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isTranslated", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + } + ], + "name": "nsIMacUtils", + "parent": "nsISupports", + "uuid": "5e9072d7-ff95-455e-9466-8af9841a72ec" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "heapMinimize", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hasretval" + ], + "name": "isLowMemoryPlatform", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + } + ], + "name": "nsIMemory", + "parent": "nsISupports", + "uuid": "1e004834-6d8f-425a-bc9c-a2812ed43bb7" + }, + { + "consts": [], + "flags": [ + "function" + ], + "methods": [ + { + "flags": [], + "name": "callback", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsIFinishDumpingCallback", + "parent": "nsISupports", + "uuid": "2dea18fc-fbfa-4bf7-ad45-0efaf5495f5e" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [], + "name": "onDump", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIFile", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIFile", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "onFinish", + "params": [] + } + ], + "name": "nsIDumpGCAndCCLogsCallback", + "parent": "nsISupports", + "uuid": "dc1b2b24-65bd-441b-b6bd-cb5825a7ed14" + }, + { + "consts": [], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [], + "name": "dumpMemoryReportsToNamedFile", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIFinishDumpingCallback", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "dumpMemoryInfoToTempDir", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "dumpGCAndCCLogsToFile", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIDumpGCAndCCLogsCallback", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "dumpGCAndCCLogsToSink", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsICycleCollectorLogSink", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsIMemoryInfoDumper", + "parent": "nsISupports", + "uuid": "48541b74-47ee-4a62-9557-7f4b809bda5c" + }, + { + "consts": [], + "flags": [ + "function" + ], + "methods": [ + { + "flags": [], + "name": "callback", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UTF8STRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT32" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT64" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_UTF8STRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsIHandleReportCallback", + "parent": "nsISupports", + "uuid": "62ef0e1c-dbd6-11e3-aa75-3c970e9f4238" + }, + { + "consts": [ + { + "name": "KIND_NONHEAP", + "type": { + "tag": "TD_INT32" + }, + "value": 0 + }, + { + "name": "KIND_HEAP", + "type": { + "tag": "TD_INT32" + }, + "value": 1 + }, + { + "name": "KIND_OTHER", + "type": { + "tag": "TD_INT32" + }, + "value": 2 + }, + { + "name": "UNITS_BYTES", + "type": { + "tag": "TD_INT32" + }, + "value": 0 + }, + { + "name": "UNITS_COUNT", + "type": { + "tag": "TD_INT32" + }, + "value": 1 + }, + { + "name": "UNITS_COUNT_CUMULATIVE", + "type": { + "tag": "TD_INT32" + }, + "value": 2 + }, + { + "name": "UNITS_PERCENTAGE", + "type": { + "tag": "TD_INT32" + }, + "value": 3 + } + ], + "flags": [], + "methods": [ + { + "flags": [], + "name": "collectReports", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIHandleReportCallback", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + } + ], + "name": "nsIMemoryReporter", + "parent": "nsISupports", + "uuid": "92a36db1-46bd-4fe6-988e-47db47236d8b" + }, + { + "consts": [], + "flags": [ + "function" + ], + "methods": [ + { + "flags": [], + "name": "callback", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsIFinishReportingCallback", + "parent": "nsISupports", + "uuid": "548b3909-c04d-4ca6-8466-b8bee3837457" + }, + { + "consts": [], + "flags": [ + "function" + ], + "methods": [ + { + "flags": [], + "name": "callback", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + } + ], + "name": "nsIHeapAllocatedCallback", + "parent": "nsISupports", + "uuid": "1a80cd0f-0d9e-4397-be69-68ad28fe5175" + }, + { + "consts": [], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [], + "name": "init", + "params": [] + }, + { + "flags": [], + "name": "registerStrongReporter", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIMemoryReporter", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "registerStrongAsyncReporter", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIMemoryReporter", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "registerWeakReporter", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIMemoryReporter", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "registerWeakAsyncReporter", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIMemoryReporter", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "unregisterStrongReporter", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIMemoryReporter", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "unregisterWeakReporter", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIMemoryReporter", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "blockRegistrationAndHideExistingReporters", + "params": [] + }, + { + "flags": [], + "name": "unblockRegistrationAndRestoreOriginalReporters", + "params": [] + }, + { + "flags": [], + "name": "registerStrongReporterEvenIfBlocked", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIMemoryReporter", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "getReports", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIHandleReportCallback", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIFinishReportingCallback", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getReportsExtended", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIHandleReportCallback", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIFinishReportingCallback", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_ASTRING" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "getReportsForThisProcessExtended", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIHandleReportCallback", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_BOOL" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsIFinishReportingCallback", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "in" + ], + "type": { + "name": "nsISupports", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "endReport", + "params": [] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "vsize", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "vsizeMaxContiguous", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "resident", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "residentFast", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "residentPeak", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "residentUnique", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "heapAllocated", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "heapOverheadFraction", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "JSMainRuntimeGCHeap", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "JSMainRuntimeTemporaryPeak", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "JSMainRuntimeCompartmentsSystem", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "JSMainRuntimeCompartmentsUser", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "JSMainRuntimeRealmsSystem", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "JSMainRuntimeRealmsUser", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "imagesContentUsedUncompressed", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "storageSQLite", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "lowMemoryEventsPhysical", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "ghostWindows", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "pageFaultsHard", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "hasMozMallocUsableSize", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isDMDEnabled", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [ + "getter", + "hasretval" + ], + "name": "isDMDRunning", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_BOOL" + } + } + ] + }, + { + "flags": [], + "name": "minimizeMemoryUsage", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "nsIRunnable", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + }, + { + "flags": [], + "name": "sizeOfTab", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "name": "mozIDOMWindowProxy", + "tag": "TD_INTERFACE_TYPE" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT64" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_DOUBLE" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_DOUBLE" + } + } + ] + } + ], + "name": "nsIMemoryReporterManager", + "parent": "nsISupports", + "uuid": "2998574d-8993-407a-b1a5-8ad7417653e1" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "QueryInterface", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_NSID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "iid_is": 0, + "tag": "TD_INTERFACE_IS_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "AddRef", + "params": [] + }, + { + "flags": [ + "hidden" + ], + "name": "Release", + "params": [] + } + ], + "name": "nsISupports", + "parent": null, + "uuid": "00000000-0000-0000-c000-000000000046" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "generateUUID", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_NSIDPTR" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "generateUUIDInPlace", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + } + ], + "name": "nsIUUIDGenerator", + "parent": "nsISupports", + "uuid": "138ad1b2-c694-41cc-b201-333ce936d8b8" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "compare", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_CSTRING" + } + }, + { + "flags": [ + "out" + ], + "type": { + "tag": "TD_INT32" + } + } + ] + } + ], + "name": "nsIVersionComparator", + "parent": "nsISupports", + "uuid": "e6cd620a-edbb-41d2-9e42-9a2ffc8107f3" + }, + { + "consts": [], + "flags": [ + "builtinclass" + ], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "QueryReferent", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_NSID" + } + }, + { + "flags": [ + "out" + ], + "type": { + "iid_is": 0, + "tag": "TD_INTERFACE_IS_TYPE" + } + } + ] + }, + { + "flags": [ + "hidden" + ], + "name": "sizeOfOnlyThis", + "params": [ + { + "flags": [ + "in" + ], + "type": { + "tag": "TD_VOID" + } + } + ] + } + ], + "name": "nsIWeakReference", + "parent": "nsISupports", + "uuid": "9188bc85-f92e-11d2-81ef-0060083a0bcf" + }, + { + "consts": [], + "flags": [], + "methods": [ + { + "flags": [ + "hasretval" + ], + "name": "GetWeakReference", + "params": [ + { + "flags": [ + "out" + ], + "type": { + "name": "nsIWeakReference", + "tag": "TD_INTERFACE_TYPE" + } + } + ] + } + ], + "name": "nsISupportsWeakReference", + "parent": "nsISupports", + "uuid": "9188bc86-f92e-11d2-81ef-0060083a0bcf" + } +] diff --git a/tools/lint/eslint/eslint-plugin-mozilla/update.sh b/tools/lint/eslint/eslint-plugin-mozilla/update.sh new file mode 100755 index 0000000000..240cfdb9c2 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/update.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Script to regenerate the npm packages used for eslint-plugin-mozilla by the builders. +# Requires + +# Force the scripts working directory to be projdir/tools/lint/eslint/eslint-plugin-mozilla. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR + +if [ -z "$TASKCLUSTER_ACCESS_TOKEN" -o -z "$TASKCLUSTER_CLIENT_ID" -o -z "$TASKCLUSTER_ROOT_URL" ]; then + echo "Please ensure you have run the taskcluster shell correctly to set" + echo "the TASKCLUSTER_ACCESS_TOKEN, TASKCLUSTER_CLIENT_ID and" + echo "TASKCLUSTER_ROOT_URL environment variables." + echo "See https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint/enabling-rules.html" + exit 1; +fi + +echo "" +echo "Removing node_modules and package-lock.json..." +# Move to the top-level directory. +rm -rf node_modules +rm package-lock.json + +echo "Installing modules for eslint-plugin-mozilla..." +../../../../mach npm install + +echo "Creating eslint-plugin-mozilla.tar.gz..." +tar cvz -f eslint-plugin-mozilla.tar.gz node_modules + +echo "Adding eslint-plugin-mozilla.tar.gz to tooltool..." +rm -f manifest.tt +../../../../python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint-plugin-mozilla.tar.gz --url="https://tooltool.mozilla-releng.net/" + +echo "Uploading eslint-plugin-mozilla.tar.gz to tooltool..." +../../../../python/mozbuild/mozbuild/action/tooltool.py upload --message "node_modules folder update for tools/lint/eslint/eslint-plugin-mozilla" --url="https://tooltool.mozilla-releng.net/" + +echo "Cleaning up..." +rm eslint-plugin-mozilla.tar.gz + +echo "" +echo "Update complete, please commit and check in your changes." diff --git a/tools/lint/eslint/eslint-plugin-spidermonkey-js/LICENSE b/tools/lint/eslint/eslint-plugin-spidermonkey-js/LICENSE new file mode 100644 index 0000000000..e87a115e46 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-spidermonkey-js/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/environments/self-hosted.js b/tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/environments/self-hosted.js new file mode 100644 index 0000000000..37ae42bfa3 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/environments/self-hosted.js @@ -0,0 +1,180 @@ +/** + * @fileoverview Add environment defaults to SpiderMonkey's self-hosted JS. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const path = require("path"); +const fs = require("fs"); + +let gRootDir = null; + +// Copied from `tools/lint/eslint/eslint-plugin-mozilla/lib/helpers.js`. +function getRootDir() { + if (!gRootDir) { + function searchUpForIgnore(dirName, filename) { + let parsed = path.parse(dirName); + while (parsed.root !== dirName) { + if (fs.existsSync(path.join(dirName, filename))) { + return dirName; + } + // Move up a level + dirName = parsed.dir; + parsed = path.parse(dirName); + } + return null; + } + + let possibleRoot = searchUpForIgnore( + path.dirname(module.filename), + ".eslintignore" + ); + if (!possibleRoot) { + possibleRoot = searchUpForIgnore(path.resolve(), ".eslintignore"); + } + if (!possibleRoot) { + possibleRoot = searchUpForIgnore(path.resolve(), "package.json"); + } + if (!possibleRoot) { + // We've couldn't find a root from the module or CWD, so lets just go + // for the CWD. We really don't want to throw if possible, as that + // tends to give confusing results when used with ESLint. + possibleRoot = process.cwd(); + } + + gRootDir = possibleRoot; + } + + return gRootDir; +} + +function tryReadFile(filePath) { + let absPath = path.join(getRootDir(), filePath); + if (!fs.existsSync(absPath)) { + // Safely handle the case when the file wasn't found, because throwing + // errors can lead to confusing result when used with ESLint. + return ""; + } + return fs.readFileSync(absPath, "utf-8"); +} + +// Search for top-level declarations, #defines, and #includes. +function addGlobalsFrom(dirName, fileName, globals) { + let filePath = path.join(dirName, fileName); + + // Definitions are separated by line. + let lines = tryReadFile(filePath).split("\n"); + + // We don't have to fully parse the source code, because it's formatted + // through "prettier", which means we know the exact code structure. + // + // |class| is disallowed in self-hosted code, so we don't have to handle it. + for (let line of lines) { + if ( + line.startsWith("function") || + line.startsWith("function*") || + line.startsWith("async function") || + line.startsWith("async function*") + ) { + let m = line.match(/^(?:async )?function(?:\*)?\s+([\w\$]+)\s*\(/); + if (m) { + globals[m[1]] = "readonly"; + } + } else if ( + line.startsWith("var") || + line.startsWith("let") || + line.startsWith("const") + ) { + let m = line.match(/^(?:var|let|const)\s+([\w\$]+)\s*[;=]/); + if (m) { + globals[m[1]] = "readonly"; + } + } else if (line.startsWith("#define")) { + let m = line.match(/^#define (\w+)/); + if (m) { + globals[m[1]] = "readonly"; + } + } else if (line.startsWith("#include")) { + let m = line.match(/^#include \"([\w\.]+)\"$/); + if (m) { + // Also process definitions from includes. + addGlobalsFrom(dirName, m[1], globals); + } + } + } +} + +function selfHostingDefines(dirName = "js/src/builtin/") { + let absDir = path.join(getRootDir(), dirName); + if (!fs.existsSync(absDir)) { + // See |tryReadFile| for why we avoid to throw any errors. + return {}; + } + + // Search sub-directories and js-files within |dirName|. + let dirs = []; + let jsFiles = []; + for (let name of fs.readdirSync(absDir)) { + let stat = fs.statSync(path.join(absDir, name)); + if (stat.isDirectory()) { + dirs.push(name); + } else if (stat.isFile() && name.endsWith(".js")) { + jsFiles.push(name); + } + } + + let globals = Object.create(null); + + // Process each js-file. + for (let jsFile of jsFiles) { + addGlobalsFrom(dirName, jsFile, globals); + } + + // Recursively traverse all sub-directories. + for (let dir of dirs) { + globals = { ...globals, ...selfHostingDefines(path.join(dirName, dir)) }; + } + + return globals; +} + +function selfHostingFunctions() { + // Definitions can be spread across multiple lines and may have extra + // whitespace, so we simply remove all whitespace and match over the complete + // file. + let content = tryReadFile("js/src/vm/SelfHosting.cpp").replace(/\s+/g, ""); + + let globals = Object.create(null); + for (let m of content.matchAll(/(?:JS_FN|JS_INLINABLE_FN)\("(\w+)"/g)) { + globals[m[1]] = "readonly"; + } + return globals; +} + +function errorNumbers() { + // Definitions are separated by line. + let lines = tryReadFile("js/public/friend/ErrorNumbers.msg").split("\n"); + + let globals = Object.create(null); + for (let line of lines) { + let m = line.match(/^MSG_DEF\((\w+),/); + if (m) { + globals[m[1]] = "readonly"; + } + } + return globals; +} + +const globals = { + ...selfHostingDefines(), + ...selfHostingFunctions(), + ...errorNumbers(), +}; + +module.exports = { + globals, +}; diff --git a/tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/index.js b/tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/index.js new file mode 100644 index 0000000000..d9d40af8c6 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/index.js @@ -0,0 +1,20 @@ +/** + * @fileoverview A processor to help parse the spidermonkey js code. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ------------------------------------------------------------------------------ +// Plugin Definition +// ------------------------------------------------------------------------------ +module.exports = { + processors: { + processor: require("../lib/processors/self-hosted"), + }, + environments: { + environment: require("../lib/environments/self-hosted"), + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/processors/self-hosted.js b/tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/processors/self-hosted.js new file mode 100644 index 0000000000..02201a329b --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/processors/self-hosted.js @@ -0,0 +1,129 @@ +/** + * @fileoverview Remove macros from SpiderMonkey's self-hosted JS. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const path = require("path"); +const fs = require("fs"); + +const selfHostedRegex = /js\/src\/builtin\/.*?\.js$/; +const macroRegex = + /\s*\#(if|ifdef|else|elif|endif|include|define|undef|error).*/; + +function isSelfHostedFile(filename) { + if (path.win32) { + filename = filename.split(path.sep).join("/"); + } + return selfHostedRegex.test(filename); +} + +function tryReadFile(filePath) { + if (!path.isAbsolute(filePath)) { + return ""; + } + if (!fs.existsSync(filePath)) { + // Safely handle the case when the file wasn't found, because throwing + // errors can lead to confusing result when used with ESLint. + return ""; + } + return fs.readFileSync(filePath, "utf-8"); +} + +// Adjust the range of fixes to match the original source code. +function createFix(lines, message) { + let { line, column, fix } = message; + + // Line and column are 1-based. Make sure we got a valid input. + if (line <= 0 || column <= 0) { + return null; + } + + // Reject to create a fix when the line is out of range for some reason. + if (line > lines.length) { + return null; + } + + // Find the absolute start position of the line in the original file. + let startOfLine = 0; + for (let i = 0; i < line - 1; ++i) { + // Add the length of the line, including its line separator. + startOfLine += lines[i].length + "\n".length; + } + + // Add the 1-based column to the start of line to get the start position. + let start = startOfLine + (column - 1); + + // Add the fix range to get the end position. + let end = start + (fix.range[1] - fix.range[0]); + + // And finally return the new fix object. + return { text: fix.text, range: [start, end] }; +} + +module.exports = { + preprocess(text, filename) { + if (!isSelfHostedFile(filename)) { + return [text]; + } + + let lines = text.split(/\n/); + for (let i = 0; i < lines.length; i++) { + if (!macroRegex.test(lines[i])) { + // No macro here, nothing to do. + continue; + } + + for (; i < lines.length; i++) { + // The macro isn't correctly indented, so we need to instruct + // prettier to ignore them. + lines[i] = "// prettier-ignore -- " + lines[i]; + + // If the line ends with a backslash (\), the next line + // is also part of part of the macro. + if (!lines[i].endsWith("\\")) { + break; + } + } + } + + return [lines.join("\n")]; + }, + + postprocess(messages, filename) { + // Don't attempt to create fixes for any non-selfhosted files. + if (!isSelfHostedFile(filename)) { + return [].concat(...messages); + } + + let lines = null; + + let result = []; + for (let message of messages.flat()) { + if (message.fix) { + if (lines === null) { + lines = tryReadFile(filename).split(/\n/); + } + + let fix = createFix(lines, message); + if (fix) { + message.fix = fix; + } else { + // We couldn't create a fix, so we better remove the passed in fix, + // because its range points to the preprocessor output, but the post- + // processor must translate it into a range of the original source. + delete message.fix; + } + } + + result.push(message); + } + return result; + }, + + supportsAutofix: true, +}; diff --git a/tools/lint/eslint/eslint-plugin-spidermonkey-js/package.json b/tools/lint/eslint/eslint-plugin-spidermonkey-js/package.json new file mode 100644 index 0000000000..7d96b5b1c4 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-spidermonkey-js/package.json @@ -0,0 +1,28 @@ +{ + "name": "eslint-plugin-spidermonkey-js", + "version": "0.1.1", + "description": "A collection of rules that help enforce JavaScript coding standard in the Mozilla SpiderMonkey project.", + "keywords": [ + "eslint", + "eslintplugin", + "eslint-plugin", + "mozilla", + "spidermonkey" + ], + "bugs": { + "url": "https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=Lint" + }, + "homepage": "http://firefox-source-docs.mozilla.org/tools/lint/linters/eslint-plugin-spidermonkey-js.html", + "repository": { + "type": "hg", + "url": "https://hg.mozilla.org/mozilla-central/" + }, + "author": "Mozilla", + "main": "lib/index.js", + "dependencies": {}, + "devDependencies": {}, + "engines": { + "node": ">=6.9.1" + }, + "license": "MPL-2.0" +} diff --git a/tools/lint/eslint/manifest.tt b/tools/lint/eslint/manifest.tt new file mode 100644 index 0000000000..c1ec658b0e --- /dev/null +++ b/tools/lint/eslint/manifest.tt @@ -0,0 +1,10 @@ +[ + { + "filename": "eslint.tar.gz", + "size": 22086762, + "algorithm": "sha512", + "digest": "abf5caa29669e1f8f889459b7af074f2744c7ba7e51901a1b3ad3e504c3f331d360616a0d9c1f0114034dce33874ca7ed3971a8ae7f38f21ae2a6b4514a0e5eb", + "unpack": true, + "visibility": "public" + } +]
\ No newline at end of file diff --git a/tools/lint/eslint/setup_helper.py b/tools/lint/eslint/setup_helper.py new file mode 100644 index 0000000000..2f2074d2cf --- /dev/null +++ b/tools/lint/eslint/setup_helper.py @@ -0,0 +1,423 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import platform +import re +import subprocess +import sys +from filecmp import dircmp + +from mozbuild.nodeutil import ( + NODE_MIN_VERSION, + NPM_MIN_VERSION, + find_node_executable, + find_npm_executable, +) +from mozfile.mozfile import remove as mozfileremove +from packaging.version import Version + +NODE_MACHING_VERSION_NOT_FOUND_MESSAGE = """ +Could not find Node.js executable later than %s. + +Executing `mach bootstrap --no-system-changes` should +install a compatible version in ~/.mozbuild on most platforms. +""".strip() + +NPM_MACHING_VERSION_NOT_FOUND_MESSAGE = """ +Could not find npm executable later than %s. + +Executing `mach bootstrap --no-system-changes` should +install a compatible version in ~/.mozbuild on most platforms. +""".strip() + +NODE_NOT_FOUND_MESSAGE = """ +nodejs is either not installed or is installed to a non-standard path. + +Executing `mach bootstrap --no-system-changes` should +install a compatible version in ~/.mozbuild on most platforms. +""".strip() + +NPM_NOT_FOUND_MESSAGE = """ +Node Package Manager (npm) is either not installed or installed to a +non-standard path. + +Executing `mach bootstrap --no-system-changes` should +install a compatible version in ~/.mozbuild on most platforms. +""".strip() + + +VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") +CARET_VERSION_RANGE_RE = re.compile(r"^\^((\d+)\.\d+\.\d+)$") + +project_root = None + + +def eslint_maybe_setup(): + """Setup ESLint only if it is needed.""" + has_issues, needs_clobber = eslint_module_needs_setup() + + if has_issues: + eslint_setup(needs_clobber) + + +def eslint_setup(should_clobber=False): + """Ensure eslint is optimally configured. + + This command will inspect your eslint configuration and + guide you through an interactive wizard helping you configure + eslint for optimal use on Mozilla projects. + """ + package_setup(get_project_root(), "eslint", should_clobber=should_clobber) + + +def remove_directory(path): + print("Clobbering %s..." % path) + if sys.platform.startswith("win") and have_winrm(): + process = subprocess.Popen(["winrm", "-rf", path]) + process.wait() + else: + mozfileremove(path) + + +def package_setup( + package_root, + package_name, + should_update=False, + should_clobber=False, + no_optional=False, +): + """Ensure `package_name` at `package_root` is installed. + + When `should_update` is true, clobber, install, and produce a new + "package-lock.json" file. + + This populates `package_root/node_modules`. + + """ + orig_project_root = get_project_root() + orig_cwd = os.getcwd() + + if should_update: + should_clobber = True + + try: + set_project_root(package_root) + sys.path.append(os.path.dirname(__file__)) + + # npm sometimes fails to respect cwd when it is run using check_call so + # we manually switch folders here instead. + project_root = get_project_root() + os.chdir(project_root) + + if should_clobber: + remove_directory(os.path.join(project_root, "node_modules")) + + # Always remove the eslint-plugin-mozilla sub-directory as that can + # sometimes conflict with the top level node_modules, see bug 1809036. + remove_directory( + os.path.join( + get_eslint_module_path(), "eslint-plugin-mozilla", "node_modules" + ) + ) + + npm_path, _ = find_npm_executable() + if not npm_path: + return 1 + + node_path, _ = find_node_executable() + if not node_path: + return 1 + + extra_parameters = ["--loglevel=error"] + + if no_optional: + extra_parameters.append("--no-optional") + + package_lock_json_path = os.path.join(get_project_root(), "package-lock.json") + + if should_update: + cmd = [npm_path, "install"] + mozfileremove(package_lock_json_path) + else: + cmd = [npm_path, "ci"] + + # On non-Windows, ensure npm is called via node, as node may not be in the + # path. + if platform.system() != "Windows": + cmd.insert(0, node_path) + + cmd.extend(extra_parameters) + + # Ensure that bare `node` and `npm` in scripts, including post-install scripts, finds the + # binary we're invoking with. Without this, it's easy for compiled extensions to get + # mismatched versions of the Node.js extension API. + path = os.environ.get("PATH", "").split(os.pathsep) + node_dir = os.path.dirname(node_path) + if node_dir not in path: + path = [node_dir] + path + + print('Installing %s for mach using "%s"...' % (package_name, " ".join(cmd))) + result = call_process( + package_name, cmd, append_env={"PATH": os.pathsep.join(path)} + ) + + if not result: + return 1 + + bin_path = os.path.join( + get_project_root(), "node_modules", ".bin", package_name + ) + + print("\n%s installed successfully!" % package_name) + print("\nNOTE: Your local %s binary is at %s\n" % (package_name, bin_path)) + + finally: + set_project_root(orig_project_root) + os.chdir(orig_cwd) + + +def call_process(name, cmd, cwd=None, append_env={}): + env = dict(os.environ) + env.update(append_env) + + try: + with open(os.devnull, "w") as fnull: + subprocess.check_call(cmd, cwd=cwd, stdout=fnull, env=env) + except subprocess.CalledProcessError: + if cwd: + print("\nError installing %s in the %s folder, aborting." % (name, cwd)) + else: + print("\nError installing %s, aborting." % name) + + return False + + return True + + +def expected_eslint_modules(): + # Read the expected version of ESLint and external modules + expected_modules_path = os.path.join(get_project_root(), "package.json") + with open(expected_modules_path, encoding="utf-8") as f: + sections = json.load(f) + expected_modules = sections.get("dependencies", {}) + expected_modules.update(sections.get("devDependencies", {})) + + # Also read the in-tree ESLint plugin mozilla information, to ensure the + # dependencies are up to date. + mozilla_json_path = os.path.join( + get_eslint_module_path(), "eslint-plugin-mozilla", "package.json" + ) + with open(mozilla_json_path, encoding="utf-8") as f: + dependencies = json.load(f).get("dependencies", {}) + expected_modules.update(dependencies) + + # Also read the in-tree ESLint plugin spidermonkey information, to ensure the + # dependencies are up to date. + mozilla_json_path = os.path.join( + get_eslint_module_path(), "eslint-plugin-spidermonkey-js", "package.json" + ) + with open(mozilla_json_path, encoding="utf-8") as f: + expected_modules.update(json.load(f).get("dependencies", {})) + + return expected_modules + + +def check_eslint_files(node_modules_path, name): + def check_file_diffs(dcmp): + # Diff files only looks at files that are different. Not for files + # that are only present on one side. This should be generally OK as + # new files will need to be added in the index.js for the package. + if dcmp.diff_files and dcmp.diff_files != ["package.json"]: + return True + + result = False + + # Again, we only look at common sub directories for the same reason + # as above. + for sub_dcmp in dcmp.subdirs.values(): + result = result or check_file_diffs(sub_dcmp) + + return result + + dcmp = dircmp( + os.path.join(node_modules_path, name), + os.path.join(get_eslint_module_path(), name), + ) + + return check_file_diffs(dcmp) + + +def eslint_module_needs_setup(): + has_issues = False + needs_clobber = False + node_modules_path = os.path.join(get_project_root(), "node_modules") + + for name, expected_data in expected_eslint_modules().items(): + # expected_eslint_modules returns a string for the version number of + # dependencies for installation of eslint generally, and an object + # for our in-tree plugins (which contains the entire module info). + if "version" in expected_data: + version_range = expected_data["version"] + else: + version_range = expected_data + + path = os.path.join(node_modules_path, name, "package.json") + + if not os.path.exists(path): + print("%s v%s needs to be installed locally." % (name, version_range)) + has_issues = True + continue + data = json.load(open(path, encoding="utf-8")) + + if version_range.startswith("file:"): + # We don't need to check local file installations for versions, as + # these are symlinked, so we'll always pick up the latest. + continue + + if name == "eslint" and Version("4.0.0") > Version(data["version"]): + print("ESLint is an old version, clobbering node_modules directory") + needs_clobber = True + has_issues = True + continue + + if not version_in_range(data["version"], version_range): + print("%s v%s should be v%s." % (name, data["version"], version_range)) + has_issues = True + continue + + return has_issues, needs_clobber + + +def version_in_range(version, version_range): + """ + Check if a module version is inside a version range. Only supports explicit versions and + caret ranges for the moment, since that's all we've used so far. + """ + if version == version_range: + return True + + version_match = VERSION_RE.match(version) + if not version_match: + raise RuntimeError("mach eslint doesn't understand module version %s" % version) + version = Version(version) + + # Caret ranges as specified by npm allow changes that do not modify the left-most non-zero + # digit in the [major, minor, patch] tuple. The code below assumes the major digit is + # non-zero. + range_match = CARET_VERSION_RANGE_RE.match(version_range) + if range_match: + range_version = range_match.group(1) + range_major = int(range_match.group(2)) + + range_min = Version(range_version) + range_max = Version("%d.0.0" % (range_major + 1)) + + return range_min <= version < range_max + + return False + + +def get_possible_node_paths_win(): + """ + Return possible nodejs paths on Windows. + """ + if platform.system() != "Windows": + return [] + + return list( + { + "%s\\nodejs" % os.environ.get("SystemDrive"), + os.path.join(os.environ.get("ProgramFiles"), "nodejs"), + os.path.join(os.environ.get("PROGRAMW6432"), "nodejs"), + os.path.join(os.environ.get("PROGRAMFILES"), "nodejs"), + } + ) + + +def get_version(path): + try: + version_str = subprocess.check_output( + [path, "--version"], stderr=subprocess.STDOUT, universal_newlines=True + ) + return version_str + except (subprocess.CalledProcessError, OSError): + return None + + +def set_project_root(root=None): + """Sets the project root to the supplied path, or works out what the root + is based on looking for 'mach'. + + Keyword arguments: + root - (optional) The path to set the root to. + """ + global project_root + + if root: + project_root = root + return + + file_found = False + folder = os.getcwd() + + while folder: + if os.path.exists(os.path.join(folder, "mach")): + file_found = True + break + else: + folder = os.path.dirname(folder) + + if file_found: + project_root = os.path.abspath(folder) + + +def get_project_root(): + """Returns the absolute path to the root of the project, see set_project_root() + for how this is determined. + """ + global project_root + + if not project_root: + set_project_root() + + return project_root + + +def get_eslint_module_path(): + return os.path.join(get_project_root(), "tools", "lint", "eslint") + + +def check_node_executables_valid(): + node_path, version = find_node_executable() + if not node_path: + print(NODE_NOT_FOUND_MESSAGE) + return False + if not version: + print(NODE_MACHING_VERSION_NOT_FOUND_MESSAGE % NODE_MIN_VERSION) + return False + + npm_path, version = find_npm_executable() + if not npm_path: + print(NPM_NOT_FOUND_MESSAGE) + return False + if not version: + print(NPM_MACHING_VERSION_NOT_FOUND_MESSAGE % NPM_MIN_VERSION) + return False + + return True + + +def have_winrm(): + # `winrm -h` should print 'winrm version ...' and exit 1 + try: + p = subprocess.Popen( + ["winrm.exe", "-h"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + return p.wait() == 1 and p.stdout.read().startswith("winrm") + except Exception: + return False diff --git a/tools/lint/eslint/update.sh b/tools/lint/eslint/update.sh new file mode 100755 index 0000000000..025f747135 --- /dev/null +++ b/tools/lint/eslint/update.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Script to regenerate the npm packages used for ESLint by the builders. +# Requires + +# Force the scripts working directory to be projdir/tools/lint/eslint. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR + +if [ -z "$TASKCLUSTER_ACCESS_TOKEN" -o -z "$TASKCLUSTER_CLIENT_ID" -o -z "$TASKCLUSTER_ROOT_URL" ]; then + echo "Please ensure you have run the taskcluster shell correctly to set" + echo "the TASKCLUSTER_ACCESS_TOKEN, TASKCLUSTER_CLIENT_ID and" + echo "TASKCLUSTER_ROOT_URL environment variables." + echo "See https://firefox-source-docs.mozilla.org/code-quality/lint/linters/eslint/enabling-rules.html" + exit 1; +fi + +echo "" +echo "Removing node_modules and package-lock.json..." +# Move to the top-level directory. +cd ../../../ +rm -rf node_modules/ +rm -rf tools/lint/eslint/eslint-plugin-mozilla/node_modules +rm package-lock.json + +echo "Installing eslint and external plugins..." +# ESLint and all _external_ plugins are listed in this directory's package.json, +# so a regular `npm install` will install them at the specified versions. +# The in-tree eslint-plugin-mozilla is kept out of this tooltool archive on +# purpose so that it can be changed by any developer without requiring tooltool +# access to make changes. +./mach npm install + +echo "Creating eslint.tar.gz..." +tar cvz --exclude=eslint-plugin-mozilla --exclude=eslint-plugin-spidermonkey-js -f eslint.tar.gz node_modules + +echo "Adding eslint.tar.gz to tooltool..." +rm tools/lint/eslint/manifest.tt +./python/mozbuild/mozbuild/action/tooltool.py add --visibility public --unpack eslint.tar.gz + +echo "Uploading eslint.tar.gz to tooltool..." +./python/mozbuild/mozbuild/action/tooltool.py upload --message "node_modules folder update for tools/lint/eslint" + +echo "Cleaning up..." +mv manifest.tt tools/lint/eslint/manifest.tt +rm eslint.tar.gz + +cd $DIR + +echo "" +echo "Update complete, please commit and check in your changes." diff --git a/tools/lint/file-perm.yml b/tools/lint/file-perm.yml new file mode 100644 index 0000000000..566472e7fb --- /dev/null +++ b/tools/lint/file-perm.yml @@ -0,0 +1,56 @@ +--- +file-perm: + description: File permission check + include: + - . + extensions: + - .build + - .c + - .cc + - .cpp + - .flac + - .h + - .html + - .idl + - .js + - .jsm + - .json + - .jsx + - .m + - .m4s + - .md + - .mjs + - .mm + - .mn + - .mozbuild + - .mp4 + - .png + - .rs + - .rst + - .svg + - .toml + - .ttf + - .wasm + - .webidl + - .xhtml + - .xml + - .yaml + - .yml + support-files: + - 'tools/lint/file-perm/**' + type: external + payload: file-perm:lint + +maybe-shebang-file-perm: + description: "File permission check for files that might have `#!` header." + include: + - . + allow-shebang: true + extensions: + - .js + - .py + - .sh + support-files: + - 'tools/lint/file-perm/**' + type: external + payload: file-perm:lint diff --git a/tools/lint/file-perm/__init__.py b/tools/lint/file-perm/__init__.py new file mode 100644 index 0000000000..c5f31c008c --- /dev/null +++ b/tools/lint/file-perm/__init__.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import platform + +from mozlint import result +from mozlint.pathutils import expand_exclusions + + +def lint(paths, config, fix=None, **lintargs): + results = [] + fixed = 0 + + if platform.system() == "Windows": + # Windows doesn't have permissions in files + # Exit now + return {"results": results, "fixed": fixed} + + files = list(expand_exclusions(paths, config, lintargs["root"])) + for f in files: + if os.access(f, os.X_OK): + if config.get("allow-shebang"): + with open(f, "r+") as content: + # Some source files have +x permissions + line = content.readline() + if line.startswith("#!"): + # Check if the file doesn't start with a shebang + # if it does, not a warning + continue + + if fix: + # We want to fix it, do it and leave + os.chmod(f, 0o644) + fixed += 1 + continue + + res = { + "path": f, + "message": "Execution permissions on a source file", + "level": "error", + } + results.append(result.from_config(config, **res)) + return {"results": results, "fixed": fixed} diff --git a/tools/lint/file-whitespace.yml b/tools/lint/file-whitespace.yml new file mode 100644 index 0000000000..91a8ed2103 --- /dev/null +++ b/tools/lint/file-whitespace.yml @@ -0,0 +1,176 @@ +--- +file-whitespace: + description: File content sanity check + include: + - . + - tools/lint/python/black_requirements.txt + - tools/lint/python/ruff_requirements.txt + - tools/lint/rst/requirements.txt + - tools/lint/tox/tox_requirements.txt + - tools/lint/spell/codespell_requirements.txt + exclude: + - accessible/tests/crashtests + - accessible/tests/mochitest + - browser/locales/en-US/chrome/browser/uiDensity.properties + - build/pgo/blueprint + - build/pgo/js-input + - devtools/client/debugger/test + - devtools/client/inspector/markup/test + - devtools/client/inspector/rules/test + - devtools/client/inspector/test + # Excluded because of python json output. + - testing/talos/talos/unittests/test_talosconfig_browser_config.json + - testing/talos/talos/unittests/test_talosconfig_test_config.json + # Excluded because tests were failing unexpectedly + - devtools/client/styleeditor/test/sync_with_csp.css + - devtools/client/webconsole/test/browser/test-message-categories-css-parser.css + - devtools/shared/webconsole/test/chrome/test_object_actor_native_getters.html + - docshell/base/crashtests + - docshell/test + - dom/base/crashtests + - dom/base/test + - dom/canvas/crashtests + - dom/canvas/test + - dom/events/crashtests + - dom/events/test + - dom/file/tests/crashtests/1748342.html + - dom/file/tests/file_mozfiledataurl_inner.html + - dom/html/crashtests + - dom/html/reftests + - dom/html/test + - dom/jsurl/crashtests/344996-1.xhtml + - dom/jsurl/test + - dom/media/mediasource/test/crashtests/926665.html + - dom/media/test + - dom/media/tests + - dom/media/webaudio/test + - dom/media/webrtc/transport/nricectx.cpp + - dom/media/webspeech/synth/test + - dom/plugins/test + - dom/smil/crashtests + - dom/smil/test + - dom/security/test + - dom/svg/crashtests + - dom/svg/test + - dom/webauthn/winwebauthn/webauthn.h + - dom/tests/mochitest + - dom/xml/crashtests + - dom/xml/test + - dom/xslt/crashtests + - dom/xslt/tests + - dom/xul/crashtests + - dom/xul/test + - editor/composer/test + - editor/composer/crashtests/removing-editable-xslt.html + - editor/libeditor/tests + - editor/libeditor/crashtests + - editor/reftests + - extensions/universalchardet + - gfx/tests/crashtests + - gfx/vr/nsFxrCommandLineHandler.cpp + - image/test/crashtests + - image/test/mochitest + - image/test/reftest + - intl/components/gtest/TestNumberRangeFormat.cpp + - intl/icu_segmenter_data + - intl/lwbrk/crashtests + - intl/lwbrk/rulebrk.c + - intl/uconv/crashtests + - intl/uconv/tests + - intl/strres/tests/unit/397093.properties + - intl/strres/tests/unit/strres.properties + - js/xpconnect/crashtests + - js/xpconnect/tests + - js/src/builtin/intl/IcuMemoryUsage.java + - js/src/frontend/BytecodeEmitter.cpp + - js/src/frontend/SharedContext.h + - layout/base/crashtests + - layout/base/tests + - layout/forms/crashtests + - layout/forms/test + - layout/generic/crashtests + - layout/generic/test + - layout/inspector/tests + - layout/painting/crashtests/1405881-1.html + - layout/painting/crashtests/1407470-1.html + - layout/reftests + - layout/style/crashtests + - layout/style/test + - layout/svg/crashtests + - layout/tables/test/test_bug337124.html + - layout/tables/crashtests + - layout/xul/crashtests + - layout/xul/reftest + - layout/xul/test + - layout/xul/tree + - modules/libjar/zipwriter/test/unit/data/test_bug399727.html + - netwerk/test/crashtests + - netwerk/test/mochitests/test1.css + - netwerk/test/mochitests/test2.css + - parser/htmlparser/tests + - parser/html/java/named-character-references.html + - parser/html/javasrc + - testing/perfdocs/generated/ + - testing/talos/talos/pageloader/chrome/pageloader.xhtml + - testing/talos/talos/tests + - testing/web-platform/mozilla/tests/editor + - testing/web-platform/mozilla/tests/focus + - testing/web-platform/tests + - testing/web-platform/tests/conformance-checkers + - testing/web-platform/tests/content-security-policy + - testing/web-platform/tests/html + - testing/web-platform/tests/tools/lint/tests/dummy/broken.html + - testing/web-platform/tests/tools/lint/tests/dummy/broken_ignored.html + - toolkit/components/passwordmgr/test/mochitest/ + - toolkit/content/tests/chrome + - toolkit/locales/en-US/chrome/passwordmgr/passwordmgr.properties + - tools/jprof/README.html + - tools/lint/eslint + - tools/lint/test/test_clang_format.py + - view/crashtests + - widget/cocoa/crashtests + - widget/nsFilePickerProxy.cpp + - widget/tests + - widget/windows/tests/TestUrisToValidate.h + - xpcom/reflect/xptcall/porting.html + - xpcom/tests/test.properties + - xpcom/tests/unit/data/bug121341.properties + # Excluded below files because tests were failing unexpectedly + - dom/bindings/test/test_barewordGetsWindow.html + - devtools/client/styleeditor/test/sourcemap-css/sourcemaps.css + - devtools/client/styleeditor/test/sourcemap-css/contained.css + - devtools/client/styleeditor/test/sourcemap-css/test-stylus.css + - dom/bindings/test/file_barewordGetsWindow_frame1.html + - dom/bindings/test/file_barewordGetsWindow_frame2.html + - devtools/perfdocs/index.rst + - python/mozperftest/perfdocs/running.rst + - python/mozperftest/perfdocs/vision.rst + - python/mozperftest/perfdocs/writing.rst + extensions: + - .c + - .cc + - .cpp + - .css + - .dtd + - .idl + - .ftl + - .h + - .html + - .java + - .json + - .kt + - .md + - .mn + - .properties + - .py + - .rs + - .rst + - .toml + - .webidl + - .yaml + - .yml + - .xhtml + support-files: + - 'tools/lint/file-whitespace/**' + type: external + payload: file-whitespace:lint diff --git a/tools/lint/file-whitespace/__init__.py b/tools/lint/file-whitespace/__init__.py new file mode 100644 index 0000000000..ab7a6944b7 --- /dev/null +++ b/tools/lint/file-whitespace/__init__.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozlint import result +from mozlint.pathutils import expand_exclusions + +results = [] + + +def lint(paths, config, fix=None, **lintargs): + files = list(expand_exclusions(paths, config, lintargs["root"])) + log = lintargs["log"] + fixed = 0 + + for f in files: + with open(f, "rb") as open_file: + hasFix = False + content_to_write = [] + + try: + lines = open_file.readlines() + # Check for Empty spaces or newline character at end of file + if lines[:].__len__() != 0 and lines[-1:][0].strip().__len__() == 0: + # return file pointer to first + open_file.seek(0) + if fix: + fixed += 1 + # fix Empty lines at end of file + for i, line in reversed(list(enumerate(open_file))): + # determine if line is empty + if line.strip() != b"": + with open(f, "wb") as write_file: + # determine if file's last line have \n, if not then add a \n + if not lines[i].endswith(b"\n"): + lines[i] = lines[i] + b"\n" + # write content to file + for e in lines[: i + 1]: + write_file.write(e) + # end the loop + break + else: + res = { + "path": f, + "message": "Empty Lines at end of file", + "level": "error", + "lineno": open_file.readlines()[:].__len__(), + } + results.append(result.from_config(config, **res)) + except Exception as ex: + log.debug("Error: " + str(ex) + ", in file: " + f) + + # return file pointer to first + open_file.seek(0) + + lines = open_file.readlines() + # Detect missing newline at the end of the file + if lines[:].__len__() != 0 and not lines[-1].endswith(b"\n"): + if fix: + fixed += 1 + with open(f, "wb") as write_file: + # add a newline character at end of file + lines[-1] = lines[-1] + b"\n" + # write content to file + for e in lines: + write_file.write(e) + else: + res = { + "path": f, + "message": "File does not end with newline character", + "level": "error", + "lineno": lines.__len__(), + } + results.append(result.from_config(config, **res)) + + # return file pointer to first + open_file.seek(0) + + for i, line in enumerate(open_file): + if line.endswith(b" \n"): + # We found a trailing whitespace + if fix: + # We want to fix it, strip the trailing spaces + content_to_write.append(line.rstrip() + b"\n") + fixed += 1 + hasFix = True + else: + res = { + "path": f, + "message": "Trailing whitespace", + "level": "error", + "lineno": i + 1, + } + results.append(result.from_config(config, **res)) + else: + if fix: + content_to_write.append(line) + if hasFix: + # Only update the file when we found a change to make + with open(f, "wb") as open_file_to_write: + open_file_to_write.write(b"".join(content_to_write)) + + # We are still using the same fp, let's return to the first + # line + open_file.seek(0) + # Open it as once as we just need to know if there is + # at least one \r\n + content = open_file.read() + + if b"\r\n" in content: + if fix: + fixed += 1 + content = content.replace(b"\r\n", b"\n") + with open(f, "wb") as open_file_to_write: + open_file_to_write.write(content) + else: + res = { + "path": f, + "message": "Windows line return", + "level": "error", + } + results.append(result.from_config(config, **res)) + + return {"results": results, "fixed": fixed} diff --git a/tools/lint/fluent-lint.yml b/tools/lint/fluent-lint.yml new file mode 100644 index 0000000000..8d8a5d956b --- /dev/null +++ b/tools/lint/fluent-lint.yml @@ -0,0 +1,13 @@ +--- +fluent-lint: + description: Linter for Fluent files + exclude: + - dom/l10n/tests/mochitest/document_l10n/non-system-principal/localization/test.ftl + extensions: ['ftl'] + support-files: + - 'tools/lint/fluent-lint**' + brand-files: + - 'browser/branding/official/locales/en-US/brand.ftl' + - 'toolkit/locales/en-US/toolkit/branding/brandings.ftl' + type: external + payload: fluent-lint:lint diff --git a/tools/lint/fluent-lint/__init__.py b/tools/lint/fluent-lint/__init__.py new file mode 100644 index 0000000000..3b4c3c570b --- /dev/null +++ b/tools/lint/fluent-lint/__init__.py @@ -0,0 +1,531 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import bisect +import os +import re +from html.parser import HTMLParser + +import mozpack.path as mozpath +import yaml +from fluent.syntax import ast, parse, visitor +from mozlint import result +from mozlint.pathutils import expand_exclusions + + +class TextElementHTMLParser(HTMLParser): + """HTML Parser for TextElement. + + TextElements may contain embedded html tags, which can include + quotes in attributes. We only want to check the actual text. + """ + + def __init__(self): + super().__init__() + self.extracted_text = [] + + def handle_data(self, data): + self.extracted_text.append(data) + + +class Linter(visitor.Visitor): + """Fluent linter implementation. + + This subclasses the Fluent AST visitor. Methods are called corresponding + to each type of node in the Fluent AST. It is possible to control + whether a node is recursed into by calling the generic_visit method on + the superclass. + + See the documentation here: + https://www.projectfluent.org/python-fluent/fluent.syntax/stable/usage.html + """ + + def __init__( + self, path, config, exclusions, contents, offsets_and_lines, brand_names=[] + ): + super().__init__() + self.path = path + self.config = config + self.exclusions = exclusions + self.contents = contents + self.offsets_and_lines = offsets_and_lines + + self.results = [] + self.identifier_re = re.compile(r"[a-z0-9-]+") + self.apostrophe_re = re.compile(r"\w'") + self.incorrect_apostrophe_re = re.compile(r"\w\u2018\w") + self.single_quote_re = re.compile(r"'(.+)'") + self.double_quote_re = re.compile(r"\".+\"") + self.ellipsis_re = re.compile(r"\.\.\.") + + self.brand_names = brand_names + self.minimum_id_length = 9 + + self.state = { + # The resource comment should be at the top of the page after the license. + "node_can_be_resource_comment": True, + # Group comments must be followed by a message. Two group comments are not + # allowed in a row. + "can_have_group_comment": True, + # Comment bound to the current message + "comment": "", + # The current group comment + "group_comment": "", + # Variables in the current message + "variables": [], + } + + attributes = [ + "label", + "value", + "accesskey", + "alt", + "title", + "tooltiptext", + "placeholder", + "aria-label", + "aria-description", + "aria-valuetext", + "style", + # For XUL key/command setup. + "key", + "keycode", + # For download filenames: + "download", + # Used in the Firefox prefs + "searchkeywords", + # Used by search-textbox.js + "searchbuttonlabel", + # Used in toolbar customization. + "toolbarname", + # Used in moz-message-bar. + "message", + # Used in dialogs (should be moved to using fluent IDs though) + "buttonlabelaccept", + "buttonaccesskeyaccept", + "buttonlabelcancel", + "buttonaccesskeycancel", + "buttonlabelextra2", + "buttonaccesskeyextra2", + # Used in app menu notifications (should be moved to use fluent IDs) + "buttonlabel", + "buttonaccesskey", + "secondarybuttonlabel", + "secondarybuttonaccesskey", + # Commonly used in Lit-based web components + "heading", + "description", + ] + self.known_attribute_list = [a.lower() for a in attributes] + + # Set this to true to debug print the root node's json. This is useful for + # writing new lint rules, or debugging existing ones. + self.debug_print_json = False + + def generic_visit(self, node): + node_name = type(node).__name__ + self.state["node_can_be_resource_comment"] = self.state[ + "node_can_be_resource_comment" + ] and ( + # This is the root node. + node_name == "Resource" + # Empty space is allowed. + or node_name == "Span" + # Comments are allowed + or node_name == "Comment" + ) + + if self.debug_print_json: + import json + + print(json.dumps(node.to_json(), indent=2)) + # Only debug print the root node. + self.debug_print_json = False + + super(Linter, self).generic_visit(node) + + def visit_Attribute(self, node): + # Only visit values for Attribute nodes, the identifier comes from dom. + super().generic_visit(node.value) + + def visit_FunctionReference(self, node): + # We don't recurse into function references, the identifiers there are + # allowed to be free form. + pass + + def visit_Message(self, node): + # There must be at least one message or term between group comments. + self.state["can_have_group_comment"] = True + self.last_message_id = node.id.name + + super().generic_visit(node) + + # Do this here instead as visit_Attribute doesn't have access to the + # message's comment. + for attr in node.attributes: + if not attr.id.name.lower() in self.known_attribute_list: + comment = self.state["comment"] + self.state["group_comment"] + if not f".{attr.id.name}" in comment: + self.add_error( + attr, + "VA01", + "Use attributes designed for localized content directly." + " If script-based processing is necessary, add a comment" + f" explaining why. The linter didn't recognize: .{attr.id.name}", + "warning", + ) + + # Check if variables are referenced in comments + if self.state["variables"]: + comments = self.state["comment"] + self.state["group_comment"] + missing_references = [ + v for v in self.state["variables"] if f"${v}" not in comments + ] + if missing_references: + self.add_error( + node, + "VC01", + "Messages including variables should have a comment " + "explaining what will replace the variable. " + "Missing references: " + + ", ".join([f"${m}" for m in missing_references]), + ) + + # Reset current comment and variable references after reading the + # message. + self.state["comment"] = "" + self.state["variables"] = [] + + def visit_Term(self, node): + # There must be at least one message or term between group comments. + self.state["can_have_group_comment"] = True + self.last_message_id = None + + super().generic_visit(node) + + # Reset current comment and variable references after reading the term. + self.state["comment"] = "" + self.state["variables"] = [] + + def visit_MessageReference(self, node): + # We don't recurse into message references, the identifiers are either + # checked elsewhere or are attributes and come from DOM. + pass + + def visit_Identifier(self, node): + if ( + self.path not in self.exclusions["ID01"]["files"] + and node.name not in self.exclusions["ID01"]["messages"] + and not self.identifier_re.fullmatch(node.name) + ): + self.add_error( + node, + "ID01", + "Identifiers may only contain lowercase characters and -", + ) + if ( + len(node.name) < self.minimum_id_length + and self.path not in self.exclusions["ID02"]["files"] + and node.name not in self.exclusions["ID02"]["messages"] + ): + self.add_error( + node, + "ID02", + f"Identifiers must be at least {self.minimum_id_length} characters long", + ) + + def visit_TextElement(self, node): + parser = TextElementHTMLParser() + parser.feed(node.value) + for text in parser.extracted_text: + # To check for apostrophes, first remove pairs of straight quotes + # used as delimiters. + cleaned_str = re.sub(self.single_quote_re, "\1", node.value) + if self.apostrophe_re.search(cleaned_str): + self.add_error( + node, + "TE01", + "Strings with apostrophes should use foo\u2019s instead of foo's.", + ) + if self.incorrect_apostrophe_re.search(text): + self.add_error( + node, + "TE02", + "Strings with apostrophes should use foo\u2019s instead of foo\u2018s.", + ) + if self.single_quote_re.search(text): + self.add_error( + node, + "TE03", + "Single-quoted strings should use Unicode \u2018foo\u2019 instead of 'foo'.", + ) + if self.double_quote_re.search(text): + self.add_error( + node, + "TE04", + 'Double-quoted strings should use Unicode \u201cfoo\u201d instead of "foo".', + ) + if self.ellipsis_re.search(text): + self.add_error( + node, + "TE05", + "Strings with an ellipsis should use the Unicode \u2026 character" + " instead of three periods", + ) + + # If part of a message, check for brand names + if ( + self.last_message_id is not None + and self.path not in self.exclusions["CO01"]["files"] + and self.last_message_id not in self.exclusions["CO01"]["messages"] + ): + found_brands = [] + for brand in self.brand_names: + if brand in text: + found_brands.append(brand) + if found_brands: + self.add_error( + node, + "CO01", + "Strings should use the corresponding terms instead of" + f" hard-coded brand names ({', '.join(found_brands)})", + ) + + def visit_ResourceComment(self, node): + # This node is a comment with: "###" + if not self.state["node_can_be_resource_comment"]: + self.add_error( + node, + "RC01", + "Resource comments (###) should be placed at the top of the file, just " + "after the license header. There should only be one resource comment " + "per file.", + ) + return + + lines_after = get_newlines_count_after(node.span, self.contents) + lines_before = get_newlines_count_before(node.span, self.contents) + + if node.span.end == len(self.contents) - 1: + # This file only contains a resource comment. + return + + if lines_after != 2: + self.add_error( + node, + "RC02", + "Resource comments (###) should be followed by one empty line.", + ) + return + + if lines_before != 2: + self.add_error( + node, + "RC03", + "Resource comments (###) should have one empty line above them.", + ) + return + + def visit_SelectExpression(self, node): + # We only want to visit the variant values, the identifiers in selectors + # and keys are allowed to be free form. + for variant in node.variants: + super().generic_visit(variant.value) + + # Store the variable used for the SelectExpression, excluding functions + # like PLATFORM() + if ( + type(node.selector) == ast.VariableReference + and node.selector.id.name not in self.state["variables"] + ): + self.state["variables"].append(node.selector.id.name) + + def visit_Comment(self, node): + # This node is a comment with: "#" + + # Store the comment + self.state["comment"] = node.content + + def visit_GroupComment(self, node): + # This node is a comment with: "##" + + # Store the group comment + self.state["group_comment"] = node.content + + if not self.state["can_have_group_comment"]: + self.add_error( + node, + "GC04", + "Group comments (##) must be followed by at least one message " + "or term. Make sure that a single group comment with multiple " + "paragraphs is not separated by whitespace, as it will be " + "interpreted as two different comments.", + ) + return + + self.state["can_have_group_comment"] = False + + lines_after = get_newlines_count_after(node.span, self.contents) + lines_before = get_newlines_count_before(node.span, self.contents) + + if node.span.end == len(self.contents) - 1: + # The group comment is the last thing in the file. + + if node.content == "": + # Empty comments are allowed at the end of the file. + return + + self.add_error( + node, + "GC01", + "Group comments (##) should not be at the end of the file, they should " + "always be above a message. Only an empty group comment is allowed at " + "the end of a file.", + ) + return + + if lines_after != 2: + self.add_error( + node, + "GC02", + "Group comments (##) should be followed by one empty line.", + ) + return + + if lines_before != 2: + self.add_error( + node, + "GC03", + "Group comments (##) should have an empty line before them.", + ) + return + + def visit_VariableReference(self, node): + # Identifiers are allowed to be free form, but need to store them + # for comment checks. + + if node.id.name not in self.state["variables"]: + self.state["variables"].append(node.id.name) + + def add_error(self, node, rule, msg, level=None): + (col, line) = self.span_to_line_and_col(node.span) + res = { + "path": self.path, + "lineno": line, + "column": col, + "rule": rule, + "message": msg, + } + if level: + res["level"] = level + + self.results.append(result.from_config(self.config, **res)) + + def span_to_line_and_col(self, span): + i = bisect.bisect_left(self.offsets_and_lines, (span.start, 0)) + if i > 0: + col = span.start - self.offsets_and_lines[i - 1][0] + else: + col = 1 + span.start + return (col, self.offsets_and_lines[i][1]) + + +def get_offsets_and_lines(contents): + """Return a list consisting of tuples of (offset, line). + + The Fluent AST contains spans of start and end offsets in the file. + This function returns a list of offsets and line numbers so that errors + can be reported using line and column. + """ + line = 1 + result = [] + for m in re.finditer(r"\n", contents): + result.append((m.start(), line)) + line += 1 + return result + + +def get_newlines_count_after(span, contents): + # Determine the number of newlines. + count = 0 + for i in range(span.end, len(contents)): + assert contents[i] != "\r", "This linter does not handle \\r characters." + if contents[i] != "\n": + break + count += 1 + + return count + + +def get_newlines_count_before(span, contents): + # Determine the range of newline characters. + count = 0 + for i in range(span.start - 1, 0, -1): + assert contents[i] != "\r", "This linter does not handle \\r characters." + if contents[i] != "\n": + break + count += 1 + + return count + + +def get_exclusions(root): + with open( + mozpath.join(root, "tools", "lint", "fluent-lint", "exclusions.yml") + ) as f: + exclusions = list(yaml.safe_load_all(f))[0] + for error_type in exclusions: + exclusions[error_type]["files"] = set( + [mozpath.join(root, x) for x in exclusions[error_type]["files"]] + ) + return exclusions + + +def get_branding_list(root, brand_files): + class MessageExtractor(visitor.Visitor): + def __init__(self): + self.brands = [] + self.last_message_id = None + + def visit_Term(self, node): + self.last_message_id = node.id.name + self.generic_visit(node) + + def visit_TextElement(self, node): + if self.last_message_id: + self.brands += [node.value] + self.last_message_id = None + self.generic_visit(node) + + extractor = MessageExtractor() + + for brand_path in brand_files: + brand_file = mozpath.join(root, brand_path) + if os.path.exists(brand_file): + with open(brand_file, encoding="utf-8") as f: + messages = parse(f.read()) + extractor.visit(messages) + + return list(set(extractor.brands)) + + +def lint(paths, config, fix=None, **lintargs): + root = lintargs["root"] + files = list(expand_exclusions(paths, config, root)) + exclusions = get_exclusions(root) + brand_files = config.get("brand-files") + brand_names = get_branding_list(root, brand_files) + results = [] + for path in files: + contents = open(path, "r", encoding="utf-8").read() + linter = Linter( + path, + config, + exclusions, + contents, + get_offsets_and_lines(contents), + brand_names, + ) + linter.visit(parse(contents)) + results.extend(linter.results) + return results diff --git a/tools/lint/fluent-lint/exclusions.yml b/tools/lint/fluent-lint/exclusions.yml new file mode 100644 index 0000000000..549eb395c2 --- /dev/null +++ b/tools/lint/fluent-lint/exclusions.yml @@ -0,0 +1,186 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Warning: Only exclusions for identifiers (ID01) are currently allowed. +--- +# Only add exceptions to this file if the ID is generated programmatically and +# can't easily be changed to follow the naming convention. +# Only lowercase letters and hyphens should be used in Fluent IDs. +ID01: + messages: + - trademarkInfo + - crashed-include-URL-2 + - blocklist-item-moz-std-listName + - blocklist-item-moz-full-listName + - shortcuts-browserAction2 + - shortcuts-pageAction + - shortcuts-sidebarAction + - about-networking-originAttributesSuffix + - size-KB + - size-MB + - size-GB + - state-dd-Disabled + - state-dd-Disabled-block-list-state + - memory-unit-B + - memory-unit-KB + - memory-unit-MB + - memory-unit-GB + - memory-unit-TB + - memory-unit-PB + - memory-unit-EB + - enableSafeBrowsing-label + - about-telemetry-show-in-Firefox-json-viewer + - url-classifier-search-listType + # aboutDialog.ftl: Do not add new exceptions for this file, + # new strings should follow the naming convention. + - aboutDialog-title + - releaseNotes-link + - update-checkForUpdatesButton + - update-updateButton + - update-checkingForUpdates + - update-noUpdatesFound + - update-otherInstanceHandlingUpdates + - warningDesc-version + - bottomLinks-license + - bottomLinks-rights + - bottomLinks-privacy + - aboutDialog-version + - aboutDialog-version-nightly + # certError.ftl: These IDs are generated programmatically + # from certificate error codes. + - connectionFailure-title + - deniedPortAccess-title + - dnsNotFound-title + - fileNotFound-title + - fileAccessDenied-title + - captivePortal-title + - malformedURI-title + - netInterrupt-title + - notCached-title + - netOffline-title + - contentEncodingError-title + - unsafeContentType-title + - netReset-title + - netTimeout-title + - unknownProtocolFound-title + - proxyConnectFailure-title + - proxyResolveFailure-title + - redirectLoop-title + - unknownSocketType-title + - nssFailure2-title + - corruptedContentError-title + - sslv3Used-title + - inadequateSecurityError-title + - blockedByPolicy-title + - clockSkewError-title + - networkProtocolError-title + - nssBadCert-title + - nssBadCert-sts-title + files: + # policies-descriptions.ftl: These IDs are generated programmatically + # from policy names. + - browser/locales/en-US/browser/policies/policies-descriptions.ftl + # The webext-perms-description-* IDs are generated programmatically + # from permission names + - toolkit/locales/en-US/toolkit/global/extensionPermissions.ftl +ID02: + messages: + # browser/components/ion/content/ion.ftl + - ion + # browser/locales/en-US/browser/aboutDialog.ftl + - helpus + # browser/locales/en-US/browser/aboutLogins.ftl + - menu + # browser/locales/en-US/browser/pageInfo.ftl + - copy + - perm-tab + # browser/locales/en-US/browser/tabContextMenu.ftl + - pin-tab + # browser/locales/en-US/browser/touchbar/touchbar.ftl + - back + - forward + - reload + - home + - find + - new-tab + - share + # toolkit/locales/en-US/toolkit/about/aboutServiceWorkers.ftl + - scope + - waiting + # toolkit/locales/en-US/toolkit/about/aboutSupport.ftl + # yaml interprets yes and no as booleans if quotes are not present. + - "yes" + - "no" + - unknown + - found + - missing + - gpu-ram + - apz-none + # toolkit/locales/en-US/toolkit/printing/printDialogs.ftl + - portrait + - scale + - print-bg + - hf-blank + - hf-title + - hf-url + - hf-page + files: [] +# Hard-coded brand names like Firefox or Mozilla should be used only in +# specific cases, in all other cases the corresponding terms should be used. +# Check with the localization team for advice. +CO01: + messages: + # browser/branding/official/locales/en-US/brand.ftl + - trademarkInfo + # toolkit/locales/en-US/toolkit/neterror/certError.ftl + - cert-error-mitm-mozilla + - cert-error-mitm-connection + # browser/locales/en-US/browser/appExtensionFields.ftl + - extension-firefox-alpenglow-name + # browser/locales/en-US/browser/browser.ftl + - identity-custom-root + - identity-description-custom-root2 + # browser/locales/en-US/browser/migrationWizard.ftl + - migration-wizard-migrator-display-name-firefox + # browser/locales/en-US/browser/newtab/onboarding.ftl + - mr1-onboarding-welcome-image-caption + - mr2022-onboarding-gratitude-subtitle + - onboarding-gratitude-security-and-privacy-subtitle + # browser/locales/en-US/browser/policies/policies-descriptions.ftl + - policy-DisableFirefoxScreenshots + # browser/locales/en-US/browser/preferences/preferences.ftl + - sync-engine-addons + - sync-mobile-promo + # devtools/client/locales/en-US/aboutdebugging.ftl + - about-debugging-setup-usb-step-enable-debug-firefox2 + - about-debugging-browser-version-too-old-fennec + - about-debugging-browser-version-too-recent + # devtools/client/locales/en-US/application.ftl + - manifest-loaded-devtools-error + # devtools/shared/locales/en-US/webconsole-commands.ftl + - webconsole-commands-usage-block + # toolkit/locales/en-US/toolkit/about/aboutAddons.ftl + - addon-badge-line3 + - recommended-theme-1 + - plugins-openh264-description + # toolkit/locales/en-US/toolkit/about/aboutGlean.ftl + - about-glean-description + # toolkit/locales/en-US/toolkit/about/aboutRights.ftl + - rights-intro-point-1 + - rights-intro-point-2 + # toolkit/locales/en-US/toolkit/about/aboutSupport.ftl + - app-basics-key-mozilla + - virtual-monitor-disp + # toolkit/locales/en-US/toolkit/about/aboutTelemetry.ftl + - about-telemetry-firefox-data-doc + - about-telemetry-telemetry-client-doc + - about-telemetry-telemetry-dashboard + # toolkit/locales/en-US/toolkit/global/extensionPermissions.ftl + - webext-perms-description-management + # toolkit/locales/en-US/toolkit/global/processTypes.ftl + - process-type-privilegedmozilla + files: + - browser/components/ion/content/ion.ftl + - browser/locales/en-US/browser/profile/default-bookmarks.ftl + - toolkit/locales/en-US/toolkit/about/aboutMozilla.ftl diff --git a/tools/lint/hooks.py b/tools/lint/hooks.py new file mode 100755 index 0000000000..92f8565bf7 --- /dev/null +++ b/tools/lint/hooks.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import signal +import subprocess +import sys + +here = os.path.dirname(os.path.realpath(__file__)) +topsrcdir = os.path.join(here, os.pardir, os.pardir) + + +def run_process(cmd): + proc = subprocess.Popen(cmd) + + # ignore SIGINT, the mozlint subprocess should exit quickly and gracefully + orig_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc.wait() + signal.signal(signal.SIGINT, orig_handler) + return proc.returncode + + +def run_mozlint(hooktype, args): + if isinstance(hooktype, bytes): + hooktype = hooktype.decode("UTF-8", "replace") + + python = shutil.which("python3") + if not python: + print("error: Python 3 not detected on your system! Please install it.") + sys.exit(1) + + cmd = [python, os.path.join(topsrcdir, "mach"), "lint"] + + if "commit" in hooktype: + # don't prevent commits, just display the lint results + run_process(cmd + ["--workdir=staged"]) + return False + elif "push" in hooktype: + return run_process(cmd + ["--outgoing"] + args) + + print("warning: '{}' is not a valid mozlint hooktype".format(hooktype)) + return False + + +def hg(ui, repo, **kwargs): + hooktype = kwargs["hooktype"] + return run_mozlint(hooktype, kwargs.get("pats", [])) + + +def git(): + hooktype = os.path.basename(__file__) + if hooktype == "hooks.py": + hooktype = "pre-push" + return run_mozlint(hooktype, []) + + +if __name__ == "__main__": + sys.exit(git()) diff --git a/tools/lint/hooks_clang_format.py b/tools/lint/hooks_clang_format.py new file mode 100755 index 0000000000..9adb81b7f0 --- /dev/null +++ b/tools/lint/hooks_clang_format.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import subprocess +import sys +from subprocess import CalledProcessError, check_output + +here = os.path.dirname(os.path.realpath(__file__)) +topsrcdir = os.path.join(here, os.pardir, os.pardir) + +EXTRA_PATHS = ( + "python/mach", + "python/mozbuild", + "python/mozversioncontrol", + "testing/mozbase/mozfile", + "third_party/python/jsmin", + "third_party/python/six", +) +sys.path[:0] = [os.path.join(topsrcdir, p) for p in EXTRA_PATHS] + +from mozversioncontrol import InvalidRepoPath, get_repository_object + + +def run_clang_format(hooktype, changedFiles): + try: + vcs = get_repository_object(topsrcdir) + except InvalidRepoPath: + return + + if not changedFiles: + # No files have been touched + return + + # We have also a copy of this list in: + # python/mozbuild/mozbuild/mach_commands.py + # version-control-tools/hgext/clang-format/__init__.py + # release-services/src/staticanalysis/bot/static_analysis_bot/config.py + # Too heavy to import the full class just for this variable + extensions = (".cpp", ".c", ".cc", ".h", ".m", ".mm") + path_list = [] + for filename in sorted(changedFiles): + # Ignore files unsupported in clang-format + if filename.endswith(extensions): + path_list.append(filename) + + if not path_list: + # No files have been touched + return + + arguments = ["clang-format", "-p"] + path_list + # On windows we need this to call the command in a shell, see Bug 1511594 + if os.name == "nt": + clang_format_cmd = [sys.executable, "mach"] + arguments + else: + clang_format_cmd = [os.path.join(topsrcdir, "mach")] + arguments + if "commit" in hooktype: + # don't prevent commits, just display the clang-format results + subprocess.call(clang_format_cmd) + + vcs.add_remove_files(*path_list) + + return False + print("warning: '{}' is not a valid clang-format hooktype".format(hooktype)) + return False + + +def hg(ui, repo, node, **kwargs): + print( + "warning: this hook has been deprecated. Please use the hg extension instead.\n" + "please add 'clang-format = ~/.mozbuild/version-control-tools/hgext/clang-format'" + " to hgrc\n" + "Or run 'mach bootstrap'" + ) + return False + + +def git(): + hooktype = os.path.basename(__file__) + if hooktype == "hooks_clang_format.py": + hooktype = "pre-push" + + try: + changedFiles = check_output( + ["git", "diff", "--staged", "--diff-filter=d", "--name-only", "HEAD"], + text=True, + ).split() + # TODO we should detect if we are in a "add -p" mode and show a warning + return run_clang_format(hooktype, changedFiles) + + except CalledProcessError: + print("Command to retrieve local files failed") + return 1 + + +if __name__ == "__main__": + sys.exit(git()) diff --git a/tools/lint/hooks_js_format.py b/tools/lint/hooks_js_format.py new file mode 100755 index 0000000000..1b0386685e --- /dev/null +++ b/tools/lint/hooks_js_format.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import subprocess +import sys +from subprocess import CalledProcessError, check_output + +here = os.path.dirname(os.path.realpath(__file__)) +topsrcdir = os.path.join(here, os.pardir, os.pardir) + +EXTRA_PATHS = ( + "python/mach", + "python/mozbuild", + "python/mozversioncontrol", + "testing/mozbase/mozfile", + "third_party/python/jsmin", +) +sys.path[:0] = [os.path.join(topsrcdir, p) for p in EXTRA_PATHS] + +from mozversioncontrol import InvalidRepoPath, get_repository_object + + +def run_js_format(hooktype, changedFiles): + try: + vcs = get_repository_object(topsrcdir) + except InvalidRepoPath: + return + + if not changedFiles: + # No files have been touched + return + + extensions = (".js", ".jsx", ".jsm", ".json", ".mjs", "sjs", "html", "xhtml") + path_list = [] + for filename in sorted(changedFiles): + # Ignore files unsupported in eslint and prettier + if filename.endswith(extensions): + path_list.append(filename) + + if not path_list: + # No files have been touched + return + + arguments = ["eslint", "--fix"] + path_list + # On windows we need this to call the command in a shell, see Bug 1511594 + if os.name == "nt": + js_format_cmd = ["sh", "mach"] + arguments + else: + js_format_cmd = [os.path.join(topsrcdir, "mach")] + arguments + if "commit" in hooktype: + # don't prevent commits, just display the eslint and prettier results + subprocess.call(js_format_cmd) + + vcs.add_remove_files(*path_list) + + return False + print("warning: '{}' is not a valid js-format hooktype".format(hooktype)) + return False + + +def git(): + hooktype = os.path.basename(__file__) + if hooktype == "hooks_js_format.py": + hooktype = "pre-push" + + try: + changedFiles = check_output( + ["git", "diff", "--staged", "--diff-filter=d", "--name-only", "HEAD"], + text=True, + ).split() + # TODO we should detect if we are in a "add -p" mode and show a warning + return run_js_format(hooktype, changedFiles) + + except CalledProcessError: + print("Command to retrieve local files failed") + return 1 + + +if __name__ == "__main__": + sys.exit(git()) diff --git a/tools/lint/l10n.yml b/tools/lint/l10n.yml new file mode 100644 index 0000000000..85ad1aaabd --- /dev/null +++ b/tools/lint/l10n.yml @@ -0,0 +1,37 @@ +--- +l10n: + description: Localization linter + # list of include directories of both + # browser and mobile/android l10n.tomls + include: + - browser/branding/official/locales/en-US + - browser/extensions/formautofill/locales/en-US + - browser/extensions/report-site-issue/locales/en-US + - browser/locales/en-US + - devtools/client/locales/en-US + - devtools/shared/locales/en-US + - devtools/startup/locales/en-US + - dom/locales/en-US + - mobile/android/locales/en-US + - netwerk/locales/en-US + - security/manager/locales/en-US + - toolkit/locales/en-US + - tools/lint/l10n.yml + # files not supported by compare-locales, + # and also not relevant to this linter + exclude: + - browser/locales/en-US/firefox-l10n.js + - mobile/android/locales/en-US/mobile-l10n.js + - toolkit/locales/en-US/chrome/global/intl.css + l10n_configs: + - browser/locales/l10n.toml + - mobile/android/locales/l10n.toml + type: external + payload: python.l10n_lint:lint + setup: python.l10n_lint:gecko_strings_setup + support-files: + - '**/l10n.toml' + - 'third_party/python/compare-locales/**' + - 'third_party/python/fluent/**' + - 'tools/lint/python/l10n_lint.py' + - 'tools/lint/l10n.yml' diff --git a/tools/lint/libpref/__init__.py b/tools/lint/libpref/__init__.py new file mode 100644 index 0000000000..48b347ab03 --- /dev/null +++ b/tools/lint/libpref/__init__.py @@ -0,0 +1,113 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +import sys + +import yaml +from mozlint import result +from mozlint.pathutils import expand_exclusions + +# This simple linter checks for duplicates from +# modules/libpref/init/StaticPrefList.yaml against modules/libpref/init/all.js + +# If for any reason a pref needs to appear in both files, add it to this set. +IGNORE_PREFS = { + "devtools.console.stdout.chrome", # Uses the 'sticky' attribute. + "devtools.console.stdout.content", # Uses the 'sticky' attribute. + "fission.autostart", # Uses the 'locked' attribute. + "browser.dom.window.dump.enabled", # Uses the 'sticky' attribute. + "apz.fling_curve_function_y2", # This pref is a part of a series. + "dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled", # NOQA: E501; Uses the 'locked' attribute. + "extensions.backgroundServiceWorkerEnabled.enabled", # NOQA: E501; Uses the 'locked' attribute. +} +PATTERN = re.compile(r"\s*pref\(\s*\"(?P<pref>.+)\"\s*,\s*(?P<val>.+)\)\s*;.*") + + +def get_names(pref_list_filename): + pref_names = {} + # We want to transform patterns like 'foo: @VAR@' into valid yaml. This + # pattern does not happen in 'name', so it's fine to ignore these. + # We also want to evaluate all branches of #ifdefs for pref names, so we + # ignore anything else preprocessor related. + file = open(pref_list_filename).read().replace("@", "") + try: + pref_list = yaml.safe_load(file) + except (IOError, ValueError) as e: + print("{}: error:\n {}".format(pref_list_filename, e), file=sys.stderr) + sys.exit(1) + + for pref in pref_list: + if pref["name"] not in IGNORE_PREFS: + pref_names[pref["name"]] = pref["value"] + + return pref_names + + +# Check the names of prefs against each other, and if the pref is a duplicate +# that has not previously been noted, add that name to the list of errors. +def check_against(path, pref_names): + errors = [] + prefs = read_prefs(path) + for pref in prefs: + if pref["name"] in pref_names: + errors.extend(check_value_for_pref(pref, pref_names[pref["name"]], path)) + return errors + + +def check_value_for_pref(some_pref, some_value, path): + errors = [] + if some_pref["value"] == some_value: + errors.append( + { + "path": path, + "message": some_pref["raw"], + "lineno": some_pref["line"], + "hint": "Remove the duplicate pref or add it to IGNORE_PREFS.", + "level": "error", + } + ) + return errors + + +# The entries in the *.js pref files are regular enough to use simple pattern +# matching to load in prefs. +def read_prefs(path): + prefs = [] + with open(path) as source: + for lineno, line in enumerate(source, start=1): + match = PATTERN.match(line) + if match: + prefs.append( + { + "name": match.group("pref"), + "value": evaluate_pref(match.group("val")), + "line": lineno, + "raw": line, + } + ) + return prefs + + +def evaluate_pref(value): + bools = {"true": True, "false": False} + if value in bools: + return bools[value] + elif value.isdigit(): + return int(value) + return value + + +def checkdupes(paths, config, **kwargs): + results = [] + errors = [] + pref_names = get_names(config["support-files"][0]) + files = list(expand_exclusions(paths, config, kwargs["root"])) + for file in files: + errors.extend(check_against(file, pref_names)) + for error in errors: + results.append(result.from_config(config, **error)) + return results diff --git a/tools/lint/license.yml b/tools/lint/license.yml new file mode 100644 index 0000000000..c1cf7e628e --- /dev/null +++ b/tools/lint/license.yml @@ -0,0 +1,104 @@ +--- +license: + description: License Check + include: + - . + exclude: + # These paths need to be triaged. + - build/pgo/js-input + # License not super clear + - browser/branding/ + # Trademarks + - browser/components/pocket/content/panels/ + - browser/components/newtab/data/content/tippytop/images/ + - toolkit/components/pdfjs/content/web/images/ + # We probably want a specific license + - browser/extensions/webcompat/injections/ + # Copied mostly verbatim from upstream. License is documented in + # Cargo.toml. + - build/rust/windows/src/lib.rs + # Different license + - build/pgo/blueprint/print.css + # Different license + - build/pgo/blueprint/screen.css + # Empty files + - config/external/nspr/_pl_bld.h + - config/external/nspr/_pr_bld.h + # Unknown origin + - gfx/2d/MMIHelpers.h + # might not work with license + - gradle.properties + # might not work with license + - gradle/wrapper/gradle-wrapper.properties + # ICU4X data + - intl/icu_segmenter_data + # Imported code that is dual Apache2 / MIT licensed + - intl/l10n/rust/l10nregistry-rs + - intl/l10n/rust/l10nregistry-tests + # tests + - js/src/devtools/rootAnalysis/t/ + - mobile/android/geckoview/src/main/AndroidManifest_overlay.jinja + - mobile/android/geckoview_example/src/main + - testing/webcompat/interventions/ + - testing/webcompat/shims/ + # might not work with license + - mobile/android/gradle/dotgradle-offline/gradle.properties + # might not work with license + - mobile/android/gradle/dotgradle-online/gradle.properties + # Almost empty file + - modules/libpref/greprefs.js + - parser/html/java/named-character-references.html + - python/mozlint/test/files/ + # By design + - python/mozrelease/mozrelease + - security/mac/hardenedruntime/v1/production/browser.xml + - security/mac/hardenedruntime/v1/developer/browser.xml + - security/mac/hardenedruntime/v2/developer/browser.xml + - security/mac/hardenedruntime/v2/developer/media-plugin-helper.xml + - security/mac/hardenedruntime/v2/developer/plugin-container.xml + - security/mac/hardenedruntime/v2/developer/utility.xml + - security/mac/hardenedruntime/v2/production/nightly.browser.xml + - security/mac/hardenedruntime/v2/production/firefox.browser.xml + - security/mac/hardenedruntime/v2/production/firefoxdeveloperedition.browser.xml + - security/mac/hardenedruntime/v2/production/media-plugin-helper.xml + - security/mac/hardenedruntime/v2/production/plugin-container.xml + - testing/marionette/harness/marionette_harness/www/ + # Browsertime can't handle this script when there's a comment at the top + - testing/raptor/browsertime/browsertime_benchmark.js + - toolkit/components/reputationservice/chromium/chrome/common/safe_browsing/csd.pb.cc + - toolkit/components/reputationservice/chromium/chrome/common/safe_browsing/csd.pb.h + - toolkit/mozapps/update/updater/crctable.h + - tools/lint/eslint/eslint-plugin-mozilla/lib/configs + # template fragments used to generate .js sources. + - toolkit/components/uniffi-bindgen-gecko-js/src/templates/js + # By design + - tools/lint/test/ + extensions: + - .c + - .cc + - .cpp + - .css + - .dtd + - .ftl + - .h + - .html + - .idl + - .java + - .js + - .jsm + - .jsx + - .m + - .mm + - .mjs + - .properties + - .py + - .rs + - .svg + - .webidl + - .xhtml + - .xml + support-files: + - 'tools/lint/license/**' + type: external + payload: license:lint + find-dotfiles: true diff --git a/tools/lint/license/__init__.py b/tools/lint/license/__init__.py new file mode 100644 index 0000000000..e49b3b3dea --- /dev/null +++ b/tools/lint/license/__init__.py @@ -0,0 +1,268 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import os +from glob import glob +from html.parser import HTMLParser + +from mozlint import result +from mozlint.pathutils import expand_exclusions + +here = os.path.abspath(os.path.dirname(__file__)) +topsrcdir = os.path.join(here, "..", "..", "..") + +# Official source: https://www.mozilla.org/en-US/MPL/headers/ +TEMPLATES = { + "mpl2_license": """ + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at https://mozilla.org/MPL/2.0/. + """.strip().splitlines(), + "public_domain_license": """ + Any copyright is dedicated to the public domain. + https://creativecommons.org/publicdomain/zero/1.0/ + """.strip().splitlines(), +} +license_list = os.path.join(here, "valid-licenses.txt") + + +def load_valid_license(): + """ + Load the list of license patterns + """ + with open(license_list) as f: + l = f.readlines() + # Remove the empty lines + return list(filter(bool, [x.replace("\n", "") for x in l])) + + +def is_valid_license(licenses, filename): + """ + From a given file, check if we can find the license patterns + in the X first lines of the file + """ + with open(filename, "r", errors="replace") as myfile: + contents = myfile.read() + # Empty files don't need a license. + if not contents: + return True + + for l in licenses: + if l.lower().strip() in contents.lower(): + return True + return False + + +def add_header(log, filename, header): + """ + Add the header to the top of the file + """ + header.append("\n") + with open(filename, "r+") as f: + # lines in list format + try: + lines = f.readlines() + except UnicodeDecodeError as e: + log.debug("Could not read file '{}'".format(f)) + log.debug("Error: {}".format(e)) + return + + i = 0 + if lines: + # if the file isn't empty (__init__.py can be empty files) + if lines[0].startswith("#!") or lines[0].startswith("<?xml "): + i = 1 + + if lines[0].startswith("/* -*- Mode"): + i = 2 + # Insert in the top of the data structure + lines[i:i] = header + f.seek(0, 0) + f.write("".join(lines)) + + +def is_test(f): + """ + is the file a test or not? + """ + if "lint/test/" in f or "lint_license_test_tmp_file.js" in f: + # For the unit tests + return False + return ( + "/tests/" in f + or "/test/" in f + or "/test_" in f + or "/gtest" in f + or "/crashtest" in f + or "/mochitest" in f + or "/reftest" in f + or "/imptest" in f + or "/androidTest" in f + or "/jit-test/" in f + or "jsapi-tests/" in f + ) + + +def fix_me(log, filename): + """ + Add the copyright notice to the top of the file + """ + _, ext = os.path.splitext(filename) + license = [] + + license_template = TEMPLATES["mpl2_license"] + test = False + + if is_test(filename): + license_template = TEMPLATES["public_domain_license"] + test = True + + if ext in [ + ".cpp", + ".c", + ".cc", + ".h", + ".m", + ".mm", + ".rs", + ".java", + ".js", + ".jsm", + ".jsx", + ".mjs", + ".css", + ".idl", + ".webidl", + ]: + for i, l in enumerate(license_template): + start = " " + end = "" + if i == 0: + # first line, we have the /* + start = "/" + if i == len(license_template) - 1: + # Last line, we end by */ + end = " */" + license.append(start + "* " + l.strip() + end + "\n") + + add_header(log, filename, license) + return True + + if ext in [".py", ".ftl", ".properties"]: + for l in license_template: + license.append("# " + l.strip() + "\n") + add_header(log, filename, license) + return True + + if ext in [".xml", ".html", ".xhtml", ".dtd", ".svg"]: + for i, l in enumerate(license_template): + start = " - " + end = "" + if i == 0: + # first line, we have the <!-- + start = "<!-- " + if i == 2 or (i == 1 and test): + # Last line, we end by --> + end = " -->" + license.append(start + l.strip() + end) + if ext != ".svg" or not end: + # When dealing with an svg, we should not have a space between + # the license and the content + license.append("\n") + add_header(log, filename, license) + return True + + # In case we don't know how to handle a specific format. + return False + + +class HTMLParseError(Exception): + def __init__(self, msg, pos): + super().__init__(msg, *pos) + + +class LicenseHTMLParser(HTMLParser): + def __init__(self): + super().__init__() + self.in_code = False + self.invalid_paths = [] + + def handle_starttag(self, tag, attrs): + if tag == "code": + if self.in_code: + raise HTMLParseError("nested code tag", self.getpos()) + self.in_code = True + + def handle_endtag(self, tag): + if tag == "code": + if not self.in_code: + raise HTMLParseError("not started code tag", self.getpos()) + self.in_code = False + + def handle_data(self, data): + if self.in_code: + path = data.strip() + abspath = os.path.join(topsrcdir, path) + if not glob(abspath): + self.invalid_paths.append((path, self.getpos())) + + +def lint_license_html(path): + parser = LicenseHTMLParser() + with open(path) as fd: + content = fd.read() + parser.feed(content) + return parser.invalid_paths + + +def is_html_licence_summary(path): + license_html = os.path.join(topsrcdir, "toolkit", "content", "license.html") + return os.path.samefile(path, license_html) + + +def lint(paths, config, fix=None, **lintargs): + results = [] + log = lintargs["log"] + files = list(expand_exclusions(paths, config, lintargs["root"])) + fixed = 0 + + licenses = load_valid_license() + for f in files: + if is_test(f): + # For now, do not do anything with test (too many) + continue + + if not is_valid_license(licenses, f): + if fix and fix_me(log, f): + fixed += 1 + else: + res = { + "path": f, + "message": "No matching license strings found in tools/lint/license/valid-licenses.txt", # noqa + "level": "error", + } + results.append(result.from_config(config, **res)) + + if is_html_licence_summary(f): + try: + for invalid_path, (lineno, column) in lint_license_html(f): + res = { + "path": f, + "message": "references unknown path {}".format(invalid_path), + "level": "error", + "lineno": lineno, + "column": column, + } + results.append(result.from_config(config, **res)) + except HTMLParseError as err: + res = { + "path": f, + "message": err.args[0], + "level": "error", + "lineno": err.args[1], + "column": err.args[2], + } + results.append(result.from_config(config, **res)) + + return {"results": results, "fixed": fixed} diff --git a/tools/lint/license/valid-licenses.txt b/tools/lint/license/valid-licenses.txt new file mode 100644 index 0000000000..8a1d39a69e --- /dev/null +++ b/tools/lint/license/valid-licenses.txt @@ -0,0 +1,30 @@ +mozilla.org/MPL/ +Licensed under the Apache License, Version 2.0 +copyright is dedicated to the Public Domain. +under the MIT +Redistributions of source code must retain the above copyright +Use of this source code is governed by a BSD-style license +The author disclaims copyright to this source code. +The author hereby disclaims copyright to this source code +Use, Modification and Redistribution (including distribution of any +author grants irrevocable permission to anyone to use, modify, +THIS FILE IS AUTO-GENERATED +Permission is hereby granted, free of charge, to any person obtaining +Permission to use, copy, modify, +License: Public domain. You are free to use this code however you +You are granted a license to use, reproduce and create derivative works +GENERATED FILE, DO NOT EDIT +This code is governed by the BSD license +This Source Code Form is subject to the terms of the Apache License +DO NOT EDIT +This program is made available under an ISC-style license. +under MIT license +License MIT per upstream +DO NOT MODIFY +GNU General Public License +FILE IS GENERATED +Generated by +do not edit +may be protected as a trademark in some jurisdictions +The ASF licenses this file to You under the Apache License, Version 2.0 +Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions diff --git a/tools/lint/lintpref.yml b/tools/lint/lintpref.yml new file mode 100644 index 0000000000..cb58d642bc --- /dev/null +++ b/tools/lint/lintpref.yml @@ -0,0 +1,17 @@ +--- +lintpref: + description: Linter for static prefs. + include: + - 'modules/libpref/init' + - 'browser/app/profile/' + - 'mobile/android/app/' + - 'devtools/client/preferences/' + - 'browser/branding/' + - 'mobile/android/installer/' + - 'mobile/android/locales/' + exclude: [] + extensions: ['js'] + type: external + payload: libpref:checkdupes + support-files: + - 'modules/libpref/init/StaticPrefList.yaml' diff --git a/tools/lint/mach_commands.py b/tools/lint/mach_commands.py new file mode 100644 index 0000000000..c9778e28a0 --- /dev/null +++ b/tools/lint/mach_commands.py @@ -0,0 +1,189 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import copy +import os + +from mach.decorators import Command, CommandArgument +from mozbuild.base import BuildEnvironmentNotFoundException +from mozbuild.base import MachCommandConditions as conditions + +here = os.path.abspath(os.path.dirname(__file__)) +EXCLUSION_FILES = [ + os.path.join("tools", "rewriting", "Generated.txt"), + os.path.join("tools", "rewriting", "ThirdPartyPaths.txt"), +] + +EXCLUSION_FILES_OPTIONAL = [] +thunderbird_excludes = os.path.join("comm", "tools", "lint", "GlobalExclude.txt") +if os.path.exists(thunderbird_excludes): + EXCLUSION_FILES_OPTIONAL.append(thunderbird_excludes) + +GLOBAL_EXCLUDES = ["**/node_modules", "tools/lint/test/files", ".hg", ".git"] + +VALID_FORMATTERS = {"black", "clang-format", "rustfmt", "isort"} +VALID_ANDROID_FORMATTERS = {"android-format"} + +# Code-review bot must index issues from the whole codebase when pushing +# to autoland or try repositories. In such cases, output warnings in the +# task's JSON artifact but do not fail if only warnings are found. +REPORT_WARNINGS = os.environ.get("GECKO_HEAD_REPOSITORY", "").rstrip("/") in ( + "https://hg.mozilla.org/mozilla-central", + "https://hg.mozilla.org/integration/autoland", + "https://hg.mozilla.org/try", +) + + +def setup_argument_parser(): + from mozlint import cli + + return cli.MozlintParser() + + +def get_global_excludes(**lintargs): + # exclude misc paths + excludes = GLOBAL_EXCLUDES[:] + topsrcdir = lintargs["root"] + + # exclude top level paths that look like objdirs + excludes.extend( + [ + name + for name in os.listdir(topsrcdir) + if name.startswith("obj") and os.path.isdir(name) + ] + ) + + if lintargs.get("include_thirdparty"): + # For some linters, we want to include the thirdparty code too. + # Example: trojan-source linter should run also on third party code. + return excludes + + for path in EXCLUSION_FILES + EXCLUSION_FILES_OPTIONAL: + with open(os.path.join(topsrcdir, path), "r") as fh: + excludes.extend([f.strip() for f in fh.readlines()]) + + return excludes + + +@Command( + "lint", + category="devenv", + description="Run linters.", + parser=setup_argument_parser, + virtualenv_name="lint", +) +def lint(command_context, *runargs, **lintargs): + """Run linters.""" + command_context.activate_virtualenv() + from mozlint import cli, parser + + try: + buildargs = {} + buildargs["substs"] = copy.deepcopy(dict(command_context.substs)) + buildargs["defines"] = copy.deepcopy(dict(command_context.defines)) + buildargs["topobjdir"] = command_context.topobjdir + lintargs.update(buildargs) + except BuildEnvironmentNotFoundException: + pass + + lintargs.setdefault("root", command_context.topsrcdir) + lintargs["exclude"] = get_global_excludes(**lintargs) + lintargs["config_paths"].insert(0, here) + lintargs["virtualenv_bin_path"] = command_context.virtualenv_manager.bin_path + lintargs["virtualenv_manager"] = command_context.virtualenv_manager + if REPORT_WARNINGS and lintargs.get("show_warnings") is None: + lintargs["show_warnings"] = "soft" + for path in EXCLUSION_FILES: + parser.GLOBAL_SUPPORT_FILES.append( + os.path.join(command_context.topsrcdir, path) + ) + setupargs = { + "mach_command_context": command_context, + } + return cli.run(*runargs, setupargs=setupargs, **lintargs) + + +@Command( + "eslint", + category="devenv", + description="Run eslint or help configure eslint for optimal development.", +) +@CommandArgument( + "paths", + default=None, + nargs="*", + help="Paths to file or directories to lint, like " + "'browser/' Defaults to the " + "current directory if not given.", +) +@CommandArgument( + "-s", + "--setup", + default=False, + action="store_true", + help="Configure eslint for optimal development.", +) +@CommandArgument("-b", "--binary", default=None, help="Path to eslint binary.") +@CommandArgument( + "--fix", + default=False, + action="store_true", + help="Request that eslint automatically fix errors, where possible.", +) +@CommandArgument( + "--rule", + default=[], + dest="rules", + action="append", + help="Specify an additional rule for ESLint to run, e.g. 'no-new-object: error'", +) +@CommandArgument( + "extra_args", + nargs=argparse.REMAINDER, + help="Extra args that will be forwarded to eslint.", +) +def eslint(command_context, paths, extra_args=[], **kwargs): + command_context._mach_context.commands.dispatch( + "lint", + command_context._mach_context, + linters=["eslint"], + paths=paths, + argv=extra_args, + **kwargs + ) + + +@Command( + "format", + category="devenv", + description="Format files, alternative to 'lint --fix' ", + parser=setup_argument_parser, +) +def format_files(command_context, paths, extra_args=[], **kwargs): + linters = kwargs["linters"] + + formatters = VALID_FORMATTERS + if conditions.is_android(command_context): + formatters |= VALID_ANDROID_FORMATTERS + + if not linters: + linters = formatters + else: + invalid_linters = set(linters) - formatters + if invalid_linters: + print( + "error: One or more linters passed are not valid formatters. " + "Note that only the following linters are valid formatters:" + ) + print("\n".join(sorted(formatters))) + return 1 + + kwargs["linters"] = list(linters) + + kwargs["fix"] = True + command_context._mach_context.commands.dispatch( + "lint", command_context._mach_context, paths=paths, argv=extra_args, **kwargs + ) diff --git a/tools/lint/mingw-capitalization.yml b/tools/lint/mingw-capitalization.yml new file mode 100644 index 0000000000..04c9c79e16 --- /dev/null +++ b/tools/lint/mingw-capitalization.yml @@ -0,0 +1,12 @@ +--- +mingw-capitalization: + description: > + "A Windows include file is not lowercase, and may break the MinGW build" + extensions: ['h', 'cpp', 'cc', 'c'] + include: ['.'] + exclude: + # We do not compile WebRTC with MinGW yet + - media/webrtc + type: external + level: error + payload: cpp.mingw-capitalization:lint diff --git a/tools/lint/mscom-init.yml b/tools/lint/mscom-init.yml new file mode 100644 index 0000000000..d818aa4ab9 --- /dev/null +++ b/tools/lint/mscom-init.yml @@ -0,0 +1,74 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +forbid-mscom-init: + description: >- + New calls to CoInitialize, CoInitializeEx, OleInitialize, RoInitialize, + CoUninitialize, OleUninitialize, or RoUninitialize are forbidden. If you + have questions, please consult a peer of the IPC: MSCOM module. + level: error + include: ['.'] + type: regex + payload: ([CR]o|Ole)(Uni|I)nitialize(Ex)? + ignore-case: false + extensions: + - h + - c + - cc + - cpp + exclude: + # These files contain utilities for using COM more safely. + - ipc/mscom/ApartmentRegion.h + - ipc/mscom/COMWrappers.cpp + - ipc/mscom/COMWrappers.h + - ipc/mscom/EnsureMTA.cpp + # These files have been reviewed by MSCOM peers, and the use of + # CoInitialize within them has been confirmed to be necessary and + # proper. + - ipc/mscom/ProcessRuntime.cpp + # These files are existing legacy uses of CoInitialize (and so forth) + # that must eventually be fixed -- probably by converting them to use + # ApartmentRegion and moving them to _that_ lint's exception-list. + - browser/components/migration/nsIEHistoryEnumerator.cpp + - browser/components/migration/tests/unit/insertIEHistory/InsertIEHistory.cpp + - browser/components/shell/nsWindowsShellService.cpp + - gfx/thebes/gfxWindowsPlatform.cpp + - image/DecodePool.cpp + - ipc/glue/BrowserProcessSubThread.cpp + - netwerk/system/win32/nsNotifyAddrListener.cpp + - toolkit/components/parentalcontrols/nsParentalControlsServiceWin.cpp + - toolkit/crashreporter/google-breakpad/src/common/windows/pdb_source_line_writer.cc + - toolkit/mozapps/defaultagent/proxy/main.cpp + - uriloader/exthandler/win/nsOSHelperAppService.cpp + - widget/windows/TaskbarPreview.cpp + - widget/windows/WinTaskbar.cpp + - widget/windows/nsAppShell.cpp + - widget/windows/nsWindow.cpp + - widget/windows/nsWindow.h + - widget/windows/tests/TestUriValidation.cpp + - xpcom/io/nsLocalFileWin.cpp + +forbid-apartment-region: + description: >- + New uses of ApartmentRegion, ApartmentRegionT, MTARegion, or STARegion + require approval by a peer of the IPC: MSCOM module. + level: error + include: ['.'] + type: regex + payload: ApartmentRegion(T)?|[MS]TARegion + ignore-case: false + extensions: + - h + - c + - cc + - cpp + exclude: + # ApartmentRegion's definition. + - ipc/mscom/ApartmentRegion.h + # These files have been reviewed and approved by MSCOM peers. + - ipc/mscom/ProcessRuntime.cpp + - ipc/mscom/ProcessRuntime.h + - widget/windows/filedialog/WinFileDialogCommands.cpp + # These files are existing uses that must eventually be fixed. + - widget/windows/LegacyJumpListBuilder.cpp diff --git a/tools/lint/perfdocs.yml b/tools/lint/perfdocs.yml new file mode 100644 index 0000000000..175781ad00 --- /dev/null +++ b/tools/lint/perfdocs.yml @@ -0,0 +1,18 @@ +--- +perfdocs: + description: Performance Documentation linter + # This task handles its own search, so just include cwd + include: [ + 'python/mozperftest', + 'testing/awsy', + 'testing/raptor', + 'testing/talos', + 'testing/performance/fxrecord', + 'testing/performance/mach-try-perf', + 'dom/indexedDB/test', + ] + exclude: [] + extensions: ['rst', 'ini', 'yml'] + support-files: [] + type: structured_log + payload: perfdocs:lint diff --git a/tools/lint/perfdocs/__init__.py b/tools/lint/perfdocs/__init__.py new file mode 100644 index 0000000000..1194d38624 --- /dev/null +++ b/tools/lint/perfdocs/__init__.py @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os + +from perfdocs import perfdocs + +here = os.path.abspath(os.path.dirname(__file__)) +PERFDOCS_REQUIREMENTS_PATH = os.path.join(here, "requirements.txt") + + +def lint(paths, config, logger, fix=False, **lintargs): + return perfdocs.run_perfdocs(config, logger=logger, paths=paths, generate=fix) diff --git a/tools/lint/perfdocs/doc_helpers.py b/tools/lint/perfdocs/doc_helpers.py new file mode 100644 index 0000000000..709f190204 --- /dev/null +++ b/tools/lint/perfdocs/doc_helpers.py @@ -0,0 +1,85 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +class MismatchedRowLengthsException(Exception): + """ + This exception is thrown when there is a mismatch between the number of items in a row, + and the number of headers defined. + """ + + pass + + +class TableBuilder(object): + """ + Helper class for building tables. + """ + + def __init__(self, title, widths, header_rows, headers, indent=0): + """ + :param title: str - Title of the table + :param widths: list of str - Widths of each column of the table + :param header_rows: int - Number of header rows + :param headers: 2D list of str - Headers + :param indent: int - Number of spaces to indent table + """ + if not isinstance(title, str): + raise TypeError("TableBuilder attribute title must be a string.") + if not isinstance(widths, list) or not isinstance(widths[0], int): + raise TypeError("TableBuilder attribute widths must be a list of integers.") + if not isinstance(header_rows, int): + raise TypeError("TableBuilder attribute header_rows must be an integer.") + if ( + not isinstance(headers, list) + or not isinstance(headers[0], list) + or not isinstance(headers[0][0], str) + ): + raise TypeError( + "TableBuilder attribute headers must be a two-dimensional list of strings." + ) + if not isinstance(indent, int): + raise TypeError("TableBuilder attribute indent must be an integer.") + + self.title = title + self.widths = widths + self.header_rows = header_rows + self.headers = headers + self.indent = " " * indent + self.table = "" + self._build_table() + + def _build_table(self): + if len(self.widths) != len(self.headers[0]): + raise MismatchedRowLengthsException( + "Number of table headers must match number of column widths." + ) + widths = " ".join(map(str, self.widths)) + self.table += ( + f"{self.indent}.. list-table:: **{self.title}**\n" + f"{self.indent} :widths: {widths}\n" + f"{self.indent} :header-rows: {self.header_rows}\n\n" + ) + self.add_rows(self.headers) + + def add_rows(self, rows): + if type(rows) != list or type(rows[0]) != list or type(rows[0][0]) != str: + raise TypeError("add_rows() requires a two-dimensional list of strings.") + for row in rows: + self.add_row(row) + + def add_row(self, values): + if len(values) != len(self.widths): + raise MismatchedRowLengthsException( + "Number of items in a row must must number of columns defined." + ) + for i, val in enumerate(values): + if i == 0: + self.table += f"{self.indent} * - **{val}**\n" + else: + self.table += f"{self.indent} - {val}\n" + + def finish_table(self): + self.table += "\n" + return self.table diff --git a/tools/lint/perfdocs/framework_gatherers.py b/tools/lint/perfdocs/framework_gatherers.py new file mode 100644 index 0000000000..4c2ca585ad --- /dev/null +++ b/tools/lint/perfdocs/framework_gatherers.py @@ -0,0 +1,571 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json +import os +import pathlib +import re + +from gecko_taskgraph.util.attributes import match_run_on_projects +from manifestparser import TestManifest +from mozperftest.script import ScriptInfo + +from perfdocs.doc_helpers import TableBuilder +from perfdocs.logger import PerfDocLogger +from perfdocs.utils import read_yaml + +logger = PerfDocLogger() + +BRANCHES = [ + "mozilla-central", + "autoland", + "mozilla-release", + "mozilla-beta", +] + +""" +This file is for framework specific gatherers since manifests +might be parsed differently in each of them. The gatherers +must implement the FrameworkGatherer class. +""" + + +class FrameworkGatherer(object): + """ + Abstract class for framework gatherers. + """ + + def __init__(self, yaml_path, workspace_dir, taskgraph={}): + """ + Generic initialization for a framework gatherer. + """ + self.workspace_dir = workspace_dir + self._yaml_path = yaml_path + self._taskgraph = taskgraph + self._suite_list = {} + self._test_list = {} + self._descriptions = {} + self._manifest_path = "" + self._manifest = None + self.script_infos = {} + self._task_list = {} + self._task_match_pattern = re.compile(r"([\w\W]*/[pgo|opt]*)-([\w\W]*)") + + def get_task_match(self, task_name): + return re.search(self._task_match_pattern, task_name) + + def get_manifest_path(self): + """ + Returns the path to the manifest based on the + manifest entry in the frameworks YAML configuration + file. + + :return str: Path to the manifest. + """ + if self._manifest_path: + return self._manifest_path + + yaml_content = read_yaml(self._yaml_path) + self._manifest_path = pathlib.Path(self.workspace_dir, yaml_content["manifest"]) + return self._manifest_path + + def get_suite_list(self): + """ + Each framework gatherer must return a dictionary with + the following structure. Note that the test names must + be relative paths so that issues can be correctly issued + by the reviewbot. + + :return dict: A dictionary with the following structure: { + "suite_name": [ + 'testing/raptor/test1', + 'testing/raptor/test2' + ] + } + """ + raise NotImplementedError + + def _build_section_with_header(self, title, content, header_type=None): + """ + Adds a section to the documentation with the title as the type mentioned + and paragraph as content mentioned. + :param title: title of the section + :param content: content of section paragraph + :param header_type: type of the title heading + """ + heading_map = {"H2": "*", "H3": "=", "H4": "-", "H5": "^"} + return [title, heading_map.get(header_type, "^") * len(title), content, ""] + + +class RaptorGatherer(FrameworkGatherer): + """ + Gatherer for the Raptor framework. + """ + + def get_suite_list(self): + """ + Returns a dictionary containing a mapping from suites + to the tests they contain. + + :return dict: A dictionary with the following structure: { + "suite_name": [ + 'testing/raptor/test1', + 'testing/raptor/test2' + ] + } + """ + if self._suite_list: + return self._suite_list + + manifest_path = self.get_manifest_path() + + # Get the tests from the manifest + test_manifest = TestManifest([str(manifest_path)], strict=False) + test_list = test_manifest.active_tests(exists=False, disabled=False) + + # Parse the tests into the expected dictionary + for test in test_list: + # Get the top-level suite + s = os.path.basename(test["here"]) + if s not in self._suite_list: + self._suite_list[s] = [] + + # Get the individual test + fpath = re.sub(".*testing", "testing", test["manifest"]) + + if fpath not in self._suite_list[s]: + self._suite_list[s].append(fpath) + + return self._suite_list + + def _get_ci_tasks(self): + for task in self._taskgraph.keys(): + if type(self._taskgraph[task]) == dict: + command = self._taskgraph[task]["task"]["payload"].get("command", []) + run_on_projects = self._taskgraph[task]["attributes"]["run_on_projects"] + else: + command = self._taskgraph[task].task["payload"].get("command", []) + run_on_projects = self._taskgraph[task].attributes["run_on_projects"] + + test_match = re.search(r"[\s']--test[\s=](.+?)[\s']", str(command)) + task_match = self.get_task_match(task) + if test_match and task_match: + test = test_match.group(1) + platform = task_match.group(1) + test_name = task_match.group(2) + + item = {"test_name": test_name, "run_on_projects": run_on_projects} + self._task_list.setdefault(test, {}).setdefault(platform, []).append( + item + ) + + def _get_subtests_from_ini(self, manifest_path, suite_name): + """ + Returns a list of (sub)tests from an ini file containing the test definitions. + + :param str manifest_path: path to the ini file + :return list: the list of the tests + """ + desc_exclusion = ["here", "manifest_relpath", "path", "relpath"] + test_manifest = TestManifest([str(manifest_path)], strict=False) + test_list = test_manifest.active_tests(exists=False, disabled=False) + subtests = {} + for subtest in test_list: + subtests[subtest["name"]] = subtest["manifest"] + + description = {} + for key, value in subtest.items(): + if key not in desc_exclusion: + description[key] = value + + # Prepare alerting metrics for verification + description["metrics"] = [ + metric.strip() + for metric in description.get("alert_on", "").split(",") + if metric.strip() != "" + ] + + subtests[subtest["name"]] = description + self._descriptions.setdefault(suite_name, []).append(description) + + self._descriptions[suite_name].sort(key=lambda item: item["name"]) + + return subtests + + def get_test_list(self): + """ + Returns a dictionary containing the tests in every suite ini file. + + :return dict: A dictionary with the following structure: { + "suite_name": { + 'raptor_test1', + 'raptor_test2' + }, + } + """ + if self._test_list: + return self._test_list + + suite_list = self.get_suite_list() + + # Iterate over each manifest path from suite_list[suite_name] + # and place the subtests into self._test_list under the same key + for suite_name, manifest_paths in suite_list.items(): + if not self._test_list.get(suite_name): + self._test_list[suite_name] = {} + for manifest_path in manifest_paths: + subtest_list = self._get_subtests_from_ini(manifest_path, suite_name) + self._test_list[suite_name].update(subtest_list) + + self._get_ci_tasks() + + return self._test_list + + def build_test_description(self, title, test_description="", suite_name=""): + matcher = [] + browsers = [ + "firefox", + "chrome", + "chromium", + "refbrow", + "fennec68", + "geckoview", + "fenix", + ] + test_name = [f"{title}-{browser}" for browser in browsers] + test_name.append(title) + + for suite, val in self._descriptions.items(): + for test in val: + if test["name"] in test_name and suite_name == suite: + matcher.append(test) + + if len(matcher) == 0: + logger.critical( + "No tests exist for the following name " + "(obtained from config.yml): {}".format(title) + ) + raise Exception( + "No tests exist for the following name " + "(obtained from config.yml): {}".format(title) + ) + + result = f".. dropdown:: {title}\n" + result += f" :class-container: anchor-id-{title}-{suite_name[0]}\n\n" + + for idx, description in enumerate(matcher): + if description["name"] != title: + result += f" {idx+1}. **{description['name']}**\n\n" + if "owner" in description.keys(): + result += f" **Owner**: {description['owner']}\n\n" + + for key in sorted(description.keys()): + if key in ["owner", "name", "manifest", "metrics"]: + continue + sub_title = key.replace("_", " ") + if key == "test_url": + if "<" in description[key] or ">" in description[key]: + description[key] = description[key].replace("<", "\<") + description[key] = description[key].replace(">", "\>") + result += f" * **{sub_title}**: `<{description[key]}>`__\n" + elif key == "secondary_url": + result += f" * **{sub_title}**: `<{description[key]}>`__\n" + elif key in ["playback_pageset_manifest"]: + result += ( + f" * **{sub_title}**: " + f"{description[key].replace('{subtest}', description['name'])}\n" + ) + else: + if "\n" in description[key]: + description[key] = description[key].replace("\n", " ") + result += f" * **{sub_title}**: {description[key]}\n" + + if self._task_list.get(title, []): + result += " * **Test Task**:\n\n" + for platform in sorted(self._task_list[title]): + self._task_list[title][platform].sort(key=lambda x: x["test_name"]) + + table = TableBuilder( + title=platform, + widths=[30] + [15 for x in BRANCHES], + header_rows=1, + headers=[["Test Name"] + BRANCHES], + indent=3, + ) + + for task in self._task_list[title][platform]: + values = [task["test_name"]] + values += [ + "\u2705" + if match_run_on_projects(x, task["run_on_projects"]) + else "\u274C" + for x in BRANCHES + ] + table.add_row(values) + result += f"{table.finish_table()}\n" + + return [result] + + def build_suite_section(self, title, content): + return self._build_section_with_header( + title.capitalize(), content, header_type="H4" + ) + + +class MozperftestGatherer(FrameworkGatherer): + """ + Gatherer for the Mozperftest framework. + """ + + def get_test_list(self): + """ + Returns a dictionary containing the tests that are in perftest.toml manifest. + + :return dict: A dictionary with the following structure: { + "suite_name": { + 'perftest_test1', + 'perftest_test2', + }, + } + """ + for path in list(pathlib.Path(self.workspace_dir).rglob("perftest.toml")): + if "obj-" in str(path): + continue + suite_name = str(path.parent).replace(str(self.workspace_dir), "") + + # If the workspace dir doesn't end with a forward-slash, + # the substitution above won't work completely + if suite_name.startswith("/") or suite_name.startswith("\\"): + suite_name = suite_name[1:] + + # We have to add new paths to the logger as we search + # because mozperftest tests exist in multiple places in-tree + PerfDocLogger.PATHS.append(suite_name) + + # Get the tests from perftest.toml + test_manifest = TestManifest([str(path)], strict=False) + test_list = test_manifest.active_tests(exists=False, disabled=False) + for test in test_list: + si = ScriptInfo(test["path"]) + self.script_infos[si["name"]] = si + self._test_list.setdefault(suite_name.replace("\\", "/"), {}).update( + {si["name"]: {"path": str(path)}} + ) + + return self._test_list + + def build_test_description(self, title, test_description="", suite_name=""): + return [str(self.script_infos[title])] + + def build_suite_section(self, title, content): + return self._build_section_with_header(title, content, header_type="H4") + + +class TalosGatherer(FrameworkGatherer): + def _get_ci_tasks(self): + with open( + pathlib.Path(self.workspace_dir, "testing", "talos", "talos.json") + ) as f: + config_suites = json.load(f)["suites"] + + for task_name in self._taskgraph.keys(): + task = self._taskgraph[task_name] + + if type(task) == dict: + is_talos = task["task"]["extra"].get("suite", []) + command = task["task"]["payload"].get("command", []) + run_on_projects = task["attributes"]["run_on_projects"] + else: + is_talos = task.task["extra"].get("suite", []) + command = task.task["payload"].get("command", []) + run_on_projects = task.attributes["run_on_projects"] + + suite_match = re.search(r"[\s']--suite[\s=](.+?)[\s']", str(command)) + task_match = self.get_task_match(task_name) + if "talos" == is_talos and task_match: + suite = suite_match.group(1) + platform = task_match.group(1) + test_name = task_match.group(2) + item = {"test_name": test_name, "run_on_projects": run_on_projects} + + for test in config_suites[suite]["tests"]: + self._task_list.setdefault(test, {}).setdefault( + platform, [] + ).append(item) + + def get_test_list(self): + from talos import test as talos_test + + test_lists = talos_test.test_dict() + mod = __import__("talos.test", fromlist=test_lists) + + suite_name = "Talos Tests" + + for test in test_lists: + self._test_list.setdefault(suite_name, {}).update({test: {}}) + + klass = getattr(mod, test) + self._descriptions.setdefault(test, klass.__dict__) + + self._get_ci_tasks() + + return self._test_list + + def build_test_description(self, title, test_description="", suite_name=""): + result = f".. dropdown:: {title}\n" + result += f" :class-container: anchor-id-{title}\n\n" + + yml_descriptions = [s.strip() for s in test_description.split("- ") if s] + for description in yml_descriptions: + if "Example Data" in description: + # Example Data for using code block + example_list = [s.strip() for s in description.split("* ")] + result += f" * {example_list[0]}\n" + result += "\n .. code-block::\n\n" + for example in example_list[1:]: + result += f" {example}\n" + result += "\n" + + elif " * " in description: + # Sub List + sub_list = [s.strip() for s in description.split(" * ")] + result += f" * {sub_list[0]}\n" + for sub in sub_list[1:]: + result += f" * {sub}\n" + + else: + # General List + result += f" * {description}\n" + + if title in self._descriptions: + for key in sorted(self._descriptions[title]): + if key.startswith("__") and key.endswith("__"): + continue + elif key == "filters": + continue + + # On windows, we get the paths in the wrong style + value = self._descriptions[title][key] + if isinstance(value, dict): + for k, v in value.items(): + if isinstance(v, str) and "\\" in v: + value[k] = str(v).replace("\\", r"/") + result += r" * " + key + r": " + str(value) + r"\n" + + # Command + result += " * Command\n\n" + result += " .. code-block::\n\n" + result += f" ./mach talos-test -a {title}\n\n" + + if self._task_list.get(title, []): + result += " * **Test Task**:\n\n" + for platform in sorted(self._task_list[title]): + self._task_list[title][platform].sort(key=lambda x: x["test_name"]) + + table = TableBuilder( + title=platform, + widths=[30] + [15 for x in BRANCHES], + header_rows=1, + headers=[["Test Name"] + BRANCHES], + indent=3, + ) + + for task in self._task_list[title][platform]: + values = [task["test_name"]] + values += [ + "\u2705" + if match_run_on_projects(x, task["run_on_projects"]) + else "\u274C" + for x in BRANCHES + ] + table.add_row(values) + result += f"{table.finish_table()}\n" + + return [result] + + def build_suite_section(self, title, content): + return self._build_section_with_header(title, content, header_type="H2") + + +class AwsyGatherer(FrameworkGatherer): + """ + Gatherer for the Awsy framework. + """ + + def _generate_ci_tasks(self): + for task_name in self._taskgraph.keys(): + task = self._taskgraph[task_name] + + if type(task) == dict: + awsy_test = task["task"]["extra"].get("suite", []) + run_on_projects = task["attributes"]["run_on_projects"] + else: + awsy_test = task.task["extra"].get("suite", []) + run_on_projects = task.attributes["run_on_projects"] + + task_match = self.get_task_match(task_name) + + if "awsy" in awsy_test and task_match: + platform = task_match.group(1) + test_name = task_match.group(2) + item = {"test_name": test_name, "run_on_projects": run_on_projects} + self._task_list.setdefault(platform, []).append(item) + + def get_suite_list(self): + self._suite_list = {"Awsy tests": ["tp6", "base", "dmd", "tp5"]} + return self._suite_list + + def get_test_list(self): + self._generate_ci_tasks() + return { + "Awsy tests": { + "tp6": {}, + "base": {}, + "dmd": {}, + "tp5": {}, + } + } + + def build_suite_section(self, title, content): + return self._build_section_with_header( + title.capitalize(), content, header_type="H4" + ) + + def build_test_description(self, title, test_description="", suite_name=""): + dropdown_suite_name = suite_name.replace(" ", "-") + result = f".. dropdown:: {title} ({test_description})\n" + result += f" :class-container: anchor-id-{title}-{dropdown_suite_name}\n\n" + + awsy_data = read_yaml(self._yaml_path)["suites"]["Awsy tests"] + if "owner" in awsy_data.keys(): + result += f" **Owner**: {awsy_data['owner']}\n\n" + result += " * **Test Task**:\n" + + # tp5 tests are represented by awsy-e10s test names + # while the others have their title in test names + search_tag = "awsy-e10s" if title == "tp5" else title + for platform in sorted(self._task_list.keys()): + result += f" * {platform}\n" + for test_dict in sorted( + self._task_list[platform], key=lambda d: d["test_name"] + ): + if search_tag in test_dict["test_name"]: + run_on_project = ": " + ( + ", ".join(test_dict["run_on_projects"]) + if test_dict["run_on_projects"] + else "None" + ) + result += ( + f" * {test_dict['test_name']}{run_on_project}\n" + ) + result += "\n" + + return [result] + + +class StaticGatherer(FrameworkGatherer): + """ + A noop gatherer for frameworks with static-only documentation. + """ + + pass diff --git a/tools/lint/perfdocs/gatherer.py b/tools/lint/perfdocs/gatherer.py new file mode 100644 index 0000000000..828c2f7f2b --- /dev/null +++ b/tools/lint/perfdocs/gatherer.py @@ -0,0 +1,156 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import pathlib + +from perfdocs.framework_gatherers import ( + AwsyGatherer, + MozperftestGatherer, + RaptorGatherer, + StaticGatherer, + TalosGatherer, +) +from perfdocs.logger import PerfDocLogger +from perfdocs.utils import read_yaml + +logger = PerfDocLogger() + +# TODO: Implement decorator/searcher to find the classes. +frameworks = { + "raptor": RaptorGatherer, + "mozperftest": MozperftestGatherer, + "talos": TalosGatherer, + "awsy": AwsyGatherer, +} + +# List of file types allowed to be used as static files +ALLOWED_STATIC_FILETYPES = ("rst", "png") + + +class Gatherer(object): + """ + Gatherer produces the tree of the perfdoc's entries found + and can obtain manifest-based test lists. Used by the Verifier. + """ + + def __init__(self, workspace_dir, taskgraph=None): + """ + Initialzie the Gatherer. + + :param str workspace_dir: Path to the gecko checkout. + """ + self.workspace_dir = workspace_dir + self.taskgraph = taskgraph + self._perfdocs_tree = [] + self._test_list = [] + self.framework_gatherers = {} + + @property + def perfdocs_tree(self): + """ + Returns the perfdocs_tree, and computes it + if it doesn't exist. + + :return dict: The perfdocs tree containing all + framework perfdoc entries. See `fetch_perfdocs_tree` + for information on the data structure. + """ + if self._perfdocs_tree: + return self._perfdocs_tree + else: + self.fetch_perfdocs_tree() + return self._perfdocs_tree + + def fetch_perfdocs_tree(self): + """ + Creates the perfdocs tree with the following structure: + [ + { + "path": Path to the perfdocs directory. + "yml": Name of the configuration YAML file. + "rst": Name of the RST file. + "static": Name of the static file. + }, ... + ] + + This method doesn't return anything. The result can be found in + the perfdocs_tree attribute. + """ + exclude_dir = [ + str(pathlib.Path(self.workspace_dir, ".hg")), + str(pathlib.Path("tools", "lint")), + str(pathlib.Path("testing", "perfdocs")), + ] + + for path in pathlib.Path(self.workspace_dir).rglob("perfdocs"): + if any(d in str(path.resolve()) for d in exclude_dir): + continue + files = [f for f in os.listdir(path)] + matched = {"path": str(path), "yml": "", "rst": "", "static": []} + + for file in files: + # Add the yml/rst/static file to its key if re finds the searched file + if file == "config.yml" or file == "config.yaml": + matched["yml"] = file + elif file == "index.rst": + matched["rst"] = file + elif file.split(".")[-1] in ALLOWED_STATIC_FILETYPES: + matched["static"].append(file) + + # Append to structdocs if all the searched files were found + if all(val for val in matched.values() if not type(val) == list): + self._perfdocs_tree.append(matched) + + logger.log( + "Found {} perfdocs directories in {}".format( + len(self._perfdocs_tree), + [d["path"] for d in self._perfdocs_tree], + ) + ) + + def get_test_list(self, sdt_entry): + """ + Use a perfdocs_tree entry to find the test list for + the framework that was found. + + :return: A framework info dictionary with fields: { + 'yml_path': Path to YAML, + 'yml_content': Content of YAML, + 'name': Name of framework, + 'test_list': Test list found for the framework + } + """ + + # If it was computed before, return it + yaml_path = pathlib.Path(sdt_entry["path"], sdt_entry["yml"]) + for entry in self._test_list: + if entry["yml_path"] == yaml_path: + return entry + + # Set up framework entry with meta data + yaml_content = read_yaml(yaml_path) + framework = { + "yml_content": yaml_content, + "yml_path": yaml_path, + "name": yaml_content["name"], + "test_list": {}, + } + + if yaml_content["static-only"]: + framework_gatherer_cls = StaticGatherer + else: + framework_gatherer_cls = frameworks[framework["name"]] + + # Get and then store the frameworks tests + framework_gatherer = self.framework_gatherers[ + framework["name"] + ] = framework_gatherer_cls( + framework["yml_path"], self.workspace_dir, self.taskgraph + ) + + if not yaml_content["static-only"]: + framework["test_list"] = framework_gatherer.get_test_list() + + self._test_list.append(framework) + return framework diff --git a/tools/lint/perfdocs/generator.py b/tools/lint/perfdocs/generator.py new file mode 100644 index 0000000000..3f3a0acefa --- /dev/null +++ b/tools/lint/perfdocs/generator.py @@ -0,0 +1,281 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import pathlib +import re +import shutil +import tempfile + +from perfdocs.logger import PerfDocLogger +from perfdocs.utils import ( + ON_TRY, + are_dirs_equal, + get_changed_files, + read_file, + read_yaml, + save_file, +) + +logger = PerfDocLogger() + + +class Generator(object): + """ + After each perfdocs directory was validated, the generator uses the templates + for each framework, fills them with the test descriptions in config and saves + the perfdocs in the form index.rst as index file and suite_name.rst for + each suite of tests in the framework. + """ + + def __init__(self, verifier, workspace, generate=False): + """ + Initialize the Generator. + + :param verifier: Verifier object. It should not be a fresh Verifier object, + but an initialized one with validate_tree() method already called + :param workspace: Path to the top-level checkout directory. + :param generate: Flag for generating the documentation + """ + self._workspace = workspace + if not self._workspace: + raise Exception("PerfDocs Generator requires a workspace directory.") + # Template documents without added information reside here + self.templates_path = pathlib.Path( + self._workspace, "tools", "lint", "perfdocs", "templates" + ) + self.perfdocs_path = pathlib.Path( + self._workspace, "testing", "perfdocs", "generated" + ) + + self._generate = generate + self._verifier = verifier + self._perfdocs_tree = self._verifier._gatherer.perfdocs_tree + + def build_perfdocs_from_tree(self): + """ + Builds up a document for each framework that was found. + + :return dict: A dictionary containing a mapping from each framework + to the document that was built for it, i.e: + { + framework_name: framework_document, + ... + } + """ + + # Using the verified `perfdocs_tree`, build up the documentation. + frameworks_info = {} + for framework in self._perfdocs_tree: + yaml_content = read_yaml(pathlib.Path(framework["path"], framework["yml"])) + rst_content = read_file( + pathlib.Path(framework["path"], framework["rst"]), stringify=True + ) + + # Gather all tests and descriptions and format them into + # documentation content + documentation = [] + suites = yaml_content["suites"] + for suite_name in sorted(suites.keys()): + suite_info = suites[suite_name] + + # Add the suite section + documentation.extend( + self._verifier._gatherer.framework_gatherers[ + yaml_content["name"] + ].build_suite_section(suite_name, suite_info["description"]) + ) + + tests = suite_info.get("tests", {}) + for test_name in sorted(tests.keys()): + gatherer = self._verifier._gatherer.framework_gatherers[ + yaml_content["name"] + ] + test_description = gatherer.build_test_description( + test_name, tests[test_name], suite_name + ) + documentation.extend(test_description) + documentation.append("") + + # Insert documentation into `.rst` file + framework_rst = re.sub( + r"{documentation}", "\n".join(documentation), rst_content + ) + frameworks_info[yaml_content["name"]] = { + "dynamic": framework_rst, + "static": [], + } + + # For static `.rst` file + for static_file in framework["static"]: + if static_file.endswith("rst"): + frameworks_info[yaml_content["name"]]["static"].append( + { + "file": static_file, + "content": read_file( + pathlib.Path(framework["path"], static_file), + stringify=True, + ), + } + ) + else: + frameworks_info[yaml_content["name"]]["static"].append( + { + "file": static_file, + "content": pathlib.Path(framework["path"], static_file), + } + ) + + return frameworks_info + + def _create_temp_dir(self): + """ + Create a temp directory as preparation of saving the documentation tree. + :return: str the location of perfdocs_tmpdir + """ + # Build the directory that will contain the final result (a tmp dir + # that will be moved to the final location afterwards) + try: + tmpdir = pathlib.Path(tempfile.mkdtemp()) + perfdocs_tmpdir = pathlib.Path(tmpdir, "generated") + perfdocs_tmpdir.mkdir(parents=True, exist_ok=True) + perfdocs_tmpdir.chmod(0o766) + except OSError as e: + logger.critical("Error creating temp file: {}".format(e)) + + if perfdocs_tmpdir.is_dir(): + return perfdocs_tmpdir + return False + + def _create_perfdocs(self): + """ + Creates the perfdocs documentation. + :return: str path of the temp dir it is saved in + """ + # All directories that are kept in the perfdocs tree are valid, + # so use it to build up the documentation. + framework_docs = self.build_perfdocs_from_tree() + perfdocs_tmpdir = self._create_temp_dir() + + # Save the documentation files + frameworks = [] + for framework_name in sorted(framework_docs.keys()): + frameworks.append(framework_name) + save_file( + framework_docs[framework_name]["dynamic"], + pathlib.Path(perfdocs_tmpdir, framework_name), + ) + + for static_name in framework_docs[framework_name]["static"]: + if static_name["file"].endswith(".rst"): + # XXX Replace this with a shutil.copy call (like below) + save_file( + static_name["content"], + pathlib.Path( + perfdocs_tmpdir, static_name["file"].split(".")[0] + ), + ) + else: + shutil.copy( + static_name["content"], + pathlib.Path(perfdocs_tmpdir, static_name["file"]), + ) + + # Get the main page and add the framework links to it + mainpage = read_file( + pathlib.Path(self.templates_path, "index.rst"), stringify=True + ) + + fmt_frameworks = "\n".join([" * :doc:`%s`" % name for name in frameworks]) + fmt_toctree = "\n".join([" %s" % name for name in frameworks]) + + fmt_mainpage = re.sub(r"{toctree_documentation}", fmt_toctree, mainpage) + fmt_mainpage = re.sub(r"{test_documentation}", fmt_frameworks, fmt_mainpage) + + save_file(fmt_mainpage, pathlib.Path(perfdocs_tmpdir, "index")) + + return perfdocs_tmpdir + + def _save_perfdocs(self, perfdocs_tmpdir): + """ + Copies the perfdocs tree after it was saved into the perfdocs_tmpdir + :param perfdocs_tmpdir: str location of the temp dir where the + perfdocs was saved + """ + # Remove the old docs and copy the new version there without + # checking if they need to be regenerated. + logger.log("Regenerating perfdocs...") + + if self.perfdocs_path.exists(): + shutil.rmtree(str(self.perfdocs_path)) + + try: + saved = shutil.copytree(str(perfdocs_tmpdir), str(self.perfdocs_path)) + if saved: + logger.log( + "Documentation saved to {}/".format( + re.sub(".*testing", "testing", str(self.perfdocs_path)) + ) + ) + except Exception as e: + logger.critical( + "There was an error while saving the documentation: {}".format(e) + ) + + def generate_perfdocs(self): + """ + Generate the performance documentation. + + If `self._generate` is True, then the documentation will be regenerated + without any checks. Otherwise, if it is False, the new documentation will be + prepare and compare with the existing documentation to determine if + it should be regenerated. + + :return bool: True/False - For True, if `self._generate` is True, then the + docs were regenerated. If `self._generate` is False, then True will mean + that the docs should be regenerated, and False means that they do not + need to be regenerated. + """ + + def get_possibly_changed_files(): + """ + Returns files that might have been modified + (used to output a linter warning for regeneration) + :return: list - files that might have been modified + """ + # Returns files that might have been modified + # (used to output a linter warning for regeneration) + files = [] + for entry in self._perfdocs_tree: + files.extend( + [ + pathlib.Path(entry["path"], entry["yml"]), + pathlib.Path(entry["path"], entry["rst"]), + ] + ) + return files + + # Throw a warning if there's no need for generating + if not self.perfdocs_path.exists() and not self._generate: + # If they don't exist and we are not generating, then throw + # a linting error and exit. + logger.warning( + "PerfDocs need to be regenerated.", files=get_possibly_changed_files() + ) + return True + + perfdocs_tmpdir = self._create_perfdocs() + if self._generate: + self._save_perfdocs(perfdocs_tmpdir) + else: + # If we are not generating, then at least check if they + # should be regenerated by comparing the directories. + if not are_dirs_equal(perfdocs_tmpdir, self.perfdocs_path): + logger.warning( + "PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` " + + "to update them. You can also apply the " + + f"{'perfdocs.diff' if ON_TRY else 'diff.txt'} patch file " + + f"{'produced from this reviewbot test ' if ON_TRY else ''}" + + "to fix the issue.", + files=get_changed_files(self._workspace), + restricted=False, + ) diff --git a/tools/lint/perfdocs/logger.py b/tools/lint/perfdocs/logger.py new file mode 100644 index 0000000000..ba02532c32 --- /dev/null +++ b/tools/lint/perfdocs/logger.py @@ -0,0 +1,95 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import pathlib + + +class PerfDocLogger(object): + """ + Logger for the PerfDoc tooling. Handles the warnings by outputting + them into through the StructuredLogger provided by lint. + """ + + TOP_DIR = "" + PATHS = [] + LOGGER = None + FAILED = False + + def __init__(self): + """Initializes the PerfDocLogger.""" + + # Set up class attributes for all logger instances + if not PerfDocLogger.LOGGER: + raise Exception( + "Missing linting LOGGER instance for PerfDocLogger initialization" + ) + if not PerfDocLogger.PATHS: + raise Exception("Missing PATHS for PerfDocLogger initialization") + self.logger = PerfDocLogger.LOGGER + + def log(self, msg): + """ + Log an info message. + + :param str msg: Message to log. + """ + self.logger.info(msg) + + def warning(self, msg, files, restricted=True): + """ + Logs a validation warning message. The warning message is + used as the error message that is output in the reviewbot. + + :param str msg: Message to log, it's also used as the error message + for the issue that is output by the reviewbot. + :param list/str files: The file(s) that this warning is about. + :param boolean restricted: If the param is False, the lint error can be used anywhere. + """ + if type(files) != list: + files = [files] + + if len(files) == 0: + self.logger.info("No file was provided for the warning") + self.logger.lint_error( + message=msg, + lineno=0, + column=None, + path=None, + linter="perfdocs", + rule="Flawless performance docs (unknown file)", + ) + + PerfDocLogger.FAILED = True + return + + # Add a reviewbot error for each file that is given + for file in files: + # Get a relative path (reviewbot can't handle absolute paths) + fpath = str(file).replace(str(PerfDocLogger.TOP_DIR), "") + + # Filter out any issues that do not relate to the paths + # that are being linted + for path in PerfDocLogger.PATHS: + if restricted and str(path) not in str(file): + continue + + # Output error entry + self.logger.lint_error( + message=msg, + lineno=0, + column=None, + path=str(pathlib.PurePosixPath(fpath)), + linter="perfdocs", + rule="Flawless performance docs.", + ) + + PerfDocLogger.FAILED = True + break + + def critical(self, msg): + """ + Log a critical message. + + :param str msg: Message to log. + """ + self.logger.critical(msg) diff --git a/tools/lint/perfdocs/perfdocs.py b/tools/lint/perfdocs/perfdocs.py new file mode 100644 index 0000000000..b41edb1979 --- /dev/null +++ b/tools/lint/perfdocs/perfdocs.py @@ -0,0 +1,95 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import pathlib + + +def run_perfdocs(config, logger=None, paths=None, generate=True): + """ + Build up performance testing documentation dynamically by combining + text data from YAML files that reside in `perfdoc` folders + across the `testing` directory. Each directory is expected to have + an `index.rst` file along with `config.yml` YAMLs defining what needs + to be added to the documentation. + + The YAML must also define the name of the "framework" that should be + used in the main index.rst for the performance testing documentation. + + The testing documentation list will be ordered alphabetically once + it's produced (to avoid unwanted shifts because of unordered dicts + and path searching). + + Note that the suite name headings will be given the H4 (---) style so it + is suggested that you use H3 (===) style as the heading for your + test section. H5 will be used be used for individual tests within each + suite. + + Usage for verification: "./mach lint -l perfdocs ." + Usage for generation: "./mach lint -l perfdocs --fix ." + + For validation, see the Verifier class for a description of how + it works. + + The run will fail if the valid result from validate_tree is not + False, implying some warning/problem was logged. + + :param dict config: The configuration given by mozlint. + :param StructuredLogger logger: The StructuredLogger instance to be used to + output the linting warnings/errors. + :param list paths: The paths that are being tested. Used to filter + out errors from files outside of these paths. + :param bool generate: If true, the docs will be (re)generated. + """ + from perfdocs.logger import PerfDocLogger + + if not os.environ.get("WORKSPACE", None): + floc = pathlib.Path(__file__).absolute() + top_dir = pathlib.Path(str(floc).split("tools")[0]).resolve() + else: + top_dir = pathlib.Path(os.environ.get("WORKSPACE")).resolve() + + PerfDocLogger.LOGGER = logger + PerfDocLogger.TOP_DIR = top_dir + + # Convert all the paths to relative ones + target_dir = [pathlib.Path(path) for path in paths] + rel_paths = [] + for path in target_dir: + try: + rel_paths.append(path.relative_to(top_dir)) + except ValueError: + rel_paths.append(path) + + PerfDocLogger.PATHS = rel_paths + + for path in target_dir: + if not path.exists(): + raise Exception("Cannot locate directory at %s" % str(path)) + + decision_task_id = os.environ.get("DECISION_TASK_ID", None) + if decision_task_id: + from taskgraph.util.taskcluster import get_artifact + + task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") + else: + from tryselect.tasks import generate_tasks + + task_graph = generate_tasks( + params=None, full=True, disable_target_task_filter=True + ).tasks + + # Late import because logger isn't defined until later + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + # Run the verifier first + verifier = Verifier(top_dir, task_graph) + verifier.validate_tree() + + if not PerfDocLogger.FAILED: + # Even if the tree is valid, we need to check if the documentation + # needs to be regenerated, and if it does, we throw a linting error. + # `generate` dictates whether or not the documentation is generated. + generator = Generator(verifier, generate=generate, workspace=top_dir) + generator.generate_perfdocs() diff --git a/tools/lint/perfdocs/templates/index.rst b/tools/lint/perfdocs/templates/index.rst new file mode 100644 index 0000000000..d2d82f6328 --- /dev/null +++ b/tools/lint/perfdocs/templates/index.rst @@ -0,0 +1,86 @@ +################### +Performance Testing +################### + +.. toctree:: + :maxdepth: 2 + :hidden: + :glob: + +{toctree_documentation} + +Performance tests are designed to catch performance regressions before they reach our +end users. At this time, there is no unified approach for these types of tests, +but `mozperftest </testing/perfdocs/mozperftest.html>`_ aims to provide this in the future. + +For more detailed information about each test suite and project, see their documentation: + +{test_documentation} + + +Here are the active PerfTest components/modules and their respective owners: + + * AWFY (Are We Fast Yet) - + - Owner: Beatrice A. + - Description: A public dashboard comparing Firefox and Chrome performance metrics + * AWSY (Are We Slim Yet) + - Owner: Alexandru I. + - Description: Project that tracks memory usage across builds + * Raptor + - Owner: Sparky + - Co-owner: Kash + - Description: Test harness that uses Browsertime (based on webdriver) as the underlying engine to run performance tests + * CondProf (Conditioned profiles) + - Owner: Sparky + - Co-owner: Jmaher + - Description: Provides tooling to build, and obtain profiles that are preconditioned in some way. + * fxrecord + - Owner: Sparky + - Co-owners: Kash, Andrej + - Description: Tool for measuring startup performance for Firefox Desktop + * Infrastructure + - Owner: Sparky + - Co-owners: Kash, Andrej + - Description: All things involving: TaskCluster, Youtube Playback, Bitbar, Mobile Configs, etc... + * Mozperftest + - Owner: Sparky + - Co-owners: Kash, Andrej + - Description: Testing framework used to run performance tests + * Mozperftest Tools + - Owner: Sparky + - Co-owner: Alexandru I. + - Description: Various tools used by performance testing team + * Mozproxy + - Owner: Sparky + - Co-owner: Kash + - Description: An http proxy used to run tests against third-party websites in a reliable and reproducible way + * PerfCompare + - Owner: Carla S. + - Co-owner: Beatrice A. + - Description: Performance comparison tool used to compare performance of different commits within a repository + * PerfDocs + - Owner: Sparky + - Co-owner: Alexandru I. + - Description: Automatically generated performance test engineering documentation + * PerfHerder + - Owner: Beatrice A + - Co-owner: Andra A. + - Description: The framework used by the performance sheriffs to find performance regressions and for storing, and visualizing our performance data. + * Performance Sheriffing + - Owner: Alexandru I. + - Co-owner: Andra A. + - Description: Performance sheriffs are responsible for finding commits that cause performance regressions and getting fixes from devs or backing out the changes + * Talos + - Owner: Sparky + - Co-owner: Andrej + - Description: Testing framework used to run Firefox-specific performance tests + * WebPageTest + - Owner: Andrej + - Co-owner: Sparky + - Description: A test running in the mozperftest framework used as a third party performance benchmark + +You can additionally reach out to our team on +the `#perftest <https://matrix.to/#/#perftest:mozilla.org>`_ channel on matrix + +For more information about the performance testing team, +`visit the wiki page <https://wiki.mozilla.org/TestEngineering/Performance>`_. diff --git a/tools/lint/perfdocs/utils.py b/tools/lint/perfdocs/utils.py new file mode 100644 index 0000000000..1ba7daeb52 --- /dev/null +++ b/tools/lint/perfdocs/utils.py @@ -0,0 +1,157 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import difflib +import filecmp +import os +import pathlib + +import yaml +from mozversioncontrol import get_repository_object + +from perfdocs.logger import PerfDocLogger + +logger = PerfDocLogger() + +ON_TRY = "MOZ_AUTOMATION" in os.environ + + +def save_file(file_content, path, extension="rst"): + """ + Saves data into a file. + + :param str path: Location and name of the file being saved + (without an extension). + :param str data: Content to write into the file. + :param str extension: Extension to save the file as. + """ + new_file = pathlib.Path("{}.{}".format(str(path), extension)) + with new_file.open("wb") as f: + f.write(file_content.encode("utf-8")) + + +def read_file(path, stringify=False): + """ + Opens a file and returns its contents. + + :param str path: Path to the file. + :return list: List containing the lines in the file. + """ + with path.open(encoding="utf-8") as f: + return f.read() if stringify else f.readlines() + + +def read_yaml(yaml_path): + """ + Opens a YAML file and returns the contents. + + :param str yaml_path: Path to the YAML to open. + :return dict: Dictionary containing the YAML content. + """ + contents = {} + try: + with yaml_path.open(encoding="utf-8") as f: + contents = yaml.safe_load(f) + except Exception as e: + logger.warning( + "Error opening file {}: {}".format(str(yaml_path), str(e)), str(yaml_path) + ) + + return contents + + +def are_dirs_equal(dir_1, dir_2): + """ + Compare two directories to see if they are equal. Files in each + directory are assumed to be equal if their names and contents + are equal. + + :param dir_1: First directory path + :param dir_2: Second directory path + :return: True if the directory trees are the same and False otherwise. + """ + + dirs_cmp = filecmp.dircmp(str(dir_1.resolve()), str(dir_2.resolve())) + if dirs_cmp.left_only or dirs_cmp.right_only or dirs_cmp.funny_files: + logger.log("Some files are missing or are funny.") + for file in dirs_cmp.left_only: + logger.log(f"Missing in existing docs: {file}") + for file in dirs_cmp.right_only: + logger.log(f"Missing in new docs: {file}") + for file in dirs_cmp.funny_files: + logger.log(f"The following file is funny: {file}") + return False + + _, mismatch, errors = filecmp.cmpfiles( + str(dir_1.resolve()), str(dir_2.resolve()), dirs_cmp.common_files, shallow=False + ) + + if mismatch or errors: + logger.log(f"Found mismatches: {mismatch}") + + # The root for where to save the diff will be different based on + # whether we are running in CI or not + os_root = pathlib.Path.cwd().anchor + diff_root = pathlib.Path(os_root, "builds", "worker") + if not ON_TRY: + diff_root = pathlib.Path(PerfDocLogger.TOP_DIR, "artifacts") + diff_root.mkdir(parents=True, exist_ok=True) + + diff_path = pathlib.Path(diff_root, "diff.txt") + with diff_path.open("w", encoding="utf-8") as diff_file: + for entry in mismatch: + logger.log(f"Mismatch found on {entry}") + + with pathlib.Path(dir_1, entry).open(encoding="utf-8") as f: + newlines = f.readlines() + with pathlib.Path(dir_2, entry).open(encoding="utf-8") as f: + baselines = f.readlines() + for line in difflib.unified_diff( + baselines, newlines, fromfile="base", tofile="new" + ): + logger.log(line) + + # Here we want to add to diff.txt in a patch format, we use + # the basedir to make the file names/paths relative and this is + # different in CI vs local runs. + basedir = pathlib.Path( + os_root, "builds", "worker", "checkouts", "gecko" + ) + if not ON_TRY: + basedir = diff_root + + relative_path = str(pathlib.Path(dir_2, entry)).split(str(basedir))[-1] + patch = difflib.unified_diff( + baselines, newlines, fromfile=relative_path, tofile=relative_path + ) + + write_header = True + for line in patch: + if write_header: + diff_file.write( + f"diff --git a/{relative_path} b/{relative_path}\n" + ) + write_header = False + diff_file.write(line) + + logger.log(f"Completed diff on {entry}") + + logger.log(f"Saved diff to {diff_path}") + + return False + + for common_dir in dirs_cmp.common_dirs: + subdir_1 = pathlib.Path(dir_1, common_dir) + subdir_2 = pathlib.Path(dir_2, common_dir) + if not are_dirs_equal(subdir_1, subdir_2): + return False + + return True + + +def get_changed_files(top_dir): + """ + Returns the changed files found with duplicates removed. + """ + repo = get_repository_object(top_dir) + return list(set(repo.get_changed_files() + repo.get_outgoing_files())) diff --git a/tools/lint/perfdocs/verifier.py b/tools/lint/perfdocs/verifier.py new file mode 100644 index 0000000000..6603b9acce --- /dev/null +++ b/tools/lint/perfdocs/verifier.py @@ -0,0 +1,601 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +import pathlib +import re + +import jsonschema + +from perfdocs.gatherer import Gatherer +from perfdocs.logger import PerfDocLogger +from perfdocs.utils import read_file, read_yaml + +logger = PerfDocLogger() + +""" +Schema for the config.yml file. +Expecting a YAML file with a format such as this: + +name: raptor +manifest: testing/raptor/raptor/raptor.toml +static-only: False +suites: + desktop: + description: "Desktop tests." + tests: + raptor-tp6: "Raptor TP6 tests." + mobile: + description: "Mobile tests" + benchmarks: + description: "Benchmark tests." + tests: + wasm: "All wasm tests." + +""" +CONFIG_SCHEMA = { + "definitions": { + "metrics_schema": { + "metric_name": { + "type": "object", + "properties": { + "aliases": {"type": "array", "items": {"type": "string"}}, + "description": {"type": "string"}, + "matcher": {"type": "string"}, + }, + "required": ["description", "aliases"], + }, + }, + }, + "type": "object", + "properties": { + "name": {"type": "string"}, + "manifest": {"type": "string"}, + "static-only": {"type": "boolean"}, + "metrics": {"$ref": "#/definitions/metrics_schema"}, + "suites": { + "type": "object", + "properties": { + "suite_name": { + "type": "object", + "properties": { + "tests": { + "type": "object", + "properties": { + "test_name": {"type": "string"}, + "metrics": {"$ref": "#/definitions/metrics_schema"}, + }, + }, + "description": {"type": "string"}, + "owner": {"type": "string"}, + "metrics": {"$ref": "#/definitions/metrics_schema"}, + }, + "required": ["description"], + } + }, + }, + }, + "required": ["name", "manifest", "static-only", "suites"], +} + + +class Verifier(object): + """ + Verifier is used for validating the perfdocs folders/tree. In the future, + the generator will make use of this class to obtain a validated set of + descriptions that can be used to build up a document. + """ + + def __init__(self, workspace_dir, taskgraph=None): + """ + Initialize the Verifier. + + :param str workspace_dir: Path to the top-level checkout directory. + """ + self.workspace_dir = workspace_dir + self._gatherer = Gatherer(workspace_dir, taskgraph) + self._compiled_matchers = {} + + def _is_yaml_test_match( + self, target_test_name, test_name, suite="", global_descriptions={} + ): + """Determine if a target name (from a YAML) matches with a test.""" + tb = os.path.basename(target_test_name) + tb = re.sub("\..*", "", tb) + if test_name == tb: + # Found an exact match for the test_name + return True + if test_name in tb: + # Found a 'fuzzy' match for the test_name + # i.e. 'wasm' could exist for all raptor wasm tests + global_descriptions.setdefault(suite, []).append(test_name) + return True + + def _validate_desc_yaml_direction( + self, suite, framework_info, yaml_content, global_descriptions + ): + """Validate the descriptions in the YAML. + + This validation ensures that all tests defined in the YAML exist in the test + harness. Failures here suggest that there's a typo in the YAML or that + a test was removed. + """ + ytests = yaml_content["suites"][suite] + global_descriptions[suite] = [] + if not ytests.get("tests"): + # It's possible a suite entry has no tests + return True + + # Suite found - now check if any tests in YAML + # definitions don't exist + ytests = ytests["tests"] + for test_name in ytests: + foundtest = False + for t in framework_info["test_list"][suite]: + if self._is_yaml_test_match( + t, test_name, suite=suite, global_descriptions=global_descriptions + ): + foundtest = True + break + if not foundtest: + logger.warning( + "Could not find an existing test for {} - bad test name?".format( + test_name + ), + framework_info["yml_path"], + ) + return False + + def _validate_desc_harness_direction( + self, suite, test_list, yaml_content, global_descriptions + ): + """Validate that the tests have a description in the YAML. + + This stage of validation ensures that all the tests have some + form of description, or that global descriptions are available. + Failures here suggest a new test was added, or the config.yml + file was changed. + """ + # If only a description is provided for the suite, assume + # that this is a suite-wide description and don't check for + # it's tests + stests = yaml_content["suites"][suite].get("tests", None) + if not stests: + return + + tests_found = 0 + missing_tests = [] + test_to_manifest = {} + for test_name, test_info in test_list.items(): + manifest_path = test_info.get("path", test_info.get("manifest", "")) + tb = os.path.basename(manifest_path) + tb = re.sub("\..*", "", tb) + if ( + stests.get(tb, None) is not None + or stests.get(test_name, None) is not None + ): + # Test description exists, continue with the next test + tests_found += 1 + continue + test_to_manifest[test_name] = manifest_path + missing_tests.append(test_name) + + # Check if global test descriptions exist (i.e. + # ones that cover all of tp6) for the missing tests + new_mtests = [] + for mt in missing_tests: + found = False + for test_name in global_descriptions[suite]: + # Global test exists for this missing test + if mt.startswith(test_name): + found = True + break + if test_name in mt: + found = True + break + if not found: + new_mtests.append(mt) + + if len(new_mtests): + # Output an error for each manifest with a missing + # test description + for test_name in new_mtests: + logger.warning( + "Could not find a test description for {}".format(test_name), + test_to_manifest[test_name], + ) + + def _match_metrics(self, target_metric_name, target_metric_info, measured_metrics): + """Find all metrics that match the given information. + + It either checks for the metric through a direct equality check, and if + a regex matcher was provided, we will use that afterwards. + """ + verified_metrics = [] + + metric_names = target_metric_info["aliases"] + [target_metric_name] + for measured_metric in measured_metrics: + if measured_metric in metric_names: + verified_metrics.append(measured_metric) + + if target_metric_info.get("matcher", ""): + # Compile the regex separately to capture issues in the regex + # compilation + matcher = self._compiled_matchers.get(target_metric_name, None) + if not matcher: + matcher = re.compile(target_metric_info.get("matcher")) + self._compiled_matchers[target_metric_name] = matcher + + # Search the measured metrics + for measured_metric in measured_metrics: + if matcher.search(measured_metric): + verified_metrics.append(measured_metric) + + return verified_metrics + + def _validate_metrics_yaml_direction( + self, suite, framework_info, yaml_content, global_metrics + ): + """Validate the metric descriptions in the YAML. + + This direction (`yaml_direction`) checks that the YAML definitions exist in + the test harness as real metrics. Failures here suggest that a metric + changed name, is missing an alias, is misnamed, duplicated, or was removed. + """ + yaml_suite = yaml_content["suites"][suite] + suite_metrics = yaml_suite.get("metrics", {}) + + # Check to make sure all the metrics with given descriptions + # are actually being measured. Add the metric to the "verified" field in + # global_metrics to use it later for "global" metrics that can + # have their descriptions removed. Start from the test level. + for test_name, test_info in yaml_suite.get("tests", {}).items(): + if not isinstance(test_info, dict): + continue + test_metrics_info = test_info.get("metrics", {}) + + # Find all tests that match with this name in case they measure + # different things + measured_metrics = [] + for t in framework_info["test_list"][suite]: + if not self._is_yaml_test_match(t, test_name): + # Check to make sure we are checking against the right + # test. Skip the metric check if we can't find the test. + continue + measured_metrics.extend( + framework_info["test_list"][suite][t].get("metrics", []) + ) + + if len(measured_metrics) == 0: + continue + + # Check if all the test metrics documented exist + for metric_name, metric_info in test_metrics_info.items(): + verified_metrics = self._match_metrics( + metric_name, metric_info, measured_metrics + ) + if len(verified_metrics) > 0: + global_metrics["yaml-verified"].extend( + [metric_name] + metric_info["aliases"] + ) + global_metrics["verified"].extend( + [metric_name] + metric_info["aliases"] + verified_metrics + ) + else: + logger.warning( + ( + "Cannot find documented metric `{}` " + "being used in the specified test `{}`." + ).format(metric_name, test_name), + framework_info["yml_path"], + ) + + # Check the suite level now + for suite_metric_name, suite_metric_info in suite_metrics.items(): + measured_metrics = [] + for _, test_info in framework_info["test_list"][suite].items(): + measured_metrics.extend(test_info.get("metrics", [])) + + verified_metrics = self._match_metrics( + suite_metric_name, suite_metric_info, measured_metrics + ) + if len(verified_metrics) > 0: + global_metrics["yaml-verified"].extend( + [suite_metric_name] + suite_metric_info["aliases"] + ) + global_metrics["verified"].extend( + [suite_metric_name] + + suite_metric_info["aliases"] + + verified_metrics + ) + else: + logger.warning( + ( + "Cannot find documented metric `{}` " + "being used in the specified suite `{}`." + ).format(suite_metric_name, suite), + framework_info["yml_path"], + ) + + # Finally check the global level (output failures later) + all_measured_metrics = [] + for _, test_info in framework_info["test_list"][suite].items(): + all_measured_metrics.extend(test_info.get("metrics", [])) + for global_metric_name, global_metric_info in global_metrics["global"].items(): + verified_metrics = self._match_metrics( + global_metric_name, global_metric_info, all_measured_metrics + ) + if global_metric_info.get("verified", False): + # We already verified this global metric, but add any + # extra verified metrics here + global_metrics["verified"].extend(verified_metrics) + continue + if len(verified_metrics) > 0: + global_metric_info["verified"] = True + global_metrics["yaml-verified"].extend( + [global_metric_name] + global_metric_info["aliases"] + ) + global_metrics["verified"].extend( + [global_metric_name] + + global_metric_info["aliases"] + + verified_metrics + ) + + def _validate_metrics_harness_direction( + self, suite, test_list, yaml_content, global_metrics + ): + """Validate that metrics in the harness are documented.""" + # Gather all the metrics being measured + all_measured_metrics = {} + for test_name, test_info in test_list.items(): + metrics = test_info.get("metrics", []) + for metric in metrics: + all_measured_metrics.setdefault(metric, []).append(test_name) + + if len(all_measured_metrics) == 0: + # There are no metrics measured by this suite + return + + for metric, tests in all_measured_metrics.items(): + if metric not in global_metrics["verified"]: + # Log a warning in all files that have this metric + for test in tests: + logger.warning( + "Missing description for the metric `{}` in test `{}`".format( + metric, test + ), + test_list[test].get( + "path", test_list[test].get("manifest", "") + ), + ) + + def validate_descriptions(self, framework_info): + """ + Cross-validate the tests found in the manifests and the YAML + test definitions. This function doesn't return a valid flag. Instead, + the StructDocLogger.VALIDATION_LOG is used to determine validity. + + The validation proceeds as follows: + 1. Check that all tests/suites in the YAML exist in the manifests. + - At the same time, build a list of global descriptions which + define descriptions for groupings of tests. + 2. Check that all tests/suites found in the manifests exist in the YAML. + - For missing tests, check if a global description for them exists. + + As the validation is completed, errors are output into the validation log + for any issues that are found. + + The same is done for the metrics field expect it also has regex matching, + and the definitions cannot be duplicated in a single harness. We make use + of two `*verified` fields to simplify the two stages/directions, and checking + for any duplication. + + :param dict framework_info: Contains information about the framework. See + `Gatherer.get_test_list` for information about its structure. + """ + yaml_content = framework_info["yml_content"] + + # Check for any bad test/suite names in the yaml config file + # TODO: Combine global settings into a single dictionary + global_descriptions = {} + global_metrics = { + "global": yaml_content.get("metrics", {}), + "verified": [], + "yaml-verified": [], + } + for suite, ytests in yaml_content["suites"].items(): + # Find the suite, then check against the tests within it + if framework_info["test_list"].get(suite, None) is None: + logger.warning( + "Could not find an existing suite for {} - bad suite name?".format( + suite + ), + framework_info["yml_path"], + ) + continue + + # Validate descriptions + self._validate_desc_yaml_direction( + suite, framework_info, yaml_content, global_descriptions + ) + + # Validate metrics + self._validate_metrics_yaml_direction( + suite, framework_info, yaml_content, global_metrics + ) + + # The suite and test levels were properly checked, but we can only + # check the global level after all suites were checked. If the metric + # isn't in the verified + for global_metric_name, _ in global_metrics["global"].items(): + if global_metric_name not in global_metrics["verified"]: + logger.warning( + ( + "Cannot find documented metric `{}` " + "being used in the specified harness `{}`." + ).format(global_metric_name, yaml_content["name"]), + framework_info["yml_path"], + ) + + # Check for duplicate metrics/aliases in the verified metrics + unique_metrics = set() + warned = set() + for metric in global_metrics["yaml-verified"]: + if ( + metric in unique_metrics or unique_metrics.add(metric) + ) and metric not in warned: + logger.warning( + "Duplicate definitions found for `{}`.".format(metric), + framework_info["yml_path"], + ) + warned.add(metric) + + # Check for duplicate metrics in the global level + unique_metrics = set() + warned = set() + for metric, metric_info in global_metrics["global"].items(): + if ( + metric in unique_metrics or unique_metrics.add(metric) + ) and metric not in warned: + logger.warning( + "Duplicate definitions found for `{}`.".format(metric), + framework_info["yml_path"], + ) + for alias in metric_info.get("aliases", []): + unique_metrics.add(alias) + warned.add(alias) + warned.add(metric) + + # Check for any missing tests/suites + for suite, test_list in framework_info["test_list"].items(): + if not yaml_content["suites"].get(suite): + # Description doesn't exist for the suite + logger.warning( + "Missing suite description for {}".format(suite), + yaml_content["manifest"], + ) + continue + + self._validate_desc_harness_direction( + suite, test_list, yaml_content, global_descriptions + ) + + self._validate_metrics_harness_direction( + suite, test_list, yaml_content, global_metrics + ) + + def validate_yaml(self, yaml_path): + """ + Validate that the YAML file has all the fields that are + required and parse the descriptions into strings in case + some are give as relative file paths. + + :param str yaml_path: Path to the YAML to validate. + :return bool: True/False => Passed/Failed Validation + """ + + def _get_description(desc): + """ + Recompute the description in case it's a file. + """ + desc_path = pathlib.Path(self.workspace_dir, desc) + + try: + if desc_path.exists() and desc_path.is_file(): + with open(desc_path, "r") as f: + desc = f.readlines() + except OSError: + pass + + return desc + + def _parse_descriptions(content): + for suite, sinfo in content.items(): + desc = sinfo["description"] + sinfo["description"] = _get_description(desc) + + # It's possible that the suite has no tests and + # only a description. If they exist, then parse them. + if "tests" in sinfo: + for test, desc in sinfo["tests"].items(): + sinfo["tests"][test] = _get_description(desc) + + valid = False + yaml_content = read_yaml(yaml_path) + + try: + jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA) + _parse_descriptions(yaml_content["suites"]) + valid = True + except Exception as e: + logger.warning("YAML ValidationError: {}".format(str(e)), yaml_path) + + return valid + + def validate_rst_content(self, rst_path): + """ + Validate that the index file given has a {documentation} entry + so that the documentation can be inserted there. + + :param str rst_path: Path to the RST file. + :return bool: True/False => Passed/Failed Validation + """ + rst_content = read_file(rst_path) + + # Check for a {documentation} entry in some line, + # if we can't find one, then the validation fails. + valid = False + docs_match = re.compile(".*{documentation}.*") + for line in rst_content: + if docs_match.search(line): + valid = True + break + if not valid: + logger.warning( # noqa: PLE1205 + "Cannot find a '{documentation}' entry in the given index file", + rst_path, + ) + + return valid + + def _check_framework_descriptions(self, item): + """ + Helper method for validating descriptions + """ + framework_info = self._gatherer.get_test_list(item) + self.validate_descriptions(framework_info) + + def validate_tree(self): + """ + Validate the `perfdocs` directory that was found. + Returns True if it is good, false otherwise. + + :return bool: True/False => Passed/Failed Validation + """ + found_good = 0 + + # For each framework, check their files and validate descriptions + for matched in self._gatherer.perfdocs_tree: + # Get the paths to the YAML and RST for this framework + matched_yml = pathlib.Path(matched["path"], matched["yml"]) + matched_rst = pathlib.Path(matched["path"], matched["rst"]) + + _valid_files = { + "yml": self.validate_yaml(matched_yml), + "rst": True, + } + if not read_yaml(matched_yml)["static-only"]: + _valid_files["rst"] = self.validate_rst_content(matched_rst) + + # Log independently the errors found for the matched files + for file_format, valid in _valid_files.items(): + if not valid: + logger.log("File validation error: {}".format(file_format)) + if not all(_valid_files.values()): + continue + found_good += 1 + + self._check_framework_descriptions(matched) + + if not found_good: + raise Exception("No valid perfdocs directories found") diff --git a/tools/lint/python/__init__.py b/tools/lint/python/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/lint/python/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/lint/python/black.py b/tools/lint/python/black.py new file mode 100644 index 0000000000..1e38dd669c --- /dev/null +++ b/tools/lint/python/black.py @@ -0,0 +1,168 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import platform +import re +import signal +import subprocess +import sys + +import mozpack.path as mozpath +from mozfile import which +from mozlint import result +from mozlint.pathutils import expand_exclusions + +here = os.path.abspath(os.path.dirname(__file__)) +BLACK_REQUIREMENTS_PATH = os.path.join(here, "black_requirements.txt") + +BLACK_INSTALL_ERROR = """ +Unable to install correct version of black +Try to install it manually with: + $ pip install -U --require-hashes -r {} +""".strip().format( + BLACK_REQUIREMENTS_PATH +) + + +def default_bindir(): + # We use sys.prefix to find executables as that gets modified with + # virtualenv's activate_this.py, whereas sys.executable doesn't. + if platform.system() == "Windows": + return os.path.join(sys.prefix, "Scripts") + else: + return os.path.join(sys.prefix, "bin") + + +def get_black_version(binary): + """ + Returns found binary's version + """ + try: + output = subprocess.check_output( + [binary, "--version"], + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + except subprocess.CalledProcessError as e: + output = e.output + try: + # Accept `black.EXE, version ...` on Windows. + # for old version of black, the output is + # black, version 21.4b2 + # From black 21.11b1, the output is like + # black, 21.11b1 (compiled: no) + return re.match(r"black.*,( version)? (\S+)", output)[2] + except TypeError as e: + print("Could not parse the version '{}'".format(output)) + print("Error: {}".format(e)) + + +def parse_issues(config, output, paths, *, log): + would_reformat = re.compile("^would reformat (.*)$", re.I) + reformatted = re.compile("^reformatted (.*)$", re.I) + cannot_reformat = re.compile("^error: cannot format (.*?): (.*)$", re.I) + results = [] + for l in output.split(b"\n"): + line = l.decode("utf-8").rstrip("\r\n") + if line.startswith("All done!") or line.startswith("Oh no!"): + break + + match = would_reformat.match(line) + if match: + res = {"path": match.group(1), "level": "error"} + results.append(result.from_config(config, **res)) + continue + + match = reformatted.match(line) + if match: + res = {"path": match.group(1), "level": "warning", "message": "reformatted"} + results.append(result.from_config(config, **res)) + continue + + match = cannot_reformat.match(line) + if match: + res = {"path": match.group(1), "level": "error", "message": match.group(2)} + results.append(result.from_config(config, **res)) + continue + + log.debug(f"Unhandled line: {line}") + return results + + +def run_process(config, cmd): + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + signal.signal(signal.SIGINT, orig) + try: + output, _ = proc.communicate() + proc.wait() + except KeyboardInterrupt: + proc.kill() + + return output + + +def setup(root, **lintargs): + log = lintargs["log"] + virtualenv_bin_path = lintargs.get("virtualenv_bin_path") + # Using `which` searches multiple directories and handles `.exe` on Windows. + binary = which("black", path=(virtualenv_bin_path, default_bindir())) + + if binary and os.path.exists(binary): + binary = mozpath.normsep(binary) + log.debug("Looking for black at {}".format(binary)) + version = get_black_version(binary) + versions = [ + line.split()[0].strip() + for line in open(BLACK_REQUIREMENTS_PATH).readlines() + if line.startswith("black==") + ] + if ["black=={}".format(version)] == versions: + log.debug("Black is present with expected version {}".format(version)) + return 0 + else: + log.debug("Black is present but unexpected version {}".format(version)) + + log.debug("Black needs to be installed or updated") + virtualenv_manager = lintargs["virtualenv_manager"] + try: + virtualenv_manager.install_pip_requirements(BLACK_REQUIREMENTS_PATH, quiet=True) + except subprocess.CalledProcessError: + print(BLACK_INSTALL_ERROR) + return 1 + + +def run_black(config, paths, fix=None, *, log, virtualenv_bin_path): + fixed = 0 + binary = os.path.join(virtualenv_bin_path or default_bindir(), "black") + + log.debug("Black version {}".format(get_black_version(binary))) + + cmd_args = [binary] + if not fix: + cmd_args.append("--check") + + base_command = cmd_args + paths + log.debug("Command: {}".format(" ".join(base_command))) + output = parse_issues(config, run_process(config, base_command), paths, log=log) + + # black returns an issue for fixed files as well + for eachIssue in output: + if eachIssue.message == "reformatted": + fixed += 1 + + return {"results": output, "fixed": fixed} + + +def lint(paths, config, fix=None, **lintargs): + files = list(expand_exclusions(paths, config, lintargs["root"])) + + return run_black( + config, + files, + fix=fix, + log=lintargs["log"], + virtualenv_bin_path=lintargs.get("virtualenv_bin_path"), + ) diff --git a/tools/lint/python/black_requirements.in b/tools/lint/python/black_requirements.in new file mode 100644 index 0000000000..dfe5a54c7b --- /dev/null +++ b/tools/lint/python/black_requirements.in @@ -0,0 +1,5 @@ +black==23.3.0 +typing-extensions==3.10.0.2 +dataclasses==0.6 +typed-ast==1.4.2; python_version < '3.8' +pkgutil-resolve-name==1.3.10 ; python_version < '3.9' diff --git a/tools/lint/python/black_requirements.txt b/tools/lint/python/black_requirements.txt new file mode 100644 index 0000000000..9e89927775 --- /dev/null +++ b/tools/lint/python/black_requirements.txt @@ -0,0 +1,115 @@ +# +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: +# +# pip-compile --config=pyproject.toml --generate-hashes --output-file=tools/lint/python/black_requirements.txt ./tools/lint/python/black_requirements.in +# +black==23.3.0 \ + --hash=sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5 \ + --hash=sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915 \ + --hash=sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326 \ + --hash=sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940 \ + --hash=sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b \ + --hash=sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30 \ + --hash=sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c \ + --hash=sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c \ + --hash=sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab \ + --hash=sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27 \ + --hash=sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2 \ + --hash=sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961 \ + --hash=sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9 \ + --hash=sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb \ + --hash=sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70 \ + --hash=sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331 \ + --hash=sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2 \ + --hash=sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266 \ + --hash=sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d \ + --hash=sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6 \ + --hash=sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b \ + --hash=sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925 \ + --hash=sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8 \ + --hash=sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4 \ + --hash=sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3 + # via -r ./tools/lint/python/black_requirements.in +click==8.0.3 \ + --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \ + --hash=sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b + # via black +dataclasses==0.6 \ + --hash=sha256:454a69d788c7fda44efd71e259be79577822f5e3f53f029a22d08004e951dc9f \ + --hash=sha256:6988bd2b895eef432d562370bb707d540f32f7360ab13da45340101bc2307d84 + # via -r ./tools/lint/python/black_requirements.in +importlib-metadata==6.7.0 \ + --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \ + --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5 + # via click +mypy-extensions==0.4.3 \ + --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ + --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 + # via black +packaging==23.1 \ + --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ + --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f + # via black +pathspec==0.9.0 \ + --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ + --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 + # via black +pkgutil-resolve-name==1.3.10 ; python_version < "3.9" \ + --hash=sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174 \ + --hash=sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e + # via -r ./tools/lint/python/black_requirements.in +platformdirs==2.4.0 \ + --hash=sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2 \ + --hash=sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d + # via black +tomli==1.2.2 \ + --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \ + --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade + # via black +typed-ast==1.4.2 ; python_version < "3.8" \ + --hash=sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1 \ + --hash=sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d \ + --hash=sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6 \ + --hash=sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd \ + --hash=sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37 \ + --hash=sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151 \ + --hash=sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07 \ + --hash=sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440 \ + --hash=sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70 \ + --hash=sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496 \ + --hash=sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea \ + --hash=sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400 \ + --hash=sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc \ + --hash=sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606 \ + --hash=sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc \ + --hash=sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581 \ + --hash=sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412 \ + --hash=sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a \ + --hash=sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2 \ + --hash=sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787 \ + --hash=sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f \ + --hash=sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937 \ + --hash=sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64 \ + --hash=sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487 \ + --hash=sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b \ + --hash=sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41 \ + --hash=sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a \ + --hash=sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3 \ + --hash=sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166 \ + --hash=sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10 + # via + # -r ./tools/lint/python/black_requirements.in + # black +typing-extensions==3.10.0.2 \ + --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ + --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ + --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 + # via + # -r ./tools/lint/python/black_requirements.in + # black + # importlib-metadata +zipp==3.15.0 \ + --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \ + --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556 + # via importlib-metadata diff --git a/tools/lint/python/l10n_lint.py b/tools/lint/python/l10n_lint.py new file mode 100644 index 0000000000..158cb5f7e6 --- /dev/null +++ b/tools/lint/python/l10n_lint.py @@ -0,0 +1,202 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from datetime import datetime, timedelta + +from compare_locales import parser +from compare_locales.lint.linter import L10nLinter +from compare_locales.lint.util import l10n_base_reference_and_tests +from compare_locales.paths import ProjectFiles, TOMLParser +from mach import util as mach_util +from mozlint import pathutils, result +from mozpack import path as mozpath +from mozversioncontrol import MissingVCSTool +from mozversioncontrol.repoupdate import update_git_repo, update_mercurial_repo + +L10N_SOURCE_NAME = "l10n-source" +L10N_SOURCE_REPO = "https://github.com/mozilla-l10n/firefox-l10n-source.git" + +STRINGS_NAME = "gecko-strings" +STRINGS_REPO = "https://hg.mozilla.org/l10n/gecko-strings" + +PULL_AFTER = timedelta(days=2) + + +# Wrapper to call lint_strings with mozilla-central configuration +# comm-central defines its own wrapper since comm-central strings are +# in separate repositories +def lint(paths, lintconfig, **lintargs): + extra_args = lintargs.get("extra_args") or [] + name = L10N_SOURCE_NAME if "--l10n-git" in extra_args else STRINGS_NAME + return lint_strings(name, paths, lintconfig, **lintargs) + + +def lint_strings(name, paths, lintconfig, **lintargs): + l10n_base = mach_util.get_state_dir() + root = lintargs["root"] + exclude = lintconfig.get("exclude") + extensions = lintconfig.get("extensions") + + # Load l10n.toml configs + l10nconfigs = load_configs(lintconfig, root, l10n_base, name) + + # Check include paths in l10n.yml if it's in our given paths + # Only the l10n.yml will show up here, but if the l10n.toml files + # change, we also get the l10n.yml as the toml files are listed as + # support files. + if lintconfig["path"] in paths: + results = validate_linter_includes(lintconfig, l10nconfigs, lintargs) + paths.remove(lintconfig["path"]) + else: + results = [] + + all_files = [] + for p in paths: + fp = pathutils.FilterPath(p) + if fp.isdir: + for _, fileobj in fp.finder: + all_files.append(fileobj.path) + if fp.isfile: + all_files.append(p) + # Filter again, our directories might have picked up files the + # explicitly excluded in the l10n.yml configuration. + # `browser/locales/en-US/firefox-l10n.js` is a good example. + all_files, _ = pathutils.filterpaths( + lintargs["root"], + all_files, + lintconfig["include"], + exclude=exclude, + extensions=extensions, + ) + # These should be excluded in l10n.yml + skips = {p for p in all_files if not parser.hasParser(p)} + results.extend( + result.from_config( + lintconfig, + level="warning", + path=path, + message="file format not supported in compare-locales", + ) + for path in skips + ) + all_files = [p for p in all_files if p not in skips] + files = ProjectFiles(name, l10nconfigs) + + get_reference_and_tests = l10n_base_reference_and_tests(files) + + linter = MozL10nLinter(lintconfig) + results += linter.lint(all_files, get_reference_and_tests) + return results + + +# Similar to the lint/lint_strings wrapper setup, for comm-central support. +def gecko_strings_setup(**lint_args): + extra_args = lint_args.get("extra_args") or [] + if "--l10n-git" in extra_args: + return source_repo_setup(L10N_SOURCE_REPO, L10N_SOURCE_NAME) + else: + return strings_repo_setup(STRINGS_REPO, STRINGS_NAME) + + +def source_repo_setup(repo: str, name: str): + gs = mozpath.join(mach_util.get_state_dir(), name) + marker = mozpath.join(gs, ".git", "l10n_pull_marker") + try: + last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime) + skip_clone = datetime.now() < last_pull + PULL_AFTER + except OSError: + skip_clone = False + if skip_clone: + return + try: + update_git_repo(repo, gs) + except MissingVCSTool: + if os.environ.get("MOZ_AUTOMATION"): + raise + print("warning: l10n linter requires Git but was unable to find 'git'") + return 1 + with open(marker, "w") as fh: + fh.flush() + + +def strings_repo_setup(repo: str, name: str): + gs = mozpath.join(mach_util.get_state_dir(), name) + marker = mozpath.join(gs, ".hg", "l10n_pull_marker") + try: + last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime) + skip_clone = datetime.now() < last_pull + PULL_AFTER + except OSError: + skip_clone = False + if skip_clone: + return + try: + update_mercurial_repo(repo, gs) + except MissingVCSTool: + if os.environ.get("MOZ_AUTOMATION"): + raise + print("warning: l10n linter requires Mercurial but was unable to find 'hg'") + return 1 + with open(marker, "w") as fh: + fh.flush() + + +def load_configs(lintconfig, root, l10n_base, locale): + """Load l10n configuration files specified in the linter configuration.""" + configs = [] + env = {"l10n_base": l10n_base} + for toml in lintconfig["l10n_configs"]: + cfg = TOMLParser().parse( + mozpath.join(root, toml), env=env, ignore_missing_includes=True + ) + cfg.set_locales([locale], deep=True) + configs.append(cfg) + return configs + + +def validate_linter_includes(lintconfig, l10nconfigs, lintargs): + """Check l10n.yml config against l10n.toml configs.""" + reference_paths = set( + mozpath.relpath(p["reference"].prefix, lintargs["root"]) + for project in l10nconfigs + for config in project.configs + for p in config.paths + ) + # Just check for directories + reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p)) + missing_in_yml = [ + refd for refd in reference_dirs if refd not in lintconfig["include"] + ] + # These might be subdirectories in the config, though + missing_in_yml = [ + d + for d in missing_in_yml + if not any(d.startswith(parent + "/") for parent in lintconfig["include"]) + ] + if missing_in_yml: + dirs = ", ".join(missing_in_yml) + return [ + result.from_config( + lintconfig, + path=lintconfig["path"], + message="l10n.yml out of sync with l10n.toml, add: " + dirs, + ) + ] + return [] + + +class MozL10nLinter(L10nLinter): + """Subclass linter to generate the right result type.""" + + def __init__(self, lintconfig): + super(MozL10nLinter, self).__init__() + self.lintconfig = lintconfig + + def lint(self, files, get_reference_and_tests): + return [ + result.from_config(self.lintconfig, **result_data) + for result_data in super(MozL10nLinter, self).lint( + files, get_reference_and_tests + ) + ] diff --git a/tools/lint/python/ruff.py b/tools/lint/python/ruff.py new file mode 100644 index 0000000000..349320e153 --- /dev/null +++ b/tools/lint/python/ruff.py @@ -0,0 +1,176 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import platform +import re +import signal +import subprocess +import sys +from pathlib import Path + +import mozfile +from mozlint import result + +here = os.path.abspath(os.path.dirname(__file__)) +RUFF_REQUIREMENTS_PATH = os.path.join(here, "ruff_requirements.txt") + +RUFF_NOT_FOUND = """ +Could not find ruff! Install ruff and try again. + + $ pip install -U --require-hashes -r {} +""".strip().format( + RUFF_REQUIREMENTS_PATH +) + + +RUFF_INSTALL_ERROR = """ +Unable to install correct version of ruff! +Try to install it manually with: + $ pip install -U --require-hashes -r {} +""".strip().format( + RUFF_REQUIREMENTS_PATH +) + + +def default_bindir(): + # We use sys.prefix to find executables as that gets modified with + # virtualenv's activate_this.py, whereas sys.executable doesn't. + if platform.system() == "Windows": + return os.path.join(sys.prefix, "Scripts") + else: + return os.path.join(sys.prefix, "bin") + + +def get_ruff_version(binary): + """ + Returns found binary's version + """ + try: + output = subprocess.check_output( + [binary, "--version"], + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError as e: + output = e.output + + matches = re.match(r"ruff ([0-9\.]+)", output) + if matches: + return matches[1] + print("Error: Could not parse the version '{}'".format(output)) + + +def setup(root, log, **lintargs): + virtualenv_bin_path = lintargs.get("virtualenv_bin_path") + binary = mozfile.which("ruff", path=(virtualenv_bin_path, default_bindir())) + + if binary and os.path.isfile(binary): + log.debug(f"Looking for ruff at {binary}") + version = get_ruff_version(binary) + versions = [ + line.split()[0].strip() + for line in open(RUFF_REQUIREMENTS_PATH).readlines() + if line.startswith("ruff==") + ] + if [f"ruff=={version}"] == versions: + log.debug("ruff is present with expected version {}".format(version)) + return 0 + else: + log.debug("ruff is present but unexpected version {}".format(version)) + + virtualenv_manager = lintargs["virtualenv_manager"] + try: + virtualenv_manager.install_pip_requirements(RUFF_REQUIREMENTS_PATH, quiet=True) + except subprocess.CalledProcessError: + print(RUFF_INSTALL_ERROR) + return 1 + + +def run_process(config, cmd, **kwargs): + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True + ) + signal.signal(signal.SIGINT, orig) + try: + output, _ = proc.communicate() + proc.wait() + except KeyboardInterrupt: + proc.kill() + + return output + + +def lint(paths, config, log, **lintargs): + fixed = 0 + results = [] + + if not paths: + return {"results": results, "fixed": fixed} + + # Currently ruff only lints non `.py` files if they are explicitly passed + # in. So we need to find any non-py files manually. This can be removed + # after https://github.com/charliermarsh/ruff/issues/3410 is fixed. + exts = [e for e in config["extensions"] if e != "py"] + non_py_files = [] + for path in paths: + p = Path(path) + if not p.is_dir(): + continue + for ext in exts: + non_py_files.extend([str(f) for f in p.glob(f"**/*.{ext}")]) + + args = ["ruff", "check", "--force-exclude"] + paths + non_py_files + + if config["exclude"]: + args.append(f"--extend-exclude={','.join(config['exclude'])}") + + process_kwargs = {"processStderrLine": lambda line: log.debug(line)} + + warning_rules = set(config.get("warning-rules", [])) + if lintargs.get("fix"): + # Do a first pass with --fix-only as the json format doesn't return the + # number of fixed issues. + fix_args = args + ["--fix-only"] + + # Don't fix warnings to limit unrelated changes sneaking into patches. + fix_args.append(f"--extend-ignore={','.join(warning_rules)}") + output = run_process(config, fix_args, **process_kwargs) + matches = re.match(r"Fixed (\d+) errors?.", output) + if matches: + fixed = int(matches[1]) + + log.debug(f"Running with args: {args}") + + output = run_process(config, args + ["--format=json"], **process_kwargs) + if not output: + return [] + + try: + issues = json.loads(output) + except json.JSONDecodeError: + log.error(f"could not parse output: {output}") + return [] + + for issue in issues: + res = { + "path": issue["filename"], + "lineno": issue["location"]["row"], + "column": issue["location"]["column"], + "lineoffset": issue["end_location"]["row"] - issue["location"]["row"], + "message": issue["message"], + "rule": issue["code"], + "level": "error", + } + if any(issue["code"].startswith(w) for w in warning_rules): + res["level"] = "warning" + + if issue["fix"]: + res["hint"] = issue["fix"]["message"] + + results.append(result.from_config(config, **res)) + + return {"results": results, "fixed": fixed} diff --git a/tools/lint/python/ruff_requirements.in b/tools/lint/python/ruff_requirements.in new file mode 100644 index 0000000000..84b2a3cfd0 --- /dev/null +++ b/tools/lint/python/ruff_requirements.in @@ -0,0 +1,2 @@ +ruff +pkgutil-resolve-name==1.3.10 ; python_version < '3.9' diff --git a/tools/lint/python/ruff_requirements.txt b/tools/lint/python/ruff_requirements.txt new file mode 100644 index 0000000000..a9db7c1097 --- /dev/null +++ b/tools/lint/python/ruff_requirements.txt @@ -0,0 +1,29 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --generate-hashes --output-file=tools/lint/python/ruff_requirements.txt tools/lint/python/ruff_requirements.in +# +pkgutil-resolve-name==1.3.10 ; python_version < "3.9" \ + --hash=sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174 \ + --hash=sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e + # via -r tools/lint/python/ruff_requirements.in +ruff==0.0.254 \ + --hash=sha256:059a380c08e849b6f312479b18cc63bba2808cff749ad71555f61dd930e3c9a2 \ + --hash=sha256:09c764bc2bd80c974f7ce1f73a46092c286085355a5711126af351b9ae4bea0c \ + --hash=sha256:0deb1d7226ea9da9b18881736d2d96accfa7f328c67b7410478cc064ad1fa6aa \ + --hash=sha256:0eb66c9520151d3bd950ea43b3a088618a8e4e10a5014a72687881e6f3606312 \ + --hash=sha256:27d39d697fdd7df1f2a32c1063756ee269ad8d5345c471ee3ca450636d56e8c6 \ + --hash=sha256:2fc21d060a3197ac463596a97d9b5db2d429395938b270ded61dd60f0e57eb21 \ + --hash=sha256:688379050ae05394a6f9f9c8471587fd5dcf22149bd4304a4ede233cc4ef89a1 \ + --hash=sha256:8deba44fd563361c488dedec90dc330763ee0c01ba54e17df54ef5820079e7e0 \ + --hash=sha256:ac1429be6d8bd3db0bf5becac3a38bd56f8421447790c50599cd90fd53417ec4 \ + --hash=sha256:b3f15d5d033fd3dcb85d982d6828ddab94134686fac2c02c13a8822aa03e1321 \ + --hash=sha256:b435afc4d65591399eaf4b2af86e441a71563a2091c386cadf33eaa11064dc09 \ + --hash=sha256:c38291bda4c7b40b659e8952167f386e86ec29053ad2f733968ff1d78b4c7e15 \ + --hash=sha256:d4385cdd30153b7aa1d8f75dfd1ae30d49c918ead7de07e69b7eadf0d5538a1f \ + --hash=sha256:dd58c500d039fb381af8d861ef456c3e94fd6855c3d267d6c6718c9a9fe07be0 \ + --hash=sha256:e15742df0f9a3615fbdc1ee9a243467e97e75bf88f86d363eee1ed42cedab1ec \ + --hash=sha256:ef20bf798ffe634090ad3dc2e8aa6a055f08c448810a2f800ab716cc18b80107 \ + --hash=sha256:f70dc93bc9db15cccf2ed2a831938919e3e630993eeea6aba5c84bc274237885 + # via -r tools/lint/python/ruff_requirements.in diff --git a/tools/lint/rejected-words.yml b/tools/lint/rejected-words.yml new file mode 100644 index 0000000000..5b5d3d696a --- /dev/null +++ b/tools/lint/rejected-words.yml @@ -0,0 +1,324 @@ +--- +avoid-blacklist-and-whitelist: + description: "Use words like 'skip', 'select', 'allow' or 'deny' instead" + level: error + include: ['.'] + type: regex + payload: (black|white)[-_]?list + ignore-case: true + # Based on codespell with idl and webidl added. + extensions: + - js + - jsm + - mjs + - jxs + - idl + - webidl + - xml + - html + - xhtml + - cpp + - c + - h + - configure + - py + - properties + - rst + - md + - ftl + - yml + - java + - kt + exclude: + - '**/.eslintrc.js' + - browser/app/profile/firefox.js + - browser/app/winlauncher/LauncherProcessWin.cpp + - browser/base/content/browser.js + - browser/base/content/contentTheme.js + - browser/base/content/test/general/browser_remoteTroubleshoot.js + - browser/base/content/test/general/browser_tab_drag_drop_perwindow.js + - browser/base/content/test/performance/browser_preferences_usage.js + - browser/base/content/test/protectionsUI/browser_protectionsUI_cryptominers.js + - browser/base/content/test/protectionsUI/browser_protectionsUI_fingerprinters.js + - browser/base/content/test/protectionsUI/browser_protectionsUI_pbmode_exceptions.js + - browser/base/content/test/protectionsUI/browser_protectionsUI_report_breakage.js + - browser/base/content/test/protectionsUI/browser_protectionsUI_socialtracking.js + - browser/base/content/test/protectionsUI/browser_protectionsUI_state.js + - browser/base/content/test/protectionsUI/browser_protectionsUI_subview_shim.js + - browser/base/content/test/siteIdentity/browser_no_mcb_for_loopback.js + - browser/base/content/test/tabMediaIndicator/browser_mute_webAudio.js + - browser/base/content/test/tabs/browser_new_file_whitelisted_http_tab.js + - browser/components/enterprisepolicies/Policies.sys.mjs + - browser/components/migration/ChromeMigrationUtils.sys.mjs + - browser/components/migration/ChromeProfileMigrator.sys.mjs + - browser/components/newtab/data/content/activity-stream.bundle.js + - browser/components/preferences/privacy.inc.xhtml + - browser/components/preferences/privacy.js + - browser/components/resistfingerprinting/test/mochitest/test_bug1354633_media_error.html + - browser/components/safebrowsing/content/test/browser_whitelisted.js + - browser/components/sessionstore/test/browser_crashedTabs.js + - browser/components/uitour/UITourChild.sys.mjs + - browser/components/urlbar/tests/browser/browser_searchSingleWordNotification.js + - browser/components/urlbar/tests/browser/browser_UrlbarInput_trimURLs.js + - browser/components/urlbar/tests/unit/test_providerHeuristicFallback.js + - browser/components/urlbar/tests/unit/test_search_suggestions.js + - browser/components/urlbar/tests/unit/test_tokenizer.js + - browser/extensions/screenshots/background/main.js + - browser/extensions/webcompat/shims/nielsen.js + - browser/modules/SitePermissions.sys.mjs + - browser/tools/mozscreenshots/mozscreenshots/extension/configurations/PermissionPrompts.sys.mjs + - build/clang-plugin/CustomMatchers.h + - build/clang-plugin/FopenUsageChecker.cpp + - build/clang-plugin/NaNExprChecker.cpp + - build/clang-plugin/NoPrincipalGetURI.cpp + - build/clang-plugin/tests/TestNANTestingExpr.cpp + - build/compare-mozconfig/compare-mozconfigs.py + - build/moz.configure/bindgen.configure + - build/moz.configure/toolchain.configure + - config/check_vanilla_allocations.py + - devtools/client/debugger/dist/parser-worker.js + - devtools/client/debugger/test/mochitest/examples/big-sourcemap_files/bundle.js + - devtools/client/debugger/test/mochitest/examples/ember/quickstart/dist/assets/vendor.js + - devtools/client/debugger/test/mochitest/examples/react/build/main.js + - devtools/client/debugger/test/mochitest/examples/react/build/service-worker.js + - devtools/client/inspector/markup/test/lib_babel_6.21.0_min.js + - devtools/client/inspector/markup/test/lib_react_dom_15.4.1.js + - docshell/base/nsDocShell.cpp + - docshell/base/URIFixup.sys.mjs + - docshell/test/unit/test_URIFixup_info.js + - dom/base/Document.cpp + - dom/base/MaybeCrossOriginObject.cpp + - dom/base/nsContentUtils.h + - dom/base/nsDataDocumentContentPolicy.cpp + - dom/base/nsGlobalWindowOuter.cpp + - dom/base/nsTreeSanitizer.cpp + - dom/base/nsTreeSanitizer.h + - dom/base/test/browser_multiple_popups.js + - dom/base/test/browser_timeout_throttling_with_audio_playback.js + - dom/base/test/chrome/test_permission_hasValidTransientUserActivation.xhtml + - dom/bindings/Codegen.py + - dom/bindings/parser/WebIDL.py + - dom/bindings/RemoteObjectProxy.cpp + - dom/events/EventStateManager.cpp + - dom/events/KeyboardEvent.cpp + - dom/html/MediaError.cpp + - dom/indexedDB/ActorsParent.cpp + - dom/ipc/ContentParent.cpp + - dom/ipc/URLClassifierParent.cpp + - dom/media/autoplay/AutoplayPolicy.cpp + - dom/media/gmp/GMPChild.cpp + - dom/media/MediaManager.cpp + - dom/media/mp4/MP4Decoder.cpp + - dom/media/platforms/apple/AppleVTDecoder.cpp + - dom/media/platforms/wmf/DXVA2Manager.cpp + - dom/media/platforms/wmf/WMFVideoMFTManager.cpp + - dom/media/autoplay/test/mochitest/file_autoplay_policy_key_blacklist.html + - dom/media/autoplay/test/mochitest/test_autoplay_policy_key_blacklist.html + - dom/media/autoplay/test/mochitest/test_autoplay_policy_permission.html + - dom/media/webm/WebMDecoder.cpp + - dom/media/webrtc/transport/stun_socket_filter.cpp + - dom/media/webrtc/transport/test/ice_unittest.cpp + - dom/tests/mochitest/dom-level0/idn_child.html + - dom/tests/mochitest/dom-level0/test_setting_document.domain_idn.html + - dom/tests/mochitest/whatwg/test_postMessage_origin.xhtml + - gfx/gl/GLContextProviderWGL.cpp + - gfx/gl/GLUploadHelpers.cpp + - gfx/tests/mochitest/test_font_whitelist.html + - gfx/thebes/gfxFT2FontList.cpp + - gfx/thebes/gfxPlatformFontList.cpp + - gfx/thebes/gfxPlatformFontList.h + - gfx/thebes/gfxUserFontSet.cpp + - gfx/thebes/gfxWindowsPlatform.cpp + - gfx/thebes/SharedFontList.cpp + - intl/strres/nsStringBundle.cpp + - ipc/glue/GeckoChildProcessHost.cpp + - js/src/debugger/DebugAPI.h + - js/src/devtools/rootAnalysis/analyzeHeapWrites.js + - js/src/jit/CodeGenerator.cpp + - js/src/jit-test/tests/auto-regress/bug687399.js + - js/src/jit-test/tests/basic/missingArgTest2.js + - js/src/tests/non262/regress/regress-450369.js + - js/xpconnect/src/Sandbox.cpp + - js/xpconnect/src/XPCJSRuntime.cpp + - js/xpconnect/src/xpcpublic.h + - js/xpconnect/src/XPCWrappedNativeScope.cpp + - js/xpconnect/tests/unit/head_watchdog.js + - js/xpconnect/wrappers/FilteringWrapper.cpp + - js/xpconnect/wrappers/XrayWrapper.cpp + - layout/base/PositionedEventTargeting.cpp + - layout/base/PresShell.cpp + - layout/base/PresShell.h + - layout/reftests/css-placeholder/css-restrictions.html + - layout/style/test/test_computed_style_difference.html + - layout/tools/reftest/mach_commands.py + - layout/tools/reftest/mach_test_package_commands.py + - layout/tools/reftest/reftestcommandline.py + - layout/tools/reftest/runreftest.py + - layout/tools/reftest/selftest/conftest.py + - mobile/android/app/geckoview-prefs.js + - mobile/android/geckoview/src/main/java/org/mozilla/gecko/util/HardwareCodecCapabilityUtils.java + - mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java + - mobile/android/geckoview/src/main/java/org/mozilla/geckoview/CrashReporter.java + - mobile/android/geckoview/src/main/java/org/mozilla/geckoview/WebAuthnTokenManager.java + - modules/libpref/Preferences.cpp + - modules/libpref/init/all.js + - netwerk/base/nsIPermissionManager.idl + - netwerk/base/nsIProtocolHandler.idl + - netwerk/base/nsIOService.cpp + - netwerk/base/nsIURI.idl + - netwerk/base/nsURLHelper.cpp + - netwerk/cookie/CookieCommons.h + - netwerk/dns/nsHostRecord.cpp + - netwerk/dns/nsIDNService.cpp + - netwerk/dns/nsIIDNService.idl + - netwerk/dns/TRR.cpp + - netwerk/ipc/DocumentLoadListener.cpp + - netwerk/protocol/http/HttpBaseChannel.cpp + - netwerk/protocol/http/nsHttpChannel.cpp + - netwerk/protocol/http/nsHttpConnectionMgr.cpp + - netwerk/protocol/viewsource/nsViewSourceChannel.cpp + - netwerk/protocol/websocket/BaseWebSocketChannel.cpp + - netwerk/socket/nsSOCKSSocketProvider.cpp + - netwerk/test/unit/test_bug464591.js + - netwerk/test/unit/test_cookie_blacklist.js + - netwerk/test/unit/test_idn_blacklist.js + - netwerk/url-classifier/UrlClassifierCommon.cpp + - netwerk/url-classifier/UrlClassifierFeatureCryptominingAnnotation.cpp + - netwerk/url-classifier/UrlClassifierFeatureCryptominingProtection.cpp + - netwerk/url-classifier/UrlClassifierFeatureFingerprintingAnnotation.cpp + - netwerk/url-classifier/UrlClassifierFeatureFingerprintingProtection.cpp + - netwerk/url-classifier/UrlClassifierFeatureSocialTrackingAnnotation.cpp + - netwerk/url-classifier/UrlClassifierFeatureSocialTrackingProtection.cpp + - netwerk/url-classifier/UrlClassifierFeatureTrackingAnnotation.cpp + - netwerk/url-classifier/UrlClassifierFeatureTrackingProtection.cpp + - python/mozbuild/mozbuild/backend/recursivemake.py + - python/mozbuild/mozbuild/configure/options.py + - python/mozbuild/mozbuild/vendor/vendor_rust.py + - remote/cdp/Protocol.sys.mjs + - security/manager/ssl/tests/unit/test_intermediate_preloads.js + - security/sandbox/linux/broker/SandboxBroker.cpp + - security/sandbox/linux/broker/SandboxBroker.h + - security/sandbox/linux/broker/SandboxBrokerPolicyFactory.cpp + - security/sandbox/linux/glue/SandboxPrefBridge.cpp + - security/sandbox/linux/gtest/TestBroker.cpp + - security/sandbox/linux/Sandbox.cpp + - security/sandbox/linux/SandboxFilter.cpp + - security/sandbox/linux/SandboxFilterUtil.h + - security/sandbox/linux/Sandbox.h + - taskcluster/ci/docker-image/kind.yml + - taskcluster/gecko_taskgraph/actions/create_interactive.py + - taskcluster/gecko_taskgraph/transforms/test/other.py + - taskcluster/gecko_taskgraph/try_option_syntax.py + - testing/condprofile/condprof/client.py + - testing/condprofile/condprof/tests/profile/prefs.js + - testing/condprofile/condprof/tests/test_client.py + - testing/firefox-ui/tests/functional/safebrowsing/test_initial_download.py + - testing/marionette/client/marionette_driver/wait.py + - testing/mochitest/browser-test.js + - testing/mochitest/mach_test_package_commands.py + - testing/mochitest/mochitest_options.py + - testing/mochitest/runtests.py + - testing/mozbase/mozprofile/mozprofile/profile.py + - testing/mozharness/configs/unittests/linux_unittest.py + - testing/mozharness/configs/unittests/mac_unittest.py + - testing/mozharness/configs/unittests/win_unittest.py + - testing/profiles/unittest-required/user.js + - testing/raptor/browsertime/browsertime_scenario.js + - testing/web-platform/tests/tools/manifest/tests/test_manifest.py + - toolkit/actors/RemotePageChild.sys.mjs + - toolkit/actors/WebChannelChild.sys.mjs + - toolkit/components/antitracking/test/browser/browser_socialtracking_save_image.js + - toolkit/components/reputationservice/ApplicationReputation.cpp + - toolkit/components/reputationservice/chromium/chrome/common/safe_browsing/csd.pb.h + - toolkit/components/reputationservice/test/unit/head_download_manager.js + - toolkit/components/reputationservice/test/unit/test_app_rep.js + - toolkit/components/reputationservice/test/unit/test_app_rep_maclinux.js + - toolkit/components/reputationservice/test/unit/test_app_rep_windows.js + - toolkit/components/satchel/test/test_form_autocomplete.html + - toolkit/components/telemetry/docs/data/environment.rst + - toolkit/components/url-classifier/nsUrlClassifierUtils.cpp + - toolkit/components/url-classifier/SafeBrowsing.sys.mjs + - toolkit/components/url-classifier/tests/mochitest/features.js + - toolkit/components/url-classifier/tests/mochitest/good.js + - toolkit/components/url-classifier/tests/mochitest/test_annotation_vs_TP.html + - toolkit/components/url-classifier/tests/mochitest/test_classified_annotations.html + - toolkit/components/url-classifier/tests/mochitest/test_classify_by_default.html + - toolkit/components/url-classifier/tests/mochitest/test_classify_ping.html + - toolkit/components/url-classifier/tests/mochitest/test_cryptomining_annotate.html + - toolkit/components/url-classifier/tests/mochitest/test_cryptomining.html + - toolkit/components/url-classifier/tests/mochitest/test_fingerprinting_annotate.html + - toolkit/components/url-classifier/tests/mochitest/test_fingerprinting.html + - toolkit/components/url-classifier/tests/mochitest/test_safebrowsing_bug1272239.html + - toolkit/components/url-classifier/tests/mochitest/test_socialtracking_annotate.html + - toolkit/components/url-classifier/tests/mochitest/test_socialtracking.html + - toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_bug1580416.html + - toolkit/components/url-classifier/tests/mochitest/test_trackingprotection_whitelist.html + - toolkit/components/url-classifier/tests/unit/head_urlclassifier.js + - toolkit/components/url-classifier/tests/unit/test_digest256.js + - toolkit/components/url-classifier/tests/unit/test_platform_specific_threats.js + - toolkit/components/url-classifier/tests/UrlClassifierTestUtils.sys.mjs + - toolkit/content/aboutTelemetry.js + - toolkit/content/tests/browser/browser_delay_autoplay_webAudio.js + - toolkit/modules/PermissionsUtils.sys.mjs + - toolkit/modules/tests/browser/browser_AsyncPrefs.js + - toolkit/modules/tests/browser/browser_web_channel.js + - toolkit/modules/tests/xpcshell/test_PermissionsUtils.js + - toolkit/modules/third_party/jsesc/jsesc.mjs + - toolkit/modules/Troubleshoot.sys.mjs + - toolkit/mozapps/extensions/internal/XPIInstall.sys.mjs + - toolkit/mozapps/extensions/test/browser/browser_html_discover_view.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_Device.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_DriverNew.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_Equal_DriverNew.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_Equal_DriverOld.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_Equal_OK.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_GTE_DriverOld.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_GTE_OK.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_No_Comparison.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_OK.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_OS.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_OSVersion_match.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_OSVersion_mismatch_DriverVersion.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_OSVersion_mismatch_OSVersion.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_prefs.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_Vendor.js + - toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_gfxBlacklist_Version.js + - toolkit/mozapps/extensions/test/xpcshell/test_permissions.js + - toolkit/mozapps/extensions/test/xpcshell/test_permissions_prefs.js + - toolkit/mozapps/extensions/test/xpinstall/browser_bug645699.js + - toolkit/mozapps/extensions/test/xpinstall/browser_doorhanger_installs.js + - toolkit/mozapps/extensions/test/xpinstall/browser_localfile3.js + - toolkit/mozapps/extensions/test/xpinstall/browser_localfile4.js + - toolkit/mozapps/extensions/test/xpinstall/browser_localfile4_postDownload.js + - toolkit/mozapps/extensions/test/xpinstall/head.js + - tools/fuzzing/messagemanager/MessageManagerFuzzer.cpp + - tools/fuzzing/messagemanager/MessageManagerFuzzer.h + - tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-define-cc-etc.js + - tools/lint/rejected-words.yml + - widget/android/GfxInfo.cpp + - widget/GfxInfoBase.cpp + - widget/gtk/IMContextWrapper.cpp + - widget/gtk/nsAppShell.cpp + - widget/windows/GfxInfo.cpp + - widget/windows/WinUtils.cpp + - widget/windows/WinUtils.h + - xpcom/io/FilePreferences.cpp + - xpcom/tests/gtest/TestFilePreferencesUnix.cpp + +# --- +# Disable for now. Needs some dev to handle this +# avoid-master-and-slave: +# description: "Use words like 'controller', 'worker' instead" +--- +avoid-gobbledygook: + description: "American English colloquialism. Use 'nonsense' instead." + level: error + include: ['.'] + type: regex + payload: \b(gobbledy)?-?gook + ignore-case: true + exclude: + - extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/orig/en_US-custom.dic + - extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/utf8/en-US-utf8.dic + - extensions/spellcheck/locales/en-US/hunspell/en-US.dic + - tools/lint/rejected-words.yml diff --git a/tools/lint/rst.yml b/tools/lint/rst.yml new file mode 100644 index 0000000000..3f35fe7def --- /dev/null +++ b/tools/lint/rst.yml @@ -0,0 +1,11 @@ +--- +rst: + description: RST linter + include: [.] + extensions: + - rst + support-files: + - 'tools/lint/rst/**' + type: external + payload: rst:lint + setup: rst:setup diff --git a/tools/lint/rst/__init__.py b/tools/lint/rst/__init__.py new file mode 100644 index 0000000000..7151c09a59 --- /dev/null +++ b/tools/lint/rst/__init__.py @@ -0,0 +1,116 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import subprocess + +from mozfile import which +from mozlint import result +from mozlint.pathutils import expand_exclusions + +# Error Levels +# (0, 'debug') +# (1, 'info') +# (2, 'warning') +# (3, 'error') +# (4, 'severe') + +abspath = os.path.abspath(os.path.dirname(__file__)) +rstcheck_requirements_file = os.path.join(abspath, "requirements.txt") + +results = [] + +RSTCHECK_NOT_FOUND = """ +Could not find rstcheck! Install rstcheck and try again. + + $ pip install -U --require-hashes -r {} +""".strip().format( + rstcheck_requirements_file +) + +RSTCHECK_INSTALL_ERROR = """ +Unable to install required version of rstcheck +Try to install it manually with: + $ pip install -U --require-hashes -r {} +""".strip().format( + rstcheck_requirements_file +) + +RSTCHECK_FORMAT_REGEX = re.compile(r"(.*):(.*): \(.*/([0-9]*)\) (.*)$") +IGNORE_NOT_REF_LINK_UPSTREAM_BUG = re.compile( + r"Hyperlink target (.*) is not referenced." +) + + +def setup(root, **lintargs): + virtualenv_manager = lintargs["virtualenv_manager"] + try: + virtualenv_manager.install_pip_requirements( + rstcheck_requirements_file, quiet=True + ) + except subprocess.CalledProcessError: + print(RSTCHECK_INSTALL_ERROR) + return 1 + + +def get_rstcheck_binary(): + """ + Returns the path of the first rstcheck binary available + if not found returns None + """ + binary = os.environ.get("RSTCHECK") + if binary: + return binary + + return which("rstcheck") + + +def parse_with_split(errors): + match = RSTCHECK_FORMAT_REGEX.match(errors) + filename, lineno, level, message = match.groups() + + return filename, lineno, level, message + + +def lint(files, config, **lintargs): + log = lintargs["log"] + config["root"] = lintargs["root"] + paths = expand_exclusions(files, config, config["root"]) + paths = list(paths) + chunk_size = 50 + binary = get_rstcheck_binary() + rstcheck_options = [ + "--ignore-language=cpp,json", + "--ignore-roles=searchfox", + ] + + while paths: + cmdargs = [which("python"), binary] + rstcheck_options + paths[:chunk_size] + log.debug("Command: {}".format(" ".join(cmdargs))) + + proc = subprocess.Popen( + cmdargs, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=os.environ, + universal_newlines=True, + ) + all_errors = proc.communicate()[1] + for errors in all_errors.split("\n"): + if len(errors) > 1: + filename, lineno, level, message = parse_with_split(errors) + if not IGNORE_NOT_REF_LINK_UPSTREAM_BUG.match(message): + # Ignore an upstream bug + # https://github.com/myint/rstcheck/issues/19 + res = { + "path": filename, + "message": message, + "lineno": lineno, + "level": "error" if int(level) >= 2 else "warning", + } + results.append(result.from_config(config, **res)) + paths = paths[chunk_size:] + + return results diff --git a/tools/lint/rst/requirements.in b/tools/lint/rst/requirements.in new file mode 100644 index 0000000000..e6b6022a47 --- /dev/null +++ b/tools/lint/rst/requirements.in @@ -0,0 +1,20 @@ +alabaster==0.7.13 +charset-normalizer==2.0.12 +docutils==0.17.1 +idna==2.10 +imagesize==1.4.1 +importlib-metadata==6.0.0 +markupsafe==2.0.1 +packaging==21.0 +requests==2.27.1 +snowballstemmer==2.2.0 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-mermaid==0.8.1 +rstcheck==3.5.0 +Pygments==2.14.0 +pytz==2022.7.1 +urllib3==1.26.9 +# We need sphinx to avoid some rstcheck errors and warnings +Sphinx==5.3.0 +pkgutil-resolve-name==1.3.10 ; python_version < '3.9' diff --git a/tools/lint/rst/requirements.txt b/tools/lint/rst/requirements.txt new file mode 100644 index 0000000000..ded13595c3 --- /dev/null +++ b/tools/lint/rst/requirements.txt @@ -0,0 +1,220 @@ +# +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: +# +# pip-compile --config=pyproject.toml --generate-hashes --output-file=tools/lint/rst/requirements.txt ./tools/lint/rst/requirements.in +# +alabaster==0.7.13 \ + --hash=sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3 \ + --hash=sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2 + # via + # -r ./tools/lint/rst/requirements.in + # sphinx +babel==2.12.1 \ + --hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \ + --hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455 + # via sphinx +certifi==2022.12.7 \ + --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ + --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 + # via requests +charset-normalizer==2.0.12 \ + --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 \ + --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df + # via + # -r ./tools/lint/rst/requirements.in + # requests +docutils==0.17.1 \ + --hash=sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125 \ + --hash=sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61 + # via + # -r ./tools/lint/rst/requirements.in + # rstcheck + # sphinx +idna==2.10 \ + --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ + --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 + # via + # -r ./tools/lint/rst/requirements.in + # requests +imagesize==1.4.1 \ + --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \ + --hash=sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a + # via + # -r ./tools/lint/rst/requirements.in + # sphinx +importlib-metadata==6.0.0 \ + --hash=sha256:7efb448ec9a5e313a57655d35aa54cd3e01b7e1fbcf72dce1bf06119420f5bad \ + --hash=sha256:e354bedeb60efa6affdcc8ae121b73544a7aa74156d047311948f6d711cd378d + # via + # -r ./tools/lint/rst/requirements.in + # sphinx +jinja2==3.1.2 \ + --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ + --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 + # via sphinx +markupsafe==2.0.1 \ + --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \ + --hash=sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64 \ + --hash=sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b \ + --hash=sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194 \ + --hash=sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567 \ + --hash=sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff \ + --hash=sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724 \ + --hash=sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74 \ + --hash=sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646 \ + --hash=sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35 \ + --hash=sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6 \ + --hash=sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a \ + --hash=sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6 \ + --hash=sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad \ + --hash=sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26 \ + --hash=sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38 \ + --hash=sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac \ + --hash=sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7 \ + --hash=sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6 \ + --hash=sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047 \ + --hash=sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75 \ + --hash=sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f \ + --hash=sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b \ + --hash=sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135 \ + --hash=sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8 \ + --hash=sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a \ + --hash=sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a \ + --hash=sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1 \ + --hash=sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9 \ + --hash=sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864 \ + --hash=sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914 \ + --hash=sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee \ + --hash=sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f \ + --hash=sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18 \ + --hash=sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8 \ + --hash=sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2 \ + --hash=sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d \ + --hash=sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b \ + --hash=sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b \ + --hash=sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86 \ + --hash=sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6 \ + --hash=sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f \ + --hash=sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb \ + --hash=sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833 \ + --hash=sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28 \ + --hash=sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e \ + --hash=sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415 \ + --hash=sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902 \ + --hash=sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f \ + --hash=sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d \ + --hash=sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9 \ + --hash=sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d \ + --hash=sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145 \ + --hash=sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066 \ + --hash=sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c \ + --hash=sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1 \ + --hash=sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a \ + --hash=sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207 \ + --hash=sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f \ + --hash=sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53 \ + --hash=sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd \ + --hash=sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134 \ + --hash=sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85 \ + --hash=sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9 \ + --hash=sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5 \ + --hash=sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94 \ + --hash=sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509 \ + --hash=sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51 \ + --hash=sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872 + # via + # -r ./tools/lint/rst/requirements.in + # jinja2 +packaging==21.0 \ + --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ + --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 + # via + # -r ./tools/lint/rst/requirements.in + # sphinx +pkgutil-resolve-name==1.3.10 ; python_version < "3.9" \ + --hash=sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174 \ + --hash=sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e + # via -r ./tools/lint/rst/requirements.in +pygments==2.14.0 \ + --hash=sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297 \ + --hash=sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717 + # via + # -r ./tools/lint/rst/requirements.in + # sphinx +pyparsing==3.0.9 \ + --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ + --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc + # via packaging +pytz==2022.7.1 \ + --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ + --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a + # via + # -r ./tools/lint/rst/requirements.in + # babel +requests==2.27.1 \ + --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \ + --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d + # via + # -r ./tools/lint/rst/requirements.in + # sphinx +rstcheck==3.5.0 \ + --hash=sha256:30c36768c4bd617a85ab93c31facaf410582e53803fde624845eb00c1430070c \ + --hash=sha256:d4b035300b7d898403544f38c3a4980171ce85f487d25e188347bbafb6ee58c0 + # via -r ./tools/lint/rst/requirements.in +snowballstemmer==2.2.0 \ + --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ + --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a + # via + # -r ./tools/lint/rst/requirements.in + # sphinx +sphinx==5.3.0 \ + --hash=sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d \ + --hash=sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5 + # via -r ./tools/lint/rst/requirements.in +sphinxcontrib-applehelp==1.0.2 \ + --hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \ + --hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58 + # via + # -r ./tools/lint/rst/requirements.in + # sphinx +sphinxcontrib-devhelp==1.0.2 \ + --hash=sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e \ + --hash=sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 \ + --hash=sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07 \ + --hash=sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2 + # via + # -r ./tools/lint/rst/requirements.in + # sphinx +sphinxcontrib-jsmath==1.0.1 \ + --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \ + --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8 + # via sphinx +sphinxcontrib-mermaid==0.8.1 \ + --hash=sha256:15491c24ec78cf1626b1e79e797a9ce87cb7959cf38f955eb72dd5512aeb6ce9 \ + --hash=sha256:fa3e5325d4ba395336e6137d113f55026b1a03ccd115dc54113d1d871a580466 + # via -r ./tools/lint/rst/requirements.in +sphinxcontrib-qthelp==1.0.3 \ + --hash=sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72 \ + --hash=sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 \ + --hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \ + --hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952 + # via sphinx +typing-extensions==4.7.1 \ + --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \ + --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2 + # via importlib-metadata +urllib3==1.26.9 \ + --hash=sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14 \ + --hash=sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e + # via + # -r ./tools/lint/rst/requirements.in + # requests +zipp==3.15.0 \ + --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \ + --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556 + # via importlib-metadata diff --git a/tools/lint/ruff.yml b/tools/lint/ruff.yml new file mode 100644 index 0000000000..59ae2a1350 --- /dev/null +++ b/tools/lint/ruff.yml @@ -0,0 +1,17 @@ +--- +ruff: + description: An extremely fast Python linter, written in Rust + # Excludes should be added to topsrcdir/pyproject.toml + exclude: [] + # The configure option is used by the build system + extensions: ["configure", "py"] + support-files: + - "**/.ruff.toml" + - "**/ruff.toml" + - "**/pyproject.toml" + - "tools/lint/python/ruff.py" + # Rules that should result in warnings rather than errors. + warning-rules: [PLR, PLW] + type: external + payload: python.ruff:lint + setup: python.ruff:setup diff --git a/tools/lint/rust/__init__.py b/tools/lint/rust/__init__.py new file mode 100644 index 0000000000..148aeeb648 --- /dev/null +++ b/tools/lint/rust/__init__.py @@ -0,0 +1,171 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import signal +import subprocess +from collections import namedtuple + +from mozboot.util import get_tools_dir +from mozfile import which +from mozlint import result +from mozlint.pathutils import expand_exclusions +from packaging.version import Version + +RUSTFMT_NOT_FOUND = """ +Could not find rustfmt! Install rustfmt and try again. + + $ rustup component add rustfmt + +And make sure that it is in the PATH +""".strip() + + +RUSTFMT_INSTALL_ERROR = """ +Unable to install correct version of rustfmt +Try to install it manually with: + $ rustup component add rustfmt +""".strip() + + +RUSTFMT_WRONG_VERSION = """ +You are probably using an old version of rustfmt. +Expected version is {version}. +Try to update it: + $ rustup update stable +""".strip() + + +def parse_issues(config, output, paths): + RustfmtDiff = namedtuple("RustfmtDiff", ["file", "line", "diff"]) + issues = [] + diff_line = re.compile("^Diff in (.*) at line ([0-9]*):") + file = "" + line_no = 0 + diff = "" + for line in output.split(b"\n"): + processed_line = ( + line.decode("utf-8", "replace") if isinstance(line, bytes) else line + ).rstrip("\r\n") + match = diff_line.match(processed_line) + if match: + if diff: + issues.append(RustfmtDiff(file, line_no, diff.rstrip("\n"))) + diff = "" + file, line_no = match.groups() + else: + diff += processed_line + "\n" + # the algorithm above will always skip adding the last issue + issues.append(RustfmtDiff(file, line_no, diff)) + file = os.path.normcase(os.path.normpath(file)) + results = [] + for issue in issues: + # rustfmt can not be supplied the paths to the files we want to analyze + # therefore, for each issue detected, we check if any of the the paths + # supplied are part of the file name. + # This just filters out the issues that are not part of paths. + if any([os.path.normcase(os.path.normpath(path)) in file for path in paths]): + res = { + "path": issue.file, + "diff": issue.diff, + "level": "warning", + "lineno": issue.line, + } + results.append(result.from_config(config, **res)) + return {"results": results, "fixed": 0} + + +def run_process(config, cmd): + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + signal.signal(signal.SIGINT, orig) + + try: + output, _ = proc.communicate() + proc.wait() + except KeyboardInterrupt: + proc.kill() + + return output + + +def get_rustfmt_binary(): + """ + Returns the path of the first rustfmt binary available + if not found returns None + """ + binary = os.environ.get("RUSTFMT") + if binary: + return binary + + rust_path = os.path.join(get_tools_dir(), "rustc", "bin") + return which("rustfmt", path=os.pathsep.join([rust_path, os.environ["PATH"]])) + + +def get_rustfmt_version(binary): + """ + Returns found binary's version + """ + try: + output = subprocess.check_output( + [binary, "--version"], + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + except subprocess.CalledProcessError as e: + output = e.output + + version = re.findall(r"\d.\d+.\d+", output)[0] + return Version(version) + + +def lint(paths, config, fix=None, **lintargs): + log = lintargs["log"] + paths = list(expand_exclusions(paths, config, lintargs["root"])) + + # An empty path array can occur when the user passes in `-n`. If we don't + # return early in this case, rustfmt will attempt to read stdin and hang. + if not paths: + return [] + + binary = get_rustfmt_binary() + + if not binary: + print(RUSTFMT_NOT_FOUND) + if "MOZ_AUTOMATION" in os.environ: + return 1 + return [] + + min_version_str = config.get("min_rustfmt_version") + min_version = Version(min_version_str) + actual_version = get_rustfmt_version(binary) + log.debug( + "Found version: {}. Minimal expected version: {}".format( + actual_version, min_version + ) + ) + + if actual_version < min_version: + print(RUSTFMT_WRONG_VERSION.format(version=min_version_str)) + return 1 + + cmd_args = [binary] + cmd_args.append("--check") + base_command = cmd_args + paths + log.debug("Command: {}".format(" ".join(cmd_args))) + output = run_process(config, base_command) + + issues = parse_issues(config, output, paths) + + if fix: + issues["fixed"] = len(issues["results"]) + issues["results"] = [] + cmd_args.remove("--check") + + base_command = cmd_args + paths + log.debug("Command: {}".format(" ".join(cmd_args))) + output = run_process(config, base_command) + + return issues diff --git a/tools/lint/rustfmt.yml b/tools/lint/rustfmt.yml new file mode 100644 index 0000000000..e317088812 --- /dev/null +++ b/tools/lint/rustfmt.yml @@ -0,0 +1,22 @@ +--- +rust: + description: Reformat rust + min_rustfmt_version: 1.4.12 + include: + - '.' + exclude: + - build/rust/windows/src/lib.rs + - dom/webauthn/libudev-sys/ + - gfx/wr/peek-poke/ + - gfx/wr/webrender_build/ + - gfx/wr/wr_malloc_size_of/ + - intl/icu_segmenter_data/ + - media/mp4parse-rust/ + - servo/ + - '**/*.mako.rs' + extensions: + - rs + support-files: + - 'tools/lint/rust/**' + type: external + payload: rust:lint diff --git a/tools/lint/shell/__init__.py b/tools/lint/shell/__init__.py new file mode 100644 index 0000000000..b75dc2d159 --- /dev/null +++ b/tools/lint/shell/__init__.py @@ -0,0 +1,148 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +from json.decoder import JSONDecodeError + +import mozpack.path as mozpath +from mozfile import which +from mozlint import result +from mozlint.util.implementation import LintProcess +from mozpack.files import FileFinder + +SHELLCHECK_NOT_FOUND = """ +Unable to locate shellcheck, please ensure it is installed and in +your PATH or set the SHELLCHECK environment variable. + +https://shellcheck.net or your system's package manager. +""".strip() + +results = [] + + +class ShellcheckProcess(LintProcess): + def process_line(self, line): + try: + data = json.loads(line) + except JSONDecodeError as e: + print("Unable to load shellcheck output ({}): {}".format(e, line)) + return + + for entry in data: + res = { + "path": entry["file"], + "message": entry["message"], + "level": "error", + "lineno": entry["line"], + "column": entry["column"], + "rule": entry["code"], + } + results.append(result.from_config(self.config, **res)) + + +def determine_shell_from_script(path): + """Returns a string identifying the shell used. + + Returns None if not identifiable. + + Copes with the following styles: + #!bash + #!/bin/bash + #!/usr/bin/env bash + """ + with open(path, "r") as f: + head = f.readline() + + if not head.startswith("#!"): + return + + # allow for parameters to the shell + shebang = head.split()[0] + + # if the first entry is a variant of /usr/bin/env + if "env" in shebang: + shebang = head.split()[1] + + if shebang.endswith("sh"): + # Strip first to avoid issues with #!bash + return shebang.strip("#!").split("/")[-1] + # make it clear we return None, rather than fall through. + return + + +def find_shell_scripts(config, paths): + found = dict() + + root = config["root"] + exclude = [mozpath.join(root, e) for e in config.get("exclude", [])] + + if config.get("extensions"): + pattern = "**/*.{}".format(config.get("extensions")[0]) + else: + pattern = "**/*.sh" + + files = [] + for path in paths: + path = mozpath.normsep(path) + ignore = [ + e[len(path) :].lstrip("/") + for e in exclude + if mozpath.commonprefix((path, e)) == path + ] + finder = FileFinder(path, ignore=ignore) + files.extend([os.path.join(path, p) for p, f in finder.find(pattern)]) + + for filename in files: + shell = determine_shell_from_script(filename) + if shell: + found[filename] = shell + return found + + +def run_process(config, cmd): + proc = ShellcheckProcess(config, cmd) + proc.run() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + + +def get_shellcheck_binary(): + """ + Returns the path of the first shellcheck binary available + if not found returns None + """ + binary = os.environ.get("SHELLCHECK") + if binary: + return binary + + return which("shellcheck") + + +def lint(paths, config, **lintargs): + log = lintargs["log"] + binary = get_shellcheck_binary() + + if not binary: + print(SHELLCHECK_NOT_FOUND) + if "MOZ_AUTOMATION" in os.environ: + return 1 + return [] + + config["root"] = lintargs["root"] + + files = find_shell_scripts(config, paths) + + base_command = [binary, "-f", "json"] + if config.get("excludecodes"): + base_command.extend(["-e", ",".join(config.get("excludecodes"))]) + + for f in files: + cmd = list(base_command) + cmd.extend(["-s", files[f], f]) + log.debug("Command: {}".format(cmd)) + run_process(config, cmd) + return results diff --git a/tools/lint/shellcheck.yml b/tools/lint/shellcheck.yml new file mode 100644 index 0000000000..0100e3d5cc --- /dev/null +++ b/tools/lint/shellcheck.yml @@ -0,0 +1,15 @@ +--- +shellcheck: + description: Shell script linter + include: + - extensions/spellcheck/locales/en-US/hunspell/dictionary-sources/ + - taskcluster/docker/ + exclude: [] + # 1090: https://github.com/koalaman/shellcheck/wiki/SC1090 + # 'Can't follow a non-constant source' + extensions: ['sh'] + support-files: + - 'tools/lint/shell/**' + excludecodes: ['1090', '1091'] + type: external + payload: shell:lint diff --git a/tools/lint/spell/__init__.py b/tools/lint/spell/__init__.py new file mode 100644 index 0000000000..65712acdd7 --- /dev/null +++ b/tools/lint/spell/__init__.py @@ -0,0 +1,168 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import subprocess + +# py2-compat +try: + from json.decoder import JSONDecodeError +except ImportError: + JSONDecodeError = ValueError + +from mozfile import which +from mozlint import result +from mozlint.util.implementation import LintProcess + +here = os.path.abspath(os.path.dirname(__file__)) +CODESPELL_REQUIREMENTS_PATH = os.path.join(here, "codespell_requirements.txt") + +CODESPELL_NOT_FOUND = """ +Could not find codespell! Install codespell and try again. + + $ pip install -U --require-hashes -r {} +""".strip().format( + CODESPELL_REQUIREMENTS_PATH +) + + +CODESPELL_INSTALL_ERROR = """ +Unable to install correct version of codespell +Try to install it manually with: + $ pip install -U --require-hashes -r {} +""".strip().format( + CODESPELL_REQUIREMENTS_PATH +) + +results = [] + +CODESPELL_FORMAT_REGEX = re.compile(r"(.*):(.*): (.*) ==> (.*)$") + + +class CodespellProcess(LintProcess): + fixed = 0 + _fix = None + + def process_line(self, line): + try: + match = CODESPELL_FORMAT_REGEX.match(line) + abspath, line, typo, correct = match.groups() + except AttributeError: + if "FIXED: " not in line: + print("Unable to match regex against output: {}".format(line)) + return + + if CodespellProcess._fix: + CodespellProcess.fixed += 1 + + # Ignore false positive like aParent (which would be fixed to apparent) + # See https://github.com/lucasdemarchi/codespell/issues/314 + m = re.match(r"^[a-z][A-Z][a-z]*", typo) + if m: + return + res = { + "path": abspath, + "message": typo.strip() + " ==> " + correct, + "level": "error", + "lineno": line, + } + results.append(result.from_config(self.config, **res)) + + +def run_process(config, cmd): + proc = CodespellProcess(config, cmd) + proc.run() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + + +def get_codespell_binary(): + """ + Returns the path of the first codespell binary available + if not found returns None + """ + binary = os.environ.get("CODESPELL") + if binary: + return binary + + return which("codespell") + + +def setup(root, **lintargs): + virtualenv_manager = lintargs["virtualenv_manager"] + try: + virtualenv_manager.install_pip_requirements( + CODESPELL_REQUIREMENTS_PATH, quiet=True + ) + except subprocess.CalledProcessError: + print(CODESPELL_INSTALL_ERROR) + return 1 + + +def get_codespell_version(binary): + return subprocess.check_output( + [which("python"), binary, "--version"], + universal_newlines=True, + stderr=subprocess.STDOUT, + ) + + +def get_ignored_words_file(config): + config_root = os.path.dirname(config["path"]) + return os.path.join(config_root, "spell", "exclude-list.txt") + + +def lint(paths, config, fix=None, **lintargs): + log = lintargs["log"] + binary = get_codespell_binary() + if not binary: + print(CODESPELL_NOT_FOUND) + if "MOZ_AUTOMATION" in os.environ: + return 1 + return [] + + config["root"] = lintargs["root"] + + exclude_list = get_ignored_words_file(config) + cmd_args = [ + which("python"), + binary, + "--disable-colors", + # Silence some warnings: + # 1: disable warnings about wrong encoding + # 2: disable warnings about binary file + # 4: shut down warnings about automatic fixes + # that were disabled in dictionary. + "--quiet-level=7", + "--ignore-words=" + exclude_list, + ] + + if "exclude" in config: + cmd_args.append("--skip=*.dic,{}".format(",".join(config["exclude"]))) + + log.debug("Command: {}".format(" ".join(cmd_args))) + log.debug("Version: {}".format(get_codespell_version(binary))) + + if fix: + CodespellProcess._fix = True + + base_command = cmd_args + paths + run_process(config, base_command) + + if fix: + global results + results = [] + cmd_args.append("--write-changes") + log.debug("Command: {}".format(" ".join(cmd_args))) + log.debug("Version: {}".format(get_codespell_version(binary))) + base_command = cmd_args + paths + run_process(config, base_command) + CodespellProcess.fixed = CodespellProcess.fixed - len(results) + else: + CodespellProcess.fixed = 0 + + return {"results": results, "fixed": CodespellProcess.fixed} diff --git a/tools/lint/spell/codespell_requirements.in b/tools/lint/spell/codespell_requirements.in new file mode 100644 index 0000000000..4b1d259fbe --- /dev/null +++ b/tools/lint/spell/codespell_requirements.in @@ -0,0 +1,2 @@ +codespell==2.2.6 +pkgutil-resolve-name==1.3.10 ; python_version < '3.9' diff --git a/tools/lint/spell/codespell_requirements.txt b/tools/lint/spell/codespell_requirements.txt new file mode 100644 index 0000000000..9b09c43897 --- /dev/null +++ b/tools/lint/spell/codespell_requirements.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes --output-file=tools/lint/spell/codespell_requirements.txt tools/lint/spell/codespell_requirements.in +# +codespell==2.2.6 \ + --hash=sha256:9ee9a3e5df0990604013ac2a9f22fa8e57669c827124a2e961fe8a1da4cacc07 \ + --hash=sha256:a8c65d8eb3faa03deabab6b3bbe798bea72e1799c7e9e955d57eca4096abcff9 + # via -r tools/lint/spell/codespell_requirements.in diff --git a/tools/lint/spell/exclude-list.txt b/tools/lint/spell/exclude-list.txt new file mode 100644 index 0000000000..7682d1a177 --- /dev/null +++ b/tools/lint/spell/exclude-list.txt @@ -0,0 +1,25 @@ +cas +optin +aparent +acount +te +wasn +incrementall +aare +whats +crate +files' +thru +referer +dur +ue +tring +delink +warmup +aNumber +falsy +rduce +complies +ehr +inout +manuel diff --git a/tools/lint/stylelint.yml b/tools/lint/stylelint.yml new file mode 100644 index 0000000000..b1b9c94dec --- /dev/null +++ b/tools/lint/stylelint.yml @@ -0,0 +1,15 @@ +--- +stylelint: + description: CSS linter + # Stylelint infra handles its own path filtering, so just include cwd + include: ['.'] + exclude: [] + extensions: ['css', 'scss'] + support-files: + - 'package.json' + - '**/.stylelintrc.js' + - '.stylelintignore' + - 'tools/lint/stylelint/**' + type: external + payload: stylelint:lint + setup: stylelint:setup diff --git a/tools/lint/stylelint/__init__.py b/tools/lint/stylelint/__init__.py new file mode 100644 index 0000000000..417ee836a2 --- /dev/null +++ b/tools/lint/stylelint/__init__.py @@ -0,0 +1,195 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import re +import signal +import subprocess +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "eslint")) +from eslint import setup_helper +from mozbuild.nodeutil import find_node_executable +from mozlint import result + +STYLELINT_ERROR_MESSAGE = """ +An error occurred running stylelint. Please check the following error messages: + +{} +""".strip() + +STYLELINT_NOT_FOUND_MESSAGE = """ +Could not find stylelint! We looked at the --binary option, at the STYLELINT +environment variable, and then at your local node_modules path. Please install +eslint, stylelint and needed plugins with: + +mach eslint --setup + +and try again. +""".strip() + +FILE_EXT_REGEX = re.compile(r"\.[a-z0-9_]{2,10}$", re.IGNORECASE) + + +def setup(root, **lintargs): + setup_helper.set_project_root(root) + + if not setup_helper.check_node_executables_valid(): + return 1 + + return setup_helper.eslint_maybe_setup() + + +def lint(paths, config, binary=None, fix=None, rules=[], setup=None, **lintargs): + """Run stylelint.""" + log = lintargs["log"] + setup_helper.set_project_root(lintargs["root"]) + module_path = setup_helper.get_project_root() + + modified_paths = [] + exts = "*.{" + ",".join(config["extensions"]) + "}" + + for path in paths: + filepath, fileext = os.path.splitext(path) + if fileext: + modified_paths += [path] + else: + joined_path = os.path.join(path, "**", exts) + if is_windows(): + joined_path = joined_path.replace("\\", "/") + modified_paths.append(joined_path) + + # Valid binaries are: + # - Any provided by the binary argument. + # - Any pointed at by the STYLELINT environmental variable. + # - Those provided by |mach lint --setup|. + + if not binary: + binary, _ = find_node_executable() + + if not binary: + print(STYLELINT_NOT_FOUND_MESSAGE) + return 1 + + extra_args = lintargs.get("extra_args") or [] + exclude_args = [] + for path in config.get("exclude", []): + exclude_args.extend( + ["--ignore-pattern", os.path.relpath(path, lintargs["root"])] + ) + + # Default to $topsrcdir/.stylelintrc.js, but allow override in stylelint.yml + stylelint_rc = config.get("stylelint-rc", ".stylelintrc.js") + + # First run Stylelint + cmd_args = ( + [ + binary, + os.path.join( + module_path, "node_modules", "stylelint", "bin", "stylelint.mjs" + ), + "--formatter", + "json", + "--allow-empty-input", + "--config", + os.path.join(lintargs["root"], stylelint_rc), + ] + + extra_args + + exclude_args + + modified_paths + ) + + if fix: + cmd_args.append("--fix") + + log.debug("Stylelint command: {}".format(" ".join(cmd_args))) + + result = run(cmd_args, config, fix) + if result == 1: + return result + + return result + + +def run(cmd_args, config, fix): + shell = False + if is_windows(): + # The stylelint binary needs to be run from a shell with msys + shell = True + encoding = "utf-8" + + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = subprocess.Popen( + cmd_args, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + signal.signal(signal.SIGINT, orig) + + try: + output, errors = proc.communicate() + except KeyboardInterrupt: + proc.kill() + return {"results": [], "fixed": 0} + + if errors: + errors = errors.decode(encoding, "replace") + print(STYLELINT_ERROR_MESSAGE.format(errors)) + + # 0 is success, 2 is there was at least 1 rule violation. Anything else + # is more serious. + if proc.returncode != 0 and proc.returncode != 2: + if proc.returncode == 78: + print("Stylelint reported an issue with its configuration file.") + print(output) + return 1 + + if not output: + return {"results": [], "fixed": 0} # no output means success + output = output.decode(encoding, "replace") + try: + jsonresult = json.loads(output) + except ValueError: + print(STYLELINT_ERROR_MESSAGE.format(output)) + return 1 + + results = [] + fixed = 0 + for obj in jsonresult: + errors = obj["warnings"] + obj["parseErrors"] + # This will return a number of fixed files, as that's the only thing + # stylelint gives us. Note that it also seems to sometimes list files + # like this where it finds nothing and fixes nothing. It's not clear + # why... but this is why we also check if we were even trying to fix + # anything. + if fix and not errors and not obj.get("ignored"): + fixed += 1 + + for err in errors: + msg = err.get("text") + if err.get("rule"): + # stylelint includes the rule id in the error message. + # All mozlint formatters that include the error message also already + # separately include the rule id, so that leads to duplication. Fix: + msg = msg.replace("(" + err.get("rule") + ")", "").strip() + err.update( + { + "message": msg, + "level": err.get("severity") or "error", + "lineno": err.get("line") or 0, + "path": obj["source"], + "rule": err.get("rule") or "parseError", + } + ) + results.append(result.from_config(config, **err)) + + return {"results": results, "fixed": fixed} + + +def is_windows(): + return ( + os.environ.get("MSYSTEM") in ("MINGW32", "MINGW64") + or "MOZILLABUILD" in os.environ + ) diff --git a/tools/lint/test-manifest-alpha.yml b/tools/lint/test-manifest-alpha.yml new file mode 100644 index 0000000000..328da4cb1b --- /dev/null +++ b/tools/lint/test-manifest-alpha.yml @@ -0,0 +1,19 @@ +--- +test-manifest-alpha: + description: Mochitest manifest tests should be in alphabetical order. + exclude: + - "**/application.ini" + - "**/l10n.ini" + - "**/xpcshell.ini" + - "**/python.ini" + - "**/manifest.ini" + - dom/canvas/test/webgl-conf/mochitest-errata.toml + - python/mozbuild/mozbuild/test/backend/data + - testing/mozbase/manifestparser/tests + - testing/web-platform + - xpcom/tests/unit/data + extensions: ['ini'] + type: external + payload: test-manifest-alpha:lint + support-files: + - 'tools/lint/test-manifest-alpha/error-level-manifests.yml' diff --git a/tools/lint/test-manifest-alpha/__init__.py b/tools/lint/test-manifest-alpha/__init__.py new file mode 100644 index 0000000000..87d6ce5c5d --- /dev/null +++ b/tools/lint/test-manifest-alpha/__init__.py @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import difflib +import os +import sys + +import yaml +from manifestparser import TestManifest +from mozlint import result +from mozlint.pathutils import expand_exclusions +from mozpack import path as mozpath + +# Since this linter matches on all files with .ini extensions, we can omit +# those extensions from this allowlist, which makes it easier to match +# against variants like "mochitest-serviceworker.ini". +FILENAME_ALLOWLIST = ["mochitest", "browser", "chrome", "a11y"] + +here = os.path.abspath(os.path.dirname(__file__)) +ERROR_LEVEL_MANIFESTS_PATH = os.path.join(here, "error-level-manifests.yml") + + +def lint(paths, config, fix=None, **lintargs): + try: + with open(ERROR_LEVEL_MANIFESTS_PATH) as f: + error_level_manifests = yaml.safe_load(f) + except (IOError, ValueError) as e: + print("{}: error:\n {}".format(ERROR_LEVEL_MANIFESTS_PATH, e), file=sys.stderr) + sys.exit(1) + + topsrcdir = lintargs["root"] + + results = [] + file_names = list(expand_exclusions(paths, config, lintargs["root"])) + + for file_name in file_names: + name = os.path.basename(file_name) + if not any(name.startswith(allowed) for allowed in FILENAME_ALLOWLIST): + continue + + manifest = TestManifest(manifests=(file_name,), strict=False) + + test_names = [test["name"] for test in manifest.tests] + sorted_test_names = sorted(test_names) + + if test_names != sorted_test_names: + rel_file_path = mozpath.relpath(file_name, topsrcdir) + level = "warning" + + if (mozpath.normsep(rel_file_path) in error_level_manifests) or ( + any( + mozpath.match(rel_file_path, e) + for e in error_level_manifests + if "*" in e + ) + ): + level = "error" + + diff_instance = difflib.Differ() + diff_result = diff_instance.compare(test_names, sorted_test_names) + diff_list = list(diff_result) + + res = { + "path": rel_file_path, + "lineno": 0, + "column": 0, + "message": ( + "The mochitest test manifest is not in alphabetical order. " + "Expected ordering: \n\n%s\n\n" % "\n".join(sorted_test_names) + ), + "level": level, + "diff": "\n".join(diff_list), + } + results.append(result.from_config(config, **res)) + + return {"results": results, "fixed": 0} diff --git a/tools/lint/test-manifest-alpha/error-level-manifests.yml b/tools/lint/test-manifest-alpha/error-level-manifests.yml new file mode 100644 index 0000000000..f2491ab97d --- /dev/null +++ b/tools/lint/test-manifest-alpha/error-level-manifests.yml @@ -0,0 +1,8 @@ +--- +# This file contains a list of manifest files or directory patterns that have +# have been put into alphabetical order already. Items in this list will +# cause the test-manifest-alpha linter to use the Error level rather than +# the Warning level. + +- browser/** +- mobile/** diff --git a/tools/lint/test-manifest-disable.yml b/tools/lint/test-manifest-disable.yml new file mode 100644 index 0000000000..404ffe6c09 --- /dev/null +++ b/tools/lint/test-manifest-disable.yml @@ -0,0 +1,16 @@ +--- +no-comment-disable: + description: > + "Use 'disabled=<reason>' to disable a test instead of a + comment" + include: ['.'] + exclude: + - "**/application.ini" + - "**/l10n.ini" + - dom/canvas/test/webgl-conf/mochitest-errata.toml + - testing/mozbase/manifestparser/tests + - testing/web-platform + - xpcom/tests/unit/data + extensions: ['ini'] + type: regex + payload: ^[ \t]*(#|;)[ \t]*\[ diff --git a/tools/lint/test-manifest-skip-if.yml b/tools/lint/test-manifest-skip-if.yml new file mode 100644 index 0000000000..393beb42e2 --- /dev/null +++ b/tools/lint/test-manifest-skip-if.yml @@ -0,0 +1,18 @@ +--- +multiline-skip-if: + description: Conditions joined by || should be on separate lines + hint: | + skip-if = + <condition one> # reason + <condition two> # reason + exclude: + - "**/application.ini" + - "**/l10n.ini" + - dom/canvas/test/webgl-conf/mochitest-errata.toml + - testing/mozbase/manifestparser/tests + - testing/web-platform + - xpcom/tests/unit/data + extensions: ['ini'] + level: warning + type: regex + payload: '^\s*(skip|fail)-if\s*=[^(]*\|\|' diff --git a/tools/lint/test-manifest-toml.yml b/tools/lint/test-manifest-toml.yml new file mode 100644 index 0000000000..d1e4b7df94 --- /dev/null +++ b/tools/lint/test-manifest-toml.yml @@ -0,0 +1,32 @@ +--- +test-manifest-toml: + description: ManifestParser TOML linter. + exclude: + - 'intl/icu/source/test/testdata/codepointtrie/' + - 'python/mozbuild/mozbuild/test/' + - 'testing/marionette/harness/marionette_harness/tests/unit-tests.toml' + - 'testing/mozbase/manifestparser/tests/' + - 'third_party/rust/' + - 'toolkit/components/featuregates/test/python/data/' + - '**/.*ruff.toml' + - '**/Cargo.toml' + - '**/Cross.toml' + - '**/Features.toml' + - '**/ServoBindings.toml' + - '**/askama.toml' + - '**/audits.toml' + - '**/cbindgen.toml' + - '**/clippy.toml' + - '**/config-lock.toml' + - '**/config.toml' + - '**/cram.toml' + - '**/empty.toml' + - '**/generated-mochitest.toml' + - '**/l10n.toml' + - '**/labels.toml' + - '**/pyproject.toml' + - '**/rustfmt.toml' + - '**/uniffi.toml' + extensions: ['toml'] + type: external + payload: test-manifest-toml:lint diff --git a/tools/lint/test-manifest-toml/__init__.py b/tools/lint/test-manifest-toml/__init__.py new file mode 100644 index 0000000000..08f0e4ed93 --- /dev/null +++ b/tools/lint/test-manifest-toml/__init__.py @@ -0,0 +1,135 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import io +import os +import re + +from manifestparser import TestManifest +from manifestparser.toml import DEFAULT_SECTION, alphabetize_toml_str, sort_paths +from mozlint import result +from mozlint.pathutils import expand_exclusions +from mozpack import path as mozpath +from tomlkit.items import Array, Table + +SECTION_REGEX = r"^\[.*\]$" +DISABLE_REGEX = r"^[ \t]*#[ \t]*\[.*\]" + + +def make_result(path, message, is_error=False): + if is_error: + level = "error" + else: + level = "warning" + result = { + "path": path, + "lineno": 0, # tomlkit does not report lineno/column + "column": 0, + "message": message, + "level": level, + } + return result + + +def lint(paths, config, fix=None, **lintargs): + results = [] + fixed = 0 + topsrcdir = lintargs["root"] + file_names = list(expand_exclusions(paths, config, topsrcdir)) + file_names = [os.path.normpath(f) for f in file_names] + section_rx = re.compile(SECTION_REGEX, flags=re.M) + disable_rx = re.compile(DISABLE_REGEX, flags=re.M) + + for file_name in file_names: + path = mozpath.relpath(file_name, topsrcdir) + os.path.basename(file_name) + parser = TestManifest(use_toml=True, document=True) + + try: + parser.read(file_name) + except Exception: + r = make_result(path, "The manifest is not valid TOML.", True) + results.append(result.from_config(config, **r)) + continue + + manifest = parser.source_documents[file_name] + manifest_str = io.open(file_name, "r", encoding="utf-8").read() + + if not DEFAULT_SECTION in manifest: + r = make_result( + path, f"The manifest does not start with a [{DEFAULT_SECTION}] section." + ) + if fix: + fixed += 1 + results.append(result.from_config(config, **r)) + + sections = [k for k in manifest.keys() if k != DEFAULT_SECTION] + sorted_sections = sort_paths(sections) + if sections != sorted_sections: + r = make_result( + path, "The manifest sections are not in alphabetical order." + ) + if fix: + fixed += 1 + results.append(result.from_config(config, **r)) + + m = section_rx.findall(manifest_str) + if len(m) > 0: + for section_match in m: + section = section_match[1:-1] + if section == DEFAULT_SECTION: + continue + if not section.startswith('"'): + r = make_result( + path, f"The section name must be double quoted: [{section}]" + ) + if fix: + fixed += 1 + results.append(result.from_config(config, **r)) + + m = disable_rx.findall(manifest_str) + if len(m) > 0: + for disabled_section in m: + r = make_result( + path, + f"Use 'disabled = \"<reason>\"' to disable a test instead of a comment: {disabled_section}", + True, + ) + results.append(result.from_config(config, **r)) + + for section, keyvals in manifest.body: + if section is None: + continue + if not isinstance(keyvals, Table): + r = make_result( + path, f"Bad assignment in preamble: {section} = {keyvals}", True + ) + results.append(result.from_config(config, **r)) + else: + for k, v in keyvals.items(): + if k.endswith("-if"): + if not isinstance(v, Array): + r = make_result( + path, + f'Value for conditional must be an array: {k} = "{v}"', + True, + ) + results.append(result.from_config(config, **r)) + else: + for e in v: + if e.find("||") > 0 and e.find("&&") < 0: + r = make_result( + path, + f'Value for conditional must not include explicit ||, instead put on multiple lines: {k} = [ ... "{e}" ... ]', + True, + ) + results.append(result.from_config(config, **r)) + + if fix: + manifest_str = alphabetize_toml_str(manifest) + fp = io.open(file_name, "w", encoding="utf-8", newline="\n") + fp.write(manifest_str) + fp.close() + + return {"results": results, "fixed": fixed} diff --git a/tools/lint/test/conftest.py b/tools/lint/test/conftest.py new file mode 100644 index 0000000000..ad88f8aa97 --- /dev/null +++ b/tools/lint/test/conftest.py @@ -0,0 +1,305 @@ +import logging +import os +import pathlib +import sys +from collections import defaultdict + +import pytest +from mozbuild.base import MozbuildObject +from mozlint.parser import Parser +from mozlint.pathutils import findobject +from mozlint.result import ResultSummary +from mozlog.structuredlog import StructuredLogger +from mozpack import path + +here = path.abspath(path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here, virtualenv_name="python-test") + +lintdir = path.dirname(here) +sys.path.insert(0, lintdir) +logger = logging.getLogger("mozlint") + + +def pytest_generate_tests(metafunc): + """Finds, loads and returns the config for the linter name specified by the + LINTER global variable in the calling module. + + This implies that each test file (that uses this fixture) should only be + used to test a single linter. If no LINTER variable is defined, the test + will fail. + """ + if "config" in metafunc.fixturenames: + if not hasattr(metafunc.module, "LINTER"): + pytest.fail( + "'config' fixture used from a module that didn't set the LINTER variable" + ) + + name = metafunc.module.LINTER + config_path = path.join(lintdir, "{}.yml".format(name)) + parser = Parser(build.topsrcdir) + configs = parser.parse(config_path) + config_names = {config["name"] for config in configs} + + marker = metafunc.definition.get_closest_marker("lint_config") + if marker: + config_name = marker.kwargs["name"] + if config_name not in config_names: + pytest.fail(f"lint config {config_name} not present in {name}.yml") + configs = [ + config for config in configs if config["name"] == marker.kwargs["name"] + ] + + ids = [config["name"] for config in configs] + metafunc.parametrize("config", configs, ids=ids) + + +@pytest.fixture(scope="module") +def root(request): + """Return the root directory for the files of the linter under test. + + For example, with LINTER=flake8 this would be tools/lint/test/files/flake8. + """ + if not hasattr(request.module, "LINTER"): + pytest.fail( + "'root' fixture used from a module that didn't set the LINTER variable" + ) + return path.join(here, "files", request.module.LINTER) + + +@pytest.fixture(scope="module") +def paths(root): + """Return a function that can resolve file paths relative to the linter + under test. + + Can be used like `paths('foo.py', 'bar/baz')`. This will return a list of + absolute paths under the `root` files directory. + """ + + def _inner(*paths): + if not paths: + return [root] + return [path.normpath(path.join(root, p)) for p in paths] + + return _inner + + +@pytest.fixture(autouse=True) +def run_setup(config): + """Make sure that if the linter named in the LINTER global variable has a + setup function, it gets called before running the tests. + """ + if "setup" not in config: + return + + if config["name"] == "clang-format": + # Skip the setup for the clang-format linter, as it requires a Mach context + # (which we may not have if pytest is invoked directly). + return + + log = logging.LoggerAdapter( + logger, {"lintname": config.get("name"), "pid": os.getpid()} + ) + + func = findobject(config["setup"]) + func( + build.topsrcdir, + virtualenv_manager=build.virtualenv_manager, + virtualenv_bin_path=build.virtualenv_manager.bin_path, + log=log, + ) + + +@pytest.fixture +def lint(config, root, request): + """Find and return the 'lint' function for the external linter named in the + LINTER global variable. + + This will automatically pass in the 'config' and 'root' arguments if not + specified. + """ + + if hasattr(request.module, "fixed"): + request.module.fixed = 0 + + try: + func = findobject(config["payload"]) + except (ImportError, ValueError): + pytest.fail( + "could not resolve a lint function from '{}'".format(config["payload"]) + ) + + ResultSummary.root = root + + def wrapper(paths, config=config, root=root, collapse_results=False, **lintargs): + logger.setLevel(logging.DEBUG) + lintargs["log"] = logging.LoggerAdapter( + logger, {"lintname": config.get("name"), "pid": os.getpid()} + ) + + results = func(paths, config, root=root, **lintargs) + if hasattr(request.module, "fixed") and isinstance(results, dict): + request.module.fixed += results["fixed"] + + if isinstance(results, dict): + results = results["results"] + + if isinstance(results, (list, tuple)): + results = sorted(results) + + if not collapse_results: + return results + + ret = defaultdict(list) + for r in results: + ret[r.relpath].append(r) + return ret + + return wrapper + + +@pytest.fixture +def structuredlog_lint(config, root, logger=None): + """Find and return the 'lint' function for the external linter named in the + LINTER global variable. This variant of the lint function is for linters that + use the 'structuredlog' type. + + This will automatically pass in the 'config' and 'root' arguments if not + specified. + """ + try: + func = findobject(config["payload"]) + except (ImportError, ValueError): + pytest.fail( + "could not resolve a lint function from '{}'".format(config["payload"]) + ) + + ResultSummary.root = root + + if not logger: + logger = structured_logger() + + def wrapper( + paths, + config=config, + root=root, + logger=logger, + collapse_results=False, + **lintargs, + ): + lintargs["log"] = logging.LoggerAdapter( + logger, {"lintname": config.get("name"), "pid": os.getpid()} + ) + results = func(paths, config, root=root, logger=logger, **lintargs) + if not collapse_results: + return results + + ret = defaultdict(list) + for r in results: + ret[r.path].append(r) + return ret + + return wrapper + + +@pytest.fixture +def global_lint(config, root, request): + try: + func = findobject(config["payload"]) + except (ImportError, ValueError): + pytest.fail( + "could not resolve a lint function from '{}'".format(config["payload"]) + ) + + ResultSummary.root = root + + def wrapper(config=config, root=root, collapse_results=False, **lintargs): + logger.setLevel(logging.DEBUG) + lintargs["log"] = logging.LoggerAdapter( + logger, {"lintname": config.get("name"), "pid": os.getpid()} + ) + results = func(config, root=root, **lintargs) + if hasattr(request.module, "fixed") and isinstance(results, dict): + request.module.fixed += results["fixed"] + + if isinstance(results, dict): + results = results["results"] + + if isinstance(results, (list, tuple)): + results = sorted(results) + + if not collapse_results: + return results + + ret = defaultdict(list) + for r in results: + ret[r.relpath].append(r) + return ret + + return wrapper + + +@pytest.fixture +def create_temp_file(tmpdir): + def inner(contents, name=None): + name = name or "temp.py" + path = tmpdir.join(name) + path.write(contents) + return path.strpath + + return inner + + +@pytest.fixture +def structured_logger(): + return StructuredLogger("logger") + + +@pytest.fixture +def perfdocs_sample(): + from test_perfdocs import ( + DYNAMIC_SAMPLE_CONFIG, + SAMPLE_CONFIG, + SAMPLE_INI, + SAMPLE_TEST, + temp_dir, + temp_file, + ) + + with temp_dir() as tmpdir: + suite_dir = pathlib.Path(tmpdir, "suite") + raptor_dir = pathlib.Path(tmpdir, "raptor") + raptor_suitedir = pathlib.Path(tmpdir, "raptor", "suite") + raptor_another_suitedir = pathlib.Path(tmpdir, "raptor", "another_suite") + perfdocs_dir = pathlib.Path(tmpdir, "perfdocs") + + perfdocs_dir.mkdir(parents=True, exist_ok=True) + suite_dir.mkdir(parents=True, exist_ok=True) + raptor_dir.mkdir(parents=True, exist_ok=True) + raptor_suitedir.mkdir(parents=True, exist_ok=True) + raptor_another_suitedir.mkdir(parents=True, exist_ok=True) + + with temp_file( + "perftest.toml", tempdir=suite_dir, content='["perftest_sample.js"]' + ) as tmpmanifest, temp_file( + "raptor_example1.ini", tempdir=raptor_suitedir, content=SAMPLE_INI + ) as tmpexample1manifest, temp_file( + "raptor_example2.ini", tempdir=raptor_another_suitedir, content=SAMPLE_INI + ) as tmpexample2manifest, temp_file( + "perftest_sample.js", tempdir=suite_dir, content=SAMPLE_TEST + ) as tmptest, temp_file( + "config.yml", tempdir=perfdocs_dir, content=SAMPLE_CONFIG + ) as tmpconfig, temp_file( + "config_2.yml", tempdir=perfdocs_dir, content=DYNAMIC_SAMPLE_CONFIG + ) as tmpconfig_2, temp_file( + "index.rst", tempdir=perfdocs_dir, content="{documentation}" + ) as tmpindex: + yield { + "top_dir": tmpdir, + "manifest": {"path": tmpmanifest}, + "example1_manifest": tmpexample1manifest, + "example2_manifest": tmpexample2manifest, + "test": tmptest, + "config": tmpconfig, + "config_2": tmpconfig_2, + "index": tmpindex, + } diff --git a/tools/lint/test/files/android-format/Bad.java b/tools/lint/test/files/android-format/Bad.java new file mode 100644 index 0000000000..13aa5d49d5 --- /dev/null +++ b/tools/lint/test/files/android-format/Bad.java @@ -0,0 +1,8 @@ +package org.mozilla.geckoview; + +import java.util.Arrays; + +public class Bad { + public static void main() { + } +} diff --git a/tools/lint/test/files/android-format/Main.kt b/tools/lint/test/files/android-format/Main.kt new file mode 100644 index 0000000000..a172cf71ee --- /dev/null +++ b/tools/lint/test/files/android-format/Main.kt @@ -0,0 +1,7 @@ +package org.mozilla.geckoview + +import java.util.Arrays; + +fun main() { +println("Hello") +} diff --git a/tools/lint/test/files/android-format/build.gradle b/tools/lint/test/files/android-format/build.gradle new file mode 100644 index 0000000000..6d2aff6d60 --- /dev/null +++ b/tools/lint/test/files/android-format/build.gradle @@ -0,0 +1 @@ +buildDir "${topobjdir}/gradle/build/tools/lint/test/files/android-format" diff --git a/tools/lint/test/files/black/bad.py b/tools/lint/test/files/black/bad.py new file mode 100644 index 0000000000..0a50df4dd9 --- /dev/null +++ b/tools/lint/test/files/black/bad.py @@ -0,0 +1,6 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +print ( + "test" + ) diff --git a/tools/lint/test/files/black/invalid.py b/tools/lint/test/files/black/invalid.py new file mode 100644 index 0000000000..079ecbad30 --- /dev/null +++ b/tools/lint/test/files/black/invalid.py @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +print( diff --git a/tools/lint/test/files/clang-format/bad/bad.cpp b/tools/lint/test/files/clang-format/bad/bad.cpp new file mode 100644 index 0000000000..f08a83f795 --- /dev/null +++ b/tools/lint/test/files/clang-format/bad/bad.cpp @@ -0,0 +1,6 @@ +int main ( ) { + +return 0; + + +} diff --git a/tools/lint/test/files/clang-format/bad/bad2.c b/tools/lint/test/files/clang-format/bad/bad2.c new file mode 100644 index 0000000000..9792e85071 --- /dev/null +++ b/tools/lint/test/files/clang-format/bad/bad2.c @@ -0,0 +1,8 @@ +#include "bad2.h" + + +int bad2() { + int a =2; + return a; + +} diff --git a/tools/lint/test/files/clang-format/bad/bad2.h b/tools/lint/test/files/clang-format/bad/bad2.h new file mode 100644 index 0000000000..a35d49d7e7 --- /dev/null +++ b/tools/lint/test/files/clang-format/bad/bad2.h @@ -0,0 +1 @@ +int bad2(void ); diff --git a/tools/lint/test/files/clang-format/bad/good.cpp b/tools/lint/test/files/clang-format/bad/good.cpp new file mode 100644 index 0000000000..76e8197013 --- /dev/null +++ b/tools/lint/test/files/clang-format/bad/good.cpp @@ -0,0 +1 @@ +int main() { return 0; } diff --git a/tools/lint/test/files/clang-format/good/foo.cpp b/tools/lint/test/files/clang-format/good/foo.cpp new file mode 100644 index 0000000000..76e8197013 --- /dev/null +++ b/tools/lint/test/files/clang-format/good/foo.cpp @@ -0,0 +1 @@ +int main() { return 0; } diff --git a/tools/lint/test/files/clippy/test1/Cargo.toml b/tools/lint/test/files/clippy/test1/Cargo.toml new file mode 100644 index 0000000000..92d5072eca --- /dev/null +++ b/tools/lint/test/files/clippy/test1/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "hello_world" # the name of the package +version = "0.1.0" # the current version, obeying semver +authors = ["Alice <a@example.com>", "Bob <b@example.com>"] + +[[bin]] +name ="good" +path = "good.rs" + +[[bin]] +name ="bad" +path = "bad.rs" + +[[bin]] +name ="bad2" +path = "bad2.rs" + diff --git a/tools/lint/test/files/clippy/test1/bad.rs b/tools/lint/test/files/clippy/test1/bad.rs new file mode 100644 index 0000000000..c403fba603 --- /dev/null +++ b/tools/lint/test/files/clippy/test1/bad.rs @@ -0,0 +1,14 @@ +fn main() { + // Statements here are executed when the compiled binary is called + + // Print text to the console + println!("Hello World!"); + // Clippy detects this as a swap and considers this as an error + let mut a=1; + let mut b=1; + + a = b; + b = a; + + +} diff --git a/tools/lint/test/files/clippy/test1/bad2.rs b/tools/lint/test/files/clippy/test1/bad2.rs new file mode 100644 index 0000000000..bf488bbe72 --- /dev/null +++ b/tools/lint/test/files/clippy/test1/bad2.rs @@ -0,0 +1,14 @@ +fn main() { + // Statements here are executed when the compiled binary is called + + // Print text to the console + println!("Hello World!"); + let mut a; + let mut b=1; + let mut vec = vec![1, 2]; + + for x in 5..10 - 5 { + a = x; + } + + } diff --git a/tools/lint/test/files/clippy/test1/good.rs b/tools/lint/test/files/clippy/test1/good.rs new file mode 100644 index 0000000000..9bcaee67b7 --- /dev/null +++ b/tools/lint/test/files/clippy/test1/good.rs @@ -0,0 +1,6 @@ +fn main() { + // Statements here are executed when the compiled binary is called + + // Print text to the console + println!("Hello World!"); +} diff --git a/tools/lint/test/files/clippy/test2/Cargo.lock b/tools/lint/test/files/clippy/test2/Cargo.lock new file mode 100644 index 0000000000..6b2bc69eeb --- /dev/null +++ b/tools/lint/test/files/clippy/test2/Cargo.lock @@ -0,0 +1,5 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "hello_world_2" +version = "0.2.0" diff --git a/tools/lint/test/files/clippy/test2/Cargo.toml b/tools/lint/test/files/clippy/test2/Cargo.toml new file mode 100644 index 0000000000..b0ac992088 --- /dev/null +++ b/tools/lint/test/files/clippy/test2/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "hello_world_2" # the name of the package +version = "0.2.0" # the current version, obeying semver +authors = ["Alice <a@example.com>", "Bob <b@example.com>"] + +[[bin]] +name = "fake_lib1" +path = "src/bad_1.rs" diff --git a/tools/lint/test/files/clippy/test2/src/bad_1.rs b/tools/lint/test/files/clippy/test2/src/bad_1.rs new file mode 100644 index 0000000000..2fe0630202 --- /dev/null +++ b/tools/lint/test/files/clippy/test2/src/bad_1.rs @@ -0,0 +1,15 @@ +mod bad_2; + +fn main() { + // Statements here are executed when the compiled binary is called + + // Print text to the console + println!("Hello World!"); + // Clippy detects this as a swap and considers this as an error + let mut a=1; + let mut b=1; + + a = b; + b = a; + +} diff --git a/tools/lint/test/files/clippy/test2/src/bad_2.rs b/tools/lint/test/files/clippy/test2/src/bad_2.rs new file mode 100644 index 0000000000..f77de330b4 --- /dev/null +++ b/tools/lint/test/files/clippy/test2/src/bad_2.rs @@ -0,0 +1,17 @@ +fn foo() { + // Statements here are executed when the compiled binary is called + + // Print text to the console + println!("Hello World!"); + let mut a; + let mut b=1; + let mut vec = Vec::new(); + vec.push(1); + vec.push(2); + + + for x in 5..10 - 5 { + a = x; + } + + } diff --git a/tools/lint/test/files/codespell/ignore.rst b/tools/lint/test/files/codespell/ignore.rst new file mode 100644 index 0000000000..1371d07054 --- /dev/null +++ b/tools/lint/test/files/codespell/ignore.rst @@ -0,0 +1,5 @@ +This is a file with some typos and informations. +But also testing false positive like optin (because this isn't always option) +or stuff related to our coding style like: +aparent (aParent). +but detects mistakes like mozila diff --git a/tools/lint/test/files/condprof-addons/browsertime.yml b/tools/lint/test/files/condprof-addons/browsertime.yml new file mode 100644 index 0000000000..7f065809d9 --- /dev/null +++ b/tools/lint/test/files/condprof-addons/browsertime.yml @@ -0,0 +1,10 @@ +--- +firefox-addons: + description: "fixture for the expected firefox-addons.tar ci fetch config" + fetch: + type: static-url + artifact-name: firefox-addons.tar.zst + add-prefix: firefox-addons/ + url: https://localhost/fake-firefox-addons.tar + sha256: 20372ff1d58fc33d1568f8922fe66e2e2e01c77663820344d2a364a8ddd68281 + size: 3584000 diff --git a/tools/lint/test/files/condprof-addons/fake-condprof-config.json b/tools/lint/test/files/condprof-addons/fake-condprof-config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/tools/lint/test/files/condprof-addons/fake-condprof-config.json @@ -0,0 +1 @@ +{} diff --git a/tools/lint/test/files/condprof-addons/fake-customizations-dir/fake-config-01.json b/tools/lint/test/files/condprof-addons/fake-customizations-dir/fake-config-01.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/tools/lint/test/files/condprof-addons/fake-customizations-dir/fake-config-01.json @@ -0,0 +1 @@ +{} diff --git a/tools/lint/test/files/condprof-addons/fake-customizations-dir/fake-config-02.json b/tools/lint/test/files/condprof-addons/fake-customizations-dir/fake-config-02.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/tools/lint/test/files/condprof-addons/fake-customizations-dir/fake-config-02.json @@ -0,0 +1 @@ +{} diff --git a/tools/lint/test/files/condprof-addons/fake-customizations-dir/fake-config-03.json b/tools/lint/test/files/condprof-addons/fake-customizations-dir/fake-config-03.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/tools/lint/test/files/condprof-addons/fake-customizations-dir/fake-config-03.json @@ -0,0 +1 @@ +{} diff --git a/tools/lint/test/files/condprof-addons/fake-fetches-dir/firefox-addons/fake-ext-01.xpi b/tools/lint/test/files/condprof-addons/fake-fetches-dir/firefox-addons/fake-ext-01.xpi new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/lint/test/files/condprof-addons/fake-fetches-dir/firefox-addons/fake-ext-01.xpi diff --git a/tools/lint/test/files/condprof-addons/fake-fetches-dir/firefox-addons/fake-ext-02.xpi b/tools/lint/test/files/condprof-addons/fake-fetches-dir/firefox-addons/fake-ext-02.xpi new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/lint/test/files/condprof-addons/fake-fetches-dir/firefox-addons/fake-ext-02.xpi diff --git a/tools/lint/test/files/condprof-addons/firefox-addons-fake.tar b/tools/lint/test/files/condprof-addons/firefox-addons-fake.tar Binary files differnew file mode 100644 index 0000000000..2b7e13b82c --- /dev/null +++ b/tools/lint/test/files/condprof-addons/firefox-addons-fake.tar diff --git a/tools/lint/test/files/condprof-addons/with-missing-xpi.json b/tools/lint/test/files/condprof-addons/with-missing-xpi.json new file mode 100644 index 0000000000..ae44833a70 --- /dev/null +++ b/tools/lint/test/files/condprof-addons/with-missing-xpi.json @@ -0,0 +1,5 @@ +{ + "addons": { + "non-existing": "http://localhost/non-existing.xpi" + } +} diff --git a/tools/lint/test/files/eslint/good.js b/tools/lint/test/files/eslint/good.js new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/lint/test/files/eslint/good.js diff --git a/tools/lint/test/files/eslint/import/bad_import.js b/tools/lint/test/files/eslint/import/bad_import.js new file mode 100644 index 0000000000..e2a8ec8de1 --- /dev/null +++ b/tools/lint/test/files/eslint/import/bad_import.js @@ -0,0 +1 @@ +/* import-globals-from notpresent/notpresent.js */ diff --git a/tools/lint/test/files/eslint/nolint/foo.txt b/tools/lint/test/files/eslint/nolint/foo.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/lint/test/files/eslint/nolint/foo.txt diff --git a/tools/lint/test/files/eslint/subdir/bad.js b/tools/lint/test/files/eslint/subdir/bad.js new file mode 100644 index 0000000000..9d2dd18f39 --- /dev/null +++ b/tools/lint/test/files/eslint/subdir/bad.js @@ -0,0 +1,2 @@ +// Missing semicolon +let foo = "bar" diff --git a/tools/lint/test/files/eslint/testprettierignore b/tools/lint/test/files/eslint/testprettierignore new file mode 100644 index 0000000000..c2df665174 --- /dev/null +++ b/tools/lint/test/files/eslint/testprettierignore @@ -0,0 +1 @@ +# Intentionally empty file. diff --git a/tools/lint/test/files/file-perm/maybe-shebang/bad.js b/tools/lint/test/files/file-perm/maybe-shebang/bad.js new file mode 100755 index 0000000000..1a0b4c5fd6 --- /dev/null +++ b/tools/lint/test/files/file-perm/maybe-shebang/bad.js @@ -0,0 +1,2 @@ +# Nothing too + diff --git a/tools/lint/test/files/file-perm/maybe-shebang/good.js b/tools/lint/test/files/file-perm/maybe-shebang/good.js new file mode 100755 index 0000000000..8149c0d4f3 --- /dev/null +++ b/tools/lint/test/files/file-perm/maybe-shebang/good.js @@ -0,0 +1,5 @@ +#!/usr/bin/env node + + +# Nothing + diff --git a/tools/lint/test/files/file-perm/no-shebang/bad-shebang.c b/tools/lint/test/files/file-perm/no-shebang/bad-shebang.c new file mode 100755 index 0000000000..7151678efa --- /dev/null +++ b/tools/lint/test/files/file-perm/no-shebang/bad-shebang.c @@ -0,0 +1,2 @@ +#!/bin/bash +int main() { return 0; } diff --git a/tools/lint/test/files/file-perm/no-shebang/bad.c b/tools/lint/test/files/file-perm/no-shebang/bad.c new file mode 100755 index 0000000000..76e8197013 --- /dev/null +++ b/tools/lint/test/files/file-perm/no-shebang/bad.c @@ -0,0 +1 @@ +int main() { return 0; } diff --git a/tools/lint/test/files/file-perm/no-shebang/bad.png b/tools/lint/test/files/file-perm/no-shebang/bad.png Binary files differnew file mode 100755 index 0000000000..db3a5fda7e --- /dev/null +++ b/tools/lint/test/files/file-perm/no-shebang/bad.png diff --git a/tools/lint/test/files/file-perm/no-shebang/good.c b/tools/lint/test/files/file-perm/no-shebang/good.c new file mode 100644 index 0000000000..76e8197013 --- /dev/null +++ b/tools/lint/test/files/file-perm/no-shebang/good.c @@ -0,0 +1 @@ +int main() { return 0; } diff --git a/tools/lint/test/files/file-whitespace/bad-newline.c b/tools/lint/test/files/file-whitespace/bad-newline.c new file mode 100644 index 0000000000..3746a0add3 --- /dev/null +++ b/tools/lint/test/files/file-whitespace/bad-newline.c @@ -0,0 +1,3 @@ +int main() { return 0; } + +
\ No newline at end of file diff --git a/tools/lint/test/files/file-whitespace/bad-windows.c b/tools/lint/test/files/file-whitespace/bad-windows.c new file mode 100644 index 0000000000..70d4c697b9 --- /dev/null +++ b/tools/lint/test/files/file-whitespace/bad-windows.c @@ -0,0 +1,3 @@ +int main(){
+ return 42;
+}
diff --git a/tools/lint/test/files/file-whitespace/bad.c b/tools/lint/test/files/file-whitespace/bad.c new file mode 100644 index 0000000000..4309b1f55d --- /dev/null +++ b/tools/lint/test/files/file-whitespace/bad.c @@ -0,0 +1,3 @@ +int main() { +return 0; +} diff --git a/tools/lint/test/files/file-whitespace/bad.js b/tools/lint/test/files/file-whitespace/bad.js new file mode 100644 index 0000000000..3441696ef1 --- /dev/null +++ b/tools/lint/test/files/file-whitespace/bad.js @@ -0,0 +1,3 @@ +# Nothing too + + diff --git a/tools/lint/test/files/file-whitespace/good.c b/tools/lint/test/files/file-whitespace/good.c new file mode 100644 index 0000000000..76e8197013 --- /dev/null +++ b/tools/lint/test/files/file-whitespace/good.c @@ -0,0 +1 @@ +int main() { return 0; } diff --git a/tools/lint/test/files/file-whitespace/good.js b/tools/lint/test/files/file-whitespace/good.js new file mode 100644 index 0000000000..8149c0d4f3 --- /dev/null +++ b/tools/lint/test/files/file-whitespace/good.js @@ -0,0 +1,5 @@ +#!/usr/bin/env node + + +# Nothing + diff --git a/tools/lint/test/files/fluent-lint/bad.ftl b/tools/lint/test/files/fluent-lint/bad.ftl new file mode 100644 index 0000000000..ebc0e1a602 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/bad.ftl @@ -0,0 +1,44 @@ +Blah-blah = Uppercase letters in identifiers are not permitted. +blah-blah = This is a legal identifier. +blah_blah = Underscores in identifiers are not permitted. + +bad-apostrophe-1 = The bee's knees +bad-apostrophe-end-1 = The bees' knees +bad-apostrophe-2 = The bee‘s knees +bad-single-quote = 'The bee’s knees' +ok-apostrophe = The bee’s knees +ok-single-quote = ‘The bee’s knees’ +bad-double-quote = "The bee’s knees" +good-double-quote = “The bee’s knees” +bad-ellipsis = The bee’s knees... +good-ellipsis = The bee’s knees… + +embedded-tag = Read more about <a data-l10n-name="privacy-policy"> our privacy policy </a>. +bad-embedded-tag = Read more about <a data-l10n-name="privacy-policy"> our 'privacy' policy </a>. + +Invalid_Id = This identifier is in the exclusions file and will not cause an error. + +good-has-attributes = + .accessKey = Attribute identifiers are not checked. + +bad-has-attributes = + .accessKey = Attribute 'values' are checked. + +good-function-call = Last modified: { DATETIME($timeChanged, day: "numeric", month: "long", year: "numeric") } + +# $engineName (String) - The engine name that will currently be used for the private window. +good-variable-identifier = { $engineName } is your default search engine in Private Windows + +short-id = I am too short + +identifiers-in-selectors-should-be-ignored = + .label = { $tabCount -> + [1] Send Tab to Device + [UPPERCASE] Send Tab to Device + *[other] Send { $tabCount } Tabs to Device + } + +this-message-reference-is-ignored = + .label = { menu-quit.label } + +ok-message-with-html-and-var = This is a <a href="{ $url }">link</a> diff --git a/tools/lint/test/files/fluent-lint/brand-names-excluded.ftl b/tools/lint/test/files/fluent-lint/brand-names-excluded.ftl new file mode 100644 index 0000000000..9f3afa28b8 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/brand-names-excluded.ftl @@ -0,0 +1,2 @@ +# Comment +bad-firefox1 = Welcome to Firefox diff --git a/tools/lint/test/files/fluent-lint/brand-names.ftl b/tools/lint/test/files/fluent-lint/brand-names.ftl new file mode 100644 index 0000000000..c338d920ca --- /dev/null +++ b/tools/lint/test/files/fluent-lint/brand-names.ftl @@ -0,0 +1,30 @@ +bad-firefox1 = Welcome to Firefox + +# Comment should be ignored when displaying the offset of the error +bad-firefox2 = Welcome to Firefox again +bad-firefox2b = <span>Welcome to Firefox<span> again +bad-firefox3 = <b>Firefox</b> +bad-firefox-excluded = <b>Firefox</b> + +bad-mozilla1 = Welcome to Mozilla +bad-mozilla2 = Welcome to Mozilla again +bad-mozilla2b = <span>Welcome to Mozilla</span> again +bad-mozilla3 = <b>Mozilla</b> + +bad-thunderbird1 = Welcome to Thunderbird +bad-thunderbird2 = <span>Welcome to Thunderbird</span> again +bad-thunderbird3 = <b>Thunderbird</b> + +good-firefox1 = Welcome to { -brand-firefox } +good-firefox2 = Welcome to { firefox-message } + +good-mozilla1 = Welcome to { -brand-mozilla } +good-mozilla2 = Welcome to { mozilla-message } + +good-thunderbird1 = Welcome to { -brand-thunderbird } +good-thunderbird2 = Welcome to { thunderbird-message } + +# There are no brand checks on terms +-brand-firefox = Firefox + +bland-message = No brands here. diff --git a/tools/lint/test/files/fluent-lint/comment-group1.ftl b/tools/lint/test/files/fluent-lint/comment-group1.ftl new file mode 100644 index 0000000000..32c19dc441 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-group1.ftl @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +### Test group comments. + +fake-identifier-1 = Fake text + +## Pass: This group comment has proper spacing. + +fake-identifier-2 = Fake text +## Fail: (GC03) Group comments should have an empty line before them. + +fake-identifier-3 = Fake text + +## Fail: (GC02) Group comments should have an empty line after them. +fake-identifier-4 = Fake text + +## Pass: A single comment is fine. + +## Fail: (GC04) A group comment must be followed by at least one message. + +fake-identifier-5 = Fake text + + +## Fail: (GC03) Only allow 1 line above. + +fake-identifier-6 = Fake text + +## Fail: (GC02) Only allow 1 line below. + + +fake-identifier-6 = Fake text + +## Fail: (GC01) Group comments should not be at the end of a file. diff --git a/tools/lint/test/files/fluent-lint/comment-group2.ftl b/tools/lint/test/files/fluent-lint/comment-group2.ftl new file mode 100644 index 0000000000..47d29fa211 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-group2.ftl @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +### Test group comments. + +## Pass: This group comment is followed by a term + +-fake-term = Fake text + +## Pass: The last group comment is allowed to be an empty ## + +fake-identifier-1 = Fake text + +## diff --git a/tools/lint/test/files/fluent-lint/comment-resource1.ftl b/tools/lint/test/files/fluent-lint/comment-resource1.ftl new file mode 100644 index 0000000000..f5d5e53d59 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-resource1.ftl @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +### Pass: This is a resource comment with proper spacing. + +fake-identifier-1 = Fake text + +### Fail: (RC01) There should not be more than one resource comment + +fake-identifier-2 = Fake text diff --git a/tools/lint/test/files/fluent-lint/comment-resource2.ftl b/tools/lint/test/files/fluent-lint/comment-resource2.ftl new file mode 100644 index 0000000000..44a77f4e73 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-resource2.ftl @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +### Fail: (RC03) There should be an empty line preceeding. + +fake-identifier-1 = Fake text diff --git a/tools/lint/test/files/fluent-lint/comment-resource3.ftl b/tools/lint/test/files/fluent-lint/comment-resource3.ftl new file mode 100644 index 0000000000..b261404380 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-resource3.ftl @@ -0,0 +1,6 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +### Fail: (RC02) There should be an empty line following. +fake-identifier-1 = Fake text diff --git a/tools/lint/test/files/fluent-lint/comment-resource4.ftl b/tools/lint/test/files/fluent-lint/comment-resource4.ftl new file mode 100644 index 0000000000..c24e8887f8 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-resource4.ftl @@ -0,0 +1,8 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +### Fail: (RC03) There should be only one space above. + +fake-identifier-1 = Fake text diff --git a/tools/lint/test/files/fluent-lint/comment-resource5.ftl b/tools/lint/test/files/fluent-lint/comment-resource5.ftl new file mode 100644 index 0000000000..60d8e8c264 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-resource5.ftl @@ -0,0 +1,8 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +### Fail: (RC02) There should be only one space below. + + +fake-identifier-1 = Fake text diff --git a/tools/lint/test/files/fluent-lint/comment-resource6.ftl b/tools/lint/test/files/fluent-lint/comment-resource6.ftl new file mode 100644 index 0000000000..a2ca9abfe7 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-resource6.ftl @@ -0,0 +1,4 @@ +### Pass: Check two conditions. +### 1. This is an edge case, but we shouldn't error if there is only a resource comment. +### 2. Make sure this linter does not error if there is no license header. The license is +### checked with `mach lint license`. diff --git a/tools/lint/test/files/fluent-lint/comment-variables1.ftl b/tools/lint/test/files/fluent-lint/comment-variables1.ftl new file mode 100644 index 0000000000..10de9de195 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-variables1.ftl @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +### Test group comments. +### $var doesn't count towards commented placeables + +message-without-comment = This string has a { $var } + +## Variables: +## $foo-group (String): group level comment + +# Variables: +# $foo1 (String): just text +message-with-comment = This string has a { $foo1 } + +message-with-group-comment = This string has a { $foo-group } + +select-without-comment1 = { + $select1 -> + [t] Foo + *[s] Bar +} + +select-without-comment2 = { + $select2 -> + [t] Foo { $select2 } + *[s] Bar +} + +## Variables: +## $select4 (Integer): a number + +# Variables: +# $select3 (Integer): a number +select-with-comment1 = { + $select3 -> + [t] Foo + *[s] Bar +} + +select-with-group-comment1 = { + $select4 -> + [t] Foo { $select4 } + *[s] Bar +} + +message-attribute-without-comment = + .label = This string as { $attr } + +# Variables: +# $attr2 (String): just text +message-attribute-with-comment = + .label = This string as { $attr2 } + +message-selection-function = + { PLATFORM() -> + [macos] foo + *[other] bar + } diff --git a/tools/lint/test/files/fluent-lint/comment-variables2.ftl b/tools/lint/test/files/fluent-lint/comment-variables2.ftl new file mode 100644 index 0000000000..2e9ae8e684 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/comment-variables2.ftl @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +## Terms are not checked for variables, no need for comments. + +-term-without-comment1 = { $term1 } +-term-without-comment2 = { + $select2 -> + [t] Foo { $term2 } + *[s] Bar +} + +# Testing that variable references from terms are not kept around when analyzing +# standard messages (see bug 1812568). +# +# Variables: +# $message1 (String) - Just text +message-with-comment = This string has a { $message1 } + +# This comment is not necessary, just making sure it doesn't get carried over to +# the following message which uses the same variable. +# +# Variables: +# $term-message (string) - Text +-term-with-variable = { $term-message } +message-without-comment = This string has a { $term-message } diff --git a/tools/lint/test/files/fluent-lint/excluded.ftl b/tools/lint/test/files/fluent-lint/excluded.ftl new file mode 100644 index 0000000000..79fe509ad6 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/excluded.ftl @@ -0,0 +1,6 @@ +# This file is used to test excluding paths from tests. +Blah-blah = Uppercase letters in identifiers are not permitted. +blah-blah = This is a legal identifier. +blah_blah = Underscores in identifiers are not permitted. + +bad-apostrophe-1 = The bee's knees diff --git a/tools/lint/test/files/fluent-lint/test-brands.ftl b/tools/lint/test/files/fluent-lint/test-brands.ftl new file mode 100644 index 0000000000..1aa6e9d0e8 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/test-brands.ftl @@ -0,0 +1,5 @@ +# These are brands used in the fluent-lint test + +-brand-first = Firefox +-brand-second = Thunderbird +-brand-third = Mozilla diff --git a/tools/lint/test/files/fluent-lint/tools/lint/fluent-lint/exclusions.yml b/tools/lint/test/files/fluent-lint/tools/lint/fluent-lint/exclusions.yml new file mode 100644 index 0000000000..1aecf8cedd --- /dev/null +++ b/tools/lint/test/files/fluent-lint/tools/lint/fluent-lint/exclusions.yml @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +ID01: + messages: + - Invalid_Id + files: + - excluded.ftl +ID02: + messages: [] + files: [] +CO01: + messages: + - bad-firefox-excluded + files: + - brand-names-excluded.ftl diff --git a/tools/lint/test/files/fluent-lint/valid-attributes.ftl b/tools/lint/test/files/fluent-lint/valid-attributes.ftl new file mode 100644 index 0000000000..c308fbd5b3 --- /dev/null +++ b/tools/lint/test/files/fluent-lint/valid-attributes.ftl @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +### Test for valid attributes. + +message-with-known-attribute = + .label = Foo + +# String has a known label but with different case, not a warning +message-with-known-attribute-case = + .Label = Foo + +# Warning: unknown attribute +message-with-unknown-attribute = + .extralabel = Foo + +# NO warning: unknown attribute, but commented +# .extralabel is known +message-with-unknown-attribute-commented = + .extralabel = Foo diff --git a/tools/lint/test/files/license/.eslintrc.js b/tools/lint/test/files/license/.eslintrc.js new file mode 100644 index 0000000000..0449fdfa33 --- /dev/null +++ b/tools/lint/test/files/license/.eslintrc.js @@ -0,0 +1,5 @@ + +// Dot file to verify that it works +// without license + +"use strict"; diff --git a/tools/lint/test/files/license/bad.c b/tools/lint/test/files/license/bad.c new file mode 100644 index 0000000000..76e8197013 --- /dev/null +++ b/tools/lint/test/files/license/bad.c @@ -0,0 +1 @@ +int main() { return 0; } diff --git a/tools/lint/test/files/license/bad.js b/tools/lint/test/files/license/bad.js new file mode 100644 index 0000000000..5de1a72f1f --- /dev/null +++ b/tools/lint/test/files/license/bad.js @@ -0,0 +1,6 @@ +/* + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Pulic Unknown License as published by + * the Free Software Foundation, version 3. + * + */ diff --git a/tools/lint/test/files/license/good-other.h b/tools/lint/test/files/license/good-other.h new file mode 100644 index 0000000000..fb915e9b26 --- /dev/null +++ b/tools/lint/test/files/license/good-other.h @@ -0,0 +1,9 @@ +/* +Permission to use, copy, modify, distribute and sell this software +and its documentation for any purpose is hereby granted without fee, +provided that the above copyright notice appear in all copies and +that both that copyright notice and this permission notice appear +in supporting documentation. Samphan Raruenrom makes no +representations about the suitability of this software for any +purpose. It is provided "as is" without express or implied warranty. +*/ diff --git a/tools/lint/test/files/license/good.c b/tools/lint/test/files/license/good.c new file mode 100644 index 0000000000..d1a6827fb1 --- /dev/null +++ b/tools/lint/test/files/license/good.c @@ -0,0 +1,8 @@ + +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +int main() { return 0; } diff --git a/tools/lint/test/files/license/good.js b/tools/lint/test/files/license/good.js new file mode 100644 index 0000000000..d10ae3a8d5 --- /dev/null +++ b/tools/lint/test/files/license/good.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +# Nothing + diff --git a/tools/lint/test/files/lintpref/bad.js b/tools/lint/test/files/lintpref/bad.js new file mode 100644 index 0000000000..ab55ba5dad --- /dev/null +++ b/tools/lint/test/files/lintpref/bad.js @@ -0,0 +1,2 @@ +// Real test pref, matching value. +pref("dom.webidl.test1", true); diff --git a/tools/lint/test/files/lintpref/good.js b/tools/lint/test/files/lintpref/good.js new file mode 100644 index 0000000000..0bf81c8f58 --- /dev/null +++ b/tools/lint/test/files/lintpref/good.js @@ -0,0 +1,6 @@ +// Fake prefs. +pref("foo.bar", 1); +pref("foo.baz", "1.234"); + +// Real test pref, different value. +pref("dom.webidl.test1", false); diff --git a/tools/lint/test/files/rst/.dotfile.rst b/tools/lint/test/files/rst/.dotfile.rst new file mode 100644 index 0000000000..be24e1d161 --- /dev/null +++ b/tools/lint/test/files/rst/.dotfile.rst @@ -0,0 +1,11 @@ +============ +Coding style +========== + +foo bar +~~~~~ + + +This file has error but should not be there +as we don't analyze dot files + diff --git a/tools/lint/test/files/rst/bad.rst b/tools/lint/test/files/rst/bad.rst new file mode 100644 index 0000000000..c9b60f613e --- /dev/null +++ b/tools/lint/test/files/rst/bad.rst @@ -0,0 +1,20 @@ +============ +Coding style +============ + + +This document attempts to explain the basic styles and patterns used in +the Mozilla codebase. New code should try to conform to these standards, +so it is as easy to maintain as existing code. There are exceptions, but +it's still important to know the rules! + + +Whitespace +~~~~~~~~ + +Line length +~~~~~~~~~~~ + +Line length +~~~~~~~~~~~ + diff --git a/tools/lint/test/files/rst/bad2.rst b/tools/lint/test/files/rst/bad2.rst new file mode 100644 index 0000000000..81c35dde06 --- /dev/null +++ b/tools/lint/test/files/rst/bad2.rst @@ -0,0 +1,4 @@ +==== +Test +=== + diff --git a/tools/lint/test/files/rst/bad3.rst b/tools/lint/test/files/rst/bad3.rst new file mode 100644 index 0000000000..b7e66e5c92 --- /dev/null +++ b/tools/lint/test/files/rst/bad3.rst @@ -0,0 +1,6 @@ + +.. _When_Should_I_Use_a_Hashtable.3F: + +When Should I Use a Hashtable? +------------------------------ + diff --git a/tools/lint/test/files/rst/good.rst b/tools/lint/test/files/rst/good.rst new file mode 100644 index 0000000000..fd12da85d3 --- /dev/null +++ b/tools/lint/test/files/rst/good.rst @@ -0,0 +1,11 @@ +============ +Coding style +============ + + +This document attempts to explain the basic styles and patterns used in +the Mozilla codebase. New code should try to conform to these standards, +so it is as easy to maintain as existing code. There are exceptions, but +it's still important to know the rules! + + diff --git a/tools/lint/test/files/ruff/bad.py b/tools/lint/test/files/ruff/bad.py new file mode 100644 index 0000000000..0015d7e7f9 --- /dev/null +++ b/tools/lint/test/files/ruff/bad.py @@ -0,0 +1,4 @@ +import distutils + +if not "foo" in "foobar": + print("oh no!") diff --git a/tools/lint/test/files/ruff/ruff.toml b/tools/lint/test/files/ruff/ruff.toml new file mode 100644 index 0000000000..34f5ca74a4 --- /dev/null +++ b/tools/lint/test/files/ruff/ruff.toml @@ -0,0 +1 @@ +# Empty config to force ruff to ignore the global one. diff --git a/tools/lint/test/files/rustfmt/subdir/bad.rs b/tools/lint/test/files/rustfmt/subdir/bad.rs new file mode 100644 index 0000000000..fb1746fafd --- /dev/null +++ b/tools/lint/test/files/rustfmt/subdir/bad.rs @@ -0,0 +1,16 @@ +fn main() { + // Statements here are executed when the compiled binary is called + + // Print text to the console + println!("Hello World!"); + // Clippy detects this as a swap and considers this as an error + let mut a = + 1; + let mut b=1; + + a = + b; + b = a; + + +} diff --git a/tools/lint/test/files/rustfmt/subdir/bad2.rs b/tools/lint/test/files/rustfmt/subdir/bad2.rs new file mode 100644 index 0000000000..a4236a2de7 --- /dev/null +++ b/tools/lint/test/files/rustfmt/subdir/bad2.rs @@ -0,0 +1,17 @@ +fn main() { + // Statements here are executed when the compiled binary is called + + // Print text to the console + println!("Hello World!"); + let mut a; + let mut b=1; + let mut vec = Vec::new(); + vec.push(1); + vec.push(2); + + + for x in 5..10 - 5 { + a = x; + } + + } diff --git a/tools/lint/test/files/rustfmt/subdir/good.rs b/tools/lint/test/files/rustfmt/subdir/good.rs new file mode 100644 index 0000000000..9bcaee67b7 --- /dev/null +++ b/tools/lint/test/files/rustfmt/subdir/good.rs @@ -0,0 +1,6 @@ +fn main() { + // Statements here are executed when the compiled binary is called + + // Print text to the console + println!("Hello World!"); +} diff --git a/tools/lint/test/files/shellcheck/bad.sh b/tools/lint/test/files/shellcheck/bad.sh new file mode 100644 index 0000000000..b2eb195558 --- /dev/null +++ b/tools/lint/test/files/shellcheck/bad.sh @@ -0,0 +1,3 @@ +#!/bin/sh +hello="Hello world" +echo $1 diff --git a/tools/lint/test/files/shellcheck/good.sh b/tools/lint/test/files/shellcheck/good.sh new file mode 100644 index 0000000000..e61d501955 --- /dev/null +++ b/tools/lint/test/files/shellcheck/good.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo "Hello world" diff --git a/tools/lint/test/files/stylelint/nolint/foo.txt b/tools/lint/test/files/stylelint/nolint/foo.txt new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/lint/test/files/stylelint/nolint/foo.txt diff --git a/tools/lint/test/files/stylelint/subdir/bad.css b/tools/lint/test/files/stylelint/subdir/bad.css new file mode 100644 index 0000000000..70004c1fb2 --- /dev/null +++ b/tools/lint/test/files/stylelint/subdir/bad.css @@ -0,0 +1,5 @@ +#foo { + /* Duplicate property: */ + font-size: 12px; + font-size: 12px; +} diff --git a/tools/lint/test/files/test-manifest-alpha/mochitest-in-order.ini b/tools/lint/test/files/test-manifest-alpha/mochitest-in-order.ini new file mode 100644 index 0000000000..9de566702a --- /dev/null +++ b/tools/lint/test/files/test-manifest-alpha/mochitest-in-order.ini @@ -0,0 +1,29 @@ +[DEFAULT] +support-files = + click-event-helper.js + head.js + test-button-overlay.html + ../../../../dom/media/test/gizmo.mp4 + ../../../../dom/media/test/owl.mp3 + +prefs = + media.videocontrols.picture-in-picture.display-text-tracks.enabled=false + +[browser_AAA_run_first_firstTimePiPToggleEvents.js] +[browser_aaa_run_first_firstTimePiPToggleEvents.js] +[browser_backgroundTab.js] +[browser_cannotTriggerFromContent.js] +[browser_closePipPause.js] +skip-if = os == "linux" && bits == 64 && os_version == "18.04" # Bug 1569205 +[browser_closePip_pageNavigationChanges.js] +[browser_closePlayer.js] +[browser_closeTab.js] +[browser_close_unpip_focus.js] +[browser_contextMenu.js] +[browser_cornerSnapping.js] +run-if = os == "mac" +[browser_dblclickFullscreen.js] +[browser_flipIconWithRTL.js] +skip-if = + os == "linux" && ccov # Bug 1678091 + tsan # Bug 1678091 diff --git a/tools/lint/test/files/test-manifest-alpha/mochitest-mostly-in-order.ini b/tools/lint/test/files/test-manifest-alpha/mochitest-mostly-in-order.ini new file mode 100644 index 0000000000..01f2551bd4 --- /dev/null +++ b/tools/lint/test/files/test-manifest-alpha/mochitest-mostly-in-order.ini @@ -0,0 +1,30 @@ +[DEFAULT] +support-files = + click-event-helper.js + head.js + test-button-overlay.html + ../../../../dom/media/test/gizmo.mp4 + ../../../../dom/media/test/owl.mp3 + +prefs = + media.videocontrols.picture-in-picture.display-text-tracks.enabled=false + +[browser_AAA_run_first_firstTimePiPToggleEvents.js] +[browser_aaa_run_first_firstTimePiPToggleEvents.js] +[browser_backgroundTab.js] +[browser_cannotTriggerFromContent.js] +[browser_closePipPause.js] +skip-if = os == "linux" && bits == 64 && os_version == "18.04" # Bug 1569205 +[browser_closePip_pageNavigationChanges.js] +[browser_closePlayer.js] +[browser_closeTab.js] +[browser_close_unpip_focus.js] +[browser_contextMenu.js] +[browser_cornerSnapping.js] +run-if = os == "mac" +[browser_dblclickFullscreen.js] +[browser_flipIconWithRTL.js] +skip-if = + os == "linux" && ccov # Bug 1678091 + tsan # Bug 1678091 +[browser_a_new_test.js] diff --git a/tools/lint/test/files/test-manifest-alpha/mochitest-very-out-of-order.ini b/tools/lint/test/files/test-manifest-alpha/mochitest-very-out-of-order.ini new file mode 100644 index 0000000000..45bfdd776b --- /dev/null +++ b/tools/lint/test/files/test-manifest-alpha/mochitest-very-out-of-order.ini @@ -0,0 +1,29 @@ +[DEFAULT] +support-files = + click-event-helper.js + head.js + test-button-overlay.html + ../../../../dom/media/test/gizmo.mp4 + ../../../../dom/media/test/owl.mp3 + +prefs = + media.videocontrols.picture-in-picture.display-text-tracks.enabled=false + +[browser_contextMenu.js] +[browser_aaa_run_first_firstTimePiPToggleEvents.js] +[browser_AAA_run_first_firstTimePiPToggleEvents.js] +[browser_backgroundTab.js] +[browser_cannotTriggerFromContent.js] +[browser_close_unpip_focus.js] +[browser_closePip_pageNavigationChanges.js] +[browser_closePipPause.js] +skip-if = os == "linux" && bits == 64 && os_version == "18.04" # Bug 1569205 +[browser_cornerSnapping.js] +run-if = os == "mac" +[browser_closePlayer.js] +[browser_closeTab.js] +[browser_dblclickFullscreen.js] +[browser_flipIconWithRTL.js] +skip-if = + os == "linux" && ccov # Bug 1678091 + tsan # Bug 1678091 diff --git a/tools/lint/test/files/test-manifest-alpha/other-ini-very-out-of-order.ini b/tools/lint/test/files/test-manifest-alpha/other-ini-very-out-of-order.ini new file mode 100644 index 0000000000..45bfdd776b --- /dev/null +++ b/tools/lint/test/files/test-manifest-alpha/other-ini-very-out-of-order.ini @@ -0,0 +1,29 @@ +[DEFAULT] +support-files = + click-event-helper.js + head.js + test-button-overlay.html + ../../../../dom/media/test/gizmo.mp4 + ../../../../dom/media/test/owl.mp3 + +prefs = + media.videocontrols.picture-in-picture.display-text-tracks.enabled=false + +[browser_contextMenu.js] +[browser_aaa_run_first_firstTimePiPToggleEvents.js] +[browser_AAA_run_first_firstTimePiPToggleEvents.js] +[browser_backgroundTab.js] +[browser_cannotTriggerFromContent.js] +[browser_close_unpip_focus.js] +[browser_closePip_pageNavigationChanges.js] +[browser_closePipPause.js] +skip-if = os == "linux" && bits == 64 && os_version == "18.04" # Bug 1569205 +[browser_cornerSnapping.js] +run-if = os == "mac" +[browser_closePlayer.js] +[browser_closeTab.js] +[browser_dblclickFullscreen.js] +[browser_flipIconWithRTL.js] +skip-if = + os == "linux" && ccov # Bug 1678091 + tsan # Bug 1678091 diff --git a/tools/lint/test/files/test-manifest-toml/comment-section.toml b/tools/lint/test/files/test-manifest-toml/comment-section.toml new file mode 100644 index 0000000000..fa7289ee90 --- /dev/null +++ b/tools/lint/test/files/test-manifest-toml/comment-section.toml @@ -0,0 +1,5 @@ +[DEFAULT] + +# ["aaa.js"] + + # ["bbb.js"] diff --git a/tools/lint/test/files/test-manifest-toml/invalid.toml b/tools/lint/test/files/test-manifest-toml/invalid.toml new file mode 100644 index 0000000000..d9e8ef6463 --- /dev/null +++ b/tools/lint/test/files/test-manifest-toml/invalid.toml @@ -0,0 +1,2 @@ +# Invalid TOML +& = @ diff --git a/tools/lint/test/files/test-manifest-toml/no-default.toml b/tools/lint/test/files/test-manifest-toml/no-default.toml new file mode 100644 index 0000000000..b891d4ffb8 --- /dev/null +++ b/tools/lint/test/files/test-manifest-toml/no-default.toml @@ -0,0 +1 @@ +# this Manifest has no DEFAULT section diff --git a/tools/lint/test/files/test-manifest-toml/non-double-quote-sections.toml b/tools/lint/test/files/test-manifest-toml/non-double-quote-sections.toml new file mode 100644 index 0000000000..abfa25eb68 --- /dev/null +++ b/tools/lint/test/files/test-manifest-toml/non-double-quote-sections.toml @@ -0,0 +1,6 @@ +[DEFAULT] + +[aaa.js] +# Every non DEFAULT section must be double quoted + +['bbb.js'] diff --git a/tools/lint/test/files/test-manifest-toml/skip-if-explicit-or.toml b/tools/lint/test/files/test-manifest-toml/skip-if-explicit-or.toml new file mode 100644 index 0000000000..082baca3c3 --- /dev/null +++ b/tools/lint/test/files/test-manifest-toml/skip-if-explicit-or.toml @@ -0,0 +1,7 @@ +[DEFAULT] + +["aaa.js"] +skip-if = ["asan || verify"] # should be two lines + +["bbb.js"] +skip-if = ["(asan || verify) && os == 'linux'"] # OK diff --git a/tools/lint/test/files/test-manifest-toml/skip-if-not-array.toml b/tools/lint/test/files/test-manifest-toml/skip-if-not-array.toml new file mode 100644 index 0000000000..e6f7460fb9 --- /dev/null +++ b/tools/lint/test/files/test-manifest-toml/skip-if-not-array.toml @@ -0,0 +1,4 @@ +[DEFAULT] + +["aaa.js"] +run-if = "os == 'linux'" diff --git a/tools/lint/test/files/test-manifest-toml/unsorted.toml b/tools/lint/test/files/test-manifest-toml/unsorted.toml new file mode 100644 index 0000000000..5e949daa04 --- /dev/null +++ b/tools/lint/test/files/test-manifest-toml/unsorted.toml @@ -0,0 +1,10 @@ +[DEFAULT] +# unsorted sections + +["ccc.js"] + +["aaa.js"] + +["bug_10.js"] + +["bug_2.js"] diff --git a/tools/lint/test/files/test-manifest-toml/valid.toml b/tools/lint/test/files/test-manifest-toml/valid.toml new file mode 100644 index 0000000000..20ff24979d --- /dev/null +++ b/tools/lint/test/files/test-manifest-toml/valid.toml @@ -0,0 +1,2 @@ +[DEFAULT] +# a minimal valid ManifestParser TOML file diff --git a/tools/lint/test/files/trojan-source/README b/tools/lint/test/files/trojan-source/README new file mode 100644 index 0000000000..343a9d0c3c --- /dev/null +++ b/tools/lint/test/files/trojan-source/README @@ -0,0 +1,5 @@ +These examples are taken from trojan source: +https://github.com/nickboucher/trojan-source + +The examples are published under the MIT license. + diff --git a/tools/lint/test/files/trojan-source/commenting-out.cpp b/tools/lint/test/files/trojan-source/commenting-out.cpp new file mode 100644 index 0000000000..d67df70ce1 --- /dev/null +++ b/tools/lint/test/files/trojan-source/commenting-out.cpp @@ -0,0 +1,9 @@ +#include <iostream> + +int main() { + bool isAdmin = false; + /* } if (isAdmin) begin admins only */ + std::cout << "You are an admin.\n"; + /* end admins only { */ + return 0; +}
\ No newline at end of file diff --git a/tools/lint/test/files/trojan-source/early-return.py b/tools/lint/test/files/trojan-source/early-return.py new file mode 100644 index 0000000000..2797d8ae9f --- /dev/null +++ b/tools/lint/test/files/trojan-source/early-return.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +bank = { 'alice': 100 } + +def subtract_funds(account: str, amount: int): + ''' Subtract funds from bank account then ''' ;return + bank[account] -= amount + return + +subtract_funds('alice', 50)
\ No newline at end of file diff --git a/tools/lint/test/files/trojan-source/invisible-function.rs b/tools/lint/test/files/trojan-source/invisible-function.rs new file mode 100644 index 0000000000..b32efb0372 --- /dev/null +++ b/tools/lint/test/files/trojan-source/invisible-function.rs @@ -0,0 +1,15 @@ +fn isAdmin() { + return false; +} + +fn isAdmin() { + return true; +} + +fn main() { + if isAdmin() { + printf("You are an admin\n"); + } else { + printf("You are NOT an admin.\n"); + } +}
\ No newline at end of file diff --git a/tools/lint/test/files/updatebot/.yamllint b/tools/lint/test/files/updatebot/.yamllint new file mode 100644 index 0000000000..4f11bbd6c5 --- /dev/null +++ b/tools/lint/test/files/updatebot/.yamllint @@ -0,0 +1,6 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +# Explicity default .yamllint to isolate tests from tree-wide yamlint config. +--- +extends: default diff --git a/tools/lint/test/files/updatebot/cargo-mismatch.yaml b/tools/lint/test/files/updatebot/cargo-mismatch.yaml new file mode 100644 index 0000000000..ac18d2b87c --- /dev/null +++ b/tools/lint/test/files/updatebot/cargo-mismatch.yaml @@ -0,0 +1,44 @@ +--- +# Version of this schema +schema: 1 + +bugzilla: + # Bugzilla product and component for this directory and subdirectories + product: Core + component: "Graphics: WebGPU" + +# Document the source of externally hosted code +origin: + + # Short name of the package/library + name: wgpu + + description: A cross-platform pure-Rust graphics API + + # Full URL for the package's homepage/etc + # Usually different from repository url + url: https://github.com/gfx-rs/wgpu + + # Human-readable identifier for this version/release + # Generally "version NNN", "tag SSS", "bookmark SSS" + release: commit 32af4f56 + + # Revision to pull in + # Must be a long or short commit SHA (long preferred) + revision: idontmatchanything + + license: ['MIT', 'Apache-2.0'] + +updatebot: + maintainer-phab: jimb + maintainer-bz: jimb@mozilla.com + tasks: + - type: vendoring + enabled: true + frequency: 1 week + +vendoring: + url: https://github.com/gfx-rs/wgpu + source-hosting: github + vendor-directory: gfx/wgpu_bindings/ + flavor: rust diff --git a/tools/lint/test/files/updatebot/good1.yaml b/tools/lint/test/files/updatebot/good1.yaml new file mode 100644 index 0000000000..f57d2c5b4c --- /dev/null +++ b/tools/lint/test/files/updatebot/good1.yaml @@ -0,0 +1,44 @@ +--- +schema: 1 + +bugzilla: + product: Core + component: Graphics + +origin: + name: angle + + description: ANGLE - Almost Native Graphics Layer Engine + + url: https://chromium.googlesource.com/angle/angle + + # Note that while the vendoring information here, including revision, + # release, and upstream repo locations refer to the third party upstream, + # Angle is vendored from a mozilla git repository that pulls from + # upstream and mainntains local patches there. + release: commit 018f85dea11fd5e41725750c6958695a6b8e8409 + revision: 018f85dea11fd5e41725750c6958695a6b8e8409 + + license: BSD-3-Clause + +updatebot: + maintainer-phab: jgilbert + maintainer-bz: jgilbert@mozilla.com + tasks: + - type: commit-alert + enabled: true + branch: chromium/4515 + needinfo: ["jgilbert@mozilla.com"] + +vendoring: + url: https://chromium.googlesource.com/angle/angle + tracking: tag + source-hosting: angle + vendor-directory: gfx/angle/checkout + skip-vendoring-steps: ["fetch", "update-moz-build"] + + update-actions: + - action: run-script + script: '{yaml_dir}/auto-update-angle.sh' + args: ['{revision}'] + cwd: '{cwd}' diff --git a/tools/lint/test/files/updatebot/good2.yaml b/tools/lint/test/files/updatebot/good2.yaml new file mode 100644 index 0000000000..0161d28b11 --- /dev/null +++ b/tools/lint/test/files/updatebot/good2.yaml @@ -0,0 +1,74 @@ +--- +# Version of this schema +schema: 1 + +bugzilla: + # Bugzilla product and component for this directory and subdirectories + product: Core + component: "Audio/Video: Playback" + +# Document the source of externally hosted code +origin: + + # Short name of the package/library + name: dav1d + + description: dav1d, a fast AV1 decoder + + # Full URL for the package's homepage/etc + # Usually different from repository url + url: https://code.videolan.org/videolan/dav1d + + # Human-readable identifier for this version/release + # Generally "version NNN", "tag SSS", "bookmark SSS" + release: ffb59680356fd210816cf9e46d9d023ade1f4d5a + + # Revision to pull in + # Must be a long or short commit SHA (long preferred) + revision: ffb59680356fd210816cf9e46d9d023ade1f4d5a + + # The package's license, where possible using the mnemonic from + # https://spdx.org/licenses/ + # Multiple licenses can be specified (as a YAML list) + # A "LICENSE" file must exist containing the full license text + license: BSD-2-Clause + + license-file: COPYING + +updatebot: + maintainer-phab: chunmin + maintainer-bz: cchang@mozilla.com + tasks: + - type: vendoring + enabled: true + frequency: release + +vendoring: + url: https://code.videolan.org/videolan/dav1d + source-hosting: gitlab + vendor-directory: third_party/dav1d + + exclude: + - build/.gitattributes + - build/.gitignore + - doc + - examples + - package + - tools + + generated: + - '{yaml_dir}/vcs_version.h' + - '{yaml_dir}/version.h' + + update-actions: + - action: copy-file + from: include/vcs_version.h.in + to: '{yaml_dir}/vcs_version.h' + - action: replace-in-file + pattern: '@VCS_TAG@' + with: '{revision}' + file: '{yaml_dir}/vcs_version.h' + - action: run-script + script: '{yaml_dir}/update-version.sh' + cwd: '{vendor_dir}' + args: ['{yaml_dir}/version.h'] diff --git a/tools/lint/test/files/updatebot/no-revision.yaml b/tools/lint/test/files/updatebot/no-revision.yaml new file mode 100644 index 0000000000..4d581508d8 --- /dev/null +++ b/tools/lint/test/files/updatebot/no-revision.yaml @@ -0,0 +1,43 @@ +--- +schema: 1 + +bugzilla: + product: Core + component: Graphics + +origin: + name: angle + + description: ANGLE - Almost Native Graphics Layer Engine + + url: https://chromium.googlesource.com/angle/angle + + # Note that while the vendoring information here, including revision, + # release, and upstream repo locations refer to the third party upstream, + # Angle is vendored from a mozilla git repository that pulls from + # upstream and mainntains local patches there. + release: commit 018f85dea11fd5e41725750c6958695a6b8e8409 + + license: BSD-3-Clause + +updatebot: + maintainer-phab: jgilbert + maintainer-bz: jgilbert@mozilla.com + tasks: + - type: commit-alert + enabled: true + branch: chromium/4515 + needinfo: ["jgilbert@mozilla.com"] + +vendoring: + url: https://chromium.googlesource.com/angle/angle + tracking: tag + source-hosting: angle + vendor-directory: gfx/angle/checkout + skip-vendoring-steps: ["fetch", "update-moz-build"] + + update-actions: + - action: run-script + script: '{yaml_dir}/auto-update-angle.sh' + args: ['{revision}'] + cwd: '{cwd}' diff --git a/tools/lint/test/files/yaml/.yamllint b/tools/lint/test/files/yaml/.yamllint new file mode 100644 index 0000000000..4f11bbd6c5 --- /dev/null +++ b/tools/lint/test/files/yaml/.yamllint @@ -0,0 +1,6 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +# Explicity default .yamllint to isolate tests from tree-wide yamlint config. +--- +extends: default diff --git a/tools/lint/test/files/yaml/bad.yml b/tools/lint/test/files/yaml/bad.yml new file mode 100644 index 0000000000..195ac7b030 --- /dev/null +++ b/tools/lint/test/files/yaml/bad.yml @@ -0,0 +1,8 @@ +--- +yamllint: + description: YAML linteraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaax + include: + - .cron.yml + - browser/config/ + - wrong + application:bar diff --git a/tools/lint/test/files/yaml/good.yml b/tools/lint/test/files/yaml/good.yml new file mode 100644 index 0000000000..b30941b797 --- /dev/null +++ b/tools/lint/test/files/yaml/good.yml @@ -0,0 +1,6 @@ +--- +yamllint: + description: YAML linter + include: + - .cron.yml + - browser/config/ diff --git a/tools/lint/test/python.toml b/tools/lint/test/python.toml new file mode 100644 index 0000000000..65036449a8 --- /dev/null +++ b/tools/lint/test/python.toml @@ -0,0 +1,60 @@ +[DEFAULT] +subsuite = "mozlint" + +["test_android_format.py"] + +["test_black.py"] +requirements = "tools/lint/python/black_requirements.txt" + +["test_clang_format.py"] + +["test_codespell.py"] + +["test_condprof_addons.py"] + +["test_eslint.py"] +skip-if = ["os == 'win'"] # busts the tree for subsequent tasks on the same worker (bug 1708591) +# Setup conflicts with stylelint setup so this should run sequentially. +sequential = true + +["test_file_license.py"] + +["test_file_perm.py"] +skip-if = ["os == 'win'"] + +["test_file_whitespace.py"] + +["test_fluent_lint.py"] + +["test_lintpref.py"] + +["test_manifest_alpha.py"] + +["test_manifest_toml.py"] + +["test_perfdocs.py"] + +["test_perfdocs_generation.py"] + +["test_perfdocs_helpers.py"] + +["test_rst.py"] +requirements = "tools/lint/rst/requirements.txt" + +["test_ruff.py"] +requirements = "tools/lint/python/ruff_requirements.txt" + +["test_rustfmt.py"] + +["test_shellcheck.py"] + +["test_stylelint.py"] +skip-if = ["os == 'win'"] # busts the tree for subsequent tasks on the same worker (bug 1708591) +# Setup conflicts with eslint setup so this should run sequentially. +sequential = true + +["test_trojan_source.py"] + +["test_updatebot.py"] + +["test_yaml.py"] diff --git a/tools/lint/test/test_android_format.py b/tools/lint/test/test_android_format.py new file mode 100644 index 0000000000..70cd1ea02e --- /dev/null +++ b/tools/lint/test/test_android_format.py @@ -0,0 +1,38 @@ +import mozunit +from conftest import build + +LINTER = "android-format" + + +def test_basic(global_lint, config): + substs = { + "GRADLE_ANDROID_FORMAT_LINT_CHECK_TASKS": [ + "spotlessJavaCheck", + "spotlessKotlinCheck", + ], + "GRADLE_ANDROID_FORMAT_LINT_FIX_TASKS": [ + "spotlessJavaApply", + "spotlessKotlinApply", + ], + "GRADLE_ANDROID_FORMAT_LINT_FOLDERS": ["tools/lint/test/files/android-format"], + } + results = global_lint( + config=config, + topobjdir=build.topobjdir, + root=build.topsrcdir, + substs=substs, + extra_args=["-PandroidFormatLintTest"], + ) + print(results) + + # When first task (spotlessJavaCheck) hits error, we won't check next Kotlin error. + # So results length will be 1. + assert len(results) == 1 + assert results[0].level == "error" + + # Since android-format is global lint, fix=True overrides repository files directly. + # No way to add this test. + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_black.py b/tools/lint/test/test_black.py new file mode 100644 index 0000000000..df0e792e68 --- /dev/null +++ b/tools/lint/test/test_black.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit + +LINTER = "black" +fixed = 0 + + +def test_lint_fix(lint, create_temp_file): + contents = """def is_unique( + s + ): + s = list(s + ) + s.sort() + + + for i in range(len(s) - 1): + if s[i] == s[i + 1]: + return 0 + else: + return 1 + + +if __name__ == "__main__": + print( + is_unique(input()) + ) """ + + path = create_temp_file(contents, "bad.py") + lint([path], fix=True) + assert fixed == 1 + + +def test_lint_black(lint, paths): + results = lint(paths()) + assert len(results) == 2 + + assert results[0].level == "error" + assert results[0].relpath == "bad.py" + + assert "EOF" in results[1].message + assert results[1].level == "error" + assert results[1].relpath == "invalid.py" + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_clang_format.py b/tools/lint/test/test_clang_format.py new file mode 100644 index 0000000000..d32e000131 --- /dev/null +++ b/tools/lint/test/test_clang_format.py @@ -0,0 +1,138 @@ +import mozunit +from conftest import build + +LINTER = "clang-format" +fixed = 0 + + +def test_good(lint, config, paths): + results = lint(paths("good/"), root=build.topsrcdir, use_filters=False) + print(results) + assert len(results) == 0 + + results = lint(paths("good/"), root=build.topsrcdir, use_filters=False, fix=True) + assert fixed == len(results) + + +def test_basic(lint, config, paths): + results = lint(paths("bad/bad.cpp"), root=build.topsrcdir, use_filters=False) + print(results) + assert len(results) == 1 + + assert "Reformat C/C++" in results[0].message + assert results[0].level == "warning" + assert results[0].lineno == 1 + assert results[0].column == 0 + assert "bad.cpp" in results[0].path + assert ( + results[0].diff + == """\ +-int main ( ) { +- +-return 0; +- +- +-} ++int main() { return 0; } +""" # noqa + ) + + +def test_dir(lint, config, paths): + results = lint(paths("bad/"), root=build.topsrcdir, use_filters=False) + print(results) + assert len(results) == 5 + + assert "Reformat C/C++" in results[0].message + assert results[0].level == "warning" + assert results[0].lineno == 1 + assert results[0].column == 0 + assert "bad.cpp" in results[0].path + assert ( + results[0].diff + == """\ +-int main ( ) { +- +-return 0; +- +- +-} ++int main() { return 0; } +""" # noqa + ) + + assert "Reformat C/C++" in results[1].message + assert results[1].level == "warning" + assert results[1].lineno == 1 + assert results[1].column == 0 + assert "bad2.c" in results[1].path + assert ( + results[1].diff + == """\ +-#include "bad2.h" +- +- +-int bad2() { ++#include "bad2.h" ++ ++int bad2() { +""" + ) + + assert "Reformat C/C++" in results[2].message + assert results[2].level == "warning" + assert results[2].lineno == 5 + assert results[2].column == 0 + assert "bad2.c" in results[2].path + assert ( + results[2].diff + == """\ +- int a =2; ++ int a = 2; +""" + ) + + assert "Reformat C/C++" in results[3].message + assert results[3].level == "warning" + assert results[3].lineno == 6 + assert results[3].column == 0 + assert "bad2.c" in results[3].path + assert ( + results[3].diff + == """\ +- return a; +- +-} ++ return a; ++} +""" + ) + + assert "Reformat C/C++" in results[4].message + assert results[4].level == "warning" + assert results[4].lineno == 1 + assert results[4].column == 0 + assert "bad2.h" in results[4].path + assert ( + results[4].diff + == """\ +-int bad2(void ); ++int bad2(void); +""" + ) + + +def test_fixed(lint, create_temp_file): + contents = """int main ( ) { \n +return 0; \n + +}""" + + path = create_temp_file(contents, "ignore.cpp") + lint([path], use_filters=False, fix=True) + + assert fixed == 1 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_codespell.py b/tools/lint/test/test_codespell.py new file mode 100644 index 0000000000..8baae66b41 --- /dev/null +++ b/tools/lint/test/test_codespell.py @@ -0,0 +1,37 @@ +import mozunit + +LINTER = "codespell" +fixed = 0 + + +def test_lint_codespell_fix(lint, create_temp_file): + contents = """This is a file with some typos and informations. +But also testing false positive like optin (because this isn't always option) +or stuff related to our coding style like: +aparent (aParent). +but detects mistakes like mozila +""".lstrip() + + path = create_temp_file(contents, "ignore.rst") + lint([path], fix=True) + + assert fixed == 2 + + +def test_lint_codespell(lint, paths): + results = lint(paths()) + assert len(results) == 2 + + assert results[0].message == "informations ==> information" + assert results[0].level == "error" + assert results[0].lineno == 1 + assert results[0].relpath == "ignore.rst" + + assert results[1].message == "mozila ==> mozilla" + assert results[1].level == "error" + assert results[1].lineno == 5 + assert results[1].relpath == "ignore.rst" + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_condprof_addons.py b/tools/lint/test/test_condprof_addons.py new file mode 100644 index 0000000000..e1401a7119 --- /dev/null +++ b/tools/lint/test/test_condprof_addons.py @@ -0,0 +1,285 @@ +import importlib +import tempfile +from pathlib import Path +from unittest import mock + +import mozunit +import requests + +LINTER = "condprof-addons" + + +def linter_module_mocks( + customizations_path=".", browsertime_fetches_path="browsertime.yml", **othermocks +): + return mock.patch.multiple( + LINTER, + CUSTOMIZATIONS_PATH=Path(customizations_path), + BROWSERTIME_FETCHES_PATH=Path(browsertime_fetches_path), + **othermocks, + ) + + +def linter_class_mocks(**mocks): + return mock.patch.multiple( + f"{LINTER}.CondprofAddonsLinter", + **mocks, + ) + + +# Sanity check (make sure linter message includes the xpi filename). +def test_get_missing_xpi_msg(lint, paths): + condprof_addons = importlib.import_module("condprof-addons") + with linter_class_mocks( + get_firefox_addons_tar_names=mock.Mock(return_value=list()), + ): + instance = condprof_addons.CondprofAddonsLinter( + topsrcdir=paths()[0], logger=mock.Mock() + ) + assert instance.get_missing_xpi_msg("test.xpi").startswith( + "test.xpi is missing" + ) + + +def test_xpi_missing_from_firefox_addons_tar(lint, paths): + fixture_customizations = paths("with-missing-xpi.json") + with linter_module_mocks(), linter_class_mocks( + get_firefox_addons_tar_names=mock.Mock(return_value=list()), + ): + logger_mock = mock.Mock() + lint(fixture_customizations, logger=logger_mock) + assert logger_mock.lint_error.call_count == 1 + assert Path(fixture_customizations[0]).samefile( + logger_mock.lint_error.call_args.kwargs["path"] + ) + importlib.import_module("condprof-addons") + assert "non-existing.xpi" in logger_mock.lint_error.call_args.args[0] + + +def test_xpi_all_found_in_firefox_addons_tar(lint, paths): + get_tarnames_mock = mock.Mock( + return_value=["an-extension.xpi", "another-extension.xpi"] + ) + read_json_mock = mock.Mock( + return_value={ + "addons": { + "an-extension": "http://localhost/ext/an-extension.xpi", + "another-extension": "http://localhost/ext/another-extension.xpi", + } + } + ) + + with linter_module_mocks(), linter_class_mocks( + get_firefox_addons_tar_names=get_tarnames_mock, read_json=read_json_mock + ): + logger_mock = mock.Mock() + # Compute a fake condprof customization path, the content is + # going to be the read_json_mock.return_value and so the + # fixture file does not actually exists. + fixture_customizations = paths("fake-condprof-config.json") + lint( + fixture_customizations, + logger=logger_mock, + config={"include": paths(), "extensions": ["json", "yml"]}, + ) + assert read_json_mock.call_count == 1 + assert get_tarnames_mock.call_count == 1 + assert logger_mock.lint_error.call_count == 0 + + +def test_lint_error_on_missing_or_invalid_firefoxaddons_fetch_task( + lint, + paths, +): + read_json_mock = mock.Mock(return_value=dict()) + read_yaml_mock = mock.Mock(return_value=dict()) + # Verify that an explicit linter error is reported if the fetch task is not found. + with linter_module_mocks(), linter_class_mocks( + read_json=read_json_mock, read_yaml=read_yaml_mock + ): + logger_mock = mock.Mock() + fixture_customizations = paths("fake-condprof-config.json") + condprof_addons = importlib.import_module("condprof-addons") + + def assert_linter_error(yaml_mock_value, expected_msg): + logger_mock.reset_mock() + read_yaml_mock.return_value = yaml_mock_value + lint(fixture_customizations, logger=logger_mock) + assert logger_mock.lint_error.call_count == 1 + expected_path = condprof_addons.BROWSERTIME_FETCHES_PATH + assert logger_mock.lint_error.call_args.kwargs["path"] == expected_path + assert logger_mock.lint_error.call_args.args[0] == expected_msg + + # Mock a yaml file that is not including the expected firefox-addons fetch task. + assert_linter_error( + yaml_mock_value=dict(), expected_msg=condprof_addons.ERR_FETCH_TASK_MISSING + ) + # Mock a yaml file where firefox-addons is missing the fetch attribute. + assert_linter_error( + yaml_mock_value={"firefox-addons": {}}, + expected_msg=condprof_addons.ERR_FETCH_TASK_MISSING, + ) + # Mock a yaml file where firefox-addons add-prefix is missing. + assert_linter_error( + yaml_mock_value={"firefox-addons": {"fetch": {}}}, + expected_msg=condprof_addons.ERR_FETCH_TASK_ADDPREFIX, + ) + # Mock a yaml file where firefox-addons add-prefix is invalid. + assert_linter_error( + yaml_mock_value={ + "firefox-addons": {"fetch": {"add-prefix": "invalid-subdir-name/"}} + }, + expected_msg=condprof_addons.ERR_FETCH_TASK_ADDPREFIX, + ) + + +def test_get_xpi_list_from_fetch_dir(lint, paths): + # Verify that when executed on the CI, the helper method looks for the xpi files + # in the MOZ_FETCHES_DIR subdir where they are expected to be unpacked by the + # fetch task. + with linter_module_mocks( + MOZ_AUTOMATION=1, MOZ_FETCHES_DIR=paths("fake-fetches-dir")[0] + ): + condprof_addons = importlib.import_module("condprof-addons") + logger_mock = mock.Mock() + Path(paths("browsertime.yml")[0]) + + linter = condprof_addons.CondprofAddonsLinter( + topsrcdir=paths()[0], logger=logger_mock + ) + results = linter.tar_xpi_filenames + + results.sort() + assert results == ["fake-ext-01.xpi", "fake-ext-02.xpi"] + + +def test_get_xpi_list_from_downloaded_tar(lint, paths): + def mocked_download_tar(firefox_addons_tar_url, tar_tmp_path): + tar_tmp_path.write_bytes(Path(paths("firefox-addons-fake.tar")[0]).read_bytes()) + + download_firefox_addons_tar_mock = mock.Mock() + download_firefox_addons_tar_mock.side_effect = mocked_download_tar + + # Verify that when executed locally on a developer machine, the tar archive is downloaded + # and the list of xpi files included in it returned by the helper method. + with tempfile.TemporaryDirectory() as tempdir, linter_module_mocks( + MOZ_AUTOMATION=0, + tempdir=tempdir, + ), linter_class_mocks( + download_firefox_addons_tar=download_firefox_addons_tar_mock, + ): + condprof_addons = importlib.import_module("condprof-addons") + logger_mock = mock.Mock() + Path(paths("browsertime.yml")[0]) + + linter = condprof_addons.CondprofAddonsLinter( + topsrcdir=paths()[0], logger=logger_mock + ) + results = linter.tar_xpi_filenames + assert len(results) > 0 + print("List of addons found in the downloaded file archive:", results) + assert all(filename.endswith(".xpi") for filename in results) + assert download_firefox_addons_tar_mock.call_count == 1 + + +@mock.patch("requests.get") +def test_error_on_downloading_tar(requests_get_mock, lint, paths): + # Verify that when executed locally and the tar archive fails to download + # the linter does report an explicit linting error with the http error included. + with tempfile.TemporaryDirectory() as tempdir, linter_module_mocks( + MOZ_AUTOMATION=0, tempdir=tempdir + ): + condprof_addons = importlib.import_module("condprof-addons") + logger_mock = mock.Mock() + response_mock = mock.Mock() + response_mock.raise_for_status.side_effect = requests.exceptions.HTTPError( + "MOCK_ERROR" + ) + requests_get_mock.return_value = response_mock + Path(paths("browsertime.yml")[0]) + + linter = condprof_addons.CondprofAddonsLinter( + topsrcdir=paths()[0], logger=logger_mock + ) + + assert ( + logger_mock.lint_error.call_args.kwargs["path"] + == condprof_addons.BROWSERTIME_FETCHES_PATH + ) + assert ( + logger_mock.lint_error.call_args.args[0] + == f"{condprof_addons.ERR_FETCH_TASK_ARCHIVE}, MOCK_ERROR" + ) + assert requests_get_mock.call_count == 1 + assert len(linter.tar_xpi_filenames) == 0 + + +@mock.patch("requests.get") +def test_error_on_opening_tar(requests_get_mock, lint, paths): + # Verify that when executed locally and the tar archive fails to open + # the linter does report an explicit linting error with the tarfile error included. + with tempfile.TemporaryDirectory() as tempdir, linter_module_mocks( + MOZ_AUTOMATION=0, tempdir=tempdir + ): + condprof_addons = importlib.import_module("condprof-addons") + logger_mock = mock.Mock() + response_mock = mock.Mock() + response_mock.raise_for_status.return_value = None + + def mock_iter_content(chunk_size): + yield b"fake tar content" + yield b"expected to trigger tarfile.ReadError" + + response_mock.iter_content.side_effect = mock_iter_content + requests_get_mock.return_value = response_mock + Path(paths("browsertime.yml")[0]) + + linter = condprof_addons.CondprofAddonsLinter( + topsrcdir=paths()[0], logger=logger_mock + ) + + assert ( + logger_mock.lint_error.call_args.kwargs["path"] + == condprof_addons.BROWSERTIME_FETCHES_PATH + ) + actual_msg = logger_mock.lint_error.call_args.args[0] + print("Got linter error message:", actual_msg) + assert actual_msg.startswith( + f"{condprof_addons.ERR_FETCH_TASK_ARCHIVE}, file could not be opened successfully" + ) + assert requests_get_mock.call_count == 1 + assert len(linter.tar_xpi_filenames) == 0 + + +def test_lint_all_customization_files_when_linting_browsertime_yml( + lint, + paths, +): + get_tarnames_mock = mock.Mock(return_value=["an-extension.xpi"]) + read_json_mock = mock.Mock( + return_value={ + "addons": {"an-extension": "http://localhost/ext/an-extension.xpi"} + } + ) + with linter_module_mocks( + customizations_path="fake-customizations-dir", + ), linter_class_mocks( + get_firefox_addons_tar_names=get_tarnames_mock, + read_json=read_json_mock, + ): + logger_mock = mock.Mock() + importlib.import_module("condprof-addons") + # When mozlint detects a change to the ci fetch browser.yml support file, + # condprof-addons linter is called for the entire customizations dir path + # and we expect that to be expanded to the list of the json customizations + # files from that directory path. + lint(paths("fake-customizations-dir"), logger=logger_mock) + # Expect read_json_mock to be called once per each of the json files + # found in the fixture dir. + assert read_json_mock.call_count == 3 + assert get_tarnames_mock.call_count == 1 + assert logger_mock.lint_error.call_count == 0 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_eslint.py b/tools/lint/test/test_eslint.py new file mode 100644 index 0000000000..b4fda2fb35 --- /dev/null +++ b/tools/lint/test/test_eslint.py @@ -0,0 +1,99 @@ +import mozunit +import pytest +from conftest import build + +LINTER = "eslint" +fixed = 0 + + +@pytest.fixture +def eslint(lint): + def inner(*args, **kwargs): + # --no-ignore is for ESLint to avoid the .eslintignore file. + # --ignore-path is because Prettier doesn't have the --no-ignore option + # and therefore needs to be given an empty file for the tests to work. + kwargs["extra_args"] = [ + "--no-ignore", + "--ignore-path=tools/lint/test/files/eslint/testprettierignore", + ] + return lint(*args, **kwargs) + + return inner + + +def test_lint_with_global_exclude(lint, config, paths): + config["exclude"] = ["subdir", "import"] + # This uses lint directly as we need to not ignore the excludes. + results = lint(paths(), config=config, root=build.topsrcdir) + assert len(results) == 0 + + +def test_no_files_to_lint(eslint, config, paths): + # A directory with no files to lint. + results = eslint(paths("nolint"), root=build.topsrcdir) + assert results == [] + + # Errors still show up even when a directory with no files is passed in. + results = eslint(paths("nolint", "subdir/bad.js"), root=build.topsrcdir) + assert len(results) == 1 + + +def test_bad_import(eslint, config, paths): + results = eslint(paths("import"), config=config, root=build.topsrcdir) + assert results == 1 + + +def test_eslint_rule(eslint, config, create_temp_file): + contents = """var re = /foo bar/; +var re = new RegExp("foo bar"); +""" + path = create_temp_file(contents, "bad.js") + results = eslint( + [path], config=config, root=build.topsrcdir, rules=["no-regex-spaces: error"] + ) + + assert len(results) == 2 + + +def test_eslint_fix(eslint, config, create_temp_file): + contents = """/*eslint no-regex-spaces: "error"*/ + +var re = /foo bar/; +var re = new RegExp("foo bar"); + +var re = /foo bar/; +var re = new RegExp("foo bar"); + +var re = /foo bar/; +var re = new RegExp("foo bar"); +""" + path = create_temp_file(contents, "bad.js") + eslint([path], config=config, root=build.topsrcdir, fix=True) + + # ESLint returns counts of files fixed, not errors fixed. + assert fixed == 1 + + +def test_prettier_rule(eslint, config, create_temp_file): + contents = """var re = /foobar/; + var re = "foo"; +""" + path = create_temp_file(contents, "bad.js") + results = eslint([path], config=config, root=build.topsrcdir) + + assert len(results) == 1 + + +def test_prettier_fix(eslint, config, create_temp_file): + contents = """var re = /foobar/; + var re = "foo"; +""" + path = create_temp_file(contents, "bad.js") + eslint([path], config=config, root=build.topsrcdir, fix=True) + + # Prettier returns counts of files fixed, not errors fixed. + assert fixed == 1 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_file_license.py b/tools/lint/test/test_file_license.py new file mode 100644 index 0000000000..250d4a0778 --- /dev/null +++ b/tools/lint/test/test_file_license.py @@ -0,0 +1,33 @@ +import mozunit + +LINTER = "license" +fixed = 0 + + +def test_lint_license(lint, paths): + results = lint(paths()) + print(results) + assert len(results) == 3 + + assert ".eslintrc.js" in results[0].relpath + + assert "No matching license strings" in results[1].message + assert results[1].level == "error" + assert "bad.c" in results[1].relpath + + assert "No matching license strings" in results[2].message + assert results[2].level == "error" + assert "bad.js" in results[2].relpath + + +def test_lint_license_fix(lint, paths, create_temp_file): + contents = """let foo = 0;""" + path = create_temp_file(contents, "lint_license_test_tmp_file.js") + results = lint([path], fix=True) + + assert len(results) == 0 + assert fixed == 1 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_file_perm.py b/tools/lint/test/test_file_perm.py new file mode 100644 index 0000000000..08d6a20eef --- /dev/null +++ b/tools/lint/test/test_file_perm.py @@ -0,0 +1,35 @@ +import mozunit +import pytest + +LINTER = "file-perm" + + +@pytest.mark.lint_config(name="file-perm") +def test_lint_file_perm(lint, paths): + results = lint(paths("no-shebang"), collapse_results=True) + + assert results.keys() == { + "no-shebang/bad.c", + "no-shebang/bad-shebang.c", + "no-shebang/bad.png", + } + + for path, issues in results.items(): + for issue in issues: + assert "permissions on a source" in issue.message + assert issue.level == "error" + + +@pytest.mark.lint_config(name="maybe-shebang-file-perm") +def test_lint_shebang_file_perm(config, lint, paths): + results = lint(paths("maybe-shebang")) + + assert len(results) == 1 + + assert "permissions on a source" in results[0].message + assert results[0].level == "error" + assert results[0].relpath == "maybe-shebang/bad.js" + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_file_whitespace.py b/tools/lint/test/test_file_whitespace.py new file mode 100644 index 0000000000..7dc815d6ca --- /dev/null +++ b/tools/lint/test/test_file_whitespace.py @@ -0,0 +1,50 @@ +import mozunit + +LINTER = "file-whitespace" +fixed = 0 + + +def test_lint_file_whitespace(lint, paths): + results = lint(paths()) + print(results) + assert len(results) == 5 + + assert "File does not end with newline character" in results[1].message + assert results[1].level == "error" + assert "bad-newline.c" in results[1].relpath + + assert "Empty Lines at end of file" in results[0].message + assert results[0].level == "error" + assert "bad-newline.c" in results[0].relpath + + assert "Windows line return" in results[2].message + assert results[2].level == "error" + assert "bad-windows.c" in results[2].relpath + + assert "Trailing whitespace" in results[3].message + assert results[3].level == "error" + assert "bad.c" in results[3].relpath + assert results[3].lineno == 1 + + assert "Trailing whitespace" in results[4].message + assert results[4].level == "error" + assert "bad.c" in results[4].relpath + assert results[4].lineno == 2 + + +def test_lint_file_whitespace_fix(lint, paths, create_temp_file): + contents = """int main() { \n + return 0; \n +} + + +""" + + path = create_temp_file(contents, "bad.cpp") + lint([path], fix=True) + # Gives a different answer on Windows. Probably because of Windows CR + assert fixed == 3 or fixed == 2 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_fluent_lint.py b/tools/lint/test/test_fluent_lint.py new file mode 100644 index 0000000000..0bbd4305d5 --- /dev/null +++ b/tools/lint/test/test_fluent_lint.py @@ -0,0 +1,164 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import mozunit + +LINTER = "fluent-lint" + + +def test_lint_exclusions(lint, paths): + results = lint(paths("excluded.ftl")) + assert len(results) == 1 + assert results[0].rule == "TE01" + assert results[0].lineno == 6 + assert results[0].column == 20 + + +def test_lint_single_file(lint, paths): + results = lint(paths("bad.ftl")) + assert len(results) == 13 + assert results[0].rule == "ID01" + assert results[0].lineno == 1 + assert results[0].column == 1 + assert results[1].rule == "ID01" + assert results[1].lineno == 3 + assert results[1].column == 1 + assert results[2].rule == "TE01" + assert results[2].lineno == 5 + assert results[2].column == 20 + assert results[3].rule == "TE01" + assert results[3].lineno == 6 + assert results[3].column == 24 + assert results[4].rule == "TE02" + assert results[4].lineno == 7 + assert results[4].column == 20 + assert results[5].rule == "TE03" + assert results[5].lineno == 8 + assert results[5].column == 20 + assert results[6].rule == "TE04" + assert results[6].lineno == 11 + assert results[6].column == 20 + assert results[7].rule == "TE05" + assert results[7].lineno == 13 + assert results[7].column == 16 + assert results[8].rule == "TE03" + assert results[8].lineno == 17 + assert results[8].column == 20 + assert results[9].rule == "TE03" + assert results[9].lineno == 25 + assert results[9].column == 18 + assert results[10].rule == "ID02" + assert results[10].lineno == 32 + assert results[10].column == 1 + assert results[11].rule == "VC01" + assert "$tabCount" in results[11].message + assert results[12].rule == "VC01" + assert "$url" in results[12].message + + +def test_comment_group(lint, paths): + results = lint(paths("comment-group1.ftl")) + assert len(results) == 6 + assert results[0].rule == "GC03" + assert results[0].lineno == 12 + assert results[0].column == 1 + assert results[1].rule == "GC02" + assert results[1].lineno == 16 + assert results[1].column == 1 + assert results[2].rule == "GC04" + assert results[2].lineno == 21 + assert results[2].column == 1 + assert results[3].rule == "GC03" + assert results[3].lineno == 26 + assert results[3].column == 1 + assert results[4].rule == "GC02" + assert results[4].lineno == 30 + assert results[4].column == 1 + assert results[5].rule == "GC01" + assert results[5].lineno == 35 + assert results[5].column == 1 + + results = lint(paths("comment-group2.ftl")) + assert (len(results)) == 0 + + +def test_comment_resource(lint, paths): + results = lint(paths("comment-resource1.ftl")) + assert len(results) == 1 + assert results[0].rule == "RC01" + assert results[0].lineno == 9 + assert results[0].column == 1 + + results = lint(paths("comment-resource2.ftl")) + assert len(results) == 1 + assert results[0].rule == "RC03" + assert results[0].lineno == 4 + assert results[0].column == 1 + + results = lint(paths("comment-resource3.ftl")) + assert len(results) == 1 + assert results[0].rule == "RC02" + assert results[0].lineno == 5 + assert results[0].column == 1 + + results = lint(paths("comment-resource4.ftl")) + assert len(results) == 1 + assert results[0].rule == "RC03" + assert results[0].lineno == 6 + assert results[0].column == 1 + + results = lint(paths("comment-resource5.ftl")) + assert len(results) == 1 + assert results[0].rule == "RC02" + assert results[0].lineno == 5 + assert results[0].column == 1 + + results = lint(paths("comment-resource6.ftl")) + assert len(results) == 0 + + +def test_brand_names(lint, paths): + results = lint(paths("brand-names.ftl"), {"brand-files": ["test-brands.ftl"]}) + assert len(results) == 11 + assert results[0].rule == "CO01" + assert results[0].lineno == 1 + assert results[0].column == 16 + assert "Firefox" in results[0].message + assert "Mozilla" not in results[0].message + assert "Thunderbird" not in results[0].message + assert results[1].rule == "CO01" + assert results[1].lineno == 4 + assert results[1].column == 16 + + results = lint(paths("brand-names-excluded.ftl")) + assert len(results) == 0 + + +def test_comment_variables(lint, paths): + results = lint(paths("comment-variables1.ftl")) + assert len(results) == 4 + assert results[0].rule == "VC01" + assert "$var" in results[0].message + assert results[1].rule == "VC01" + assert "$select1" in results[1].message + assert results[2].rule == "VC01" + assert "$select2" in results[2].message + assert results[3].rule == "VC01" + assert "$attr" in results[3].message + + results = lint(paths("comment-variables2.ftl")) + assert len(results) == 1 + assert results[0].rule == "VC01" + assert "$term-message" in results[0].message + + +def test_valid_attributes(lint, paths): + results = lint(paths("valid-attributes.ftl")) + print(results) + assert len(results) == 1 + assert results[0].rule == "VA01" + assert ".extralabel" in results[0].message + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_lintpref.py b/tools/lint/test/test_lintpref.py new file mode 100644 index 0000000000..3e75b1675e --- /dev/null +++ b/tools/lint/test/test_lintpref.py @@ -0,0 +1,16 @@ +import mozunit + +LINTER = "lintpref" + + +def test_lintpref(lint, paths): + results = lint(paths()) + assert len(results) == 1 + assert results[0].level == "error" + assert 'pref("dom.webidl.test1", true);' in results[0].message + assert "bad.js" in results[0].relpath + assert results[0].lineno == 2 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_manifest_alpha.py b/tools/lint/test/test_manifest_alpha.py new file mode 100644 index 0000000000..2e8e1a6c77 --- /dev/null +++ b/tools/lint/test/test_manifest_alpha.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import mozunit + +LINTER = "test-manifest-alpha" + + +def test_very_out_of_order(lint, paths): + results = lint(paths("mochitest-very-out-of-order.ini")) + assert len(results) == 1 + assert results[0].diff + + +def test_in_order(lint, paths): + results = lint(paths("mochitest-in-order.ini")) + assert len(results) == 0 + + +def test_mostly_in_order(lint, paths): + results = lint(paths("mochitest-mostly-in-order.ini")) + assert len(results) == 1 + assert results[0].diff + + +def test_other_ini_very_out_of_order(lint, paths): + """Test that an .ini file outside of the allowlist is ignored.""" + results = lint(paths("other-ini-very-out-of-order.ini")) + assert len(results) == 0 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_manifest_toml.py b/tools/lint/test/test_manifest_toml.py new file mode 100644 index 0000000000..f205a1cd1f --- /dev/null +++ b/tools/lint/test/test_manifest_toml.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import mozunit + +LINTER = "test-manifest-toml" +fixed = 0 + + +def test_valid(lint, paths): + results = lint(paths("valid.toml")) + assert len(results) == 0 + + +def test_invalid(lint, paths): + results = lint(paths("invalid.toml")) + assert len(results) == 1 + assert results[0].message == "The manifest is not valid TOML." + + +def test_no_default(lint, paths): + """Test verifying [DEFAULT] section.""" + results = lint(paths("no-default.toml")) + assert len(results) == 1 + assert results[0].message == "The manifest does not start with a [DEFAULT] section." + + +def test_no_default_fix(lint, paths, create_temp_file): + """Test fixing missing [DEFAULT] section.""" + contents = "# this Manifest has no DEFAULT section\n" + path = create_temp_file(contents, "no-default.toml") + results = lint([path], fix=True) + assert len(results) == 1 + assert results[0].message == "The manifest does not start with a [DEFAULT] section." + assert fixed == 1 + + +def test_non_double_quote_sections(lint, paths): + """Test verifying [DEFAULT] section.""" + results = lint(paths("non-double-quote-sections.toml")) + assert len(results) == 2 + assert results[0].message.startswith("The section name must be double quoted:") + + +def test_unsorted(lint, paths): + """Test sections in alpha order.""" + results = lint(paths("unsorted.toml")) + assert len(results) == 1 + assert results[0].message == "The manifest sections are not in alphabetical order." + + +def test_comment_section(lint, paths): + """Test for commented sections.""" + results = lint(paths("comment-section.toml")) + assert len(results) == 2 + assert results[0].message.startswith( + "Use 'disabled = \"<reason>\"' to disable a test instead of a comment:" + ) + + +def test_skip_if_not_array(lint, paths): + """Test for non-array skip-if value.""" + results = lint(paths("skip-if-not-array.toml")) + assert len(results) == 1 + assert results[0].message.startswith("Value for conditional must be an array:") + + +def test_skip_if_explicit_or(lint, paths): + """Test for explicit || in skip-if.""" + results = lint(paths("skip-if-explicit-or.toml")) + assert len(results) == 1 + assert results[0].message.startswith( + "Value for conditional must not include explicit ||, instead put on multiple lines:" + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_perfdocs.py b/tools/lint/test/test_perfdocs.py new file mode 100644 index 0000000000..4ee834ad68 --- /dev/null +++ b/tools/lint/test/test_perfdocs.py @@ -0,0 +1,858 @@ +import contextlib +import os +import pathlib +import shutil +import tempfile +from unittest import mock + +import mozunit +import pytest + +LINTER = "perfdocs" + + +class PerfDocsLoggerMock: + LOGGER = None + PATHS = [] + FAILED = True + + +""" +This is a sample mozperftest test that we use for testing +the verification process. +""" +SAMPLE_TEST = """ +"use strict"; + +async function setUp(context) { + context.log.info("setUp example!"); +} + +async function test(context, commands) { + context.log.info("Test with setUp/tearDown example!"); + await commands.measure.start("https://www.sitespeed.io/"); + await commands.measure.start("https://www.mozilla.org/en-US/"); +} + +async function tearDown(context) { + context.log.info("tearDown example!"); +} + +module.noexport = {}; + +module.exports = { + setUp, + tearDown, + test, + owner: "Performance Testing Team", + name: "Example", + description: "The description of the example test.", + longDescription: ` + This is a longer description of the test perhaps including information + about how it should be run locally or links to relevant information. + ` +}; +""" + + +SAMPLE_CONFIG = """ +name: mozperftest +manifest: None +static-only: False +suites: + suite: + description: "Performance tests from the 'suite' folder." + tests: + Example: "" +""" + + +DYNAMIC_SAMPLE_CONFIG = """ +name: {} +manifest: None +static-only: False +suites: + suite: + description: "Performance tests from the 'suite' folder." + tests: + Example: "Performance test Example from suite." + another_suite: + description: "Performance tests from the 'another_suite' folder." + tests: + Example: "Performance test Example from another_suite." +""" + + +SAMPLE_METRICS_CONFIG = """ +name: raptor +manifest: "None"{} +static-only: False +suites: + suite: + description: "Performance tests from the 'suite' folder."{} + tests: + Example: "Performance test Example from another_suite." + another_suite: + description: "Performance tests from the 'another_suite' folder." + tests: + Example: "Performance test Example from another_suite." +""" + + +SAMPLE_INI = """ +[Example] +test_url = Example_url +alert_on = fcp +""" + +SAMPLE_METRICS_INI = """ +[Example] +test_url = Example_url +alert_on = fcp,SpeedIndex +""" + + +@contextlib.contextmanager +def temp_file(name="temp", tempdir=None, content=None): + if tempdir is None: + tempdir = tempfile.mkdtemp() + path = pathlib.Path(tempdir, name) + if content is not None: + with path.open("w", newline="\n") as f: + f.write(content) + try: + yield path + finally: + try: + shutil.rmtree(str(tempdir)) + except FileNotFoundError: + pass + + +@contextlib.contextmanager +def temp_dir(): + tempdir = pathlib.Path(tempfile.mkdtemp()) + try: + yield tempdir + finally: + try: + shutil.rmtree(str(tempdir)) + except FileNotFoundError: + pass + + +def setup_sample_logger(logger, structured_logger, top_dir): + from perfdocs.logger import PerfDocLogger + + PerfDocLogger.LOGGER = structured_logger + PerfDocLogger.PATHS = ["perfdocs"] + PerfDocLogger.TOP_DIR = top_dir + + import perfdocs.gatherer as gt + import perfdocs.generator as gn + import perfdocs.verifier as vf + + gt.logger = logger + vf.logger = logger + gn.logger = logger + + +@mock.patch("taskgraph.util.taskcluster.get_artifact") +@mock.patch("tryselect.tasks.generate_tasks") +@mock.patch("perfdocs.generator.Generator") +@mock.patch("perfdocs.verifier.Verifier") +@mock.patch("perfdocs.logger.PerfDocLogger", new=PerfDocsLoggerMock) +def test_perfdocs_start_and_fail( + verifier, + generator, + get_artifact_mock, + gen_tasks_mock, + structured_logger, + config, + paths, +): + from perfdocs.perfdocs import run_perfdocs + + with temp_file("bad", content="foo") as temp: + run_perfdocs( + config, logger=structured_logger, paths=[str(temp)], generate=False + ) + assert PerfDocsLoggerMock.LOGGER == structured_logger + assert PerfDocsLoggerMock.PATHS == [temp] + assert PerfDocsLoggerMock.FAILED + + assert verifier.call_count == 1 + assert mock.call().validate_tree() in verifier.mock_calls + assert generator.call_count == 0 + + +@mock.patch("taskgraph.util.taskcluster.get_artifact") +@mock.patch("tryselect.tasks.generate_tasks") +@mock.patch("perfdocs.generator.Generator") +@mock.patch("perfdocs.verifier.Verifier") +@mock.patch("perfdocs.logger.PerfDocLogger", new=PerfDocsLoggerMock) +def test_perfdocs_start_and_pass(verifier, generator, structured_logger, config, paths): + from perfdocs.perfdocs import run_perfdocs + + PerfDocsLoggerMock.FAILED = False + with temp_file("bad", content="foo") as temp: + run_perfdocs( + config, logger=structured_logger, paths=[str(temp)], generate=False + ) + assert PerfDocsLoggerMock.LOGGER == structured_logger + assert PerfDocsLoggerMock.PATHS == [temp] + assert not PerfDocsLoggerMock.FAILED + + assert verifier.call_count == 1 + assert mock.call().validate_tree() in verifier.mock_calls + assert generator.call_count == 1 + assert mock.call().generate_perfdocs() in generator.mock_calls + + +@mock.patch("perfdocs.logger.PerfDocLogger", new=PerfDocsLoggerMock) +def test_perfdocs_bad_paths(structured_logger, config, paths): + from perfdocs.perfdocs import run_perfdocs + + with pytest.raises(Exception): + run_perfdocs(config, logger=structured_logger, paths=["bad"], generate=False) + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_gatherer_fetch_perfdocs_tree( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.gatherer import Gatherer + + gatherer = Gatherer(top_dir) + assert not gatherer._perfdocs_tree + + gatherer.fetch_perfdocs_tree() + + expected = "Found 1 perfdocs directories" + args, _ = logger.log.call_args + + assert expected in args[0] + assert logger.log.call_count == 1 + assert gatherer._perfdocs_tree + + expected = ["path", "yml", "rst", "static"] + for i, key in enumerate(gatherer._perfdocs_tree[0].keys()): + assert key == expected[i] + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_gatherer_get_test_list(logger, structured_logger, perfdocs_sample): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.gatherer import Gatherer + + gatherer = Gatherer(top_dir) + gatherer.fetch_perfdocs_tree() + framework = gatherer.get_test_list(gatherer._perfdocs_tree[0]) + + expected = ["name", "test_list", "yml_content", "yml_path"] + for i, key in enumerate(sorted(framework.keys())): + assert key == expected[i] + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verification(logger, structured_logger, perfdocs_sample): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier.validate_tree() + + # Make sure that we had no warnings + assert logger.warning.call_count == 0 + assert logger.log.call_count == 1 + assert len(logger.mock_calls) == 1 + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_validate_yaml_pass( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + yaml_path = perfdocs_sample["config"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.verifier import Verifier + + valid = Verifier(top_dir).validate_yaml(pathlib.Path(yaml_path)) + + assert valid + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_invalid_yaml(logger, structured_logger, perfdocs_sample): + top_dir = perfdocs_sample["top_dir"] + yaml_path = perfdocs_sample["config"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.verifier import Verifier + + verifier = Verifier("top_dir") + with open(yaml_path, "r", newline="\n") as f: + lines = f.readlines() + print(lines) + with open(yaml_path, "w", newline="\n") as f: + f.write("\n".join(lines[2:])) + valid = verifier.validate_yaml(yaml_path) + + expected = ("YAML ValidationError: 'name' is a required property\n", yaml_path) + args, _ = logger.warning.call_args + + assert logger.warning.call_count == 1 + assert expected[0] in args[0] + assert not valid + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_validate_rst_pass( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + rst_path = perfdocs_sample["index"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.verifier import Verifier + + valid = Verifier(top_dir).validate_rst_content(pathlib.Path(rst_path)) + + assert valid + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_invalid_rst(logger, structured_logger, perfdocs_sample): + top_dir = perfdocs_sample["top_dir"] + rst_path = perfdocs_sample["index"] + setup_sample_logger(logger, structured_logger, top_dir) + + # Replace the target string to invalid Keyword for test + with open(rst_path, "r") as file: + filedata = file.read() + + filedata = filedata.replace("documentation", "Invalid Keyword") + + with open(rst_path, "w", newline="\n") as file: + file.write(filedata) + + from perfdocs.verifier import Verifier + + verifier = Verifier("top_dir") + valid = verifier.validate_rst_content(rst_path) + + expected = ( + "Cannot find a '{documentation}' entry in the given index file", + rst_path, + ) + args, _ = logger.warning.call_args + + assert logger.warning.call_count == 1 + assert args == expected + assert not valid + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_validate_descriptions_pass( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier._check_framework_descriptions(verifier._gatherer.perfdocs_tree[0]) + + assert logger.warning.call_count == 0 + assert logger.log.call_count == 1 + assert len(logger.mock_calls) == 1 + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_not_existing_suite_in_test_list( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + manifest_path = perfdocs_sample["manifest"]["path"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + os.remove(manifest_path) + verifier._check_framework_descriptions(verifier._gatherer.perfdocs_tree[0]) + + expected = ( + "Could not find an existing suite for suite - bad suite name?", + perfdocs_sample["config"], + ) + args, _ = logger.warning.call_args + + assert logger.warning.call_count == 1 + assert args == expected + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_not_existing_tests_in_suites( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + with open(perfdocs_sample["config"], "r") as file: + filedata = file.read() + filedata = filedata.replace("Example", "DifferentName") + with open(perfdocs_sample["config"], "w", newline="\n") as file: + file.write(filedata) + + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier._check_framework_descriptions(verifier._gatherer.perfdocs_tree[0]) + + expected = [ + "Could not find an existing test for DifferentName - bad test name?", + "Could not find a test description for Example", + ] + + assert logger.warning.call_count == 2 + for i, call in enumerate(logger.warning.call_args_list): + args, _ = call + assert args[0] == expected[i] + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_missing_contents_in_suite( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + with open(perfdocs_sample["config"], "r") as file: + filedata = file.read() + filedata = filedata.replace("suite:", "InvalidSuite:") + with open(perfdocs_sample["config"], "w", newline="\n") as file: + file.write(filedata) + + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier._check_framework_descriptions(verifier._gatherer.perfdocs_tree[0]) + + expected = ( + "Could not find an existing suite for InvalidSuite - bad suite name?", + "Missing suite description for suite", + ) + + assert logger.warning.call_count == 2 + for i, call in enumerate(logger.warning.call_args_list): + args, _ = call + assert args[0] == expected[i] + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_invalid_dir(logger, structured_logger, perfdocs_sample): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.verifier import Verifier + + verifier = Verifier("invalid_path") + with pytest.raises(Exception) as exceinfo: + verifier.validate_tree() + + assert str(exceinfo.value) == "No valid perfdocs directories found" + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_file_invalidation( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.verifier import Verifier + + with mock.patch("perfdocs.verifier.Verifier.validate_yaml", return_value=False): + verifier = Verifier(top_dir) + with pytest.raises(Exception): + verifier.validate_tree() + + # Check if "File validation error" log is called + # and Called with a log inside perfdocs_tree(). + assert logger.log.call_count == 2 + assert len(logger.mock_calls) == 2 + + +@pytest.mark.parametrize( + "manifest, metric_definitions, expected", + [ + [ + SAMPLE_INI, + """ +metrics: + "FirstPaint": + aliases: + - fcp + description: "Example" """, + 1, + ], + [ + SAMPLE_METRICS_INI, + """ +metrics: + FirstPaint: + aliases: + - fcp + description: Example + SpeedIndex: + aliases: + - speedindex + - si + description: Example + """, + 2, + ], + ], +) +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_nonexistent_documented_metrics( + logger, structured_logger, perfdocs_sample, manifest, metric_definitions, expected +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + with open(perfdocs_sample["config"], "w", newline="\n") as f: + f.write(SAMPLE_METRICS_CONFIG.format(metric_definitions, "")) + with open(perfdocs_sample["manifest"]["path"], "w", newline="\n") as f: + f.write(manifest) + + sample_gatherer_result = { + "suite": {"Example": {}}, + "another_suite": {"Example": {}}, + } + + from perfdocs.verifier import Verifier + + with mock.patch("perfdocs.framework_gatherers.RaptorGatherer.get_test_list") as m: + m.return_value = sample_gatherer_result + verifier = Verifier(top_dir) + verifier.validate_tree() + + assert len(logger.warning.call_args_list) == expected + for args, _ in logger.warning.call_args_list: + assert "Cannot find documented metric" in args[0] + assert "being used" in args[0] + + +@pytest.mark.parametrize( + "manifest, metric_definitions", + [ + [ + SAMPLE_INI, + """ +metrics: + "FirstPaint": + aliases: + - fcp + description: "Example" """, + ], + [ + SAMPLE_METRICS_INI, + """ +metrics: + SpeedIndex: + aliases: + - speedindex + - si + description: Example + """, + ], + ], +) +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_undocumented_metrics( + logger, structured_logger, perfdocs_sample, manifest, metric_definitions +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + with open(perfdocs_sample["config"], "w", newline="\n") as f: + f.write(SAMPLE_METRICS_CONFIG.format(metric_definitions, "")) + with open(perfdocs_sample["manifest"]["path"], "w", newline="\n") as f: + f.write(manifest) + + sample_gatherer_result = { + "suite": {"Example": {"metrics": ["fcp", "SpeedIndex"]}}, + "another_suite": {"Example": {}}, + } + + from perfdocs.verifier import Verifier + + with mock.patch("perfdocs.framework_gatherers.RaptorGatherer.get_test_list") as m: + m.return_value = sample_gatherer_result + verifier = Verifier(top_dir) + verifier.validate_tree() + + assert len(logger.warning.call_args_list) == 1 + for args, _ in logger.warning.call_args_list: + assert "Missing description for the metric" in args[0] + + +@pytest.mark.parametrize( + "manifest, metric_definitions, expected", + [ + [ + SAMPLE_INI, + """ +metrics: + "FirstPaint": + aliases: + - fcp + - SpeedIndex + description: "Example" """, + 3, + ], + [ + SAMPLE_METRICS_INI, + """ +metrics: + FirstPaint: + aliases: + - fcp + description: Example + SpeedIndex: + aliases: + - speedindex + - si + description: Example + """, + 5, + ], + ], +) +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_duplicate_metrics( + logger, structured_logger, perfdocs_sample, manifest, metric_definitions, expected +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + with open(perfdocs_sample["config"], "w", newline="\n") as f: + indented_defs = "\n".join( + [(" " * 8) + metric_line for metric_line in metric_definitions.split("\n")] + ) + f.write(SAMPLE_METRICS_CONFIG.format(metric_definitions, indented_defs)) + with open(perfdocs_sample["manifest"]["path"], "w", newline="\n") as f: + f.write(manifest) + + sample_gatherer_result = { + "suite": {"Example": {"metrics": ["fcp", "SpeedIndex"]}}, + "another_suite": {"Example": {}}, + } + + from perfdocs.verifier import Verifier + + with mock.patch("perfdocs.framework_gatherers.RaptorGatherer.get_test_list") as m: + m.return_value = sample_gatherer_result + verifier = Verifier(top_dir) + verifier.validate_tree() + + assert len(logger.warning.call_args_list) == expected + for args, _ in logger.warning.call_args_list: + assert "Duplicate definitions found for " in args[0] + + +@pytest.mark.parametrize( + "manifest, metric_definitions", + [ + [ + SAMPLE_INI, + """ +metrics: + "FirstPaint": + aliases: + - fcp + - SpeedIndex + description: "Example" """, + ], + [ + SAMPLE_METRICS_INI, + """ +metrics: + FirstPaint: + aliases: + - fcp + description: Example + SpeedIndex: + aliases: + - speedindex + - si + description: Example + """, + ], + ], +) +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_verifier_valid_metrics( + logger, structured_logger, perfdocs_sample, manifest, metric_definitions +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + with open(perfdocs_sample["config"], "w", newline="\n") as f: + f.write(SAMPLE_METRICS_CONFIG.format(metric_definitions, "")) + with open(perfdocs_sample["manifest"]["path"], "w", newline="\n") as f: + f.write(manifest) + + sample_gatherer_result = { + "suite": {"Example": {"metrics": ["fcp", "SpeedIndex"]}}, + "another_suite": {"Example": {}}, + } + + from perfdocs.verifier import Verifier + + with mock.patch("perfdocs.framework_gatherers.RaptorGatherer.get_test_list") as m: + m.return_value = sample_gatherer_result + verifier = Verifier(top_dir) + verifier.validate_tree() + + assert len(logger.warning.call_args_list) == 0 + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_framework_gatherers(logger, structured_logger, perfdocs_sample): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + # Check to make sure that every single framework + # gatherer that has been implemented produces a test list + # in every suite that contains a test with an associated + # manifest. + from perfdocs.gatherer import frameworks + + for framework, gatherer in frameworks.items(): + with open(perfdocs_sample["config"], "w", newline="\n") as f: + f.write(DYNAMIC_SAMPLE_CONFIG.format(framework)) + + fg = gatherer(perfdocs_sample["config"], top_dir) + if getattr(fg, "get_test_list", None) is None: + # Skip framework gatherers that have not + # implemented a method to build a test list. + continue + + # Setup some framework-specific things here if needed + if framework == "raptor": + fg._manifest_path = perfdocs_sample["manifest"]["path"] + fg._get_subtests_from_ini = mock.Mock() + fg._get_subtests_from_ini.return_value = { + "Example": perfdocs_sample["manifest"], + } + + if framework == "talos": + fg._get_ci_tasks = mock.Mock() + for suite, suitetests in fg.get_test_list().items(): + assert suite == "Talos Tests" + assert suitetests + continue + + if framework == "awsy": + for suite, suitetests in fg.get_test_list().items(): + assert suite == "Awsy tests" + assert suitetests + continue + + for suite, suitetests in fg.get_test_list().items(): + assert suite == "suite" + for test, manifest in suitetests.items(): + assert test == "Example" + assert ( + pathlib.Path(manifest["path"]) + == perfdocs_sample["manifest"]["path"] + ) + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_framework_gatherers_urls(logger, structured_logger, perfdocs_sample): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.gatherer import frameworks + from perfdocs.generator import Generator + from perfdocs.utils import read_yaml + from perfdocs.verifier import Verifier + + # This test is only for raptor + gatherer = frameworks["raptor"] + with open(perfdocs_sample["config"], "w", newline="\n") as f: + f.write(DYNAMIC_SAMPLE_CONFIG.format("raptor")) + + fg = gatherer(perfdocs_sample["config_2"], top_dir) + fg.get_suite_list = mock.Mock() + fg.get_suite_list.return_value = { + "suite": [perfdocs_sample["example1_manifest"]], + "another_suite": [perfdocs_sample["example2_manifest"]], + } + + v = Verifier(top_dir) + gn = Generator(v, generate=True, workspace=top_dir) + + # Check to make sure that if a test is present under multiple + # suties the urls are generated correctly for the test under + # every suite + for suite, suitetests in fg.get_test_list().items(): + url = fg._descriptions.get(suite) + assert url is not None + assert url[0]["name"] == "Example" + assert url[0]["test_url"] == "Example_url" + + perfdocs_tree = gn._perfdocs_tree[0] + yaml_content = read_yaml( + pathlib.Path( + os.path.join(os.path.join(perfdocs_tree["path"], perfdocs_tree["yml"])) + ) + ) + suites = yaml_content["suites"] + + # Check that the sections for each suite are generated correctly + for suite_name, suite_details in suites.items(): + gn._verifier._gatherer = mock.Mock(framework_gatherers={"raptor": gatherer}) + section = gn._verifier._gatherer.framework_gatherers[ + "raptor" + ].build_suite_section(fg, suite_name, suites.get(suite_name)["description"]) + assert suite_name.capitalize() == section[0] + assert suite_name in section[2] + + tests = suites.get(suite_name).get("tests", {}) + for test_name in tests.keys(): + desc = gn._verifier._gatherer.framework_gatherers[ + "raptor" + ].build_test_description(fg, test_name, tests[test_name], suite_name) + assert f"**test url**: `<{url[0]['test_url']}>`__" in desc[0] + assert f"**expected**: {url[0]['expected']}" in desc[0] + assert test_name in desc[0] + + +def test_perfdocs_logger_failure(config, paths): + from perfdocs.logger import PerfDocLogger + + PerfDocLogger.LOGGER = None + with pytest.raises(Exception): + PerfDocLogger() + + PerfDocLogger.PATHS = [] + with pytest.raises(Exception): + PerfDocLogger() + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_perfdocs_generation.py b/tools/lint/test/test_perfdocs_generation.py new file mode 100644 index 0000000000..b9b540d234 --- /dev/null +++ b/tools/lint/test/test_perfdocs_generation.py @@ -0,0 +1,350 @@ +import os +import pathlib +from unittest import mock + +import mozunit + +LINTER = "perfdocs" + + +def setup_sample_logger(logger, structured_logger, top_dir): + from perfdocs.logger import PerfDocLogger + + PerfDocLogger.LOGGER = structured_logger + PerfDocLogger.PATHS = ["perfdocs"] + PerfDocLogger.TOP_DIR = top_dir + + import perfdocs.gatherer as gt + import perfdocs.generator as gn + import perfdocs.utils as utls + import perfdocs.verifier as vf + + gt.logger = logger + vf.logger = logger + gn.logger = logger + utls.logger = logger + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_generator_generate_perfdocs_pass( + logger, structured_logger, perfdocs_sample +): + from test_perfdocs import temp_file + + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates") + templates_dir.mkdir(parents=True, exist_ok=True) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=True, workspace=top_dir) + with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"): + generator.generate_perfdocs() + + assert logger.warning.call_count == 0 + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_generator_needed_regeneration( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=False, workspace=top_dir) + generator.generate_perfdocs() + + expected = "PerfDocs need to be regenerated." + args, _ = logger.warning.call_args + + assert logger.warning.call_count == 1 + assert args[0] == expected + + +@mock.patch("perfdocs.generator.get_changed_files", new=lambda x: []) +@mock.patch("perfdocs.generator.ON_TRY", new=True) +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_generator_needed_update(logger, structured_logger, perfdocs_sample): + from test_perfdocs import temp_file + + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates") + templates_dir.mkdir(parents=True, exist_ok=True) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + # Initializing perfdocs + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=True, workspace=top_dir) + with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"): + generator.generate_perfdocs() + + # Removed file for testing and run again + generator._generate = False + files = [f for f in os.listdir(generator.perfdocs_path)] + for f in files: + os.remove(str(pathlib.Path(generator.perfdocs_path, f))) + + generator.generate_perfdocs() + + expected = ( + "PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` to update them. " + "You can also apply the perfdocs.diff patch file produced from this " + "reviewbot test to fix the issue." + ) + args, _ = logger.warning.call_args + + assert logger.warning.call_count == 1 + assert args[0] == expected + + # Check to ensure a diff was produced + assert logger.log.call_count == 6 + + logs = [v[0][0] for v in logger.log.call_args_list] + for failure_log in ( + "Some files are missing or are funny.", + "Missing in existing docs: index.rst", + "Missing in existing docs: mozperftest.rst", + ): + assert failure_log in logs + + +@mock.patch("perfdocs.generator.get_changed_files", new=lambda x: []) +@mock.patch("perfdocs.generator.ON_TRY", new=True) +def test_perfdocs_generator_update_with_no_changes(structured_logger, perfdocs_sample): + """This test ensures that when no changed files exist, we'll still trigger a failure.""" + from perfdocs.logger import PerfDocLogger + from test_perfdocs import temp_file + + top_dir = perfdocs_sample["top_dir"] + + logger_mock = mock.MagicMock() + PerfDocLogger.LOGGER = logger_mock + PerfDocLogger.PATHS = ["perfdocs"] + PerfDocLogger.TOP_DIR = top_dir + logger = PerfDocLogger() + + setup_sample_logger(logger, logger_mock, top_dir) + + templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates") + templates_dir.mkdir(parents=True, exist_ok=True) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + # Initializing perfdocs + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=True, workspace=top_dir) + with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"): + generator.generate_perfdocs() + + # Removed file for testing and run again + generator._generate = False + files = [f for f in os.listdir(generator.perfdocs_path)] + for f in files: + os.remove(str(pathlib.Path(generator.perfdocs_path, f))) + + generator.generate_perfdocs() + + expected = ( + "PerfDocs are outdated, run ./mach lint -l perfdocs --fix .` to update them. " + "You can also apply the perfdocs.diff patch file produced from this " + "reviewbot test to fix the issue." + ) + assert logger.LOGGER.lint_error.call_args is not None + _, msg = logger.LOGGER.lint_error.call_args + + assert logger.FAILED + assert logger.LOGGER.lint_error.call_count == 1 + assert msg["message"] == expected + assert msg["rule"] == "Flawless performance docs (unknown file)" + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_generator_created_perfdocs( + logger, structured_logger, perfdocs_sample +): + from test_perfdocs import temp_file + + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates") + templates_dir.mkdir(parents=True, exist_ok=True) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=True, workspace=top_dir) + with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"): + perfdocs_tmpdir = generator._create_perfdocs() + + files = [f for f in os.listdir(perfdocs_tmpdir)] + files.sort() + expected_files = ["index.rst", "mozperftest.rst"] + + for i, file in enumerate(files): + assert file == expected_files[i] + + with pathlib.Path(perfdocs_tmpdir, expected_files[0]).open() as f: + filedata = f.readlines() + assert "".join(filedata) == " * :doc:`mozperftest`" + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_generator_build_perfdocs(logger, structured_logger, perfdocs_sample): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=True, workspace=top_dir) + frameworks_info = generator.build_perfdocs_from_tree() + + expected = ["dynamic", "static"] + + for framework in sorted(frameworks_info.keys()): + for i, framework_info in enumerate(frameworks_info[framework]): + assert framework_info == expected[i] + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_generator_create_temp_dir(logger, structured_logger, perfdocs_sample): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=True, workspace=top_dir) + tmpdir = generator._create_temp_dir() + + assert pathlib.Path(tmpdir).is_dir() + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_generator_create_temp_dir_fail( + logger, structured_logger, perfdocs_sample +): + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=True, workspace=top_dir) + with mock.patch("perfdocs.generator.pathlib") as path_mock: + path_mock.Path().mkdir.side_effect = OSError() + path_mock.Path().is_dir.return_value = False + tmpdir = generator._create_temp_dir() + + expected = "Error creating temp file: " + args, _ = logger.critical.call_args + + assert not tmpdir + assert logger.critical.call_count == 1 + assert args[0] == expected + + +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_generator_save_perfdocs_pass( + logger, structured_logger, perfdocs_sample +): + from test_perfdocs import temp_file + + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates") + templates_dir.mkdir(parents=True, exist_ok=True) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=True, workspace=top_dir) + + assert not generator.perfdocs_path.is_dir() + + with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"): + perfdocs_tmpdir = generator._create_perfdocs() + + generator._save_perfdocs(perfdocs_tmpdir) + + expected = ["index.rst", "mozperftest.rst"] + files = [f for f in os.listdir(generator.perfdocs_path)] + files.sort() + + for i, file in enumerate(files): + assert file == expected[i] + + +@mock.patch("perfdocs.generator.shutil") +@mock.patch("perfdocs.logger.PerfDocLogger") +def test_perfdocs_generator_save_perfdocs_fail( + logger, shutil, structured_logger, perfdocs_sample +): + from test_perfdocs import temp_file + + top_dir = perfdocs_sample["top_dir"] + setup_sample_logger(logger, structured_logger, top_dir) + + templates_dir = pathlib.Path(top_dir, "tools", "lint", "perfdocs", "templates") + templates_dir.mkdir(parents=True, exist_ok=True) + + from perfdocs.generator import Generator + from perfdocs.verifier import Verifier + + verifier = Verifier(top_dir) + verifier.validate_tree() + + generator = Generator(verifier, generate=True, workspace=top_dir) + with temp_file("index.rst", tempdir=templates_dir, content="{test_documentation}"): + perfdocs_tmpdir = generator._create_perfdocs() + + shutil.copytree = mock.Mock(side_effect=Exception()) + generator._save_perfdocs(perfdocs_tmpdir) + + expected = "There was an error while saving the documentation: " + args, _ = logger.critical.call_args + + assert logger.critical.call_count == 1 + assert args[0] == expected + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_perfdocs_helpers.py b/tools/lint/test/test_perfdocs_helpers.py new file mode 100644 index 0000000000..02c1abfecc --- /dev/null +++ b/tools/lint/test/test_perfdocs_helpers.py @@ -0,0 +1,206 @@ +import mozunit +import pytest + +LINTER = "perfdocs" + +testdata = [ + { + "table_specifications": { + "title": ["not a string"], + "widths": [10, 10, 10, 10], + "header_rows": 1, + "headers": [["Coconut 1", "Coconut 2", "Coconut 3"]], + "indent": 2, + }, + "error_msg": "TableBuilder attribute title must be a string.", + }, + { + "table_specifications": { + "title": "I've got a lovely bunch of coconuts", + "widths": ("not", "a", "list"), + "header_rows": 1, + "headers": [["Coconut 1", "Coconut 2", "Coconut 3"]], + "indent": 2, + }, + "error_msg": "TableBuilder attribute widths must be a list of integers.", + }, + { + "table_specifications": { + "title": "There they are all standing in a row", + "widths": ["not an integer"], + "header_rows": 1, + "headers": [["Coconut 1", "Coconut 2", "Coconut 3"]], + "indent": 2, + }, + "error_msg": "TableBuilder attribute widths must be a list of integers.", + }, + { + "table_specifications": { + "title": "Big ones, small ones", + "widths": [10, 10, 10, 10], + "header_rows": "not an integer", + "headers": [["Coconut 1", "Coconut 2", "Coconut 3"]], + "indent": 2, + }, + "error_msg": "TableBuilder attribute header_rows must be an integer.", + }, + { + "table_specifications": { + "title": "Some as big as your head!", + "widths": [10, 10, 10, 10], + "header_rows": 1, + "headers": ("not", "a", "list"), + "indent": 2, + }, + "error_msg": "TableBuilder attribute headers must be a two-dimensional list of strings.", + }, + { + "table_specifications": { + "title": "(And bigger)", + "widths": [10, 10, 10, 10], + "header_rows": 1, + "headers": ["not", "two", "dimensional"], + "indent": 2, + }, + "error_msg": "TableBuilder attribute headers must be a two-dimensional list of strings.", + }, + { + "table_specifications": { + "title": "Give 'em a twist, a flick of the wrist'", + "widths": [10, 10, 10, 10], + "header_rows": 1, + "headers": [[1, 2, 3]], + "indent": 2, + }, + "error_msg": "TableBuilder attribute headers must be a two-dimensional list of strings.", + }, + { + "table_specifications": { + "title": "That's what the showman said!", + "widths": [10, 10, 10, 10], + "header_rows": 1, + "headers": [["Coconut 1", "Coconut 2", "Coconut 3"]], + "indent": "not an integer", + }, + "error_msg": "TableBuilder attribute indent must be an integer.", + }, +] + +table_specifications = { + "title": "I've got a lovely bunch of coconuts", + "widths": [10, 10, 10], + "header_rows": 1, + "headers": [["Coconut 1", "Coconut 2", "Coconut 3"]], + "indent": 2, +} + + +@pytest.mark.parametrize("testdata", testdata) +def test_table_builder_invalid_attributes(testdata): + from perfdocs.doc_helpers import TableBuilder + + table_specifications = testdata["table_specifications"] + error_msg = testdata["error_msg"] + + with pytest.raises(TypeError) as error: + TableBuilder( + table_specifications["title"], + table_specifications["widths"], + table_specifications["header_rows"], + table_specifications["headers"], + table_specifications["indent"], + ) + + assert str(error.value) == error_msg + + +def test_table_builder_mismatched_columns(): + from perfdocs.doc_helpers import MismatchedRowLengthsException, TableBuilder + + table_specifications = { + "title": "I've got a lovely bunch of coconuts", + "widths": [10, 10, 10, 42], + "header_rows": 1, + "headers": [["Coconut 1", "Coconut 2", "Coconut 3"]], + "indent": 2, + } + + with pytest.raises(MismatchedRowLengthsException) as error: + TableBuilder( + table_specifications["title"], + table_specifications["widths"], + table_specifications["header_rows"], + table_specifications["headers"], + table_specifications["indent"], + ) + assert ( + str(error.value) + == "Number of table headers must match number of column widths." + ) + + +def test_table_builder_add_row_too_long(): + from perfdocs.doc_helpers import MismatchedRowLengthsException, TableBuilder + + table = TableBuilder( + table_specifications["title"], + table_specifications["widths"], + table_specifications["header_rows"], + table_specifications["headers"], + table_specifications["indent"], + ) + with pytest.raises(MismatchedRowLengthsException) as error: + table.add_row( + ["big ones", "small ones", "some as big as your head!", "(and bigger)"] + ) + assert ( + str(error.value) + == "Number of items in a row must must number of columns defined." + ) + + +def test_table_builder_add_rows_type_error(): + from perfdocs.doc_helpers import TableBuilder + + table = TableBuilder( + table_specifications["title"], + table_specifications["widths"], + table_specifications["header_rows"], + table_specifications["headers"], + table_specifications["indent"], + ) + with pytest.raises(TypeError) as error: + table.add_rows( + ["big ones", "small ones", "some as big as your head!", "(and bigger)"] + ) + assert str(error.value) == "add_rows() requires a two-dimensional list of strings." + + +def test_table_builder_validate(): + from perfdocs.doc_helpers import TableBuilder + + table = TableBuilder( + table_specifications["title"], + table_specifications["widths"], + table_specifications["header_rows"], + table_specifications["headers"], + table_specifications["indent"], + ) + table.add_row(["big ones", "small ones", "some as big as your head!"]) + table.add_row( + ["Give 'em a twist", "A flick of the wrist", "That's what the showman said!"] + ) + table = table.finish_table() + print(table) + assert ( + table == " .. list-table:: **I've got a lovely bunch of coconuts**\n" + " :widths: 10 10 10\n :header-rows: 1\n\n" + " * - **Coconut 1**\n - Coconut 2\n - Coconut 3\n" + " * - **big ones**\n - small ones\n - some as big as your head!\n" + " * - **Give 'em a twist**\n - A flick of the wrist\n" + " - That's what the showman said!\n\n" + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_rst.py b/tools/lint/test/test_rst.py new file mode 100644 index 0000000000..e540081a94 --- /dev/null +++ b/tools/lint/test/test_rst.py @@ -0,0 +1,25 @@ +import mozunit +import pytest +from mozfile import which + +LINTER = "rst" +pytestmark = pytest.mark.skipif( + not which("rstcheck"), reason="rstcheck is not installed" +) + + +def test_basic(lint, paths): + results = lint(paths()) + assert len(results) == 2 + + assert "Title underline too short" in results[0].message + assert results[0].level == "error" + assert results[0].relpath == "bad.rst" + + assert "Title overline & underline mismatch" in results[1].message + assert results[1].level == "error" + assert results[1].relpath == "bad2.rst" + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_ruff.py b/tools/lint/test/test_ruff.py new file mode 100644 index 0000000000..fbb483780e --- /dev/null +++ b/tools/lint/test/test_ruff.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from pprint import pprint +from textwrap import dedent + +import mozunit + +LINTER = "ruff" +fixed = 0 + + +def test_lint_fix(lint, create_temp_file): + contents = dedent( + """ + import distutils + print("hello!") + """ + ) + + path = create_temp_file(contents, "bad.py") + lint([path], fix=True) + assert fixed == 1 + + +def test_lint_ruff(lint, paths): + results = lint(paths()) + pprint(results, indent=2) + assert len(results) == 2 + assert results[0].level == "error" + assert results[0].relpath == "bad.py" + assert "`distutils` imported but unused" in results[0].message + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_rustfmt.py b/tools/lint/test/test_rustfmt.py new file mode 100644 index 0000000000..f1793be383 --- /dev/null +++ b/tools/lint/test/test_rustfmt.py @@ -0,0 +1,69 @@ +import mozunit + +LINTER = "rustfmt" +fixed = 0 + + +def test_good(lint, config, paths): + results = lint(paths("subdir/good.rs")) + print(results) + assert len(results) == 0 + + +def test_basic(lint, config, paths): + results = lint(paths("subdir/bad.rs")) + print(results) + assert len(results) >= 1 + + assert "Reformat rust" in results[0].message + assert results[0].level == "warning" + assert results[0].lineno == 4 + assert "bad.rs" in results[0].path + assert "Print text to the console" in results[0].diff + + +def test_dir(lint, config, paths): + results = lint(paths("subdir/")) + print(results) + assert len(results) >= 4 + + assert "Reformat rust" in results[0].message + assert results[0].level == "warning" + assert results[0].lineno == 4 + assert "bad.rs" in results[0].path + assert "Print text to the console" in results[0].diff + + assert "Reformat rust" in results[1].message + assert results[1].level == "warning" + assert results[1].lineno == 4 + assert "bad2.rs" in results[1].path + assert "Print text to the console" in results[1].diff + + +def test_fix(lint, create_temp_file): + contents = """fn main() { + // Statements here are executed when the compiled binary is called + + // Print text to the console + println!("Hello World!"); + let mut a; + let mut b=1; + let mut vec = Vec::new(); + vec.push(1); + vec.push(2); + + + for x in 5..10 - 5 { + a = x; + } + + } +""" + + path = create_temp_file(contents, "bad.rs") + lint([path], fix=True) + assert fixed == 3 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_shellcheck.py b/tools/lint/test/test_shellcheck.py new file mode 100644 index 0000000000..1b41e298bd --- /dev/null +++ b/tools/lint/test/test_shellcheck.py @@ -0,0 +1,26 @@ +import mozunit +import pytest +from mozfile import which + +LINTER = "shellcheck" +pytestmark = pytest.mark.skipif( + not which("shellcheck"), reason="shellcheck is not installed" +) + + +def test_basic(lint, paths): + results = lint(paths()) + print(results) + assert len(results) == 2 + + assert "hello appears unused" in results[0].message + assert results[0].level == "error" + assert results[0].relpath == "bad.sh" + + assert "Double quote to prevent" in results[1].message + assert results[1].level == "error" + assert results[1].relpath == "bad.sh" + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_stylelint.py b/tools/lint/test/test_stylelint.py new file mode 100644 index 0000000000..5d758ad318 --- /dev/null +++ b/tools/lint/test/test_stylelint.py @@ -0,0 +1,65 @@ +import mozunit +import pytest +from conftest import build + +LINTER = "stylelint" +fixed = 0 + + +@pytest.fixture +def stylelint(lint): + def inner(*args, **kwargs): + # --ignore-path is because stylelint doesn't have the --no-ignore option + # and therefore needs to be given an empty file for the tests to work. + kwargs["extra_args"] = [ + "--ignore-path=tools/lint/test/files/eslint/testprettierignore", + ] + return lint(*args, **kwargs) + + return inner + + +def test_lint_with_global_exclude(lint, config, paths): + config["exclude"] = ["subdir", "import"] + # This uses lint directly as we need to not ignore the excludes. + results = lint(paths(), config=config, root=build.topsrcdir) + assert len(results) == 0 + + +def test_no_files_to_lint(stylelint, config, paths): + # A directory with no files to lint. + results = stylelint(paths("nolint"), root=build.topsrcdir) + assert results == [] + + # Errors still show up even when a directory with no files is passed in. + results = stylelint(paths("nolint", "subdir/bad.css"), root=build.topsrcdir) + assert len(results) == 1 + + +def test_stylelint(stylelint, config, create_temp_file): + contents = """#foo { + font-size: 12px; + font-size: 12px; +} +""" + path = create_temp_file(contents, "bad.css") + results = stylelint([path], config=config, root=build.topsrcdir) + + assert len(results) == 1 + + +def test_stylelint_fix(stylelint, config, create_temp_file): + contents = """#foo { + font-size: 12px; + font-size: 12px; +} +""" + path = create_temp_file(contents, "bad.css") + stylelint([path], config=config, root=build.topsrcdir, fix=True) + + # stylelint returns counts of files fixed, not errors fixed. + assert fixed == 1 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_trojan_source.py b/tools/lint/test/test_trojan_source.py new file mode 100644 index 0000000000..64a3789c37 --- /dev/null +++ b/tools/lint/test/test_trojan_source.py @@ -0,0 +1,25 @@ +import mozunit + +LINTER = "trojan-source" + + +def test_lint_trojan_source(lint, paths): + results = lint(paths()) + print(results) + assert len(results) == 3 + + assert "disallowed characters" in results[0].message + assert results[0].level == "error" + assert "commenting-out.cpp" in results[0].relpath + + assert "disallowed characters" in results[1].message + assert results[1].level == "error" + assert "early-return.py" in results[1].relpath + + assert "disallowed characters" in results[2].message + assert results[2].level == "error" + assert "invisible-function.rs" in results[2].relpath + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_updatebot.py b/tools/lint/test/test_updatebot.py new file mode 100644 index 0000000000..5763cc1a7a --- /dev/null +++ b/tools/lint/test/test_updatebot.py @@ -0,0 +1,44 @@ +import os + +import mozunit + +LINTER = "updatebot" + + +def test_basic(lint, paths): + results = [] + + for p in paths(): + for root, dirs, files in os.walk(p): + for f in files: + if f == ".yamllint": + continue + + filepath = os.path.join(root, f) + result = lint(filepath, testing=True) + if result: + results.append(result) + + assert len(results) == 2 + + expected_results = 0 + + for r in results: + if "no-revision.yaml" in r[0].path: + expected_results += 1 + assert "no-revision.yaml" in r[0].path + assert ( + 'If "vendoring" is present, "revision" must be present in "origin"' + in r[0].message + ) + + if "cargo-mismatch.yaml" in r[0].path: + expected_results += 1 + assert "cargo-mismatch.yaml" in r[0].path + assert "wasn't found in Cargo.lock" in r[0].message + + assert expected_results == 2 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/test/test_yaml.py b/tools/lint/test/test_yaml.py new file mode 100644 index 0000000000..63b678d152 --- /dev/null +++ b/tools/lint/test/test_yaml.py @@ -0,0 +1,28 @@ +import mozunit + +LINTER = "yaml" + + +def test_basic(lint, paths): + results = lint(paths()) + + assert len(results) == 3 + + assert "line too long (122 > 80 characters)" in results[0].message + assert results[0].level == "error" + assert "bad.yml" in results[0].relpath + assert results[0].lineno == 3 + + assert "wrong indentation: expected 4 but found 8" in results[1].message + assert results[1].level == "error" + assert "bad.yml" in results[1].relpath + assert results[0].lineno == 3 + + assert "could not find expected" in results[2].message + assert results[2].level == "error" + assert "bad.yml" in results[2].relpath + assert results[2].lineno == 9 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/lint/tox/tox_requirements.txt b/tools/lint/tox/tox_requirements.txt new file mode 100644 index 0000000000..00cec3ff5d --- /dev/null +++ b/tools/lint/tox/tox_requirements.txt @@ -0,0 +1,10 @@ +pluggy==0.13.1 --hash=sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d +importlib-metadata==0.23 --hash=sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af +more-itertools==7.2.0 --hash=sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4 +zipp==0.6.0 --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335 +py==1.11.0 --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 +tox==2.7.0 --hash=sha256:0f37ea637ead4a5bbae91531b0bf8fd327c7152e20255e5960ee180598228d21 +virtualenv==20.24.7 --hash=sha256:a18b3fd0314ca59a2e9f4b556819ed07183b3e9a3702ecfe213f593d44f7b3fd +distlib==0.3.7 --hash=sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057 +filelock==3.13.1 --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c +platformdirs==4.0.0 --hash=sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b diff --git a/tools/lint/trojan-source.yml b/tools/lint/trojan-source.yml new file mode 100644 index 0000000000..9b25962b2e --- /dev/null +++ b/tools/lint/trojan-source.yml @@ -0,0 +1,27 @@ +--- +trojan-source: + description: Trojan Source attack - CVE-2021-42572 + include: + - . + exclude: + - intl/lwbrk/rulebrk.c + - testing/web-platform/tests/conformance-checkers/tools/ins-del-datetime.py + - modules/freetype2/src/autofit/afblue.c + - modules/freetype2/builds/amiga/include/config/ftconfig.h + - modules/freetype2/builds/amiga/include/config/ftmodule.h + - modules/freetype2/builds/amiga/src/base/ftsystem.c + - third_party/rust/chardetng/src/data.rs + - third_party/rust/error-chain/tests/tests.rs + - third_party/rust/unicode-width/src/tests.rs + - security/nss/gtests/mozpkix_gtest/pkixnames_tests.cpp + extensions: + - .c + - .cc + - .cpp + - .h + - .py + - .rs + support-files: + - 'tools/lint/trojan-source/**' + type: external + payload: trojan-source:lint diff --git a/tools/lint/trojan-source/__init__.py b/tools/lint/trojan-source/__init__.py new file mode 100644 index 0000000000..a20c10203d --- /dev/null +++ b/tools/lint/trojan-source/__init__.py @@ -0,0 +1,67 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import sys +import unicodedata + +from mozlint import result +from mozlint.pathutils import expand_exclusions + +# Code inspired by Red Hat +# https://github.com/siddhesh/find-unicode-control/ +# published under the 'BSD 3-Clause' license +# https://access.redhat.com/security/vulnerabilities/RHSB-2021-007 + +results = [] + +disallowed = set( + chr(c) for c in range(sys.maxunicode) if unicodedata.category(chr(c)) == "Cf" +) + + +def getfiletext(config, filename): + # Make a text string from a file, attempting to decode from latin1 if necessary. + # Other non-utf-8 locales are not supported at the moment. + with open(filename, "rb") as infile: + try: + return infile.read().decode("utf-8") + except Exception as e: + res = { + "path": filename, + "message": "Could not open file as utf-8 - maybe an encoding error: %s" + % e, + "level": "error", + } + results.append(result.from_config(config, **res)) + return None + + return None + + +def analyze_text(filename, text, disallowed): + line = 0 + for t in text.splitlines(): + line = line + 1 + subset = [c for c in t if chr(ord(c)) in disallowed] + if subset: + return (subset, line) + + return ("", 0) + + +def lint(paths, config, **lintargs): + files = list(expand_exclusions(paths, config, lintargs["root"])) + for f in files: + text = getfiletext(config, f) + if text: + (subset, line) = analyze_text(f, text, disallowed) + if subset: + res = { + "path": f, + "lineno": line, + "message": "disallowed characters: %s" % subset, + "level": "error", + } + results.append(result.from_config(config, **res)) + + return {"results": results, "fixed": 0} diff --git a/tools/lint/updatebot.yml b/tools/lint/updatebot.yml new file mode 100644 index 0000000000..90f13e8cf4 --- /dev/null +++ b/tools/lint/updatebot.yml @@ -0,0 +1,9 @@ +--- +updatebot: + description: > + "Ensure moz.yaml files are valid" + extensions: ['yaml'] + include: ['.'] + type: external-file + level: error + payload: updatebot.validate_yaml:lint diff --git a/tools/lint/updatebot/__init__.py b/tools/lint/updatebot/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/lint/updatebot/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/lint/updatebot/validate_yaml.py b/tools/lint/updatebot/validate_yaml.py new file mode 100644 index 0000000000..802a9033fa --- /dev/null +++ b/tools/lint/updatebot/validate_yaml.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozbuild.vendor.moz_yaml import load_moz_yaml +from mozlint import result +from mozlint.pathutils import expand_exclusions + + +class UpdatebotValidator: + def lint_file(self, path, **kwargs): + if not kwargs.get("testing", False) and not path.endswith("moz.yaml"): + # When testing, process all files provided + return None + if not kwargs.get("testing", False) and "test/files/updatebot" in path: + # When not testing, ignore the test files + return None + + try: + yaml = load_moz_yaml(path) + + if "vendoring" in yaml and yaml["vendoring"].get("flavor", None) == "rust": + yaml_revision = yaml["origin"]["revision"] + + with open("Cargo.lock", "r") as f: + for line in f: + if yaml_revision in line: + return None + + return f"Revision {yaml_revision} specified in {path} wasn't found in Cargo.lock" + + return None + except Exception as e: + return f"Could not load {path} according to schema in moz_yaml.py: {e}" + + +def lint(paths, config, **lintargs): + # expand_exclusions expects a list, and will convert a string + # into it if it doesn't receive one + if not isinstance(paths, list): + paths = [paths] + + errors = [] + files = list(expand_exclusions(paths, config, lintargs["root"])) + + m = UpdatebotValidator() + for f in files: + message = m.lint_file(f, **lintargs) + if message: + errors.append(result.from_config(config, path=f, message=message)) + + return errors diff --git a/tools/lint/wpt.yml b/tools/lint/wpt.yml new file mode 100644 index 0000000000..dd3c0dd042 --- /dev/null +++ b/tools/lint/wpt.yml @@ -0,0 +1,10 @@ +--- +wpt: + description: web-platform-tests lint + include: + - testing/web-platform/tests + exclude: [] + support-files: + - tools/lint/wpt/wpt.py + type: external + payload: wpt.wpt:lint diff --git a/tools/lint/wpt/__init__.py b/tools/lint/wpt/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/lint/wpt/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/lint/wpt/wpt.py b/tools/lint/wpt/wpt.py new file mode 100644 index 0000000000..6f4f40492a --- /dev/null +++ b/tools/lint/wpt/wpt.py @@ -0,0 +1,64 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import subprocess +import sys + +from mozlint import result + +results = [] + + +def lint(files, config, **kwargs): + log = kwargs["log"] + tests_dir = os.path.join(kwargs["root"], "testing", "web-platform", "tests") + + def process_line(line): + try: + data = json.loads(line) + except ValueError: + print( + f"Got non-JSON output: {line}", + file=sys.stderr, + ) + return + + data["level"] = "error" + data["path"] = os.path.relpath( + os.path.join(tests_dir, data["path"]), kwargs["root"] + ) + data.setdefault("lineno", 0) + results.append(result.from_config(config, **data)) + + if files == [tests_dir]: + print( + "No specific files specified, running the full wpt lint" " (this is slow)", + file=sys.stderr, + ) + files = ["--all"] + cmd = ["python3", os.path.join(tests_dir, "wpt"), "lint", "--json"] + files + log.debug("Command: {}".format(" ".join(cmd))) + + proc = subprocess.Popen( + cmd, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) + try: + for line in proc.stdout: + process_line(line.rstrip("\r\n")) + proc.wait() + if proc.returncode != 0: + results.append( + result.from_config( + config, + message="Lint process exited with return code %s" % proc.returncode, + ) + ) + except KeyboardInterrupt: + proc.kill() + + return results diff --git a/tools/lint/yaml.yml b/tools/lint/yaml.yml new file mode 100644 index 0000000000..47cde80177 --- /dev/null +++ b/tools/lint/yaml.yml @@ -0,0 +1,19 @@ +--- +yamllint: + description: YAML linter + include: + - .cron.yml + - .taskcluster.yml + - browser/config/ + - python/mozlint/ + - security/nss/.taskcluster.yml + - taskcluster + - testing/mozharness + - tools + - build/cargo + extensions: ['yml', 'yaml'] + support-files: + - '**/.yamllint' + - 'tools/lint/yamllint_/**' + type: external + payload: yamllint_:lint diff --git a/tools/lint/yamllint_/__init__.py b/tools/lint/yamllint_/__init__.py new file mode 100644 index 0000000000..2244fabd3f --- /dev/null +++ b/tools/lint/yamllint_/__init__.py @@ -0,0 +1,101 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import sys +from collections import defaultdict + +from mozbuild.base import MozbuildObject + +topsrcdir = MozbuildObject.from_environment().topsrcdir + +from mozlint import result +from mozlint.pathutils import get_ancestors_by_name +from mozlint.util.implementation import LintProcess + +YAMLLINT_FORMAT_REGEX = re.compile("(.*):(.*):(.*): \[(error|warning)\] (.*) \((.*)\)$") + +results = [] + + +class YAMLLintProcess(LintProcess): + def process_line(self, line): + try: + match = YAMLLINT_FORMAT_REGEX.match(line) + abspath, line, col, level, message, code = match.groups() + except AttributeError: + print("Unable to match yaml regex against output: {}".format(line)) + return + + res = { + "path": os.path.relpath(str(abspath), self.config["root"]), + "message": str(message), + "level": "error", + "lineno": line, + "column": col, + "rule": code, + } + + results.append(result.from_config(self.config, **res)) + + +def get_yamllint_version(): + from yamllint import APP_VERSION + + return APP_VERSION + + +def run_process(config, cmd): + proc = YAMLLintProcess(config, cmd) + proc.run() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + + +def gen_yamllint_args(cmdargs, paths=None, conf_file=None): + args = cmdargs[:] + if isinstance(paths, str): + paths = [paths] + if conf_file and conf_file != "default": + return args + ["-c", conf_file] + paths + return args + paths + + +def lint(files, config, **lintargs): + log = lintargs["log"] + + log.debug("Version: {}".format(get_yamllint_version())) + + cmdargs = [ + sys.executable, + os.path.join(topsrcdir, "mach"), + "python", + "--", + "-m", + "yamllint", + "-f", + "parsable", + ] + log.debug("Command: {}".format(" ".join(cmdargs))) + + config = config.copy() + config["root"] = lintargs["root"] + + # Run any paths with a .yamllint file in the directory separately so + # it gets picked up. This means only .yamllint files that live in + # directories that are explicitly included will be considered. + paths_by_config = defaultdict(list) + for f in files: + conf_files = get_ancestors_by_name(".yamllint", f, config["root"]) + paths_by_config[conf_files[0] if conf_files else "default"].append(f) + + for conf_file, paths in paths_by_config.items(): + run_process( + config, gen_yamllint_args(cmdargs, conf_file=conf_file, paths=paths) + ) + + return results diff --git a/tools/mach_commands.py b/tools/mach_commands.py new file mode 100644 index 0000000000..9b96047f71 --- /dev/null +++ b/tools/mach_commands.py @@ -0,0 +1,594 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import logging +import subprocess +import sys +from datetime import datetime, timedelta +from operator import itemgetter + +from mach.decorators import Command, CommandArgument, SubCommand +from mozbuild.base import MozbuildObject + + +def _get_busted_bugs(payload): + import requests + + payload = dict(payload) + payload["include_fields"] = "id,summary,last_change_time,resolution" + payload["blocks"] = 1543241 + response = requests.get("https://bugzilla.mozilla.org/rest/bug", payload) + response.raise_for_status() + return response.json().get("bugs", []) + + +@Command( + "busted", + category="misc", + description="Query known bugs in our tooling, and file new ones.", +) +def busted_default(command_context): + unresolved = _get_busted_bugs({"resolution": "---"}) + creation_time = datetime.now() - timedelta(days=15) + creation_time = creation_time.strftime("%Y-%m-%dT%H-%M-%SZ") + resolved = _get_busted_bugs({"creation_time": creation_time}) + resolved = [bug for bug in resolved if bug["resolution"]] + all_bugs = sorted( + unresolved + resolved, key=itemgetter("last_change_time"), reverse=True + ) + if all_bugs: + for bug in all_bugs: + print( + "[%s] Bug %s - %s" + % ( + "UNRESOLVED" + if not bug["resolution"] + else "RESOLVED - %s" % bug["resolution"], + bug["id"], + bug["summary"], + ) + ) + else: + print("No known tooling issues found.") + + +@SubCommand("busted", "file", description="File a bug for busted tooling.") +@CommandArgument( + "against", + help=( + "The specific mach command that is busted (i.e. if you encountered " + "an error with `mach build`, run `mach busted file build`). If " + "the issue is not connected to any particular mach command, you " + "can also run `mach busted file general`." + ), +) +def busted_file(command_context, against): + import webbrowser + + if ( + against != "general" + and against not in command_context._mach_context.commands.command_handlers + ): + print( + "%s is not a valid value for `against`. `against` must be " + "the name of a `mach` command, or else the string " + '"general".' % against + ) + return 1 + + if against == "general": + product = "Firefox Build System" + component = "General" + else: + import inspect + + import mozpack.path as mozpath + + # Look up the file implementing that command, then cross-refernce + # moz.build files to get the product/component. + handler = command_context._mach_context.commands.command_handlers[against] + sourcefile = mozpath.relpath( + inspect.getsourcefile(handler.func), command_context.topsrcdir + ) + reader = command_context.mozbuild_reader(config_mode="empty") + try: + res = reader.files_info([sourcefile])[sourcefile]["BUG_COMPONENT"] + product, component = res.product, res.component + except TypeError: + # The file might not have a bug set. + product = "Firefox Build System" + component = "General" + + uri = ( + "https://bugzilla.mozilla.org/enter_bug.cgi?" + "product=%s&component=%s&blocked=1543241" % (product, component) + ) + webbrowser.open_new_tab(uri) + + +MACH_PASTEBIN_DURATIONS = { + "onetime": "onetime", + "hour": "3600", + "day": "86400", + "week": "604800", + "month": "2073600", +} + +EXTENSION_TO_HIGHLIGHTER = { + ".hgrc": "ini", + "Dockerfile": "docker", + "Makefile": "make", + "applescript": "applescript", + "arduino": "arduino", + "bash": "bash", + "bat": "bat", + "c": "c", + "clojure": "clojure", + "cmake": "cmake", + "coffee": "coffee-script", + "console": "console", + "cpp": "cpp", + "cs": "csharp", + "css": "css", + "cu": "cuda", + "cuda": "cuda", + "dart": "dart", + "delphi": "delphi", + "diff": "diff", + "django": "django", + "docker": "docker", + "elixir": "elixir", + "erlang": "erlang", + "go": "go", + "h": "c", + "handlebars": "handlebars", + "haskell": "haskell", + "hs": "haskell", + "html": "html", + "ini": "ini", + "ipy": "ipythonconsole", + "ipynb": "ipythonconsole", + "irc": "irc", + "j2": "django", + "java": "java", + "js": "js", + "json": "json", + "jsx": "jsx", + "kt": "kotlin", + "less": "less", + "lisp": "common-lisp", + "lsp": "common-lisp", + "lua": "lua", + "m": "objective-c", + "make": "make", + "matlab": "matlab", + "md": "_markdown", + "nginx": "nginx", + "numpy": "numpy", + "patch": "diff", + "perl": "perl", + "php": "php", + "pm": "perl", + "postgresql": "postgresql", + "py": "python", + "rb": "rb", + "rs": "rust", + "rst": "rst", + "sass": "sass", + "scss": "scss", + "sh": "bash", + "sol": "sol", + "sql": "sql", + "swift": "swift", + "tex": "tex", + "typoscript": "typoscript", + "vim": "vim", + "xml": "xml", + "xslt": "xslt", + "yaml": "yaml", + "yml": "yaml", +} + + +def guess_highlighter_from_path(path): + """Return a known highlighter from a given path + + Attempt to select a highlighter by checking the file extension in the mapping + of extensions to highlighter. If that fails, attempt to pass the basename of + the file. Return `_code` as the default highlighter if that fails. + """ + import os + + _name, ext = os.path.splitext(path) + + if ext.startswith("."): + ext = ext[1:] + + if ext in EXTENSION_TO_HIGHLIGHTER: + return EXTENSION_TO_HIGHLIGHTER[ext] + + basename = os.path.basename(path) + + return EXTENSION_TO_HIGHLIGHTER.get(basename, "_code") + + +PASTEMO_MAX_CONTENT_LENGTH = 250 * 1024 * 1024 + +PASTEMO_URL = "https://paste.mozilla.org/api/" + + +@Command( + "pastebin", + category="misc", + description="Command line interface to paste.mozilla.org.", +) +@CommandArgument( + "--list-highlighters", + action="store_true", + help="List known highlighters and exit", +) +@CommandArgument( + "--highlighter", default=None, help="Syntax highlighting to use for paste" +) +@CommandArgument( + "--expires", + default="week", + choices=sorted(MACH_PASTEBIN_DURATIONS.keys()), + help="Expire paste after given time duration (default: %(default)s)", +) +@CommandArgument( + "--verbose", + action="store_true", + help="Print extra info such as selected syntax highlighter", +) +@CommandArgument( + "path", + nargs="?", + default=None, + help="Path to file for upload to paste.mozilla.org", +) +def pastebin(command_context, list_highlighters, highlighter, expires, verbose, path): + """Command line interface to `paste.mozilla.org`. + + Takes either a filename whose content should be pasted, or reads + content from standard input. If a highlighter is specified it will + be used, otherwise the file name will be used to determine an + appropriate highlighter. + """ + + import requests + + def verbose_print(*args, **kwargs): + """Print a string if `--verbose` flag is set""" + if verbose: + print(*args, **kwargs) + + # Show known highlighters and exit. + if list_highlighters: + lexers = set(EXTENSION_TO_HIGHLIGHTER.values()) + print("Available lexers:\n - %s" % "\n - ".join(sorted(lexers))) + return 0 + + # Get a correct expiry value. + try: + verbose_print("Setting expiry from %s" % expires) + expires = MACH_PASTEBIN_DURATIONS[expires] + verbose_print("Using %s as expiry" % expires) + except KeyError: + print( + "%s is not a valid duration.\n" + "(hint: try one of %s)" + % (expires, ", ".join(MACH_PASTEBIN_DURATIONS.keys())) + ) + return 1 + + data = { + "format": "json", + "expires": expires, + } + + # Get content to be pasted. + if path: + verbose_print("Reading content from %s" % path) + try: + with open(path, "r") as f: + content = f.read() + except IOError: + print("ERROR. No such file %s" % path) + return 1 + + lexer = guess_highlighter_from_path(path) + if lexer: + data["lexer"] = lexer + else: + verbose_print("Reading content from stdin") + content = sys.stdin.read() + + # Assert the length of content to be posted does not exceed the maximum. + content_length = len(content) + verbose_print("Checking size of content is okay (%d)" % content_length) + if content_length > PASTEMO_MAX_CONTENT_LENGTH: + print( + "Paste content is too large (%d, maximum %d)" + % (content_length, PASTEMO_MAX_CONTENT_LENGTH) + ) + return 1 + + data["content"] = content + + # Highlight as specified language, overwriting value set from filename. + if highlighter: + verbose_print("Setting %s as highlighter" % highlighter) + data["lexer"] = highlighter + + try: + verbose_print("Sending request to %s" % PASTEMO_URL) + resp = requests.post(PASTEMO_URL, data=data) + + # Error code should always be 400. + # Response content will include a helpful error message, + # so print it here (for example, if an invalid highlighter is + # provided, it will return a list of valid highlighters). + if resp.status_code >= 400: + print("Error code %d: %s" % (resp.status_code, resp.content)) + return 1 + + verbose_print("Pasted successfully") + + response_json = resp.json() + + verbose_print("Paste highlighted as %s" % response_json["lexer"]) + print(response_json["url"]) + + return 0 + except Exception as e: + print("ERROR. Paste failed.") + print("%s" % e) + return 1 + + +class PypiBasedTool: + """ + Helper for loading a tool that is hosted on pypi. The package is expected + to expose a `mach_interface` module which has `new_release_on_pypi`, + `parser`, and `run` functions. + """ + + def __init__(self, module_name, pypi_name=None): + self.name = module_name + self.pypi_name = pypi_name or module_name + + def _import(self): + # Lazy loading of the tools mach interface. + # Note that only the mach_interface module should be used from this file. + import importlib + + try: + return importlib.import_module("%s.mach_interface" % self.name) + except ImportError: + return None + + def create_parser(self, subcommand=None): + # Create the command line parser. + # If the tool is not installed, or not up to date, it will + # first be installed. + cmd = MozbuildObject.from_environment() + cmd.activate_virtualenv() + tool = self._import() + if not tool: + # The tool is not here at all, install it + cmd.virtualenv_manager.install_pip_package(self.pypi_name) + print( + "%s was installed. please re-run your" + " command. If you keep getting this message please " + " manually run: 'pip install -U %s'." % (self.pypi_name, self.pypi_name) + ) + else: + # Check if there is a new release available + release = tool.new_release_on_pypi() + if release: + print(release) + # there is one, so install it. Note that install_pip_package + # does not work here, so just run pip directly. + subprocess.check_call( + [ + cmd.virtualenv_manager.python_path, + "-m", + "pip", + "install", + f"{self.pypi_name}=={release}", + ] + ) + print( + "%s was updated to version %s. please" + " re-run your command." % (self.pypi_name, release) + ) + else: + # Tool is up to date, return the parser. + if subcommand: + return tool.parser(subcommand) + else: + return tool.parser() + # exit if we updated or installed mozregression because + # we may have already imported mozregression and running it + # as this may cause issues. + sys.exit(0) + + def run(self, **options): + tool = self._import() + tool.run(options) + + +def mozregression_create_parser(): + # Create the mozregression command line parser. + # if mozregression is not installed, or not up to date, it will + # first be installed. + loader = PypiBasedTool("mozregression") + return loader.create_parser() + + +@Command( + "mozregression", + category="misc", + description="Regression range finder for nightly and inbound builds.", + parser=mozregression_create_parser, +) +def run(command_context, **options): + command_context.activate_virtualenv() + mozregression = PypiBasedTool("mozregression") + mozregression.run(**options) + + +@Command( + "node", + category="devenv", + description="Run the NodeJS interpreter used for building.", +) +@CommandArgument("args", nargs=argparse.REMAINDER) +def node(command_context, args): + from mozbuild.nodeutil import find_node_executable + + # Avoid logging the command + command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) + + node_path, _ = find_node_executable() + + return command_context.run_process( + [node_path] + args, + pass_thru=True, # Allow user to run Node interactively. + ensure_exit_code=False, # Don't throw on non-zero exit code. + ) + + +@Command( + "npm", + category="devenv", + description="Run the npm executable from the NodeJS used for building.", +) +@CommandArgument("args", nargs=argparse.REMAINDER) +def npm(command_context, args): + from mozbuild.nodeutil import find_npm_executable + + # Avoid logging the command + command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) + + import os + + # Add node and npm from mozbuild to front of system path + # + # This isn't pretty, but npm currently executes itself with + # `#!/usr/bin/env node`, which means it just uses the node in the + # current PATH. As a result, stuff gets built wrong and installed + # in the wrong places and probably other badness too without this: + npm_path, _ = find_npm_executable() + if not npm_path: + exit(-1, "could not find npm executable") + path = os.path.abspath(os.path.dirname(npm_path)) + os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"]) + + # karma-firefox-launcher needs the path to firefox binary. + firefox_bin = command_context.get_binary_path(validate_exists=False) + if os.path.exists(firefox_bin): + os.environ["FIREFOX_BIN"] = firefox_bin + + return command_context.run_process( + [npm_path, "--scripts-prepend-node-path=auto"] + args, + pass_thru=True, # Avoid eating npm output/error messages + ensure_exit_code=False, # Don't throw on non-zero exit code. + ) + + +def logspam_create_parser(subcommand): + # Create the logspam command line parser. + # if logspam is not installed, or not up to date, it will + # first be installed. + loader = PypiBasedTool("logspam", "mozilla-log-spam") + return loader.create_parser(subcommand) + + +from functools import partial + + +@Command( + "logspam", + category="misc", + description="Warning categorizer for treeherder test runs.", +) +def logspam(command_context): + pass + + +@SubCommand("logspam", "report", parser=partial(logspam_create_parser, "report")) +def report(command_context, **options): + command_context.activate_virtualenv() + logspam = PypiBasedTool("logspam") + logspam.run(command="report", **options) + + +@SubCommand("logspam", "bisect", parser=partial(logspam_create_parser, "bisect")) +def bisect(command_context, **options): + command_context.activate_virtualenv() + logspam = PypiBasedTool("logspam") + logspam.run(command="bisect", **options) + + +@SubCommand("logspam", "file", parser=partial(logspam_create_parser, "file")) +def create(command_context, **options): + command_context.activate_virtualenv() + logspam = PypiBasedTool("logspam") + logspam.run(command="file", **options) + + +# mots_loader will be used when running commands and subcommands, as well as +# when creating the parsers. +mots_loader = PypiBasedTool("mots") + + +def mots_create_parser(subcommand=None): + return mots_loader.create_parser(subcommand) + + +def mots_run_subcommand(command, command_context, **options): + command_context.activate_virtualenv() + mots_loader.run(command=command, **options) + + +class motsSubCommand(SubCommand): + """A helper subclass that reduces repitition when defining subcommands.""" + + def __init__(self, subcommand): + super().__init__( + "mots", + subcommand, + parser=partial(mots_create_parser, subcommand), + ) + + +@Command( + "mots", + category="misc", + description="Manage module information in-tree using the mots CLI.", + parser=mots_create_parser, +) +def mots(command_context, **options): + """The main mots command call.""" + command_context.activate_virtualenv() + mots_loader.run(**options) + + +# Define subcommands that will be proxied through mach. +for sc in ( + "clean", + "check-hashes", + "export", + "export-and-clean", + "module", + "query", + "settings", + "user", + "validate", +): + # Pass through args and kwargs, but add the subcommand string as the first argument. + motsSubCommand(sc)(lambda *a, **kw: mots_run_subcommand(sc, *a, **kw)) diff --git a/tools/moz.build b/tools/moz.build new file mode 100644 index 0000000000..bca499db61 --- /dev/null +++ b/tools/moz.build @@ -0,0 +1,79 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Core", "General") + +with Files("code-coverage/**"): + BUG_COMPONENT = ("Testing", "Code Coverage") + +with Files("compare-locales/mach_commands.py"): + BUG_COMPONENT = ("Localization Infrastructure and Tools", "compare-locales") + +with Files("github-sync/**"): + BUG_COMPONENT = ("Core", "Graphics") + +with Files("lint/**"): + BUG_COMPONENT = ("Developer Infrastructure", "Lint and Formatting") + +with Files("moztreedocs/**"): + BUG_COMPONENT = ("Developer Infrastructure", "Source Documentation") + SCHEDULES.exclusive = ["docs"] + +with Files("profiler/**"): + BUG_COMPONENT = ("Core", "Gecko Profiler") + +with Files("performance/**"): + BUG_COMPONENT = ("Core", "Gecko Profiler") + +with Files("quitter/**"): + BUG_COMPONENT = ("Testing", "General") + +with Files("rb/**"): + BUG_COMPONENT = ("Core", "XPCOM") + +with Files("rewriting/**"): + BUG_COMPONENT = ("Firefox Build System", "Source Code Analysis") + +with Files("tryselect/**"): + BUG_COMPONENT = ("Developer Infrastructure", "Try") + +with Files("tryselect/selectors/release.py"): + BUG_COMPONENT = ("Release Engineering", "General") + +with Files("update-packaging/**"): + BUG_COMPONENT = ("Release Engineering", "General") + +with Files("update-verify/**"): + BUG_COMPONENT = ("Release Engineering", "Release Automation: Updates") + +with Files("vcs/**"): + BUG_COMPONENT = ("Firefox Build System", "General") + +SPHINX_TREES["moztreedocs"] = "moztreedocs/docs" + +SPHINX_TREES["try"] = "tryselect/docs" + +SPHINX_TREES["fuzzing"] = "fuzzing/docs" + +SPHINX_TREES["sanitizer"] = "sanitizer/docs" + +SPHINX_TREES["code-coverage"] = "code-coverage/docs" + +SPHINX_TREES["profiler"] = "profiler/docs" + +with Files("tryselect/docs/**"): + SCHEDULES.exclusive = ["docs"] + +CRAMTEST_MANIFESTS += [ + "tryselect/test/cram.toml", +] + +PYTHON_UNITTEST_MANIFESTS += [ + "fuzzing/smoke/python.toml", + "lint/test/python.toml", + "tryselect/test/python.toml", +] diff --git a/tools/moztreedocs/__init__.py b/tools/moztreedocs/__init__.py new file mode 100644 index 0000000000..7b8aed6059 --- /dev/null +++ b/tools/moztreedocs/__init__.py @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from pathlib import PurePath + +import sphinx +import sphinx.ext.apidoc +import yaml +from mozbuild.base import MozbuildObject +from mozbuild.frontend.reader import BuildReader +from mozbuild.util import memoize +from mozpack.copier import FileCopier +from mozpack.files import FileFinder +from mozpack.manifests import InstallManifest + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + +MAIN_DOC_PATH = os.path.normpath(os.path.join(build.topsrcdir, "docs")) + +logger = sphinx.util.logging.getLogger(__name__) + + +@memoize +def read_build_config(docdir): + """Read the active build config and return the relevant doc paths. + + The return value is cached so re-generating with the same docdir won't + invoke the build system a second time.""" + trees = {} + python_package_dirs = set() + + is_main = docdir == MAIN_DOC_PATH + relevant_mozbuild_path = None if is_main else docdir + + # Reading the Sphinx variables doesn't require a full build context. + # Only define the parts we need. + class fakeconfig(object): + topsrcdir = build.topsrcdir + + variables = ("SPHINX_TREES", "SPHINX_PYTHON_PACKAGE_DIRS") + reader = BuildReader(fakeconfig()) + result = reader.find_variables_from_ast(variables, path=relevant_mozbuild_path) + for path, name, key, value in result: + reldir = os.path.dirname(path) + + if name == "SPHINX_TREES": + # If we're building a subtree, only process that specific subtree. + # topsrcdir always uses POSIX-style path, normalize it for proper comparison. + absdir = os.path.normpath(os.path.join(build.topsrcdir, reldir, value)) + if not is_main and absdir not in (docdir, MAIN_DOC_PATH): + # allow subpaths of absdir (i.e. docdir = <absdir>/sub/path/) + if docdir.startswith(absdir): + key = os.path.join(key, docdir.split(f"{key}/")[-1]) + else: + continue + + assert key + if key.startswith("/"): + key = key[1:] + else: + key = os.path.normpath(os.path.join(reldir, key)) + + if key in trees: + raise Exception( + "%s has already been registered as a destination." % key + ) + trees[key] = os.path.join(reldir, value) + + if name == "SPHINX_PYTHON_PACKAGE_DIRS": + python_package_dirs.add(os.path.join(reldir, value)) + + return trees, python_package_dirs + + +class _SphinxManager(object): + """Manages the generation of Sphinx documentation for the tree.""" + + NO_AUTODOC = False + + def __init__(self, topsrcdir, main_path): + self.topsrcdir = topsrcdir + self.conf_py_path = os.path.join(main_path, "conf.py") + self.index_path = os.path.join(main_path, "index.rst") + + # Instance variables that get set in self.generate_docs() + self.staging_dir = None + self.trees = None + self.python_package_dirs = None + + def generate_docs(self, app): + """Generate/stage documentation.""" + if self.NO_AUTODOC: + logger.info("Python/JS API documentation generation will be skipped") + app.config["extensions"].remove("sphinx.ext.autodoc") + app.config["extensions"].remove("sphinx_js") + self.staging_dir = os.path.join(app.outdir, "_staging") + + logger.info("Reading Sphinx metadata from build configuration") + self.trees, self.python_package_dirs = read_build_config(app.srcdir) + + logger.info("Staging static documentation") + self._synchronize_docs(app) + + if not self.NO_AUTODOC: + self._generate_python_api_docs() + + def _generate_python_api_docs(self): + """Generate Python API doc files.""" + out_dir = os.path.join(self.staging_dir, "python") + base_args = ["--no-toc", "-o", out_dir] + + for p in sorted(self.python_package_dirs): + full = os.path.join(self.topsrcdir, p) + + finder = FileFinder(full) + dirs = {os.path.dirname(f[0]) for f in finder.find("**")} + + test_dirs = {"test", "tests"} + excludes = {d for d in dirs if set(PurePath(d).parts) & test_dirs} + + args = list(base_args) + args.append(full) + args.extend(excludes) + + sphinx.ext.apidoc.main(argv=args) + + def _synchronize_docs(self, app): + m = InstallManifest() + + with open(os.path.join(MAIN_DOC_PATH, "config.yml"), "r") as fh: + tree_config = yaml.safe_load(fh)["categories"] + + m.add_link(self.conf_py_path, "conf.py") + + for dest, source in sorted(self.trees.items()): + source_dir = os.path.join(self.topsrcdir, source) + for root, _, files in os.walk(source_dir): + for f in files: + source_path = os.path.normpath(os.path.join(root, f)) + rel_source = source_path[len(source_dir) + 1 :] + target = os.path.normpath(os.path.join(dest, rel_source)) + m.add_link(source_path, target) + + copier = FileCopier() + m.populate_registry(copier) + + # In the case of livereload, we don't want to delete unmodified (unaccounted) files. + copier.copy( + self.staging_dir, remove_empty_directories=False, remove_unaccounted=False + ) + + with open(self.index_path, "r") as fh: + data = fh.read() + + def is_toplevel(key): + """Whether the tree is nested under the toplevel index, or is + nested under another tree's index. + """ + for k in self.trees: + if k == key: + continue + if key.startswith(k): + return False + return True + + def format_paths(paths): + source_doc = ["%s/index" % p for p in paths] + return "\n ".join(source_doc) + + toplevel_trees = {k: v for k, v in self.trees.items() if is_toplevel(k)} + + CATEGORIES = {} + # generate the datastructure to deal with the tree + for t in tree_config: + CATEGORIES[t] = format_paths(tree_config[t]) + + # During livereload, we don't correctly rebuild the full document + # tree (Bug 1557020). The page is no longer referenced within the index + # tree, thus we shall check categorisation only if complete tree is being rebuilt. + if app.srcdir == self.topsrcdir: + indexes = set( + [ + os.path.normpath(os.path.join(p, "index")) + for p in toplevel_trees.keys() + ] + ) + # Format categories like indexes + cats = "\n".join(CATEGORIES.values()).split("\n") + # Remove heading spaces + cats = [os.path.normpath(x.strip()) for x in cats] + indexes = tuple(set(indexes) - set(cats)) + if indexes: + # In case a new doc isn't categorized + print(indexes) + raise Exception( + "Uncategorized documentation. Please add it in docs/config.yml" + ) + + data = data.format(**CATEGORIES) + + with open(os.path.join(self.staging_dir, "index.rst"), "w") as fh: + fh.write(data) + + +manager = _SphinxManager(build.topsrcdir, MAIN_DOC_PATH) diff --git a/tools/moztreedocs/docs/adding-documentation.rst b/tools/moztreedocs/docs/adding-documentation.rst new file mode 100644 index 0000000000..9abb6a2f84 --- /dev/null +++ b/tools/moztreedocs/docs/adding-documentation.rst @@ -0,0 +1,30 @@ +Adding Documentation +-------------------- + +To add new documentation, define the ``SPHINX_TREES`` and +``SPHINX_PYTHON_PACKAGE_DIRS`` variables in ``moz.build`` files in +the tree and documentation will automatically get picked up. + +Say you have a directory ``featureX`` you would like to write some +documentation for. Here are the steps to create Sphinx documentation +for it: + +1. Create a directory for the docs. This is typically ``docs``. e.g. + ``featureX/docs``. +2. Create an ``index.rst`` file in this directory. The ``index.rst`` file + is the root documentation for that section. See ``build/docs/index.rst`` + for an example file. +3. In a ``moz.build`` file (typically the one in the parent directory of + the ``docs`` directory), define ``SPHINX_TREES`` to hook up the plumbing. + e.g. ``SPHINX_TREES['featureX'] = 'docs'``. This says *the ``docs`` + directory under the current directory should be installed into the + Sphinx documentation tree under ``/featureX``*. +4. If you have Python packages you would like to generate Python API + documentation for, you can use ``SPHINX_PYTHON_PACKAGE_DIRS`` to + declare directories containing Python packages. e.g. + ``SPHINX_PYTHON_PACKAGE_DIRS += ['mozpackage']``. +5. In ``docs/config.yml``, defines in which category the doc + should go. +6. Verify the rst syntax using `./mach lint -l rst`_ + +.. _./mach lint -l rst: /tools/lint/linters/rstlinter.html diff --git a/tools/moztreedocs/docs/architecture.rst b/tools/moztreedocs/docs/architecture.rst new file mode 100644 index 0000000000..fc502f847f --- /dev/null +++ b/tools/moztreedocs/docs/architecture.rst @@ -0,0 +1,51 @@ +Documentation architecture +========================== + +The documentation relies on Sphinx and many Sphinx extensions. + +The documentation code is in two main directories: + +* https://searchfox.org/mozilla-central/source/docs +* https://searchfox.org/mozilla-central/source/tools/moztreedocs + +Our documentation supports both rst & markdown syntaxes. + +Configuration +------------- + +The main configuration file is: + +https://searchfox.org/mozilla-central/source/docs/config.yml + +It contains the categories, the redirects, the warnings and others configuration aspects. + +The dependencies are listed in: + +https://searchfox.org/mozilla-central/source/tools/moztreedocs/requirements.in + +Be aware that Python libraries stored in `third_party/python` are used in priority (not always for good reasons). See :ref:`Vendor the source of the Python package in-tree <python-vendor>` for more details. + + +Architecture +------------ + + +`mach_commands <https://searchfox.org/mozilla-central/source/tools/moztreedocs/mach_commands.py>`__ +contains: + +* `mach doc` arguments managements +* Detection/configuration of the environment (nodejs for jsdoc, pip for dependencies, etc) +* Symlink the doc sources (.rst & .md) from the source tree into the staging directory +* Fails the build if any critical warnings have been identified +* Starts the sphinx build (and serve it if the option is set) +* Manages telemetry + +`docs/conf.py <https://searchfox.org/mozilla-central/source/docs/conf.py>`__ defines: + +* The list of extensions +* JS source paths +* Various sphinx configuration + +At the end of the build documentation process, files will be uploaded to a CDN: + +https://searchfox.org/mozilla-central/source/tools/moztreedocs/upload.py diff --git a/tools/moztreedocs/docs/index.rst b/tools/moztreedocs/docs/index.rst new file mode 100644 index 0000000000..fc03756fbe --- /dev/null +++ b/tools/moztreedocs/docs/index.rst @@ -0,0 +1,24 @@ +Managing Documentation +====================== + +Documentation is hard. It's difficult to write, difficult to find and always out +of date. That's why we implemented our in-tree documentation system that +underpins firefox-source-docs.mozilla.org. The documentation lives next to the +code that it documents, so it can be updated within the same commit that makes +the underlying changes. + +This documentation is generated via the +`Sphinx <http://sphinx-doc.org/>`_ tool from sources in the tree. + +To build the documentation, run ``mach doc``. Run +``mach help doc`` to see configurable options. + +The review group in Phabricator is ``#firefox-source-docs-reviewers``. +For simple documentation changes, reviews are not required. + +.. toctree:: + :caption: Documentation + :maxdepth: 2 + :glob: + + * diff --git a/tools/moztreedocs/docs/jsdoc-support.rst b/tools/moztreedocs/docs/jsdoc-support.rst new file mode 100644 index 0000000000..100fb92dac --- /dev/null +++ b/tools/moztreedocs/docs/jsdoc-support.rst @@ -0,0 +1,16 @@ +jsdoc support +============= + +Here is a quick example, for the public AddonManager :ref:`API <AddonManager Reference>` + +To use it for your own code: + +#. Check that JSDoc generates the output you expect (you may need to use a @class annotation on "object initializer"-style class definitions for instance) + +#. Create an `.rst file`, which may contain explanatory text as well as the API docs. The minimum will look something like + `this <https://firefox-source-docs.mozilla.org/_sources/toolkit/mozapps/extensions/addon-manager/AddonManager.rst.txt>`__ + +#. Ensure your component is on the js_source_path here in the sphinx + config: https://hg.mozilla.org/mozilla-central/file/72ee4800d415/tools/docs/conf.py#l46 + +#. Run `mach doc` locally to generate the output and confirm that it looks correct. diff --git a/tools/moztreedocs/docs/mdn-import.rst b/tools/moztreedocs/docs/mdn-import.rst new file mode 100644 index 0000000000..69d2f56df4 --- /dev/null +++ b/tools/moztreedocs/docs/mdn-import.rst @@ -0,0 +1,34 @@ +Importing documentation from MDN +-------------------------------- + +As MDN should not be used for documenting mozilla-central specific code or process, +the documentation should be migrated in this repository. + +The meta bug is `Bug 1617963 <https://bugzilla.mozilla.org/show_bug.cgi?id=migrate-from-mdn>`__. + +Fortunately, there is an easy way to import the doc from MDN using GitHub +to the firefox source docs. + +1. Install https://pandoc.org/ - If you are using packages provided by your distribution, + make sure that the version is not too old. + +2. Identify where your page is located on the GitHub repository ( https://github.com/mdn/archived-content/tree/main/files/en-us/mozilla ). + Get the raw URL + +3. Run pandoc the following way: + +.. code-block:: shell + + $ pandoc -t rst https://github.com/mdn/archived-content/tree/main/files/en-us/mozilla/firefox/performance_best_practices_for_firefox_fe_engineers > doc.rst + +4. In the new doc.rst, identify the images and wget/curl them into `img/`. + +5. Verify the rst syntax using `./mach lint -l rst`_ + +.. _./mach lint -l rst: /tools/lint/linters/rstlinter.html + +6. If relevant, remove unbreakable spaces (rendered with a "!" on Phabricator) + +.. code-block:: shell + + $ sed -i -e 's/\xc2\xa0/ /g' doc.rst diff --git a/tools/moztreedocs/docs/mermaid-integration.rst b/tools/moztreedocs/docs/mermaid-integration.rst new file mode 100644 index 0000000000..ec0da3dd74 --- /dev/null +++ b/tools/moztreedocs/docs/mermaid-integration.rst @@ -0,0 +1,104 @@ +Mermaid Integration +=================== + +Mermaid is a tool that lets you generate flow charts, sequence diagrams, gantt +charts, class diagrams and vcs graphs from a simple markup language. This +allows charts and diagrams to be embedded and edited directly in the +documentation source files rather than creating them as images using some +external tool and checking the images into the tree. + +To add a diagram, simply put something like this into your page: + +.. These two examples come from the upstream website (https://mermaid-js.github.io/mermaid/#/) + +.. code-block:: rst + :caption: .rst + + .. mermaid:: + + graph TD; + A-->B; + A-->C; + B-->D; + C-->D; + +.. code-block:: md + :caption: .md + + ```{mermaid} + graph TD; + A-->B; + A-->C; + B-->D; + C-->D; + ``` + +The result will be: + +.. mermaid:: + + graph TD; + A-->B; + A-->C; + B-->D; + C-->D; + +Or + +.. code-block:: rst + :caption: .rst + + .. mermaid:: + + sequenceDiagram + participant Alice + participant Bob + Alice->>John: Hello John, how are you? + loop Healthcheck + John->>John: Fight against hypochondria + end + Note right of John: Rational thoughts <br/>prevail! + John-->>Alice: Great! + John->>Bob: How about you? + Bob-->>John: Jolly good! + +.. code-block:: markdown + :caption: .md + + ```{mermaid} + sequenceDiagram + participant Alice + participant Bob + Alice->>John: Hello John, how are you? + loop Healthcheck + John->>John: Fight against hypochondria + end + Note right of John: Rational thoughts <br/>prevail! + John-->>Alice: Great! + John->>Bob: How about you? + Bob-->>John: Jolly good! + ``` + + + +will show: + +.. mermaid:: + + sequenceDiagram + participant Alice + participant Bob + Alice->>John: Hello John, how are you? + loop Healthcheck + John->>John: Fight against hypochondria + end + Note right of John: Rational thoughts <br/>prevail! + John-->>Alice: Great! + John->>Bob: How about you? + Bob-->>John: Jolly good! + + +See `Mermaid's official <https://mermaid-js.github.io/mermaid/#/>`__ docs for +more details on the syntax, and use the +`Mermaid Live Editor <https://mermaidjs.github.io/mermaid-live-editor/>`__ to +experiment with creating your own diagrams. diff --git a/tools/moztreedocs/docs/nested-docs.rst b/tools/moztreedocs/docs/nested-docs.rst new file mode 100644 index 0000000000..e2eb03b42d --- /dev/null +++ b/tools/moztreedocs/docs/nested-docs.rst @@ -0,0 +1,14 @@ +Nested Doc Trees +================ + +This feature essentially means we can now group related docs together under +common "landing pages". This will allow us to refactor the docs into a structure that makes more sense. For example we could have a landing page for docs describing Gecko's internals, and another one for docs describing developer workflows in `mozilla-central`. + + +To clarify a few things: + +#. The path specified in `SPHINX_TREES` does not need to correspond to a path in `mozilla-central`. For example, I could register my docs using `SPHINX_TREES["/foo"] = "docs"`, which would make that doc tree accessible at `firefox-source-docs.mozilla.org/foo`. + +#. Any subtrees that are nested under another index will automatically be hidden from the main index. This means you should make sure to link to any subtrees from somewhere in the landing page. So given my earlier doc tree at `/foo`, if I now created a subtree and registered it using `SPHINX_TREES["/foo/bar"] = "docs"`, those docs would not show up in the main index. + +#. The relation between subtrees and their parents does not necessarily have any bearing with their relation on the file system. For example, a doc tree that lives under `/devtools` can be nested under an index that lives under `/browser`. diff --git a/tools/moztreedocs/docs/redirect.rst b/tools/moztreedocs/docs/redirect.rst new file mode 100644 index 0000000000..6ec29cdfd0 --- /dev/null +++ b/tools/moztreedocs/docs/redirect.rst @@ -0,0 +1,11 @@ +Redirects +========= + +We now have the ability to define redirects in-tree! This will allow us to +refactor and move docs around to our hearts content without needing to worry +about stale external URLs. To set up a redirect simply add a line to this file under ``redirects`` key: + +https://searchfox.org/mozilla-central/source/docs/config.yml + +Any request starting with the prefix on the left, will be rewritten to the prefix on the right by the server. So for example a request to +``/testing/marionette/marionette/index.html`` will be re-written to ``/testing/marionette/index.html``. Amazon's API only supports prefix redirects, so anything more complex isn't supported. diff --git a/tools/moztreedocs/docs/rstlint.rst b/tools/moztreedocs/docs/rstlint.rst new file mode 100644 index 0000000000..230ba2e812 --- /dev/null +++ b/tools/moztreedocs/docs/rstlint.rst @@ -0,0 +1,12 @@ +ReStructuredText Linter +----------------------- + +RST isn't the easiest of markup languages, but it's powerful and what `Sphinx` (the library used to build our docs) uses, so we're stuck with it. But at least we now have a linter which will catch basic problems in `.rst` files early. Be sure to run: + +.. code-block:: shell + + mach lint -l rst + +to test your outgoing changes before submitting to review. + +`More information <RST Linter>`__. diff --git a/tools/moztreedocs/docs/run-try-job.rst b/tools/moztreedocs/docs/run-try-job.rst new file mode 100644 index 0000000000..d7fe6b20b8 --- /dev/null +++ b/tools/moztreedocs/docs/run-try-job.rst @@ -0,0 +1,27 @@ +Running a try job for Documentation +----------------------------------- + +Documentation has two try jobs associated: + + - ``doc-generate`` - This generates the documentation with the committed changes on the try server and gives the same output as if it has landed on regular integration branch. + + .. code-block:: shell + + mach try fuzzy -q "'doc-generate" + + - ``doc-upload`` - This uploads documentation to `gecko-l1 bucket <http://gecko-docs.mozilla.org-l1.s3.us-west-2.amazonaws.com/index.html>`__ with the committed changes. + + .. code-block:: shell + + mach try fuzzy -q "'doc-upload" + +When the documentation is modified, at review phase, reviewbot will automatically generate a temporary documentation with a direct link to the modified pages. + +.. important:: + + Running try jobs require the user to have try server access. + +.. note:: + + To learn more about setting up try server or + using a different selector head over to :ref:`try server documentation <Pushing to Try>` diff --git a/tools/moztreedocs/docs/server-synchronization.rst b/tools/moztreedocs/docs/server-synchronization.rst new file mode 100644 index 0000000000..b47b66503e --- /dev/null +++ b/tools/moztreedocs/docs/server-synchronization.rst @@ -0,0 +1,5 @@ +Server Synchronization +====================== + +We now compare all the files that exist on the server against the list of source files in `mozilla-central`. +Any files on the server that no longer exist in `mozilla-central` are removed. diff --git a/tools/moztreedocs/mach_commands.py b/tools/moztreedocs/mach_commands.py new file mode 100644 index 0000000000..ad63ae0e44 --- /dev/null +++ b/tools/moztreedocs/mach_commands.py @@ -0,0 +1,539 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import fnmatch +import json +import multiprocessing +import os +import re +import subprocess +import sys +import tempfile +import time +import uuid +from functools import partial +from pprint import pprint + +import mozpack.path as mozpath +import sentry_sdk +import yaml +from mach.decorators import Command, CommandArgument, SubCommand +from mach.registrar import Registrar +from mozbuild.util import memoize +from mozfile import load_source + +here = os.path.abspath(os.path.dirname(__file__)) +topsrcdir = os.path.abspath(os.path.dirname(os.path.dirname(here))) +DOC_ROOT = os.path.join(topsrcdir, "docs") +BASE_LINK = "http://gecko-docs.mozilla.org-l1.s3-website.us-west-2.amazonaws.com/" + + +# Helps manage in-tree documentation. + + +@Command( + "doc", + category="devenv", + virtualenv_name="docs", + description="Generate and serve documentation from the tree.", +) +@CommandArgument( + "path", + default=None, + metavar="DIRECTORY", + nargs="?", + help="Path to documentation to build and display.", +) +@CommandArgument( + "--format", default="html", dest="fmt", help="Documentation format to write." +) +@CommandArgument( + "--outdir", default=None, metavar="DESTINATION", help="Where to write output." +) +@CommandArgument( + "--archive", + action="store_true", + help="Write a gzipped tarball of generated docs.", +) +@CommandArgument( + "--no-open", + dest="auto_open", + default=True, + action="store_false", + help="Don't automatically open HTML docs in a browser.", +) +@CommandArgument( + "--no-serve", + dest="serve", + default=True, + action="store_false", + help="Don't serve the generated docs after building.", +) +@CommandArgument( + "--http", + default="localhost:5500", + metavar="ADDRESS", + help="Serve documentation on the specified host and port, " + 'default "localhost:5500".', +) +@CommandArgument("--upload", action="store_true", help="Upload generated files to S3.") +@CommandArgument( + "-j", + "--jobs", + default=str(multiprocessing.cpu_count()), + dest="jobs", + help="Distribute the build over N processes in parallel.", +) +@CommandArgument("--write-url", default=None, help="Write S3 Upload URL to text file") +@CommandArgument( + "--linkcheck", action="store_true", help="Check if the links are still valid" +) +@CommandArgument( + "--dump-trees", default=None, help="Dump the Sphinx trees to specified file." +) +@CommandArgument( + "--fatal-warnings", + dest="enable_fatal_warnings", + action="store_true", + help="Enable fatal warnings.", +) +@CommandArgument( + "--check-num-warnings", + action="store_true", + help="Check that the upper bound on the number of warnings is respected.", +) +@CommandArgument("--verbose", action="store_true", help="Run Sphinx in verbose mode") +@CommandArgument( + "--no-autodoc", + action="store_true", + help="Disable generating Python/JS API documentation", +) +def build_docs( + command_context, + path=None, + fmt="html", + outdir=None, + auto_open=True, + serve=True, + http=None, + archive=False, + upload=False, + jobs=None, + write_url=None, + linkcheck=None, + dump_trees=None, + enable_fatal_warnings=False, + check_num_warnings=False, + verbose=None, + no_autodoc=False, +): + # TODO: Bug 1704891 - move the ESLint setup tools to a shared place. + import setup_helper + + setup_helper.set_project_root(command_context.topsrcdir) + + if not setup_helper.check_node_executables_valid(): + return 1 + + setup_helper.eslint_maybe_setup() + + # Set the path so that Sphinx can find jsdoc, unfortunately there isn't + # a way to pass this to Sphinx itself at the moment. + os.environ["PATH"] = ( + mozpath.join(command_context.topsrcdir, "node_modules", ".bin") + + os.pathsep + + _node_path() + + os.pathsep + + os.environ["PATH"] + ) + + import webbrowser + + from livereload import Server + + from moztreedocs.package import create_tarball + + unique_id = "%s/%s" % (project(), str(uuid.uuid1())) + + outdir = outdir or os.path.join(command_context.topobjdir, "docs") + savedir = os.path.join(outdir, fmt) + + if path is None: + path = command_context.topsrcdir + if os.environ.get("MOZ_AUTOMATION") != "1": + print( + "\nBuilding the full documentation tree.\n" + "Did you mean to only build part of the documentation?\n" + "For a faster command, consider running:\n" + " ./mach doc path/to/docs\n" + ) + path = os.path.normpath(os.path.abspath(path)) + + docdir = _find_doc_dir(path) + if not docdir: + print(_dump_sphinx_backtrace()) + return die( + "failed to generate documentation:\n" + "%s: could not find docs at this location" % path + ) + + if linkcheck: + # We want to verify if the links are valid or not + fmt = "linkcheck" + if no_autodoc: + if check_num_warnings: + return die( + "'--no-autodoc' flag may not be used with '--check-num-warnings'" + ) + toggle_no_autodoc() + + status, warnings = _run_sphinx(docdir, savedir, fmt=fmt, jobs=jobs, verbose=verbose) + if status != 0: + print(_dump_sphinx_backtrace()) + return die( + "failed to generate documentation:\n" + "%s: sphinx return code %d" % (path, status) + ) + else: + print("\nGenerated documentation:\n%s" % savedir) + msg = "" + + if enable_fatal_warnings: + fatal_warnings = _check_sphinx_fatal_warnings(warnings) + if fatal_warnings: + msg += f"Error: Got fatal warnings:\n{''.join(fatal_warnings)}" + if check_num_warnings: + [num_new, num_actual] = _check_sphinx_num_warnings(warnings) + print("Logged %s warnings\n" % num_actual) + if num_new: + msg += f"Error: {num_new} new warnings have been introduced compared to the limit in docs/config.yml" + if msg: + return dieWithTestFailure(msg) + + # Upload the artifact containing the link to S3 + # This would be used by code-review to post the link to Phabricator + if write_url is not None: + unique_link = BASE_LINK + unique_id + "/index.html" + with open(write_url, "w") as fp: + fp.write(unique_link) + fp.flush() + print("Generated " + write_url) + + if dump_trees is not None: + parent = os.path.dirname(dump_trees) + if parent and not os.path.isdir(parent): + os.makedirs(parent) + with open(dump_trees, "w") as fh: + json.dump(manager().trees, fh) + + if archive: + archive_path = os.path.join(outdir, "%s.tar.gz" % project()) + create_tarball(archive_path, savedir) + print("Archived to %s" % archive_path) + + if upload: + _s3_upload(savedir, project(), unique_id, version()) + + if not serve: + index_path = os.path.join(savedir, "index.html") + if auto_open and os.path.isfile(index_path): + webbrowser.open(index_path) + return + + # Create livereload server. Any files modified in the specified docdir + # will cause a re-build and refresh of the browser (if open). + try: + host, port = http.split(":", 1) + port = int(port) + except ValueError: + return die("invalid address: %s" % http) + + server = Server() + + sphinx_trees = manager().trees or {savedir: docdir} + for _, src in sphinx_trees.items(): + run_sphinx = partial( + _run_sphinx, src, savedir, fmt=fmt, jobs=jobs, verbose=verbose + ) + server.watch(src, run_sphinx) + server.serve( + host=host, + port=port, + root=savedir, + open_url_delay=0.1 if auto_open else None, + ) + + +def _dump_sphinx_backtrace(): + """ + If there is a sphinx dump file, read and return + its content. + By default, it isn't displayed. + """ + pattern = "sphinx-err-*" + output = "" + tmpdir = "/tmp" + + if not os.path.isdir(tmpdir): + # Only run it on Linux + return + files = os.listdir(tmpdir) + for name in files: + if fnmatch.fnmatch(name, pattern): + pathFile = os.path.join(tmpdir, name) + stat = os.stat(pathFile) + output += "Name: {0} / Creation date: {1}\n".format( + pathFile, time.ctime(stat.st_mtime) + ) + with open(pathFile) as f: + output += f.read() + return output + + +def _run_sphinx(docdir, savedir, config=None, fmt="html", jobs=None, verbose=None): + import sphinx.cmd.build + + config = config or manager().conf_py_path + # When running sphinx with sentry, it adds significant overhead + # and makes the build generation very very very slow + # So, disable it to generate the doc faster + sentry_sdk.init(None) + warn_fd, warn_path = tempfile.mkstemp() + os.close(warn_fd) + try: + args = [ + "-T", + "-b", + fmt, + "-c", + os.path.dirname(config), + "-w", + warn_path, + docdir, + savedir, + ] + if jobs: + args.extend(["-j", jobs]) + if verbose: + args.extend(["-v", "-v"]) + print("Run sphinx with:") + print(args) + status = sphinx.cmd.build.build_main(args) + with open(warn_path) as warn_file: + warnings = warn_file.readlines() + return status, warnings + finally: + try: + os.unlink(warn_path) + except Exception as ex: + print(ex) + + +def _check_sphinx_fatal_warnings(warnings): + with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh: + fatal_warnings_src = yaml.safe_load(fh)["fatal warnings"] + fatal_warnings_regex = [re.compile(item) for item in fatal_warnings_src] + fatal_warnings = [] + for warning in warnings: + if any(item.search(warning) for item in fatal_warnings_regex): + fatal_warnings.append(warning) + return fatal_warnings + + +def _check_sphinx_num_warnings(warnings): + # warnings file contains other strings as well + num_warnings = len([w for w in warnings if "WARNING" in w]) + with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh: + max_num = yaml.safe_load(fh)["max_num_warnings"] + if num_warnings > max_num: + return [num_warnings - max_num, num_warnings] + return [0, num_warnings] + + +def manager(): + from moztreedocs import manager + + return manager + + +def toggle_no_autodoc(): + import moztreedocs + + moztreedocs._SphinxManager.NO_AUTODOC = True + + +@memoize +def _read_project_properties(): + path = os.path.normpath(manager().conf_py_path) + conf = load_source("doc_conf", path) + + # Prefer the Mozilla project name, falling back to Sphinx's + # default variable if it isn't defined. + project = getattr(conf, "moz_project_name", None) + if not project: + project = conf.project.replace(" ", "_") + + return {"project": project, "version": getattr(conf, "version", None)} + + +def project(): + return _read_project_properties()["project"] + + +def version(): + return _read_project_properties()["version"] + + +def _node_path(): + from mozbuild.nodeutil import find_node_executable + + node, _ = find_node_executable() + + return os.path.dirname(node) + + +def _find_doc_dir(path): + if os.path.isfile(path): + return + + valid_doc_dirs = ("doc", "docs") + for d in valid_doc_dirs: + p = os.path.join(path, d) + if os.path.isdir(p): + path = p + + for index_file in ["index.rst", "index.md"]: + if os.path.exists(os.path.join(path, index_file)): + return path + + +def _s3_upload(root, project, unique_id, version=None): + # Workaround the issue + # BlockingIOError: [Errno 11] write could not complete without blocking + # https://github.com/travis-ci/travis-ci/issues/8920 + import fcntl + + from moztreedocs.package import distribution_files + from moztreedocs.upload import s3_set_redirects, s3_upload + + fcntl.fcntl(1, fcntl.F_SETFL, 0) + + # Files are uploaded to multiple locations: + # + # <project>/latest + # <project>/<version> + # + # This allows multiple projects and versions to be stored in the + # S3 bucket. + + files = list(distribution_files(root)) + key_prefixes = [] + if version: + key_prefixes.append("%s/%s" % (project, version)) + + # Until we redirect / to main/latest, upload the main docs + # to the root. + if project == "main": + key_prefixes.append("") + + key_prefixes.append(unique_id) + + with open(os.path.join(DOC_ROOT, "config.yml"), "r") as fh: + redirects = yaml.safe_load(fh)["redirects"] + + redirects = {k.strip("/"): v.strip("/") for k, v in redirects.items()} + + all_redirects = {} + + for prefix in key_prefixes: + s3_upload(files, prefix) + + # Don't setup redirects for the "version" or "uuid" prefixes since + # we are exceeding a 50 redirect limit and external things are + # unlikely to link there anyway (see bug 1614908). + if (version and prefix.endswith(version)) or prefix == unique_id: + continue + + if prefix: + prefix += "/" + all_redirects.update({prefix + k: prefix + v for k, v in redirects.items()}) + + print("Redirects currently staged") + pprint(all_redirects, indent=1) + + s3_set_redirects(all_redirects) + + unique_link = BASE_LINK + unique_id + "/index.html" + print("Uploaded documentation can be accessed here " + unique_link) + + +@SubCommand( + "doc", + "mach-telemetry", + description="Generate documentation from Glean metrics.yaml files", +) +def generate_telemetry_docs(command_context): + args = [ + sys.executable, + "-m" "glean_parser", + "translate", + "-f", + "markdown", + "-o", + os.path.join(topsrcdir, "python/mach/docs/"), + os.path.join(topsrcdir, "python/mach/pings.yaml"), + os.path.join(topsrcdir, "python/mach/metrics.yaml"), + ] + metrics_paths = [ + handler.metrics_path + for handler in Registrar.command_handlers.values() + if handler.metrics_path is not None + ] + args.extend( + [os.path.join(command_context.topsrcdir, path) for path in set(metrics_paths)] + ) + subprocess.check_call(args) + + +@SubCommand( + "doc", + "show-targets", + description="List all reference targets. Requires the docs to have been built.", +) +@CommandArgument( + "--format", default="html", dest="fmt", help="Documentation format used." +) +@CommandArgument( + "--outdir", default=None, metavar="DESTINATION", help="Where output was written." +) +def show_reference_targets(command_context, fmt="html", outdir=None): + command_context.activate_virtualenv() + command_context.virtualenv_manager.install_pip_requirements( + os.path.join(here, "requirements.txt") + ) + + import sphinx.ext.intersphinx + + outdir = outdir or os.path.join(command_context.topobjdir, "docs") + inv_path = os.path.join(outdir, fmt, "objects.inv") + + if not os.path.exists(inv_path): + return die( + "object inventory not found: {inv_path}.\n" + "Rebuild the docs and rerun this command" + ) + sphinx.ext.intersphinx.inspect_main([inv_path]) + + +def die(msg, exit_code=1): + msg = "%s %s: %s" % (sys.argv[0], sys.argv[1], msg) + print(msg, file=sys.stderr) + return exit_code + + +def dieWithTestFailure(msg, exit_code=1): + for m in msg.split("\n"): + msg = "TEST-UNEXPECTED-FAILURE | %s %s | %s" % (sys.argv[0], sys.argv[1], m) + print(msg, file=sys.stderr) + return exit_code diff --git a/tools/moztreedocs/package.py b/tools/moztreedocs/package.py new file mode 100644 index 0000000000..b8db23ee87 --- /dev/null +++ b/tools/moztreedocs/package.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from mozpack.archive import create_tar_gz_from_files +from mozpack.files import FileFinder + + +def distribution_files(root): + """Find all files suitable for distributing. + + Given the path to generated Sphinx documentation, returns an iterable + of (path, BaseFile) for files that should be archived, uploaded, etc. + Paths are relative to given root directory. + """ + finder = FileFinder(root, ignore=("_staging", "_venv")) + return finder.find("**") + + +def create_tarball(filename, root): + """Create a tar.gz archive of docs in a directory.""" + files = dict(distribution_files(root)) + + with open(filename, "wb") as fh: + create_tar_gz_from_files( + fh, files, filename=os.path.basename(filename), compresslevel=6 + ) diff --git a/tools/moztreedocs/upload.py b/tools/moztreedocs/upload.py new file mode 100644 index 0000000000..a607135174 --- /dev/null +++ b/tools/moztreedocs/upload.py @@ -0,0 +1,171 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import io +import mimetypes +import os +import sys +from concurrent import futures +from pprint import pprint + +import boto3 +import botocore +import requests +from mozbuild.util import memoize + + +@memoize +def create_aws_session(): + """ + This function creates an aws session that is + shared between upload and delete both. + """ + region = "us-west-2" + level = os.environ.get("MOZ_SCM_LEVEL", "1") + bucket = { + "1": "gecko-docs.mozilla.org-l1", + "2": "gecko-docs.mozilla.org-l2", + "3": "gecko-docs.mozilla.org", + }[level] + secrets_url = "http://taskcluster/secrets/v1/secret/" + secrets_url += "project/releng/gecko/build/level-{}/gecko-docs-upload".format(level) + + # Get the credentials from the TC secrets service. Note that these + # differ per SCM level + if "TASK_ID" in os.environ: + print("Using AWS credentials from the secrets service") + session = requests.Session() + res = session.get(secrets_url) + res.raise_for_status() + secret = res.json()["secret"] + session = boto3.session.Session( + aws_access_key_id=secret["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=secret["AWS_SECRET_ACCESS_KEY"], + region_name=region, + ) + else: + print("Trying to use your AWS credentials..") + session = boto3.session.Session(region_name=region) + + s3 = session.client("s3", config=botocore.client.Config(max_pool_connections=20)) + + return s3, bucket + + +@memoize +def get_s3_keys(s3, bucket): + kwargs = {"Bucket": bucket} + all_keys = [] + while True: + response = s3.list_objects_v2(**kwargs) + for obj in response["Contents"]: + all_keys.append(obj["Key"]) + + try: + kwargs["ContinuationToken"] = response["NextContinuationToken"] + except KeyError: + break + + return all_keys + + +def s3_set_redirects(redirects): + s3, bucket = create_aws_session() + + configuration = {"IndexDocument": {"Suffix": "index.html"}, "RoutingRules": []} + + for path, redirect in redirects.items(): + rule = { + "Condition": {"KeyPrefixEquals": path}, + "Redirect": {"ReplaceKeyPrefixWith": redirect}, + } + if os.environ.get("MOZ_SCM_LEVEL") == "3": + rule["Redirect"]["HostName"] = "firefox-source-docs.mozilla.org" + + configuration["RoutingRules"].append(rule) + + s3.put_bucket_website( + Bucket=bucket, + WebsiteConfiguration=configuration, + ) + + +def s3_delete_missing(files, key_prefix=None): + """Delete files in the S3 bucket. + + Delete files on the S3 bucket that doesn't match the files + given as the param. If the key_prefix is not specified, missing + files that has main/ as a prefix will be removed. Otherwise, it + will remove files with the same prefix as key_prefix. + """ + s3, bucket = create_aws_session() + files_on_server = get_s3_keys(s3, bucket) + if key_prefix: + files_on_server = [ + path for path in files_on_server if path.startswith(key_prefix) + ] + else: + files_on_server = [ + path for path in files_on_server if not path.startswith("main/") + ] + files = [key_prefix + "/" + path if key_prefix else path for path, f in files] + files_to_delete = [path for path in files_on_server if path not in files] + + query_size = 1000 + while files_to_delete: + keys_to_remove = [{"Key": key} for key in files_to_delete[:query_size]] + response = s3.delete_objects( + Bucket=bucket, + Delete={ + "Objects": keys_to_remove, + }, # NOQA + ) + pprint(response, indent=2) + files_to_delete = files_to_delete[query_size:] + + +def s3_upload(files, key_prefix=None): + """Upload files to an S3 bucket. + + ``files`` is an iterable of ``(path, BaseFile)`` (typically from a + mozpack Finder). + + Keys in the bucket correspond to source filenames. If ``key_prefix`` is + defined, key names will be ``<key_prefix>/<path>``. + """ + s3, bucket = create_aws_session() + + def upload(f, path, bucket, key, extra_args): + # Need to flush to avoid buffering/interleaving from multiple threads. + sys.stdout.write("uploading %s to %s\n" % (path, key)) + sys.stdout.flush() + s3.upload_fileobj(f, bucket, key, ExtraArgs=extra_args) + + fs = [] + with futures.ThreadPoolExecutor(20) as e: + for path, f in files: + content_type, content_encoding = mimetypes.guess_type(path) + extra_args = {} + if content_type: + if content_type.startswith("text/"): + content_type += '; charset="utf-8"' + extra_args["ContentType"] = content_type + if content_encoding: + extra_args["ContentEncoding"] = content_encoding + + if key_prefix: + key = "%s/%s" % (key_prefix, path) + else: + key = path + + # The file types returned by mozpack behave like file objects. But + # they don't accept an argument to read(). So we wrap in a BytesIO. + fs.append( + e.submit(upload, io.BytesIO(f.read()), path, bucket, key, extra_args) + ) + + s3_delete_missing(files, key_prefix) + # Need to do this to catch any exceptions. + for f in fs: + f.result() diff --git a/tools/performance/PerfStats.cpp b/tools/performance/PerfStats.cpp new file mode 100644 index 0000000000..d15977ddbf --- /dev/null +++ b/tools/performance/PerfStats.cpp @@ -0,0 +1,317 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "PerfStats.h" +#include "nsAppRunner.h" +#include "mozilla/dom/BrowserParent.h" +#include "mozilla/dom/CanonicalBrowsingContext.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/dom/ContentProcessManager.h" +#include "mozilla/dom/WindowGlobalParent.h" +#include "mozilla/gfx/GPUChild.h" +#include "mozilla/gfx/GPUProcessManager.h" +#include "mozilla/JSONStringWriteFuncs.h" + +using namespace mozilla::dom; +using namespace mozilla::gfx; + +namespace mozilla { + +#define METRIC_NAME(metric) #metric, +static const char* const sMetricNames[] = { + FOR_EACH_PERFSTATS_METRIC(METRIC_NAME) +#undef METRIC_NAME + "Invalid"}; + +PerfStats::MetricMask PerfStats::sCollectionMask = 0; +StaticMutex PerfStats::sMutex; +StaticAutoPtr<PerfStats> PerfStats::sSingleton; + +void PerfStats::SetCollectionMask(MetricMask aMask) { + sCollectionMask = aMask; + GetSingleton()->ResetCollection(); + + if (!XRE_IsParentProcess()) { + return; + } + + GPUProcessManager* gpuManager = GPUProcessManager::Get(); + GPUChild* gpuChild = nullptr; + + if (gpuManager) { + gpuChild = gpuManager->GetGPUChild(); + if (gpuChild) { + gpuChild->SendUpdatePerfStatsCollectionMask(aMask); + } + } + + nsTArray<ContentParent*> contentParents; + ContentParent::GetAll(contentParents); + + for (ContentParent* parent : contentParents) { + Unused << parent->SendUpdatePerfStatsCollectionMask(aMask); + } +} + +PerfStats::MetricMask PerfStats::GetCollectionMask() { return sCollectionMask; } + +PerfStats* PerfStats::GetSingleton() { + if (!sSingleton) { + sSingleton = new PerfStats; + } + + return sSingleton.get(); +} + +void PerfStats::RecordMeasurementStartInternal(Metric aMetric) { + StaticMutexAutoLock lock(sMutex); + + GetSingleton()->mRecordedStarts[static_cast<size_t>(aMetric)] = + TimeStamp::Now(); +} + +void PerfStats::RecordMeasurementEndInternal(Metric aMetric) { + StaticMutexAutoLock lock(sMutex); + + MOZ_ASSERT(sSingleton); + + sSingleton->mRecordedTimes[static_cast<size_t>(aMetric)] += + (TimeStamp::Now() - + sSingleton->mRecordedStarts[static_cast<size_t>(aMetric)]) + .ToMilliseconds(); + sSingleton->mRecordedCounts[static_cast<size_t>(aMetric)]++; +} + +void PerfStats::RecordMeasurementInternal(Metric aMetric, + TimeDuration aDuration) { + StaticMutexAutoLock lock(sMutex); + + MOZ_ASSERT(sSingleton); + + sSingleton->mRecordedTimes[static_cast<size_t>(aMetric)] += + aDuration.ToMilliseconds(); + sSingleton->mRecordedCounts[static_cast<size_t>(aMetric)]++; +} + +void PerfStats::RecordMeasurementCounterInternal(Metric aMetric, + uint64_t aIncrementAmount) { + StaticMutexAutoLock lock(sMutex); + + MOZ_ASSERT(sSingleton); + + sSingleton->mRecordedTimes[static_cast<size_t>(aMetric)] += + double(aIncrementAmount); + sSingleton->mRecordedCounts[static_cast<size_t>(aMetric)]++; +} + +void AppendJSONStringAsProperty(nsCString& aDest, const char* aPropertyName, + const nsACString& aJSON) { + // We need to manually append into the string here, since JSONWriter has no + // way to allow us to write an existing JSON object into a property. + aDest.Append(",\n\""); + aDest.Append(aPropertyName); + aDest.Append("\": "); + aDest.Append(aJSON); +} + +static void WriteContentParent(nsCString& aRawString, JSONWriter& aWriter, + const nsACString& aString, + ContentParent* aParent) { + aWriter.StringProperty("type", "content"); + aWriter.IntProperty("id", aParent->ChildID()); + const ManagedContainer<PBrowserParent>& browsers = + aParent->ManagedPBrowserParent(); + + aWriter.StartArrayProperty("urls"); + for (const auto& key : browsers) { + // This only reports -current- URLs, not ones that may have been here in + // the past, this is unfortunate especially for processes which are dying + // and that have no more active URLs. + RefPtr<BrowserParent> parent = BrowserParent::GetFrom(key); + + CanonicalBrowsingContext* ctx = parent->GetBrowsingContext(); + if (!ctx) { + continue; + } + + WindowGlobalParent* windowGlobal = ctx->GetCurrentWindowGlobal(); + if (!windowGlobal) { + continue; + } + + RefPtr<nsIURI> uri = windowGlobal->GetDocumentURI(); + if (!uri) { + continue; + } + + nsAutoCString url; + uri->GetSpec(url); + + aWriter.StringElement(url); + } + aWriter.EndArray(); + AppendJSONStringAsProperty(aRawString, "perfstats", aString); +} + +struct PerfStatsCollector { + PerfStatsCollector() : writer(MakeUnique<JSONStringRefWriteFunc>(string)) {} + + void AppendPerfStats(const nsCString& aString, ContentParent* aParent) { + writer.StartObjectElement(); + WriteContentParent(string, writer, aString, aParent); + writer.EndObject(); + } + + void AppendPerfStats(const nsCString& aString, GPUChild* aChild) { + writer.StartObjectElement(); + writer.StringProperty("type", "gpu"); + writer.IntProperty("id", aChild->Id()); + AppendJSONStringAsProperty(string, "perfstats", aString); + writer.EndObject(); + } + + ~PerfStatsCollector() { + writer.EndArray(); + writer.End(); + promise.Resolve(string, __func__); + } + nsCString string; + JSONWriter writer; + MozPromiseHolder<PerfStats::PerfStatsPromise> promise; +}; + +void PerfStats::ResetCollection() { + for (uint64_t i = 0; i < static_cast<uint64_t>(Metric::Max); i++) { + if (!(sCollectionMask & 1 << i)) { + continue; + } + + mRecordedTimes[i] = 0; + mRecordedCounts[i] = 0; + } + + mStoredPerfStats.Clear(); +} + +void PerfStats::StorePerfStatsInternal(dom::ContentParent* aParent, + const nsACString& aPerfStats) { + nsCString jsonString; + JSONStringRefWriteFunc jw(jsonString); + JSONWriter w(jw); + + // To generate correct JSON here we don't call start and end. That causes + // this to use Single Line mode, sadly. + WriteContentParent(jsonString, w, aPerfStats, aParent); + + mStoredPerfStats.AppendElement(jsonString); +} + +auto PerfStats::CollectPerfStatsJSONInternal() -> RefPtr<PerfStatsPromise> { + if (!PerfStats::sCollectionMask) { + return PerfStatsPromise::CreateAndReject(false, __func__); + } + + if (!XRE_IsParentProcess()) { + return PerfStatsPromise::CreateAndResolve( + CollectLocalPerfStatsJSONInternal(), __func__); + } + + std::shared_ptr<PerfStatsCollector> collector = + std::make_shared<PerfStatsCollector>(); + + JSONWriter& w = collector->writer; + + w.Start(); + { + w.StartArrayProperty("processes"); + { + w.StartObjectElement(); + { + w.StringProperty("type", "parent"); + AppendJSONStringAsProperty(collector->string, "perfstats", + CollectLocalPerfStatsJSONInternal()); + } + w.EndObject(); + + // Append any processes that closed earlier. + for (nsCString& string : mStoredPerfStats) { + w.StartObjectElement(); + // This trick makes indentation even more messed up than it already + // was. However it produces technically correct JSON. + collector->string.Append(string); + w.EndObject(); + } + // We do not clear this, we only clear stored perfstats when the mask is + // reset. + + GPUProcessManager* gpuManager = GPUProcessManager::Get(); + GPUChild* gpuChild = nullptr; + + if (gpuManager) { + gpuChild = gpuManager->GetGPUChild(); + } + nsTArray<ContentParent*> contentParents; + ContentParent::GetAll(contentParents); + + if (gpuChild) { + gpuChild->SendCollectPerfStatsJSON( + [collector, gpuChild = RefPtr{gpuChild}](const nsCString& aString) { + collector->AppendPerfStats(aString, gpuChild); + }, + // The only feasible errors here are if something goes wrong in the + // the bridge, we choose to ignore those. + [](mozilla::ipc::ResponseRejectReason) {}); + } + for (ContentParent* parent : contentParents) { + RefPtr<ContentParent> parentRef = parent; + parent->SendCollectPerfStatsJSON( + [collector, parentRef](const nsCString& aString) { + collector->AppendPerfStats(aString, parentRef.get()); + }, + // The only feasible errors here are if something goes wrong in the + // the bridge, we choose to ignore those. + [](mozilla::ipc::ResponseRejectReason) {}); + } + } + } + + return collector->promise.Ensure(__func__); +} + +nsCString PerfStats::CollectLocalPerfStatsJSONInternal() { + StaticMutexAutoLock lock(PerfStats::sMutex); + + nsCString jsonString; + + JSONStringRefWriteFunc jw(jsonString); + JSONWriter w(jw); + w.Start(); + { + w.StartArrayProperty("metrics"); + { + for (uint64_t i = 0; i < static_cast<uint64_t>(Metric::Max); i++) { + if (!(sCollectionMask & (1 << i))) { + continue; + } + + w.StartObjectElement(); + { + w.IntProperty("id", i); + w.StringProperty("metric", MakeStringSpan(sMetricNames[i])); + w.DoubleProperty("time", mRecordedTimes[i]); + w.IntProperty("count", mRecordedCounts[i]); + } + w.EndObject(); + } + } + w.EndArray(); + } + w.End(); + + return jsonString; +} + +} // namespace mozilla diff --git a/tools/performance/PerfStats.h b/tools/performance/PerfStats.h new file mode 100644 index 0000000000..a1f7e37fdd --- /dev/null +++ b/tools/performance/PerfStats.h @@ -0,0 +1,171 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef PerfStats_h +#define PerfStats_h + +#include "mozilla/TimeStamp.h" +#include "mozilla/StaticMutex.h" +#include "mozilla/StaticPtr.h" +#include "mozilla/MozPromise.h" +#include <memory> +#include <string> +#include <limits> + +// PerfStats +// +// Framework for low overhead selective collection of internal performance +// metrics through ChromeUtils. +// +// Gathering: in C++, wrap execution in an RAII class +// PerfStats::AutoMetricRecording<PerfStats::Metric::MyMetric> or call +// PerfStats::RecordMeasurement{Start,End} manually. Use +// RecordMeasurementCount() for incrementing counters. +// +// Controlling: Use ChromeUtils.SetPerfStatsCollectionMask(mask), where mask=0 +// disables all metrics and mask=0xFFFFFFFF enables all of them. +// +// Reporting: Results can be accessed with ChromeUtils.CollectPerfStats(). +// Browsertime will sum results across processes and report them. + +// Define a new metric by adding it to this list. It will be created as a class +// enum value mozilla::PerfStats::Metric::MyMetricName. +#define FOR_EACH_PERFSTATS_METRIC(MACRO) \ + MACRO(DisplayListBuilding) \ + MACRO(Rasterizing) \ + MACRO(WrDisplayListBuilding) \ + MACRO(LayerTransactions) \ + MACRO(Compositing) \ + MACRO(Reflowing) \ + MACRO(Styling) \ + MACRO(HttpChannelCompletion) \ + MACRO(HttpChannelCompletion_Network) \ + MACRO(HttpChannelCompletion_Cache) \ + MACRO(HttpChannelAsyncOpenToTransactionPending) \ + MACRO(HttpChannelResponseStartParentToContent) \ + MACRO(HttpChannelResponseEndParentToContent) \ + MACRO(HttpTransactionWaitTime) \ + MACRO(ResponseEndSocketToParent) \ + MACRO(OnStartRequestSocketToParent) \ + MACRO(OnDataAvailableSocketToParent) \ + MACRO(OnStopRequestSocketToParent) \ + MACRO(OnStartRequestToContent) \ + MACRO(OnDataAvailableToContent) \ + MACRO(OnStopRequestToContent) \ + MACRO(JSBC_Compression) \ + MACRO(JSBC_Decompression) \ + MACRO(JSBC_IO_Read) \ + MACRO(JSBC_IO_Write) \ + MACRO(MinorGC) \ + MACRO(MajorGC) \ + MACRO(NonIdleMajorGC) \ + MACRO(A11Y_DoInitialUpdate) \ + MACRO(A11Y_ProcessQueuedCacheUpdate) + +namespace mozilla { + +namespace dom { +// Forward declaration. +class ContentParent; +} // namespace dom + +class PerfStats { + public: + typedef MozPromise<nsCString, bool, true> PerfStatsPromise; + + enum class Metric : uint32_t { +#define DECLARE_ENUM(metric) metric, + FOR_EACH_PERFSTATS_METRIC(DECLARE_ENUM) +#undef DECLARE_ENUM + Max + }; + + // MetricMask is a bitmask based on 'Metric', i.e. Metric::LayerBuilding (2) + // is synonymous to 1 << 2 in MetricMask. + using MetricMask = uint64_t; + + static void RecordMeasurementStart(Metric aMetric) { + if (!(sCollectionMask & (1 << static_cast<uint64_t>(aMetric)))) { + return; + } + RecordMeasurementStartInternal(aMetric); + } + + static void RecordMeasurementEnd(Metric aMetric) { + if (!(sCollectionMask & (1 << static_cast<uint64_t>(aMetric)))) { + return; + } + RecordMeasurementEndInternal(aMetric); + } + + static void RecordMeasurement(Metric aMetric, TimeDuration aDuration) { + if (!(sCollectionMask & (1 << static_cast<uint64_t>(aMetric)))) { + return; + } + RecordMeasurementInternal(aMetric, aDuration); + } + + static void RecordMeasurementCounter(Metric aMetric, + uint64_t aIncrementAmount) { + if (!(sCollectionMask & (1 << static_cast<uint64_t>(aMetric)))) { + return; + } + RecordMeasurementCounterInternal(aMetric, aIncrementAmount); + } + + template <Metric N> + class AutoMetricRecording { + public: + AutoMetricRecording() { PerfStats::RecordMeasurementStart(N); } + ~AutoMetricRecording() { PerfStats::RecordMeasurementEnd(N); } + }; + + static void SetCollectionMask(MetricMask aMask); + static MetricMask GetCollectionMask(); + + static RefPtr<PerfStatsPromise> CollectPerfStatsJSON() { + return GetSingleton()->CollectPerfStatsJSONInternal(); + } + + static nsCString CollectLocalPerfStatsJSON() { + return GetSingleton()->CollectLocalPerfStatsJSONInternal(); + } + + static void StorePerfStats(dom::ContentParent* aParent, + const nsACString& aPerfStats) { + GetSingleton()->StorePerfStatsInternal(aParent, aPerfStats); + } + + private: + static PerfStats* GetSingleton(); + static void RecordMeasurementStartInternal(Metric aMetric); + static void RecordMeasurementEndInternal(Metric aMetric); + static void RecordMeasurementInternal(Metric aMetric, TimeDuration aDuration); + static void RecordMeasurementCounterInternal(Metric aMetric, + uint64_t aIncrementAmount); + + void ResetCollection(); + void StorePerfStatsInternal(dom::ContentParent* aParent, + const nsACString& aPerfStats); + RefPtr<PerfStatsPromise> CollectPerfStatsJSONInternal(); + nsCString CollectLocalPerfStatsJSONInternal(); + + static MetricMask sCollectionMask; + static StaticMutex sMutex MOZ_UNANNOTATED; + static StaticAutoPtr<PerfStats> sSingleton; + TimeStamp mRecordedStarts[static_cast<size_t>(Metric::Max)]; + double mRecordedTimes[static_cast<size_t>(Metric::Max)]; + uint32_t mRecordedCounts[static_cast<size_t>(Metric::Max)]; + nsTArray<nsCString> mStoredPerfStats; +}; + +static_assert(1 << (static_cast<uint64_t>(PerfStats::Metric::Max) - 1) <= + std::numeric_limits<PerfStats::MetricMask>::max(), + "More metrics than can fit into sCollectionMask bitmask"); + +} // namespace mozilla + +#endif // PerfStats_h diff --git a/tools/performance/moz.build b/tools/performance/moz.build new file mode 100644 index 0000000000..ea82c83126 --- /dev/null +++ b/tools/performance/moz.build @@ -0,0 +1,17 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + "PerfStats.cpp", +] + +EXPORTS.mozilla += [ + "PerfStats.h", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul" diff --git a/tools/phabricator/mach_commands.py b/tools/phabricator/mach_commands.py new file mode 100644 index 0000000000..657267a510 --- /dev/null +++ b/tools/phabricator/mach_commands.py @@ -0,0 +1,140 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozfile +from mach.decorators import Command, CommandArgument +from mach.site import MozSiteMetadata + + +@Command( + "install-moz-phab", + category="misc", + description="Install patch submission tool.", +) +@CommandArgument( + "--force", + "-f", + action="store_true", + help="Force installation even if already installed.", +) +def install_moz_phab(command_context, force=False): + import logging + import os + import re + import subprocess + import sys + + existing = mozfile.which("moz-phab") + if existing and not force: + command_context.log( + logging.ERROR, + "already_installed", + {}, + "moz-phab is already installed in %s." % existing, + ) + sys.exit(1) + + active_metadata = MozSiteMetadata.from_runtime() + original_python = active_metadata.original_python.python_path + is_external_python_virtualenv = ( + subprocess.check_output( + [ + original_python, + "-c", + "import sys; print(sys.prefix != sys.base_prefix)", + ] + ).strip() + == b"True" + ) + + # pip3 is part of Python since 3.4, however some distros choose to + # remove core components from languages, so show a useful error message + # if pip3 is missing. + has_pip = subprocess.run([original_python, "-c", "import pip"]).returncode == 0 + if not has_pip: + command_context.log( + logging.ERROR, + "pip_not_installed", + {}, + "Python 3's `pip` is not installed. Try installing it with your system " + "package manager.", + ) + sys.exit(1) + + command = [original_python, "-m", "pip", "install", "--upgrade", "MozPhab"] + + if ( + sys.platform.startswith("linux") + or sys.platform.startswith("openbsd") + or sys.platform.startswith("dragonfly") + or sys.platform.startswith("freebsd") + or sys.platform.startswith("netbsd") + ): + # On all Linux and BSD distros we consider doing a user installation. + platform_prefers_user_install = True + + elif sys.platform.startswith("darwin"): + # On MacOS we require brew or ports, which work better without --user. + platform_prefers_user_install = False + + elif sys.platform.startswith("win32") or sys.platform.startswith("msys"): + # Likewise for Windows we assume a system level install is preferred. + platform_prefers_user_install = False + + else: + # Unsupported, default to --user. + command_context.log( + logging.WARNING, + "unsupported_platform", + {}, + "Unsupported platform (%s), assuming per-user installation is " + "preferred." % sys.platform, + ) + platform_prefers_user_install = True + + command_env = os.environ.copy() + + if platform_prefers_user_install and not is_external_python_virtualenv: + # Virtual environments don't see user packages, so only perform a user + # installation if we're not within one. + command.append("--user") + # This is needed to work around a problem on Ubuntu 23.04 and Debian 12 + # See bug 1831442 for more details + command_env["PIP_BREAK_SYSTEM_PACKAGES"] = "1" + + command_context.log(logging.INFO, "run", {}, "Installing moz-phab") + subprocess.run(command, env=command_env) + + # There isn't an elegant way of determining the CLI location of a pip-installed package. + # The viable mechanism used here is to: + # 1. Get the list of info about the installed package via pip + # 2. Parse out the install location. This gives us the python environment in which the + # package is installed + # 3. Parse out the relative location of the cli script + # 4. Join the two paths, and execute the script at that location + + info = subprocess.check_output( + [original_python, "-m", "pip", "show", "-f", "MozPhab"], + universal_newlines=True, + ) + mozphab_package_location = re.compile(r"Location: (.*)").search(info).group(1) + # This needs to match "moz-phab" (*nix) and "moz-phab.exe" (Windows) while missing + # "moz-phab-script.py" (Windows). + potential_cli_paths = re.compile( + r"([^\s]*moz-phab(?:\.exe)?)$", re.MULTILINE + ).findall(info) + + if len(potential_cli_paths) != 1: + command_context.log( + logging.WARNING, + "no_mozphab_console_script", + {}, + "Could not find the CLI script for moz-phab. Skipping install-certificate step.", + ) + sys.exit(1) + + console_script = os.path.realpath( + os.path.join(mozphab_package_location, potential_cli_paths[0]) + ) + subprocess.run([console_script, "install-certificate"]) diff --git a/tools/power/mach_commands.py b/tools/power/mach_commands.py new file mode 100644 index 0000000000..117a96e1c7 --- /dev/null +++ b/tools/power/mach_commands.py @@ -0,0 +1,148 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mach.decorators import Command, CommandArgument +from packaging.version import Version + + +def is_osx_10_10_or_greater(cls): + import platform + + release = platform.mac_ver()[0] + return release and Version(release) >= Version("10.10") + + +# Get system power consumption and related measurements. +@Command( + "power", + category="misc", + conditions=[is_osx_10_10_or_greater], + description="Get system power consumption and related measurements for " + "all running browsers. Available only on Mac OS X 10.10 and above. " + "Requires root access.", +) +@CommandArgument( + "-i", + "--interval", + type=int, + default=30000, + help="The sample period, measured in milliseconds. Defaults to 30000.", +) +def power(command_context, interval): + """ + Get system power consumption and related measurements. + """ + import os + import re + import subprocess + + rapl = os.path.join(command_context.topobjdir, "dist", "bin", "rapl") + + interval = str(interval) + + # Run a trivial command with |sudo| to gain temporary root privileges + # before |rapl| and |powermetrics| are called. This ensures that |rapl| + # doesn't start measuring while |powermetrics| is waiting for the root + # password to be entered. + try: + subprocess.check_call(["sudo", "true"]) + except Exception: + print("\nsudo failed; aborting") + return 1 + + # This runs rapl in the background because nothing in this script + # depends on the output. This is good because we want |rapl| and + # |powermetrics| to run at the same time. + subprocess.Popen([rapl, "-n", "1", "-i", interval]) + + lines = subprocess.check_output( + [ + "sudo", + "powermetrics", + "--samplers", + "tasks", + "--show-process-coalition", + "--show-process-gpu", + "-n", + "1", + "-i", + interval, + ], + universal_newlines=True, + ) + + # When run with --show-process-coalition, |powermetrics| groups outputs + # into process coalitions, each of which has a leader. + # + # For example, when Firefox runs from the dock, its coalition looks + # like this: + # + # org.mozilla.firefox + # firefox + # plugin-container + # + # When Safari runs from the dock: + # + # com.apple.Safari + # Safari + # com.apple.WebKit.Networking + # com.apple.WebKit.WebContent + # com.apple.WebKit.WebContent + # + # When Chrome runs from the dock: + # + # com.google.Chrome + # Google Chrome + # Google Chrome Helper + # Google Chrome Helper + # + # In these cases, we want to print the whole coalition. + # + # Also, when you run any of them from the command line, things are the + # same except that the leader is com.apple.Terminal and there may be + # non-browser processes in the coalition, e.g.: + # + # com.apple.Terminal + # firefox + # plugin-container + # <and possibly other, non-browser processes> + # + # Also, the WindowServer and kernel coalitions and processes are often + # relevant. + # + # We want to print all these but omit uninteresting coalitions. We + # could do this by properly parsing powermetrics output, but it's + # simpler and more robust to just grep for a handful of identifying + # strings. + + print() # blank line between |rapl| output and |powermetrics| output + + for line in lines.splitlines(): + # Search for the following things. + # + # - '^Name' is for the columns headings line. + # + # - 'firefox' and 'plugin-container' are for Firefox + # + # - 'Safari\b' and 'WebKit' are for Safari. The '\b' excludes + # SafariCloudHistoryPush, which is a process that always + # runs, even when Safari isn't open. + # + # - 'Chrome' is for Chrome. + # + # - 'Terminal' is for the terminal. If no browser is running from + # within the terminal, it will show up unnecessarily. This is a + # minor disadvantage of this very simple parsing strategy. + # + # - 'WindowServer' is for the WindowServer. + # + # - 'kernel' is for the kernel. + # + if re.search( + r"(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)", # NOQA: E501 + line, + ): + print(line) + + return 0 diff --git a/tools/power/moz.build b/tools/power/moz.build new file mode 100644 index 0000000000..ec855a0b86 --- /dev/null +++ b/tools/power/moz.build @@ -0,0 +1,26 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +do_rapl = False + +if CONFIG["OS_ARCH"] == "Darwin" and CONFIG["TARGET_CPU"] == "x86_64": + do_rapl = True + +if ( + CONFIG["OS_ARCH"] == "Linux" + and CONFIG["OS_TARGET"] != "Android" + and CONFIG["TARGET_CPU"] in ("x86", "x86_64") +): + do_rapl = True + +if do_rapl: + SimplePrograms( + [ + "rapl", + ] + ) + +DisableStlWrapping() diff --git a/tools/power/rapl.cpp b/tools/power/rapl.cpp new file mode 100644 index 0000000000..1aa5fcf6ee --- /dev/null +++ b/tools/power/rapl.cpp @@ -0,0 +1,874 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This program provides processor power estimates. It does this by reading +// model-specific registers (MSRs) that are part Intel's Running Average Power +// Limit (RAPL) interface. These MSRs provide good quality estimates of the +// energy consumption of up to four system components: +// - PKG: the entire processor package; +// - PP0: the cores (a subset of the package); +// - PP1: the GPU (a subset of the package); +// - DRAM: main memory. +// +// For more details about RAPL, see section 14.9 of Volume 3 of the "Intel 64 +// and IA-32 Architecture's Software Developer's Manual", Order Number 325384. +// +// This program exists because there are no existing tools on Mac that can +// obtain all four RAPL estimates. (|powermetrics| can obtain the package +// estimate, but not the others. Intel Power Gadget can obtain the package and +// cores estimates.) +// +// On Linux |perf| can obtain all four estimates (as Joules, which are easily +// converted to Watts), but this program is implemented for Linux because it's +// not too hard to do, and that gives us multi-platform consistency. +// +// This program does not support Windows, unfortunately. It's not obvious how +// to access the RAPL MSRs on Windows. +// +// This program deliberately uses only standard libraries and avoids +// Mozilla-specific code, to make it easy to compile and test on different +// machines. + +#include <assert.h> +#include <getopt.h> +#include <math.h> +#include <signal.h> +#include <stdarg.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/time.h> +#include <unistd.h> + +#include <algorithm> +#include <numeric> +#include <vector> + +//--------------------------------------------------------------------------- +// Utilities +//--------------------------------------------------------------------------- + +// The value of argv[0] passed to main(). Used in error messages. +static const char* gArgv0; + +static void Abort(const char* aFormat, ...) { + va_list vargs; + va_start(vargs, aFormat); + fprintf(stderr, "%s: ", gArgv0); + vfprintf(stderr, aFormat, vargs); + fprintf(stderr, "\n"); + va_end(vargs); + + exit(1); +} + +static void CmdLineAbort(const char* aMsg) { + if (aMsg) { + fprintf(stderr, "%s: %s\n", gArgv0, aMsg); + } + fprintf(stderr, "Use --help for more information.\n"); + exit(1); +} + +// A special value that represents an estimate from an unsupported RAPL domain. +static const double kUnsupported_j = -1.0; + +// Print to stdout and flush it, so that the output appears immediately even if +// being redirected through |tee| or anything like that. +static void PrintAndFlush(const char* aFormat, ...) { + va_list vargs; + va_start(vargs, aFormat); + vfprintf(stdout, aFormat, vargs); + va_end(vargs); + + fflush(stdout); +} + +//--------------------------------------------------------------------------- +// Mac-specific code +//--------------------------------------------------------------------------- + +#if defined(__APPLE__) + +// Because of the pkg_energy_statistics_t::pkes_version check below, the +// earliest OS X version this code will work with is 10.9.0 (xnu-2422.1.72). + +# include <sys/types.h> +# include <sys/sysctl.h> + +// OS X has four kinds of system calls: +// +// 1. Mach traps; +// 2. UNIX system calls; +// 3. machine-dependent calls; +// 4. diagnostic calls. +// +// (See "Mac OS X and iOS Internals" by Jonathan Levin for more details.) +// +// The last category has a single call named diagCall() or diagCall64(). Its +// mode is controlled by its first argument, and one of the modes allows access +// to the Intel RAPL MSRs. +// +// The interface to diagCall64() is not exported, so we have to import some +// definitions from the XNU kernel. All imported definitions are annotated with +// the XNU source file they come from, and information about what XNU versions +// they were introduced in and (if relevant) modified. + +// The diagCall64() mode. +// From osfmk/i386/Diagnostics.h +// - In 10.8.4 (xnu-2050.24.15) this value was introduced. (In 10.8.3 the value +// 17 was used for dgGzallocTest.) +# define dgPowerStat 17 + +// From osfmk/i386/cpu_data.h +// - In 10.8.5 these values were introduced, along with core_energy_stat_t. +# define CPU_RTIME_BINS (12) +# define CPU_ITIME_BINS (CPU_RTIME_BINS) + +// core_energy_stat_t and pkg_energy_statistics_t are both from +// osfmk/i386/Diagnostics.c. +// - In 10.8.4 (xnu-2050.24.15) both structs were introduced, but with many +// fewer fields. +// - In 10.8.5 (xnu-2050.48.11) both structs were substantially expanded, with +// numerous new fields. +// - In 10.9.0 (xnu-2422.1.72) pkg_energy_statistics_t::pkes_version was added. +// diagCall64(dgPowerStat) fills it with '1' in all versions since (up to +// 10.10.2 at time of writing). +// - in 10.10.2 (xnu-2782.10.72) core_energy_stat_t::gpmcs was conditionally +// added, if DIAG_ALL_PMCS is true. (DIAG_ALL_PMCS is not even defined in the +// source code, but it could be defined at compile-time via compiler flags.) +// pkg_energy_statistics_t::pkes_version did not change, though. + +typedef struct { + uint64_t caperf; + uint64_t cmperf; + uint64_t ccres[6]; + uint64_t crtimes[CPU_RTIME_BINS]; + uint64_t citimes[CPU_ITIME_BINS]; + uint64_t crtime_total; + uint64_t citime_total; + uint64_t cpu_idle_exits; + uint64_t cpu_insns; + uint64_t cpu_ucc; + uint64_t cpu_urc; +# if DIAG_ALL_PMCS // Added in 10.10.2 (xnu-2782.10.72). + uint64_t gpmcs[4]; // Added in 10.10.2 (xnu-2782.10.72). +# endif /* DIAG_ALL_PMCS */ // Added in 10.10.2 (xnu-2782.10.72). +} core_energy_stat_t; + +typedef struct { + uint64_t pkes_version; // Added in 10.9.0 (xnu-2422.1.72). + uint64_t pkg_cres[2][7]; + + // This is read from MSR 0x606, which Intel calls MSR_RAPL_POWER_UNIT + // and XNU calls MSR_IA32_PKG_POWER_SKU_UNIT. + uint64_t pkg_power_unit; + + // These are the four fields for the four RAPL domains. For each field + // we list: + // + // - the corresponding MSR number; + // - Intel's name for that MSR; + // - XNU's name for that MSR; + // - which Intel processors the MSR is supported on. + // + // The last of these is determined from chapter 35 of Volume 3 of the + // "Intel 64 and IA-32 Architecture's Software Developer's Manual", + // Order Number 325384. (Note that chapter 35 contradicts section 14.9 + // to some degree.) + + // 0x611 == MSR_PKG_ENERGY_STATUS == MSR_IA32_PKG_ENERGY_STATUS + // Atom (various), Sandy Bridge, Next Gen Xeon Phi (model 0x57). + uint64_t pkg_energy; + + // 0x639 == MSR_PP0_ENERGY_STATUS == MSR_IA32_PP0_ENERGY_STATUS + // Atom (various), Sandy Bridge, Next Gen Xeon Phi (model 0x57). + uint64_t pp0_energy; + + // 0x641 == MSR_PP1_ENERGY_STATUS == MSR_PP1_ENERGY_STATUS + // Sandy Bridge, Haswell. + uint64_t pp1_energy; + + // 0x619 == MSR_DRAM_ENERGY_STATUS == MSR_IA32_DDR_ENERGY_STATUS + // Xeon E5, Xeon E5 v2, Haswell/Haswell-E, Next Gen Xeon Phi (model + // 0x57) + uint64_t ddr_energy; + + uint64_t llc_flushed_cycles; + uint64_t ring_ratio_instantaneous; + uint64_t IA_frequency_clipping_cause; + uint64_t GT_frequency_clipping_cause; + uint64_t pkg_idle_exits; + uint64_t pkg_rtimes[CPU_RTIME_BINS]; + uint64_t pkg_itimes[CPU_ITIME_BINS]; + uint64_t mbus_delay_time; + uint64_t mint_delay_time; + uint32_t ncpus; + core_energy_stat_t cest[]; +} pkg_energy_statistics_t; + +static int diagCall64(uint64_t aMode, void* aBuf) { + // We cannot use syscall() here because it doesn't work with diagnostic + // system calls -- it raises SIGSYS if you try. So we have to use asm. + +# ifdef __x86_64__ + // The 0x40000 prefix indicates it's a diagnostic system call. The 0x01 + // suffix indicates the syscall number is 1, which also happens to be the + // only diagnostic system call. See osfmk/mach/i386/syscall_sw.h for more + // details. + static const uint64_t diagCallNum = 0x4000001; + uint64_t rv; + + __asm__ __volatile__( + "syscall" + + // Return value goes in "a" (%rax). + : /* outputs */ "=a"(rv) + + // The syscall number goes in "0", a synonym (from outputs) for "a" + // (%rax). The syscall arguments go in "D" (%rdi) and "S" (%rsi). + : /* inputs */ "0"(diagCallNum), "D"(aMode), "S"(aBuf) + + // The |syscall| instruction clobbers %rcx, %r11, and %rflags ("cc"). And + // this particular syscall also writes memory (aBuf). + : /* clobbers */ "rcx", "r11", "cc", "memory"); + return rv; +# else +# error Sorry, only x86-64 is supported +# endif +} + +static void diagCall64_dgPowerStat(pkg_energy_statistics_t* aPkes) { + static const uint64_t supported_version = 1; + + // Write an unsupported version number into pkes_version so that the check + // below cannot succeed by dumb luck. + aPkes->pkes_version = supported_version - 1; + + // diagCall64() returns 1 on success, and 0 on failure (which can only happen + // if the mode is unrecognized, e.g. in 10.7.x or earlier versions). + if (diagCall64(dgPowerStat, aPkes) != 1) { + Abort("diagCall64() failed"); + } + + if (aPkes->pkes_version != 1) { + Abort("unexpected pkes_version: %llu", aPkes->pkes_version); + } +} + +class RAPL { + bool mIsGpuSupported; // Is the GPU domain supported by the processor? + bool mIsRamSupported; // Is the RAM domain supported by the processor? + + // The DRAM domain on Haswell servers has a fixed energy unit (1/65536 J == + // 15.3 microJoules) which is different to the power unit MSR. (See the + // "Intel Xeon Processor E5-1600 and E5-2600 v3 Product Families, Volume 2 of + // 2, Registers" datasheet, September 2014, Reference Number: 330784-001.) + // This field records whether the quirk is present. + bool mHasRamUnitsQuirk; + + // The abovementioned 15.3 microJoules value. + static const double kQuirkyRamJoulesPerTick; + + // The previous sample's MSR values. + uint64_t mPrevPkgTicks; + uint64_t mPrevPp0Ticks; + uint64_t mPrevPp1Ticks; + uint64_t mPrevDdrTicks; + + // The struct passed to diagCall64(). + pkg_energy_statistics_t* mPkes; + + public: + RAPL() : mHasRamUnitsQuirk(false) { + // Work out which RAPL MSRs this CPU model supports. + int cpuModel; + size_t size = sizeof(cpuModel); + if (sysctlbyname("machdep.cpu.model", &cpuModel, &size, NULL, 0) != 0) { + Abort("sysctlbyname(\"machdep.cpu.model\") failed"); + } + + // This is similar to arch/x86/kernel/cpu/perf_event_intel_rapl.c in + // linux-4.1.5/. + // + // By linux-5.6.14/, this stuff had moved into + // arch/x86/events/intel/rapl.c, which references processor families in + // arch/x86/include/asm/intel-family.h. + switch (cpuModel) { + case 0x2a: // Sandy Bridge + case 0x3a: // Ivy Bridge + // Supports package, cores, GPU. + mIsGpuSupported = true; + mIsRamSupported = false; + break; + + case 0x3f: // Haswell X + case 0x4f: // Broadwell X + case 0x55: // Skylake X + case 0x56: // Broadwell D + // Supports package, cores, RAM. Has the units quirk. + mIsGpuSupported = false; + mIsRamSupported = true; + mHasRamUnitsQuirk = true; + break; + + case 0x2d: // Sandy Bridge X + case 0x3e: // Ivy Bridge X + // Supports package, cores, RAM. + mIsGpuSupported = false; + mIsRamSupported = true; + break; + + case 0x3c: // Haswell + case 0x3d: // Broadwell + case 0x45: // Haswell L + case 0x46: // Haswell G + case 0x47: // Broadwell G + // Supports package, cores, GPU, RAM. + mIsGpuSupported = true; + mIsRamSupported = true; + break; + + case 0x4e: // Skylake L + case 0x5e: // Skylake + case 0x8e: // Kaby Lake L + case 0x9e: // Kaby Lake + case 0x66: // Cannon Lake L + case 0x7d: // Ice Lake + case 0x7e: // Ice Lake L + case 0xa5: // Comet Lake + case 0xa6: // Comet Lake L + // Supports package, cores, GPU, RAM, PSYS. + // XXX: this tool currently doesn't measure PSYS. + mIsGpuSupported = true; + mIsRamSupported = true; + break; + + default: + Abort("unknown CPU model: %d", cpuModel); + break; + } + + // Get the maximum number of logical CPUs so that we know how big to make + // |mPkes|. + int logicalcpu_max; + size = sizeof(logicalcpu_max); + if (sysctlbyname("hw.logicalcpu_max", &logicalcpu_max, &size, NULL, 0) != + 0) { + Abort("sysctlbyname(\"hw.logicalcpu_max\") failed"); + } + + // Over-allocate by 1024 bytes per CPU to allow for the uncertainty around + // core_energy_stat_t::gpmcs and for any other future extensions to that + // struct. (The fields we read all come before the core_energy_stat_t + // array, so it won't matter to us whether gpmcs is present or not.) + size_t pkesSize = sizeof(pkg_energy_statistics_t) + + logicalcpu_max * sizeof(core_energy_stat_t) + + logicalcpu_max * 1024; + mPkes = (pkg_energy_statistics_t*)malloc(pkesSize); + if (!mPkes) { + Abort("malloc() failed"); + } + + // Do an initial measurement so that the first sample's diffs are sensible. + double dummy1, dummy2, dummy3, dummy4; + EnergyEstimates(dummy1, dummy2, dummy3, dummy4); + } + + ~RAPL() { free(mPkes); } + + static double Joules(uint64_t aTicks, double aJoulesPerTick) { + return double(aTicks) * aJoulesPerTick; + } + + void EnergyEstimates(double& aPkg_J, double& aCores_J, double& aGpu_J, + double& aRam_J) { + diagCall64_dgPowerStat(mPkes); + + // Bits 12:8 are the ESU. + // Energy measurements come in multiples of 1/(2^ESU). + uint32_t energyStatusUnits = (mPkes->pkg_power_unit >> 8) & 0x1f; + double joulesPerTick = ((double)1 / (1 << energyStatusUnits)); + + aPkg_J = Joules(mPkes->pkg_energy - mPrevPkgTicks, joulesPerTick); + aCores_J = Joules(mPkes->pp0_energy - mPrevPp0Ticks, joulesPerTick); + aGpu_J = mIsGpuSupported + ? Joules(mPkes->pp1_energy - mPrevPp1Ticks, joulesPerTick) + : kUnsupported_j; + aRam_J = mIsRamSupported + ? Joules(mPkes->ddr_energy - mPrevDdrTicks, + mHasRamUnitsQuirk ? kQuirkyRamJoulesPerTick + : joulesPerTick) + : kUnsupported_j; + + mPrevPkgTicks = mPkes->pkg_energy; + mPrevPp0Ticks = mPkes->pp0_energy; + if (mIsGpuSupported) { + mPrevPp1Ticks = mPkes->pp1_energy; + } + if (mIsRamSupported) { + mPrevDdrTicks = mPkes->ddr_energy; + } + } +}; + +/* static */ const double RAPL::kQuirkyRamJoulesPerTick = (double)1 / 65536; + +//--------------------------------------------------------------------------- +// Linux-specific code +//--------------------------------------------------------------------------- + +#elif defined(__linux__) + +# include <linux/perf_event.h> +# include <sys/syscall.h> + +// There is no glibc wrapper for this system call so we provide our own. +static int perf_event_open(struct perf_event_attr* aAttr, pid_t aPid, int aCpu, + int aGroupFd, unsigned long aFlags) { + return syscall(__NR_perf_event_open, aAttr, aPid, aCpu, aGroupFd, aFlags); +} + +// Returns false if the file cannot be opened. +template <typename T> +static bool ReadValueFromPowerFile(const char* aStr1, const char* aStr2, + const char* aStr3, const char* aScanfString, + T* aOut) { + // The filenames going into this buffer are under our control and the longest + // one is "/sys/bus/event_source/devices/power/events/energy-cores.scale". + // So 256 chars is plenty. + char filename[256]; + + sprintf(filename, "/sys/bus/event_source/devices/power/%s%s%s", aStr1, aStr2, + aStr3); + FILE* fp = fopen(filename, "r"); + if (!fp) { + return false; + } + if (fscanf(fp, aScanfString, aOut) != 1) { + Abort("fscanf() failed"); + } + fclose(fp); + + return true; +} + +// This class encapsulates the reading of a single RAPL domain. +class Domain { + bool mIsSupported; // Is the domain supported by the processor? + + // These three are only set if |mIsSupported| is true. + double mJoulesPerTick; // How many Joules each tick of the MSR represents. + int mFd; // The fd through which the MSR is read. + double mPrevTicks; // The previous sample's MSR value. + + public: + enum IsOptional { Optional, NonOptional }; + + Domain(const char* aName, uint32_t aType, + IsOptional aOptional = NonOptional) { + uint64_t config; + if (!ReadValueFromPowerFile("events/energy-", aName, "", "event=%llx", + &config)) { + // Failure is allowed for optional domains. + if (aOptional == NonOptional) { + Abort( + "failed to open file for non-optional domain '%s'\n" + "- Is your kernel version 3.14 or later, as required? " + "Run |uname -r| to see.", + aName); + } + mIsSupported = false; + return; + } + + mIsSupported = true; + + if (!ReadValueFromPowerFile("events/energy-", aName, ".scale", "%lf", + &mJoulesPerTick)) { + Abort("failed to read from .scale file"); + } + + // The unit should be "Joules", so 128 chars should be plenty. + char unit[128]; + if (!ReadValueFromPowerFile("events/energy-", aName, ".unit", "%127s", + unit)) { + Abort("failed to read from .unit file"); + } + if (strcmp(unit, "Joules") != 0) { + Abort("unexpected unit '%s' in .unit file", unit); + } + + struct perf_event_attr attr; + memset(&attr, 0, sizeof(attr)); + attr.type = aType; + attr.size = uint32_t(sizeof(attr)); + attr.config = config; + + // Measure all processes/threads. The specified CPU doesn't matter. + mFd = perf_event_open(&attr, /* aPid = */ -1, /* aCpu = */ 0, + /* aGroupFd = */ -1, /* aFlags = */ 0); + if (mFd < 0) { + Abort( + "perf_event_open() failed\n" + "- Did you run as root (e.g. with |sudo|) or set\n" + " /proc/sys/kernel/perf_event_paranoid to 0, as required?"); + } + + mPrevTicks = 0; + } + + ~Domain() { + if (mIsSupported) { + close(mFd); + } + } + + double EnergyEstimate() { + if (!mIsSupported) { + return kUnsupported_j; + } + + uint64_t thisTicks; + if (read(mFd, &thisTicks, sizeof(uint64_t)) != sizeof(uint64_t)) { + Abort("read() failed"); + } + + uint64_t ticks = thisTicks - mPrevTicks; + mPrevTicks = thisTicks; + double joules = ticks * mJoulesPerTick; + return joules; + } +}; + +class RAPL { + Domain* mPkg; + Domain* mCores; + Domain* mGpu; + Domain* mRam; + + public: + RAPL() { + uint32_t type; + if (!ReadValueFromPowerFile("type", "", "", "%u", &type)) { + Abort("failed to read from type file"); + } + + mPkg = new Domain("pkg", type); + mCores = new Domain("cores", type); + mGpu = new Domain("gpu", type, Domain::Optional); + mRam = new Domain("ram", type, Domain::Optional); + if (!mPkg || !mCores || !mGpu || !mRam) { + Abort("new Domain() failed"); + } + } + + ~RAPL() { + delete mPkg; + delete mCores; + delete mGpu; + delete mRam; + } + + void EnergyEstimates(double& aPkg_J, double& aCores_J, double& aGpu_J, + double& aRam_J) { + aPkg_J = mPkg->EnergyEstimate(); + aCores_J = mCores->EnergyEstimate(); + aGpu_J = mGpu->EnergyEstimate(); + aRam_J = mRam->EnergyEstimate(); + } +}; + +#else + +//--------------------------------------------------------------------------- +// Unsupported platforms +//--------------------------------------------------------------------------- + +# error Sorry, this platform is not supported + +#endif // platform + +//--------------------------------------------------------------------------- +// The main loop +//--------------------------------------------------------------------------- + +// The sample interval, measured in seconds. +static double gSampleInterval_sec; + +// The platform-specific RAPL-reading machinery. +static RAPL* gRapl; + +// All the sampled "total" values, in Watts. +static std::vector<double> gTotals_W; + +// Power = Energy / Time, where power is measured in Watts, Energy is measured +// in Joules, and Time is measured in seconds. +static double JoulesToWatts(double aJoules) { + return aJoules / gSampleInterval_sec; +} + +// "Normalize" here means convert kUnsupported_j to zero so it can be used in +// additive expressions. All printed values are 5 or maybe 6 chars (though 6 +// chars would require a value > 100 W, which is unlikely). Values above 1000 W +// are normalized to " n/a ", so 6 chars is the longest that may be printed. +static void NormalizeAndPrintAsWatts(char* aBuf, double& aValue_J) { + if (aValue_J == kUnsupported_j || aValue_J >= 1000) { + aValue_J = 0; + sprintf(aBuf, "%s", " n/a "); + } else { + sprintf(aBuf, "%5.2f", JoulesToWatts(aValue_J)); + } +} + +static void SigAlrmHandler(int aSigNum, siginfo_t* aInfo, void* aContext) { + static int sampleNumber = 1; + + double pkg_J, cores_J, gpu_J, ram_J; + gRapl->EnergyEstimates(pkg_J, cores_J, gpu_J, ram_J); + + // We should have pkg and cores estimates, but might not have gpu and ram + // estimates. + assert(pkg_J != kUnsupported_j); + assert(cores_J != kUnsupported_j); + + // This needs to be big enough to print watt values to two decimal places. 16 + // should be plenty. + static const size_t kNumStrLen = 16; + + static char pkgStr[kNumStrLen], coresStr[kNumStrLen], gpuStr[kNumStrLen], + ramStr[kNumStrLen]; + NormalizeAndPrintAsWatts(pkgStr, pkg_J); + NormalizeAndPrintAsWatts(coresStr, cores_J); + NormalizeAndPrintAsWatts(gpuStr, gpu_J); + NormalizeAndPrintAsWatts(ramStr, ram_J); + + // Core and GPU power are a subset of the package power. + assert(pkg_J >= cores_J + gpu_J); + + // Compute "other" (i.e. rest of the package) and "total" only after the + // other values have been normalized. + + char otherStr[kNumStrLen]; + double other_J = pkg_J - cores_J - gpu_J; + NormalizeAndPrintAsWatts(otherStr, other_J); + + char totalStr[kNumStrLen]; + double total_J = pkg_J + ram_J; + NormalizeAndPrintAsWatts(totalStr, total_J); + + gTotals_W.push_back(JoulesToWatts(total_J)); + + // Print and flush so that the output appears immediately even if being + // redirected through |tee| or anything like that. + PrintAndFlush("#%02d %s W = %s (%s + %s + %s) + %s W\n", sampleNumber++, + totalStr, pkgStr, coresStr, gpuStr, otherStr, ramStr); +} + +static void Finish() { + size_t n = gTotals_W.size(); + + // This time calculation assumes that the timers are perfectly accurate which + // is not true but the inaccuracy should be small in practice. + double time = n * gSampleInterval_sec; + + printf("\n"); + printf("%d sample%s taken over a period of %.3f second%s\n", int(n), + n == 1 ? "" : "s", n * gSampleInterval_sec, time == 1.0 ? "" : "s"); + + if (n == 0 || n == 1) { + exit(0); + } + + // Compute the mean. + double sum = std::accumulate(gTotals_W.begin(), gTotals_W.end(), 0.0); + double mean = sum / n; + + // Compute the *population* standard deviation: + // + // popStdDev = sqrt(Sigma(x - m)^2 / n) + // + // where |x| is the sum variable, |m| is the mean, and |n| is the + // population size. + // + // This is different from the *sample* standard deviation, which divides by + // |n - 1|, and would be appropriate if we were using a random sample of a + // larger population. + double sumOfSquaredDeviations = 0; + for (double& iter : gTotals_W) { + double deviation = (iter - mean); + sumOfSquaredDeviations += deviation * deviation; + } + double popStdDev = sqrt(sumOfSquaredDeviations / n); + + // Sort so that percentiles can be determined. We use the "Nearest Rank" + // method of determining percentiles, which is simplest to compute and which + // chooses values from those that appear in the input set. + std::sort(gTotals_W.begin(), gTotals_W.end()); + + printf("\n"); + printf("Distribution of 'total' values:\n"); + printf(" mean = %5.2f W\n", mean); + printf(" std dev = %5.2f W\n", popStdDev); + printf(" 0th percentile = %5.2f W (min)\n", gTotals_W[0]); + printf(" 5th percentile = %5.2f W\n", gTotals_W[ceil(0.05 * n) - 1]); + printf(" 25th percentile = %5.2f W\n", gTotals_W[ceil(0.25 * n) - 1]); + printf(" 50th percentile = %5.2f W\n", gTotals_W[ceil(0.50 * n) - 1]); + printf(" 75th percentile = %5.2f W\n", gTotals_W[ceil(0.75 * n) - 1]); + printf(" 95th percentile = %5.2f W\n", gTotals_W[ceil(0.95 * n) - 1]); + printf("100th percentile = %5.2f W (max)\n", gTotals_W[n - 1]); + + exit(0); +} + +static void SigIntHandler(int aSigNum, siginfo_t* aInfo, void* aContext) { + Finish(); +} + +static void PrintUsage() { + printf( + "usage: rapl [options]\n" + "\n" + "Options:\n" + "\n" + " -h --help show this message\n" + " -i --sample-interval <N> sample every N ms [default=1000]\n" + " -n --sample-count <N> get N samples (0 means unlimited) " + "[default=0]\n" + "\n" +#if defined(__APPLE__) + "On Mac this program can be run by any user.\n" +#elif defined(__linux__) + "On Linux this program can only be run by the super-user unless the " + "contents\n" + "of /proc/sys/kernel/perf_event_paranoid is set to 0 or lower.\n" +#else +# error Sorry, this platform is not supported +#endif + "\n"); +} + +int main(int argc, char** argv) { + // Process command line options. + + gArgv0 = argv[0]; + + // Default values. + int sampleInterval_msec = 1000; + int sampleCount = 0; + + struct option longOptions[] = { + {"help", no_argument, NULL, 'h'}, + {"sample-interval", required_argument, NULL, 'i'}, + {"sample-count", required_argument, NULL, 'n'}, + {NULL, 0, NULL, 0}}; + const char* shortOptions = "hi:n:"; + + int c; + char* endPtr; + while ((c = getopt_long(argc, argv, shortOptions, longOptions, NULL)) != -1) { + switch (c) { + case 'h': + PrintUsage(); + exit(0); + + case 'i': + sampleInterval_msec = strtol(optarg, &endPtr, /* base = */ 10); + if (*endPtr) { + CmdLineAbort("sample interval is not an integer"); + } + if (sampleInterval_msec < 1 || sampleInterval_msec > 3600000) { + CmdLineAbort("sample interval must be in the range 1..3600000 ms"); + } + break; + + case 'n': + sampleCount = strtol(optarg, &endPtr, /* base = */ 10); + if (*endPtr) { + CmdLineAbort("sample count is not an integer"); + } + if (sampleCount < 0 || sampleCount > 1000000) { + CmdLineAbort("sample count must be in the range 0..1000000"); + } + break; + + default: + CmdLineAbort(NULL); + } + } + + // The RAPL MSRs update every ~1 ms, but the measurement period isn't exactly + // 1 ms, which means the sample periods are not exact. "Power Measurement + // Techniques on Standard Compute Nodes: A Quantitative Comparison" by + // Hackenberg et al. suggests the following. + // + // "RAPL provides energy (and not power) consumption data without + // timestamps associated to each counter update. This makes sampling rates + // above 20 Samples/s unfeasible if the systematic error should be below + // 5%... Constantly polling the RAPL registers will both occupy a processor + // core and distort the measurement itself." + // + // So warn about this case. + if (sampleInterval_msec < 50) { + fprintf(stderr, + "\nWARNING: sample intervals < 50 ms are likely to produce " + "inaccurate estimates\n\n"); + } + gSampleInterval_sec = double(sampleInterval_msec) / 1000; + + // Initialize the platform-specific RAPL reading machinery. + gRapl = new RAPL(); + if (!gRapl) { + Abort("new RAPL() failed"); + } + + // Install the signal handlers. + + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_flags = SA_RESTART | SA_SIGINFO; + // The extra parens around (0) suppress a -Wunreachable-code warning on OS X + // where sigemptyset() is a macro that can never fail and always returns 0. + if (sigemptyset(&sa.sa_mask) < (0)) { + Abort("sigemptyset() failed"); + } + sa.sa_sigaction = SigAlrmHandler; + if (sigaction(SIGALRM, &sa, NULL) < 0) { + Abort("sigaction(SIGALRM) failed"); + } + sa.sa_sigaction = SigIntHandler; + if (sigaction(SIGINT, &sa, NULL) < 0) { + Abort("sigaction(SIGINT) failed"); + } + + // Set up the timer. + struct itimerval timer; + timer.it_interval.tv_sec = sampleInterval_msec / 1000; + timer.it_interval.tv_usec = (sampleInterval_msec % 1000) * 1000; + timer.it_value = timer.it_interval; + if (setitimer(ITIMER_REAL, &timer, NULL) < 0) { + Abort("setitimer() failed"); + } + + // Print header. + PrintAndFlush(" total W = _pkg_ (cores + _gpu_ + other) + _ram_ W\n"); + + // Take samples. + if (sampleCount == 0) { + while (true) { + pause(); + } + } else { + for (int i = 0; i < sampleCount; i++) { + pause(); + } + } + + Finish(); + + return 0; +} diff --git a/tools/profiler/core/EHABIStackWalk.cpp b/tools/profiler/core/EHABIStackWalk.cpp new file mode 100644 index 0000000000..e3099b89ec --- /dev/null +++ b/tools/profiler/core/EHABIStackWalk.cpp @@ -0,0 +1,597 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * This is an implementation of stack unwinding according to a subset + * of the ARM Exception Handling ABI, as described in: + * http://infocenter.arm.com/help/topic/com.arm.doc.ihi0038a/IHI0038A_ehabi.pdf + * + * This handles only the ARM-defined "personality routines" (chapter + * 9), and don't track the value of FP registers, because profiling + * needs only chain of PC/SP values. + * + * Because the exception handling info may not be accurate for all + * possible places where an async signal could occur (e.g., in a + * prologue or epilogue), this bounds-checks all stack accesses. + * + * This file uses "struct" for structures in the exception tables and + * "class" otherwise. We should avoid violating the C++11 + * standard-layout rules in the former. + */ + +#include "EHABIStackWalk.h" + +#include "shared-libraries.h" +#include "platform.h" + +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/EndianUtils.h" + +#include <algorithm> +#include <elf.h> +#include <stdint.h> +#include <vector> +#include <string> + +#ifndef PT_ARM_EXIDX +# define PT_ARM_EXIDX 0x70000001 +#endif + +namespace mozilla { + +struct PRel31 { + uint32_t mBits; + bool topBit() const { return mBits & 0x80000000; } + uint32_t value() const { return mBits & 0x7fffffff; } + int32_t offset() const { return (static_cast<int32_t>(mBits) << 1) >> 1; } + const void* compute() const { + return reinterpret_cast<const char*>(this) + offset(); + } + + private: + PRel31(const PRel31& copied) = delete; + PRel31() = delete; +}; + +struct EHEntry { + PRel31 startPC; + PRel31 exidx; + + private: + EHEntry(const EHEntry& copied) = delete; + EHEntry() = delete; +}; + +class EHState { + // Note that any core register can be used as a "frame pointer" to + // influence the unwinding process, so this must track all of them. + uint32_t mRegs[16]; + + public: + bool unwind(const EHEntry* aEntry, const void* stackBase); + uint32_t& operator[](int i) { return mRegs[i]; } + const uint32_t& operator[](int i) const { return mRegs[i]; } + explicit EHState(const mcontext_t&); +}; + +enum { R_SP = 13, R_LR = 14, R_PC = 15 }; + +class EHTable { + uint32_t mStartPC; + uint32_t mEndPC; + uint32_t mBaseAddress; + const EHEntry* mEntriesBegin; + const EHEntry* mEntriesEnd; + std::string mName; + + public: + EHTable(const void* aELF, size_t aSize, const std::string& aName); + const EHEntry* lookup(uint32_t aPC) const; + bool isValid() const { return mEntriesEnd != mEntriesBegin; } + const std::string& name() const { return mName; } + uint32_t startPC() const { return mStartPC; } + uint32_t endPC() const { return mEndPC; } + uint32_t baseAddress() const { return mBaseAddress; } +}; + +class EHAddrSpace { + std::vector<uint32_t> mStarts; + std::vector<EHTable> mTables; + static mozilla::Atomic<const EHAddrSpace*> sCurrent; + + public: + explicit EHAddrSpace(const std::vector<EHTable>& aTables); + const EHTable* lookup(uint32_t aPC) const; + static void Update(); + static const EHAddrSpace* Get(); +}; + +void EHABIStackWalkInit() { EHAddrSpace::Update(); } + +size_t EHABIStackWalk(const mcontext_t& aContext, void* stackBase, void** aSPs, + void** aPCs, const size_t aNumFrames) { + const EHAddrSpace* space = EHAddrSpace::Get(); + EHState state(aContext); + size_t count = 0; + + while (count < aNumFrames) { + uint32_t pc = state[R_PC], sp = state[R_SP]; + + // ARM instructions are always aligned to 2 or 4 bytes. + // The last bit of the pc / lr indicates ARM or Thumb mode. + // We're only interested in the instruction address, so we mask off that + // bit. + constexpr uint32_t instrAddrMask = ~1; + uint32_t instrAddress = pc & instrAddrMask; + + aPCs[count] = reinterpret_cast<void*>(instrAddress); + aSPs[count] = reinterpret_cast<void*>(sp); + count++; + + if (!space) break; + // TODO: cache these lookups. Binary-searching libxul is + // expensive (possibly more expensive than doing the actual + // unwind), and even a small cache should help. + const EHTable* table = space->lookup(pc); + if (!table) break; + const EHEntry* entry = table->lookup(pc); + if (!entry) break; + if (!state.unwind(entry, stackBase)) break; + } + + return count; +} + +class EHInterp { + public: + // Note that stackLimit is exclusive and stackBase is inclusive + // (i.e, stackLimit < SP <= stackBase), following the convention + // set by the AAPCS spec. + EHInterp(EHState& aState, const EHEntry* aEntry, uint32_t aStackLimit, + uint32_t aStackBase) + : mState(aState), + mStackLimit(aStackLimit), + mStackBase(aStackBase), + mNextWord(0), + mWordsLeft(0), + mFailed(false) { + const PRel31& exidx = aEntry->exidx; + uint32_t firstWord; + + if (exidx.mBits == 1) { // EXIDX_CANTUNWIND + mFailed = true; + return; + } + if (exidx.topBit()) { + firstWord = exidx.mBits; + } else { + mNextWord = reinterpret_cast<const uint32_t*>(exidx.compute()); + firstWord = *mNextWord++; + } + + switch (firstWord >> 24) { + case 0x80: // short + mWord = firstWord << 8; + mBytesLeft = 3; + break; + case 0x81: + case 0x82: // long; catch descriptor size ignored + mWord = firstWord << 16; + mBytesLeft = 2; + mWordsLeft = (firstWord >> 16) & 0xff; + break; + default: + // unknown personality + mFailed = true; + } + } + + bool unwind(); + + private: + // TODO: GCC has been observed not CSEing repeated reads of + // mState[R_SP] with writes to mFailed between them, suggesting that + // it hasn't determined that they can't alias and is thus missing + // optimization opportunities. So, we may want to flatten EHState + // into this class; this may also make the code simpler. + EHState& mState; + uint32_t mStackLimit; + uint32_t mStackBase; + const uint32_t* mNextWord; + uint32_t mWord; + uint8_t mWordsLeft; + uint8_t mBytesLeft; + bool mFailed; + + enum { + I_ADDSP = 0x00, // 0sxxxxxx (subtract if s) + M_ADDSP = 0x80, + I_POPMASK = 0x80, // 1000iiii iiiiiiii (if any i set) + M_POPMASK = 0xf0, + I_MOVSP = 0x90, // 1001nnnn + M_MOVSP = 0xf0, + I_POPN = 0xa0, // 1010lnnn + M_POPN = 0xf0, + I_FINISH = 0xb0, // 10110000 + I_POPLO = 0xb1, // 10110001 0000iiii (if any i set) + I_ADDSPBIG = 0xb2, // 10110010 uleb128 + I_POPFDX = 0xb3, // 10110011 sssscccc + I_POPFDX8 = 0xb8, // 10111nnn + M_POPFDX8 = 0xf8, + // "Intel Wireless MMX" extensions omitted. + I_POPFDD = 0xc8, // 1100100h sssscccc + M_POPFDD = 0xfe, + I_POPFDD8 = 0xd0, // 11010nnn + M_POPFDD8 = 0xf8 + }; + + uint8_t next() { + if (mBytesLeft == 0) { + if (mWordsLeft == 0) { + return I_FINISH; + } + mWordsLeft--; + mWord = *mNextWord++; + mBytesLeft = 4; + } + mBytesLeft--; + mWord = (mWord << 8) | (mWord >> 24); // rotate + return mWord; + } + + uint32_t& vSP() { return mState[R_SP]; } + uint32_t* ptrSP() { return reinterpret_cast<uint32_t*>(vSP()); } + + void checkStackBase() { + if (vSP() > mStackBase) mFailed = true; + } + void checkStackLimit() { + if (vSP() <= mStackLimit) mFailed = true; + } + void checkStackAlign() { + if ((vSP() & 3) != 0) mFailed = true; + } + void checkStack() { + checkStackBase(); + checkStackLimit(); + checkStackAlign(); + } + + void popRange(uint8_t first, uint8_t last, uint16_t mask) { + bool hasSP = false; + uint32_t tmpSP; + if (mask == 0) mFailed = true; + for (uint8_t r = first; r <= last; ++r) { + if (mask & 1) { + if (r == R_SP) { + hasSP = true; + tmpSP = *ptrSP(); + } else + mState[r] = *ptrSP(); + vSP() += 4; + checkStackBase(); + if (mFailed) return; + } + mask >>= 1; + } + if (hasSP) { + vSP() = tmpSP; + checkStack(); + } + } +}; + +bool EHState::unwind(const EHEntry* aEntry, const void* stackBasePtr) { + // The unwinding program cannot set SP to less than the initial value. + uint32_t stackLimit = mRegs[R_SP] - 4; + uint32_t stackBase = reinterpret_cast<uint32_t>(stackBasePtr); + EHInterp interp(*this, aEntry, stackLimit, stackBase); + return interp.unwind(); +} + +bool EHInterp::unwind() { + mState[R_PC] = 0; + checkStack(); + while (!mFailed) { + uint8_t insn = next(); +#if DEBUG_EHABI_UNWIND + LOG("unwind insn = %02x", (unsigned)insn); +#endif + // Try to put the common cases first. + + // 00xxxxxx: vsp = vsp + (xxxxxx << 2) + 4 + // 01xxxxxx: vsp = vsp - (xxxxxx << 2) - 4 + if ((insn & M_ADDSP) == I_ADDSP) { + uint32_t offset = ((insn & 0x3f) << 2) + 4; + if (insn & 0x40) { + vSP() -= offset; + checkStackLimit(); + } else { + vSP() += offset; + checkStackBase(); + } + continue; + } + + // 10100nnn: Pop r4-r[4+nnn] + // 10101nnn: Pop r4-r[4+nnn], r14 + if ((insn & M_POPN) == I_POPN) { + uint8_t n = (insn & 0x07) + 1; + bool lr = insn & 0x08; + uint32_t* ptr = ptrSP(); + vSP() += (n + (lr ? 1 : 0)) * 4; + checkStackBase(); + for (uint8_t r = 4; r < 4 + n; ++r) mState[r] = *ptr++; + if (lr) mState[R_LR] = *ptr++; + continue; + } + + // 1011000: Finish + if (insn == I_FINISH) { + if (mState[R_PC] == 0) { + mState[R_PC] = mState[R_LR]; + // Non-standard change (bug 916106): Prevent the caller from + // re-using LR. Since the caller is by definition not a leaf + // routine, it will have to restore LR from somewhere to + // return to its own caller, so we can safely zero it here. + // This makes a difference only if an error in unwinding + // (e.g., caused by starting from within a prologue/epilogue) + // causes us to load a pointer to a leaf routine as LR; if we + // don't do something, we'll go into an infinite loop of + // "returning" to that same function. + mState[R_LR] = 0; + } + return true; + } + + // 1001nnnn: Set vsp = r[nnnn] + if ((insn & M_MOVSP) == I_MOVSP) { + vSP() = mState[insn & 0x0f]; + checkStack(); + continue; + } + + // 11001000 sssscccc: Pop VFP regs D[16+ssss]-D[16+ssss+cccc] (as FLDMFDD) + // 11001001 sssscccc: Pop VFP regs D[ssss]-D[ssss+cccc] (as FLDMFDD) + if ((insn & M_POPFDD) == I_POPFDD) { + uint8_t n = (next() & 0x0f) + 1; + // Note: if the 16+ssss+cccc > 31, the encoding is reserved. + // As the space is currently unused, we don't try to check. + vSP() += 8 * n; + checkStackBase(); + continue; + } + + // 11010nnn: Pop VFP regs D[8]-D[8+nnn] (as FLDMFDD) + if ((insn & M_POPFDD8) == I_POPFDD8) { + uint8_t n = (insn & 0x07) + 1; + vSP() += 8 * n; + checkStackBase(); + continue; + } + + // 10110010 uleb128: vsp = vsp + 0x204 + (uleb128 << 2) + if (insn == I_ADDSPBIG) { + uint32_t acc = 0; + uint8_t shift = 0; + uint8_t byte; + do { + if (shift >= 32) return false; + byte = next(); + acc |= (byte & 0x7f) << shift; + shift += 7; + } while (byte & 0x80); + uint32_t offset = 0x204 + (acc << 2); + // The calculations above could have overflowed. + // But the one we care about is this: + if (vSP() + offset < vSP()) mFailed = true; + vSP() += offset; + // ...so that this is the only other check needed: + checkStackBase(); + continue; + } + + // 1000iiii iiiiiiii (i not all 0): Pop under masks {r15-r12}, {r11-r4} + if ((insn & M_POPMASK) == I_POPMASK) { + popRange(4, 15, ((insn & 0x0f) << 8) | next()); + continue; + } + + // 1011001 0000iiii (i not all 0): Pop under mask {r3-r0} + if (insn == I_POPLO) { + popRange(0, 3, next() & 0x0f); + continue; + } + + // 10110011 sssscccc: Pop VFP regs D[ssss]-D[ssss+cccc] (as FLDMFDX) + if (insn == I_POPFDX) { + uint8_t n = (next() & 0x0f) + 1; + vSP() += 8 * n + 4; + checkStackBase(); + continue; + } + + // 10111nnn: Pop VFP regs D[8]-D[8+nnn] (as FLDMFDX) + if ((insn & M_POPFDX8) == I_POPFDX8) { + uint8_t n = (insn & 0x07) + 1; + vSP() += 8 * n + 4; + checkStackBase(); + continue; + } + + // unhandled instruction +#ifdef DEBUG_EHABI_UNWIND + LOG("Unhandled EHABI instruction 0x%02x", insn); +#endif + mFailed = true; + } + return false; +} + +bool operator<(const EHTable& lhs, const EHTable& rhs) { + return lhs.startPC() < rhs.startPC(); +} + +// Async signal unsafe. +EHAddrSpace::EHAddrSpace(const std::vector<EHTable>& aTables) + : mTables(aTables) { + std::sort(mTables.begin(), mTables.end()); + DebugOnly<uint32_t> lastEnd = 0; + for (std::vector<EHTable>::iterator i = mTables.begin(); i != mTables.end(); + ++i) { + MOZ_ASSERT(i->startPC() >= lastEnd); + mStarts.push_back(i->startPC()); + lastEnd = i->endPC(); + } +} + +const EHTable* EHAddrSpace::lookup(uint32_t aPC) const { + ptrdiff_t i = (std::upper_bound(mStarts.begin(), mStarts.end(), aPC) - + mStarts.begin()) - + 1; + + if (i < 0 || aPC >= mTables[i].endPC()) return 0; + return &mTables[i]; +} + +const EHEntry* EHTable::lookup(uint32_t aPC) const { + MOZ_ASSERT(aPC >= mStartPC); + if (aPC >= mEndPC) return nullptr; + + const EHEntry* begin = mEntriesBegin; + const EHEntry* end = mEntriesEnd; + MOZ_ASSERT(begin < end); + if (aPC < reinterpret_cast<uint32_t>(begin->startPC.compute())) + return nullptr; + + while (end - begin > 1) { +#ifdef EHABI_UNWIND_MORE_ASSERTS + if ((end - 1)->startPC.compute() < begin->startPC.compute()) { + MOZ_CRASH("unsorted exidx"); + } +#endif + const EHEntry* mid = begin + (end - begin) / 2; + if (aPC < reinterpret_cast<uint32_t>(mid->startPC.compute())) + end = mid; + else + begin = mid; + } + return begin; +} + +#if MOZ_LITTLE_ENDIAN() +static const unsigned char hostEndian = ELFDATA2LSB; +#elif MOZ_BIG_ENDIAN() +static const unsigned char hostEndian = ELFDATA2MSB; +#else +# error "No endian?" +#endif + +// Async signal unsafe: std::vector::reserve, std::string copy ctor. +EHTable::EHTable(const void* aELF, size_t aSize, const std::string& aName) + : mStartPC(~0), // largest uint32_t + mEndPC(0), + mEntriesBegin(nullptr), + mEntriesEnd(nullptr), + mName(aName) { + const uint32_t fileHeaderAddr = reinterpret_cast<uint32_t>(aELF); + + if (aSize < sizeof(Elf32_Ehdr)) return; + + const Elf32_Ehdr& file = *(reinterpret_cast<Elf32_Ehdr*>(fileHeaderAddr)); + if (memcmp(&file.e_ident[EI_MAG0], ELFMAG, SELFMAG) != 0 || + file.e_ident[EI_CLASS] != ELFCLASS32 || + file.e_ident[EI_DATA] != hostEndian || + file.e_ident[EI_VERSION] != EV_CURRENT || file.e_machine != EM_ARM || + file.e_version != EV_CURRENT) + // e_flags? + return; + + MOZ_ASSERT(file.e_phoff + file.e_phnum * file.e_phentsize <= aSize); + const Elf32_Phdr *exidxHdr = 0, *zeroHdr = 0; + for (unsigned i = 0; i < file.e_phnum; ++i) { + const Elf32_Phdr& phdr = *(reinterpret_cast<Elf32_Phdr*>( + fileHeaderAddr + file.e_phoff + i * file.e_phentsize)); + if (phdr.p_type == PT_ARM_EXIDX) { + exidxHdr = &phdr; + } else if (phdr.p_type == PT_LOAD) { + if (phdr.p_offset == 0) { + zeroHdr = &phdr; + } + if (phdr.p_flags & PF_X) { + mStartPC = std::min(mStartPC, phdr.p_vaddr); + mEndPC = std::max(mEndPC, phdr.p_vaddr + phdr.p_memsz); + } + } + } + if (!exidxHdr) return; + if (!zeroHdr) return; + mBaseAddress = fileHeaderAddr - zeroHdr->p_vaddr; + mStartPC += mBaseAddress; + mEndPC += mBaseAddress; + mEntriesBegin = + reinterpret_cast<const EHEntry*>(mBaseAddress + exidxHdr->p_vaddr); + mEntriesEnd = reinterpret_cast<const EHEntry*>( + mBaseAddress + exidxHdr->p_vaddr + exidxHdr->p_memsz); +} + +mozilla::Atomic<const EHAddrSpace*> EHAddrSpace::sCurrent(nullptr); + +// Async signal safe; can fail if Update() hasn't returned yet. +const EHAddrSpace* EHAddrSpace::Get() { return sCurrent; } + +// Collect unwinding information from loaded objects. Calls after the +// first have no effect. Async signal unsafe. +void EHAddrSpace::Update() { + const EHAddrSpace* space = sCurrent; + if (space) return; + + SharedLibraryInfo info = SharedLibraryInfo::GetInfoForSelf(); + std::vector<EHTable> tables; + + for (size_t i = 0; i < info.GetSize(); ++i) { + const SharedLibrary& lib = info.GetEntry(i); + // FIXME: This isn't correct if the start address isn't p_offset 0, because + // the start address will not point at the file header. But this is worked + // around by magic number checks in the EHTable constructor. + EHTable tab(reinterpret_cast<const void*>(lib.GetStart()), + lib.GetEnd() - lib.GetStart(), lib.GetNativeDebugPath()); + if (tab.isValid()) tables.push_back(tab); + } + space = new EHAddrSpace(tables); + + if (!sCurrent.compareExchange(nullptr, space)) { + delete space; + space = sCurrent; + } +} + +EHState::EHState(const mcontext_t& context) { +#ifdef linux + mRegs[0] = context.arm_r0; + mRegs[1] = context.arm_r1; + mRegs[2] = context.arm_r2; + mRegs[3] = context.arm_r3; + mRegs[4] = context.arm_r4; + mRegs[5] = context.arm_r5; + mRegs[6] = context.arm_r6; + mRegs[7] = context.arm_r7; + mRegs[8] = context.arm_r8; + mRegs[9] = context.arm_r9; + mRegs[10] = context.arm_r10; + mRegs[11] = context.arm_fp; + mRegs[12] = context.arm_ip; + mRegs[13] = context.arm_sp; + mRegs[14] = context.arm_lr; + mRegs[15] = context.arm_pc; +#else +# error "Unhandled OS for ARM EHABI unwinding" +#endif +} + +} // namespace mozilla diff --git a/tools/profiler/core/EHABIStackWalk.h b/tools/profiler/core/EHABIStackWalk.h new file mode 100644 index 0000000000..61286290b8 --- /dev/null +++ b/tools/profiler/core/EHABIStackWalk.h @@ -0,0 +1,28 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * This is an implementation of stack unwinding according to a subset + * of the ARM Exception Handling ABI; see the comment at the top of + * the .cpp file for details. + */ + +#ifndef mozilla_EHABIStackWalk_h__ +#define mozilla_EHABIStackWalk_h__ + +#include <stddef.h> +#include <ucontext.h> + +namespace mozilla { + +void EHABIStackWalkInit(); + +size_t EHABIStackWalk(const mcontext_t& aContext, void* stackBase, void** aSPs, + void** aPCs, size_t aNumFrames); + +} // namespace mozilla + +#endif diff --git a/tools/profiler/core/ETWTools.cpp b/tools/profiler/core/ETWTools.cpp new file mode 100644 index 0000000000..b8fab9b316 --- /dev/null +++ b/tools/profiler/core/ETWTools.cpp @@ -0,0 +1,46 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ETWTools.h" + +#include <atomic> + +namespace ETW { +std::atomic<ULONGLONG> gETWCollectionMask = 0; + +// Define a handle to a TraceLogging provider +TRACELOGGING_DEFINE_PROVIDER(kFirefoxTraceLoggingProvider, + "Mozilla.FirefoxTraceLogger", + // This GUID is a hash generated based on the + // above string. + // {c923f508-96e4-5515-e32c-7539d1b10504} + (0xc923f508, 0x96e4, 0x5515, 0xe3, 0x3c, 0x75, + 0x39, 0xd1, 0xb1, 0x05, 0x04)); + +static void NTAPI ETWEnableCallback(LPCGUID aSourceId, ULONG aIsEnabled, + UCHAR aLevel, ULONGLONG aMatchAnyKeyword, + ULONGLONG aMatchAllKeyword, + PEVENT_FILTER_DESCRIPTOR aFilterData, + PVOID aCallbackContext) { + // This is called on a CRT worker thread. This means this might race a bit + // with our main thread, but that is okay. + if (aIsEnabled) { + mozilla::profiler::detail::RacyFeatures::SetETWCollectionActive(); + } else { + mozilla::profiler::detail::RacyFeatures::SetETWCollectionInactive(); + } + // The lower 48 bits of the provider flags are used to mask markers. + gETWCollectionMask = aMatchAnyKeyword; +} + +void Init() { + TraceLoggingRegisterEx(kFirefoxTraceLoggingProvider, ETWEnableCallback, + nullptr); +} + +void Shutdown() { TraceLoggingUnregister(kFirefoxTraceLoggingProvider); } + +} // namespace ETW diff --git a/tools/profiler/core/MicroGeckoProfiler.cpp b/tools/profiler/core/MicroGeckoProfiler.cpp new file mode 100644 index 0000000000..bedb755742 --- /dev/null +++ b/tools/profiler/core/MicroGeckoProfiler.cpp @@ -0,0 +1,203 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "GeckoProfiler.h" + +#include "mozilla/Maybe.h" +#include "nsPrintfCString.h" +#include "public/GeckoTraceEvent.h" + +using namespace mozilla; +using webrtc::trace_event_internal::TraceValueUnion; + +void uprofiler_register_thread(const char* name, void* stacktop) { +#ifdef MOZ_GECKO_PROFILER + profiler_register_thread(name, stacktop); +#endif // MOZ_GECKO_PROFILER +} + +void uprofiler_unregister_thread() { +#ifdef MOZ_GECKO_PROFILER + profiler_unregister_thread(); +#endif // MOZ_GECKO_PROFILER +} + +#ifdef MOZ_GECKO_PROFILER +namespace { +Maybe<MarkerTiming> ToTiming(char phase) { + switch (phase) { + case 'B': + return Some(MarkerTiming::IntervalStart()); + case 'E': + return Some(MarkerTiming::IntervalEnd()); + case 'I': + return Some(MarkerTiming::InstantNow()); + default: + return Nothing(); + } +} + +struct TraceOption { + bool mPassed = false; + ProfilerString8View mName; + Variant<int64_t, bool, double, ProfilerString8View> mValue = AsVariant(false); +}; + +struct TraceMarker { + static constexpr int MAX_NUM_ARGS = 2; + using OptionsType = std::tuple<TraceOption, TraceOption>; + static constexpr mozilla::Span<const char> MarkerTypeName() { + return MakeStringSpan("TraceEvent"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter, + const OptionsType& aArgs) { + auto writeValue = [&](const auto& aName, const auto& aVariant) { + aVariant.match( + [&](const int64_t& aValue) { aWriter.IntProperty(aName, aValue); }, + [&](const bool& aValue) { aWriter.BoolProperty(aName, aValue); }, + [&](const double& aValue) { aWriter.DoubleProperty(aName, aValue); }, + [&](const ProfilerString8View& aValue) { + aWriter.StringProperty(aName, aValue); + }); + }; + if (const auto& arg = std::get<0>(aArgs); arg.mPassed) { + aWriter.StringProperty("name1", arg.mName); + writeValue("val1", arg.mValue); + } + if (const auto& arg = std::get<1>(aArgs); arg.mPassed) { + aWriter.StringProperty("name2", arg.mName); + writeValue("val2", arg.mValue); + } + } + static mozilla::MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable}; + schema.SetChartLabel("{marker.name}"); + schema.SetTableLabel( + "{marker.name} {marker.data.name1} {marker.data.val1} " + "{marker.data.name2} {marker.data.val2}"); + schema.AddKeyLabelFormatSearchable("name1", "Key 1", MS::Format::String, + MS::Searchable::Searchable); + schema.AddKeyLabelFormatSearchable("val1", "Value 1", MS::Format::String, + MS::Searchable::Searchable); + schema.AddKeyLabelFormatSearchable("name2", "Key 2", MS::Format::String, + MS::Searchable::Searchable); + schema.AddKeyLabelFormatSearchable("val2", "Value 2", MS::Format::String, + MS::Searchable::Searchable); + return schema; + } +}; +} // namespace + +namespace mozilla { +template <> +struct ProfileBufferEntryWriter::Serializer<TraceOption> { + static Length Bytes(const TraceOption& aOption) { + // 1 byte to store passed flag, then object size if passed. + return aOption.mPassed ? (1 + SumBytes(aOption.mName, aOption.mValue)) : 1; + } + + static void Write(ProfileBufferEntryWriter& aEW, const TraceOption& aOption) { + // 'T'/'t' is just an arbitrary 1-byte value to distinguish states. + if (aOption.mPassed) { + aEW.WriteObject<char>('T'); + // Use the Serializer for the name/value pair. + aEW.WriteObject(aOption.mName); + aEW.WriteObject(aOption.mValue); + } else { + aEW.WriteObject<char>('t'); + } + } +}; + +template <> +struct ProfileBufferEntryReader::Deserializer<TraceOption> { + static void ReadInto(ProfileBufferEntryReader& aER, TraceOption& aOption) { + char c = aER.ReadObject<char>(); + if ((aOption.mPassed = (c == 'T'))) { + aER.ReadIntoObject(aOption.mName); + aER.ReadIntoObject(aOption.mValue); + } else { + MOZ_ASSERT(c == 't'); + } + } + + static TraceOption Read(ProfileBufferEntryReader& aER) { + TraceOption option; + ReadInto(aER, option); + return option; + } +}; +} // namespace mozilla +#endif // MOZ_GECKO_PROFILER + +void uprofiler_simple_event_marker(const char* name, char phase, int num_args, + const char** arg_names, + const unsigned char* arg_types, + const unsigned long long* arg_values) { +#ifdef MOZ_GECKO_PROFILER + if (!profiler_thread_is_being_profiled_for_markers()) { + return; + } + Maybe<MarkerTiming> timing = ToTiming(phase); + if (!timing) { + if (getenv("MOZ_LOG_UNKNOWN_TRACE_EVENT_PHASES")) { + fprintf(stderr, "XXX UProfiler: phase not handled: '%c'\n", phase); + } + return; + } + MOZ_ASSERT(num_args <= TraceMarker::MAX_NUM_ARGS); + TraceMarker::OptionsType tuple; + TraceOption* args[2] = {&std::get<0>(tuple), &std::get<1>(tuple)}; + for (int i = 0; i < std::min(num_args, TraceMarker::MAX_NUM_ARGS); ++i) { + auto& arg = *args[i]; + arg.mPassed = true; + arg.mName = ProfilerString8View::WrapNullTerminatedString(arg_names[i]); + switch (arg_types[i]) { + case TRACE_VALUE_TYPE_UINT: + MOZ_ASSERT(arg_values[i] <= std::numeric_limits<int64_t>::max()); + arg.mValue = AsVariant(static_cast<int64_t>( + reinterpret_cast<const TraceValueUnion*>(&arg_values[i])->as_uint)); + break; + case TRACE_VALUE_TYPE_INT: + arg.mValue = AsVariant(static_cast<int64_t>( + reinterpret_cast<const TraceValueUnion*>(&arg_values[i])->as_int)); + break; + case TRACE_VALUE_TYPE_BOOL: + arg.mValue = AsVariant( + reinterpret_cast<const TraceValueUnion*>(&arg_values[i])->as_bool); + break; + case TRACE_VALUE_TYPE_DOUBLE: + arg.mValue = + AsVariant(reinterpret_cast<const TraceValueUnion*>(&arg_values[i]) + ->as_double); + break; + case TRACE_VALUE_TYPE_POINTER: + arg.mValue = AsVariant(ProfilerString8View(nsPrintfCString( + "%p", reinterpret_cast<const TraceValueUnion*>(&arg_values[i]) + ->as_pointer))); + break; + case TRACE_VALUE_TYPE_STRING: + arg.mValue = AsVariant(ProfilerString8View::WrapNullTerminatedString( + reinterpret_cast<const TraceValueUnion*>(&arg_values[i]) + ->as_string)); + break; + case TRACE_VALUE_TYPE_COPY_STRING: + arg.mValue = AsVariant(ProfilerString8View( + nsCString(reinterpret_cast<const TraceValueUnion*>(&arg_values[i]) + ->as_string))); + break; + default: + MOZ_ASSERT_UNREACHABLE("Unexpected trace value type"); + arg.mValue = AsVariant(ProfilerString8View( + nsPrintfCString("Unexpected type: %u", arg_types[i]))); + break; + } + } + profiler_add_marker(ProfilerString8View::WrapNullTerminatedString(name), + geckoprofiler::category::MEDIA_RT, {timing.extract()}, + TraceMarker{}, tuple); +#endif // MOZ_GECKO_PROFILER +} diff --git a/tools/profiler/core/PageInformation.cpp b/tools/profiler/core/PageInformation.cpp new file mode 100644 index 0000000000..83d2d508a1 --- /dev/null +++ b/tools/profiler/core/PageInformation.cpp @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "PageInformation.h" + +#include "mozilla/ProfileJSONWriter.h" + +PageInformation::PageInformation(uint64_t aTabID, uint64_t aInnerWindowID, + const nsCString& aUrl, + uint64_t aEmbedderInnerWindowID, + bool aIsPrivateBrowsing) + : mTabID(aTabID), + mInnerWindowID(aInnerWindowID), + mUrl(aUrl), + mEmbedderInnerWindowID(aEmbedderInnerWindowID), + mIsPrivateBrowsing(aIsPrivateBrowsing) {} + +bool PageInformation::Equals(PageInformation* aOtherPageInfo) const { + // It's enough to check inner window IDs because they are unique for each + // page. Therefore, we don't have to check the tab ID or url. + return InnerWindowID() == aOtherPageInfo->InnerWindowID(); +} + +void PageInformation::StreamJSON(SpliceableJSONWriter& aWriter) const { + // Here, we are converting uint64_t to double. Both tab and Inner + // Window IDs are created using `nsContentUtils::GenerateProcessSpecificId`, + // which is specifically designed to only use 53 of the 64 bits to be lossless + // when passed into and out of JS as a double. + aWriter.StartObjectElement(); + aWriter.DoubleProperty("tabID", TabID()); + aWriter.DoubleProperty("innerWindowID", InnerWindowID()); + aWriter.StringProperty("url", Url()); + aWriter.DoubleProperty("embedderInnerWindowID", EmbedderInnerWindowID()); + aWriter.BoolProperty("isPrivateBrowsing", IsPrivateBrowsing()); + aWriter.EndObject(); +} + +size_t PageInformation::SizeOfIncludingThis( + mozilla::MallocSizeOf aMallocSizeOf) const { + return aMallocSizeOf(this); +} diff --git a/tools/profiler/core/PageInformation.h b/tools/profiler/core/PageInformation.h new file mode 100644 index 0000000000..6c9039b9a4 --- /dev/null +++ b/tools/profiler/core/PageInformation.h @@ -0,0 +1,68 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef PageInformation_h +#define PageInformation_h + +#include "mozilla/Maybe.h" +#include "mozilla/MemoryReporting.h" +#include "nsISupportsImpl.h" +#include "nsString.h" + +namespace mozilla { +namespace baseprofiler { +class SpliceableJSONWriter; +} // namespace baseprofiler +} // namespace mozilla + +// This class contains information that's relevant to a single page only +// while the page information is important and registered with the profiler, +// but regardless of whether the profiler is running. All accesses to it are +// protected by the profiler state lock. +// When the page gets unregistered, we keep the profiler buffer position +// to determine if we are still using this page. If not, we unregister +// it in the next page registration. +class PageInformation final { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(PageInformation) + PageInformation(uint64_t aTabID, uint64_t aInnerWindowID, + const nsCString& aUrl, uint64_t aEmbedderInnerWindowID, + bool aIsPrivateBrowsing); + + size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) const; + bool Equals(PageInformation* aOtherPageInfo) const; + void StreamJSON(mozilla::baseprofiler::SpliceableJSONWriter& aWriter) const; + + uint64_t InnerWindowID() const { return mInnerWindowID; } + uint64_t TabID() const { return mTabID; } + const nsCString& Url() const { return mUrl; } + uint64_t EmbedderInnerWindowID() const { return mEmbedderInnerWindowID; } + bool IsPrivateBrowsing() const { return mIsPrivateBrowsing; } + + mozilla::Maybe<uint64_t> BufferPositionWhenUnregistered() const { + return mBufferPositionWhenUnregistered; + } + + void NotifyUnregistered(uint64_t aBufferPosition) { + mBufferPositionWhenUnregistered = mozilla::Some(aBufferPosition); + } + + private: + const uint64_t mTabID; + const uint64_t mInnerWindowID; + const nsCString mUrl; + const uint64_t mEmbedderInnerWindowID; + const bool mIsPrivateBrowsing; + + // Holds the buffer position when page is unregistered. + // It's used to determine if we still use this page in the profiler or + // not. + mozilla::Maybe<uint64_t> mBufferPositionWhenUnregistered; + + virtual ~PageInformation() = default; +}; + +#endif // PageInformation_h diff --git a/tools/profiler/core/PlatformMacros.h b/tools/profiler/core/PlatformMacros.h new file mode 100644 index 0000000000..c72e94c128 --- /dev/null +++ b/tools/profiler/core/PlatformMacros.h @@ -0,0 +1,130 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef PLATFORM_MACROS_H +#define PLATFORM_MACROS_H + +// Define platform selection macros in a consistent way. Don't add anything +// else to this file, so it can remain freestanding. The primary factorisation +// is on (ARCH,OS) pairs ("PLATforms") but ARCH_ and OS_ macros are defined +// too, since they are sometimes convenient. +// +// Note: "GP" is short for "Gecko Profiler". + +#undef GP_PLAT_x86_android +#undef GP_PLAT_amd64_android +#undef GP_PLAT_arm_android +#undef GP_PLAT_arm64_android +#undef GP_PLAT_x86_linux +#undef GP_PLAT_amd64_linux +#undef GP_PLAT_arm_linux +#undef GP_PLAT_mips64_linux +#undef GP_PLAT_amd64_darwin +#undef GP_PLAT_arm64_darwin +#undef GP_PLAT_x86_windows +#undef GP_PLAT_amd64_windows +#undef GP_PLAT_arm64_windows + +#undef GP_ARCH_x86 +#undef GP_ARCH_amd64 +#undef GP_ARCH_arm +#undef GP_ARCH_arm64 +#undef GP_ARCH_mips64 + +#undef GP_OS_android +#undef GP_OS_linux +#undef GP_OS_darwin +#undef GP_OS_windows + +// We test __ANDROID__ before __linux__ because __linux__ is defined on both +// Android and Linux, whereas GP_OS_android is not defined on vanilla Linux. + +#if defined(__ANDROID__) && defined(__i386__) +# define GP_PLAT_x86_android 1 +# define GP_ARCH_x86 1 +# define GP_OS_android 1 + +#elif defined(__ANDROID__) && defined(__x86_64__) +# define GP_PLAT_amd64_android 1 +# define GP_ARCH_amd64 1 +# define GP_OS_android 1 + +#elif defined(__ANDROID__) && defined(__arm__) +# define GP_PLAT_arm_android 1 +# define GP_ARCH_arm 1 +# define GP_OS_android 1 + +#elif defined(__ANDROID__) && defined(__aarch64__) +# define GP_PLAT_arm64_android 1 +# define GP_ARCH_arm64 1 +# define GP_OS_android 1 + +#elif defined(__linux__) && defined(__i386__) +# define GP_PLAT_x86_linux 1 +# define GP_ARCH_x86 1 +# define GP_OS_linux 1 + +#elif defined(__linux__) && defined(__x86_64__) +# define GP_PLAT_amd64_linux 1 +# define GP_ARCH_amd64 1 +# define GP_OS_linux 1 + +#elif defined(__linux__) && defined(__arm__) +# define GP_PLAT_arm_linux 1 +# define GP_ARCH_arm 1 +# define GP_OS_linux 1 + +#elif defined(__linux__) && defined(__aarch64__) +# define GP_PLAT_arm64_linux 1 +# define GP_ARCH_arm64 1 +# define GP_OS_linux 1 + +#elif defined(__linux__) && defined(__mips64) +# define GP_PLAT_mips64_linux 1 +# define GP_ARCH_mips64 1 +# define GP_OS_linux 1 + +#elif defined(__APPLE__) && defined(__aarch64__) +# define GP_PLAT_arm64_darwin 1 +# define GP_ARCH_arm64 1 +# define GP_OS_darwin 1 + +#elif defined(__APPLE__) && defined(__x86_64__) +# define GP_PLAT_amd64_darwin 1 +# define GP_ARCH_amd64 1 +# define GP_OS_darwin 1 + +#elif defined(__FreeBSD__) && defined(__x86_64__) +# define GP_PLAT_amd64_freebsd 1 +# define GP_ARCH_amd64 1 +# define GP_OS_freebsd 1 + +#elif defined(__FreeBSD__) && defined(__aarch64__) +# define GP_PLAT_arm64_freebsd 1 +# define GP_ARCH_arm64 1 +# define GP_OS_freebsd 1 + +#elif (defined(_MSC_VER) || defined(__MINGW32__)) && \ + (defined(_M_IX86) || defined(__i386__)) +# define GP_PLAT_x86_windows 1 +# define GP_ARCH_x86 1 +# define GP_OS_windows 1 + +#elif (defined(_MSC_VER) || defined(__MINGW32__)) && \ + (defined(_M_X64) || defined(__x86_64__)) +# define GP_PLAT_amd64_windows 1 +# define GP_ARCH_amd64 1 +# define GP_OS_windows 1 + +#elif defined(_MSC_VER) && defined(_M_ARM64) +# define GP_PLAT_arm64_windows 1 +# define GP_ARCH_arm64 1 +# define GP_OS_windows 1 + +#else +# error "Unsupported platform" +#endif + +#endif /* ndef PLATFORM_MACROS_H */ diff --git a/tools/profiler/core/PowerCounters-linux.cpp b/tools/profiler/core/PowerCounters-linux.cpp new file mode 100644 index 0000000000..006cea4867 --- /dev/null +++ b/tools/profiler/core/PowerCounters-linux.cpp @@ -0,0 +1,287 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "PowerCounters.h" +#include "nsXULAppAPI.h" +#include "mozilla/Maybe.h" +#include "mozilla/Logging.h" + +#include <sys/syscall.h> +#include <sys/ioctl.h> +#include <unistd.h> + +#include <cerrno> +#include <cinttypes> +#include <cstdio> +#include <cstdlib> +#include <fstream> +#include <string> + +#include <linux/perf_event.h> + +// From the kernel rapl_scale() function: +// +// > users must then scale back: count * 1/(1e9*2^32) to get Joules +#define PERF_EVENT_SCALE_NANOJOULES 2.3283064365386962890625e-1 +#define SCALE_NANOJOULES_TO_PICOWATTHOUR 3.6 +#define SYSFS_PERF_POWER_TYPE_PATH "/sys/bus/event_source/devices/power/type" + +static mozilla::LazyLogModule sRaplEventLog("profiler.rapl"); +#define RAPL_LOG(...) \ + MOZ_LOG(sRaplEventLog, mozilla::LogLevel::Debug, (__VA_ARGS__)); + +enum class RaplEventType : uint64_t { + RAPL_ENERGY_CORES = 0x01, + RAPL_ENERGY_PKG = 0x02, + RAPL_ENERGY_DRAM = 0x03, + RAPL_ENERGY_GPU = 0x04, + RAPL_ENERGY_PSYS = 0x05, +}; + +struct RaplDomain { + RaplEventType mRaplEventType; + const char* mLabel; + const char* mDescription; +}; + +constexpr RaplDomain kSupportedRaplDomains[] = { + {RaplEventType::RAPL_ENERGY_CORES, "Power: CPU cores", + "Consumption of all physical cores"}, + { + RaplEventType::RAPL_ENERGY_PKG, + "Power: CPU package", + "Consumption of the whole processor package", + }, + { + RaplEventType::RAPL_ENERGY_DRAM, + "Power: DRAM", + "Consumption of the dram domain", + }, + { + RaplEventType::RAPL_ENERGY_GPU, + "Power: iGPU", + "Consumption of the builtin-gpu domain", + }, + { + RaplEventType::RAPL_ENERGY_PSYS, + "Power: System", + "Consumption of the builtin-psys domain", + }}; + +static std::string GetSysfsFileID(RaplEventType aEventType) { + switch (aEventType) { + case RaplEventType::RAPL_ENERGY_CORES: + return "cores"; + case RaplEventType::RAPL_ENERGY_PKG: + return "pkg"; + case RaplEventType::RAPL_ENERGY_DRAM: + return "ram"; + case RaplEventType::RAPL_ENERGY_GPU: + return "gpu"; + case RaplEventType::RAPL_ENERGY_PSYS: + return "psys"; + } + + return ""; +} + +static double GetRaplPerfEventScale(RaplEventType aEventType) { + const std::string sysfsFileName = + "/sys/bus/event_source/devices/power/events/energy-" + + GetSysfsFileID(aEventType) + ".scale"; + std::ifstream sysfsFile(sysfsFileName); + + if (!sysfsFile) { + return PERF_EVENT_SCALE_NANOJOULES; + } + + double scale; + + if (sysfsFile >> scale) { + RAPL_LOG("Read scale from %s: %.22e", sysfsFileName.c_str(), scale); + return scale * 1e9; + } + + return PERF_EVENT_SCALE_NANOJOULES; +} + +static uint64_t GetRaplPerfEventConfig(RaplEventType aEventType) { + const std::string sysfsFileName = + "/sys/bus/event_source/devices/power/events/energy-" + + GetSysfsFileID(aEventType); + std::ifstream sysfsFile(sysfsFileName); + + if (!sysfsFile) { + return static_cast<uint64_t>(aEventType); + } + + char buffer[7] = {}; + const std::string key = "event="; + + if (!sysfsFile.get(buffer, static_cast<std::streamsize>(key.length()) + 1) || + key != buffer) { + return static_cast<uint64_t>(aEventType); + } + + uint64_t config; + + if (sysfsFile >> std::hex >> config) { + RAPL_LOG("Read config from %s: 0x%" PRIx64, sysfsFileName.c_str(), config); + return config; + } + + return static_cast<uint64_t>(aEventType); +} + +class RaplProfilerCount final : public BaseProfilerCount { + public: + explicit RaplProfilerCount(int aPerfEventType, + const RaplEventType& aPerfEventConfig, + const char* aLabel, const char* aDescription) + : BaseProfilerCount(aLabel, nullptr, nullptr, "power", aDescription), + mLastResult(0), + mPerfEventFd(-1) { + RAPL_LOG("Creating RAPL Event for type: %s", mLabel); + + // Optimize for ease of use and do not set an excludes value. This + // ensures we do not require PERF_PMU_CAP_NO_EXCLUDE. + struct perf_event_attr attr = {0}; + memset(&attr, 0, sizeof(attr)); + attr.type = aPerfEventType; + attr.size = sizeof(struct perf_event_attr); + attr.config = GetRaplPerfEventConfig(aPerfEventConfig); + attr.sample_period = 0; + attr.sample_type = PERF_SAMPLE_IDENTIFIER; + attr.inherit = 1; + + RAPL_LOG("Config for event %s: 0x%llx", mLabel, attr.config); + + mEventScale = GetRaplPerfEventScale(aPerfEventConfig); + RAPL_LOG("Scale for event %s: %.22e", mLabel, mEventScale); + + long fd = syscall(__NR_perf_event_open, &attr, -1, 0, -1, 0); + if (fd < 0) { + RAPL_LOG("Event descriptor creation failed for event: %s", mLabel); + mPerfEventFd = -1; + return; + } + + RAPL_LOG("Created descriptor for event: %s", mLabel) + mPerfEventFd = static_cast<int>(fd); + } + + ~RaplProfilerCount() { + if (ValidPerfEventFd()) { + ioctl(mPerfEventFd, PERF_EVENT_IOC_DISABLE, 0); + close(mPerfEventFd); + } + } + + RaplProfilerCount(const RaplProfilerCount&) = delete; + RaplProfilerCount& operator=(const RaplProfilerCount&) = delete; + + CountSample Sample() override { + CountSample result = { + .count = 0, + .number = 0, + .isSampleNew = false, + }; + mozilla::Maybe<uint64_t> raplEventResult = ReadEventFd(); + + if (raplEventResult.isNothing()) { + return result; + } + + // We need to return picowatthour to be consistent with the Windows + // EMI API. As a result, the scale calculation should: + // + // - Convert the returned value to nanojoules + // - Convert nanojoules to picowatthour + double nanojoules = + static_cast<double>(raplEventResult.value()) * mEventScale; + double picowatthours = nanojoules / SCALE_NANOJOULES_TO_PICOWATTHOUR; + RAPL_LOG("Sample %s { count: %lu, last-result: %lu } = %lfJ", mLabel, + raplEventResult.value(), mLastResult, nanojoules * 1e-9); + + result.count = static_cast<int64_t>(picowatthours); + + // If the tick count is the same as the returned value or if this is the + // first sample, treat this sample as a duplicate. + result.isSampleNew = + (mLastResult != 0 && mLastResult != raplEventResult.value() && + result.count >= 0); + mLastResult = raplEventResult.value(); + + return result; + } + + bool ValidPerfEventFd() { return mPerfEventFd >= 0; } + + private: + mozilla::Maybe<uint64_t> ReadEventFd() { + MOZ_ASSERT(ValidPerfEventFd()); + + uint64_t eventResult; + ssize_t readBytes = read(mPerfEventFd, &eventResult, sizeof(uint64_t)); + if (readBytes != sizeof(uint64_t)) { + RAPL_LOG("Invalid RAPL event read size: %ld", readBytes); + return mozilla::Nothing(); + } + + return mozilla::Some(eventResult); + } + + uint64_t mLastResult; + int mPerfEventFd; + double mEventScale; +}; + +static int GetRaplPerfEventType() { + FILE* fp = fopen(SYSFS_PERF_POWER_TYPE_PATH, "r"); + if (!fp) { + RAPL_LOG("Open of " SYSFS_PERF_POWER_TYPE_PATH " failed"); + return -1; + } + + int readTypeValue = -1; + if (fscanf(fp, "%d", &readTypeValue) != 1) { + RAPL_LOG("Read of " SYSFS_PERF_POWER_TYPE_PATH " failed"); + } + fclose(fp); + + return readTypeValue; +} + +PowerCounters::PowerCounters() { + if (!XRE_IsParentProcess()) { + // Energy meters are global, so only sample them on the parent. + return; + } + + // Get the value perf_event_attr.type should be set to for RAPL + // perf events. + int perfEventType = GetRaplPerfEventType(); + if (perfEventType < 0) { + RAPL_LOG("Failed to find the event type for RAPL perf events."); + return; + } + + for (const auto& raplEventDomain : kSupportedRaplDomains) { + RaplProfilerCount* raplEvent = new RaplProfilerCount( + perfEventType, raplEventDomain.mRaplEventType, raplEventDomain.mLabel, + raplEventDomain.mDescription); + if (!raplEvent->ValidPerfEventFd() || !mCounters.emplaceBack(raplEvent)) { + delete raplEvent; + } + } +} + +PowerCounters::~PowerCounters() { + for (auto* raplEvent : mCounters) { + delete raplEvent; + } + mCounters.clear(); +} + +void PowerCounters::Sample() {} diff --git a/tools/profiler/core/PowerCounters-mac-amd64.cpp b/tools/profiler/core/PowerCounters-mac-amd64.cpp new file mode 100644 index 0000000000..540cee155d --- /dev/null +++ b/tools/profiler/core/PowerCounters-mac-amd64.cpp @@ -0,0 +1,419 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "PowerCounters.h" +#include "nsDebug.h" +#include "nsPrintfCString.h" +#include "nsXULAppAPI.h" // for XRE_IsParentProcess + +// Because of the pkg_energy_statistics_t::pkes_version check below, the +// earliest OS X version this code will work with is 10.9.0 (xnu-2422.1.72). + +#include <sys/types.h> +#include <sys/sysctl.h> + +// OS X has four kinds of system calls: +// +// 1. Mach traps; +// 2. UNIX system calls; +// 3. machine-dependent calls; +// 4. diagnostic calls. +// +// (See "Mac OS X and iOS Internals" by Jonathan Levin for more details.) +// +// The last category has a single call named diagCall() or diagCall64(). Its +// mode is controlled by its first argument, and one of the modes allows access +// to the Intel RAPL MSRs. +// +// The interface to diagCall64() is not exported, so we have to import some +// definitions from the XNU kernel. All imported definitions are annotated with +// the XNU source file they come from, and information about what XNU versions +// they were introduced in and (if relevant) modified. + +// The diagCall64() mode. +// From osfmk/i386/Diagnostics.h +// - In 10.8.4 (xnu-2050.24.15) this value was introduced. (In 10.8.3 the value +// 17 was used for dgGzallocTest.) +#define dgPowerStat 17 + +// From osfmk/i386/cpu_data.h +// - In 10.8.5 these values were introduced, along with core_energy_stat_t. +#define CPU_RTIME_BINS (12) +#define CPU_ITIME_BINS (CPU_RTIME_BINS) + +// core_energy_stat_t and pkg_energy_statistics_t are both from +// osfmk/i386/Diagnostics.c. +// - In 10.8.4 (xnu-2050.24.15) both structs were introduced, but with many +// fewer fields. +// - In 10.8.5 (xnu-2050.48.11) both structs were substantially expanded, with +// numerous new fields. +// - In 10.9.0 (xnu-2422.1.72) pkg_energy_statistics_t::pkes_version was added. +// diagCall64(dgPowerStat) fills it with '1' in all versions since (up to +// 10.10.2 at time of writing). +// - in 10.10.2 (xnu-2782.10.72) core_energy_stat_t::gpmcs was conditionally +// added, if DIAG_ALL_PMCS is true. (DIAG_ALL_PMCS is not even defined in the +// source code, but it could be defined at compile-time via compiler flags.) +// pkg_energy_statistics_t::pkes_version did not change, though. + +typedef struct { + uint64_t caperf; + uint64_t cmperf; + uint64_t ccres[6]; + uint64_t crtimes[CPU_RTIME_BINS]; + uint64_t citimes[CPU_ITIME_BINS]; + uint64_t crtime_total; + uint64_t citime_total; + uint64_t cpu_idle_exits; + uint64_t cpu_insns; + uint64_t cpu_ucc; + uint64_t cpu_urc; +#if DIAG_ALL_PMCS // Added in 10.10.2 (xnu-2782.10.72). + uint64_t gpmcs[4]; // Added in 10.10.2 (xnu-2782.10.72). +#endif /* DIAG_ALL_PMCS */ // Added in 10.10.2 (xnu-2782.10.72). +} core_energy_stat_t; + +typedef struct { + uint64_t pkes_version; // Added in 10.9.0 (xnu-2422.1.72). + uint64_t pkg_cres[2][7]; + + // This is read from MSR 0x606, which Intel calls MSR_RAPL_POWER_UNIT + // and XNU calls MSR_IA32_PKG_POWER_SKU_UNIT. + uint64_t pkg_power_unit; + + // These are the four fields for the four RAPL domains. For each field + // we list: + // + // - the corresponding MSR number; + // - Intel's name for that MSR; + // - XNU's name for that MSR; + // - which Intel processors the MSR is supported on. + // + // The last of these is determined from chapter 35 of Volume 3 of the + // "Intel 64 and IA-32 Architecture's Software Developer's Manual", + // Order Number 325384. (Note that chapter 35 contradicts section 14.9 + // to some degree.) + + // 0x611 == MSR_PKG_ENERGY_STATUS == MSR_IA32_PKG_ENERGY_STATUS + // Atom (various), Sandy Bridge, Next Gen Xeon Phi (model 0x57). + uint64_t pkg_energy; + + // 0x639 == MSR_PP0_ENERGY_STATUS == MSR_IA32_PP0_ENERGY_STATUS + // Atom (various), Sandy Bridge, Next Gen Xeon Phi (model 0x57). + uint64_t pp0_energy; + + // 0x641 == MSR_PP1_ENERGY_STATUS == MSR_PP1_ENERGY_STATUS + // Sandy Bridge, Haswell. + uint64_t pp1_energy; + + // 0x619 == MSR_DRAM_ENERGY_STATUS == MSR_IA32_DDR_ENERGY_STATUS + // Xeon E5, Xeon E5 v2, Haswell/Haswell-E, Next Gen Xeon Phi (model + // 0x57) + uint64_t ddr_energy; + + uint64_t llc_flushed_cycles; + uint64_t ring_ratio_instantaneous; + uint64_t IA_frequency_clipping_cause; + uint64_t GT_frequency_clipping_cause; + uint64_t pkg_idle_exits; + uint64_t pkg_rtimes[CPU_RTIME_BINS]; + uint64_t pkg_itimes[CPU_ITIME_BINS]; + uint64_t mbus_delay_time; + uint64_t mint_delay_time; + uint32_t ncpus; + core_energy_stat_t cest[]; +} pkg_energy_statistics_t; + +static int diagCall64(uint64_t aMode, void* aBuf) { + // We cannot use syscall() here because it doesn't work with diagnostic + // system calls -- it raises SIGSYS if you try. So we have to use asm. + +#ifdef __x86_64__ + // The 0x40000 prefix indicates it's a diagnostic system call. The 0x01 + // suffix indicates the syscall number is 1, which also happens to be the + // only diagnostic system call. See osfmk/mach/i386/syscall_sw.h for more + // details. + static const uint64_t diagCallNum = 0x4000001; + uint64_t rv; + + __asm__ __volatile__( + "syscall" + + // Return value goes in "a" (%rax). + : /* outputs */ "=a"(rv) + + // The syscall number goes in "0", a synonym (from outputs) for "a" + // (%rax). The syscall arguments go in "D" (%rdi) and "S" (%rsi). + : /* inputs */ "0"(diagCallNum), "D"(aMode), "S"(aBuf) + + // The |syscall| instruction clobbers %rcx, %r11, and %rflags ("cc"). And + // this particular syscall also writes memory (aBuf). + : /* clobbers */ "rcx", "r11", "cc", "memory"); + return rv; +#else +# error Sorry, only x86-64 is supported +#endif +} + +// This is a counter to collect power utilization during profiling. +// It cannot be a raw `ProfilerCounter` because we need to manually add/remove +// it while the profiler lock is already held. +class RaplDomain final : public BaseProfilerCount { + public: + explicit RaplDomain(const char* aLabel, const char* aDescription) + : BaseProfilerCount(aLabel, nullptr, nullptr, "power", aDescription), + mSample(0), + mEnergyStatusUnits(0), + mWrapAroundCount(0), + mIsSampleNew(false) {} + + CountSample Sample() override { + CountSample result; + + // To be consistent with the Windows EMI API, + // return values in picowatt-hour. + constexpr double NANOJOULES_PER_JOULE = 1'000'000'000; + constexpr double NANOJOULES_TO_PICOWATTHOUR = 3.6; + + uint64_t ticks = (uint64_t(mWrapAroundCount) << 32) + mSample; + double joulesPerTick = (double)1 / (1 << mEnergyStatusUnits); + result.count = static_cast<double>(ticks) * joulesPerTick * + NANOJOULES_PER_JOULE / NANOJOULES_TO_PICOWATTHOUR; + + result.number = 0; + result.isSampleNew = mIsSampleNew; + mIsSampleNew = false; + return result; + } + + void AddSample(uint32_t aSample, uint32_t aEnergyStatusUnits) { + if (aSample == mSample) { + return; + } + + mEnergyStatusUnits = aEnergyStatusUnits; + + if (aSample > mSample) { + mIsSampleNew = true; + mSample = aSample; + return; + } + + // Despite being returned in uint64_t fields, the power counter values + // only use the lowest 32 bits of their fields, and we need to handle + // wraparounds to avoid our power tracks stopping after a few hours. + constexpr uint32_t highestBit = 1 << 31; + if ((mSample & highestBit) && !(aSample & highestBit)) { + mIsSampleNew = true; + ++mWrapAroundCount; + mSample = aSample; + } else { + NS_WARNING("unexpected sample with smaller value"); + } + } + + private: + uint32_t mSample; + uint32_t mEnergyStatusUnits; + uint32_t mWrapAroundCount; + bool mIsSampleNew; +}; + +class RAPL { + bool mIsGpuSupported; // Is the GPU domain supported by the processor? + bool mIsRamSupported; // Is the RAM domain supported by the processor? + + // The DRAM domain on Haswell servers has a fixed energy unit (1/65536 J == + // 15.3 microJoules) which is different to the power unit MSR. (See the + // "Intel Xeon Processor E5-1600 and E5-2600 v3 Product Families, Volume 2 of + // 2, Registers" datasheet, September 2014, Reference Number: 330784-001.) + // This field records whether the quirk is present. + bool mHasRamUnitsQuirk; + + // The abovementioned 15.3 microJoules value. (2^16 = 65536) + static constexpr double kQuirkyRamEnergyStatusUnits = 16; + + // The struct passed to diagCall64(). + pkg_energy_statistics_t* mPkes; + + RaplDomain* mPkg = nullptr; + RaplDomain* mCores = nullptr; + RaplDomain* mGpu = nullptr; + RaplDomain* mRam = nullptr; + + public: + explicit RAPL(PowerCounters::CountVector& aCounters) + : mHasRamUnitsQuirk(false) { + // Work out which RAPL MSRs this CPU model supports. + int cpuModel; + size_t size = sizeof(cpuModel); + if (sysctlbyname("machdep.cpu.model", &cpuModel, &size, NULL, 0) != 0) { + NS_WARNING("sysctlbyname(\"machdep.cpu.model\") failed"); + return; + } + + // This is similar to arch/x86/kernel/cpu/perf_event_intel_rapl.c in + // linux-4.1.5/. + // + // By linux-5.6.14/, this stuff had moved into + // arch/x86/events/intel/rapl.c, which references processor families in + // arch/x86/include/asm/intel-family.h. + switch (cpuModel) { + case 0x2a: // Sandy Bridge + case 0x3a: // Ivy Bridge + // Supports package, cores, GPU. + mIsGpuSupported = true; + mIsRamSupported = false; + break; + + case 0x3f: // Haswell X + case 0x4f: // Broadwell X + case 0x55: // Skylake X + case 0x56: // Broadwell D + // Supports package, cores, RAM. Has the units quirk. + mIsGpuSupported = false; + mIsRamSupported = true; + mHasRamUnitsQuirk = true; + break; + + case 0x2d: // Sandy Bridge X + case 0x3e: // Ivy Bridge X + // Supports package, cores, RAM. + mIsGpuSupported = false; + mIsRamSupported = true; + break; + + case 0x3c: // Haswell + case 0x3d: // Broadwell + case 0x45: // Haswell L + case 0x46: // Haswell G + case 0x47: // Broadwell G + // Supports package, cores, GPU, RAM. + mIsGpuSupported = true; + mIsRamSupported = true; + break; + + case 0x4e: // Skylake L + case 0x5e: // Skylake + case 0x8e: // Kaby Lake L + case 0x9e: // Kaby Lake + case 0x66: // Cannon Lake L + case 0x7d: // Ice Lake + case 0x7e: // Ice Lake L + case 0xa5: // Comet Lake + case 0xa6: // Comet Lake L + // Supports package, cores, GPU, RAM, PSYS. + // XXX: this tool currently doesn't measure PSYS. + mIsGpuSupported = true; + mIsRamSupported = true; + break; + + default: + NS_WARNING(nsPrintfCString("unknown CPU model: %d", cpuModel).get()); + return; + } + + // Get the maximum number of logical CPUs so that we know how big to make + // |mPkes|. + int logicalcpu_max; + size = sizeof(logicalcpu_max); + if (sysctlbyname("hw.logicalcpu_max", &logicalcpu_max, &size, NULL, 0) != + 0) { + NS_WARNING("sysctlbyname(\"hw.logicalcpu_max\") failed"); + return; + } + + // Over-allocate by 1024 bytes per CPU to allow for the uncertainty around + // core_energy_stat_t::gpmcs and for any other future extensions to that + // struct. (The fields we read all come before the core_energy_stat_t + // array, so it won't matter to us whether gpmcs is present or not.) + size_t pkesSize = sizeof(pkg_energy_statistics_t) + + logicalcpu_max * sizeof(core_energy_stat_t) + + logicalcpu_max * 1024; + mPkes = (pkg_energy_statistics_t*)malloc(pkesSize); + if (mPkes && aCounters.reserve(4)) { + mPkg = new RaplDomain("Power: CPU package", "RAPL PKG"); + aCounters.infallibleAppend(mPkg); + + mCores = new RaplDomain("Power: CPU cores", "RAPL PP0"); + aCounters.infallibleAppend(mCores); + + if (mIsGpuSupported) { + mGpu = new RaplDomain("Power: iGPU", "RAPL PP1"); + aCounters.infallibleAppend(mGpu); + } + + if (mIsRamSupported) { + mRam = new RaplDomain("Power: DRAM", "RAPL DRAM"); + aCounters.infallibleAppend(mRam); + } + } + } + + ~RAPL() { + free(mPkes); + delete mPkg; + delete mCores; + delete mGpu; + delete mRam; + } + + void Sample() { + constexpr uint64_t kSupportedVersion = 1; + + // If we failed to allocate the memory for package energy statistics, we + // have nothing to sample. + if (MOZ_UNLIKELY(!mPkes)) { + return; + } + + // Write an unsupported version number into pkes_version so that the check + // below cannot succeed by dumb luck. + mPkes->pkes_version = kSupportedVersion - 1; + + // diagCall64() returns 1 on success, and 0 on failure (which can only + // happen if the mode is unrecognized, e.g. in 10.7.x or earlier versions). + if (diagCall64(dgPowerStat, mPkes) != 1) { + NS_WARNING("diagCall64() failed"); + return; + } + + if (mPkes->pkes_version != kSupportedVersion) { + NS_WARNING( + nsPrintfCString("unexpected pkes_version: %llu", mPkes->pkes_version) + .get()); + return; + } + + // Bits 12:8 are the ESU. + // Energy measurements come in multiples of 1/(2^ESU). + uint32_t energyStatusUnits = (mPkes->pkg_power_unit >> 8) & 0x1f; + mPkg->AddSample(mPkes->pkg_energy, energyStatusUnits); + mCores->AddSample(mPkes->pp0_energy, energyStatusUnits); + if (mIsGpuSupported) { + mGpu->AddSample(mPkes->pp1_energy, energyStatusUnits); + } + if (mIsRamSupported) { + mRam->AddSample(mPkes->ddr_energy, mHasRamUnitsQuirk + ? kQuirkyRamEnergyStatusUnits + : energyStatusUnits); + } + } +}; + +PowerCounters::PowerCounters() { + // RAPL values are global, so only sample them on the parent. + mRapl = XRE_IsParentProcess() ? new RAPL(mCounters) : nullptr; +} + +PowerCounters::~PowerCounters() { + mCounters.clear(); + delete mRapl; + mRapl = nullptr; +} + +void PowerCounters::Sample() { + if (mRapl) { + mRapl->Sample(); + } +} diff --git a/tools/profiler/core/PowerCounters-mac-arm64.cpp b/tools/profiler/core/PowerCounters-mac-arm64.cpp new file mode 100644 index 0000000000..3a84a479ef --- /dev/null +++ b/tools/profiler/core/PowerCounters-mac-arm64.cpp @@ -0,0 +1,47 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "PowerCounters.h" + +#include <mach/mach.h> + +class ProcessPower final : public BaseProfilerCount { + public: + ProcessPower() + : BaseProfilerCount("Process Power", nullptr, nullptr, "power", + "Power utilization") {} + + CountSample Sample() override { + CountSample result; + result.count = GetTaskEnergy(); + result.number = 0; + result.isSampleNew = true; + return result; + } + + private: + int64_t GetTaskEnergy() { + task_power_info_v2_data_t task_power_info; + mach_msg_type_number_t count = TASK_POWER_INFO_V2_COUNT; + kern_return_t kr = task_info(mach_task_self(), TASK_POWER_INFO_V2, + (task_info_t)&task_power_info, &count); + if (kr != KERN_SUCCESS) { + return 0; + } + + // task_energy is in nanojoules. To be consistent with the Windows EMI + // API, return values in picowatt-hour. + return task_power_info.task_energy / 3.6; + } +}; + +PowerCounters::PowerCounters() : mProcessPower(new ProcessPower()) { + if (mProcessPower) { + (void)mCounters.append(mProcessPower.get()); + } +} + +PowerCounters::~PowerCounters() { mCounters.clear(); } + +void PowerCounters::Sample() {} diff --git a/tools/profiler/core/PowerCounters-win.cpp b/tools/profiler/core/PowerCounters-win.cpp new file mode 100644 index 0000000000..6e8f492d6d --- /dev/null +++ b/tools/profiler/core/PowerCounters-win.cpp @@ -0,0 +1,310 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "PowerCounters.h" +#include "nsXULAppAPI.h" // for XRE_IsParentProcess +#include "nsString.h" + +#include <windows.h> +#include <devioctl.h> +#include <setupapi.h> // for SetupDi* +// LogSeverity, defined by setupapi.h to DWORD, messes with other code. +#undef LogSeverity + +#include <emi.h> + +using namespace mozilla; + +// This is a counter to collect power utilization during profiling. +// It cannot be a raw `ProfilerCounter` because we need to manually add/remove +// it while the profiler lock is already held. +class PowerMeterChannel final : public BaseProfilerCount { + public: + explicit PowerMeterChannel(const WCHAR* aChannelName, ULONGLONG aInitialValue, + ULONGLONG aInitialTime) + : BaseProfilerCount(nullptr, nullptr, nullptr, "power", + "Power utilization"), + mChannelName(NS_ConvertUTF16toUTF8(aChannelName)), + mPreviousValue(aInitialValue), + mPreviousTime(aInitialTime), + mIsSampleNew(true) { + if (mChannelName.Equals("RAPL_Package0_PKG")) { + mLabel = "Power: CPU package"; + mDescription = mChannelName.get(); + } else if (mChannelName.Equals("RAPL_Package0_PP0")) { + mLabel = "Power: CPU cores"; + mDescription = mChannelName.get(); + } else if (mChannelName.Equals("RAPL_Package0_PP1")) { + mLabel = "Power: iGPU"; + mDescription = mChannelName.get(); + } else if (mChannelName.Equals("RAPL_Package0_DRAM")) { + mLabel = "Power: DRAM"; + mDescription = mChannelName.get(); + } else { + unsigned int coreId; + if (sscanf(mChannelName.get(), "RAPL_Package0_Core%u_CORE", &coreId) == + 1) { + mLabelString = "Power: CPU core "; + mLabelString.AppendInt(coreId); + mLabel = mLabelString.get(); + mDescription = mChannelName.get(); + } else { + mLabel = mChannelName.get(); + } + } + } + + CountSample Sample() override { + CountSample result; + result.count = mCounter; + result.number = 0; + result.isSampleNew = mIsSampleNew; + mIsSampleNew = false; + return result; + } + + void AddSample(ULONGLONG aAbsoluteEnergy, ULONGLONG aAbsoluteTime) { + // aAbsoluteTime is the time since the system start in 100ns increments. + if (aAbsoluteTime == mPreviousTime) { + return; + } + + if (aAbsoluteEnergy > mPreviousValue) { + int64_t increment = aAbsoluteEnergy - mPreviousValue; + mCounter += increment; + mPreviousValue += increment; + mPreviousTime = aAbsoluteTime; + } + + mIsSampleNew = true; + } + + private: + int64_t mCounter; + nsCString mChannelName; + + // Used as a storage when the label can not be a literal string. + nsCString mLabelString; + + ULONGLONG mPreviousValue; + ULONGLONG mPreviousTime; + bool mIsSampleNew; +}; + +class PowerMeterDevice { + public: + explicit PowerMeterDevice(LPCTSTR aDevicePath) { + mHandle = ::CreateFile(aDevicePath, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (mHandle == INVALID_HANDLE_VALUE) { + return; + } + + EMI_VERSION version = {0}; + DWORD dwOut; + + if (!::DeviceIoControl(mHandle, IOCTL_EMI_GET_VERSION, nullptr, 0, &version, + sizeof(version), &dwOut, nullptr) || + (version.EmiVersion != EMI_VERSION_V1 && + version.EmiVersion != EMI_VERSION_V2)) { + return; + } + + EMI_METADATA_SIZE size = {0}; + if (!::DeviceIoControl(mHandle, IOCTL_EMI_GET_METADATA_SIZE, nullptr, 0, + &size, sizeof(size), &dwOut, nullptr) || + !size.MetadataSize) { + return; + } + + UniquePtr<uint8_t[]> metadata(new (std::nothrow) + uint8_t[size.MetadataSize]); + if (!metadata) { + return; + } + + if (version.EmiVersion == EMI_VERSION_V2) { + EMI_METADATA_V2* metadata2 = + reinterpret_cast<EMI_METADATA_V2*>(metadata.get()); + if (!::DeviceIoControl(mHandle, IOCTL_EMI_GET_METADATA, nullptr, 0, + metadata2, size.MetadataSize, &dwOut, nullptr)) { + return; + } + + if (!mChannels.reserve(metadata2->ChannelCount)) { + return; + } + + mDataBuffer = + MakeUnique<EMI_CHANNEL_MEASUREMENT_DATA[]>(metadata2->ChannelCount); + if (!mDataBuffer) { + return; + } + + if (!::DeviceIoControl( + mHandle, IOCTL_EMI_GET_MEASUREMENT, nullptr, 0, mDataBuffer.get(), + sizeof(EMI_CHANNEL_MEASUREMENT_DATA[metadata2->ChannelCount]), + &dwOut, nullptr)) { + return; + } + + EMI_CHANNEL_V2* channel = &metadata2->Channels[0]; + for (int i = 0; i < metadata2->ChannelCount; ++i) { + EMI_CHANNEL_MEASUREMENT_DATA* channel_data = &mDataBuffer[i]; + mChannels.infallibleAppend(new PowerMeterChannel( + channel->ChannelName, channel_data->AbsoluteEnergy, + channel_data->AbsoluteTime)); + channel = EMI_CHANNEL_V2_NEXT_CHANNEL(channel); + } + } else if (version.EmiVersion == EMI_VERSION_V1) { + EMI_METADATA_V1* metadata1 = + reinterpret_cast<EMI_METADATA_V1*>(metadata.get()); + if (!::DeviceIoControl(mHandle, IOCTL_EMI_GET_METADATA, nullptr, 0, + metadata1, size.MetadataSize, &dwOut, nullptr)) { + return; + } + + mDataBuffer = MakeUnique<EMI_CHANNEL_MEASUREMENT_DATA[]>(1); + if (!mDataBuffer) { + return; + } + + if (!::DeviceIoControl( + mHandle, IOCTL_EMI_GET_MEASUREMENT, nullptr, 0, mDataBuffer.get(), + sizeof(EMI_CHANNEL_MEASUREMENT_DATA), &dwOut, nullptr)) { + return; + } + + (void)mChannels.append(new PowerMeterChannel( + metadata1->MeteredHardwareName, mDataBuffer[0].AbsoluteEnergy, + mDataBuffer[0].AbsoluteTime)); + } + } + + ~PowerMeterDevice() { + if (mHandle != INVALID_HANDLE_VALUE) { + ::CloseHandle(mHandle); + } + } + + void Sample() { + MOZ_ASSERT(HasChannels()); + MOZ_ASSERT(mDataBuffer); + + DWORD dwOut; + if (!::DeviceIoControl( + mHandle, IOCTL_EMI_GET_MEASUREMENT, nullptr, 0, mDataBuffer.get(), + sizeof(EMI_CHANNEL_MEASUREMENT_DATA[mChannels.length()]), &dwOut, + nullptr)) { + return; + } + + for (size_t i = 0; i < mChannels.length(); ++i) { + EMI_CHANNEL_MEASUREMENT_DATA* channel_data = &mDataBuffer[i]; + mChannels[i]->AddSample(channel_data->AbsoluteEnergy, + channel_data->AbsoluteTime); + } + } + + bool HasChannels() { return mChannels.length() != 0; } + void AppendCountersTo(PowerCounters::CountVector& aCounters) { + if (aCounters.reserve(aCounters.length() + mChannels.length())) { + for (auto& channel : mChannels) { + aCounters.infallibleAppend(channel.get()); + } + } + } + + private: + Vector<UniquePtr<PowerMeterChannel>, 4> mChannels; + HANDLE mHandle = INVALID_HANDLE_VALUE; + UniquePtr<EMI_CHANNEL_MEASUREMENT_DATA[]> mDataBuffer; +}; + +PowerCounters::PowerCounters() { + class MOZ_STACK_CLASS HDevInfoHolder final { + public: + explicit HDevInfoHolder(HDEVINFO aHandle) : mHandle(aHandle) {} + + ~HDevInfoHolder() { ::SetupDiDestroyDeviceInfoList(mHandle); } + + private: + HDEVINFO mHandle; + }; + + if (!XRE_IsParentProcess()) { + // Energy meters are global, so only sample them on the parent. + return; + } + + // Energy Metering Device Interface + // {45BD8344-7ED6-49cf-A440-C276C933B053} + // + // Using GUID_DEVICE_ENERGY_METER does not compile as the symbol does not + // exist before Windows 10. + GUID my_GUID_DEVICE_ENERGY_METER = { + 0x45bd8344, + 0x7ed6, + 0x49cf, + {0xa4, 0x40, 0xc2, 0x76, 0xc9, 0x33, 0xb0, 0x53}}; + + HDEVINFO hdev = + ::SetupDiGetClassDevs(&my_GUID_DEVICE_ENERGY_METER, nullptr, nullptr, + DIGCF_PRESENT | DIGCF_DEVICEINTERFACE); + if (hdev == INVALID_HANDLE_VALUE) { + return; + } + + HDevInfoHolder hdevHolder(hdev); + + DWORD i = 0; + SP_DEVICE_INTERFACE_DATA did = {0}; + did.cbSize = sizeof(did); + + while (::SetupDiEnumDeviceInterfaces( + hdev, nullptr, &my_GUID_DEVICE_ENERGY_METER, i++, &did)) { + DWORD bufferSize = 0; + ::SetupDiGetDeviceInterfaceDetail(hdev, &did, nullptr, 0, &bufferSize, + nullptr); + if (::GetLastError() != ERROR_INSUFFICIENT_BUFFER) { + continue; + } + + UniquePtr<uint8_t[]> buffer(new (std::nothrow) uint8_t[bufferSize]); + if (!buffer) { + continue; + } + + PSP_DEVICE_INTERFACE_DETAIL_DATA pdidd = + reinterpret_cast<PSP_DEVICE_INTERFACE_DETAIL_DATA>(buffer.get()); + MOZ_ASSERT(uintptr_t(buffer.get()) % + alignof(PSP_DEVICE_INTERFACE_DETAIL_DATA) == + 0); + pdidd->cbSize = sizeof(*pdidd); + if (!::SetupDiGetDeviceInterfaceDetail(hdev, &did, pdidd, bufferSize, + &bufferSize, nullptr)) { + continue; + } + + UniquePtr<PowerMeterDevice> pmd = + MakeUnique<PowerMeterDevice>(pdidd->DevicePath); + if (!pmd->HasChannels() || + !mPowerMeterDevices.emplaceBack(std::move(pmd))) { + NS_WARNING("PowerMeterDevice without measurement channel (or OOM)"); + } + } + + for (auto& device : mPowerMeterDevices) { + device->AppendCountersTo(mCounters); + } +} + +PowerCounters::~PowerCounters() { mCounters.clear(); } + +void PowerCounters::Sample() { + for (auto& device : mPowerMeterDevices) { + device->Sample(); + } +} diff --git a/tools/profiler/core/PowerCounters.h b/tools/profiler/core/PowerCounters.h new file mode 100644 index 0000000000..2fd8d5892c --- /dev/null +++ b/tools/profiler/core/PowerCounters.h @@ -0,0 +1,52 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef TOOLS_POWERCOUNTERS_H_ +#define TOOLS_POWERCOUNTERS_H_ + +#include "PlatformMacros.h" +#include "mozilla/ProfilerCounts.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Vector.h" + +#if defined(_MSC_VER) +class PowerMeterDevice; +#endif +#if defined(GP_PLAT_arm64_darwin) +class ProcessPower; +#endif +#if defined(GP_PLAT_amd64_darwin) +class RAPL; +#endif + +class PowerCounters { + public: +#if defined(_MSC_VER) || defined(GP_OS_darwin) || defined(GP_PLAT_amd64_linux) + explicit PowerCounters(); + ~PowerCounters(); + void Sample(); +#else + explicit PowerCounters(){}; + ~PowerCounters(){}; + void Sample(){}; +#endif + + using CountVector = mozilla::Vector<BaseProfilerCount*, 4>; + const CountVector& GetCounters() { return mCounters; } + + private: + CountVector mCounters; + +#if defined(_MSC_VER) + mozilla::Vector<mozilla::UniquePtr<PowerMeterDevice>> mPowerMeterDevices; +#endif +#if defined(GP_PLAT_arm64_darwin) + mozilla::UniquePtr<ProcessPower> mProcessPower; +#endif +#if defined(GP_PLAT_amd64_darwin) + RAPL* mRapl; +#endif +}; + +#endif /* ndef TOOLS_POWERCOUNTERS_H_ */ diff --git a/tools/profiler/core/ProfileAdditionalInformation.cpp b/tools/profiler/core/ProfileAdditionalInformation.cpp new file mode 100644 index 0000000000..5c9a0d52e4 --- /dev/null +++ b/tools/profiler/core/ProfileAdditionalInformation.cpp @@ -0,0 +1,94 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfileAdditionalInformation.h" + +#include "jsapi.h" +#include "js/JSON.h" +#include "js/PropertyAndElement.h" +#include "js/Value.h" +#include "mozilla/JSONStringWriteFuncs.h" +#include "mozilla/ipc/IPDLParamTraits.h" + +#ifdef MOZ_GECKO_PROFILER +# include "platform.h" + +void mozilla::ProfileGenerationAdditionalInformation::ToJSValue( + JSContext* aCx, JS::MutableHandle<JS::Value> aRetVal) const { + // Get the shared libraries array. + JS::Rooted<JS::Value> sharedLibrariesVal(aCx); + { + JSONStringWriteFunc<nsCString> buffer; + JSONWriter w(buffer, JSONWriter::SingleLineStyle); + w.StartArrayElement(); + AppendSharedLibraries(w, mSharedLibraries); + w.EndArray(); + NS_ConvertUTF8toUTF16 buffer16(buffer.StringCRef()); + MOZ_ALWAYS_TRUE(JS_ParseJSON(aCx, + static_cast<const char16_t*>(buffer16.get()), + buffer16.Length(), &sharedLibrariesVal)); + } + + JS::Rooted<JSObject*> additionalInfoObj(aCx, JS_NewPlainObject(aCx)); + JS_SetProperty(aCx, additionalInfoObj, "sharedLibraries", sharedLibrariesVal); + aRetVal.setObject(*additionalInfoObj); +} +#endif // MOZ_GECKO_PROFILER + +namespace IPC { + +void IPC::ParamTraits<SharedLibrary>::Write(MessageWriter* aWriter, + const paramType& aParam) { + WriteParam(aWriter, aParam.mStart); + WriteParam(aWriter, aParam.mEnd); + WriteParam(aWriter, aParam.mOffset); + WriteParam(aWriter, aParam.mBreakpadId); + WriteParam(aWriter, aParam.mCodeId); + WriteParam(aWriter, aParam.mModuleName); + WriteParam(aWriter, aParam.mModulePath); + WriteParam(aWriter, aParam.mDebugName); + WriteParam(aWriter, aParam.mDebugPath); + WriteParam(aWriter, aParam.mVersion); + WriteParam(aWriter, aParam.mArch); +} + +bool IPC::ParamTraits<SharedLibrary>::Read(MessageReader* aReader, + paramType* aResult) { + return ReadParam(aReader, &aResult->mStart) && + ReadParam(aReader, &aResult->mEnd) && + ReadParam(aReader, &aResult->mOffset) && + ReadParam(aReader, &aResult->mBreakpadId) && + ReadParam(aReader, &aResult->mCodeId) && + ReadParam(aReader, &aResult->mModuleName) && + ReadParam(aReader, &aResult->mModulePath) && + ReadParam(aReader, &aResult->mDebugName) && + ReadParam(aReader, &aResult->mDebugPath) && + ReadParam(aReader, &aResult->mVersion) && + ReadParam(aReader, &aResult->mArch); +} + +void IPC::ParamTraits<SharedLibraryInfo>::Write(MessageWriter* aWriter, + const paramType& aParam) { + paramType& p = const_cast<paramType&>(aParam); + WriteParam(aWriter, p.mEntries); +} + +bool IPC::ParamTraits<SharedLibraryInfo>::Read(MessageReader* aReader, + paramType* aResult) { + return ReadParam(aReader, &aResult->mEntries); +} + +void IPC::ParamTraits<mozilla::ProfileGenerationAdditionalInformation>::Write( + MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, aParam.mSharedLibraries); +} + +bool IPC::ParamTraits<mozilla::ProfileGenerationAdditionalInformation>::Read( + MessageReader* aReader, paramType* aResult) { + return ReadParam(aReader, &aResult->mSharedLibraries); +} + +} // namespace IPC diff --git a/tools/profiler/core/ProfileBuffer.cpp b/tools/profiler/core/ProfileBuffer.cpp new file mode 100644 index 0000000000..bc6314fa32 --- /dev/null +++ b/tools/profiler/core/ProfileBuffer.cpp @@ -0,0 +1,244 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfileBuffer.h" + +#include "BaseProfiler.h" +#include "js/ColumnNumber.h" // JS::LimitedColumnNumberOneOrigin +#include "js/GCAPI.h" +#include "jsfriendapi.h" +#include "mozilla/MathAlgorithms.h" +#include "nsJSPrincipals.h" +#include "nsScriptSecurityManager.h" + +using namespace mozilla; + +ProfileBuffer::ProfileBuffer(ProfileChunkedBuffer& aBuffer) + : mEntries(aBuffer) { + // Assume the given buffer is in-session. + MOZ_ASSERT(mEntries.IsInSession()); +} + +/* static */ +ProfileBufferBlockIndex ProfileBuffer::AddEntry( + ProfileChunkedBuffer& aProfileChunkedBuffer, + const ProfileBufferEntry& aEntry) { + switch (aEntry.GetKind()) { +#define SWITCH_KIND(KIND, TYPE, SIZE) \ + case ProfileBufferEntry::Kind::KIND: { \ + return aProfileChunkedBuffer.PutFrom(&aEntry, 1 + (SIZE)); \ + } + + FOR_EACH_PROFILE_BUFFER_ENTRY_KIND(SWITCH_KIND) + +#undef SWITCH_KIND + default: + MOZ_ASSERT(false, "Unhandled ProfilerBuffer entry KIND"); + return ProfileBufferBlockIndex{}; + } +} + +// Called from signal, call only reentrant functions +uint64_t ProfileBuffer::AddEntry(const ProfileBufferEntry& aEntry) { + return AddEntry(mEntries, aEntry).ConvertToProfileBufferIndex(); +} + +/* static */ +ProfileBufferBlockIndex ProfileBuffer::AddThreadIdEntry( + ProfileChunkedBuffer& aProfileChunkedBuffer, ProfilerThreadId aThreadId) { + return AddEntry(aProfileChunkedBuffer, + ProfileBufferEntry::ThreadId(aThreadId)); +} + +uint64_t ProfileBuffer::AddThreadIdEntry(ProfilerThreadId aThreadId) { + return AddThreadIdEntry(mEntries, aThreadId).ConvertToProfileBufferIndex(); +} + +void ProfileBuffer::CollectCodeLocation( + const char* aLabel, const char* aStr, uint32_t aFrameFlags, + uint64_t aInnerWindowID, const Maybe<uint32_t>& aLineNumber, + const Maybe<uint32_t>& aColumnNumber, + const Maybe<JS::ProfilingCategoryPair>& aCategoryPair) { + AddEntry(ProfileBufferEntry::Label(aLabel)); + AddEntry(ProfileBufferEntry::FrameFlags(uint64_t(aFrameFlags))); + + if (aStr) { + // Store the string using one or more DynamicStringFragment entries. + size_t strLen = strlen(aStr) + 1; // +1 for the null terminator + // If larger than the prescribed limit, we will cut the string and end it + // with an ellipsis. + const bool tooBig = strLen > kMaxFrameKeyLength; + if (tooBig) { + strLen = kMaxFrameKeyLength; + } + char chars[ProfileBufferEntry::kNumChars]; + for (size_t j = 0;; j += ProfileBufferEntry::kNumChars) { + // Store up to kNumChars characters in the entry. + size_t len = ProfileBufferEntry::kNumChars; + const bool last = j + len >= strLen; + if (last) { + // Only the last entry may be smaller than kNumChars. + len = strLen - j; + if (tooBig) { + // That last entry is part of a too-big string, replace the end + // characters with an ellipsis "...". + len = std::max(len, size_t(4)); + chars[len - 4] = '.'; + chars[len - 3] = '.'; + chars[len - 2] = '.'; + chars[len - 1] = '\0'; + // Make sure the memcpy will not overwrite our ellipsis! + len -= 4; + } + } + memcpy(chars, &aStr[j], len); + AddEntry(ProfileBufferEntry::DynamicStringFragment(chars)); + if (last) { + break; + } + } + } + + if (aInnerWindowID) { + AddEntry(ProfileBufferEntry::InnerWindowID(aInnerWindowID)); + } + + if (aLineNumber) { + AddEntry(ProfileBufferEntry::LineNumber(*aLineNumber)); + } + + if (aColumnNumber) { + AddEntry(ProfileBufferEntry::ColumnNumber(*aColumnNumber)); + } + + if (aCategoryPair.isSome()) { + AddEntry(ProfileBufferEntry::CategoryPair(int(*aCategoryPair))); + } +} + +size_t ProfileBuffer::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { + // Measurement of the following members may be added later if DMD finds it + // is worthwhile: + // - memory pointed to by the elements within mEntries + return mEntries.SizeOfExcludingThis(aMallocSizeOf); +} + +size_t ProfileBuffer::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); +} + +void ProfileBuffer::CollectOverheadStats(double aSamplingTimeMs, + TimeDuration aLocking, + TimeDuration aCleaning, + TimeDuration aCounters, + TimeDuration aThreads) { + double timeUs = aSamplingTimeMs * 1000.0; + if (mFirstSamplingTimeUs == 0.0) { + mFirstSamplingTimeUs = timeUs; + } else { + // Note that we'll have 1 fewer interval than other numbers (because + // we need both ends of an interval to know its duration). The final + // difference should be insignificant over the expected many thousands + // of iterations. + mIntervalsUs.Count(timeUs - mLastSamplingTimeUs); + } + mLastSamplingTimeUs = timeUs; + double locking = aLocking.ToMilliseconds() * 1000.0; + double cleaning = aCleaning.ToMilliseconds() * 1000.0; + double counters = aCounters.ToMilliseconds() * 1000.0; + double threads = aThreads.ToMilliseconds() * 1000.0; + + mOverheadsUs.Count(locking + cleaning + counters + threads); + mLockingsUs.Count(locking); + mCleaningsUs.Count(cleaning); + mCountersUs.Count(counters); + mThreadsUs.Count(threads); + + static const bool sRecordSamplingOverhead = []() { + const char* recordOverheads = getenv("MOZ_PROFILER_RECORD_OVERHEADS"); + return recordOverheads && recordOverheads[0] != '\0'; + }(); + if (sRecordSamplingOverhead) { + AddEntry(ProfileBufferEntry::ProfilerOverheadTime(aSamplingTimeMs)); + AddEntry(ProfileBufferEntry::ProfilerOverheadDuration(locking)); + AddEntry(ProfileBufferEntry::ProfilerOverheadDuration(cleaning)); + AddEntry(ProfileBufferEntry::ProfilerOverheadDuration(counters)); + AddEntry(ProfileBufferEntry::ProfilerOverheadDuration(threads)); + } +} + +ProfilerBufferInfo ProfileBuffer::GetProfilerBufferInfo() const { + return {BufferRangeStart(), + BufferRangeEnd(), + static_cast<uint32_t>(*mEntries.BufferLength() / + 8), // 8 bytes per entry. + mIntervalsUs, + mOverheadsUs, + mLockingsUs, + mCleaningsUs, + mCountersUs, + mThreadsUs}; +} + +/* ProfileBufferCollector */ + +void ProfileBufferCollector::CollectNativeLeafAddr(void* aAddr) { + mBuf.AddEntry(ProfileBufferEntry::NativeLeafAddr(aAddr)); +} + +void ProfileBufferCollector::CollectJitReturnAddr(void* aAddr) { + mBuf.AddEntry(ProfileBufferEntry::JitReturnAddr(aAddr)); +} + +void ProfileBufferCollector::CollectWasmFrame(const char* aLabel) { + mBuf.CollectCodeLocation("", aLabel, 0, 0, Nothing(), Nothing(), + Some(JS::ProfilingCategoryPair::JS_Wasm)); +} + +void ProfileBufferCollector::CollectProfilingStackFrame( + const js::ProfilingStackFrame& aFrame) { + // WARNING: this function runs within the profiler's "critical section". + + MOZ_ASSERT(aFrame.isLabelFrame() || + (aFrame.isJsFrame() && !aFrame.isOSRFrame())); + + const char* label = aFrame.label(); + const char* dynamicString = aFrame.dynamicString(); + Maybe<uint32_t> line; + Maybe<uint32_t> column; + + if (aFrame.isJsFrame()) { + // There are two kinds of JS frames that get pushed onto the ProfilingStack. + // + // - label = "", dynamic string = <something> + // - label = "js::RunScript", dynamic string = nullptr + // + // The line number is only interesting in the first case. + + if (label[0] == '\0') { + MOZ_ASSERT(dynamicString); + + // We call aFrame.script() repeatedly -- rather than storing the result in + // a local variable in order -- to avoid rooting hazards. + if (aFrame.script()) { + if (aFrame.pc()) { + JS::LimitedColumnNumberOneOrigin col; + line = Some(JS_PCToLineNumber(aFrame.script(), aFrame.pc(), &col)); + column = Some(col.oneOriginValue()); + } + } + + } else { + MOZ_ASSERT(strcmp(label, "js::RunScript") == 0 && !dynamicString); + } + } else { + MOZ_ASSERT(aFrame.isLabelFrame()); + } + + mBuf.CollectCodeLocation(label, dynamicString, aFrame.flags(), + aFrame.realmID(), line, column, + Some(aFrame.categoryPair())); +} diff --git a/tools/profiler/core/ProfileBuffer.h b/tools/profiler/core/ProfileBuffer.h new file mode 100644 index 0000000000..5da34909cc --- /dev/null +++ b/tools/profiler/core/ProfileBuffer.h @@ -0,0 +1,260 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZ_PROFILE_BUFFER_H +#define MOZ_PROFILE_BUFFER_H + +#include "GeckoProfiler.h" +#include "ProfileBufferEntry.h" + +#include "mozilla/Maybe.h" +#include "mozilla/PowerOfTwo.h" +#include "mozilla/ProfileBufferChunkManagerSingle.h" +#include "mozilla/ProfileChunkedBuffer.h" + +class ProcessStreamingContext; +class RunningTimes; + +// Class storing most profiling data in a ProfileChunkedBuffer. +// +// This class is used as a queue of entries which, after construction, never +// allocates. This makes it safe to use in the profiler's "critical section". +class ProfileBuffer final { + public: + // ProfileBuffer constructor + // @param aBuffer The in-session ProfileChunkedBuffer to use as buffer + // manager. + explicit ProfileBuffer(mozilla::ProfileChunkedBuffer& aBuffer); + + mozilla::ProfileChunkedBuffer& UnderlyingChunkedBuffer() const { + return mEntries; + } + + bool IsThreadSafe() const { return mEntries.IsThreadSafe(); } + + // Add |aEntry| to the buffer, ignoring what kind of entry it is. + uint64_t AddEntry(const ProfileBufferEntry& aEntry); + + // Add to the buffer a sample start (ThreadId) entry for aThreadId. + // Returns the position of the entry. + uint64_t AddThreadIdEntry(ProfilerThreadId aThreadId); + + void CollectCodeLocation( + const char* aLabel, const char* aStr, uint32_t aFrameFlags, + uint64_t aInnerWindowID, const mozilla::Maybe<uint32_t>& aLineNumber, + const mozilla::Maybe<uint32_t>& aColumnNumber, + const mozilla::Maybe<JS::ProfilingCategoryPair>& aCategoryPair); + + // Maximum size of a frameKey string that we'll handle. + static const size_t kMaxFrameKeyLength = 512; + + // Add JIT frame information to aJITFrameInfo for any JitReturnAddr entries + // that are currently in the buffer at or after aRangeStart, in samples + // for the given thread. + void AddJITInfoForRange(uint64_t aRangeStart, ProfilerThreadId aThreadId, + JSContext* aContext, JITFrameInfo& aJITFrameInfo, + mozilla::ProgressLogger aProgressLogger) const; + + // Stream JSON for samples in the buffer to aWriter, using the supplied + // UniqueStacks object. + // Only streams samples for the given thread ID and which were taken at or + // after aSinceTime. If ID is 0, ignore the stored thread ID; this should only + // be used when the buffer contains only one sample. + // aUniqueStacks needs to contain information about any JIT frames that we + // might encounter in the buffer, before this method is called. In other + // words, you need to have called AddJITInfoForRange for every range that + // might contain JIT frame information before calling this method. + // Return the thread ID of the streamed sample(s), or 0. + ProfilerThreadId StreamSamplesToJSON( + SpliceableJSONWriter& aWriter, ProfilerThreadId aThreadId, + double aSinceTime, UniqueStacks& aUniqueStacks, + mozilla::ProgressLogger aProgressLogger) const; + + void StreamMarkersToJSON(SpliceableJSONWriter& aWriter, + ProfilerThreadId aThreadId, + const mozilla::TimeStamp& aProcessStartTime, + double aSinceTime, UniqueStacks& aUniqueStacks, + mozilla::ProgressLogger aProgressLogger) const; + + // Stream samples and markers from all threads that `aProcessStreamingContext` + // accepts. + void StreamSamplesAndMarkersToJSON( + ProcessStreamingContext& aProcessStreamingContext, + mozilla::ProgressLogger aProgressLogger) const; + + void StreamPausedRangesToJSON(SpliceableJSONWriter& aWriter, + double aSinceTime, + mozilla::ProgressLogger aProgressLogger) const; + void StreamProfilerOverheadToJSON( + SpliceableJSONWriter& aWriter, + const mozilla::TimeStamp& aProcessStartTime, double aSinceTime, + mozilla::ProgressLogger aProgressLogger) const; + void StreamCountersToJSON(SpliceableJSONWriter& aWriter, + const mozilla::TimeStamp& aProcessStartTime, + double aSinceTime, + mozilla::ProgressLogger aProgressLogger) const; + + // Find (via |aLastSample|) the most recent sample for the thread denoted by + // |aThreadId| and clone it, patching in the current time as appropriate. + // Mutate |aLastSample| to point to the newly inserted sample. + // Returns whether duplication was successful. + bool DuplicateLastSample(ProfilerThreadId aThreadId, double aSampleTimeMs, + mozilla::Maybe<uint64_t>& aLastSample, + const RunningTimes& aRunningTimes); + + void DiscardSamplesBeforeTime(double aTime); + + // Read an entry in the buffer. + ProfileBufferEntry GetEntry(uint64_t aPosition) const { + return mEntries.ReadAt( + mozilla::ProfileBufferBlockIndex::CreateFromProfileBufferIndex( + aPosition), + [&](mozilla::Maybe<mozilla::ProfileBufferEntryReader>&& aMER) { + ProfileBufferEntry entry; + if (aMER.isSome()) { + if (aMER->CurrentBlockIndex().ConvertToProfileBufferIndex() == + aPosition) { + // If we're here, it means `aPosition` pointed at a valid block. + MOZ_RELEASE_ASSERT(aMER->RemainingBytes() <= sizeof(entry)); + aMER->ReadBytes(&entry, aMER->RemainingBytes()); + } else { + // EntryReader at the wrong position, pretend to have read + // everything. + aMER->SetRemainingBytes(0); + } + } + return entry; + }); + } + + size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const; + size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) const; + + void CollectOverheadStats(double aSamplingTimeMs, + mozilla::TimeDuration aLocking, + mozilla::TimeDuration aCleaning, + mozilla::TimeDuration aCounters, + mozilla::TimeDuration aThreads); + + ProfilerBufferInfo GetProfilerBufferInfo() const; + + private: + // Add |aEntry| to the provided ProfileChunkedBuffer. + // `static` because it may be used to add an entry to a `ProfileChunkedBuffer` + // that is not attached to a `ProfileBuffer`. + static mozilla::ProfileBufferBlockIndex AddEntry( + mozilla::ProfileChunkedBuffer& aProfileChunkedBuffer, + const ProfileBufferEntry& aEntry); + + // Add a sample start (ThreadId) entry for aThreadId to the provided + // ProfileChunkedBuffer. Returns the position of the entry. + // `static` because it may be used to add an entry to a `ProfileChunkedBuffer` + // that is not attached to a `ProfileBuffer`. + static mozilla::ProfileBufferBlockIndex AddThreadIdEntry( + mozilla::ProfileChunkedBuffer& aProfileChunkedBuffer, + ProfilerThreadId aThreadId); + + // The storage in which this ProfileBuffer stores its entries. + mozilla::ProfileChunkedBuffer& mEntries; + + public: + // `BufferRangeStart()` and `BufferRangeEnd()` return `uint64_t` values + // corresponding to the first entry and past the last entry stored in + // `mEntries`. + // + // The returned values are not guaranteed to be stable, because other threads + // may also be accessing the buffer concurrently. But they will always + // increase, and can therefore give an indication of how far these values have + // *at least* reached. In particular: + // - Entries whose index is strictly less that `BufferRangeStart()` have been + // discarded by now, so any related data may also be safely discarded. + // - It is safe to try and read entries at any index strictly less than + // `BufferRangeEnd()` -- but note that these reads may fail by the time you + // request them, as old entries get overwritten by new ones. + uint64_t BufferRangeStart() const { return mEntries.GetState().mRangeStart; } + uint64_t BufferRangeEnd() const { return mEntries.GetState().mRangeEnd; } + + private: + // Single pre-allocated chunk (to avoid spurious mallocs), used when: + // - Duplicating sleeping stacks (hence scExpectedMaximumStackSize). + // - Adding JIT info. + // - Streaming stacks to JSON. + // Mutable because it's accessed from non-multithreaded const methods. + mutable mozilla::Maybe<mozilla::ProfileBufferChunkManagerSingle> + mMaybeWorkerChunkManager; + mozilla::ProfileBufferChunkManagerSingle& WorkerChunkManager() const { + if (mMaybeWorkerChunkManager.isNothing()) { + // Only actually allocate it on first use. (Some ProfileBuffers are + // temporary and don't actually need this.) + mMaybeWorkerChunkManager.emplace( + mozilla::ProfileBufferChunk::SizeofChunkMetadata() + + mozilla::ProfileBufferChunkManager::scExpectedMaximumStackSize); + } + return *mMaybeWorkerChunkManager; + } + + // GetStreamingParametersForThreadCallback: + // (ProfilerThreadId) -> Maybe<StreamingParametersForThread> + template <typename GetStreamingParametersForThreadCallback> + ProfilerThreadId DoStreamSamplesAndMarkersToJSON( + mozilla::FailureLatch& aFailureLatch, + GetStreamingParametersForThreadCallback&& + aGetStreamingParametersForThreadCallback, + double aSinceTime, ProcessStreamingContext* aStreamingContextForMarkers, + mozilla::ProgressLogger aProgressLogger) const; + + double mFirstSamplingTimeUs = 0.0; + double mLastSamplingTimeUs = 0.0; + ProfilerStats mIntervalsUs; + ProfilerStats mOverheadsUs; + ProfilerStats mLockingsUs; + ProfilerStats mCleaningsUs; + ProfilerStats mCountersUs; + ProfilerStats mThreadsUs; +}; + +/** + * Helper type used to implement ProfilerStackCollector. This type is used as + * the collector for MergeStacks by ProfileBuffer. It holds a reference to the + * buffer, as well as additional feature flags which are needed to control the + * data collection strategy + */ +class ProfileBufferCollector final : public ProfilerStackCollector { + public: + ProfileBufferCollector(ProfileBuffer& aBuf, uint64_t aSamplePos, + uint64_t aBufferRangeStart) + : mBuf(aBuf), + mSamplePositionInBuffer(aSamplePos), + mBufferRangeStart(aBufferRangeStart) { + MOZ_ASSERT( + mSamplePositionInBuffer >= mBufferRangeStart, + "The sample position should always be after the buffer range start"); + } + + // Position at which the sample starts in the profiler buffer (which may be + // different from the buffer in which the sample data is collected here). + mozilla::Maybe<uint64_t> SamplePositionInBuffer() override { + return mozilla::Some(mSamplePositionInBuffer); + } + + // Profiler buffer's range start (which may be different from the buffer in + // which the sample data is collected here). + mozilla::Maybe<uint64_t> BufferRangeStart() override { + return mozilla::Some(mBufferRangeStart); + } + + virtual void CollectNativeLeafAddr(void* aAddr) override; + virtual void CollectJitReturnAddr(void* aAddr) override; + virtual void CollectWasmFrame(const char* aLabel) override; + virtual void CollectProfilingStackFrame( + const js::ProfilingStackFrame& aFrame) override; + + private: + ProfileBuffer& mBuf; + uint64_t mSamplePositionInBuffer; + uint64_t mBufferRangeStart; +}; + +#endif diff --git a/tools/profiler/core/ProfileBufferEntry.cpp b/tools/profiler/core/ProfileBufferEntry.cpp new file mode 100644 index 0000000000..8bf2cbe30d --- /dev/null +++ b/tools/profiler/core/ProfileBufferEntry.cpp @@ -0,0 +1,2294 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfileBufferEntry.h" + +#include "mozilla/ProfilerMarkers.h" +#include "platform.h" +#include "ProfileBuffer.h" +#include "ProfiledThreadData.h" +#include "ProfilerBacktrace.h" +#include "ProfilerRustBindings.h" + +#include "js/ProfilingFrameIterator.h" +#include "jsapi.h" +#include "jsfriendapi.h" +#include "mozilla/Logging.h" +#include "mozilla/JSONStringWriteFuncs.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Sprintf.h" +#include "mozilla/StackWalk.h" +#include "nsThreadUtils.h" +#include "nsXULAppAPI.h" +#include "ProfilerCodeAddressService.h" + +#include <ostream> +#include <type_traits> + +using namespace mozilla; +using namespace mozilla::literals::ProportionValue_literals; + +//////////////////////////////////////////////////////////////////////// +// BEGIN ProfileBufferEntry + +ProfileBufferEntry::ProfileBufferEntry() + : mKind(Kind::INVALID), mStorage{0, 0, 0, 0, 0, 0, 0, 0} {} + +// aString must be a static string. +ProfileBufferEntry::ProfileBufferEntry(Kind aKind, const char* aString) + : mKind(aKind) { + MOZ_ASSERT(aKind == Kind::Label); + memcpy(mStorage, &aString, sizeof(aString)); +} + +ProfileBufferEntry::ProfileBufferEntry(Kind aKind, char aChars[kNumChars]) + : mKind(aKind) { + MOZ_ASSERT(aKind == Kind::DynamicStringFragment); + memcpy(mStorage, aChars, kNumChars); +} + +ProfileBufferEntry::ProfileBufferEntry(Kind aKind, void* aPtr) : mKind(aKind) { + memcpy(mStorage, &aPtr, sizeof(aPtr)); +} + +ProfileBufferEntry::ProfileBufferEntry(Kind aKind, double aDouble) + : mKind(aKind) { + memcpy(mStorage, &aDouble, sizeof(aDouble)); +} + +ProfileBufferEntry::ProfileBufferEntry(Kind aKind, int aInt) : mKind(aKind) { + memcpy(mStorage, &aInt, sizeof(aInt)); +} + +ProfileBufferEntry::ProfileBufferEntry(Kind aKind, int64_t aInt64) + : mKind(aKind) { + memcpy(mStorage, &aInt64, sizeof(aInt64)); +} + +ProfileBufferEntry::ProfileBufferEntry(Kind aKind, uint64_t aUint64) + : mKind(aKind) { + memcpy(mStorage, &aUint64, sizeof(aUint64)); +} + +ProfileBufferEntry::ProfileBufferEntry(Kind aKind, ProfilerThreadId aThreadId) + : mKind(aKind) { + static_assert(std::is_trivially_copyable_v<ProfilerThreadId>); + static_assert(sizeof(aThreadId) <= sizeof(mStorage)); + memcpy(mStorage, &aThreadId, sizeof(aThreadId)); +} + +const char* ProfileBufferEntry::GetString() const { + const char* result; + memcpy(&result, mStorage, sizeof(result)); + return result; +} + +void* ProfileBufferEntry::GetPtr() const { + void* result; + memcpy(&result, mStorage, sizeof(result)); + return result; +} + +double ProfileBufferEntry::GetDouble() const { + double result; + memcpy(&result, mStorage, sizeof(result)); + return result; +} + +int ProfileBufferEntry::GetInt() const { + int result; + memcpy(&result, mStorage, sizeof(result)); + return result; +} + +int64_t ProfileBufferEntry::GetInt64() const { + int64_t result; + memcpy(&result, mStorage, sizeof(result)); + return result; +} + +uint64_t ProfileBufferEntry::GetUint64() const { + uint64_t result; + memcpy(&result, mStorage, sizeof(result)); + return result; +} + +ProfilerThreadId ProfileBufferEntry::GetThreadId() const { + ProfilerThreadId result; + static_assert(std::is_trivially_copyable_v<ProfilerThreadId>); + memcpy(&result, mStorage, sizeof(result)); + return result; +} + +void ProfileBufferEntry::CopyCharsInto(char (&aOutArray)[kNumChars]) const { + memcpy(aOutArray, mStorage, kNumChars); +} + +// END ProfileBufferEntry +//////////////////////////////////////////////////////////////////////// + +struct TypeInfo { + Maybe<nsCString> mKeyedBy; + Maybe<nsCString> mName; + Maybe<nsCString> mLocation; + Maybe<unsigned> mLineNumber; +}; + +// As mentioned in ProfileBufferEntry.h, the JSON format contains many +// arrays whose elements are laid out according to various schemas to help +// de-duplication. This RAII class helps write these arrays by keeping track of +// the last non-null element written and adding the appropriate number of null +// elements when writing new non-null elements. It also automatically opens and +// closes an array element on the given JSON writer. +// +// You grant the AutoArraySchemaWriter exclusive access to the JSONWriter and +// the UniqueJSONStrings objects for the lifetime of AutoArraySchemaWriter. Do +// not access them independently while the AutoArraySchemaWriter is alive. +// If you need to add complex objects, call FreeFormElement(), which will give +// you temporary access to the writer. +// +// Example usage: +// +// // Define the schema of elements in this type of array: [FOO, BAR, BAZ] +// enum Schema : uint32_t { +// FOO = 0, +// BAR = 1, +// BAZ = 2 +// }; +// +// AutoArraySchemaWriter writer(someJsonWriter, someUniqueStrings); +// if (shouldWriteFoo) { +// writer.IntElement(FOO, getFoo()); +// } +// ... etc ... +// +// The elements need to be added in-order. +class MOZ_RAII AutoArraySchemaWriter { + public: + explicit AutoArraySchemaWriter(SpliceableJSONWriter& aWriter) + : mJSONWriter(aWriter), mNextFreeIndex(0) { + mJSONWriter.StartArrayElement(); + } + + ~AutoArraySchemaWriter() { mJSONWriter.EndArray(); } + + template <typename T> + void IntElement(uint32_t aIndex, T aValue) { + static_assert(!std::is_same_v<T, uint64_t>, + "Narrowing uint64 -> int64 conversion not allowed"); + FillUpTo(aIndex); + mJSONWriter.IntElement(static_cast<int64_t>(aValue)); + } + + void DoubleElement(uint32_t aIndex, double aValue) { + FillUpTo(aIndex); + mJSONWriter.DoubleElement(aValue); + } + + void TimeMsElement(uint32_t aIndex, double aTime_ms) { + FillUpTo(aIndex); + mJSONWriter.TimeDoubleMsElement(aTime_ms); + } + + void BoolElement(uint32_t aIndex, bool aValue) { + FillUpTo(aIndex); + mJSONWriter.BoolElement(aValue); + } + + protected: + SpliceableJSONWriter& Writer() { return mJSONWriter; } + + void FillUpTo(uint32_t aIndex) { + MOZ_ASSERT(aIndex >= mNextFreeIndex); + mJSONWriter.NullElements(aIndex - mNextFreeIndex); + mNextFreeIndex = aIndex + 1; + } + + private: + SpliceableJSONWriter& mJSONWriter; + uint32_t mNextFreeIndex; +}; + +// Same as AutoArraySchemaWriter, but this can also write strings (output as +// indexes into the table of unique strings). +class MOZ_RAII AutoArraySchemaWithStringsWriter : public AutoArraySchemaWriter { + public: + AutoArraySchemaWithStringsWriter(SpliceableJSONWriter& aWriter, + UniqueJSONStrings& aStrings) + : AutoArraySchemaWriter(aWriter), mStrings(aStrings) {} + + void StringElement(uint32_t aIndex, const Span<const char>& aValue) { + FillUpTo(aIndex); + mStrings.WriteElement(Writer(), aValue); + } + + private: + UniqueJSONStrings& mStrings; +}; + +Maybe<UniqueStacks::StackKey> UniqueStacks::BeginStack(const FrameKey& aFrame) { + if (Maybe<uint32_t> frameIndex = GetOrAddFrameIndex(aFrame); frameIndex) { + return Some(StackKey(*frameIndex)); + } + return Nothing{}; +} + +Vector<JITFrameInfoForBufferRange>&& +JITFrameInfo::MoveRangesWithNewFailureLatch(FailureLatch& aFailureLatch) && { + aFailureLatch.SetFailureFrom(mLocalFailureLatchSource); + return std::move(mRanges); +} + +UniquePtr<UniqueJSONStrings>&& +JITFrameInfo::MoveUniqueStringsWithNewFailureLatch( + FailureLatch& aFailureLatch) && { + if (mUniqueStrings) { + mUniqueStrings->ChangeFailureLatchAndForwardState(aFailureLatch); + } else { + aFailureLatch.SetFailureFrom(mLocalFailureLatchSource); + } + return std::move(mUniqueStrings); +} + +Maybe<UniqueStacks::StackKey> UniqueStacks::AppendFrame( + const StackKey& aStack, const FrameKey& aFrame) { + if (Maybe<uint32_t> stackIndex = GetOrAddStackIndex(aStack); stackIndex) { + if (Maybe<uint32_t> frameIndex = GetOrAddFrameIndex(aFrame); frameIndex) { + return Some(StackKey(aStack, *stackIndex, *frameIndex)); + } + } + return Nothing{}; +} + +JITFrameInfoForBufferRange JITFrameInfoForBufferRange::Clone() const { + JITFrameInfoForBufferRange::JITAddressToJITFramesMap jitAddressToJITFramesMap; + MOZ_RELEASE_ASSERT( + jitAddressToJITFramesMap.reserve(mJITAddressToJITFramesMap.count())); + for (auto iter = mJITAddressToJITFramesMap.iter(); !iter.done(); + iter.next()) { + const mozilla::Vector<JITFrameKey>& srcKeys = iter.get().value(); + mozilla::Vector<JITFrameKey> destKeys; + MOZ_RELEASE_ASSERT(destKeys.appendAll(srcKeys)); + jitAddressToJITFramesMap.putNewInfallible(iter.get().key(), + std::move(destKeys)); + } + + JITFrameInfoForBufferRange::JITFrameToFrameJSONMap jitFrameToFrameJSONMap; + MOZ_RELEASE_ASSERT( + jitFrameToFrameJSONMap.reserve(mJITFrameToFrameJSONMap.count())); + for (auto iter = mJITFrameToFrameJSONMap.iter(); !iter.done(); iter.next()) { + jitFrameToFrameJSONMap.putNewInfallible(iter.get().key(), + iter.get().value()); + } + + return JITFrameInfoForBufferRange{mRangeStart, mRangeEnd, + std::move(jitAddressToJITFramesMap), + std::move(jitFrameToFrameJSONMap)}; +} + +JITFrameInfo::JITFrameInfo(const JITFrameInfo& aOther, + mozilla::ProgressLogger aProgressLogger) + : mUniqueStrings(MakeUniqueFallible<UniqueJSONStrings>( + mLocalFailureLatchSource, *aOther.mUniqueStrings, + aProgressLogger.CreateSubLoggerFromTo( + 0_pc, "Creating JIT frame info unique strings...", 49_pc, + "Created JIT frame info unique strings"))) { + if (!mUniqueStrings) { + mLocalFailureLatchSource.SetFailure( + "OOM in JITFrameInfo allocating mUniqueStrings"); + return; + } + + if (mRanges.reserve(aOther.mRanges.length())) { + for (auto&& [i, progressLogger] : + aProgressLogger.CreateLoopSubLoggersFromTo(50_pc, 100_pc, + aOther.mRanges.length(), + "Copying JIT frame info")) { + mRanges.infallibleAppend(aOther.mRanges[i].Clone()); + } + } else { + mLocalFailureLatchSource.SetFailure("OOM in JITFrameInfo resizing mRanges"); + } +} + +bool UniqueStacks::FrameKey::NormalFrameData::operator==( + const NormalFrameData& aOther) const { + return mLocation == aOther.mLocation && + mRelevantForJS == aOther.mRelevantForJS && + mBaselineInterp == aOther.mBaselineInterp && + mInnerWindowID == aOther.mInnerWindowID && mLine == aOther.mLine && + mColumn == aOther.mColumn && mCategoryPair == aOther.mCategoryPair; +} + +bool UniqueStacks::FrameKey::JITFrameData::operator==( + const JITFrameData& aOther) const { + return mCanonicalAddress == aOther.mCanonicalAddress && + mDepth == aOther.mDepth && mRangeIndex == aOther.mRangeIndex; +} + +// Consume aJITFrameInfo by stealing its string table and its JIT frame info +// ranges. The JIT frame info contains JSON which refers to strings from the +// JIT frame info's string table, so our string table needs to have the same +// strings at the same indices. +UniqueStacks::UniqueStacks( + FailureLatch& aFailureLatch, JITFrameInfo&& aJITFrameInfo, + ProfilerCodeAddressService* aCodeAddressService /* = nullptr */) + : mUniqueStrings(std::move(aJITFrameInfo) + .MoveUniqueStringsWithNewFailureLatch(aFailureLatch)), + mCodeAddressService(aCodeAddressService), + mFrameTableWriter(aFailureLatch), + mStackTableWriter(aFailureLatch), + mJITInfoRanges(std::move(aJITFrameInfo) + .MoveRangesWithNewFailureLatch(aFailureLatch)) { + if (!mUniqueStrings) { + SetFailure("Did not get mUniqueStrings from JITFrameInfo"); + return; + } + + mFrameTableWriter.StartBareList(); + mStackTableWriter.StartBareList(); +} + +Maybe<uint32_t> UniqueStacks::GetOrAddStackIndex(const StackKey& aStack) { + if (Failed()) { + return Nothing{}; + } + + uint32_t count = mStackToIndexMap.count(); + auto entry = mStackToIndexMap.lookupForAdd(aStack); + if (entry) { + MOZ_ASSERT(entry->value() < count); + return Some(entry->value()); + } + + if (!mStackToIndexMap.add(entry, aStack, count)) { + SetFailure("OOM in UniqueStacks::GetOrAddStackIndex"); + return Nothing{}; + } + StreamStack(aStack); + return Some(count); +} + +Maybe<Vector<UniqueStacks::FrameKey>> +UniqueStacks::LookupFramesForJITAddressFromBufferPos(void* aJITAddress, + uint64_t aBufferPos) { + JITFrameInfoForBufferRange* rangeIter = + std::lower_bound(mJITInfoRanges.begin(), mJITInfoRanges.end(), aBufferPos, + [](const JITFrameInfoForBufferRange& aRange, + uint64_t aPos) { return aRange.mRangeEnd < aPos; }); + MOZ_RELEASE_ASSERT( + rangeIter != mJITInfoRanges.end() && + rangeIter->mRangeStart <= aBufferPos && + aBufferPos < rangeIter->mRangeEnd, + "Buffer position of jit address needs to be in one of the ranges"); + + using JITFrameKey = JITFrameInfoForBufferRange::JITFrameKey; + + const JITFrameInfoForBufferRange& jitFrameInfoRange = *rangeIter; + auto jitFrameKeys = + jitFrameInfoRange.mJITAddressToJITFramesMap.lookup(aJITAddress); + if (!jitFrameKeys) { + return Nothing(); + } + + // Map the array of JITFrameKeys to an array of FrameKeys, and ensure that + // each of the FrameKeys exists in mFrameToIndexMap. + Vector<FrameKey> frameKeys; + MOZ_RELEASE_ASSERT(frameKeys.initCapacity(jitFrameKeys->value().length())); + for (const JITFrameKey& jitFrameKey : jitFrameKeys->value()) { + FrameKey frameKey(jitFrameKey.mCanonicalAddress, jitFrameKey.mDepth, + rangeIter - mJITInfoRanges.begin()); + uint32_t index = mFrameToIndexMap.count(); + auto entry = mFrameToIndexMap.lookupForAdd(frameKey); + if (!entry) { + // We need to add this frame to our frame table. The JSON for this frame + // already exists in jitFrameInfoRange, we just need to splice it into + // the frame table and give it an index. + auto frameJSON = + jitFrameInfoRange.mJITFrameToFrameJSONMap.lookup(jitFrameKey); + MOZ_RELEASE_ASSERT(frameJSON, "Should have cached JSON for this frame"); + mFrameTableWriter.Splice(frameJSON->value()); + MOZ_RELEASE_ASSERT(mFrameToIndexMap.add(entry, frameKey, index)); + } + MOZ_RELEASE_ASSERT(frameKeys.append(std::move(frameKey))); + } + return Some(std::move(frameKeys)); +} + +Maybe<uint32_t> UniqueStacks::GetOrAddFrameIndex(const FrameKey& aFrame) { + if (Failed()) { + return Nothing{}; + } + + uint32_t count = mFrameToIndexMap.count(); + auto entry = mFrameToIndexMap.lookupForAdd(aFrame); + if (entry) { + MOZ_ASSERT(entry->value() < count); + return Some(entry->value()); + } + + if (!mFrameToIndexMap.add(entry, aFrame, count)) { + SetFailure("OOM in UniqueStacks::GetOrAddFrameIndex"); + return Nothing{}; + } + StreamNonJITFrame(aFrame); + return Some(count); +} + +void UniqueStacks::SpliceFrameTableElements(SpliceableJSONWriter& aWriter) { + mFrameTableWriter.EndBareList(); + aWriter.TakeAndSplice(mFrameTableWriter.TakeChunkedWriteFunc()); +} + +void UniqueStacks::SpliceStackTableElements(SpliceableJSONWriter& aWriter) { + mStackTableWriter.EndBareList(); + aWriter.TakeAndSplice(mStackTableWriter.TakeChunkedWriteFunc()); +} + +[[nodiscard]] nsAutoCString UniqueStacks::FunctionNameOrAddress(void* aPC) { + nsAutoCString nameOrAddress; + + if (!mCodeAddressService || + !mCodeAddressService->GetFunction(aPC, nameOrAddress) || + nameOrAddress.IsEmpty()) { + nameOrAddress.AppendASCII("0x"); + // `AppendInt` only knows `uint32_t` or `uint64_t`, but because these are + // just aliases for *two* of (`unsigned`, `unsigned long`, and `unsigned + // long long`), a call with `uintptr_t` could use the third type and + // therefore would be ambiguous. + // So we want to force using exactly `uint32_t` or `uint64_t`, whichever + // matches the size of `uintptr_t`. + // (The outer cast to `uint` should then be a no-op.) + using uint = std::conditional_t<sizeof(uintptr_t) <= sizeof(uint32_t), + uint32_t, uint64_t>; + nameOrAddress.AppendInt(static_cast<uint>(reinterpret_cast<uintptr_t>(aPC)), + 16); + } + + return nameOrAddress; +} + +void UniqueStacks::StreamStack(const StackKey& aStack) { + enum Schema : uint32_t { PREFIX = 0, FRAME = 1 }; + + AutoArraySchemaWriter writer(mStackTableWriter); + if (aStack.mPrefixStackIndex.isSome()) { + writer.IntElement(PREFIX, *aStack.mPrefixStackIndex); + } + writer.IntElement(FRAME, aStack.mFrameIndex); +} + +void UniqueStacks::StreamNonJITFrame(const FrameKey& aFrame) { + if (Failed()) { + return; + } + + using NormalFrameData = FrameKey::NormalFrameData; + + enum Schema : uint32_t { + LOCATION = 0, + RELEVANT_FOR_JS = 1, + INNER_WINDOW_ID = 2, + IMPLEMENTATION = 3, + LINE = 4, + COLUMN = 5, + CATEGORY = 6, + SUBCATEGORY = 7 + }; + + AutoArraySchemaWithStringsWriter writer(mFrameTableWriter, *mUniqueStrings); + + const NormalFrameData& data = aFrame.mData.as<NormalFrameData>(); + writer.StringElement(LOCATION, data.mLocation); + writer.BoolElement(RELEVANT_FOR_JS, data.mRelevantForJS); + + // It's okay to convert uint64_t to double here because DOM always creates IDs + // that are convertible to double. + writer.DoubleElement(INNER_WINDOW_ID, data.mInnerWindowID); + + // The C++ interpreter is the default implementation so we only emit element + // for Baseline Interpreter frames. + if (data.mBaselineInterp) { + writer.StringElement(IMPLEMENTATION, MakeStringSpan("blinterp")); + } + + if (data.mLine.isSome()) { + writer.IntElement(LINE, *data.mLine); + } + if (data.mColumn.isSome()) { + writer.IntElement(COLUMN, *data.mColumn); + } + if (data.mCategoryPair.isSome()) { + const JS::ProfilingCategoryPairInfo& info = + JS::GetProfilingCategoryPairInfo(*data.mCategoryPair); + writer.IntElement(CATEGORY, uint32_t(info.mCategory)); + writer.IntElement(SUBCATEGORY, info.mSubcategoryIndex); + } +} + +static void StreamJITFrame(JSContext* aContext, SpliceableJSONWriter& aWriter, + UniqueJSONStrings& aUniqueStrings, + const JS::ProfiledFrameHandle& aJITFrame) { + enum Schema : uint32_t { + LOCATION = 0, + RELEVANT_FOR_JS = 1, + INNER_WINDOW_ID = 2, + IMPLEMENTATION = 3, + LINE = 4, + COLUMN = 5, + CATEGORY = 6, + SUBCATEGORY = 7 + }; + + AutoArraySchemaWithStringsWriter writer(aWriter, aUniqueStrings); + + writer.StringElement(LOCATION, MakeStringSpan(aJITFrame.label())); + writer.BoolElement(RELEVANT_FOR_JS, false); + + // It's okay to convert uint64_t to double here because DOM always creates IDs + // that are convertible to double. + // Realm ID is the name of innerWindowID inside JS code. + writer.DoubleElement(INNER_WINDOW_ID, aJITFrame.realmID()); + + JS::ProfilingFrameIterator::FrameKind frameKind = aJITFrame.frameKind(); + MOZ_ASSERT(frameKind == JS::ProfilingFrameIterator::Frame_Ion || + frameKind == JS::ProfilingFrameIterator::Frame_Baseline); + writer.StringElement(IMPLEMENTATION, + frameKind == JS::ProfilingFrameIterator::Frame_Ion + ? MakeStringSpan("ion") + : MakeStringSpan("baseline")); + + const JS::ProfilingCategoryPairInfo& info = JS::GetProfilingCategoryPairInfo( + frameKind == JS::ProfilingFrameIterator::Frame_Ion + ? JS::ProfilingCategoryPair::JS_IonMonkey + : JS::ProfilingCategoryPair::JS_Baseline); + writer.IntElement(CATEGORY, uint32_t(info.mCategory)); + writer.IntElement(SUBCATEGORY, info.mSubcategoryIndex); +} + +static nsCString JSONForJITFrame(JSContext* aContext, + const JS::ProfiledFrameHandle& aJITFrame, + UniqueJSONStrings& aUniqueStrings) { + nsCString json; + JSONStringRefWriteFunc jw(json); + SpliceableJSONWriter writer(jw, aUniqueStrings.SourceFailureLatch()); + StreamJITFrame(aContext, writer, aUniqueStrings, aJITFrame); + return json; +} + +void JITFrameInfo::AddInfoForRange( + uint64_t aRangeStart, uint64_t aRangeEnd, JSContext* aCx, + const std::function<void(const std::function<void(void*)>&)>& + aJITAddressProvider) { + if (mLocalFailureLatchSource.Failed()) { + return; + } + + if (aRangeStart == aRangeEnd) { + return; + } + + MOZ_RELEASE_ASSERT(aRangeStart < aRangeEnd); + + if (!mRanges.empty()) { + const JITFrameInfoForBufferRange& prevRange = mRanges.back(); + MOZ_RELEASE_ASSERT(prevRange.mRangeEnd <= aRangeStart, + "Ranges must be non-overlapping and added in-order."); + } + + using JITFrameKey = JITFrameInfoForBufferRange::JITFrameKey; + + JITFrameInfoForBufferRange::JITAddressToJITFramesMap jitAddressToJITFrameMap; + JITFrameInfoForBufferRange::JITFrameToFrameJSONMap jitFrameToFrameJSONMap; + + aJITAddressProvider([&](void* aJITAddress) { + // Make sure that we have cached data for aJITAddress. + auto addressEntry = jitAddressToJITFrameMap.lookupForAdd(aJITAddress); + if (!addressEntry) { + Vector<JITFrameKey> jitFrameKeys; + for (JS::ProfiledFrameHandle handle : + JS::GetProfiledFrames(aCx, aJITAddress)) { + uint32_t depth = jitFrameKeys.length(); + JITFrameKey jitFrameKey{handle.canonicalAddress(), depth}; + auto frameEntry = jitFrameToFrameJSONMap.lookupForAdd(jitFrameKey); + if (!frameEntry) { + if (!jitFrameToFrameJSONMap.add( + frameEntry, jitFrameKey, + JSONForJITFrame(aCx, handle, *mUniqueStrings))) { + mLocalFailureLatchSource.SetFailure( + "OOM in JITFrameInfo::AddInfoForRange adding jit->frame map"); + return; + } + } + if (!jitFrameKeys.append(jitFrameKey)) { + mLocalFailureLatchSource.SetFailure( + "OOM in JITFrameInfo::AddInfoForRange adding jit frame key"); + return; + } + } + if (!jitAddressToJITFrameMap.add(addressEntry, aJITAddress, + std::move(jitFrameKeys))) { + mLocalFailureLatchSource.SetFailure( + "OOM in JITFrameInfo::AddInfoForRange adding addr->jit map"); + return; + } + } + }); + + if (!mRanges.append(JITFrameInfoForBufferRange{ + aRangeStart, aRangeEnd, std::move(jitAddressToJITFrameMap), + std::move(jitFrameToFrameJSONMap)})) { + mLocalFailureLatchSource.SetFailure( + "OOM in JITFrameInfo::AddInfoForRange adding range"); + return; + } +} + +struct ProfileSample { + uint32_t mStack = 0; + double mTime = 0.0; + Maybe<double> mResponsiveness; + RunningTimes mRunningTimes; +}; + +// Write CPU measurements with "Delta" unit, which is some amount of work that +// happened since the previous sample. +static void WriteDelta(AutoArraySchemaWriter& aSchemaWriter, uint32_t aProperty, + uint64_t aDelta) { + aSchemaWriter.IntElement(aProperty, int64_t(aDelta)); +} + +static void WriteSample(SpliceableJSONWriter& aWriter, + const ProfileSample& aSample) { + enum Schema : uint32_t { + STACK = 0, + TIME = 1, + EVENT_DELAY = 2 +#define RUNNING_TIME_SCHEMA(index, name, unit, jsonProperty) , name + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_SCHEMA) +#undef RUNNING_TIME_SCHEMA + }; + + AutoArraySchemaWriter writer(aWriter); + + writer.IntElement(STACK, aSample.mStack); + + writer.TimeMsElement(TIME, aSample.mTime); + + if (aSample.mResponsiveness.isSome()) { + writer.DoubleElement(EVENT_DELAY, *aSample.mResponsiveness); + } + +#define RUNNING_TIME_STREAM(index, name, unit, jsonProperty) \ + aSample.mRunningTimes.GetJson##name##unit().apply( \ + [&writer](const uint64_t& aValue) { \ + Write##unit(writer, name, aValue); \ + }); + + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_STREAM) + +#undef RUNNING_TIME_STREAM +} + +static void StreamMarkerAfterKind( + ProfileBufferEntryReader& aER, + ProcessStreamingContext& aProcessStreamingContext) { + ThreadStreamingContext* threadData = nullptr; + mozilla::base_profiler_markers_detail::DeserializeAfterKindAndStream( + aER, + [&](ProfilerThreadId aThreadId) -> baseprofiler::SpliceableJSONWriter* { + threadData = + aProcessStreamingContext.GetThreadStreamingContext(aThreadId); + return threadData ? &threadData->mMarkersDataWriter : nullptr; + }, + [&](ProfileChunkedBuffer& aChunkedBuffer) { + ProfilerBacktrace backtrace("", &aChunkedBuffer); + MOZ_ASSERT(threadData, + "threadData should have been set before calling here"); + backtrace.StreamJSON(threadData->mMarkersDataWriter, + aProcessStreamingContext.ProcessStartTime(), + *threadData->mUniqueStacks); + }, + [&](mozilla::base_profiler_markers_detail::Streaming::DeserializerTag + aTag) { + MOZ_ASSERT(threadData, + "threadData should have been set before calling here"); + + size_t payloadSize = aER.RemainingBytes(); + + ProfileBufferEntryReader::DoubleSpanOfConstBytes spans = + aER.ReadSpans(payloadSize); + if (MOZ_LIKELY(spans.IsSingleSpan())) { + // Only a single span, we can just refer to it directly + // instead of copying it. + profiler::ffi::gecko_profiler_serialize_marker_for_tag( + aTag, spans.mFirstOrOnly.Elements(), payloadSize, + &threadData->mMarkersDataWriter); + } else { + // Two spans, we need to concatenate them by copying. + uint8_t* payloadBuffer = new uint8_t[payloadSize]; + spans.CopyBytesTo(payloadBuffer); + profiler::ffi::gecko_profiler_serialize_marker_for_tag( + aTag, payloadBuffer, payloadSize, + &threadData->mMarkersDataWriter); + delete[] payloadBuffer; + } + }); +} + +class EntryGetter { + public: + explicit EntryGetter( + ProfileChunkedBuffer::Reader& aReader, + mozilla::FailureLatch& aFailureLatch, + mozilla::ProgressLogger aProgressLogger = {}, + uint64_t aInitialReadPos = 0, + ProcessStreamingContext* aStreamingContextForMarkers = nullptr) + : mFailureLatch(aFailureLatch), + mStreamingContextForMarkers(aStreamingContextForMarkers), + mBlockIt( + aReader.At(ProfileBufferBlockIndex::CreateFromProfileBufferIndex( + aInitialReadPos))), + mBlockItEnd(aReader.end()), + mRangeStart(mBlockIt.BufferRangeStart().ConvertToProfileBufferIndex()), + mRangeSize( + double(mBlockIt.BufferRangeEnd().ConvertToProfileBufferIndex() - + mRangeStart)), + mProgressLogger(std::move(aProgressLogger)) { + SetLocalProgress(ProgressLogger::NO_LOCATION_UPDATE); + if (!ReadLegacyOrEnd()) { + // Find and read the next non-legacy entry. + Next(); + } + } + + bool Has() const { + return (!mFailureLatch.Failed()) && (mBlockIt != mBlockItEnd); + } + + const ProfileBufferEntry& Get() const { + MOZ_ASSERT(Has() || mFailureLatch.Failed(), + "Caller should have checked `Has()` before `Get()`"); + return mEntry; + } + + void Next() { + MOZ_ASSERT(Has() || mFailureLatch.Failed(), + "Caller should have checked `Has()` before `Next()`"); + ++mBlockIt; + ReadUntilLegacyOrEnd(); + } + + // Hand off the current iterator to the caller, which may be used to read + // any kind of entries (legacy or modern). + ProfileChunkedBuffer::BlockIterator Iterator() const { return mBlockIt; } + + // After `Iterator()` was used, we can restart from *after* its updated + // position. + void RestartAfter(const ProfileChunkedBuffer::BlockIterator& it) { + mBlockIt = it; + if (!Has()) { + return; + } + Next(); + } + + ProfileBufferBlockIndex CurBlockIndex() const { + return mBlockIt.CurrentBlockIndex(); + } + + uint64_t CurPos() const { + return CurBlockIndex().ConvertToProfileBufferIndex(); + } + + void SetLocalProgress(const char* aLocation) { + mProgressLogger.SetLocalProgress( + ProportionValue{double(CurBlockIndex().ConvertToProfileBufferIndex() - + mRangeStart) / + mRangeSize}, + aLocation); + } + + private: + // Try to read the entry at the current `mBlockIt` position. + // * If we're at the end of the buffer, just return `true`. + // * If there is a "legacy" entry (containing a real `ProfileBufferEntry`), + // read it into `mEntry`, and return `true` as well. + // * Otherwise the entry contains a "modern" type that cannot be read into + // `mEntry`, return `false` (so `EntryGetter` can skip to another entry). + bool ReadLegacyOrEnd() { + if (!Has()) { + return true; + } + // Read the entry "kind", which is always at the start of all entries. + ProfileBufferEntryReader er = *mBlockIt; + auto type = static_cast<ProfileBufferEntry::Kind>( + er.ReadObject<ProfileBufferEntry::KindUnderlyingType>()); + MOZ_ASSERT(static_cast<ProfileBufferEntry::KindUnderlyingType>(type) < + static_cast<ProfileBufferEntry::KindUnderlyingType>( + ProfileBufferEntry::Kind::MODERN_LIMIT)); + if (type >= ProfileBufferEntry::Kind::LEGACY_LIMIT) { + if (type == ProfileBufferEntry::Kind::Marker && + mStreamingContextForMarkers) { + StreamMarkerAfterKind(er, *mStreamingContextForMarkers); + if (!Has()) { + return true; + } + SetLocalProgress("Processed marker"); + } + er.SetRemainingBytes(0); + return false; + } + // Here, we have a legacy item, we need to read it from the start. + // Because the above `ReadObject` moved the reader, we ned to reset it to + // the start of the entry before reading the whole entry. + er = *mBlockIt; + er.ReadBytes(&mEntry, er.RemainingBytes()); + return true; + } + + void ReadUntilLegacyOrEnd() { + for (;;) { + if (ReadLegacyOrEnd()) { + // Either we're at the end, or we could read a legacy entry -> Done. + break; + } + // Otherwise loop around until we hit a legacy entry or the end. + ++mBlockIt; + } + SetLocalProgress(ProgressLogger::NO_LOCATION_UPDATE); + } + + mozilla::FailureLatch& mFailureLatch; + + ProcessStreamingContext* const mStreamingContextForMarkers; + + ProfileBufferEntry mEntry; + ProfileChunkedBuffer::BlockIterator mBlockIt; + const ProfileChunkedBuffer::BlockIterator mBlockItEnd; + + // Progress logger, and the data needed to compute the current relative + // position in the buffer. + const mozilla::ProfileBufferIndex mRangeStart; + const double mRangeSize; + mozilla::ProgressLogger mProgressLogger; +}; + +// The following grammar shows legal sequences of profile buffer entries. +// The sequences beginning with a ThreadId entry are known as "samples". +// +// ( +// ( /* Samples */ +// ThreadId +// TimeBeforeCompactStack +// RunningTimes? +// UnresponsivenessDurationMs? +// CompactStack +// /* internally including: +// ( NativeLeafAddr +// | Label FrameFlags? DynamicStringFragment* +// LineNumber? CategoryPair? +// | JitReturnAddr +// )+ +// */ +// ) +// | ( /* Reference to a previous identical sample */ +// ThreadId +// TimeBeforeSameSample +// RunningTimes? +// SameSample +// ) +// | Marker +// | ( /* Counters */ +// CounterId +// Time +// ( +// CounterKey +// Count +// Number? +// )* +// ) +// | CollectionStart +// | CollectionEnd +// | Pause +// | Resume +// | ( ProfilerOverheadTime /* Sampling start timestamp */ +// ProfilerOverheadDuration /* Lock acquisition */ +// ProfilerOverheadDuration /* Expired markers cleaning */ +// ProfilerOverheadDuration /* Counters */ +// ProfilerOverheadDuration /* Threads */ +// ) +// )* +// +// The most complicated part is the stack entry sequence that begins with +// Label. Here are some examples. +// +// - ProfilingStack frames without a dynamic string: +// +// Label("js::RunScript") +// CategoryPair(JS::ProfilingCategoryPair::JS) +// +// Label("XREMain::XRE_main") +// LineNumber(4660) +// CategoryPair(JS::ProfilingCategoryPair::OTHER) +// +// Label("ElementRestyler::ComputeStyleChangeFor") +// LineNumber(3003) +// CategoryPair(JS::ProfilingCategoryPair::CSS) +// +// - ProfilingStack frames with a dynamic string: +// +// Label("nsObserverService::NotifyObservers") +// FrameFlags(uint64_t(ProfilingStackFrame::Flags::IS_LABEL_FRAME)) +// DynamicStringFragment("domwindo") +// DynamicStringFragment("wopened") +// LineNumber(291) +// CategoryPair(JS::ProfilingCategoryPair::OTHER) +// +// Label("") +// FrameFlags(uint64_t(ProfilingStackFrame::Flags::IS_JS_FRAME)) +// DynamicStringFragment("closeWin") +// DynamicStringFragment("dow (chr") +// DynamicStringFragment("ome://gl") +// DynamicStringFragment("obal/con") +// DynamicStringFragment("tent/glo") +// DynamicStringFragment("balOverl") +// DynamicStringFragment("ay.js:5)") +// DynamicStringFragment("") # this string holds the closing '\0' +// LineNumber(25) +// CategoryPair(JS::ProfilingCategoryPair::JS) +// +// Label("") +// FrameFlags(uint64_t(ProfilingStackFrame::Flags::IS_JS_FRAME)) +// DynamicStringFragment("bound (s") +// DynamicStringFragment("elf-host") +// DynamicStringFragment("ed:914)") +// LineNumber(945) +// CategoryPair(JS::ProfilingCategoryPair::JS) +// +// - A profiling stack frame with an overly long dynamic string: +// +// Label("") +// FrameFlags(uint64_t(ProfilingStackFrame::Flags::IS_LABEL_FRAME)) +// DynamicStringFragment("(too lon") +// DynamicStringFragment("g)") +// LineNumber(100) +// CategoryPair(JS::ProfilingCategoryPair::NETWORK) +// +// - A wasm JIT frame: +// +// Label("") +// FrameFlags(uint64_t(0)) +// DynamicStringFragment("wasm-fun") +// DynamicStringFragment("ction[87") +// DynamicStringFragment("36] (blo") +// DynamicStringFragment("b:http:/") +// DynamicStringFragment("/webasse") +// DynamicStringFragment("mbly.org") +// DynamicStringFragment("/3dc5759") +// DynamicStringFragment("4-ce58-4") +// DynamicStringFragment("626-975b") +// DynamicStringFragment("-08ad116") +// DynamicStringFragment("30bc1:38") +// DynamicStringFragment("29856)") +// +// - A JS frame in a synchronous sample: +// +// Label("") +// FrameFlags(uint64_t(ProfilingStackFrame::Flags::IS_LABEL_FRAME)) +// DynamicStringFragment("u (https") +// DynamicStringFragment("://perf-") +// DynamicStringFragment("html.io/") +// DynamicStringFragment("ac0da204") +// DynamicStringFragment("aaa44d75") +// DynamicStringFragment("a800.bun") +// DynamicStringFragment("dle.js:2") +// DynamicStringFragment("5)") + +// Because this is a format entirely internal to the Profiler, any parsing +// error indicates a bug in the ProfileBuffer writing or the parser itself, +// or possibly flaky hardware. +#define ERROR_AND_CONTINUE(msg) \ + { \ + fprintf(stderr, "ProfileBuffer parse error: %s", msg); \ + MOZ_ASSERT(false, msg); \ + continue; \ + } + +struct StreamingParametersForThread { + SpliceableJSONWriter& mWriter; + UniqueStacks& mUniqueStacks; + ThreadStreamingContext::PreviousStackState& mPreviousStackState; + uint32_t& mPreviousStack; + + StreamingParametersForThread( + SpliceableJSONWriter& aWriter, UniqueStacks& aUniqueStacks, + ThreadStreamingContext::PreviousStackState& aPreviousStackState, + uint32_t& aPreviousStack) + : mWriter(aWriter), + mUniqueStacks(aUniqueStacks), + mPreviousStackState(aPreviousStackState), + mPreviousStack(aPreviousStack) {} +}; + +// GetStreamingParametersForThreadCallback: +// (ProfilerThreadId) -> Maybe<StreamingParametersForThread> +template <typename GetStreamingParametersForThreadCallback> +ProfilerThreadId ProfileBuffer::DoStreamSamplesAndMarkersToJSON( + mozilla::FailureLatch& aFailureLatch, + GetStreamingParametersForThreadCallback&& + aGetStreamingParametersForThreadCallback, + double aSinceTime, ProcessStreamingContext* aStreamingContextForMarkers, + mozilla::ProgressLogger aProgressLogger) const { + UniquePtr<char[]> dynStrBuf = MakeUnique<char[]>(kMaxFrameKeyLength); + + return mEntries.Read([&](ProfileChunkedBuffer::Reader* aReader) { + MOZ_ASSERT(aReader, + "ProfileChunkedBuffer cannot be out-of-session when sampler is " + "running"); + + ProfilerThreadId processedThreadId; + + EntryGetter e(*aReader, aFailureLatch, std::move(aProgressLogger), + /* aInitialReadPos */ 0, aStreamingContextForMarkers); + + for (;;) { + // This block skips entries until we find the start of the next sample. + // This is useful in three situations. + // + // - The circular buffer overwrites old entries, so when we start parsing + // we might be in the middle of a sample, and we must skip forward to + // the start of the next sample. + // + // - We skip samples that don't have an appropriate ThreadId or Time. + // + // - We skip range Pause, Resume, CollectionStart, Marker, Counter + // and CollectionEnd entries between samples. + while (e.Has()) { + if (e.Get().IsThreadId()) { + break; + } + e.Next(); + } + + if (!e.Has()) { + break; + } + + // Due to the skip_to_next_sample block above, if we have an entry here it + // must be a ThreadId entry. + MOZ_ASSERT(e.Get().IsThreadId()); + + ProfilerThreadId threadId = e.Get().GetThreadId(); + e.Next(); + + Maybe<StreamingParametersForThread> streamingParameters = + std::forward<GetStreamingParametersForThreadCallback>( + aGetStreamingParametersForThreadCallback)(threadId); + + // Ignore samples that are for the wrong thread. + if (!streamingParameters) { + continue; + } + + SpliceableJSONWriter& writer = streamingParameters->mWriter; + UniqueStacks& uniqueStacks = streamingParameters->mUniqueStacks; + ThreadStreamingContext::PreviousStackState& previousStackState = + streamingParameters->mPreviousStackState; + uint32_t& previousStack = streamingParameters->mPreviousStack; + + auto ReadStack = [&](EntryGetter& e, double time, uint64_t entryPosition, + const Maybe<double>& unresponsiveDuration, + const RunningTimes& runningTimes) { + if (writer.Failed()) { + return; + } + + Maybe<UniqueStacks::StackKey> maybeStack = + uniqueStacks.BeginStack(UniqueStacks::FrameKey("(root)")); + if (!maybeStack) { + writer.SetFailure("BeginStack failure"); + return; + } + + UniqueStacks::StackKey stack = *maybeStack; + + int numFrames = 0; + while (e.Has()) { + if (e.Get().IsNativeLeafAddr()) { + numFrames++; + + void* pc = e.Get().GetPtr(); + e.Next(); + + nsAutoCString functionNameOrAddress = + uniqueStacks.FunctionNameOrAddress(pc); + + maybeStack = uniqueStacks.AppendFrame( + stack, UniqueStacks::FrameKey(functionNameOrAddress.get())); + if (!maybeStack) { + writer.SetFailure("AppendFrame failure"); + return; + } + stack = *maybeStack; + + } else if (e.Get().IsLabel()) { + numFrames++; + + const char* label = e.Get().GetString(); + e.Next(); + + using FrameFlags = js::ProfilingStackFrame::Flags; + uint32_t frameFlags = 0; + if (e.Has() && e.Get().IsFrameFlags()) { + frameFlags = uint32_t(e.Get().GetUint64()); + e.Next(); + } + + bool relevantForJS = + frameFlags & uint32_t(FrameFlags::RELEVANT_FOR_JS); + + bool isBaselineInterp = + frameFlags & uint32_t(FrameFlags::IS_BLINTERP_FRAME); + + // Copy potential dynamic string fragments into dynStrBuf, so that + // dynStrBuf will then contain the entire dynamic string. + size_t i = 0; + dynStrBuf[0] = '\0'; + while (e.Has()) { + if (e.Get().IsDynamicStringFragment()) { + char chars[ProfileBufferEntry::kNumChars]; + e.Get().CopyCharsInto(chars); + for (char c : chars) { + if (i < kMaxFrameKeyLength) { + dynStrBuf[i] = c; + i++; + } + } + e.Next(); + } else { + break; + } + } + dynStrBuf[kMaxFrameKeyLength - 1] = '\0'; + bool hasDynamicString = (i != 0); + + nsAutoCStringN<1024> frameLabel; + if (label[0] != '\0' && hasDynamicString) { + if (frameFlags & uint32_t(FrameFlags::STRING_TEMPLATE_METHOD)) { + frameLabel.AppendPrintf("%s.%s", label, dynStrBuf.get()); + } else if (frameFlags & + uint32_t(FrameFlags::STRING_TEMPLATE_GETTER)) { + frameLabel.AppendPrintf("get %s.%s", label, dynStrBuf.get()); + } else if (frameFlags & + uint32_t(FrameFlags::STRING_TEMPLATE_SETTER)) { + frameLabel.AppendPrintf("set %s.%s", label, dynStrBuf.get()); + } else { + frameLabel.AppendPrintf("%s %s", label, dynStrBuf.get()); + } + } else if (hasDynamicString) { + frameLabel.Append(dynStrBuf.get()); + } else { + frameLabel.Append(label); + } + + uint64_t innerWindowID = 0; + if (e.Has() && e.Get().IsInnerWindowID()) { + innerWindowID = uint64_t(e.Get().GetUint64()); + e.Next(); + } + + Maybe<unsigned> line; + if (e.Has() && e.Get().IsLineNumber()) { + line = Some(unsigned(e.Get().GetInt())); + e.Next(); + } + + Maybe<unsigned> column; + if (e.Has() && e.Get().IsColumnNumber()) { + column = Some(unsigned(e.Get().GetInt())); + e.Next(); + } + + Maybe<JS::ProfilingCategoryPair> categoryPair; + if (e.Has() && e.Get().IsCategoryPair()) { + categoryPair = + Some(JS::ProfilingCategoryPair(uint32_t(e.Get().GetInt()))); + e.Next(); + } + + maybeStack = uniqueStacks.AppendFrame( + stack, + UniqueStacks::FrameKey(std::move(frameLabel), relevantForJS, + isBaselineInterp, innerWindowID, line, + column, categoryPair)); + if (!maybeStack) { + writer.SetFailure("AppendFrame failure"); + return; + } + stack = *maybeStack; + + } else if (e.Get().IsJitReturnAddr()) { + numFrames++; + + // A JIT frame may expand to multiple frames due to inlining. + void* pc = e.Get().GetPtr(); + const Maybe<Vector<UniqueStacks::FrameKey>>& frameKeys = + uniqueStacks.LookupFramesForJITAddressFromBufferPos( + pc, entryPosition ? entryPosition : e.CurPos()); + MOZ_RELEASE_ASSERT( + frameKeys, + "Attempting to stream samples for a buffer range " + "for which we don't have JITFrameInfo?"); + for (const UniqueStacks::FrameKey& frameKey : *frameKeys) { + maybeStack = uniqueStacks.AppendFrame(stack, frameKey); + if (!maybeStack) { + writer.SetFailure("AppendFrame failure"); + return; + } + stack = *maybeStack; + } + + e.Next(); + + } else { + break; + } + } + + // Even if this stack is considered empty, it contains the root frame, + // which needs to be in the JSON output because following "same samples" + // may refer to it when reusing this sample.mStack. + const Maybe<uint32_t> stackIndex = + uniqueStacks.GetOrAddStackIndex(stack); + if (!stackIndex) { + writer.SetFailure("Can't add unique string for stack"); + return; + } + + // And store that possibly-empty stack in case it's followed by "same + // sample" entries. + previousStack = *stackIndex; + previousStackState = (numFrames == 0) + ? ThreadStreamingContext::eStackWasEmpty + : ThreadStreamingContext::eStackWasNotEmpty; + + // Even if too old or empty, we did process a sample for this thread id. + processedThreadId = threadId; + + // Discard samples that are too old. + if (time < aSinceTime) { + return; + } + + if (numFrames == 0 && runningTimes.IsEmpty()) { + // It is possible to have empty stacks if native stackwalking is + // disabled. Skip samples with empty stacks, unless we have useful + // running times. + return; + } + + WriteSample(writer, ProfileSample{*stackIndex, time, + unresponsiveDuration, runningTimes}); + }; // End of `ReadStack(EntryGetter&)` lambda. + + if (e.Has() && e.Get().IsTime()) { + double time = e.Get().GetDouble(); + e.Next(); + // Note: Even if this sample is too old (before aSinceTime), we still + // need to read it, so that its frames are in the tables, in case there + // is a same-sample following it that would be after aSinceTime, which + // would need these frames to be present. + + ReadStack(e, time, 0, Nothing{}, RunningTimes{}); + + e.SetLocalProgress("Processed sample"); + } else if (e.Has() && e.Get().IsTimeBeforeCompactStack()) { + double time = e.Get().GetDouble(); + // Note: Even if this sample is too old (before aSinceTime), we still + // need to read it, so that its frames are in the tables, in case there + // is a same-sample following it that would be after aSinceTime, which + // would need these frames to be present. + + RunningTimes runningTimes; + Maybe<double> unresponsiveDuration; + + ProfileChunkedBuffer::BlockIterator it = e.Iterator(); + for (;;) { + ++it; + if (it.IsAtEnd()) { + break; + } + ProfileBufferEntryReader er = *it; + ProfileBufferEntry::Kind kind = + er.ReadObject<ProfileBufferEntry::Kind>(); + + // There may be running times before the CompactStack. + if (kind == ProfileBufferEntry::Kind::RunningTimes) { + er.ReadIntoObject(runningTimes); + continue; + } + + // There may be an UnresponsiveDurationMs before the CompactStack. + if (kind == ProfileBufferEntry::Kind::UnresponsiveDurationMs) { + unresponsiveDuration = Some(er.ReadObject<double>()); + continue; + } + + if (kind == ProfileBufferEntry::Kind::CompactStack) { + ProfileChunkedBuffer tempBuffer( + ProfileChunkedBuffer::ThreadSafety::WithoutMutex, + WorkerChunkManager()); + er.ReadIntoObject(tempBuffer); + tempBuffer.Read([&](ProfileChunkedBuffer::Reader* aReader) { + MOZ_ASSERT(aReader, + "Local ProfileChunkedBuffer cannot be out-of-session"); + // This is a compact stack, it should only contain one sample. + EntryGetter stackEntryGetter(*aReader, aFailureLatch); + ReadStack(stackEntryGetter, time, + it.CurrentBlockIndex().ConvertToProfileBufferIndex(), + unresponsiveDuration, runningTimes); + }); + WorkerChunkManager().Reset(tempBuffer.GetAllChunks()); + break; + } + + if (kind == ProfileBufferEntry::Kind::Marker && + aStreamingContextForMarkers) { + StreamMarkerAfterKind(er, *aStreamingContextForMarkers); + continue; + } + + MOZ_ASSERT(kind >= ProfileBufferEntry::Kind::LEGACY_LIMIT, + "There should be no legacy entries between " + "TimeBeforeCompactStack and CompactStack"); + er.SetRemainingBytes(0); + } + + e.RestartAfter(it); + + e.SetLocalProgress("Processed compact sample"); + } else if (e.Has() && e.Get().IsTimeBeforeSameSample()) { + if (previousStackState == ThreadStreamingContext::eNoStackYet) { + // We don't have any full sample yet, we cannot duplicate a "previous" + // one. This should only happen at most once per thread, for the very + // first sample. + continue; + } + + ProfileSample sample; + + // Keep the same `mStack` as previously output. + // Note that it may be empty, this is checked below before writing it. + sample.mStack = previousStack; + + sample.mTime = e.Get().GetDouble(); + + // Ignore samples that are too old. + if (sample.mTime < aSinceTime) { + e.Next(); + continue; + } + + sample.mResponsiveness = Nothing{}; + + sample.mRunningTimes.Clear(); + + ProfileChunkedBuffer::BlockIterator it = e.Iterator(); + for (;;) { + ++it; + if (it.IsAtEnd()) { + break; + } + ProfileBufferEntryReader er = *it; + ProfileBufferEntry::Kind kind = + er.ReadObject<ProfileBufferEntry::Kind>(); + + // There may be running times before the SameSample. + if (kind == ProfileBufferEntry::Kind::RunningTimes) { + er.ReadIntoObject(sample.mRunningTimes); + continue; + } + + if (kind == ProfileBufferEntry::Kind::SameSample) { + if (previousStackState == ThreadStreamingContext::eStackWasEmpty && + sample.mRunningTimes.IsEmpty()) { + // Skip samples with empty stacks, unless we have useful running + // times. + break; + } + WriteSample(writer, sample); + break; + } + + if (kind == ProfileBufferEntry::Kind::Marker && + aStreamingContextForMarkers) { + StreamMarkerAfterKind(er, *aStreamingContextForMarkers); + continue; + } + + MOZ_ASSERT(kind >= ProfileBufferEntry::Kind::LEGACY_LIMIT, + "There should be no legacy entries between " + "TimeBeforeSameSample and SameSample"); + er.SetRemainingBytes(0); + } + + e.RestartAfter(it); + + e.SetLocalProgress("Processed repeated sample"); + } else { + ERROR_AND_CONTINUE("expected a Time entry"); + } + } + + return processedThreadId; + }); +} + +ProfilerThreadId ProfileBuffer::StreamSamplesToJSON( + SpliceableJSONWriter& aWriter, ProfilerThreadId aThreadId, + double aSinceTime, UniqueStacks& aUniqueStacks, + mozilla::ProgressLogger aProgressLogger) const { + ThreadStreamingContext::PreviousStackState previousStackState = + ThreadStreamingContext::eNoStackYet; + uint32_t stack = 0u; +#ifdef DEBUG + int processedCount = 0; +#endif // DEBUG + return DoStreamSamplesAndMarkersToJSON( + aWriter.SourceFailureLatch(), + [&](ProfilerThreadId aReadThreadId) { + Maybe<StreamingParametersForThread> streamingParameters; +#ifdef DEBUG + ++processedCount; + MOZ_ASSERT( + aThreadId.IsSpecified() || + (processedCount == 1 && aReadThreadId.IsSpecified()), + "Unspecified aThreadId should only be used with 1-sample buffer"); +#endif // DEBUG + if (!aThreadId.IsSpecified() || aThreadId == aReadThreadId) { + streamingParameters.emplace(aWriter, aUniqueStacks, + previousStackState, stack); + } + return streamingParameters; + }, + aSinceTime, /* aStreamingContextForMarkers */ nullptr, + std::move(aProgressLogger)); +} + +void ProfileBuffer::StreamSamplesAndMarkersToJSON( + ProcessStreamingContext& aProcessStreamingContext, + mozilla::ProgressLogger aProgressLogger) const { + (void)DoStreamSamplesAndMarkersToJSON( + aProcessStreamingContext.SourceFailureLatch(), + [&](ProfilerThreadId aReadThreadId) { + Maybe<StreamingParametersForThread> streamingParameters; + ThreadStreamingContext* threadData = + aProcessStreamingContext.GetThreadStreamingContext(aReadThreadId); + if (threadData) { + streamingParameters.emplace( + threadData->mSamplesDataWriter, *threadData->mUniqueStacks, + threadData->mPreviousStackState, threadData->mPreviousStack); + } + return streamingParameters; + }, + aProcessStreamingContext.GetSinceTime(), &aProcessStreamingContext, + std::move(aProgressLogger)); +} + +void ProfileBuffer::AddJITInfoForRange( + uint64_t aRangeStart, ProfilerThreadId aThreadId, JSContext* aContext, + JITFrameInfo& aJITFrameInfo, + mozilla::ProgressLogger aProgressLogger) const { + // We can only process JitReturnAddr entries if we have a JSContext. + MOZ_RELEASE_ASSERT(aContext); + + aRangeStart = std::max(aRangeStart, BufferRangeStart()); + aJITFrameInfo.AddInfoForRange( + aRangeStart, BufferRangeEnd(), aContext, + [&](const std::function<void(void*)>& aJITAddressConsumer) { + // Find all JitReturnAddr entries in the given range for the given + // thread, and call aJITAddressConsumer with those addresses. + + mEntries.Read([&](ProfileChunkedBuffer::Reader* aReader) { + MOZ_ASSERT(aReader, + "ProfileChunkedBuffer cannot be out-of-session when " + "sampler is running"); + + EntryGetter e(*aReader, aJITFrameInfo.LocalFailureLatchSource(), + std::move(aProgressLogger), aRangeStart); + + while (true) { + // Advance to the next ThreadId entry. + while (e.Has() && !e.Get().IsThreadId()) { + e.Next(); + } + if (!e.Has()) { + break; + } + + MOZ_ASSERT(e.Get().IsThreadId()); + ProfilerThreadId threadId = e.Get().GetThreadId(); + e.Next(); + + // Ignore samples that are for a different thread. + if (threadId != aThreadId) { + continue; + } + + if (e.Has() && e.Get().IsTime()) { + // Legacy stack. + e.Next(); + while (e.Has() && !e.Get().IsThreadId()) { + if (e.Get().IsJitReturnAddr()) { + aJITAddressConsumer(e.Get().GetPtr()); + } + e.Next(); + } + } else if (e.Has() && e.Get().IsTimeBeforeCompactStack()) { + // Compact stack. + ProfileChunkedBuffer::BlockIterator it = e.Iterator(); + for (;;) { + ++it; + if (it.IsAtEnd()) { + break; + } + ProfileBufferEntryReader er = *it; + ProfileBufferEntry::Kind kind = + er.ReadObject<ProfileBufferEntry::Kind>(); + if (kind == ProfileBufferEntry::Kind::CompactStack) { + ProfileChunkedBuffer tempBuffer( + ProfileChunkedBuffer::ThreadSafety::WithoutMutex, + WorkerChunkManager()); + er.ReadIntoObject(tempBuffer); + tempBuffer.Read([&](ProfileChunkedBuffer::Reader* aReader) { + MOZ_ASSERT( + aReader, + "Local ProfileChunkedBuffer cannot be out-of-session"); + EntryGetter stackEntryGetter( + *aReader, aJITFrameInfo.LocalFailureLatchSource()); + while (stackEntryGetter.Has()) { + if (stackEntryGetter.Get().IsJitReturnAddr()) { + aJITAddressConsumer(stackEntryGetter.Get().GetPtr()); + } + stackEntryGetter.Next(); + } + }); + WorkerChunkManager().Reset(tempBuffer.GetAllChunks()); + break; + } + + MOZ_ASSERT(kind >= ProfileBufferEntry::Kind::LEGACY_LIMIT, + "There should be no legacy entries between " + "TimeBeforeCompactStack and CompactStack"); + er.SetRemainingBytes(0); + } + + e.Next(); + } else if (e.Has() && e.Get().IsTimeBeforeSameSample()) { + // Sample index, nothing to do. + + } else { + ERROR_AND_CONTINUE("expected a Time entry"); + } + } + }); + }); +} + +void ProfileBuffer::StreamMarkersToJSON( + SpliceableJSONWriter& aWriter, ProfilerThreadId aThreadId, + const TimeStamp& aProcessStartTime, double aSinceTime, + UniqueStacks& aUniqueStacks, + mozilla::ProgressLogger aProgressLogger) const { + mEntries.ReadEach([&](ProfileBufferEntryReader& aER) { + auto type = static_cast<ProfileBufferEntry::Kind>( + aER.ReadObject<ProfileBufferEntry::KindUnderlyingType>()); + MOZ_ASSERT(static_cast<ProfileBufferEntry::KindUnderlyingType>(type) < + static_cast<ProfileBufferEntry::KindUnderlyingType>( + ProfileBufferEntry::Kind::MODERN_LIMIT)); + if (type == ProfileBufferEntry::Kind::Marker) { + mozilla::base_profiler_markers_detail::DeserializeAfterKindAndStream( + aER, + [&](const ProfilerThreadId& aMarkerThreadId) { + return (!aThreadId.IsSpecified() || aMarkerThreadId == aThreadId) + ? &aWriter + : nullptr; + }, + [&](ProfileChunkedBuffer& aChunkedBuffer) { + ProfilerBacktrace backtrace("", &aChunkedBuffer); + backtrace.StreamJSON(aWriter, aProcessStartTime, aUniqueStacks); + }, + [&](mozilla::base_profiler_markers_detail::Streaming::DeserializerTag + aTag) { + size_t payloadSize = aER.RemainingBytes(); + + ProfileBufferEntryReader::DoubleSpanOfConstBytes spans = + aER.ReadSpans(payloadSize); + if (MOZ_LIKELY(spans.IsSingleSpan())) { + // Only a single span, we can just refer to it directly + // instead of copying it. + profiler::ffi::gecko_profiler_serialize_marker_for_tag( + aTag, spans.mFirstOrOnly.Elements(), payloadSize, &aWriter); + } else { + // Two spans, we need to concatenate them by copying. + uint8_t* payloadBuffer = new uint8_t[payloadSize]; + spans.CopyBytesTo(payloadBuffer); + profiler::ffi::gecko_profiler_serialize_marker_for_tag( + aTag, payloadBuffer, payloadSize, &aWriter); + delete[] payloadBuffer; + } + }); + } else { + // The entry was not a marker, we need to skip to the end. + aER.SetRemainingBytes(0); + } + }); +} + +void ProfileBuffer::StreamProfilerOverheadToJSON( + SpliceableJSONWriter& aWriter, const TimeStamp& aProcessStartTime, + double aSinceTime, mozilla::ProgressLogger aProgressLogger) const { + mEntries.Read([&](ProfileChunkedBuffer::Reader* aReader) { + MOZ_ASSERT(aReader, + "ProfileChunkedBuffer cannot be out-of-session when sampler is " + "running"); + + EntryGetter e(*aReader, aWriter.SourceFailureLatch(), + std::move(aProgressLogger)); + + enum Schema : uint32_t { + TIME = 0, + LOCKING = 1, + MARKER_CLEANING = 2, + COUNTERS = 3, + THREADS = 4 + }; + + aWriter.StartObjectProperty("profilerOverhead"); + aWriter.StartObjectProperty("samples"); + // Stream all sampling overhead data. We skip other entries, because we + // process them in StreamSamplesToJSON()/etc. + { + JSONSchemaWriter schema(aWriter); + schema.WriteField("time"); + schema.WriteField("locking"); + schema.WriteField("expiredMarkerCleaning"); + schema.WriteField("counters"); + schema.WriteField("threads"); + } + + aWriter.StartArrayProperty("data"); + double firstTime = 0.0; + double lastTime = 0.0; + ProfilerStats intervals, overheads, lockings, cleanings, counters, threads; + while (e.Has()) { + // valid sequence: ProfilerOverheadTime, ProfilerOverheadDuration * 4 + if (e.Get().IsProfilerOverheadTime()) { + double time = e.Get().GetDouble(); + if (time >= aSinceTime) { + e.Next(); + if (!e.Has() || !e.Get().IsProfilerOverheadDuration()) { + ERROR_AND_CONTINUE( + "expected a ProfilerOverheadDuration entry after " + "ProfilerOverheadTime"); + } + double locking = e.Get().GetDouble(); + e.Next(); + if (!e.Has() || !e.Get().IsProfilerOverheadDuration()) { + ERROR_AND_CONTINUE( + "expected a ProfilerOverheadDuration entry after " + "ProfilerOverheadTime,ProfilerOverheadDuration"); + } + double cleaning = e.Get().GetDouble(); + e.Next(); + if (!e.Has() || !e.Get().IsProfilerOverheadDuration()) { + ERROR_AND_CONTINUE( + "expected a ProfilerOverheadDuration entry after " + "ProfilerOverheadTime,ProfilerOverheadDuration*2"); + } + double counter = e.Get().GetDouble(); + e.Next(); + if (!e.Has() || !e.Get().IsProfilerOverheadDuration()) { + ERROR_AND_CONTINUE( + "expected a ProfilerOverheadDuration entry after " + "ProfilerOverheadTime,ProfilerOverheadDuration*3"); + } + double thread = e.Get().GetDouble(); + + if (firstTime == 0.0) { + firstTime = time; + } else { + // Note that we'll have 1 fewer interval than other numbers (because + // we need both ends of an interval to know its duration). The final + // difference should be insignificant over the expected many + // thousands of iterations. + intervals.Count(time - lastTime); + } + lastTime = time; + overheads.Count(locking + cleaning + counter + thread); + lockings.Count(locking); + cleanings.Count(cleaning); + counters.Count(counter); + threads.Count(thread); + + AutoArraySchemaWriter writer(aWriter); + writer.TimeMsElement(TIME, time); + writer.DoubleElement(LOCKING, locking); + writer.DoubleElement(MARKER_CLEANING, cleaning); + writer.DoubleElement(COUNTERS, counter); + writer.DoubleElement(THREADS, thread); + } + } + e.Next(); + } + aWriter.EndArray(); // data + aWriter.EndObject(); // samples + + // Only output statistics if there is at least one full interval (and + // therefore at least two samplings.) + if (intervals.n > 0) { + aWriter.StartObjectProperty("statistics"); + aWriter.DoubleProperty("profiledDuration", lastTime - firstTime); + aWriter.IntProperty("samplingCount", overheads.n); + aWriter.DoubleProperty("overheadDurations", overheads.sum); + aWriter.DoubleProperty("overheadPercentage", + overheads.sum / (lastTime - firstTime)); +#define PROFILER_STATS(name, var) \ + aWriter.DoubleProperty("mean" name, (var).sum / (var).n); \ + aWriter.DoubleProperty("min" name, (var).min); \ + aWriter.DoubleProperty("max" name, (var).max); + PROFILER_STATS("Interval", intervals); + PROFILER_STATS("Overhead", overheads); + PROFILER_STATS("Lockings", lockings); + PROFILER_STATS("Cleaning", cleanings); + PROFILER_STATS("Counter", counters); + PROFILER_STATS("Thread", threads); +#undef PROFILER_STATS + aWriter.EndObject(); // statistics + } + aWriter.EndObject(); // profilerOverhead + }); +} + +struct CounterSample { + double mTime; + uint64_t mNumber; + int64_t mCount; +}; + +using CounterSamples = Vector<CounterSample>; + +static LazyLogModule sFuzzyfoxLog("Fuzzyfox"); + +// HashMap lookup, if not found, a default value is inserted. +// Returns reference to (existing or new) value inside the HashMap. +template <typename HashM, typename Key> +static auto& LookupOrAdd(HashM& aMap, Key&& aKey) { + auto addPtr = aMap.lookupForAdd(aKey); + if (!addPtr) { + MOZ_RELEASE_ASSERT(aMap.add(addPtr, std::forward<Key>(aKey), + typename HashM::Entry::ValueType{})); + MOZ_ASSERT(!!addPtr); + } + return addPtr->value(); +} + +void ProfileBuffer::StreamCountersToJSON( + SpliceableJSONWriter& aWriter, const TimeStamp& aProcessStartTime, + double aSinceTime, mozilla::ProgressLogger aProgressLogger) const { + // Because this is a format entirely internal to the Profiler, any parsing + // error indicates a bug in the ProfileBuffer writing or the parser itself, + // or possibly flaky hardware. + + mEntries.Read([&](ProfileChunkedBuffer::Reader* aReader) { + MOZ_ASSERT(aReader, + "ProfileChunkedBuffer cannot be out-of-session when sampler is " + "running"); + + EntryGetter e(*aReader, aWriter.SourceFailureLatch(), + std::move(aProgressLogger)); + + enum Schema : uint32_t { TIME = 0, COUNT = 1, NUMBER = 2 }; + + // Stream all counters. We skip other entries, because we process them in + // StreamSamplesToJSON()/etc. + // + // Valid sequence in the buffer: + // CounterID + // Time + // ( Count Number? )* + // + // And the JSON (example): + // "counters": { + // "name": "malloc", + // "category": "Memory", + // "description": "Amount of allocated memory", + // "samples": { + // "schema": {"time": 0, "count": 1, "number": 2}, + // "data": [ + // [ + // 16117.033968000002, + // 2446216, + // 6801320 + // ], + // [ + // 16118.037638, + // 2446216, + // 6801320 + // ], + // ], + // }, + // } + + // Build the map of counters and populate it + HashMap<void*, CounterSamples> counters; + + while (e.Has()) { + // skip all non-Counters, including if we start in the middle of a counter + if (e.Get().IsCounterId()) { + void* id = e.Get().GetPtr(); + CounterSamples& data = LookupOrAdd(counters, id); + e.Next(); + if (!e.Has() || !e.Get().IsTime()) { + ERROR_AND_CONTINUE("expected a Time entry"); + } + double time = e.Get().GetDouble(); + e.Next(); + if (time >= aSinceTime) { + if (!e.Has() || !e.Get().IsCount()) { + ERROR_AND_CONTINUE("expected a Count entry"); + } + int64_t count = e.Get().GetUint64(); + e.Next(); + uint64_t number; + if (!e.Has() || !e.Get().IsNumber()) { + number = 0; + } else { + number = e.Get().GetInt64(); + e.Next(); + } + CounterSample sample = {time, number, count}; + MOZ_RELEASE_ASSERT(data.append(sample)); + } else { + // skip counter sample - only need to skip the initial counter + // id, then let the loop at the top skip the rest + } + } else { + e.Next(); + } + } + // we have a map of counter entries; dump them to JSON + if (counters.count() == 0) { + return; + } + + aWriter.StartArrayProperty("counters"); + for (auto iter = counters.iter(); !iter.done(); iter.next()) { + CounterSamples& samples = iter.get().value(); + size_t size = samples.length(); + if (size == 0) { + continue; + } + const BaseProfilerCount* base_counter = + static_cast<const BaseProfilerCount*>(iter.get().key()); + + aWriter.Start(); + aWriter.StringProperty("name", MakeStringSpan(base_counter->mLabel)); + aWriter.StringProperty("category", + MakeStringSpan(base_counter->mCategory)); + aWriter.StringProperty("description", + MakeStringSpan(base_counter->mDescription)); + + bool hasNumber = false; + for (size_t i = 0; i < size; i++) { + if (samples[i].mNumber != 0) { + hasNumber = true; + break; + } + } + aWriter.StartObjectProperty("samples"); + { + JSONSchemaWriter schema(aWriter); + schema.WriteField("time"); + schema.WriteField("count"); + if (hasNumber) { + schema.WriteField("number"); + } + } + + aWriter.StartArrayProperty("data"); + double previousSkippedTime = 0.0; + uint64_t previousNumber = 0; + int64_t previousCount = 0; + for (size_t i = 0; i < size; i++) { + // Encode as deltas, and only encode if different than the previous + // or next sample; Always write the first and last samples. + if (i == 0 || i == size - 1 || samples[i].mNumber != previousNumber || + samples[i].mCount != previousCount || + // Ensure we ouput the first 0 before skipping samples. + (i >= 2 && (samples[i - 2].mNumber != previousNumber || + samples[i - 2].mCount != previousCount))) { + if (i != 0 && samples[i].mTime >= samples[i - 1].mTime) { + MOZ_LOG(sFuzzyfoxLog, mozilla::LogLevel::Error, + ("Fuzzyfox Profiler Assertion: %f >= %f", samples[i].mTime, + samples[i - 1].mTime)); + } + MOZ_ASSERT(i == 0 || samples[i].mTime >= samples[i - 1].mTime); + MOZ_ASSERT(samples[i].mNumber >= previousNumber); + MOZ_ASSERT(samples[i].mNumber - previousNumber <= + uint64_t(std::numeric_limits<int64_t>::max())); + + int64_t numberDelta = + static_cast<int64_t>(samples[i].mNumber - previousNumber); + int64_t countDelta = samples[i].mCount - previousCount; + + if (previousSkippedTime != 0.0 && + (numberDelta != 0 || countDelta != 0)) { + // Write the last skipped sample, unless the new one is all + // zeroes (that'd be redundant) This is useful to know when a + // certain value was last sampled, so that the front-end graph + // will be more correct. + AutoArraySchemaWriter writer(aWriter); + writer.TimeMsElement(TIME, previousSkippedTime); + // The deltas are effectively zeroes, since no change happened + // between the last actually-written sample and the last skipped + // one. + writer.IntElement(COUNT, 0); + if (hasNumber) { + writer.IntElement(NUMBER, 0); + } + } + + AutoArraySchemaWriter writer(aWriter); + writer.TimeMsElement(TIME, samples[i].mTime); + writer.IntElement(COUNT, countDelta); + if (hasNumber) { + writer.IntElement(NUMBER, numberDelta); + } + + previousSkippedTime = 0.0; + previousNumber = samples[i].mNumber; + previousCount = samples[i].mCount; + } else { + previousSkippedTime = samples[i].mTime; + } + } + aWriter.EndArray(); // data + aWriter.EndObject(); // samples + aWriter.End(); // for each counter + } + aWriter.EndArray(); // counters + }); +} + +#undef ERROR_AND_CONTINUE + +static void AddPausedRange(SpliceableJSONWriter& aWriter, const char* aReason, + const Maybe<double>& aStartTime, + const Maybe<double>& aEndTime) { + aWriter.Start(); + if (aStartTime) { + aWriter.TimeDoubleMsProperty("startTime", *aStartTime); + } else { + aWriter.NullProperty("startTime"); + } + if (aEndTime) { + aWriter.TimeDoubleMsProperty("endTime", *aEndTime); + } else { + aWriter.NullProperty("endTime"); + } + aWriter.StringProperty("reason", MakeStringSpan(aReason)); + aWriter.End(); +} + +void ProfileBuffer::StreamPausedRangesToJSON( + SpliceableJSONWriter& aWriter, double aSinceTime, + mozilla::ProgressLogger aProgressLogger) const { + mEntries.Read([&](ProfileChunkedBuffer::Reader* aReader) { + MOZ_ASSERT(aReader, + "ProfileChunkedBuffer cannot be out-of-session when sampler is " + "running"); + + EntryGetter e(*aReader, aWriter.SourceFailureLatch(), + aProgressLogger.CreateSubLoggerFromTo( + 1_pc, "Streaming pauses...", 99_pc, "Streamed pauses")); + + Maybe<double> currentPauseStartTime; + Maybe<double> currentCollectionStartTime; + + while (e.Has()) { + if (e.Get().IsPause()) { + currentPauseStartTime = Some(e.Get().GetDouble()); + } else if (e.Get().IsResume()) { + AddPausedRange(aWriter, "profiler-paused", currentPauseStartTime, + Some(e.Get().GetDouble())); + currentPauseStartTime = Nothing(); + } else if (e.Get().IsCollectionStart()) { + currentCollectionStartTime = Some(e.Get().GetDouble()); + } else if (e.Get().IsCollectionEnd()) { + AddPausedRange(aWriter, "collecting", currentCollectionStartTime, + Some(e.Get().GetDouble())); + currentCollectionStartTime = Nothing(); + } + e.Next(); + } + + if (currentPauseStartTime) { + AddPausedRange(aWriter, "profiler-paused", currentPauseStartTime, + Nothing()); + } + if (currentCollectionStartTime) { + AddPausedRange(aWriter, "collecting", currentCollectionStartTime, + Nothing()); + } + }); +} + +bool ProfileBuffer::DuplicateLastSample(ProfilerThreadId aThreadId, + double aSampleTimeMs, + Maybe<uint64_t>& aLastSample, + const RunningTimes& aRunningTimes) { + if (!aLastSample) { + return false; + } + + if (mEntries.IsIndexInCurrentChunk(ProfileBufferIndex{*aLastSample})) { + // The last (fully-written) sample is in this chunk, we can refer to it. + + // Note that between now and when we write the SameSample below, another + // chunk could have been started, so the SameSample will in fact refer to a + // block in a previous chunk. This is okay, because: + // - When serializing to JSON, if that chunk is still there, we'll still be + // able to find that old stack, so nothing will be lost. + // - If unfortunately that chunk has been destroyed, we will lose this + // sample. But this will only happen to the first sample (per thread) in + // in the whole JSON output, because the next time we're here to duplicate + // the same sample again, IsIndexInCurrentChunk will say `false` and we + // will fall back to the normal copy or even re-sample. Losing the first + // sample out of many in a whole recording is acceptable. + // + // |---| = chunk, S = Sample, D = Duplicate, s = same sample + // |---S-s-s--| |s-D--s--s-| |s-D--s---s| + // Later, the first chunk is destroyed/recycled: + // |s-D--s--s-| |s-D--s---s| |-... + // Output: ^ ^ ^ ^ + // `-|--|-------|--- Same but no previous -> lost. + // `--|-------|--- Full duplicate sample. + // `-------|--- Same with previous -> okay. + // `--- Same but now we have a previous -> okay! + + AUTO_PROFILER_STATS(DuplicateLastSample_SameSample); + + // Add the thread id first. We don't update `aLastSample` because we are not + // writing a full sample. + (void)AddThreadIdEntry(aThreadId); + + // Copy the new time, to be followed by a SameSample. + AddEntry(ProfileBufferEntry::TimeBeforeSameSample(aSampleTimeMs)); + + // Add running times if they have data. + if (!aRunningTimes.IsEmpty()) { + mEntries.PutObjects(ProfileBufferEntry::Kind::RunningTimes, + aRunningTimes); + } + + // Finish with a SameSample entry. + mEntries.PutObjects(ProfileBufferEntry::Kind::SameSample); + + return true; + } + + AUTO_PROFILER_STATS(DuplicateLastSample_copy); + + ProfileChunkedBuffer tempBuffer( + ProfileChunkedBuffer::ThreadSafety::WithoutMutex, WorkerChunkManager()); + + auto retrieveWorkerChunk = MakeScopeExit( + [&]() { WorkerChunkManager().Reset(tempBuffer.GetAllChunks()); }); + + const bool ok = mEntries.Read([&](ProfileChunkedBuffer::Reader* aReader) { + MOZ_ASSERT(aReader, + "ProfileChunkedBuffer cannot be out-of-session when sampler is " + "running"); + + // DuplicateLastSample is only called during profiling, so we don't need a + // progress logger (only useful when capturing the final profile). + EntryGetter e(*aReader, mozilla::FailureLatchInfallibleSource::Singleton(), + ProgressLogger{}, *aLastSample); + + if (e.CurPos() != *aLastSample) { + // The last sample is no longer within the buffer range, so we cannot + // use it. Reset the stored buffer position to Nothing(). + aLastSample.reset(); + return false; + } + + MOZ_RELEASE_ASSERT(e.Has() && e.Get().IsThreadId() && + e.Get().GetThreadId() == aThreadId); + + e.Next(); + + // Go through the whole entry and duplicate it, until we find the next + // one. + while (e.Has()) { + switch (e.Get().GetKind()) { + case ProfileBufferEntry::Kind::Pause: + case ProfileBufferEntry::Kind::Resume: + case ProfileBufferEntry::Kind::PauseSampling: + case ProfileBufferEntry::Kind::ResumeSampling: + case ProfileBufferEntry::Kind::CollectionStart: + case ProfileBufferEntry::Kind::CollectionEnd: + case ProfileBufferEntry::Kind::ThreadId: + case ProfileBufferEntry::Kind::TimeBeforeSameSample: + // We're done. + return true; + case ProfileBufferEntry::Kind::Time: + // Copy with new time + AddEntry(tempBuffer, ProfileBufferEntry::Time(aSampleTimeMs)); + break; + case ProfileBufferEntry::Kind::TimeBeforeCompactStack: { + // Copy with new time, followed by a compact stack. + AddEntry(tempBuffer, + ProfileBufferEntry::TimeBeforeCompactStack(aSampleTimeMs)); + + // Add running times if they have data. + if (!aRunningTimes.IsEmpty()) { + tempBuffer.PutObjects(ProfileBufferEntry::Kind::RunningTimes, + aRunningTimes); + } + + // The `CompactStack` *must* be present afterwards, but may not + // immediately follow `TimeBeforeCompactStack` (e.g., some markers + // could be written in-between), so we need to look for it in the + // following entries. + ProfileChunkedBuffer::BlockIterator it = e.Iterator(); + for (;;) { + ++it; + if (it.IsAtEnd()) { + break; + } + ProfileBufferEntryReader er = *it; + auto kind = static_cast<ProfileBufferEntry::Kind>( + er.ReadObject<ProfileBufferEntry::KindUnderlyingType>()); + MOZ_ASSERT( + static_cast<ProfileBufferEntry::KindUnderlyingType>(kind) < + static_cast<ProfileBufferEntry::KindUnderlyingType>( + ProfileBufferEntry::Kind::MODERN_LIMIT)); + if (kind == ProfileBufferEntry::Kind::CompactStack) { + // Found our CompactStack, just make a copy of the whole entry. + er = *it; + auto bytes = er.RemainingBytes(); + MOZ_ASSERT(bytes < + ProfileBufferChunkManager::scExpectedMaximumStackSize); + tempBuffer.Put(bytes, [&](Maybe<ProfileBufferEntryWriter>& aEW) { + MOZ_ASSERT(aEW.isSome(), "tempBuffer cannot be out-of-session"); + aEW->WriteFromReader(er, bytes); + }); + // CompactStack marks the end, we're done. + break; + } + + MOZ_ASSERT(kind >= ProfileBufferEntry::Kind::LEGACY_LIMIT, + "There should be no legacy entries between " + "TimeBeforeCompactStack and CompactStack"); + er.SetRemainingBytes(0); + // Here, we have encountered a non-legacy entry that was not the + // CompactStack we're looking for; just continue the search... + } + // We're done. + return true; + } + case ProfileBufferEntry::Kind::Number: + case ProfileBufferEntry::Kind::Count: + // Don't copy anything not part of a thread's stack sample + break; + case ProfileBufferEntry::Kind::CounterId: + // CounterId is normally followed by Time - if so, we'd like + // to skip it. If we duplicate Time, it won't hurt anything, just + // waste buffer space (and this can happen if the CounterId has + // fallen off the end of the buffer, but Time (and Number/Count) + // are still in the buffer). + e.Next(); + if (e.Has() && e.Get().GetKind() != ProfileBufferEntry::Kind::Time) { + // this would only happen if there was an invalid sequence + // in the buffer. Don't skip it. + continue; + } + // we've skipped Time + break; + case ProfileBufferEntry::Kind::ProfilerOverheadTime: + // ProfilerOverheadTime is normally followed by + // ProfilerOverheadDuration*4 - if so, we'd like to skip it. Don't + // duplicate, as we are in the middle of a sampling and will soon + // capture its own overhead. + e.Next(); + // A missing Time would only happen if there was an invalid + // sequence in the buffer. Don't skip unexpected entry. + if (e.Has() && + e.Get().GetKind() != + ProfileBufferEntry::Kind::ProfilerOverheadDuration) { + continue; + } + e.Next(); + if (e.Has() && + e.Get().GetKind() != + ProfileBufferEntry::Kind::ProfilerOverheadDuration) { + continue; + } + e.Next(); + if (e.Has() && + e.Get().GetKind() != + ProfileBufferEntry::Kind::ProfilerOverheadDuration) { + continue; + } + e.Next(); + if (e.Has() && + e.Get().GetKind() != + ProfileBufferEntry::Kind::ProfilerOverheadDuration) { + continue; + } + // we've skipped ProfilerOverheadTime and + // ProfilerOverheadDuration*4. + break; + default: { + // Copy anything else we don't know about. + AddEntry(tempBuffer, e.Get()); + break; + } + } + e.Next(); + } + return true; + }); + + if (!ok) { + return false; + } + + // If the buffer was big enough, there won't be any cleared blocks. + if (tempBuffer.GetState().mClearedBlockCount != 0) { + // No need to try to read stack again as it won't fit. Reset the stored + // buffer position to Nothing(). + aLastSample.reset(); + return false; + } + + aLastSample = Some(AddThreadIdEntry(aThreadId)); + + mEntries.AppendContents(tempBuffer); + + return true; +} + +void ProfileBuffer::DiscardSamplesBeforeTime(double aTime) { + // This function does nothing! + // The duration limit will be removed from Firefox, see bug 1632365. + Unused << aTime; +} + +// END ProfileBuffer +//////////////////////////////////////////////////////////////////////// diff --git a/tools/profiler/core/ProfileBufferEntry.h b/tools/profiler/core/ProfileBufferEntry.h new file mode 100644 index 0000000000..bfee4923a3 --- /dev/null +++ b/tools/profiler/core/ProfileBufferEntry.h @@ -0,0 +1,532 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfileBufferEntry_h +#define ProfileBufferEntry_h + +#include <cstdint> +#include <cstdlib> +#include <functional> +#include <utility> +#include <type_traits> +#include "gtest/MozGtestFriend.h" +#include "js/ProfilingCategory.h" +#include "mozilla/Attributes.h" +#include "mozilla/HashFunctions.h" +#include "mozilla/HashTable.h" +#include "mozilla/Maybe.h" +#include "mozilla/ProfileBufferEntryKinds.h" +#include "mozilla/ProfileJSONWriter.h" +#include "mozilla/ProfilerUtils.h" +#include "mozilla/UniquePtrExtensions.h" +#include "mozilla/Variant.h" +#include "mozilla/Vector.h" +#include "nsString.h" + +class ProfilerCodeAddressService; +struct JSContext; + +class ProfileBufferEntry { + public: + using KindUnderlyingType = + std::underlying_type_t<::mozilla::ProfileBufferEntryKind>; + using Kind = mozilla::ProfileBufferEntryKind; + + ProfileBufferEntry(); + + static constexpr size_t kNumChars = mozilla::ProfileBufferEntryNumChars; + + private: + // aString must be a static string. + ProfileBufferEntry(Kind aKind, const char* aString); + ProfileBufferEntry(Kind aKind, char aChars[kNumChars]); + ProfileBufferEntry(Kind aKind, void* aPtr); + ProfileBufferEntry(Kind aKind, double aDouble); + ProfileBufferEntry(Kind aKind, int64_t aInt64); + ProfileBufferEntry(Kind aKind, uint64_t aUint64); + ProfileBufferEntry(Kind aKind, int aInt); + ProfileBufferEntry(Kind aKind, ProfilerThreadId aThreadId); + + public: +#define CTOR(KIND, TYPE, SIZE) \ + static ProfileBufferEntry KIND(TYPE aVal) { \ + return ProfileBufferEntry(Kind::KIND, aVal); \ + } + FOR_EACH_PROFILE_BUFFER_ENTRY_KIND(CTOR) +#undef CTOR + + Kind GetKind() const { return mKind; } + +#define IS_KIND(KIND, TYPE, SIZE) \ + bool Is##KIND() const { return mKind == Kind::KIND; } + FOR_EACH_PROFILE_BUFFER_ENTRY_KIND(IS_KIND) +#undef IS_KIND + + private: + FRIEND_TEST(ThreadProfile, InsertOneEntry); + FRIEND_TEST(ThreadProfile, InsertOneEntryWithTinyBuffer); + FRIEND_TEST(ThreadProfile, InsertEntriesNoWrap); + FRIEND_TEST(ThreadProfile, InsertEntriesWrap); + FRIEND_TEST(ThreadProfile, MemoryMeasure); + friend class ProfileBuffer; + + Kind mKind; + uint8_t mStorage[kNumChars]; + + const char* GetString() const; + void* GetPtr() const; + double GetDouble() const; + int GetInt() const; + int64_t GetInt64() const; + uint64_t GetUint64() const; + ProfilerThreadId GetThreadId() const; + void CopyCharsInto(char (&aOutArray)[kNumChars]) const; +}; + +// Packed layout: 1 byte for the tag + 8 bytes for the value. +static_assert(sizeof(ProfileBufferEntry) == 9, "bad ProfileBufferEntry size"); + +// Contains all the information about JIT frames that is needed to stream stack +// frames for JitReturnAddr entries in the profiler buffer. +// Every return address (void*) is mapped to one or more JITFrameKeys, and +// every JITFrameKey is mapped to a JSON string for that frame. +// mRangeStart and mRangeEnd describe the range in the buffer for which this +// mapping is valid. Only JitReturnAddr entries within that buffer range can be +// processed using this JITFrameInfoForBufferRange object. +struct JITFrameInfoForBufferRange final { + JITFrameInfoForBufferRange Clone() const; + + uint64_t mRangeStart; + uint64_t mRangeEnd; // mRangeEnd marks the first invalid index. + + struct JITFrameKey { + bool operator==(const JITFrameKey& aOther) const { + return mCanonicalAddress == aOther.mCanonicalAddress && + mDepth == aOther.mDepth; + } + bool operator!=(const JITFrameKey& aOther) const { + return !(*this == aOther); + } + + void* mCanonicalAddress; + uint32_t mDepth; + }; + struct JITFrameKeyHasher { + using Lookup = JITFrameKey; + + static mozilla::HashNumber hash(const JITFrameKey& aLookup) { + mozilla::HashNumber hash = 0; + hash = mozilla::AddToHash(hash, aLookup.mCanonicalAddress); + hash = mozilla::AddToHash(hash, aLookup.mDepth); + return hash; + } + + static bool match(const JITFrameKey& aKey, const JITFrameKey& aLookup) { + return aKey == aLookup; + } + + static void rekey(JITFrameKey& aKey, const JITFrameKey& aNewKey) { + aKey = aNewKey; + } + }; + + using JITAddressToJITFramesMap = + mozilla::HashMap<void*, mozilla::Vector<JITFrameKey>>; + JITAddressToJITFramesMap mJITAddressToJITFramesMap; + using JITFrameToFrameJSONMap = + mozilla::HashMap<JITFrameKey, nsCString, JITFrameKeyHasher>; + JITFrameToFrameJSONMap mJITFrameToFrameJSONMap; +}; + +// Contains JITFrameInfoForBufferRange objects for multiple profiler buffer +// ranges. +class JITFrameInfo final { + public: + JITFrameInfo() + : mUniqueStrings(mozilla::MakeUniqueFallible<UniqueJSONStrings>( + mLocalFailureLatchSource)) { + if (!mUniqueStrings) { + mLocalFailureLatchSource.SetFailure( + "OOM in JITFrameInfo allocating mUniqueStrings"); + } + } + + MOZ_IMPLICIT JITFrameInfo(const JITFrameInfo& aOther, + mozilla::ProgressLogger aProgressLogger); + + // Creates a new JITFrameInfoForBufferRange object in mRanges by looking up + // information about the provided JIT return addresses using aCx. + // Addresses are provided like this: + // The caller of AddInfoForRange supplies a function in aJITAddressProvider. + // This function will be called once, synchronously, with an + // aJITAddressConsumer argument, which is a function that needs to be called + // for every address. That function can be called multiple times for the same + // address. + void AddInfoForRange( + uint64_t aRangeStart, uint64_t aRangeEnd, JSContext* aCx, + const std::function<void(const std::function<void(void*)>&)>& + aJITAddressProvider); + + // Returns whether the information stored in this object is still relevant + // for any entries in the buffer. + bool HasExpired(uint64_t aCurrentBufferRangeStart) const { + if (mRanges.empty()) { + // No information means no relevant information. Allow this object to be + // discarded. + return true; + } + return mRanges.back().mRangeEnd <= aCurrentBufferRangeStart; + } + + mozilla::FailureLatch& LocalFailureLatchSource() { + return mLocalFailureLatchSource; + } + + // The encapsulated data points at the local FailureLatch, so on the way out + // they must be given a new external FailureLatch to start using instead. + mozilla::Vector<JITFrameInfoForBufferRange>&& MoveRangesWithNewFailureLatch( + mozilla::FailureLatch& aFailureLatch) &&; + mozilla::UniquePtr<UniqueJSONStrings>&& MoveUniqueStringsWithNewFailureLatch( + mozilla::FailureLatch& aFailureLatch) &&; + + private: + // JITFrameInfo's may exist during profiling, so it carries its own fallible + // FailureLatch. If&when the data below is finally extracted, any error is + // forwarded to the caller. + mozilla::FailureLatchSource mLocalFailureLatchSource; + + // The array of ranges of JIT frame information, sorted by buffer position. + // Ranges are non-overlapping. + // The JSON of the cached frames can contain string indexes, which refer + // to strings in mUniqueStrings. + mozilla::Vector<JITFrameInfoForBufferRange> mRanges; + + // The string table which contains strings used in the frame JSON that's + // cached in mRanges. + mozilla::UniquePtr<UniqueJSONStrings> mUniqueStrings; +}; + +class UniqueStacks final : public mozilla::FailureLatch { + public: + struct FrameKey { + explicit FrameKey(const char* aLocation) + : mData(NormalFrameData{nsCString(aLocation), false, false, 0, + mozilla::Nothing(), mozilla::Nothing()}) {} + + FrameKey(nsCString&& aLocation, bool aRelevantForJS, bool aBaselineInterp, + uint64_t aInnerWindowID, const mozilla::Maybe<unsigned>& aLine, + const mozilla::Maybe<unsigned>& aColumn, + const mozilla::Maybe<JS::ProfilingCategoryPair>& aCategoryPair) + : mData(NormalFrameData{aLocation, aRelevantForJS, aBaselineInterp, + aInnerWindowID, aLine, aColumn, + aCategoryPair}) {} + + FrameKey(void* aJITAddress, uint32_t aJITDepth, uint32_t aRangeIndex) + : mData(JITFrameData{aJITAddress, aJITDepth, aRangeIndex}) {} + + FrameKey(const FrameKey& aToCopy) = default; + + uint32_t Hash() const; + bool operator==(const FrameKey& aOther) const { + return mData == aOther.mData; + } + + struct NormalFrameData { + bool operator==(const NormalFrameData& aOther) const; + + nsCString mLocation; + bool mRelevantForJS; + bool mBaselineInterp; + uint64_t mInnerWindowID; + mozilla::Maybe<unsigned> mLine; + mozilla::Maybe<unsigned> mColumn; + mozilla::Maybe<JS::ProfilingCategoryPair> mCategoryPair; + }; + struct JITFrameData { + bool operator==(const JITFrameData& aOther) const; + + void* mCanonicalAddress; + uint32_t mDepth; + uint32_t mRangeIndex; + }; + mozilla::Variant<NormalFrameData, JITFrameData> mData; + }; + + struct FrameKeyHasher { + using Lookup = FrameKey; + + static mozilla::HashNumber hash(const FrameKey& aLookup) { + mozilla::HashNumber hash = 0; + if (aLookup.mData.is<FrameKey::NormalFrameData>()) { + const FrameKey::NormalFrameData& data = + aLookup.mData.as<FrameKey::NormalFrameData>(); + if (!data.mLocation.IsEmpty()) { + hash = mozilla::AddToHash(hash, + mozilla::HashString(data.mLocation.get())); + } + hash = mozilla::AddToHash(hash, data.mRelevantForJS); + hash = mozilla::AddToHash(hash, data.mBaselineInterp); + hash = mozilla::AddToHash(hash, data.mInnerWindowID); + if (data.mLine.isSome()) { + hash = mozilla::AddToHash(hash, *data.mLine); + } + if (data.mColumn.isSome()) { + hash = mozilla::AddToHash(hash, *data.mColumn); + } + if (data.mCategoryPair.isSome()) { + hash = mozilla::AddToHash(hash, + static_cast<uint32_t>(*data.mCategoryPair)); + } + } else { + const FrameKey::JITFrameData& data = + aLookup.mData.as<FrameKey::JITFrameData>(); + hash = mozilla::AddToHash(hash, data.mCanonicalAddress); + hash = mozilla::AddToHash(hash, data.mDepth); + hash = mozilla::AddToHash(hash, data.mRangeIndex); + } + return hash; + } + + static bool match(const FrameKey& aKey, const FrameKey& aLookup) { + return aKey == aLookup; + } + + static void rekey(FrameKey& aKey, const FrameKey& aNewKey) { + aKey = aNewKey; + } + }; + + struct StackKey { + mozilla::Maybe<uint32_t> mPrefixStackIndex; + uint32_t mFrameIndex; + + explicit StackKey(uint32_t aFrame) + : mFrameIndex(aFrame), mHash(mozilla::HashGeneric(aFrame)) {} + + StackKey(const StackKey& aPrefix, uint32_t aPrefixStackIndex, + uint32_t aFrame) + : mPrefixStackIndex(mozilla::Some(aPrefixStackIndex)), + mFrameIndex(aFrame), + mHash(mozilla::AddToHash(aPrefix.mHash, aFrame)) {} + + mozilla::HashNumber Hash() const { return mHash; } + + bool operator==(const StackKey& aOther) const { + return mPrefixStackIndex == aOther.mPrefixStackIndex && + mFrameIndex == aOther.mFrameIndex; + } + + private: + mozilla::HashNumber mHash; + }; + + struct StackKeyHasher { + using Lookup = StackKey; + + static mozilla::HashNumber hash(const StackKey& aLookup) { + return aLookup.Hash(); + } + + static bool match(const StackKey& aKey, const StackKey& aLookup) { + return aKey == aLookup; + } + + static void rekey(StackKey& aKey, const StackKey& aNewKey) { + aKey = aNewKey; + } + }; + + UniqueStacks(mozilla::FailureLatch& aFailureLatch, + JITFrameInfo&& aJITFrameInfo, + ProfilerCodeAddressService* aCodeAddressService = nullptr); + + // Return a StackKey for aFrame as the stack's root frame (no prefix). + [[nodiscard]] mozilla::Maybe<StackKey> BeginStack(const FrameKey& aFrame); + + // Return a new StackKey that is obtained by appending aFrame to aStack. + [[nodiscard]] mozilla::Maybe<StackKey> AppendFrame(const StackKey& aStack, + const FrameKey& aFrame); + + // Look up frame keys for the given JIT address, and ensure that our frame + // table has entries for the returned frame keys. The JSON for these frames + // is taken from mJITInfoRanges. + // aBufferPosition is needed in order to look up the correct JIT frame info + // object in mJITInfoRanges. + [[nodiscard]] mozilla::Maybe<mozilla::Vector<UniqueStacks::FrameKey>> + LookupFramesForJITAddressFromBufferPos(void* aJITAddress, + uint64_t aBufferPosition); + + [[nodiscard]] mozilla::Maybe<uint32_t> GetOrAddFrameIndex( + const FrameKey& aFrame); + [[nodiscard]] mozilla::Maybe<uint32_t> GetOrAddStackIndex( + const StackKey& aStack); + + void SpliceFrameTableElements(SpliceableJSONWriter& aWriter); + void SpliceStackTableElements(SpliceableJSONWriter& aWriter); + + [[nodiscard]] UniqueJSONStrings& UniqueStrings() { + MOZ_RELEASE_ASSERT(mUniqueStrings.get()); + return *mUniqueStrings; + } + + // Find the function name at the given PC (if a ProfilerCodeAddressService was + // provided), otherwise just stringify that PC. + [[nodiscard]] nsAutoCString FunctionNameOrAddress(void* aPC); + + FAILURELATCH_IMPL_PROXY(mFrameTableWriter) + + private: + void StreamNonJITFrame(const FrameKey& aFrame); + void StreamStack(const StackKey& aStack); + + mozilla::UniquePtr<UniqueJSONStrings> mUniqueStrings; + + ProfilerCodeAddressService* mCodeAddressService = nullptr; + + SpliceableChunkedJSONWriter mFrameTableWriter; + mozilla::HashMap<FrameKey, uint32_t, FrameKeyHasher> mFrameToIndexMap; + + SpliceableChunkedJSONWriter mStackTableWriter; + mozilla::HashMap<StackKey, uint32_t, StackKeyHasher> mStackToIndexMap; + + mozilla::Vector<JITFrameInfoForBufferRange> mJITInfoRanges; +}; + +// +// Thread profile JSON Format +// -------------------------- +// +// The profile contains much duplicate information. The output JSON of the +// profile attempts to deduplicate strings, frames, and stack prefixes, to cut +// down on size and to increase JSON streaming speed. Deduplicated values are +// streamed as indices into their respective tables. +// +// Further, arrays of objects with the same set of properties (e.g., samples, +// frames) are output as arrays according to a schema instead of an object +// with property names. A property that is not present is represented in the +// array as null or undefined. +// +// The format of the thread profile JSON is shown by the following example +// with 1 sample and 1 marker: +// +// { +// "name": "Foo", +// "tid": 42, +// "samples": +// { +// "schema": +// { +// "stack": 0, /* index into stackTable */ +// "time": 1, /* number */ +// "eventDelay": 2, /* number */ +// "ThreadCPUDelta": 3, /* optional number */ +// }, +// "data": +// [ +// [ 1, 0.0, 0.0 ] /* { stack: 1, time: 0.0, eventDelay: 0.0 } */ +// ] +// }, +// +// "markers": +// { +// "schema": +// { +// "name": 0, /* index into stringTable */ +// "time": 1, /* number */ +// "data": 2 /* arbitrary JSON */ +// }, +// "data": +// [ +// [ 3, 0.1 ] /* { name: 'example marker', time: 0.1 } */ +// ] +// }, +// +// "stackTable": +// { +// "schema": +// { +// "prefix": 0, /* index into stackTable */ +// "frame": 1 /* index into frameTable */ +// }, +// "data": +// [ +// [ null, 0 ], /* (root) */ +// [ 0, 1 ] /* (root) > foo.js */ +// ] +// }, +// +// "frameTable": +// { +// "schema": +// { +// "location": 0, /* index into stringTable */ +// "relevantForJS": 1, /* bool */ +// "innerWindowID": 2, /* inner window ID of global JS `window` object */ +// "implementation": 3, /* index into stringTable */ +// "line": 4, /* number */ +// "column": 5, /* number */ +// "category": 6, /* index into profile.meta.categories */ +// "subcategory": 7 /* index into +// profile.meta.categories[category].subcategories */ +// }, +// "data": +// [ +// [ 0 ], /* { location: '(root)' } */ +// [ 1, null, null, 2 ] /* { location: 'foo.js', +// implementation: 'baseline' } */ +// ] +// }, +// +// "stringTable": +// [ +// "(root)", +// "foo.js", +// "baseline", +// "example marker" +// ] +// } +// +// Process: +// { +// "name": "Bar", +// "pid": 24, +// "threads": +// [ +// <0-N threads from above> +// ], +// "counters": /* includes the memory counter */ +// [ +// { +// "name": "qwerty", +// "category": "uiop", +// "description": "this is qwerty uiop", +// "sample_groups: +// [ +// { +// "id": 42, /* number (thread id, or object identifier (tab), etc) */ +// "samples: +// { +// "schema": +// { +// "time": 1, /* number */ +// "number": 2, /* number (of times the counter was touched) */ +// "count": 3 /* number (total for the counter) */ +// }, +// "data": +// [ +// [ 0.1, 1824, +// 454622 ] /* { time: 0.1, number: 1824, count: 454622 } */ +// ] +// }, +// }, +// /* more sample-group objects with different id's */ +// ] +// }, +// /* more counters */ +// ], +// } +// +#endif /* ndef ProfileBufferEntry_h */ diff --git a/tools/profiler/core/ProfiledThreadData.cpp b/tools/profiler/core/ProfiledThreadData.cpp new file mode 100644 index 0000000000..febda0d85b --- /dev/null +++ b/tools/profiler/core/ProfiledThreadData.cpp @@ -0,0 +1,455 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfiledThreadData.h" + +#include "platform.h" +#include "ProfileBuffer.h" + +#include "mozilla/OriginAttributes.h" +#include "mozilla/Span.h" +#include "nsXULAppAPI.h" + +#if defined(GP_OS_darwin) +# include <pthread.h> +#endif + +using namespace mozilla::literals::ProportionValue_literals; + +ProfiledThreadData::ProfiledThreadData( + const mozilla::profiler::ThreadRegistrationInfo& aThreadInfo) + : mThreadInfo(aThreadInfo.Name(), aThreadInfo.ThreadId(), + aThreadInfo.IsMainThread(), aThreadInfo.RegisterTime()) { + MOZ_COUNT_CTOR(ProfiledThreadData); +} + +ProfiledThreadData::ProfiledThreadData( + mozilla::profiler::ThreadRegistrationInfo&& aThreadInfo) + : mThreadInfo(std::move(aThreadInfo)) { + MOZ_COUNT_CTOR(ProfiledThreadData); +} + +ProfiledThreadData::~ProfiledThreadData() { + MOZ_COUNT_DTOR(ProfiledThreadData); +} + +static void StreamTables(UniqueStacks&& aUniqueStacks, JSContext* aCx, + SpliceableJSONWriter& aWriter, + const mozilla::TimeStamp& aProcessStartTime, + mozilla::ProgressLogger aProgressLogger) { + aWriter.StartObjectProperty("stackTable"); + { + { + JSONSchemaWriter schema(aWriter); + schema.WriteField("prefix"); + schema.WriteField("frame"); + } + + aWriter.StartArrayProperty("data"); + { + aProgressLogger.SetLocalProgress(1_pc, "Splicing stack table..."); + aUniqueStacks.SpliceStackTableElements(aWriter); + aProgressLogger.SetLocalProgress(30_pc, "Spliced stack table"); + } + aWriter.EndArray(); + } + aWriter.EndObject(); + + aWriter.StartObjectProperty("frameTable"); + { + { + JSONSchemaWriter schema(aWriter); + schema.WriteField("location"); + schema.WriteField("relevantForJS"); + schema.WriteField("innerWindowID"); + schema.WriteField("implementation"); + schema.WriteField("line"); + schema.WriteField("column"); + schema.WriteField("category"); + schema.WriteField("subcategory"); + } + + aWriter.StartArrayProperty("data"); + { + aProgressLogger.SetLocalProgress(30_pc, "Splicing frame table..."); + aUniqueStacks.SpliceFrameTableElements(aWriter); + aProgressLogger.SetLocalProgress(60_pc, "Spliced frame table"); + } + aWriter.EndArray(); + } + aWriter.EndObject(); + + aWriter.StartArrayProperty("stringTable"); + { + aProgressLogger.SetLocalProgress(60_pc, "Splicing string table..."); + std::move(aUniqueStacks.UniqueStrings()).SpliceStringTableElements(aWriter); + aProgressLogger.SetLocalProgress(90_pc, "Spliced string table"); + } + aWriter.EndArray(); +} + +mozilla::NotNull<mozilla::UniquePtr<UniqueStacks>> +ProfiledThreadData::PrepareUniqueStacks( + const ProfileBuffer& aBuffer, JSContext* aCx, + mozilla::FailureLatch& aFailureLatch, ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger) { + if (mJITFrameInfoForPreviousJSContexts && + mJITFrameInfoForPreviousJSContexts->HasExpired( + aBuffer.BufferRangeStart())) { + mJITFrameInfoForPreviousJSContexts = nullptr; + } + aProgressLogger.SetLocalProgress(1_pc, "Checked JIT frame info presence"); + + // If we have an existing JITFrameInfo in mJITFrameInfoForPreviousJSContexts, + // copy the data from it. + JITFrameInfo jitFrameInfo = + mJITFrameInfoForPreviousJSContexts + ? JITFrameInfo(*mJITFrameInfoForPreviousJSContexts, + aProgressLogger.CreateSubLoggerTo( + "Retrieving JIT frame info...", 10_pc, + "Retrieved JIT frame info")) + : JITFrameInfo(); + + if (aCx && mBufferPositionWhenReceivedJSContext) { + aBuffer.AddJITInfoForRange( + *mBufferPositionWhenReceivedJSContext, mThreadInfo.ThreadId(), aCx, + jitFrameInfo, + aProgressLogger.CreateSubLoggerTo("Adding JIT info...", 90_pc, + "Added JIT info")); + } else { + aProgressLogger.SetLocalProgress(90_pc, "No JIT info"); + } + + return mozilla::MakeNotNull<mozilla::UniquePtr<UniqueStacks>>( + aFailureLatch, std::move(jitFrameInfo), aService); +} + +void ProfiledThreadData::StreamJSON( + const ProfileBuffer& aBuffer, JSContext* aCx, SpliceableJSONWriter& aWriter, + const nsACString& aProcessName, const nsACString& aETLDplus1, + const mozilla::TimeStamp& aProcessStartTime, double aSinceTime, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger) { + mozilla::NotNull<mozilla::UniquePtr<UniqueStacks>> uniqueStacks = + PrepareUniqueStacks(aBuffer, aCx, aWriter.SourceFailureLatch(), aService, + aProgressLogger.CreateSubLoggerFromTo( + 0_pc, "Preparing unique stacks...", 10_pc, + "Prepared Unique stacks")); + + aWriter.SetUniqueStrings(uniqueStacks->UniqueStrings()); + + aWriter.Start(); + { + StreamSamplesAndMarkers( + mThreadInfo.Name(), mThreadInfo.ThreadId(), aBuffer, aWriter, + aProcessName, aETLDplus1, aProcessStartTime, mThreadInfo.RegisterTime(), + mUnregisterTime, aSinceTime, *uniqueStacks, + aProgressLogger.CreateSubLoggerTo( + 90_pc, + "ProfiledThreadData::StreamJSON: Streamed samples and markers")); + + StreamTables(std::move(*uniqueStacks), aCx, aWriter, aProcessStartTime, + aProgressLogger.CreateSubLoggerTo( + 99_pc, "Streamed tables and trace logger")); + } + aWriter.End(); + + aWriter.ResetUniqueStrings(); +} + +void ProfiledThreadData::StreamJSON( + ThreadStreamingContext&& aThreadStreamingContext, + SpliceableJSONWriter& aWriter, const nsACString& aProcessName, + const nsACString& aETLDplus1, const mozilla::TimeStamp& aProcessStartTime, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger) { + aWriter.Start(); + { + StreamSamplesAndMarkers( + mThreadInfo.Name(), aThreadStreamingContext, aWriter, aProcessName, + aETLDplus1, aProcessStartTime, mThreadInfo.RegisterTime(), + mUnregisterTime, + aProgressLogger.CreateSubLoggerFromTo( + 1_pc, "ProfiledThreadData::StreamJSON(context): Streaming...", + 90_pc, + "ProfiledThreadData::StreamJSON(context): Streamed samples and " + "markers")); + + StreamTables( + std::move(*aThreadStreamingContext.mUniqueStacks), + aThreadStreamingContext.mJSContext, aWriter, aProcessStartTime, + aProgressLogger.CreateSubLoggerTo( + "ProfiledThreadData::StreamJSON(context): Streaming tables...", + 99_pc, "ProfiledThreadData::StreamJSON(context): Streamed tables")); + } + aWriter.End(); +} + +// StreamSamplesDataCallback: (ProgressLogger) -> ProfilerThreadId +// StreamMarkersDataCallback: (ProgressLogger) -> void +// Returns the ProfilerThreadId returned by StreamSamplesDataCallback, which +// should be the thread id of the last sample that was processed (if any; +// otherwise it is left unspecified). This is mostly useful when the caller +// doesn't know where the sample comes from, e.g., when it's a backtrace in a +// marker. +template <typename StreamSamplesDataCallback, + typename StreamMarkersDataCallback> +ProfilerThreadId DoStreamSamplesAndMarkers( + const char* aName, SpliceableJSONWriter& aWriter, + const nsACString& aProcessName, const nsACString& aETLDplus1, + const mozilla::TimeStamp& aProcessStartTime, + const mozilla::TimeStamp& aRegisterTime, + const mozilla::TimeStamp& aUnregisterTime, + mozilla::ProgressLogger aProgressLogger, + StreamSamplesDataCallback&& aStreamSamplesDataCallback, + StreamMarkersDataCallback&& aStreamMarkersDataCallback) { + ProfilerThreadId processedThreadId; + + aWriter.StringProperty("processType", + mozilla::MakeStringSpan(XRE_GetProcessTypeString())); + + aWriter.StringProperty("name", mozilla::MakeStringSpan(aName)); + + // Use given process name (if any), unless we're the parent process. + if (XRE_IsParentProcess()) { + aWriter.StringProperty("processName", "Parent Process"); + } else if (!aProcessName.IsEmpty()) { + aWriter.StringProperty("processName", aProcessName); + } + if (!aETLDplus1.IsEmpty()) { + nsAutoCString originNoSuffix; + mozilla::OriginAttributes attrs; + if (!attrs.PopulateFromOrigin(aETLDplus1, originNoSuffix)) { + aWriter.StringProperty("eTLD+1", aETLDplus1); + } else { + aWriter.StringProperty("eTLD+1", originNoSuffix); + aWriter.BoolProperty("isPrivateBrowsing", attrs.mPrivateBrowsingId > 0); + aWriter.IntProperty("userContextId", attrs.mUserContextId); + } + } + + if (aRegisterTime) { + aWriter.DoubleProperty( + "registerTime", (aRegisterTime - aProcessStartTime).ToMilliseconds()); + } else { + aWriter.NullProperty("registerTime"); + } + + if (aUnregisterTime) { + aWriter.DoubleProperty( + "unregisterTime", + (aUnregisterTime - aProcessStartTime).ToMilliseconds()); + } else { + aWriter.NullProperty("unregisterTime"); + } + + aWriter.StartObjectProperty("samples"); + { + { + JSONSchemaWriter schema(aWriter); + schema.WriteField("stack"); + schema.WriteField("time"); + schema.WriteField("eventDelay"); +#define RUNNING_TIME_FIELD(index, name, unit, jsonProperty) \ + schema.WriteField(#jsonProperty); + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_FIELD) +#undef RUNNING_TIME_FIELD + } + + aWriter.StartArrayProperty("data"); + { + processedThreadId = std::forward<StreamSamplesDataCallback>( + aStreamSamplesDataCallback)(aProgressLogger.CreateSubLoggerFromTo( + 1_pc, "Streaming samples...", 49_pc, "Streamed samples")); + } + aWriter.EndArray(); + } + aWriter.EndObject(); + + aWriter.StartObjectProperty("markers"); + { + { + JSONSchemaWriter schema(aWriter); + schema.WriteField("name"); + schema.WriteField("startTime"); + schema.WriteField("endTime"); + schema.WriteField("phase"); + schema.WriteField("category"); + schema.WriteField("data"); + } + + aWriter.StartArrayProperty("data"); + { + std::forward<StreamMarkersDataCallback>(aStreamMarkersDataCallback)( + aProgressLogger.CreateSubLoggerFromTo(50_pc, "Streaming markers...", + 99_pc, "Streamed markers")); + } + aWriter.EndArray(); + } + aWriter.EndObject(); + + // Tech note: If `ToNumber()` returns a uint64_t, the conversion to int64_t is + // "implementation-defined" before C++20. This is acceptable here, because + // this is a one-way conversion to a unique identifier that's used to visually + // separate data by thread on the front-end. + aWriter.IntProperty( + "pid", static_cast<int64_t>(profiler_current_process_id().ToNumber())); + aWriter.IntProperty("tid", + static_cast<int64_t>(processedThreadId.ToNumber())); + + return processedThreadId; +} + +ProfilerThreadId StreamSamplesAndMarkers( + const char* aName, ProfilerThreadId aThreadId, const ProfileBuffer& aBuffer, + SpliceableJSONWriter& aWriter, const nsACString& aProcessName, + const nsACString& aETLDplus1, const mozilla::TimeStamp& aProcessStartTime, + const mozilla::TimeStamp& aRegisterTime, + const mozilla::TimeStamp& aUnregisterTime, double aSinceTime, + UniqueStacks& aUniqueStacks, mozilla::ProgressLogger aProgressLogger) { + return DoStreamSamplesAndMarkers( + aName, aWriter, aProcessName, aETLDplus1, aProcessStartTime, + aRegisterTime, aUnregisterTime, std::move(aProgressLogger), + [&](mozilla::ProgressLogger aSubProgressLogger) { + ProfilerThreadId processedThreadId = aBuffer.StreamSamplesToJSON( + aWriter, aThreadId, aSinceTime, aUniqueStacks, + std::move(aSubProgressLogger)); + return aThreadId.IsSpecified() ? aThreadId : processedThreadId; + }, + [&](mozilla::ProgressLogger aSubProgressLogger) { + aBuffer.StreamMarkersToJSON(aWriter, aThreadId, aProcessStartTime, + aSinceTime, aUniqueStacks, + std::move(aSubProgressLogger)); + }); +} + +void StreamSamplesAndMarkers(const char* aName, + ThreadStreamingContext& aThreadData, + SpliceableJSONWriter& aWriter, + const nsACString& aProcessName, + const nsACString& aETLDplus1, + const mozilla::TimeStamp& aProcessStartTime, + const mozilla::TimeStamp& aRegisterTime, + const mozilla::TimeStamp& aUnregisterTime, + mozilla::ProgressLogger aProgressLogger) { + (void)DoStreamSamplesAndMarkers( + aName, aWriter, aProcessName, aETLDplus1, aProcessStartTime, + aRegisterTime, aUnregisterTime, std::move(aProgressLogger), + [&](mozilla::ProgressLogger aSubProgressLogger) { + aWriter.TakeAndSplice( + aThreadData.mSamplesDataWriter.TakeChunkedWriteFunc()); + return aThreadData.mProfiledThreadData.Info().ThreadId(); + }, + [&](mozilla::ProgressLogger aSubProgressLogger) { + aWriter.TakeAndSplice( + aThreadData.mMarkersDataWriter.TakeChunkedWriteFunc()); + }); +} + +void ProfiledThreadData::NotifyAboutToLoseJSContext( + JSContext* aContext, const mozilla::TimeStamp& aProcessStartTime, + ProfileBuffer& aBuffer) { + if (!mBufferPositionWhenReceivedJSContext) { + return; + } + + MOZ_RELEASE_ASSERT(aContext); + + if (mJITFrameInfoForPreviousJSContexts && + mJITFrameInfoForPreviousJSContexts->HasExpired( + aBuffer.BufferRangeStart())) { + mJITFrameInfoForPreviousJSContexts = nullptr; + } + + mozilla::UniquePtr<JITFrameInfo> jitFrameInfo = + mJITFrameInfoForPreviousJSContexts + ? std::move(mJITFrameInfoForPreviousJSContexts) + : mozilla::MakeUnique<JITFrameInfo>(); + + aBuffer.AddJITInfoForRange(*mBufferPositionWhenReceivedJSContext, + mThreadInfo.ThreadId(), aContext, *jitFrameInfo, + mozilla::ProgressLogger{}); + + mJITFrameInfoForPreviousJSContexts = std::move(jitFrameInfo); + mBufferPositionWhenReceivedJSContext = mozilla::Nothing(); +} + +ThreadStreamingContext::ThreadStreamingContext( + ProfiledThreadData& aProfiledThreadData, const ProfileBuffer& aBuffer, + JSContext* aCx, mozilla::FailureLatch& aFailureLatch, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger) + : mProfiledThreadData(aProfiledThreadData), + mJSContext(aCx), + mSamplesDataWriter(aFailureLatch), + mMarkersDataWriter(aFailureLatch), + mUniqueStacks(mProfiledThreadData.PrepareUniqueStacks( + aBuffer, aCx, aFailureLatch, aService, + aProgressLogger.CreateSubLoggerFromTo( + 0_pc, "Preparing thread streaming context unique stacks...", + 99_pc, "Prepared thread streaming context Unique stacks"))) { + if (aFailureLatch.Failed()) { + return; + } + mSamplesDataWriter.SetUniqueStrings(mUniqueStacks->UniqueStrings()); + mSamplesDataWriter.StartBareList(); + mMarkersDataWriter.SetUniqueStrings(mUniqueStacks->UniqueStrings()); + mMarkersDataWriter.StartBareList(); +} + +void ThreadStreamingContext::FinalizeWriter() { + mSamplesDataWriter.EndBareList(); + mMarkersDataWriter.EndBareList(); +} + +ProcessStreamingContext::ProcessStreamingContext( + size_t aThreadCount, mozilla::FailureLatch& aFailureLatch, + const mozilla::TimeStamp& aProcessStartTime, double aSinceTime) + : mFailureLatch(aFailureLatch), + mProcessStartTime(aProcessStartTime), + mSinceTime(aSinceTime) { + if (mFailureLatch.Failed()) { + return; + } + if (!mTIDList.initCapacity(aThreadCount)) { + mFailureLatch.SetFailure( + "OOM in ProcessStreamingContext allocating TID list"); + return; + } + if (!mThreadStreamingContextList.initCapacity(aThreadCount)) { + mFailureLatch.SetFailure( + "OOM in ProcessStreamingContext allocating context list"); + mTIDList.clear(); + return; + } +} + +ProcessStreamingContext::~ProcessStreamingContext() { + if (mFailureLatch.Failed()) { + return; + } + MOZ_ASSERT(mTIDList.length() == mThreadStreamingContextList.length()); + MOZ_ASSERT(mTIDList.length() == mTIDList.capacity(), + "Didn't pre-allocate exactly right"); +} + +void ProcessStreamingContext::AddThreadStreamingContext( + ProfiledThreadData& aProfiledThreadData, const ProfileBuffer& aBuffer, + JSContext* aCx, ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger) { + if (mFailureLatch.Failed()) { + return; + } + MOZ_ASSERT(mTIDList.length() == mThreadStreamingContextList.length()); + MOZ_ASSERT(mTIDList.length() < mTIDList.capacity(), + "Didn't pre-allocate enough"); + mTIDList.infallibleAppend(aProfiledThreadData.Info().ThreadId()); + mThreadStreamingContextList.infallibleEmplaceBack( + aProfiledThreadData, aBuffer, aCx, mFailureLatch, aService, + aProgressLogger.CreateSubLoggerFromTo( + 1_pc, "Prepared streaming thread id", 100_pc, + "Added thread streaming context")); +} diff --git a/tools/profiler/core/ProfiledThreadData.h b/tools/profiler/core/ProfiledThreadData.h new file mode 100644 index 0000000000..47ae0c579c --- /dev/null +++ b/tools/profiler/core/ProfiledThreadData.h @@ -0,0 +1,250 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfiledThreadData_h +#define ProfiledThreadData_h + +#include "platform.h" +#include "ProfileBuffer.h" +#include "ProfileBufferEntry.h" + +#include "mozilla/FailureLatch.h" +#include "mozilla/Maybe.h" +#include "mozilla/NotNull.h" +#include "mozilla/ProfileJSONWriter.h" +#include "mozilla/ProfilerThreadRegistrationInfo.h" +#include "mozilla/RefPtr.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Vector.h" +#include "nsStringFwd.h" + +class nsIEventTarget; +class ProfilerCodeAddressService; +struct JSContext; +struct ThreadStreamingContext; + +// This class contains information about a thread that is only relevant while +// the profiler is running, for any threads (both alive and dead) whose thread +// name matches the "thread filter" in the current profiler run. +// ProfiledThreadData objects may be kept alive even after the thread is +// unregistered, as long as there is still data for that thread in the profiler +// buffer. +// +// Accesses to this class are protected by the profiler state lock. +// +// Created as soon as the following are true for the thread: +// - The profiler is running, and +// - the thread matches the profiler's thread filter, and +// - the thread is registered with the profiler. +// So it gets created in response to either (1) the profiler being started (for +// an existing registered thread) or (2) the thread being registered (if the +// profiler is already running). +// +// The thread may be unregistered during the lifetime of ProfiledThreadData. +// If that happens, NotifyUnregistered() is called. +// +// This class is the right place to store buffer positions. Profiler buffer +// positions become invalid if the profiler buffer is destroyed, which happens +// when the profiler is stopped. +class ProfiledThreadData final { + public: + explicit ProfiledThreadData( + const mozilla::profiler::ThreadRegistrationInfo& aThreadInfo); + explicit ProfiledThreadData( + mozilla::profiler::ThreadRegistrationInfo&& aThreadInfo); + ~ProfiledThreadData(); + + void NotifyUnregistered(uint64_t aBufferPosition) { + mLastSample = mozilla::Nothing(); + MOZ_ASSERT(!mBufferPositionWhenReceivedJSContext, + "JSContext should have been cleared before the thread was " + "unregistered"); + mUnregisterTime = mozilla::TimeStamp::Now(); + mBufferPositionWhenUnregistered = mozilla::Some(aBufferPosition); + mPreviousThreadRunningTimes.Clear(); + } + mozilla::Maybe<uint64_t> BufferPositionWhenUnregistered() { + return mBufferPositionWhenUnregistered; + } + + mozilla::Maybe<uint64_t>& LastSample() { return mLastSample; } + + mozilla::NotNull<mozilla::UniquePtr<UniqueStacks>> PrepareUniqueStacks( + const ProfileBuffer& aBuffer, JSContext* aCx, + mozilla::FailureLatch& aFailureLatch, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger); + + void StreamJSON(const ProfileBuffer& aBuffer, JSContext* aCx, + SpliceableJSONWriter& aWriter, const nsACString& aProcessName, + const nsACString& aETLDplus1, + const mozilla::TimeStamp& aProcessStartTime, + double aSinceTime, ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger); + void StreamJSON(ThreadStreamingContext&& aThreadStreamingContext, + SpliceableJSONWriter& aWriter, const nsACString& aProcessName, + const nsACString& aETLDplus1, + const mozilla::TimeStamp& aProcessStartTime, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger); + + const mozilla::profiler::ThreadRegistrationInfo& Info() const { + return mThreadInfo; + } + + void NotifyReceivedJSContext(uint64_t aCurrentBufferPosition) { + mBufferPositionWhenReceivedJSContext = + mozilla::Some(aCurrentBufferPosition); + } + + // Call this method when the JS entries inside the buffer are about to + // become invalid, i.e., just before JS shutdown. + void NotifyAboutToLoseJSContext(JSContext* aCx, + const mozilla::TimeStamp& aProcessStartTime, + ProfileBuffer& aBuffer); + + RunningTimes& PreviousThreadRunningTimesRef() { + return mPreviousThreadRunningTimes; + } + + private: + // Group A: + // The following fields are interesting for the entire lifetime of a + // ProfiledThreadData object. + + // This thread's thread info. Local copy because the one in ThreadRegistration + // may be destroyed while ProfiledThreadData stays alive. + const mozilla::profiler::ThreadRegistrationInfo mThreadInfo; + + // Contains JSON for JIT frames from any JSContexts that were used for this + // thread in the past. + // Null if this thread has never lost a JSContext or if all samples from + // previous JSContexts have been evicted from the profiler buffer. + mozilla::UniquePtr<JITFrameInfo> mJITFrameInfoForPreviousJSContexts; + + // Group B: + // The following fields are only used while this thread is alive and + // registered. They become Nothing() or empty once the thread is unregistered. + + // When sampling, this holds the position in ActivePS::mBuffer of the most + // recent sample for this thread, or Nothing() if there is no sample for this + // thread in the buffer. + mozilla::Maybe<uint64_t> mLastSample; + + // Only non-Nothing() if the thread currently has a JSContext. + mozilla::Maybe<uint64_t> mBufferPositionWhenReceivedJSContext; + + // RunningTimes at the previous sample if any, or empty. + RunningTimes mPreviousThreadRunningTimes; + + // Group C: + // The following fields are only used once this thread has been unregistered. + + mozilla::Maybe<uint64_t> mBufferPositionWhenUnregistered; + mozilla::TimeStamp mUnregisterTime; +}; + +// This class will be used when outputting the profile data for one thread. +struct ThreadStreamingContext { + ProfiledThreadData& mProfiledThreadData; + JSContext* mJSContext; + SpliceableChunkedJSONWriter mSamplesDataWriter; + SpliceableChunkedJSONWriter mMarkersDataWriter; + mozilla::NotNull<mozilla::UniquePtr<UniqueStacks>> mUniqueStacks; + + // These are updated when writing samples, and reused for "same-sample"s. + enum PreviousStackState { eNoStackYet, eStackWasNotEmpty, eStackWasEmpty }; + PreviousStackState mPreviousStackState = eNoStackYet; + uint32_t mPreviousStack = 0; + + ThreadStreamingContext(ProfiledThreadData& aProfiledThreadData, + const ProfileBuffer& aBuffer, JSContext* aCx, + mozilla::FailureLatch& aFailureLatch, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger); + + void FinalizeWriter(); +}; + +// This class will be used when outputting the profile data for all threads. +class ProcessStreamingContext final : public mozilla::FailureLatch { + public: + // Pre-allocate space for `aThreadCount` threads. + ProcessStreamingContext(size_t aThreadCount, + mozilla::FailureLatch& aFailureLatch, + const mozilla::TimeStamp& aProcessStartTime, + double aSinceTime); + + ~ProcessStreamingContext(); + + // Add the streaming context corresponding to each profiled thread. This + // should be called exactly the number of times specified in the constructor. + void AddThreadStreamingContext(ProfiledThreadData& aProfiledThreadData, + const ProfileBuffer& aBuffer, JSContext* aCx, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger); + + // Retrieve the ThreadStreamingContext for a given thread id. + // Returns null if that thread id doesn't correspond to any profiled thread. + ThreadStreamingContext* GetThreadStreamingContext( + const ProfilerThreadId& aThreadId) { + for (size_t i = 0; i < mTIDList.length(); ++i) { + if (mTIDList[i] == aThreadId) { + return &mThreadStreamingContextList[i]; + } + } + return nullptr; + } + + const mozilla::TimeStamp& ProcessStartTime() const { + return mProcessStartTime; + } + + double GetSinceTime() const { return mSinceTime; } + + ThreadStreamingContext* begin() { + return mThreadStreamingContextList.begin(); + }; + ThreadStreamingContext* end() { return mThreadStreamingContextList.end(); }; + + FAILURELATCH_IMPL_PROXY(mFailureLatch) + + private: + // Separate list of thread ids, it's much faster to do a linear search + // here than a vector of bigger items like mThreadStreamingContextList. + mozilla::Vector<ProfilerThreadId> mTIDList; + // Contexts corresponding to the thread id at the same indexes. + mozilla::Vector<ThreadStreamingContext> mThreadStreamingContextList; + + mozilla::FailureLatch& mFailureLatch; + + const mozilla::TimeStamp mProcessStartTime; + + const double mSinceTime; +}; + +// Stream all samples and markers from aBuffer with the given aThreadId (or 0 +// for everything, which is assumed to be a single backtrace sample.) +// Returns the thread id of the output sample(s), or 0 if none was present. +ProfilerThreadId StreamSamplesAndMarkers( + const char* aName, ProfilerThreadId aThreadId, const ProfileBuffer& aBuffer, + SpliceableJSONWriter& aWriter, const nsACString& aProcessName, + const nsACString& aETLDplus1, const mozilla::TimeStamp& aProcessStartTime, + const mozilla::TimeStamp& aRegisterTime, + const mozilla::TimeStamp& aUnregisterTime, double aSinceTime, + UniqueStacks& aUniqueStacks, mozilla::ProgressLogger aProgressLogger); +void StreamSamplesAndMarkers(const char* aName, + ThreadStreamingContext& aThreadData, + SpliceableJSONWriter& aWriter, + const nsACString& aProcessName, + const nsACString& aETLDplus1, + const mozilla::TimeStamp& aProcessStartTime, + const mozilla::TimeStamp& aRegisterTime, + const mozilla::TimeStamp& aUnregisterTime, + mozilla::ProgressLogger aProgressLogger); + +#endif // ProfiledThreadData_h diff --git a/tools/profiler/core/ProfilerBacktrace.cpp b/tools/profiler/core/ProfilerBacktrace.cpp new file mode 100644 index 0000000000..a264d85d64 --- /dev/null +++ b/tools/profiler/core/ProfilerBacktrace.cpp @@ -0,0 +1,101 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfilerBacktrace.h" + +#include "ProfileBuffer.h" +#include "ProfiledThreadData.h" + +#include "mozilla/ProfileJSONWriter.h" + +ProfilerBacktrace::ProfilerBacktrace( + const char* aName, + mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> + aProfileChunkedBufferStorage, + mozilla::UniquePtr<ProfileBuffer> + aProfileBufferStorageOrNull /* = nullptr */) + : mName(aName), + mOptionalProfileChunkedBufferStorage( + std::move(aProfileChunkedBufferStorage)), + mProfileChunkedBuffer(mOptionalProfileChunkedBufferStorage.get()), + mOptionalProfileBufferStorage(std::move(aProfileBufferStorageOrNull)), + mProfileBuffer(mOptionalProfileBufferStorage.get()) { + MOZ_COUNT_CTOR(ProfilerBacktrace); + if (mProfileBuffer) { + MOZ_RELEASE_ASSERT(mProfileChunkedBuffer, + "If we take ownership of a ProfileBuffer, we must also " + "receive ownership of a ProfileChunkedBuffer"); + MOZ_RELEASE_ASSERT( + mProfileChunkedBuffer == &mProfileBuffer->UnderlyingChunkedBuffer(), + "If we take ownership of a ProfileBuffer, we must also receive " + "ownership of its ProfileChunkedBuffer"); + } + MOZ_ASSERT( + !mProfileChunkedBuffer || !mProfileChunkedBuffer->IsThreadSafe(), + "ProfilerBacktrace only takes a non-thread-safe ProfileChunkedBuffer"); +} + +ProfilerBacktrace::ProfilerBacktrace( + const char* aName, + mozilla::ProfileChunkedBuffer* aExternalProfileChunkedBuffer, + ProfileBuffer* aExternalProfileBuffer) + : mName(aName), + mProfileChunkedBuffer(aExternalProfileChunkedBuffer), + mProfileBuffer(aExternalProfileBuffer) { + MOZ_COUNT_CTOR(ProfilerBacktrace); + if (!mProfileChunkedBuffer) { + if (mProfileBuffer) { + // We don't have a ProfileChunkedBuffer but we have a ProfileBuffer, use + // the latter's ProfileChunkedBuffer. + mProfileChunkedBuffer = &mProfileBuffer->UnderlyingChunkedBuffer(); + MOZ_ASSERT(!mProfileChunkedBuffer->IsThreadSafe(), + "ProfilerBacktrace only takes a non-thread-safe " + "ProfileChunkedBuffer"); + } + } else { + if (mProfileBuffer) { + MOZ_RELEASE_ASSERT( + mProfileChunkedBuffer == &mProfileBuffer->UnderlyingChunkedBuffer(), + "If we reference both ProfileChunkedBuffer and ProfileBuffer, they " + "must already be connected"); + } + MOZ_ASSERT(!mProfileChunkedBuffer->IsThreadSafe(), + "ProfilerBacktrace only takes a non-thread-safe " + "ProfileChunkedBuffer"); + } +} + +ProfilerBacktrace::~ProfilerBacktrace() { MOZ_COUNT_DTOR(ProfilerBacktrace); } + +ProfilerThreadId ProfilerBacktrace::StreamJSON( + SpliceableJSONWriter& aWriter, const mozilla::TimeStamp& aProcessStartTime, + UniqueStacks& aUniqueStacks) { + ProfilerThreadId processedThreadId; + + // Unlike ProfiledThreadData::StreamJSON, we don't need to call + // ProfileBuffer::AddJITInfoForRange because ProfileBuffer does not contain + // any JitReturnAddr entries. For synchronous samples, JIT frames get expanded + // at sample time. + if (mProfileBuffer) { + processedThreadId = StreamSamplesAndMarkers( + mName.c_str(), ProfilerThreadId{}, *mProfileBuffer, aWriter, ""_ns, + ""_ns, aProcessStartTime, + /* aRegisterTime */ mozilla::TimeStamp(), + /* aUnregisterTime */ mozilla::TimeStamp(), + /* aSinceTime */ 0, aUniqueStacks, mozilla::ProgressLogger{}); + } else if (mProfileChunkedBuffer) { + ProfileBuffer profileBuffer(*mProfileChunkedBuffer); + processedThreadId = StreamSamplesAndMarkers( + mName.c_str(), ProfilerThreadId{}, profileBuffer, aWriter, ""_ns, ""_ns, + aProcessStartTime, + /* aRegisterTime */ mozilla::TimeStamp(), + /* aUnregisterTime */ mozilla::TimeStamp(), + /* aSinceTime */ 0, aUniqueStacks, mozilla::ProgressLogger{}); + } + // If there are no buffers, the backtrace is empty and nothing is streamed. + + return processedThreadId; +} diff --git a/tools/profiler/core/ProfilerBacktrace.h b/tools/profiler/core/ProfilerBacktrace.h new file mode 100644 index 0000000000..55811f4422 --- /dev/null +++ b/tools/profiler/core/ProfilerBacktrace.h @@ -0,0 +1,184 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef __PROFILER_BACKTRACE_H +#define __PROFILER_BACKTRACE_H + +#include "ProfileBuffer.h" + +#include "mozilla/ProfileBufferEntrySerialization.h" +#include "mozilla/UniquePtrExtensions.h" + +#include <string> + +class ProfileBuffer; +class ProfilerCodeAddressService; +class ThreadInfo; +class UniqueStacks; + +namespace mozilla { +class ProfileChunkedBuffer; +class TimeStamp; +namespace baseprofiler { +class SpliceableJSONWriter; +} // namespace baseprofiler +} // namespace mozilla + +// ProfilerBacktrace encapsulates a synchronous sample. +// It can work with a ProfileBuffer and/or a ProfileChunkedBuffer (if both, they +// must already be linked together). The ProfileChunkedBuffer contains all the +// data; the ProfileBuffer is not strictly needed, only provide it if it is +// already available at the call site. +// And these buffers can either be: +// - owned here, so that the ProfilerBacktrace object can be kept for later +// use), OR +// - referenced through pointers (in cases where the backtrace is immediately +// streamed out, so we only need temporary references to external buffers); +// these pointers may be null for empty backtraces. +class ProfilerBacktrace { + public: + // Take ownership of external buffers and use them to keep, and to stream a + // backtrace. If a ProfileBuffer is given, its underlying chunked buffer must + // be provided as well. + explicit ProfilerBacktrace( + const char* aName, + mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> + aProfileChunkedBufferStorage, + mozilla::UniquePtr<ProfileBuffer> aProfileBufferStorageOrNull = nullptr); + + // Take pointers to external buffers and use them to stream a backtrace. + // If null, the backtrace is effectively empty. + // If both are provided, they must already be connected. + explicit ProfilerBacktrace( + const char* aName, + mozilla::ProfileChunkedBuffer* aExternalProfileChunkedBufferOrNull = + nullptr, + ProfileBuffer* aExternalProfileBufferOrNull = nullptr); + + ~ProfilerBacktrace(); + + [[nodiscard]] bool IsEmpty() const { + return !mProfileChunkedBuffer || + mozilla::ProfileBufferEntryWriter::Serializer< + mozilla::ProfileChunkedBuffer>::Bytes(*mProfileChunkedBuffer) <= + mozilla::ULEB128Size(0u); + } + + // ProfilerBacktraces' stacks are deduplicated in the context of the + // profile that contains the backtrace as a marker payload. + // + // That is, markers that contain backtraces should not need their own stack, + // frame, and string tables. They should instead reuse their parent + // profile's tables. + ProfilerThreadId StreamJSON( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter, + const mozilla::TimeStamp& aProcessStartTime, UniqueStacks& aUniqueStacks); + + private: + // Used to serialize a ProfilerBacktrace. + friend struct mozilla::ProfileBufferEntryWriter::Serializer< + ProfilerBacktrace>; + friend struct mozilla::ProfileBufferEntryReader::Deserializer< + ProfilerBacktrace>; + + std::string mName; + + // `ProfileChunkedBuffer` in which `mProfileBuffer` stores its data; must be + // located before `mProfileBuffer` so that it's destroyed after. + mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> + mOptionalProfileChunkedBufferStorage; + // If null, there is no need to check mProfileBuffer's (if present) underlying + // buffer because this is done when constructed. + mozilla::ProfileChunkedBuffer* mProfileChunkedBuffer; + + mozilla::UniquePtr<ProfileBuffer> mOptionalProfileBufferStorage; + ProfileBuffer* mProfileBuffer; +}; + +namespace mozilla { + +// Format: [ UniquePtr<BlockRingsBuffer> | name ] +// Initial len==0 marks a nullptr or empty backtrace. +template <> +struct mozilla::ProfileBufferEntryWriter::Serializer<ProfilerBacktrace> { + static Length Bytes(const ProfilerBacktrace& aBacktrace) { + if (!aBacktrace.mProfileChunkedBuffer) { + // No buffer. + return ULEB128Size(0u); + } + auto bufferBytes = SumBytes(*aBacktrace.mProfileChunkedBuffer); + if (bufferBytes <= ULEB128Size(0u)) { + // Empty buffer. + return ULEB128Size(0u); + } + return bufferBytes + SumBytes(aBacktrace.mName); + } + + static void Write(mozilla::ProfileBufferEntryWriter& aEW, + const ProfilerBacktrace& aBacktrace) { + if (!aBacktrace.mProfileChunkedBuffer || + SumBytes(*aBacktrace.mProfileChunkedBuffer) <= ULEB128Size(0u)) { + // No buffer, or empty buffer. + aEW.WriteULEB128(0u); + return; + } + aEW.WriteObject(*aBacktrace.mProfileChunkedBuffer); + aEW.WriteObject(aBacktrace.mName); + } +}; + +template <typename Destructor> +struct mozilla::ProfileBufferEntryWriter::Serializer< + mozilla::UniquePtr<ProfilerBacktrace, Destructor>> { + static Length Bytes( + const mozilla::UniquePtr<ProfilerBacktrace, Destructor>& aBacktrace) { + if (!aBacktrace) { + // Null backtrace pointer (treated like an empty backtrace). + return ULEB128Size(0u); + } + return SumBytes(*aBacktrace); + } + + static void Write( + mozilla::ProfileBufferEntryWriter& aEW, + const mozilla::UniquePtr<ProfilerBacktrace, Destructor>& aBacktrace) { + if (!aBacktrace) { + // Null backtrace pointer (treated like an empty backtrace). + aEW.WriteULEB128(0u); + return; + } + aEW.WriteObject(*aBacktrace); + } +}; + +template <typename Destructor> +struct mozilla::ProfileBufferEntryReader::Deserializer< + mozilla::UniquePtr<ProfilerBacktrace, Destructor>> { + static void ReadInto( + mozilla::ProfileBufferEntryReader& aER, + mozilla::UniquePtr<ProfilerBacktrace, Destructor>& aBacktrace) { + aBacktrace = Read(aER); + } + + static mozilla::UniquePtr<ProfilerBacktrace, Destructor> Read( + mozilla::ProfileBufferEntryReader& aER) { + auto profileChunkedBuffer = + aER.ReadObject<UniquePtr<ProfileChunkedBuffer>>(); + if (!profileChunkedBuffer) { + return nullptr; + } + MOZ_ASSERT( + !profileChunkedBuffer->IsThreadSafe(), + "ProfilerBacktrace only stores non-thread-safe ProfileChunkedBuffers"); + std::string name = aER.ReadObject<std::string>(); + return UniquePtr<ProfilerBacktrace, Destructor>{ + new ProfilerBacktrace(name.c_str(), std::move(profileChunkedBuffer))}; + } +}; + +} // namespace mozilla + +#endif // __PROFILER_BACKTRACE_H diff --git a/tools/profiler/core/ProfilerBindings.cpp b/tools/profiler/core/ProfilerBindings.cpp new file mode 100644 index 0000000000..df180733a1 --- /dev/null +++ b/tools/profiler/core/ProfilerBindings.cpp @@ -0,0 +1,395 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* FFI functions for Profiler Rust API to call into profiler */ + +#include "ProfilerBindings.h" + +#include "GeckoProfiler.h" + +#include <set> +#include <type_traits> + +void gecko_profiler_register_thread(const char* aName) { + PROFILER_REGISTER_THREAD(aName); +} + +void gecko_profiler_unregister_thread() { PROFILER_UNREGISTER_THREAD(); } + +void gecko_profiler_construct_label(mozilla::AutoProfilerLabel* aAutoLabel, + JS::ProfilingCategoryPair aCategoryPair) { +#ifdef MOZ_GECKO_PROFILER + new (aAutoLabel) mozilla::AutoProfilerLabel( + "", nullptr, aCategoryPair, + uint32_t( + js::ProfilingStackFrame::Flags::LABEL_DETERMINED_BY_CATEGORY_PAIR)); +#endif +} + +void gecko_profiler_destruct_label(mozilla::AutoProfilerLabel* aAutoLabel) { +#ifdef MOZ_GECKO_PROFILER + aAutoLabel->~AutoProfilerLabel(); +#endif +} + +void gecko_profiler_construct_timestamp_now(mozilla::TimeStamp* aTimeStamp) { + new (aTimeStamp) mozilla::TimeStamp(mozilla::TimeStamp::Now()); +} + +void gecko_profiler_clone_timestamp(const mozilla::TimeStamp* aSrcTimeStamp, + mozilla::TimeStamp* aDestTimeStamp) { + new (aDestTimeStamp) mozilla::TimeStamp(*aSrcTimeStamp); +} + +void gecko_profiler_destruct_timestamp(mozilla::TimeStamp* aTimeStamp) { + aTimeStamp->~TimeStamp(); +} + +void gecko_profiler_add_timestamp(const mozilla::TimeStamp* aTimeStamp, + mozilla::TimeStamp* aDestTimeStamp, + double aMicroseconds) { + new (aDestTimeStamp) mozilla::TimeStamp( + *aTimeStamp + mozilla::TimeDuration::FromMicroseconds(aMicroseconds)); +} + +void gecko_profiler_subtract_timestamp(const mozilla::TimeStamp* aTimeStamp, + mozilla::TimeStamp* aDestTimeStamp, + double aMicroseconds) { + new (aDestTimeStamp) mozilla::TimeStamp( + *aTimeStamp - mozilla::TimeDuration::FromMicroseconds(aMicroseconds)); +} + +void gecko_profiler_construct_marker_timing_instant_at( + mozilla::MarkerTiming* aMarkerTiming, const mozilla::TimeStamp* aTime) { +#ifdef MOZ_GECKO_PROFILER + static_assert(std::is_trivially_copyable_v<mozilla::MarkerTiming>); + mozilla::MarkerTiming::UnsafeConstruct(aMarkerTiming, *aTime, + mozilla::TimeStamp{}, + mozilla::MarkerTiming::Phase::Instant); +#endif +} + +void gecko_profiler_construct_marker_timing_instant_now( + mozilla::MarkerTiming* aMarkerTiming) { +#ifdef MOZ_GECKO_PROFILER + static_assert(std::is_trivially_copyable_v<mozilla::MarkerTiming>); + mozilla::MarkerTiming::UnsafeConstruct( + aMarkerTiming, mozilla::TimeStamp::Now(), mozilla::TimeStamp{}, + mozilla::MarkerTiming::Phase::Instant); +#endif +} + +void gecko_profiler_construct_marker_timing_interval( + mozilla::MarkerTiming* aMarkerTiming, const mozilla::TimeStamp* aStartTime, + const mozilla::TimeStamp* aEndTime) { +#ifdef MOZ_GECKO_PROFILER + static_assert(std::is_trivially_copyable_v<mozilla::MarkerTiming>); + mozilla::MarkerTiming::UnsafeConstruct( + aMarkerTiming, *aStartTime, *aEndTime, + mozilla::MarkerTiming::Phase::Interval); +#endif +} + +void gecko_profiler_construct_marker_timing_interval_until_now_from( + mozilla::MarkerTiming* aMarkerTiming, + const mozilla::TimeStamp* aStartTime) { +#ifdef MOZ_GECKO_PROFILER + static_assert(std::is_trivially_copyable_v<mozilla::MarkerTiming>); + mozilla::MarkerTiming::UnsafeConstruct( + aMarkerTiming, *aStartTime, mozilla::TimeStamp::Now(), + mozilla::MarkerTiming::Phase::Interval); +#endif +} + +void gecko_profiler_construct_marker_timing_interval_start( + mozilla::MarkerTiming* aMarkerTiming, const mozilla::TimeStamp* aTime) { +#ifdef MOZ_GECKO_PROFILER + static_assert(std::is_trivially_copyable_v<mozilla::MarkerTiming>); + mozilla::MarkerTiming::UnsafeConstruct( + aMarkerTiming, *aTime, mozilla::TimeStamp{}, + mozilla::MarkerTiming::Phase::IntervalStart); +#endif +} + +void gecko_profiler_construct_marker_timing_interval_end( + mozilla::MarkerTiming* aMarkerTiming, const mozilla::TimeStamp* aTime) { +#ifdef MOZ_GECKO_PROFILER + static_assert(std::is_trivially_copyable_v<mozilla::MarkerTiming>); + mozilla::MarkerTiming::UnsafeConstruct( + aMarkerTiming, mozilla::TimeStamp{}, *aTime, + mozilla::MarkerTiming::Phase::IntervalEnd); +#endif +} + +void gecko_profiler_destruct_marker_timing( + mozilla::MarkerTiming* aMarkerTiming) { +#ifdef MOZ_GECKO_PROFILER + aMarkerTiming->~MarkerTiming(); +#endif +} + +void gecko_profiler_construct_marker_schema( + mozilla::MarkerSchema* aMarkerSchema, + const mozilla::MarkerSchema::Location* aLocations, size_t aLength) { +#ifdef MOZ_GECKO_PROFILER + new (aMarkerSchema) mozilla::MarkerSchema(aLocations, aLength); +#endif +} + +void gecko_profiler_construct_marker_schema_with_special_front_end_location( + mozilla::MarkerSchema* aMarkerSchema) { +#ifdef MOZ_GECKO_PROFILER + new (aMarkerSchema) + mozilla::MarkerSchema(mozilla::MarkerSchema::SpecialFrontendLocation{}); +#endif +} + +void gecko_profiler_destruct_marker_schema( + mozilla::MarkerSchema* aMarkerSchema) { +#ifdef MOZ_GECKO_PROFILER + aMarkerSchema->~MarkerSchema(); +#endif +} + +void gecko_profiler_marker_schema_set_chart_label( + mozilla::MarkerSchema* aSchema, const char* aLabel, size_t aLabelLength) { +#ifdef MOZ_GECKO_PROFILER + aSchema->SetChartLabel(std::string(aLabel, aLabelLength)); +#endif +} + +void gecko_profiler_marker_schema_set_tooltip_label( + mozilla::MarkerSchema* aSchema, const char* aLabel, size_t aLabelLength) { +#ifdef MOZ_GECKO_PROFILER + aSchema->SetTooltipLabel(std::string(aLabel, aLabelLength)); +#endif +} + +void gecko_profiler_marker_schema_set_table_label( + mozilla::MarkerSchema* aSchema, const char* aLabel, size_t aLabelLength) { +#ifdef MOZ_GECKO_PROFILER + aSchema->SetTableLabel(std::string(aLabel, aLabelLength)); +#endif +} + +void gecko_profiler_marker_schema_set_all_labels(mozilla::MarkerSchema* aSchema, + const char* aLabel, + size_t aLabelLength) { +#ifdef MOZ_GECKO_PROFILER + aSchema->SetAllLabels(std::string(aLabel, aLabelLength)); +#endif +} + +void gecko_profiler_marker_schema_add_key_format( + mozilla::MarkerSchema* aSchema, const char* aKey, size_t aKeyLength, + mozilla::MarkerSchema::Format aFormat) { +#ifdef MOZ_GECKO_PROFILER + aSchema->AddKeyFormat(std::string(aKey, aKeyLength), aFormat); +#endif +} + +void gecko_profiler_marker_schema_add_key_label_format( + mozilla::MarkerSchema* aSchema, const char* aKey, size_t aKeyLength, + const char* aLabel, size_t aLabelLength, + mozilla::MarkerSchema::Format aFormat) { +#ifdef MOZ_GECKO_PROFILER + aSchema->AddKeyLabelFormat(std::string(aKey, aKeyLength), + std::string(aLabel, aLabelLength), aFormat); +#endif +} + +void gecko_profiler_marker_schema_add_key_format_searchable( + mozilla::MarkerSchema* aSchema, const char* aKey, size_t aKeyLength, + mozilla::MarkerSchema::Format aFormat, + mozilla::MarkerSchema::Searchable aSearchable) { +#ifdef MOZ_GECKO_PROFILER + aSchema->AddKeyFormatSearchable(std::string(aKey, aKeyLength), aFormat, + aSearchable); +#endif +} + +void gecko_profiler_marker_schema_add_key_label_format_searchable( + mozilla::MarkerSchema* aSchema, const char* aKey, size_t aKeyLength, + const char* aLabel, size_t aLabelLength, + mozilla::MarkerSchema::Format aFormat, + mozilla::MarkerSchema::Searchable aSearchable) { +#ifdef MOZ_GECKO_PROFILER + aSchema->AddKeyLabelFormatSearchable(std::string(aKey, aKeyLength), + std::string(aLabel, aLabelLength), + aFormat, aSearchable); +#endif +} + +void gecko_profiler_marker_schema_add_static_label_value( + mozilla::MarkerSchema* aSchema, const char* aLabel, size_t aLabelLength, + const char* aValue, size_t aValueLength) { +#ifdef MOZ_GECKO_PROFILER + aSchema->AddStaticLabelValue(std::string(aLabel, aLabelLength), + std::string(aValue, aValueLength)); +#endif +} + +void gecko_profiler_marker_schema_stream( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, mozilla::MarkerSchema* aMarkerSchema, + void* aStreamedNamesSet) { +#ifdef MOZ_GECKO_PROFILER + auto* streamedNames = static_cast<std::set<std::string>*>(aStreamedNamesSet); + // std::set.insert(T&&) returns a pair, its `second` is true if the element + // was actually inserted (i.e., it was not there yet.). + const bool didInsert = + streamedNames->insert(std::string(aName, aNameLength)).second; + if (didInsert) { + std::move(*aMarkerSchema) + .Stream(*aWriter, mozilla::Span(aName, aNameLength)); + } +#endif +} + +void gecko_profiler_json_writer_int_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, int64_t aValue) { +#ifdef MOZ_GECKO_PROFILER + aWriter->IntProperty(mozilla::Span(aName, aNameLength), aValue); +#endif +} + +void gecko_profiler_json_writer_float_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, double aValue) { +#ifdef MOZ_GECKO_PROFILER + aWriter->DoubleProperty(mozilla::Span(aName, aNameLength), aValue); +#endif +} + +void gecko_profiler_json_writer_bool_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, bool aValue) { +#ifdef MOZ_GECKO_PROFILER + aWriter->BoolProperty(mozilla::Span(aName, aNameLength), aValue); +#endif +} +void gecko_profiler_json_writer_string_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, const char* aValue, size_t aValueLength) { +#ifdef MOZ_GECKO_PROFILER + aWriter->StringProperty(mozilla::Span(aName, aNameLength), + mozilla::Span(aValue, aValueLength)); +#endif +} + +void gecko_profiler_json_writer_unique_string_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, const char* aValue, size_t aValueLength) { +#ifdef MOZ_GECKO_PROFILER + aWriter->UniqueStringProperty(mozilla::Span(aName, aNameLength), + mozilla::Span(aValue, aValueLength)); +#endif +} + +void gecko_profiler_json_writer_null_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength) { +#ifdef MOZ_GECKO_PROFILER + aWriter->NullProperty(mozilla::Span(aName, aNameLength)); +#endif +} + +void gecko_profiler_add_marker_untyped( + const char* aName, size_t aNameLength, + mozilla::baseprofiler::ProfilingCategoryPair aCategoryPair, + mozilla::MarkerTiming* aMarkerTiming, + mozilla::StackCaptureOptions aStackCaptureOptions) { +#ifdef MOZ_GECKO_PROFILER + profiler_add_marker( + mozilla::ProfilerString8View(aName, aNameLength), + mozilla::MarkerCategory{aCategoryPair}, + mozilla::MarkerOptions( + std::move(*aMarkerTiming), + mozilla::MarkerStack::WithCaptureOptions(aStackCaptureOptions))); +#endif +} + +void gecko_profiler_add_marker_text( + const char* aName, size_t aNameLength, + mozilla::baseprofiler::ProfilingCategoryPair aCategoryPair, + mozilla::MarkerTiming* aMarkerTiming, + mozilla::StackCaptureOptions aStackCaptureOptions, const char* aText, + size_t aTextLength) { +#ifdef MOZ_GECKO_PROFILER + profiler_add_marker( + mozilla::ProfilerString8View(aName, aNameLength), + mozilla::MarkerCategory{aCategoryPair}, + mozilla::MarkerOptions( + std::move(*aMarkerTiming), + mozilla::MarkerStack::WithCaptureOptions(aStackCaptureOptions)), + geckoprofiler::markers::TextMarker{}, + mozilla::ProfilerString8View(aText, aTextLength)); +#endif +} + +void gecko_profiler_add_marker( + const char* aName, size_t aNameLength, + mozilla::baseprofiler::ProfilingCategoryPair aCategoryPair, + mozilla::MarkerTiming* aMarkerTiming, + mozilla::StackCaptureOptions aStackCaptureOptions, uint8_t aMarkerTag, + const uint8_t* aPayload, size_t aPayloadSize) { +#ifdef MOZ_GECKO_PROFILER + // Copy the marker timing and create the marker option. + mozilla::MarkerOptions markerOptions( + std::move(*aMarkerTiming), + mozilla::MarkerStack::WithCaptureOptions(aStackCaptureOptions)); + + // Currently it's not possible to add a threadId option, but we will + // have it soon. + if (markerOptions.ThreadId().IsUnspecified()) { + // If yet unspecified, set thread to this thread where the marker is added. + markerOptions.Set(mozilla::MarkerThreadId::CurrentThread()); + } + + auto& buffer = profiler_get_core_buffer(); + mozilla::Span payload(aPayload, aPayloadSize); + + mozilla::StackCaptureOptions captureOptions = + markerOptions.Stack().CaptureOptions(); + if (captureOptions != mozilla::StackCaptureOptions::NoStack && + // Do not capture a stack if the NoMarkerStacks feature is set. + profiler_active_without_feature(ProfilerFeature::NoMarkerStacks)) { + // A capture was requested, let's attempt to do it here&now. This avoids a + // lot of allocations that would be necessary if capturing a backtrace + // separately. + // TODO use a local on-stack byte buffer to remove last allocation. + // TODO reduce internal profiler stack levels, see bug 1659872. + mozilla::ProfileBufferChunkManagerSingle chunkManager( + mozilla::ProfileBufferChunkManager::scExpectedMaximumStackSize); + mozilla::ProfileChunkedBuffer chunkedBuffer( + mozilla::ProfileChunkedBuffer::ThreadSafety::WithoutMutex, + chunkManager); + markerOptions.StackRef().UseRequestedBacktrace( + profiler_capture_backtrace_into(chunkedBuffer, captureOptions) + ? &chunkedBuffer + : nullptr); + + // This call must be made from here, while chunkedBuffer is in scope. + buffer.PutObjects( + mozilla::ProfileBufferEntryKind::Marker, markerOptions, + mozilla::ProfilerString8View(aName, aNameLength), + mozilla::MarkerCategory{aCategoryPair}, + mozilla::base_profiler_markers_detail::Streaming::DeserializerTag( + aMarkerTag), + mozilla::MarkerPayloadType::Rust, payload); + return; + } + + buffer.PutObjects( + mozilla::ProfileBufferEntryKind::Marker, markerOptions, + mozilla::ProfilerString8View(aName, aNameLength), + mozilla::MarkerCategory{aCategoryPair}, + mozilla::base_profiler_markers_detail::Streaming::DeserializerTag( + aMarkerTag), + mozilla::MarkerPayloadType::Rust, payload); +#endif +} diff --git a/tools/profiler/core/ProfilerCPUFreq-linux-android.cpp b/tools/profiler/core/ProfilerCPUFreq-linux-android.cpp new file mode 100644 index 0000000000..c7efbbd75f --- /dev/null +++ b/tools/profiler/core/ProfilerCPUFreq-linux-android.cpp @@ -0,0 +1,73 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfilerCPUFreq.h" +#include "nsThreadUtils.h" +#include <fcntl.h> +#include <unistd.h> + +ProfilerCPUFreq::ProfilerCPUFreq() { + if (!mCPUCounters.resize(mozilla::GetNumberOfProcessors())) { + NS_WARNING("failing to resize the mCPUCounters vector"); + return; + } + + for (unsigned cpuId = 0; cpuId < mCPUCounters.length(); ++cpuId) { + const size_t buf_sz = 64; + char buf[buf_sz]; + int rv = sprintf( + buf, "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq", cpuId); + if (NS_WARN_IF(rv < 0)) { + continue; + } + + int fd = open(buf, O_RDONLY); + if (NS_WARN_IF(!fd)) { + continue; + } + + mCPUCounters[cpuId].fd = fd; + } +} + +ProfilerCPUFreq::~ProfilerCPUFreq() { + for (CPUCounterInfo& CPUCounter : mCPUCounters) { + int fd = CPUCounter.fd; + if (NS_WARN_IF(!fd)) { + continue; + } + close(fd); + CPUCounter.fd = 0; + } +} + +uint32_t ProfilerCPUFreq::GetCPUSpeedMHz(unsigned cpuId) { + MOZ_ASSERT(cpuId < mCPUCounters.length()); + int fd = mCPUCounters[cpuId].fd; + if (NS_WARN_IF(!fd)) { + return 0; + } + + long rv = lseek(fd, 0, SEEK_SET); + if (NS_WARN_IF(rv < 0)) { + return 0; + } + + const size_t buf_sz = 64; + char buf[buf_sz]; + rv = read(fd, buf, buf_sz); + if (NS_WARN_IF(rv < 0)) { + return 0; + } + + int cpufreq = 0; + rv = sscanf(buf, "%u", &cpufreq); + if (NS_WARN_IF(rv != 1)) { + return 0; + } + + // Convert kHz to MHz, rounding to the nearst 10Mhz, to ignore tiny + // variations that are likely due to rounding errors. + return uint32_t(cpufreq / 10000) * 10; +} diff --git a/tools/profiler/core/ProfilerCPUFreq-win.cpp b/tools/profiler/core/ProfilerCPUFreq-win.cpp new file mode 100644 index 0000000000..e66f2757ea --- /dev/null +++ b/tools/profiler/core/ProfilerCPUFreq-win.cpp @@ -0,0 +1,244 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfilerCPUFreq.h" +#include "nsString.h" +#include "nsThreadUtils.h" +#ifdef DEBUG +# include "nsPrintfCString.h" +#endif + +#include <stdio.h> +#include <strsafe.h> +#include <winperf.h> + +#pragma comment(lib, "advapi32.lib") + +using namespace mozilla; + +ProfilerCPUFreq::ProfilerCPUFreq() { + // Query the size of the text data so you can allocate the buffer. + DWORD dwBufferSize = 0; + LONG status = RegQueryValueEx(HKEY_PERFORMANCE_DATA, L"Counter 9", NULL, NULL, + NULL, &dwBufferSize); + if (ERROR_SUCCESS != status) { + NS_WARNING(nsPrintfCString("RegQueryValueEx failed getting required buffer " + "size. Error is 0x%lx.\n", + status) + .get()); + return; + } + + // Allocate the text buffer and query the text. + LPWSTR pBuffer = (LPWSTR)malloc(dwBufferSize); + if (!pBuffer) { + NS_WARNING("failed to allocate buffer"); + return; + } + status = RegQueryValueEx(HKEY_PERFORMANCE_DATA, L"Counter 9", NULL, NULL, + (LPBYTE)pBuffer, &dwBufferSize); + if (ERROR_SUCCESS != status) { + NS_WARNING( + nsPrintfCString("RegQueryValueEx failed with 0x%lx.\n", status).get()); + free(pBuffer); + return; + } + + LPWSTR pwszCounterText = pBuffer; // Used to cycle through the Counter text + // Ignore first pair. + pwszCounterText += (wcslen(pwszCounterText) + 1); + pwszCounterText += (wcslen(pwszCounterText) + 1); + + for (; *pwszCounterText; pwszCounterText += (wcslen(pwszCounterText) + 1)) { + // Keep a pointer to the counter index, to read the index later if the name + // is the one we are looking for. + LPWSTR counterIndex = pwszCounterText; + pwszCounterText += (wcslen(pwszCounterText) + 1); // Skip past index value + + if (!wcscmp(L"Processor Information", pwszCounterText)) { + mBlockIndex = _wcsdup(counterIndex); + } else if (!wcscmp(L"% Processor Performance", pwszCounterText)) { + mCounterNameIndex = _wtoi(counterIndex); + if (mBlockIndex) { + // We have found all the indexes we were looking for. + break; + } + } + } + free(pBuffer); + + if (!mBlockIndex) { + NS_WARNING("index of the performance counter block not found"); + return; + } + + mBuffer = (LPBYTE)malloc(mBufferSize); + if (!mBuffer) { + NS_WARNING("failed to allocate initial buffer"); + return; + } + dwBufferSize = mBufferSize; + + // Typically RegQueryValueEx will set the size variable to the required size. + // But this does not work when querying object index values, and the buffer + // size has to be increased in a loop until RegQueryValueEx no longer returns + // ERROR_MORE_DATA. + while (ERROR_MORE_DATA == + (status = RegQueryValueEx(HKEY_PERFORMANCE_DATA, mBlockIndex, NULL, + NULL, mBuffer, &dwBufferSize))) { + mBufferSize *= 2; + auto* oldBuffer = mBuffer; + mBuffer = (LPBYTE)realloc(mBuffer, mBufferSize); + if (!mBuffer) { + NS_WARNING("failed to reallocate buffer"); + free(oldBuffer); + return; + } + dwBufferSize = mBufferSize; + } + + if (ERROR_SUCCESS != status) { + NS_WARNING(nsPrintfCString("RegQueryValueEx failed getting required buffer " + "size. Error is 0x%lx.\n", + status) + .get()); + free(mBuffer); + mBuffer = nullptr; + return; + } + + PERF_DATA_BLOCK* dataBlock = (PERF_DATA_BLOCK*)mBuffer; + LPBYTE pObject = mBuffer + dataBlock->HeaderLength; + PERF_OBJECT_TYPE* object = (PERF_OBJECT_TYPE*)pObject; + PERF_COUNTER_DEFINITION* counter = nullptr; + { + PERF_COUNTER_DEFINITION* pCounter = + (PERF_COUNTER_DEFINITION*)(pObject + object->HeaderLength); + for (DWORD i = 0; i < object->NumCounters; i++) { + if (mCounterNameIndex == pCounter->CounterNameTitleIndex) { + counter = pCounter; + break; + } + pCounter++; + } + } + if (!counter || !mCPUCounters.resize(GetNumberOfProcessors())) { + NS_WARNING("failing to find counter or resize the mCPUCounters vector"); + free(mBuffer); + mBuffer = nullptr; + return; + } + + MOZ_ASSERT(counter->CounterType == PERF_AVERAGE_BULK); + PERF_COUNTER_DEFINITION* baseCounter = counter + 1; + MOZ_ASSERT((baseCounter->CounterType & PERF_COUNTER_BASE) == + PERF_COUNTER_BASE); + + PERF_INSTANCE_DEFINITION* instanceDef = + (PERF_INSTANCE_DEFINITION*)(pObject + object->DefinitionLength); + for (LONG i = 0; i < object->NumInstances; i++) { + PERF_COUNTER_BLOCK* counterBlock = + (PERF_COUNTER_BLOCK*)((LPBYTE)instanceDef + instanceDef->ByteLength); + + LPWSTR name = (LPWSTR)(((LPBYTE)instanceDef) + instanceDef->NameOffset); + unsigned int cpuId, coreId; + if (swscanf(name, L"%u,%u", &cpuId, &coreId) == 2 && cpuId == 0 && + coreId < mCPUCounters.length()) { + auto& CPUCounter = mCPUCounters[coreId]; + CPUCounter.data = *(UNALIGNED ULONGLONG*)((LPBYTE)counterBlock + + counter->CounterOffset); + CPUCounter.base = + *(DWORD*)((LPBYTE)counterBlock + baseCounter->CounterOffset); + + // Now get the nominal core frequency. + HKEY key; + nsAutoString keyName( + L"HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\"); + keyName.AppendInt(coreId); + + if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, keyName.get(), 0, KEY_QUERY_VALUE, + &key) == ERROR_SUCCESS) { + DWORD data, len; + len = sizeof(data); + + if (RegQueryValueEx(key, L"~Mhz", 0, 0, reinterpret_cast<LPBYTE>(&data), + &len) == ERROR_SUCCESS) { + CPUCounter.nominalFrequency = data; + } + } + } + instanceDef = (PERF_INSTANCE_DEFINITION*)((LPBYTE)counterBlock + + counterBlock->ByteLength); + } +} + +ProfilerCPUFreq::~ProfilerCPUFreq() { + RegCloseKey(HKEY_PERFORMANCE_DATA); + free(mBlockIndex); + mBlockIndex = nullptr; + free(mBuffer); + mBuffer = nullptr; +} + +void ProfilerCPUFreq::Sample() { + DWORD dwBufferSize = mBufferSize; + if (!mBuffer || + (ERROR_SUCCESS != RegQueryValueEx(HKEY_PERFORMANCE_DATA, mBlockIndex, + NULL, NULL, mBuffer, &dwBufferSize))) { + NS_WARNING("failed to query performance data"); + return; + } + + PERF_DATA_BLOCK* dataBlock = (PERF_DATA_BLOCK*)mBuffer; + LPBYTE pObject = mBuffer + dataBlock->HeaderLength; + PERF_OBJECT_TYPE* object = (PERF_OBJECT_TYPE*)pObject; + PERF_COUNTER_DEFINITION* counter = nullptr; + { + PERF_COUNTER_DEFINITION* pCounter = + (PERF_COUNTER_DEFINITION*)(pObject + object->HeaderLength); + for (DWORD i = 0; i < object->NumCounters; i++) { + if (mCounterNameIndex == pCounter->CounterNameTitleIndex) { + counter = pCounter; + break; + } + pCounter++; + } + } + if (!counter) { + NS_WARNING("failed to find counter"); + return; + } + + MOZ_ASSERT(counter->CounterType == PERF_AVERAGE_BULK); + PERF_COUNTER_DEFINITION* baseCounter = counter + 1; + MOZ_ASSERT((baseCounter->CounterType & PERF_COUNTER_BASE) == + PERF_COUNTER_BASE); + + PERF_INSTANCE_DEFINITION* instanceDef = + (PERF_INSTANCE_DEFINITION*)(pObject + object->DefinitionLength); + for (LONG i = 0; i < object->NumInstances; i++) { + PERF_COUNTER_BLOCK* counterBlock = + (PERF_COUNTER_BLOCK*)((LPBYTE)instanceDef + instanceDef->ByteLength); + + LPWSTR name = (LPWSTR)(((LPBYTE)instanceDef) + instanceDef->NameOffset); + unsigned int cpuId, coreId; + if (swscanf(name, L"%u,%u", &cpuId, &coreId) == 2 && cpuId == 0 && + coreId < mCPUCounters.length()) { + auto& CPUCounter = mCPUCounters[coreId]; + ULONGLONG prevData = CPUCounter.data; + DWORD prevBase = CPUCounter.base; + CPUCounter.data = *(UNALIGNED ULONGLONG*)((LPBYTE)counterBlock + + counter->CounterOffset); + CPUCounter.base = + *(DWORD*)((LPBYTE)counterBlock + baseCounter->CounterOffset); + if (prevBase && prevBase != CPUCounter.base) { + CPUCounter.freq = CPUCounter.nominalFrequency * + (CPUCounter.data - prevData) / + (CPUCounter.base - prevBase) / 1000 * 10; + } + } + instanceDef = (PERF_INSTANCE_DEFINITION*)((LPBYTE)counterBlock + + counterBlock->ByteLength); + } +} diff --git a/tools/profiler/core/ProfilerCPUFreq.h b/tools/profiler/core/ProfilerCPUFreq.h new file mode 100644 index 0000000000..a01f1277c9 --- /dev/null +++ b/tools/profiler/core/ProfilerCPUFreq.h @@ -0,0 +1,68 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef TOOLS_PROFILERCPUFREQ_H_ +#define TOOLS_PROFILERCPUFREQ_H_ + +#include "PlatformMacros.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Vector.h" + +#if defined(GP_OS_windows) || defined(GP_OS_linux) || defined(GP_OS_android) +# define HAVE_CPU_FREQ_SUPPORT +#endif + +#if defined(GP_OS_windows) +# include <windows.h> + +struct CPUCounterInfo { + ULONGLONG data = 0; + DWORD base = 0; + uint32_t freq = 0; + DWORD nominalFrequency = 0; +}; +#endif + +#if defined(GP_OS_linux) || defined(GP_OS_android) +struct CPUCounterInfo { + int fd = 0; +}; +#endif + +class ProfilerCPUFreq { + public: +#if defined(HAVE_CPU_FREQ_SUPPORT) + explicit ProfilerCPUFreq(); + ~ProfilerCPUFreq(); +# if defined(GP_OS_windows) + void Sample(); + uint32_t GetCPUSpeedMHz(unsigned cpuId) { + MOZ_ASSERT(cpuId < mCPUCounters.length()); + return mCPUCounters[cpuId].freq; + } +# else + void Sample() {} + uint32_t GetCPUSpeedMHz(unsigned cpuId); +# endif +#else + explicit ProfilerCPUFreq(){}; + ~ProfilerCPUFreq(){}; + void Sample(){}; +#endif + + private: +#if defined(HAVE_CPU_FREQ_SUPPORT) +# if defined(GP_OS_windows) + LPWSTR mBlockIndex = nullptr; + DWORD mCounterNameIndex = 0; + // The size of the counter block is about 8kB for a machine with 20 cores, + // so 32kB should be plenty. + DWORD mBufferSize = 32768; + LPBYTE mBuffer = nullptr; +# endif + mozilla::Vector<CPUCounterInfo> mCPUCounters; +#endif +}; + +#endif /* ndef TOOLS_PROFILERCPUFREQ_H_ */ diff --git a/tools/profiler/core/ProfilerCodeAddressService.cpp b/tools/profiler/core/ProfilerCodeAddressService.cpp new file mode 100644 index 0000000000..5a65e06379 --- /dev/null +++ b/tools/profiler/core/ProfilerCodeAddressService.cpp @@ -0,0 +1,75 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfilerCodeAddressService.h" + +#include "platform.h" +#include "mozilla/StackWalk.h" + +using namespace mozilla; + +#if defined(XP_LINUX) || defined(XP_FREEBSD) +static char* SearchSymbolTable(SymbolTable& aTable, uint32_t aOffset) { + size_t index; + bool exact = + BinarySearch(aTable.mAddrs, 0, aTable.mAddrs.Length(), aOffset, &index); + + if (index == 0 && !exact) { + // Our offset is before the first symbol in the table; no result. + return nullptr; + } + + // Extract the (mangled) symbol name out of the string table. + auto strings = reinterpret_cast<char*>(aTable.mBuffer.Elements()); + nsCString symbol; + symbol.Append(strings + aTable.mIndex[index - 1], + aTable.mIndex[index] - aTable.mIndex[index - 1]); + + // First try demangling as a Rust identifier. + char demangled[1024]; + if (!profiler_demangle_rust(symbol.get(), demangled, + ArrayLength(demangled))) { + // Then as a C++ identifier. + DemangleSymbol(symbol.get(), demangled, ArrayLength(demangled)); + } + demangled[ArrayLength(demangled) - 1] = '\0'; + + // Use the mangled name if we didn't successfully demangle. + return strdup(demangled[0] != '\0' ? demangled : symbol.get()); +} +#endif + +bool ProfilerCodeAddressService::GetFunction(const void* aPc, + nsACString& aResult) { + Entry& entry = GetEntry(aPc); + +#if defined(XP_LINUX) || defined(XP_FREEBSD) + // On Linux, most symbols will not be found by the MozDescribeCodeAddress call + // that GetEntry does. So we read the symbol table directly from the ELF + // image. + + // SymbolTable currently assumes library offsets will not be larger than + // 4 GiB. + if (entry.mLOffset <= 0xFFFFFFFF && !entry.mFunction) { + auto p = mSymbolTables.lookupForAdd(entry.mLibrary); + if (!p) { + if (!mSymbolTables.add(p, entry.mLibrary, SymbolTable())) { + MOZ_CRASH("ProfilerCodeAddressService OOM"); + } + profiler_get_symbol_table(entry.mLibrary, nullptr, &p->value()); + } + entry.mFunction = + SearchSymbolTable(p->value(), static_cast<uint32_t>(entry.mLOffset)); + } +#endif + + if (!entry.mFunction || entry.mFunction[0] == '\0') { + return false; + } + + aResult = nsDependentCString(entry.mFunction); + return true; +} diff --git a/tools/profiler/core/ProfilerMarkers.cpp b/tools/profiler/core/ProfilerMarkers.cpp new file mode 100644 index 0000000000..9b20c46d6b --- /dev/null +++ b/tools/profiler/core/ProfilerMarkers.cpp @@ -0,0 +1,32 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ProfilerMarkers.h" + +template mozilla::ProfileBufferBlockIndex AddMarkerToBuffer( + mozilla::ProfileChunkedBuffer&, const mozilla::ProfilerString8View&, + const mozilla::MarkerCategory&, mozilla::MarkerOptions&&, + mozilla::baseprofiler::markers::NoPayload); + +template mozilla::ProfileBufferBlockIndex AddMarkerToBuffer( + mozilla::ProfileChunkedBuffer&, const mozilla::ProfilerString8View&, + const mozilla::MarkerCategory&, mozilla::MarkerOptions&&, + mozilla::baseprofiler::markers::TextMarker, const std::string&); + +template mozilla::ProfileBufferBlockIndex profiler_add_marker_impl( + const mozilla::ProfilerString8View&, const mozilla::MarkerCategory&, + mozilla::MarkerOptions&&, mozilla::baseprofiler::markers::TextMarker, + const std::string&); + +template mozilla::ProfileBufferBlockIndex profiler_add_marker_impl( + const mozilla::ProfilerString8View&, const mozilla::MarkerCategory&, + mozilla::MarkerOptions&&, mozilla::baseprofiler::markers::TextMarker, + const nsCString&); + +template mozilla::ProfileBufferBlockIndex profiler_add_marker_impl( + const mozilla::ProfilerString8View&, const mozilla::MarkerCategory&, + mozilla::MarkerOptions&&, mozilla::baseprofiler::markers::Tracing, + const mozilla::ProfilerString8View&); diff --git a/tools/profiler/core/ProfilerThreadRegistration.cpp b/tools/profiler/core/ProfilerThreadRegistration.cpp new file mode 100644 index 0000000000..c81d00573d --- /dev/null +++ b/tools/profiler/core/ProfilerThreadRegistration.cpp @@ -0,0 +1,198 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ProfilerThreadRegistration.h" + +#include "mozilla/ProfilerMarkers.h" +#include "mozilla/ProfilerThreadRegistry.h" +#include "nsString.h" +#ifdef MOZ_GECKO_PROFILER +# include "platform.h" +#else +# define profiler_mark_thread_awake() +# define profiler_mark_thread_asleep() +#endif + +namespace mozilla::profiler { + +/* static */ +MOZ_THREAD_LOCAL(ThreadRegistration*) ThreadRegistration::tlsThreadRegistration; + +ThreadRegistration::ThreadRegistration(const char* aName, const void* aStackTop) + : mData(aName, aStackTop) { + auto* tls = GetTLS(); + if (MOZ_UNLIKELY(!tls)) { + // No TLS, nothing can be done without it. + return; + } + + if (ThreadRegistration* rootRegistration = tls->get(); rootRegistration) { + // This is a nested ThreadRegistration object, so the thread is already + // registered in the TLS and ThreadRegistry and we don't need to register + // again. + MOZ_ASSERT( + mData.Info().ThreadId() == rootRegistration->mData.Info().ThreadId(), + "Thread being re-registered has changed its TID"); + // TODO: Use new name. This is currently not possible because the + // TLS-stored RegisteredThread's ThreadInfo cannot be changed. + // In the meantime, we record a marker that could be used in the frontend. + PROFILER_MARKER_TEXT("Nested ThreadRegistration()", OTHER_Profiling, + MarkerOptions{}, + ProfilerString8View::WrapNullTerminatedString(aName)); + return; + } + + tls->set(this); + ThreadRegistry::Register(OnThreadRef{*this}); + profiler_mark_thread_awake(); +} + +ThreadRegistration::~ThreadRegistration() { + MOZ_ASSERT(profiler_current_thread_id() == mData.mInfo.ThreadId(), + "ThreadRegistration must be destroyed on its thread"); + MOZ_ASSERT(!mDataMutex.IsLockedOnCurrentThread(), + "Mutex shouldn't be locked here, as it's about to be destroyed " + "in ~ThreadRegistration()"); + auto* tls = GetTLS(); + if (MOZ_UNLIKELY(!tls)) { + // No TLS, nothing can be done without it. + return; + } + + if (ThreadRegistration* rootRegistration = tls->get(); rootRegistration) { + if (rootRegistration != this) { + // `this` is not in the TLS, so it was a nested registration, there is + // nothing to unregister yet. + PROFILER_MARKER_TEXT( + "Nested ~ThreadRegistration()", OTHER_Profiling, MarkerOptions{}, + ProfilerString8View::WrapNullTerminatedString(mData.Info().Name())); + return; + } + + profiler_mark_thread_asleep(); +#ifdef NIGHTLY_BUILD + mData.RecordWakeCount(); +#endif + ThreadRegistry::Unregister(OnThreadRef{*this}); +#ifdef DEBUG + // After ThreadRegistry::Unregister, other threads should not be able to + // find this ThreadRegistration, and shouldn't have kept any reference to + // it across the ThreadRegistry mutex. + MOZ_ASSERT(mDataMutex.TryLock(), + "Mutex shouldn't be locked in any thread, as it's about to be " + "destroyed in ~ThreadRegistration()"); + // Undo the above successful TryLock. + mDataMutex.Unlock(); +#endif // DEBUG + + tls->set(nullptr); + return; + } + + // Already removed from the TLS!? This could happen with improperly-nested + // register/unregister calls, and the first ThreadRegistration has already + // been unregistered. + // We cannot record a marker on this thread because it was already + // unregistered. Send it to the main thread (unless this *is* already the + // main thread, which has been unregistered); this may be useful to catch + // mismatched register/unregister pairs in Firefox. + if (!profiler_is_main_thread()) { + nsAutoCString threadId("thread id: "); + threadId.AppendInt(profiler_current_thread_id().ToNumber()); + threadId.AppendLiteral(", name: \""); + threadId.AppendASCII(mData.Info().Name()); + threadId.AppendLiteral("\""); + PROFILER_MARKER_TEXT( + "~ThreadRegistration() but TLS is empty", OTHER_Profiling, + MarkerOptions(MarkerThreadId::MainThread(), MarkerStack::Capture()), + threadId); + } +} + +/* static */ +ProfilingStack* ThreadRegistration::RegisterThread(const char* aName, + const void* aStackTop) { + auto* tls = GetTLS(); + if (MOZ_UNLIKELY(!tls)) { + // No TLS, nothing can be done without it. + return nullptr; + } + + if (ThreadRegistration* rootRegistration = tls->get(); rootRegistration) { + // Already registered, record the extra depth to ignore the matching + // UnregisterThread. + ++rootRegistration->mOtherRegistrations; + // TODO: Use new name. This is currently not possible because the + // TLS-stored RegisteredThread's ThreadInfo cannot be changed. + // In the meantime, we record a marker that could be used in the frontend. + PROFILER_MARKER_TEXT("Nested ThreadRegistration::RegisterThread()", + OTHER_Profiling, MarkerOptions{}, + ProfilerString8View::WrapNullTerminatedString(aName)); + return &rootRegistration->mData.mProfilingStack; + } + + // Create on heap, it self-registers with the TLS (its effective owner, so + // we can forget the pointer after this), and with the Profiler. + ThreadRegistration* tr = new ThreadRegistration(aName, aStackTop); + tr->mIsOnHeap = true; + return &tr->mData.mProfilingStack; +} + +/* static */ +void ThreadRegistration::UnregisterThread() { + auto* tls = GetTLS(); + if (MOZ_UNLIKELY(!tls)) { + // No TLS, nothing can be done without it. + return; + } + + if (ThreadRegistration* rootRegistration = tls->get(); rootRegistration) { + if (rootRegistration->mOtherRegistrations != 0) { + // This is assumed to be a matching UnregisterThread() for a nested + // RegisterThread(). Decrease depth and we're done. + --rootRegistration->mOtherRegistrations; + // We don't know what name was used in the related RegisterThread(). + PROFILER_MARKER_UNTYPED("Nested ThreadRegistration::UnregisterThread()", + OTHER_Profiling); + return; + } + + if (!rootRegistration->mIsOnHeap) { + // The root registration was not added by `RegisterThread()`, so it + // shouldn't be deleted! + // This could happen if there are un-paired `UnregisterThread` calls when + // the initial registration (still alive) was done on the stack. We don't + // know what name was used in the related RegisterThread(). + PROFILER_MARKER_UNTYPED("Excess ThreadRegistration::UnregisterThread()", + OTHER_Profiling, MarkerStack::Capture()); + return; + } + + // This is the last `UnregisterThread()` that should match the first + // `RegisterThread()` that created this ThreadRegistration on the heap. + // Just delete this root registration, it will de-register itself from the + // TLS (and from the Profiler). + delete rootRegistration; + return; + } + + // There is no known ThreadRegistration for this thread, ignore this + // request. We cannot record a marker on this thread because it was already + // unregistered. Send it to the main thread (unless this *is* already the + // main thread, which has been unregistered); this may be useful to catch + // mismatched register/unregister pairs in Firefox. + if (!profiler_is_main_thread()) { + nsAutoCString threadId("thread id: "); + threadId.AppendInt(profiler_current_thread_id().ToNumber()); + PROFILER_MARKER_TEXT( + "ThreadRegistration::UnregisterThread() but TLS is empty", + OTHER_Profiling, + MarkerOptions(MarkerThreadId::MainThread(), MarkerStack::Capture()), + threadId); + } +} + +} // namespace mozilla::profiler diff --git a/tools/profiler/core/ProfilerThreadRegistrationData.cpp b/tools/profiler/core/ProfilerThreadRegistrationData.cpp new file mode 100644 index 0000000000..e70f9e749a --- /dev/null +++ b/tools/profiler/core/ProfilerThreadRegistrationData.cpp @@ -0,0 +1,303 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ProfilerThreadRegistrationData.h" + +#include "mozilla/FOGIPC.h" +#include "mozilla/glean/GleanMetrics.h" +#include "mozilla/ProfilerMarkers.h" +#include "js/AllocationRecording.h" +#include "js/ProfilingStack.h" + +#if defined(XP_WIN) +# include <windows.h> +#elif defined(XP_DARWIN) +# include <pthread.h> +#endif + +#ifdef NIGHTLY_BUILD +namespace geckoprofiler::markers { + +using namespace mozilla; + +struct ThreadCpuUseMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("ThreadCpuUse"); + } + static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter, + ProfilerThreadId aThreadId, + int64_t aCpuTimeMs, int64_t aWakeUps, + const ProfilerString8View& aThreadName) { + aWriter.IntProperty("threadId", static_cast<int64_t>(aThreadId.ToNumber())); + aWriter.IntProperty("time", aCpuTimeMs); + aWriter.IntProperty("wakeups", aWakeUps); + aWriter.StringProperty("label", aThreadName); + } + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable}; + schema.AddKeyLabelFormat("time", "CPU Time", MS::Format::Milliseconds); + schema.AddKeyLabelFormat("wakeups", "Wake ups", MS::Format::Integer); + schema.SetTooltipLabel("{marker.name} - {marker.data.label}"); + schema.SetTableLabel( + "{marker.name} - {marker.data.label}: {marker.data.time} of CPU time, " + "{marker.data.wakeups} wake ups"); + return schema; + } +}; + +} // namespace geckoprofiler::markers +#endif + +namespace mozilla::profiler { + +ThreadRegistrationData::ThreadRegistrationData(const char* aName, + const void* aStackTop) + : mInfo(aName), + mPlatformData(mInfo.ThreadId()), + mStackTop( +#if defined(XP_WIN) + // We don't have to guess on Windows. + reinterpret_cast<const void*>( + reinterpret_cast<PNT_TIB>(NtCurrentTeb())->StackBase) +#elif defined(XP_DARWIN) + // We don't have to guess on Mac/Darwin. + reinterpret_cast<const void*>( + pthread_get_stackaddr_np(pthread_self())) +#else + // Otherwise use the given guess. + aStackTop +#endif + ) { +} + +// This is a simplified version of profiler_add_marker that can be easily passed +// into the JS engine. +static void profiler_add_js_marker(const char* aMarkerName, + const char* aMarkerText) { + PROFILER_MARKER_TEXT( + mozilla::ProfilerString8View::WrapNullTerminatedString(aMarkerName), JS, + {}, mozilla::ProfilerString8View::WrapNullTerminatedString(aMarkerText)); +} + +static void profiler_add_js_allocation_marker(JS::RecordAllocationInfo&& info) { + if (!profiler_thread_is_being_profiled_for_markers()) { + return; + } + + struct JsAllocationMarker { + static constexpr mozilla::Span<const char> MarkerTypeName() { + return mozilla::MakeStringSpan("JS allocation"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter, + const mozilla::ProfilerString16View& aTypeName, + const mozilla::ProfilerString8View& aClassName, + const mozilla::ProfilerString16View& aDescriptiveTypeName, + const mozilla::ProfilerString8View& aCoarseType, uint64_t aSize, + bool aInNursery) { + if (aClassName.Length() != 0) { + aWriter.StringProperty("className", aClassName); + } + if (aTypeName.Length() != 0) { + aWriter.StringProperty("typeName", NS_ConvertUTF16toUTF8(aTypeName)); + } + if (aDescriptiveTypeName.Length() != 0) { + aWriter.StringProperty("descriptiveTypeName", + NS_ConvertUTF16toUTF8(aDescriptiveTypeName)); + } + aWriter.StringProperty("coarseType", aCoarseType); + aWriter.IntProperty("size", aSize); + aWriter.BoolProperty("inNursery", aInNursery); + } + static mozilla::MarkerSchema MarkerTypeDisplay() { + return mozilla::MarkerSchema::SpecialFrontendLocation{}; + } + }; + + profiler_add_marker( + "JS allocation", geckoprofiler::category::JS, + mozilla::MarkerStack::Capture(), JsAllocationMarker{}, + mozilla::ProfilerString16View::WrapNullTerminatedString(info.typeName), + mozilla::ProfilerString8View::WrapNullTerminatedString(info.className), + mozilla::ProfilerString16View::WrapNullTerminatedString( + info.descriptiveTypeName), + mozilla::ProfilerString8View::WrapNullTerminatedString(info.coarseType), + info.size, info.inNursery); +} + +void ThreadRegistrationLockedRWFromAnyThread::SetProfilingFeaturesAndData( + ThreadProfilingFeatures aProfilingFeatures, + ProfiledThreadData* aProfiledThreadData, const PSAutoLock&) { + MOZ_ASSERT(mProfilingFeatures == ThreadProfilingFeatures::NotProfiled); + mProfilingFeatures = aProfilingFeatures; + + MOZ_ASSERT(!mProfiledThreadData); + MOZ_ASSERT(aProfiledThreadData); + mProfiledThreadData = aProfiledThreadData; + + if (mJSContext) { + // The thread is now being profiled, and we already have a JSContext, + // allocate a JsFramesBuffer to allow profiler-unlocked on-thread sampling. + MOZ_ASSERT(!mJsFrameBuffer); + mJsFrameBuffer = new JsFrame[MAX_JS_FRAMES]; + } + + // Check invariants. + MOZ_ASSERT((mProfilingFeatures != ThreadProfilingFeatures::NotProfiled) == + !!mProfiledThreadData); + MOZ_ASSERT((mJSContext && + (mProfilingFeatures != ThreadProfilingFeatures::NotProfiled)) == + !!mJsFrameBuffer); +} + +void ThreadRegistrationLockedRWFromAnyThread::ClearProfilingFeaturesAndData( + const PSAutoLock&) { + mProfilingFeatures = ThreadProfilingFeatures::NotProfiled; + mProfiledThreadData = nullptr; + + if (mJsFrameBuffer) { + delete[] mJsFrameBuffer; + mJsFrameBuffer = nullptr; + } + + // Check invariants. + MOZ_ASSERT((mProfilingFeatures != ThreadProfilingFeatures::NotProfiled) == + !!mProfiledThreadData); + MOZ_ASSERT((mJSContext && + (mProfilingFeatures != ThreadProfilingFeatures::NotProfiled)) == + !!mJsFrameBuffer); +} + +void ThreadRegistrationLockedRWOnThread::SetJSContext(JSContext* aJSContext) { + MOZ_ASSERT(aJSContext && !mJSContext); + + mJSContext = aJSContext; + + if (mProfiledThreadData) { + MOZ_ASSERT((mProfilingFeatures != ThreadProfilingFeatures::NotProfiled) == + !!mProfiledThreadData); + // We now have a JSContext, and the thread is already being profiled, + // allocate a JsFramesBuffer to allow profiler-unlocked on-thread sampling. + MOZ_ASSERT(!mJsFrameBuffer); + mJsFrameBuffer = new JsFrame[MAX_JS_FRAMES]; + } + + // We give the JS engine a non-owning reference to the ProfilingStack. It's + // important that the JS engine doesn't touch this once the thread dies. + js::SetContextProfilingStack(aJSContext, &ProfilingStackRef()); + + // Check invariants. + MOZ_ASSERT((mJSContext && + (mProfilingFeatures != ThreadProfilingFeatures::NotProfiled)) == + !!mJsFrameBuffer); +} + +void ThreadRegistrationLockedRWOnThread::ClearJSContext() { + mJSContext = nullptr; + + if (mJsFrameBuffer) { + delete[] mJsFrameBuffer; + mJsFrameBuffer = nullptr; + } + + // Check invariants. + MOZ_ASSERT((mJSContext && + (mProfilingFeatures != ThreadProfilingFeatures::NotProfiled)) == + !!mJsFrameBuffer); +} + +void ThreadRegistrationLockedRWOnThread::PollJSSampling() { + // We can't start/stop profiling until we have the thread's JSContext. + if (mJSContext) { + // It is possible for mJSSampling to go through the following sequences. + // + // - INACTIVE, ACTIVE_REQUESTED, INACTIVE_REQUESTED, INACTIVE + // + // - ACTIVE, INACTIVE_REQUESTED, ACTIVE_REQUESTED, ACTIVE + // + // Therefore, the if and else branches here aren't always interleaved. + // This is ok because the JS engine can handle that. + // + if (mJSSampling == ACTIVE_REQUESTED) { + mJSSampling = ACTIVE; + js::EnableContextProfilingStack(mJSContext, true); + + if (JSAllocationsEnabled()) { + // TODO - This probability should not be hardcoded. See Bug 1547284. + JS::EnableRecordingAllocations(mJSContext, + profiler_add_js_allocation_marker, 0.01); + } + js::RegisterContextProfilingEventMarker(mJSContext, + profiler_add_js_marker); + + } else if (mJSSampling == INACTIVE_REQUESTED) { + mJSSampling = INACTIVE; + js::EnableContextProfilingStack(mJSContext, false); + + if (JSAllocationsEnabled()) { + JS::DisableRecordingAllocations(mJSContext); + } + } + } +} + +#ifdef NIGHTLY_BUILD +void ThreadRegistrationUnlockedConstReaderAndAtomicRW::RecordWakeCount() const { + baseprofiler::detail::BaseProfilerAutoLock lock(mRecordWakeCountMutex); + + uint64_t newWakeCount = mWakeCount - mAlreadyRecordedWakeCount; + if (newWakeCount == 0 && mSleep != AWAKE) { + // If no new wake-up was counted, and the thread is not marked awake, + // we can be pretty sure there is no CPU activity to record. + // Threads that are never annotated as asleep/awake (typically rust threads) + // start as awake. + return; + } + + uint64_t cpuTimeNs; + if (!GetCpuTimeSinceThreadStartInNs(&cpuTimeNs, PlatformDataCRef())) { + cpuTimeNs = 0; + } + + constexpr uint64_t NS_PER_MS = 1'000'000; + uint64_t cpuTimeMs = cpuTimeNs / NS_PER_MS; + + uint64_t newCpuTimeMs = MOZ_LIKELY(cpuTimeMs > mAlreadyRecordedCpuTimeInMs) + ? cpuTimeMs - mAlreadyRecordedCpuTimeInMs + : 0; + + if (!newWakeCount && !newCpuTimeMs) { + // Nothing to report, avoid computing the Glean friendly thread name. + return; + } + + nsAutoCString threadName(mInfo.Name()); + // Trim the trailing number of threads that are part of a thread pool. + for (size_t length = threadName.Length(); length > 0; --length) { + const char c = threadName.CharAt(length - 1); + if ((c < '0' || c > '9') && c != '#' && c != ' ') { + if (length != threadName.Length()) { + threadName.SetLength(length); + } + break; + } + } + + mozilla::glean::RecordThreadCpuUse(threadName, newCpuTimeMs, newWakeCount); + + // The thread id is provided as part of the payload because this call is + // inside a ThreadRegistration data function, which could be invoked with + // the ThreadRegistry locked. We cannot call any function/option that could + // attempt to lock the ThreadRegistry again, like MarkerThreadId. + PROFILER_MARKER("Thread CPU use", OTHER, {}, ThreadCpuUseMarker, + mInfo.ThreadId(), newCpuTimeMs, newWakeCount, threadName); + mAlreadyRecordedCpuTimeInMs = cpuTimeMs; + mAlreadyRecordedWakeCount += newWakeCount; +} +#endif + +} // namespace mozilla::profiler diff --git a/tools/profiler/core/ProfilerThreadRegistry.cpp b/tools/profiler/core/ProfilerThreadRegistry.cpp new file mode 100644 index 0000000000..cb456471d9 --- /dev/null +++ b/tools/profiler/core/ProfilerThreadRegistry.cpp @@ -0,0 +1,40 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ProfilerThreadRegistry.h" + +namespace mozilla::profiler { + +/* static */ +ThreadRegistry::RegistryContainer ThreadRegistry::sRegistryContainer; + +/* static */ +ThreadRegistry::RegistryMutex ThreadRegistry::sRegistryMutex; + +#if !defined(MOZ_GECKO_PROFILER) +// When MOZ_GECKO_PROFILER is not defined, the function definitions in +// platform.cpp are not built, causing link errors. So we keep these simple +// definitions here. + +/* static */ +void ThreadRegistry::Register(ThreadRegistration::OnThreadRef aOnThreadRef) { + LockedRegistry lock; + MOZ_RELEASE_ASSERT(sRegistryContainer.append(OffThreadRef{aOnThreadRef})); +} + +/* static */ +void ThreadRegistry::Unregister(ThreadRegistration::OnThreadRef aOnThreadRef) { + LockedRegistry lock; + for (OffThreadRef& thread : sRegistryContainer) { + if (thread.IsPointingAt(*aOnThreadRef.mThreadRegistration)) { + sRegistryContainer.erase(&thread); + break; + } + } +} +#endif // !defined(MOZ_GECKO_PROFILER) + +} // namespace mozilla::profiler diff --git a/tools/profiler/core/ProfilerUtils.cpp b/tools/profiler/core/ProfilerUtils.cpp new file mode 100644 index 0000000000..6a46878ad7 --- /dev/null +++ b/tools/profiler/core/ProfilerUtils.cpp @@ -0,0 +1,118 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This file implements functions from ProfilerUtils.h on all platforms. +// Functions with platform-specific implementations are separated in #if blocks +// below, with each block being self-contained with all the #includes and +// definitions it needs, to keep platform code easier to maintain in isolation. + +#include "mozilla/ProfilerUtils.h" + +// --------------------------------------------- Windows process & thread ids +#if defined(XP_WIN) + +# include <process.h> +# include <processthreadsapi.h> + +ProfilerProcessId profiler_current_process_id() { + return ProfilerProcessId::FromNativeId(_getpid()); +} + +ProfilerThreadId profiler_current_thread_id() { + static_assert(std::is_same_v<ProfilerThreadId::NativeType, + decltype(GetCurrentThreadId())>, + "ProfilerThreadId::NativeType must be exactly the type " + "returned by GetCurrentThreadId()"); + return ProfilerThreadId::FromNativeId(GetCurrentThreadId()); +} + +// --------------------------------------------- Non-Windows process id +#else +// All non-Windows platforms are assumed to be POSIX, which has getpid(). + +# include <unistd.h> + +ProfilerProcessId profiler_current_process_id() { + return ProfilerProcessId::FromNativeId(getpid()); +} + +// --------------------------------------------- Non-Windows thread id +// ------------------------------------------------------- macOS +# if defined(XP_MACOSX) + +# include <pthread.h> + +ProfilerThreadId profiler_current_thread_id() { + uint64_t tid; + if (pthread_threadid_np(nullptr, &tid) != 0) { + return ProfilerThreadId{}; + } + return ProfilerThreadId::FromNativeId(tid); +} + +// ------------------------------------------------------- Android +// Test Android before Linux, because Linux includes Android. +# elif defined(__ANDROID__) || defined(ANDROID) + +ProfilerThreadId profiler_current_thread_id() { + return ProfilerThreadId::FromNativeId(gettid()); +} + +// ------------------------------------------------------- Linux +# elif defined(XP_LINUX) + +# include <sys/syscall.h> + +ProfilerThreadId profiler_current_thread_id() { + // glibc doesn't provide a wrapper for gettid() until 2.30 + return ProfilerThreadId::FromNativeId(syscall(SYS_gettid)); +} + +// ------------------------------------------------------- FreeBSD +# elif defined(XP_FREEBSD) + +# include <sys/thr.h> + +ProfilerThreadId profiler_current_thread_id() { + long id; + if (thr_self(&id) != 0) { + return ProfilerThreadId{}; + } + return ProfilerThreadId::FromNativeId(id); +} + +// ------------------------------------------------------- Others +# else + +ProfilerThreadId profiler_current_thread_id() { + return ProfilerThreadId::FromNativeId(std::this_thread::get_id()); +} + +# endif +#endif // End of non-XP_WIN. + +// --------------------------------------------- Platform-agnostic definitions + +#include "MainThreadUtils.h" +#include "mozilla/Assertions.h" + +static ProfilerThreadId scProfilerMainThreadId; + +void profiler_init_main_thread_id() { + MOZ_ASSERT(NS_IsMainThread()); + mozilla::baseprofiler::profiler_init_main_thread_id(); + if (!scProfilerMainThreadId.IsSpecified()) { + scProfilerMainThreadId = profiler_current_thread_id(); + } +} + +[[nodiscard]] ProfilerThreadId profiler_main_thread_id() { + return scProfilerMainThreadId; +} + +[[nodiscard]] bool profiler_is_main_thread() { + return profiler_current_thread_id() == scProfilerMainThreadId; +} diff --git a/tools/profiler/core/VTuneProfiler.cpp b/tools/profiler/core/VTuneProfiler.cpp new file mode 100644 index 0000000000..58a39c51ee --- /dev/null +++ b/tools/profiler/core/VTuneProfiler.cpp @@ -0,0 +1,80 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef XP_WIN +# undef UNICODE +# undef _UNICODE +#endif + +#include "VTuneProfiler.h" +#include "mozilla/Bootstrap.h" +#include <memory> + +VTuneProfiler* VTuneProfiler::mInstance = nullptr; + +void VTuneProfiler::Initialize() { + // This is just a 'dirty trick' to find out if the ittnotify DLL was found. + // If it wasn't this function always returns 0, otherwise it returns + // incrementing numbers, if the library was found this wastes 2 events but + // that should be okay. + __itt_event testEvent = + __itt_event_create("Test event", strlen("Test event")); + testEvent = __itt_event_create("Test event 2", strlen("Test event 2")); + + if (testEvent) { + mInstance = new VTuneProfiler(); + } +} + +void VTuneProfiler::Shutdown() {} + +void VTuneProfiler::TraceInternal(const char* aName, TracingKind aKind) { + std::string str(aName); + + auto iter = mStrings.find(str); + + __itt_event event; + if (iter != mStrings.end()) { + event = iter->second; + } else { + event = __itt_event_create(aName, str.length()); + mStrings.insert({str, event}); + } + + if (aKind == TRACING_INTERVAL_START || aKind == TRACING_EVENT) { + // VTune will consider starts not matched with an end to be single point in + // time events. + __itt_event_start(event); + } else { + __itt_event_end(event); + } +} + +void VTuneProfiler::RegisterThreadInternal(const char* aName) { + std::string str(aName); + + if (!str.compare("GeckoMain")) { + // Process main thread. + switch (XRE_GetProcessType()) { + case GeckoProcessType::GeckoProcessType_Default: + __itt_thread_set_name("Main Process"); + break; + case GeckoProcessType::GeckoProcessType_Content: + __itt_thread_set_name("Content Process"); + break; + case GeckoProcessType::GeckoProcessType_GMPlugin: + __itt_thread_set_name("Plugin Process"); + break; + case GeckoProcessType::GeckoProcessType_GPU: + __itt_thread_set_name("GPU Process"); + break; + default: + __itt_thread_set_name("Unknown Process"); + } + return; + } + __itt_thread_set_name(aName); +} diff --git a/tools/profiler/core/VTuneProfiler.h b/tools/profiler/core/VTuneProfiler.h new file mode 100644 index 0000000000..e3abe6b90d --- /dev/null +++ b/tools/profiler/core/VTuneProfiler.h @@ -0,0 +1,78 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef VTuneProfiler_h +#define VTuneProfiler_h + +// The intent here is to add 0 overhead for regular users. In order to build +// the VTune profiler code at all --enable-vtune-instrumentation needs to be +// set as a build option. Even then, when none of the environment variables +// is specified that allow us to find the ittnotify DLL, these functions +// should be minimal overhead. When starting Firefox under VTune, these +// env vars will be automatically defined, otherwise INTEL_LIBITTNOTIFY32/64 +// should be set to point at the ittnotify DLL. +#ifndef MOZ_VTUNE_INSTRUMENTATION + +# define VTUNE_INIT() +# define VTUNE_SHUTDOWN() + +# define VTUNE_TRACING(name, kind) +# define VTUNE_REGISTER_THREAD(name) + +#else + +# include "GeckoProfiler.h" + +// This is the regular Intel header, these functions are actually defined for +// us inside js/src/vtune by an intel C file which actually dynamically resolves +// them to the correct DLL. Through libxul these will 'magically' resolve. +# include "vtune/ittnotify.h" + +# include <stddef.h> +# include <unordered_map> +# include <string> + +class VTuneProfiler { + public: + static void Initialize(); + static void Shutdown(); + + enum TracingKind { + TRACING_EVENT, + TRACING_INTERVAL_START, + TRACING_INTERVAL_END, + }; + + static void Trace(const char* aName, TracingKind aKind) { + if (mInstance) { + mInstance->TraceInternal(aName, aKind); + } + } + static void RegisterThread(const char* aName) { + if (mInstance) { + mInstance->RegisterThreadInternal(aName); + } + } + + private: + void TraceInternal(const char* aName, TracingKind aKind); + void RegisterThreadInternal(const char* aName); + + // This is null when the ittnotify DLL could not be found. + static VTuneProfiler* mInstance; + + std::unordered_map<std::string, __itt_event> mStrings; +}; + +# define VTUNE_INIT() VTuneProfiler::Initialize() +# define VTUNE_SHUTDOWN() VTuneProfiler::Shutdown() + +# define VTUNE_TRACING(name, kind) VTuneProfiler::Trace(name, kind) +# define VTUNE_REGISTER_THREAD(name) VTuneProfiler::RegisterThread(name) + +#endif + +#endif /* VTuneProfiler_h */ diff --git a/tools/profiler/core/memory_hooks.cpp b/tools/profiler/core/memory_hooks.cpp new file mode 100644 index 0000000000..e13a109aab --- /dev/null +++ b/tools/profiler/core/memory_hooks.cpp @@ -0,0 +1,638 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "memory_hooks.h" + +#include "nscore.h" + +#include "mozilla/Assertions.h" +#include "mozilla/Atomics.h" +#include "mozilla/FastBernoulliTrial.h" +#include "mozilla/IntegerPrintfMacros.h" +#include "mozilla/JSONWriter.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/PlatformMutex.h" +#include "mozilla/ProfilerCounts.h" +#include "mozilla/ThreadLocal.h" +#include "mozilla/ThreadSafety.h" + +#include "GeckoProfiler.h" +#include "prenv.h" +#include "replace_malloc.h" + +#include <ctype.h> +#include <errno.h> +#include <limits.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#ifdef XP_WIN +# include <windows.h> +# include <process.h> +#else +# include <pthread.h> +# include <sys/types.h> +# include <unistd.h> +#endif + +#ifdef ANDROID +# include <android/log.h> +#endif + +// The counters start out as a nullptr, and then get initialized only once. They +// are never destroyed, as it would cause race conditions for the memory hooks +// that use the counters. This helps guard against potentially expensive +// operations like using a mutex. +// +// In addition, this is a raw pointer and not a UniquePtr, as the counter +// machinery will try and de-register itself from the profiler. This could +// happen after the profiler and its PSMutex was already destroyed, resulting in +// a crash. +static ProfilerCounterTotal* sCounter; + +// The gBernoulli value starts out as a nullptr, and only gets initialized once. +// It then lives for the entire lifetime of the process. It cannot be deleted +// without additional multi-threaded protections, since if we deleted it during +// profiler_stop then there could be a race between threads already in a +// memory hook that might try to access the value after or during deletion. +static mozilla::FastBernoulliTrial* gBernoulli; + +namespace mozilla::profiler { + +//--------------------------------------------------------------------------- +// Utilities +//--------------------------------------------------------------------------- + +// Returns true or or false depending on whether the marker was actually added +// or not. +static bool profiler_add_native_allocation_marker(int64_t aSize, + uintptr_t aMemoryAddress) { + if (!profiler_thread_is_being_profiled_for_markers( + profiler_main_thread_id())) { + return false; + } + + // Because native allocations may be intercepted anywhere, blocking while + // locking the profiler mutex here could end up causing a deadlock if another + // mutex is taken, which the profiler may indirectly need elsewhere. + // See bug 1642726 for such a scenario. + // So instead we bail out if the mutex is already locked. Native allocations + // are statistically sampled anyway, so missing a few because of this is + // acceptable. + if (profiler_is_locked_on_current_thread()) { + return false; + } + + struct NativeAllocationMarker { + static constexpr mozilla::Span<const char> MarkerTypeName() { + return mozilla::MakeStringSpan("Native allocation"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter, int64_t aSize, + uintptr_t aMemoryAddress, ProfilerThreadId aThreadId) { + aWriter.IntProperty("size", aSize); + aWriter.IntProperty("memoryAddress", + static_cast<int64_t>(aMemoryAddress)); + // Tech note: If `ToNumber()` returns a uint64_t, the conversion to + // int64_t is "implementation-defined" before C++20. This is acceptable + // here, because this is a one-way conversion to a unique identifier + // that's used to visually separate data by thread on the front-end. + aWriter.IntProperty("threadId", + static_cast<int64_t>(aThreadId.ToNumber())); + } + static mozilla::MarkerSchema MarkerTypeDisplay() { + return mozilla::MarkerSchema::SpecialFrontendLocation{}; + } + }; + + profiler_add_marker("Native allocation", geckoprofiler::category::OTHER, + {MarkerThreadId::MainThread(), MarkerStack::Capture()}, + NativeAllocationMarker{}, aSize, aMemoryAddress, + profiler_current_thread_id()); + return true; +} + +static malloc_table_t gMallocTable; + +// This is only needed because of the |const void*| vs |void*| arg mismatch. +static size_t MallocSizeOf(const void* aPtr) { + return gMallocTable.malloc_usable_size(const_cast<void*>(aPtr)); +} + +// The values for the Bernoulli trial are taken from DMD. According to DMD: +// +// In testing, a probability of 0.003 resulted in ~25% of heap blocks getting +// a stack trace and ~80% of heap bytes getting a stack trace. (This is +// possible because big heap blocks are more likely to get a stack trace.) +// +// The random number seeds are arbitrary and were obtained from random.org. +// +// However this value resulted in a lot of slowdown since the profiler stacks +// are pretty heavy to collect. The value was lowered to 10% of the original to +// 0.0003. +static void EnsureBernoulliIsInstalled() { + if (!gBernoulli) { + // This is only installed once. See the gBernoulli definition for more + // information. + gBernoulli = + new FastBernoulliTrial(0.0003, 0x8e26eeee166bc8ca, 0x56820f304a9c9ae0); + } +} + +// This class provides infallible allocations (they abort on OOM) like +// mozalloc's InfallibleAllocPolicy, except that memory hooks are bypassed. This +// policy is used by the HashSet. +class InfallibleAllocWithoutHooksPolicy { + static void ExitOnFailure(const void* aP) { + if (!aP) { + MOZ_CRASH("Profiler memory hooks out of memory; aborting"); + } + } + + public: + template <typename T> + static T* maybe_pod_malloc(size_t aNumElems) { + if (aNumElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value) { + return nullptr; + } + return (T*)gMallocTable.malloc(aNumElems * sizeof(T)); + } + + template <typename T> + static T* maybe_pod_calloc(size_t aNumElems) { + return (T*)gMallocTable.calloc(aNumElems, sizeof(T)); + } + + template <typename T> + static T* maybe_pod_realloc(T* aPtr, size_t aOldSize, size_t aNewSize) { + if (aNewSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value) { + return nullptr; + } + return (T*)gMallocTable.realloc(aPtr, aNewSize * sizeof(T)); + } + + template <typename T> + static T* pod_malloc(size_t aNumElems) { + T* p = maybe_pod_malloc<T>(aNumElems); + ExitOnFailure(p); + return p; + } + + template <typename T> + static T* pod_calloc(size_t aNumElems) { + T* p = maybe_pod_calloc<T>(aNumElems); + ExitOnFailure(p); + return p; + } + + template <typename T> + static T* pod_realloc(T* aPtr, size_t aOldSize, size_t aNewSize) { + T* p = maybe_pod_realloc(aPtr, aOldSize, aNewSize); + ExitOnFailure(p); + return p; + } + + template <typename T> + static void free_(T* aPtr, size_t aSize = 0) { + gMallocTable.free(aPtr); + } + + static void reportAllocOverflow() { ExitOnFailure(nullptr); } + bool checkSimulatedOOM() const { return true; } +}; + +// We can't use mozilla::Mutex because it causes re-entry into the memory hooks. +// Define a custom implementation here. +class MOZ_CAPABILITY("mutex") Mutex : private ::mozilla::detail::MutexImpl { + public: + Mutex() = default; + + void Lock() MOZ_CAPABILITY_ACQUIRE() { ::mozilla::detail::MutexImpl::lock(); } + void Unlock() MOZ_CAPABILITY_RELEASE() { + ::mozilla::detail::MutexImpl::unlock(); + } +}; + +class MOZ_SCOPED_CAPABILITY MutexAutoLock { + MutexAutoLock(const MutexAutoLock&) = delete; + void operator=(const MutexAutoLock&) = delete; + + Mutex& mMutex; + + public: + explicit MutexAutoLock(Mutex& aMutex) MOZ_CAPABILITY_ACQUIRE(aMutex) + : mMutex(aMutex) { + mMutex.Lock(); + } + ~MutexAutoLock() MOZ_CAPABILITY_RELEASE() { mMutex.Unlock(); } +}; + +//--------------------------------------------------------------------------- +// Tracked allocations +//--------------------------------------------------------------------------- + +// The allocation tracker is shared between multiple threads, and is the +// coordinator for knowing when allocations have been tracked. The mutable +// internal state is protected by a mutex, and managed by the methods. +// +// The tracker knows about all the allocations that we have added to the +// profiler. This way, whenever any given piece of memory is freed, we can see +// if it was previously tracked, and we can track its deallocation. + +class AllocationTracker { + // This type tracks all of the allocations that we have captured. This way, we + // can see if a deallocation is inside of this set. We want to provide a + // balanced view into the allocations and deallocations. + typedef mozilla::HashSet<const void*, mozilla::DefaultHasher<const void*>, + InfallibleAllocWithoutHooksPolicy> + AllocationSet; + + public: + AllocationTracker() = default; + + void AddMemoryAddress(const void* memoryAddress) { + MutexAutoLock lock(mMutex); + if (!mAllocations.put(memoryAddress)) { + MOZ_CRASH("Out of memory while tracking native allocations."); + }; + } + + void Reset() { + MutexAutoLock lock(mMutex); + mAllocations.clearAndCompact(); + } + + // Returns true when the memory address is found and removed, otherwise that + // memory address is not being tracked and it returns false. + bool RemoveMemoryAddressIfFound(const void* memoryAddress) { + MutexAutoLock lock(mMutex); + + auto ptr = mAllocations.lookup(memoryAddress); + if (ptr) { + // The memory was present. It no longer needs to be tracked. + mAllocations.remove(ptr); + return true; + } + + return false; + } + + private: + AllocationSet mAllocations; + Mutex mMutex MOZ_UNANNOTATED; +}; + +static AllocationTracker* gAllocationTracker; + +static void EnsureAllocationTrackerIsInstalled() { + if (!gAllocationTracker) { + // This is only installed once. + gAllocationTracker = new AllocationTracker(); + } +} + +//--------------------------------------------------------------------------- +// Per-thread blocking of intercepts +//--------------------------------------------------------------------------- + +// On MacOS, and Linux the first __thread/thread_local access calls malloc, +// which leads to an infinite loop. So we use pthread-based TLS instead, which +// somehow doesn't have this problem. +#if !defined(XP_DARWIN) && !defined(XP_LINUX) +# define PROFILER_THREAD_LOCAL(T) MOZ_THREAD_LOCAL(T) +#else +# define PROFILER_THREAD_LOCAL(T) \ + ::mozilla::detail::ThreadLocal<T, ::mozilla::detail::ThreadLocalKeyStorage> +#endif + +// This class is used to determine if allocations on this thread should be +// intercepted or not. +// Creating a ThreadIntercept object on the stack will implicitly block nested +// ones. There are other reasons to block: The feature is off, or we're inside a +// profiler function that is locking a mutex. +class MOZ_RAII ThreadIntercept { + // When set to true, malloc does not intercept additional allocations. This is + // needed because collecting stacks creates new allocations. When blocked, + // these allocations are then ignored by the memory hook. + static PROFILER_THREAD_LOCAL(bool) tlsIsBlocked; + + // This is a quick flag to check and see if the allocations feature is enabled + // or disabled. + static mozilla::Atomic<bool, mozilla::Relaxed> sAllocationsFeatureEnabled; + + // True if this ThreadIntercept has set tlsIsBlocked. + bool mIsBlockingTLS; + + // True if interception is blocked for any reason. + bool mIsBlocked; + + public: + static void Init() { + tlsIsBlocked.infallibleInit(); + // infallibleInit should zero-initialize, which corresponds to `false`. + MOZ_ASSERT(!tlsIsBlocked.get()); + } + + ThreadIntercept() { + // If the allocation interception feature is enabled, and the TLS is not + // blocked yet, we will block the TLS now, and unblock on destruction. + mIsBlockingTLS = sAllocationsFeatureEnabled && !tlsIsBlocked.get(); + if (mIsBlockingTLS) { + MOZ_ASSERT(!tlsIsBlocked.get()); + tlsIsBlocked.set(true); + // Since this is the top-level ThreadIntercept, interceptions are not + // blocked unless the profiler itself holds a locked mutex, in which case + // we don't want to intercept allocations that originate from such a + // profiler call. + mIsBlocked = profiler_is_locked_on_current_thread(); + } else { + // The feature is off, or the TLS was already blocked, then we block this + // interception. + mIsBlocked = true; + } + } + + ~ThreadIntercept() { + if (mIsBlockingTLS) { + MOZ_ASSERT(tlsIsBlocked.get()); + tlsIsBlocked.set(false); + } + } + + // Is this ThreadIntercept effectively blocked? (Feature is off, or this + // ThreadIntercept is nested, or we're inside a locked-Profiler function.) + bool IsBlocked() const { return mIsBlocked; } + + static void EnableAllocationFeature() { sAllocationsFeatureEnabled = true; } + + static void DisableAllocationFeature() { sAllocationsFeatureEnabled = false; } +}; + +PROFILER_THREAD_LOCAL(bool) ThreadIntercept::tlsIsBlocked; + +mozilla::Atomic<bool, mozilla::Relaxed> + ThreadIntercept::sAllocationsFeatureEnabled(false); + +//--------------------------------------------------------------------------- +// malloc/free callbacks +//--------------------------------------------------------------------------- + +static void AllocCallback(void* aPtr, size_t aReqSize) { + if (!aPtr) { + return; + } + + // The first part of this function does not allocate. + size_t actualSize = gMallocTable.malloc_usable_size(aPtr); + if (actualSize > 0) { + sCounter->Add(actualSize); + } + + ThreadIntercept threadIntercept; + if (threadIntercept.IsBlocked()) { + // Either the native allocations feature is not turned on, or we may be + // recursing into a memory hook, return. We'll still collect counter + // information about this allocation, but no stack. + return; + } + + AUTO_PROFILER_LABEL("AllocCallback", PROFILER); + + // Perform a bernoulli trial, which will return true or false based on its + // configured probability. It takes into account the byte size so that + // larger allocations are weighted heavier than smaller allocations. + MOZ_ASSERT(gBernoulli, + "gBernoulli must be properly installed for the memory hooks."); + if ( + // First perform the Bernoulli trial. + gBernoulli->trial(actualSize) && + // Second, attempt to add a marker if the Bernoulli trial passed. + profiler_add_native_allocation_marker( + static_cast<int64_t>(actualSize), + reinterpret_cast<uintptr_t>(aPtr))) { + MOZ_ASSERT(gAllocationTracker, + "gAllocationTracker must be properly installed for the memory " + "hooks."); + // Only track the memory if the allocation marker was actually added to the + // profiler. + gAllocationTracker->AddMemoryAddress(aPtr); + } + + // We're ignoring aReqSize here +} + +static void FreeCallback(void* aPtr) { + if (!aPtr) { + return; + } + + // The first part of this function does not allocate. + size_t unsignedSize = MallocSizeOf(aPtr); + int64_t signedSize = -(static_cast<int64_t>(unsignedSize)); + sCounter->Add(signedSize); + + ThreadIntercept threadIntercept; + if (threadIntercept.IsBlocked()) { + // Either the native allocations feature is not turned on, or we may be + // recursing into a memory hook, return. We'll still collect counter + // information about this allocation, but no stack. + return; + } + + AUTO_PROFILER_LABEL("FreeCallback", PROFILER); + + // Perform a bernoulli trial, which will return true or false based on its + // configured probability. It takes into account the byte size so that + // larger allocations are weighted heavier than smaller allocations. + MOZ_ASSERT( + gAllocationTracker, + "gAllocationTracker must be properly installed for the memory hooks."); + if (gAllocationTracker->RemoveMemoryAddressIfFound(aPtr)) { + // This size here is negative, indicating a deallocation. + profiler_add_native_allocation_marker(signedSize, + reinterpret_cast<uintptr_t>(aPtr)); + } +} + +} // namespace mozilla::profiler + +//--------------------------------------------------------------------------- +// malloc/free interception +//--------------------------------------------------------------------------- + +using namespace mozilla::profiler; + +static void* replace_malloc(size_t aSize) { + // This must be a call to malloc from outside. Intercept it. + void* ptr = gMallocTable.malloc(aSize); + AllocCallback(ptr, aSize); + return ptr; +} + +static void* replace_calloc(size_t aCount, size_t aSize) { + void* ptr = gMallocTable.calloc(aCount, aSize); + AllocCallback(ptr, aCount * aSize); + return ptr; +} + +static void* replace_realloc(void* aOldPtr, size_t aSize) { + // If |aOldPtr| is nullptr, the call is equivalent to |malloc(aSize)|. + if (!aOldPtr) { + return replace_malloc(aSize); + } + + FreeCallback(aOldPtr); + void* ptr = gMallocTable.realloc(aOldPtr, aSize); + if (ptr) { + AllocCallback(ptr, aSize); + } else { + // If realloc fails, we undo the prior operations by re-inserting the old + // pointer into the live block table. We don't have to do anything with the + // dead block list because the dead block hasn't yet been inserted. The + // block will end up looking like it was allocated for the first time here, + // which is untrue, and the slop bytes will be zero, which may be untrue. + // But this case is rare and doing better isn't worth the effort. + AllocCallback(aOldPtr, gMallocTable.malloc_usable_size(aOldPtr)); + } + return ptr; +} + +static void* replace_memalign(size_t aAlignment, size_t aSize) { + void* ptr = gMallocTable.memalign(aAlignment, aSize); + AllocCallback(ptr, aSize); + return ptr; +} + +static void replace_free(void* aPtr) { + FreeCallback(aPtr); + gMallocTable.free(aPtr); +} + +static void* replace_moz_arena_malloc(arena_id_t aArena, size_t aSize) { + void* ptr = gMallocTable.moz_arena_malloc(aArena, aSize); + AllocCallback(ptr, aSize); + return ptr; +} + +static void* replace_moz_arena_calloc(arena_id_t aArena, size_t aCount, + size_t aSize) { + void* ptr = gMallocTable.moz_arena_calloc(aArena, aCount, aSize); + AllocCallback(ptr, aCount * aSize); + return ptr; +} + +static void* replace_moz_arena_realloc(arena_id_t aArena, void* aPtr, + size_t aSize) { + void* ptr = gMallocTable.moz_arena_realloc(aArena, aPtr, aSize); + AllocCallback(ptr, aSize); + return ptr; +} + +static void replace_moz_arena_free(arena_id_t aArena, void* aPtr) { + FreeCallback(aPtr); + gMallocTable.moz_arena_free(aArena, aPtr); +} + +static void* replace_moz_arena_memalign(arena_id_t aArena, size_t aAlignment, + size_t aSize) { + void* ptr = gMallocTable.moz_arena_memalign(aArena, aAlignment, aSize); + AllocCallback(ptr, aSize); + return ptr; +} + +// we have to replace these or jemalloc will assume we don't implement any +// of the arena replacements! +static arena_id_t replace_moz_create_arena_with_params( + arena_params_t* aParams) { + return gMallocTable.moz_create_arena_with_params(aParams); +} + +static void replace_moz_dispose_arena(arena_id_t aArenaId) { + return gMallocTable.moz_dispose_arena(aArenaId); +} + +static void replace_moz_set_max_dirty_page_modifier(int32_t aModifier) { + return gMallocTable.moz_set_max_dirty_page_modifier(aModifier); +} + +// Must come after all the replace_* funcs +void replace_init(malloc_table_t* aMallocTable, ReplaceMallocBridge** aBridge) { + gMallocTable = *aMallocTable; +#define MALLOC_FUNCS (MALLOC_FUNCS_MALLOC_BASE | MALLOC_FUNCS_ARENA) +#define MALLOC_DECL(name, ...) aMallocTable->name = replace_##name; +#include "malloc_decls.h" +} + +void profiler_replace_remove() {} + +namespace mozilla::profiler { +//--------------------------------------------------------------------------- +// Initialization +//--------------------------------------------------------------------------- + +BaseProfilerCount* install_memory_hooks() { + if (!sCounter) { + sCounter = new ProfilerCounterTotal("malloc", "Memory", + "Amount of allocated memory"); + // Also initialize the ThreadIntercept, even if native allocation tracking + // won't be turned on. This way the TLS will be initialized. + ThreadIntercept::Init(); + } else { + sCounter->Clear(); + } + jemalloc_replace_dynamic(replace_init); + return sCounter; +} + +// Remove the hooks, but leave the sCounter machinery. Deleting the counter +// would race with any existing memory hooks that are currently running. Rather +// than adding overhead here of mutexes it's cheaper for the performance to just +// leak these values. +void remove_memory_hooks() { jemalloc_replace_dynamic(nullptr); } + +void enable_native_allocations() { + // The bloat log tracks allocations and deallocations. This can conflict + // with the memory hook machinery, as the bloat log creates its own + // allocations. This means we can re-enter inside the bloat log machinery. At + // this time, the bloat log does not know about cannot handle the native + // allocation feature. + // + // At the time of this writing, we hit this assertion: + // IsIdle(oldState) || IsRead(oldState) in Checker::StartReadOp() + // + // #01: GetBloatEntry(char const*, unsigned int) + // #02: NS_LogCtor + // #03: profiler_get_backtrace() + // #04: profiler_add_native_allocation_marker(long long) + // #05: mozilla::profiler::AllocCallback(void*, unsigned long) + // #06: replace_calloc(unsigned long, unsigned long) + // #07: PLDHashTable::ChangeTable(int) + // #08: PLDHashTable::Add(void const*, std::nothrow_t const&) + // #09: nsBaseHashtable<nsDepCharHashKey, nsAutoPtr<BloatEntry>, ... + // #10: GetBloatEntry(char const*, unsigned int) + // #11: NS_LogCtor + // #12: profiler_get_backtrace() + // ... + MOZ_ASSERT(!PR_GetEnv("XPCOM_MEM_BLOAT_LOG"), + "The bloat log feature is not compatible with the native " + "allocations instrumentation."); + + EnsureBernoulliIsInstalled(); + EnsureAllocationTrackerIsInstalled(); + ThreadIntercept::EnableAllocationFeature(); +} + +// This is safe to call even if native allocations hasn't been enabled. +void disable_native_allocations() { + ThreadIntercept::DisableAllocationFeature(); + if (gAllocationTracker) { + gAllocationTracker->Reset(); + } +} + +} // namespace mozilla::profiler diff --git a/tools/profiler/core/memory_hooks.h b/tools/profiler/core/memory_hooks.h new file mode 100644 index 0000000000..a6ace771dd --- /dev/null +++ b/tools/profiler/core/memory_hooks.h @@ -0,0 +1,25 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef memory_hooks_h +#define memory_hooks_h + +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) +class BaseProfilerCount; + +namespace mozilla { +namespace profiler { + +BaseProfilerCount* install_memory_hooks(); +void remove_memory_hooks(); +void enable_native_allocations(); +void disable_native_allocations(); + +} // namespace profiler +} // namespace mozilla +#endif + +#endif diff --git a/tools/profiler/core/platform-linux-android.cpp b/tools/profiler/core/platform-linux-android.cpp new file mode 100644 index 0000000000..11af93456c --- /dev/null +++ b/tools/profiler/core/platform-linux-android.cpp @@ -0,0 +1,637 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +// Copyright (c) 2006-2011 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in +// the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google, Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +// AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +// SUCH DAMAGE. + +// This file is used for both Linux and Android as well as FreeBSD. + +#include <stdio.h> +#include <math.h> + +#include <pthread.h> +#if defined(GP_OS_freebsd) +# include <sys/thr.h> +#endif +#include <semaphore.h> +#include <signal.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <stdlib.h> +#include <sched.h> +#include <ucontext.h> +// Ubuntu Dapper requires memory pages to be marked as +// executable. Otherwise, OS raises an exception when executing code +// in that page. +#include <sys/types.h> // mmap & munmap +#include <sys/mman.h> // mmap & munmap +#include <sys/stat.h> // open +#include <fcntl.h> // open +#include <unistd.h> // sysconf +#include <semaphore.h> +#ifdef __GLIBC__ +# include <execinfo.h> // backtrace, backtrace_symbols +#endif // def __GLIBC__ +#include <strings.h> // index +#include <errno.h> +#include <stdarg.h> + +#include "prenv.h" +#include "mozilla/PodOperations.h" +#include "mozilla/DebugOnly.h" +#if defined(GP_OS_linux) || defined(GP_OS_android) +# include "common/linux/breakpad_getcontext.h" +#endif + +#include <string.h> +#include <list> + +using namespace mozilla; + +static void PopulateRegsFromContext(Registers& aRegs, ucontext_t* aContext) { + aRegs.mContext = aContext; + mcontext_t& mcontext = aContext->uc_mcontext; + + // Extracting the sample from the context is extremely machine dependent. +#if defined(GP_PLAT_x86_linux) || defined(GP_PLAT_x86_android) + aRegs.mPC = reinterpret_cast<Address>(mcontext.gregs[REG_EIP]); + aRegs.mSP = reinterpret_cast<Address>(mcontext.gregs[REG_ESP]); + aRegs.mFP = reinterpret_cast<Address>(mcontext.gregs[REG_EBP]); + aRegs.mEcx = reinterpret_cast<Address>(mcontext.gregs[REG_ECX]); + aRegs.mEdx = reinterpret_cast<Address>(mcontext.gregs[REG_EDX]); +#elif defined(GP_PLAT_amd64_linux) || defined(GP_PLAT_amd64_android) + aRegs.mPC = reinterpret_cast<Address>(mcontext.gregs[REG_RIP]); + aRegs.mSP = reinterpret_cast<Address>(mcontext.gregs[REG_RSP]); + aRegs.mFP = reinterpret_cast<Address>(mcontext.gregs[REG_RBP]); + aRegs.mR10 = reinterpret_cast<Address>(mcontext.gregs[REG_R10]); + aRegs.mR12 = reinterpret_cast<Address>(mcontext.gregs[REG_R12]); +#elif defined(GP_PLAT_amd64_freebsd) + aRegs.mPC = reinterpret_cast<Address>(mcontext.mc_rip); + aRegs.mSP = reinterpret_cast<Address>(mcontext.mc_rsp); + aRegs.mFP = reinterpret_cast<Address>(mcontext.mc_rbp); + aRegs.mR10 = reinterpret_cast<Address>(mcontext.mc_r10); + aRegs.mR12 = reinterpret_cast<Address>(mcontext.mc_r12); +#elif defined(GP_PLAT_arm_linux) || defined(GP_PLAT_arm_android) + aRegs.mPC = reinterpret_cast<Address>(mcontext.arm_pc); + aRegs.mSP = reinterpret_cast<Address>(mcontext.arm_sp); + aRegs.mFP = reinterpret_cast<Address>(mcontext.arm_fp); + aRegs.mLR = reinterpret_cast<Address>(mcontext.arm_lr); + aRegs.mR7 = reinterpret_cast<Address>(mcontext.arm_r7); +#elif defined(GP_PLAT_arm64_linux) || defined(GP_PLAT_arm64_android) + aRegs.mPC = reinterpret_cast<Address>(mcontext.pc); + aRegs.mSP = reinterpret_cast<Address>(mcontext.sp); + aRegs.mFP = reinterpret_cast<Address>(mcontext.regs[29]); + aRegs.mLR = reinterpret_cast<Address>(mcontext.regs[30]); + aRegs.mR11 = reinterpret_cast<Address>(mcontext.regs[11]); +#elif defined(GP_PLAT_arm64_freebsd) + aRegs.mPC = reinterpret_cast<Address>(mcontext.mc_gpregs.gp_elr); + aRegs.mSP = reinterpret_cast<Address>(mcontext.mc_gpregs.gp_sp); + aRegs.mFP = reinterpret_cast<Address>(mcontext.mc_gpregs.gp_x[29]); + aRegs.mLR = reinterpret_cast<Address>(mcontext.mc_gpregs.gp_lr); + aRegs.mR11 = reinterpret_cast<Address>(mcontext.mc_gpregs.gp_x[11]); +#elif defined(GP_PLAT_mips64_linux) || defined(GP_PLAT_mips64_android) + aRegs.mPC = reinterpret_cast<Address>(mcontext.pc); + aRegs.mSP = reinterpret_cast<Address>(mcontext.gregs[29]); + aRegs.mFP = reinterpret_cast<Address>(mcontext.gregs[30]); + +#else +# error "bad platform" +#endif +} + +#if defined(GP_OS_android) +# define SYS_tgkill __NR_tgkill +#endif + +#if defined(GP_OS_linux) || defined(GP_OS_android) +int tgkill(pid_t tgid, pid_t tid, int signalno) { + return syscall(SYS_tgkill, tgid, tid, signalno); +} +#endif + +#if defined(GP_OS_freebsd) +# define tgkill thr_kill2 +#endif + +mozilla::profiler::PlatformData::PlatformData(ProfilerThreadId aThreadId) { + MOZ_ASSERT(aThreadId == profiler_current_thread_id()); + if (clockid_t clockid; pthread_getcpuclockid(pthread_self(), &clockid) == 0) { + mClockId = Some(clockid); + } +} + +mozilla::profiler::PlatformData::~PlatformData() = default; + +//////////////////////////////////////////////////////////////////////// +// BEGIN Sampler target specifics + +// The only way to reliably interrupt a Linux thread and inspect its register +// and stack state is by sending a signal to it, and doing the work inside the +// signal handler. But we don't want to run much code inside the signal +// handler, since POSIX severely restricts what we can do in signal handlers. +// So we use a system of semaphores to suspend the thread and allow the +// sampler thread to do all the work of unwinding and copying out whatever +// data it wants. +// +// A four-message protocol is used to reliably suspend and later resume the +// thread to be sampled (the samplee): +// +// Sampler (signal sender) thread Samplee (thread to be sampled) +// +// Prepare the SigHandlerCoordinator +// and point sSigHandlerCoordinator at it +// +// send SIGPROF to samplee ------- MSG 1 ----> (enter signal handler) +// wait(mMessage2) Copy register state +// into sSigHandlerCoordinator +// <------ MSG 2 ----- post(mMessage2) +// Samplee is now suspended. wait(mMessage3) +// Examine its stack/register +// state at leisure +// +// Release samplee: +// post(mMessage3) ------- MSG 3 -----> +// wait(mMessage4) Samplee now resumes. Tell +// the sampler that we are done. +// <------ MSG 4 ------ post(mMessage4) +// Now we know the samplee's signal (leave signal handler) +// handler has finished using +// sSigHandlerCoordinator. We can +// safely reuse it for some other thread. +// + +// A type used to coordinate between the sampler (signal sending) thread and +// the thread currently being sampled (the samplee, which receives the +// signals). +// +// The first message is sent using a SIGPROF signal delivery. The subsequent +// three are sent using sem_wait/sem_post pairs. They are named accordingly +// in the following struct. +struct SigHandlerCoordinator { + SigHandlerCoordinator() { + PodZero(&mUContext); + int r = sem_init(&mMessage2, /* pshared */ 0, 0); + r |= sem_init(&mMessage3, /* pshared */ 0, 0); + r |= sem_init(&mMessage4, /* pshared */ 0, 0); + MOZ_ASSERT(r == 0); + (void)r; + } + + ~SigHandlerCoordinator() { + int r = sem_destroy(&mMessage2); + r |= sem_destroy(&mMessage3); + r |= sem_destroy(&mMessage4); + MOZ_ASSERT(r == 0); + (void)r; + } + + sem_t mMessage2; // To sampler: "context is in sSigHandlerCoordinator" + sem_t mMessage3; // To samplee: "resume" + sem_t mMessage4; // To sampler: "finished with sSigHandlerCoordinator" + ucontext_t mUContext; // Context at signal +}; + +struct SigHandlerCoordinator* Sampler::sSigHandlerCoordinator = nullptr; + +static void SigprofHandler(int aSignal, siginfo_t* aInfo, void* aContext) { + // Avoid TSan warning about clobbering errno. + int savedErrno = errno; + + MOZ_ASSERT(aSignal == SIGPROF); + MOZ_ASSERT(Sampler::sSigHandlerCoordinator); + + // By sending us this signal, the sampler thread has sent us message 1 in + // the comment above, with the meaning "|sSigHandlerCoordinator| is ready + // for use, please copy your register context into it." + Sampler::sSigHandlerCoordinator->mUContext = + *static_cast<ucontext_t*>(aContext); + + // Send message 2: tell the sampler thread that the context has been copied + // into |sSigHandlerCoordinator->mUContext|. sem_post can never fail by + // being interrupted by a signal, so there's no loop around this call. + int r = sem_post(&Sampler::sSigHandlerCoordinator->mMessage2); + MOZ_ASSERT(r == 0); + + // At this point, the sampler thread assumes we are suspended, so we must + // not touch any global state here. + + // Wait for message 3: the sampler thread tells us to resume. + while (true) { + r = sem_wait(&Sampler::sSigHandlerCoordinator->mMessage3); + if (r == -1 && errno == EINTR) { + // Interrupted by a signal. Try again. + continue; + } + // We don't expect any other kind of failure + MOZ_ASSERT(r == 0); + break; + } + + // Send message 4: tell the sampler thread that we are finished accessing + // |sSigHandlerCoordinator|. After this point it is not safe to touch + // |sSigHandlerCoordinator|. + r = sem_post(&Sampler::sSigHandlerCoordinator->mMessage4); + MOZ_ASSERT(r == 0); + + errno = savedErrno; +} + +Sampler::Sampler(PSLockRef aLock) : mMyPid(profiler_current_process_id()) { +#if defined(USE_EHABI_STACKWALK) + mozilla::EHABIStackWalkInit(); +#endif + + // NOTE: We don't initialize LUL here, instead initializing it in + // SamplerThread's constructor. This is because with the + // profiler_suspend_and_sample_thread entry point, we want to be able to + // sample without waiting for LUL to be initialized. + + // Request profiling signals. + struct sigaction sa; + sa.sa_sigaction = SigprofHandler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART | SA_SIGINFO; + if (sigaction(SIGPROF, &sa, &mOldSigprofHandler) != 0) { + MOZ_CRASH("Error installing SIGPROF handler in the profiler"); + } +} + +void Sampler::Disable(PSLockRef aLock) { + // Restore old signal handler. This is global state so it's important that + // we do it now, while gPSMutex is locked. + sigaction(SIGPROF, &mOldSigprofHandler, 0); +} + +static void StreamMetaPlatformSampleUnits(PSLockRef aLock, + SpliceableJSONWriter& aWriter) { + aWriter.StringProperty("threadCPUDelta", "ns"); +} + +/* static */ +uint64_t RunningTimes::ConvertRawToJson(uint64_t aRawValue) { + return aRawValue; +} + +namespace mozilla::profiler { +bool GetCpuTimeSinceThreadStartInNs( + uint64_t* aResult, const mozilla::profiler::PlatformData& aPlatformData) { + Maybe<clockid_t> maybeCid = aPlatformData.GetClockId(); + if (MOZ_UNLIKELY(!maybeCid)) { + return false; + } + + timespec t; + if (clock_gettime(*maybeCid, &t) != 0) { + return false; + } + + *aResult = uint64_t(t.tv_sec) * 1'000'000'000u + uint64_t(t.tv_nsec); + return true; +} +} // namespace mozilla::profiler + +static RunningTimes GetProcessRunningTimesDiff( + PSLockRef aLock, RunningTimes& aPreviousRunningTimesToBeUpdated) { + AUTO_PROFILER_STATS(GetProcessRunningTimes); + + RunningTimes newRunningTimes; + { + AUTO_PROFILER_STATS(GetProcessRunningTimes_clock_gettime); + if (timespec ts; clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts) == 0) { + newRunningTimes.SetThreadCPUDelta(uint64_t(ts.tv_sec) * 1'000'000'000u + + uint64_t(ts.tv_nsec)); + } + newRunningTimes.SetPostMeasurementTimeStamp(TimeStamp::Now()); + }; + + const RunningTimes diff = newRunningTimes - aPreviousRunningTimesToBeUpdated; + aPreviousRunningTimesToBeUpdated = newRunningTimes; + return diff; +} + +static RunningTimes GetThreadRunningTimesDiff( + PSLockRef aLock, + ThreadRegistration::UnlockedRWForLockedProfiler& aThreadData) { + AUTO_PROFILER_STATS(GetRunningTimes_clock_gettime_thread); + + const mozilla::profiler::PlatformData& platformData = + aThreadData.PlatformDataCRef(); + Maybe<clockid_t> maybeCid = platformData.GetClockId(); + + if (MOZ_UNLIKELY(!maybeCid)) { + // No clock id -> Nothing to measure apart from the timestamp. + RunningTimes emptyRunningTimes; + emptyRunningTimes.SetPostMeasurementTimeStamp(TimeStamp::Now()); + return emptyRunningTimes; + } + + const RunningTimes newRunningTimes = GetRunningTimesWithTightTimestamp( + [cid = *maybeCid](RunningTimes& aRunningTimes) { + AUTO_PROFILER_STATS(GetRunningTimes_clock_gettime); + if (timespec ts; clock_gettime(cid, &ts) == 0) { + aRunningTimes.ResetThreadCPUDelta( + uint64_t(ts.tv_sec) * 1'000'000'000u + uint64_t(ts.tv_nsec)); + } else { + aRunningTimes.ClearThreadCPUDelta(); + } + }); + + ProfiledThreadData* profiledThreadData = + aThreadData.GetProfiledThreadData(aLock); + MOZ_ASSERT(profiledThreadData); + RunningTimes& previousRunningTimes = + profiledThreadData->PreviousThreadRunningTimesRef(); + const RunningTimes diff = newRunningTimes - previousRunningTimes; + previousRunningTimes = newRunningTimes; + return diff; +} + +static void DiscardSuspendedThreadRunningTimes( + PSLockRef aLock, + ThreadRegistration::UnlockedRWForLockedProfiler& aThreadData) { + AUTO_PROFILER_STATS(DiscardSuspendedThreadRunningTimes); + + // On Linux, suspending a thread uses a signal that makes that thread work + // to handle it. So we want to discard any added running time since the call + // to GetThreadRunningTimesDiff, which is done by overwriting the thread's + // PreviousThreadRunningTimesRef() with the current running time now. + + const mozilla::profiler::PlatformData& platformData = + aThreadData.PlatformDataCRef(); + Maybe<clockid_t> maybeCid = platformData.GetClockId(); + + if (MOZ_UNLIKELY(!maybeCid)) { + // No clock id -> Nothing to measure. + return; + } + + ProfiledThreadData* profiledThreadData = + aThreadData.GetProfiledThreadData(aLock); + MOZ_ASSERT(profiledThreadData); + RunningTimes& previousRunningTimes = + profiledThreadData->PreviousThreadRunningTimesRef(); + + if (timespec ts; clock_gettime(*maybeCid, &ts) == 0) { + previousRunningTimes.ResetThreadCPUDelta( + uint64_t(ts.tv_sec) * 1'000'000'000u + uint64_t(ts.tv_nsec)); + } else { + previousRunningTimes.ClearThreadCPUDelta(); + } +} + +template <typename Func> +void Sampler::SuspendAndSampleAndResumeThread( + PSLockRef aLock, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const TimeStamp& aNow, const Func& aProcessRegs) { + // Only one sampler thread can be sampling at once. So we expect to have + // complete control over |sSigHandlerCoordinator|. + MOZ_ASSERT(!sSigHandlerCoordinator); + + if (!mSamplerTid.IsSpecified()) { + mSamplerTid = profiler_current_thread_id(); + } + ProfilerThreadId sampleeTid = aThreadData.Info().ThreadId(); + MOZ_RELEASE_ASSERT(sampleeTid != mSamplerTid); + + //----------------------------------------------------------------// + // Suspend the samplee thread and get its context. + + SigHandlerCoordinator coord; // on sampler thread's stack + sSigHandlerCoordinator = &coord; + + // Send message 1 to the samplee (the thread to be sampled), by + // signalling at it. + // This could fail if the thread doesn't exist anymore. + int r = tgkill(mMyPid.ToNumber(), sampleeTid.ToNumber(), SIGPROF); + if (r == 0) { + // Wait for message 2 from the samplee, indicating that the context + // is available and that the thread is suspended. + while (true) { + r = sem_wait(&sSigHandlerCoordinator->mMessage2); + if (r == -1 && errno == EINTR) { + // Interrupted by a signal. Try again. + continue; + } + // We don't expect any other kind of failure. + MOZ_ASSERT(r == 0); + break; + } + + //----------------------------------------------------------------// + // Sample the target thread. + + // WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING + // + // The profiler's "critical section" begins here. In the critical section, + // we must not do any dynamic memory allocation, nor try to acquire any lock + // or any other unshareable resource. This is because the thread to be + // sampled has been suspended at some entirely arbitrary point, and we have + // no idea which unsharable resources (locks, essentially) it holds. So any + // attempt to acquire any lock, including the implied locks used by the + // malloc implementation, risks deadlock. This includes TimeStamp::Now(), + // which gets a lock on Windows. + + // The samplee thread is now frozen and sSigHandlerCoordinator->mUContext is + // valid. We can poke around in it and unwind its stack as we like. + + // Extract the current register values. + Registers regs; + PopulateRegsFromContext(regs, &sSigHandlerCoordinator->mUContext); + aProcessRegs(regs, aNow); + + //----------------------------------------------------------------// + // Resume the target thread. + + // Send message 3 to the samplee, which tells it to resume. + r = sem_post(&sSigHandlerCoordinator->mMessage3); + MOZ_ASSERT(r == 0); + + // Wait for message 4 from the samplee, which tells us that it has + // finished with |sSigHandlerCoordinator|. + while (true) { + r = sem_wait(&sSigHandlerCoordinator->mMessage4); + if (r == -1 && errno == EINTR) { + continue; + } + MOZ_ASSERT(r == 0); + break; + } + + // The profiler's critical section ends here. After this point, none of the + // critical section limitations documented above apply. + // + // WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING + } + + // This isn't strictly necessary, but doing so does help pick up anomalies + // in which the signal handler is running when it shouldn't be. + sSigHandlerCoordinator = nullptr; +} + +// END Sampler target specifics +//////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////// +// BEGIN SamplerThread target specifics + +static void* ThreadEntry(void* aArg) { + auto thread = static_cast<SamplerThread*>(aArg); + thread->Run(); + return nullptr; +} + +SamplerThread::SamplerThread(PSLockRef aLock, uint32_t aActivityGeneration, + double aIntervalMilliseconds, uint32_t aFeatures) + : mSampler(aLock), + mActivityGeneration(aActivityGeneration), + mIntervalMicroseconds( + std::max(1, int(floor(aIntervalMilliseconds * 1000 + 0.5)))) { +#if defined(USE_LUL_STACKWALK) + lul::LUL* lul = CorePS::Lul(); + if (!lul && ProfilerFeature::HasStackWalk(aFeatures)) { + CorePS::SetLul(MakeUnique<lul::LUL>(logging_sink_for_LUL)); + // Read all the unwind info currently available. + lul = CorePS::Lul(); + read_procmaps(lul); + + // Switch into unwind mode. After this point, we can't add or remove any + // unwind info to/from this LUL instance. The only thing we can do with + // it is Unwind() calls. + lul->EnableUnwinding(); + + // Has a test been requested? + if (PR_GetEnv("MOZ_PROFILER_LUL_TEST")) { + int nTests = 0, nTestsPassed = 0; + RunLulUnitTests(&nTests, &nTestsPassed, lul); + } + } +#endif + + // Start the sampling thread. It repeatedly sends a SIGPROF signal. Sending + // the signal ourselves instead of relying on itimer provides much better + // accuracy. + // + // At least 350 KiB of stack space are needed when built with TSAN. This + // includes lul::N_STACK_BYTES plus whatever else is needed for the sampler + // thread. Set the stack size to 800 KiB to keep a safe margin above that. + pthread_attr_t attr; + if (pthread_attr_init(&attr) != 0 || + pthread_attr_setstacksize(&attr, 800 * 1024) != 0 || + pthread_create(&mThread, &attr, ThreadEntry, this) != 0) { + MOZ_CRASH("pthread_create failed"); + } + pthread_attr_destroy(&attr); +} + +SamplerThread::~SamplerThread() { + pthread_join(mThread, nullptr); + // Just in the unlikely case some callbacks were added between the end of the + // thread and now. + InvokePostSamplingCallbacks(std::move(mPostSamplingCallbackList), + SamplingState::JustStopped); +} + +void SamplerThread::SleepMicro(uint32_t aMicroseconds) { + if (aMicroseconds >= 1000000) { + // Use usleep for larger intervals, because the nanosleep + // code below only supports intervals < 1 second. + MOZ_ALWAYS_TRUE(!::usleep(aMicroseconds)); + return; + } + + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = aMicroseconds * 1000UL; + + int rv = ::nanosleep(&ts, &ts); + + while (rv != 0 && errno == EINTR) { + // Keep waiting in case of interrupt. + // nanosleep puts the remaining time back into ts. + rv = ::nanosleep(&ts, &ts); + } + + MOZ_ASSERT(!rv, "nanosleep call failed"); +} + +void SamplerThread::Stop(PSLockRef aLock) { + // Restore old signal handler. This is global state so it's important that + // we do it now, while gPSMutex is locked. It's safe to do this now even + // though this SamplerThread is still alive, because the next time the main + // loop of Run() iterates it won't get past the mActivityGeneration check, + // and so won't send any signals. + mSampler.Disable(aLock); +} + +// END SamplerThread target specifics +//////////////////////////////////////////////////////////////////////// + +#if defined(GP_OS_linux) || defined(GP_OS_freebsd) + +// We use pthread_atfork() to temporarily disable signal delivery during any +// fork() call. Without that, fork() can be repeatedly interrupted by signal +// delivery, requiring it to be repeatedly restarted, which can lead to *long* +// delays. See bug 837390. +// +// We provide no paf_child() function to run in the child after forking. This +// is fine because we always immediately exec() after fork(), and exec() +// clobbers all process state. Also, we don't want the sampler to resume in the +// child process between fork() and exec(), it would be wasteful. +// +// Unfortunately all this is only doable on non-Android because Bionic doesn't +// have pthread_atfork. + +// In the parent, before the fork, increase gSkipSampling to ensure that +// profiler sampling loops will be skipped. There could be one in progress now, +// causing a small delay, but further sampling will be skipped, allowing `fork` +// to complete. +static void paf_prepare() { ++gSkipSampling; } + +// In the parent, after the fork, decrease gSkipSampling to let the sampler +// resume sampling (unless other places have made it non-zero as well). +static void paf_parent() { --gSkipSampling; } + +static void PlatformInit(PSLockRef aLock) { + // Set up the fork handlers. + pthread_atfork(paf_prepare, paf_parent, nullptr); +} + +#else + +static void PlatformInit(PSLockRef aLock) {} + +#endif + +#if defined(HAVE_NATIVE_UNWIND) +# define REGISTERS_SYNC_POPULATE(regs) \ + if (!getcontext(®s.mContextSyncStorage)) { \ + PopulateRegsFromContext(regs, ®s.mContextSyncStorage); \ + } +#endif diff --git a/tools/profiler/core/platform-macos.cpp b/tools/profiler/core/platform-macos.cpp new file mode 100644 index 0000000000..356d9f803e --- /dev/null +++ b/tools/profiler/core/platform-macos.cpp @@ -0,0 +1,298 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <unistd.h> +#include <sys/mman.h> +#include <mach/mach_init.h> +#include <mach-o/getsect.h> + +#include <AvailabilityMacros.h> + +#include <pthread.h> +#include <semaphore.h> +#include <signal.h> +#include <libkern/OSAtomic.h> +#include <libproc.h> +#include <mach/mach.h> +#include <mach/semaphore.h> +#include <mach/task.h> +#include <mach/thread_act.h> +#include <mach/vm_statistics.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/syscall.h> +#include <sys/types.h> +#include <sys/sysctl.h> +#include <stdarg.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <math.h> + +// this port is based off of v8 svn revision 9837 + +mozilla::profiler::PlatformData::PlatformData(ProfilerThreadId aThreadId) + : mProfiledThread(mach_thread_self()) {} + +mozilla::profiler::PlatformData::~PlatformData() { + // Deallocate Mach port for thread. + mach_port_deallocate(mach_task_self(), mProfiledThread); +} + +//////////////////////////////////////////////////////////////////////// +// BEGIN Sampler target specifics + +Sampler::Sampler(PSLockRef aLock) {} + +void Sampler::Disable(PSLockRef aLock) {} + +static void StreamMetaPlatformSampleUnits(PSLockRef aLock, + SpliceableJSONWriter& aWriter) { + // Microseconds. + aWriter.StringProperty("threadCPUDelta", "\u00B5s"); +} + +/* static */ +uint64_t RunningTimes::ConvertRawToJson(uint64_t aRawValue) { + return aRawValue; +} + +namespace mozilla::profiler { +bool GetCpuTimeSinceThreadStartInNs( + uint64_t* aResult, const mozilla::profiler::PlatformData& aPlatformData) { + thread_extended_info_data_t threadInfoData; + mach_msg_type_number_t count = THREAD_EXTENDED_INFO_COUNT; + if (thread_info(aPlatformData.ProfiledThread(), THREAD_EXTENDED_INFO, + (thread_info_t)&threadInfoData, &count) != KERN_SUCCESS) { + return false; + } + + *aResult = threadInfoData.pth_user_time + threadInfoData.pth_system_time; + return true; +} +} // namespace mozilla::profiler + +static RunningTimes GetProcessRunningTimesDiff( + PSLockRef aLock, RunningTimes& aPreviousRunningTimesToBeUpdated) { + AUTO_PROFILER_STATS(GetProcessRunningTimes); + + RunningTimes newRunningTimes; + { + AUTO_PROFILER_STATS(GetProcessRunningTimes_task_info); + + static const auto pid = getpid(); + struct proc_taskinfo pti; + if ((unsigned long)proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &pti, + PROC_PIDTASKINFO_SIZE) >= + PROC_PIDTASKINFO_SIZE) { + newRunningTimes.SetThreadCPUDelta(pti.pti_total_user + + pti.pti_total_system); + } + newRunningTimes.SetPostMeasurementTimeStamp(TimeStamp::Now()); + }; + + const RunningTimes diff = newRunningTimes - aPreviousRunningTimesToBeUpdated; + aPreviousRunningTimesToBeUpdated = newRunningTimes; + return diff; +} + +static RunningTimes GetThreadRunningTimesDiff( + PSLockRef aLock, + ThreadRegistration::UnlockedRWForLockedProfiler& aThreadData) { + AUTO_PROFILER_STATS(GetRunningTimes); + + const mozilla::profiler::PlatformData& platformData = + aThreadData.PlatformDataCRef(); + + const RunningTimes newRunningTimes = GetRunningTimesWithTightTimestamp( + [&platformData](RunningTimes& aRunningTimes) { + AUTO_PROFILER_STATS(GetRunningTimes_thread_info); + thread_basic_info_data_t threadBasicInfo; + mach_msg_type_number_t basicCount = THREAD_BASIC_INFO_COUNT; + if (thread_info(platformData.ProfiledThread(), THREAD_BASIC_INFO, + reinterpret_cast<thread_info_t>(&threadBasicInfo), + &basicCount) == KERN_SUCCESS && + basicCount == THREAD_BASIC_INFO_COUNT) { + uint64_t userTimeUs = + uint64_t(threadBasicInfo.user_time.seconds) * + uint64_t(USEC_PER_SEC) + + uint64_t(threadBasicInfo.user_time.microseconds); + uint64_t systemTimeUs = + uint64_t(threadBasicInfo.system_time.seconds) * + uint64_t(USEC_PER_SEC) + + uint64_t(threadBasicInfo.system_time.microseconds); + aRunningTimes.ResetThreadCPUDelta(userTimeUs + systemTimeUs); + } else { + aRunningTimes.ClearThreadCPUDelta(); + } + }); + + ProfiledThreadData* profiledThreadData = + aThreadData.GetProfiledThreadData(aLock); + MOZ_ASSERT(profiledThreadData); + RunningTimes& previousRunningTimes = + profiledThreadData->PreviousThreadRunningTimesRef(); + const RunningTimes diff = newRunningTimes - previousRunningTimes; + previousRunningTimes = newRunningTimes; + return diff; +} + +static void DiscardSuspendedThreadRunningTimes( + PSLockRef aLock, + ThreadRegistration::UnlockedRWForLockedProfiler& aThreadData) { + // Nothing to do! + // On macOS, suspending a thread doesn't make that thread work. +} + +template <typename Func> +void Sampler::SuspendAndSampleAndResumeThread( + PSLockRef aLock, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const TimeStamp& aNow, const Func& aProcessRegs) { + thread_act_t samplee_thread = aThreadData.PlatformDataCRef().ProfiledThread(); + + //----------------------------------------------------------------// + // Suspend the samplee thread and get its context. + + // We're using thread_suspend on OS X because pthread_kill (which is what we + // at one time used on Linux) has less consistent performance and causes + // strange crashes, see bug 1166778 and bug 1166808. thread_suspend + // is also just a lot simpler to use. + + if (KERN_SUCCESS != thread_suspend(samplee_thread)) { + return; + } + + //----------------------------------------------------------------// + // Sample the target thread. + + // WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING + // + // The profiler's "critical section" begins here. We must be very careful + // what we do here, or risk deadlock. See the corresponding comment in + // platform-linux-android.cpp for details. + +#if defined(__x86_64__) + thread_state_flavor_t flavor = x86_THREAD_STATE64; + x86_thread_state64_t state; + mach_msg_type_number_t count = x86_THREAD_STATE64_COUNT; +# if __DARWIN_UNIX03 +# define REGISTER_FIELD(name) __r##name +# else +# define REGISTER_FIELD(name) r##name +# endif // __DARWIN_UNIX03 +#elif defined(__aarch64__) + thread_state_flavor_t flavor = ARM_THREAD_STATE64; + arm_thread_state64_t state; + mach_msg_type_number_t count = ARM_THREAD_STATE64_COUNT; +# if __DARWIN_UNIX03 +# define REGISTER_FIELD(name) __##name +# else +# define REGISTER_FIELD(name) name +# endif // __DARWIN_UNIX03 +#else +# error "unknown architecture" +#endif + + if (thread_get_state(samplee_thread, flavor, + reinterpret_cast<natural_t*>(&state), + &count) == KERN_SUCCESS) { + Registers regs; +#if defined(__x86_64__) + regs.mPC = reinterpret_cast<Address>(state.REGISTER_FIELD(ip)); + regs.mSP = reinterpret_cast<Address>(state.REGISTER_FIELD(sp)); + regs.mFP = reinterpret_cast<Address>(state.REGISTER_FIELD(bp)); + regs.mR10 = reinterpret_cast<Address>(state.REGISTER_FIELD(10)); + regs.mR12 = reinterpret_cast<Address>(state.REGISTER_FIELD(12)); +#elif defined(__aarch64__) + regs.mPC = reinterpret_cast<Address>(state.REGISTER_FIELD(pc)); + regs.mSP = reinterpret_cast<Address>(state.REGISTER_FIELD(sp)); + regs.mFP = reinterpret_cast<Address>(state.REGISTER_FIELD(fp)); + regs.mLR = reinterpret_cast<Address>(state.REGISTER_FIELD(lr)); + regs.mR11 = reinterpret_cast<Address>(state.REGISTER_FIELD(x[11])); +#else +# error "unknown architecture" +#endif + + aProcessRegs(regs, aNow); + } + +#undef REGISTER_FIELD + + //----------------------------------------------------------------// + // Resume the target thread. + + thread_resume(samplee_thread); + + // The profiler's critical section ends here. + // + // WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING +} + +// END Sampler target specifics +//////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////// +// BEGIN SamplerThread target specifics + +static void* ThreadEntry(void* aArg) { + auto thread = static_cast<SamplerThread*>(aArg); + thread->Run(); + return nullptr; +} + +SamplerThread::SamplerThread(PSLockRef aLock, uint32_t aActivityGeneration, + double aIntervalMilliseconds, uint32_t aFeatures) + : mSampler(aLock), + mActivityGeneration(aActivityGeneration), + mIntervalMicroseconds( + std::max(1, int(floor(aIntervalMilliseconds * 1000 + 0.5)))), + mThread{nullptr} { + pthread_attr_t* attr_ptr = nullptr; + if (pthread_create(&mThread, attr_ptr, ThreadEntry, this) != 0) { + MOZ_CRASH("pthread_create failed"); + } +} + +SamplerThread::~SamplerThread() { + pthread_join(mThread, nullptr); + // Just in the unlikely case some callbacks were added between the end of the + // thread and now. + InvokePostSamplingCallbacks(std::move(mPostSamplingCallbackList), + SamplingState::JustStopped); +} + +void SamplerThread::SleepMicro(uint32_t aMicroseconds) { + usleep(aMicroseconds); + // FIXME: the OSX 10.12 page for usleep says "The usleep() function is + // obsolescent. Use nanosleep(2) instead." This implementation could be + // merged with the linux-android version. Also, this doesn't handle the + // case where the usleep call is interrupted by a signal. +} + +void SamplerThread::Stop(PSLockRef aLock) { mSampler.Disable(aLock); } + +// END SamplerThread target specifics +//////////////////////////////////////////////////////////////////////// + +static void PlatformInit(PSLockRef aLock) {} + +// clang-format off +#if defined(HAVE_NATIVE_UNWIND) +// Derive the stack pointer from the frame pointer. The 0x10 offset is +// 8 bytes for the previous frame pointer and 8 bytes for the return +// address both stored on the stack after at the beginning of the current +// frame. +# define REGISTERS_SYNC_POPULATE(regs) \ + regs.mSP = reinterpret_cast<Address>(__builtin_frame_address(0)) + 0x10; \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wframe-address\"") \ + regs.mFP = reinterpret_cast<Address>(__builtin_frame_address(1)); \ + _Pragma("GCC diagnostic pop") \ + regs.mPC = reinterpret_cast<Address>( \ + __builtin_extract_return_addr(__builtin_return_address(0))); +#endif +// clang-format on diff --git a/tools/profiler/core/platform-win32.cpp b/tools/profiler/core/platform-win32.cpp new file mode 100644 index 0000000000..0e5c1c9dbb --- /dev/null +++ b/tools/profiler/core/platform-win32.cpp @@ -0,0 +1,483 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +// Copyright (c) 2006-2011 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in +// the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google, Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +// AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +// SUCH DAMAGE. + +#include <windows.h> +#include <mmsystem.h> +#include <process.h> + +#include "mozilla/WindowsVersion.h" + +#include <type_traits> + +static void PopulateRegsFromContext(Registers& aRegs, CONTEXT* aContext) { +#if defined(GP_ARCH_amd64) + aRegs.mPC = reinterpret_cast<Address>(aContext->Rip); + aRegs.mSP = reinterpret_cast<Address>(aContext->Rsp); + aRegs.mFP = reinterpret_cast<Address>(aContext->Rbp); + aRegs.mR10 = reinterpret_cast<Address>(aContext->R10); + aRegs.mR12 = reinterpret_cast<Address>(aContext->R12); +#elif defined(GP_ARCH_x86) + aRegs.mPC = reinterpret_cast<Address>(aContext->Eip); + aRegs.mSP = reinterpret_cast<Address>(aContext->Esp); + aRegs.mFP = reinterpret_cast<Address>(aContext->Ebp); + aRegs.mEcx = reinterpret_cast<Address>(aContext->Ecx); + aRegs.mEdx = reinterpret_cast<Address>(aContext->Edx); +#elif defined(GP_ARCH_arm64) + aRegs.mPC = reinterpret_cast<Address>(aContext->Pc); + aRegs.mSP = reinterpret_cast<Address>(aContext->Sp); + aRegs.mFP = reinterpret_cast<Address>(aContext->Fp); + aRegs.mLR = reinterpret_cast<Address>(aContext->Lr); + aRegs.mR11 = reinterpret_cast<Address>(aContext->X11); +#else +# error "bad arch" +#endif +} + +// Gets a real (i.e. not pseudo) handle for the current thread, with the +// permissions needed for profiling. +// @return a real HANDLE for the current thread. +static HANDLE GetRealCurrentThreadHandleForProfiling() { + HANDLE realCurrentThreadHandle; + if (!::DuplicateHandle( + ::GetCurrentProcess(), ::GetCurrentThread(), ::GetCurrentProcess(), + &realCurrentThreadHandle, + THREAD_GET_CONTEXT | THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, + FALSE, 0)) { + return nullptr; + } + + return realCurrentThreadHandle; +} + +static_assert( + std::is_same_v<mozilla::profiler::PlatformData::WindowsHandle, HANDLE>); + +mozilla::profiler::PlatformData::PlatformData(ProfilerThreadId aThreadId) + : mProfiledThread(GetRealCurrentThreadHandleForProfiling()) { + MOZ_ASSERT(aThreadId == ProfilerThreadId::FromNumber(::GetCurrentThreadId())); +} + +mozilla::profiler::PlatformData::~PlatformData() { + if (mProfiledThread) { + CloseHandle(mProfiledThread); + mProfiledThread = nullptr; + } +} + +static const HANDLE kNoThread = INVALID_HANDLE_VALUE; + +//////////////////////////////////////////////////////////////////////// +// BEGIN Sampler target specifics + +Sampler::Sampler(PSLockRef aLock) {} + +void Sampler::Disable(PSLockRef aLock) {} + +static void StreamMetaPlatformSampleUnits(PSLockRef aLock, + SpliceableJSONWriter& aWriter) { + static const Span<const char> units = + (GetCycleTimeFrequencyMHz() != 0) ? MakeStringSpan("ns") + : MakeStringSpan("variable CPU cycles"); + aWriter.StringProperty("threadCPUDelta", units); +} + +/* static */ +uint64_t RunningTimes::ConvertRawToJson(uint64_t aRawValue) { + static const uint64_t cycleTimeFrequencyMHz = GetCycleTimeFrequencyMHz(); + if (cycleTimeFrequencyMHz == 0u) { + return aRawValue; + } + + constexpr uint64_t GHZ_PER_MHZ = 1'000u; + // To get ns, we need to divide cycles by a frequency in GHz, i.e.: + // cycles / (f_MHz / GHZ_PER_MHZ). To avoid losing the integer precision of + // f_MHz, this is computed as (cycles * GHZ_PER_MHZ) / f_MHz. + // Adding GHZ_PER_MHZ/2 to (cycles * GHZ_PER_MHZ) will round to nearest when + // the result of the division is truncated. + return (aRawValue * GHZ_PER_MHZ + (GHZ_PER_MHZ / 2u)) / cycleTimeFrequencyMHz; +} + +static inline uint64_t ToNanoSeconds(const FILETIME& aFileTime) { + // FILETIME values are 100-nanoseconds units, converting + ULARGE_INTEGER usec = {{aFileTime.dwLowDateTime, aFileTime.dwHighDateTime}}; + return usec.QuadPart * 100; +} + +namespace mozilla::profiler { +bool GetCpuTimeSinceThreadStartInNs( + uint64_t* aResult, const mozilla::profiler::PlatformData& aPlatformData) { + const HANDLE profiledThread = aPlatformData.ProfiledThread(); + int frequencyInMHz = GetCycleTimeFrequencyMHz(); + if (frequencyInMHz) { + uint64_t cpuCycleCount; + if (!QueryThreadCycleTime(profiledThread, &cpuCycleCount)) { + return false; + } + + constexpr uint64_t USEC_PER_NSEC = 1000L; + *aResult = cpuCycleCount * USEC_PER_NSEC / frequencyInMHz; + return true; + } + + FILETIME createTime, exitTime, kernelTime, userTime; + if (!GetThreadTimes(profiledThread, &createTime, &exitTime, &kernelTime, + &userTime)) { + return false; + } + + *aResult = ToNanoSeconds(kernelTime) + ToNanoSeconds(userTime); + return true; +} +} // namespace mozilla::profiler + +static RunningTimes GetProcessRunningTimesDiff( + PSLockRef aLock, RunningTimes& aPreviousRunningTimesToBeUpdated) { + AUTO_PROFILER_STATS(GetProcessRunningTimes); + + static const HANDLE processHandle = GetCurrentProcess(); + + RunningTimes newRunningTimes; + { + AUTO_PROFILER_STATS(GetProcessRunningTimes_QueryProcessCycleTime); + if (ULONG64 cycles; QueryProcessCycleTime(processHandle, &cycles) != 0) { + newRunningTimes.SetThreadCPUDelta(cycles); + } + newRunningTimes.SetPostMeasurementTimeStamp(TimeStamp::Now()); + }; + + const RunningTimes diff = newRunningTimes - aPreviousRunningTimesToBeUpdated; + aPreviousRunningTimesToBeUpdated = newRunningTimes; + return diff; +} + +static RunningTimes GetThreadRunningTimesDiff( + PSLockRef aLock, + ThreadRegistration::UnlockedRWForLockedProfiler& aThreadData) { + AUTO_PROFILER_STATS(GetThreadRunningTimes); + + const mozilla::profiler::PlatformData& platformData = + aThreadData.PlatformDataCRef(); + const HANDLE profiledThread = platformData.ProfiledThread(); + + const RunningTimes newRunningTimes = GetRunningTimesWithTightTimestamp( + [profiledThread](RunningTimes& aRunningTimes) { + AUTO_PROFILER_STATS(GetThreadRunningTimes_QueryThreadCycleTime); + if (ULONG64 cycles; + QueryThreadCycleTime(profiledThread, &cycles) != 0) { + aRunningTimes.ResetThreadCPUDelta(cycles); + } else { + aRunningTimes.ClearThreadCPUDelta(); + } + }); + + ProfiledThreadData* profiledThreadData = + aThreadData.GetProfiledThreadData(aLock); + MOZ_ASSERT(profiledThreadData); + RunningTimes& previousRunningTimes = + profiledThreadData->PreviousThreadRunningTimesRef(); + const RunningTimes diff = newRunningTimes - previousRunningTimes; + previousRunningTimes = newRunningTimes; + return diff; +} + +static void DiscardSuspendedThreadRunningTimes( + PSLockRef aLock, + ThreadRegistration::UnlockedRWForLockedProfiler& aThreadData) { + AUTO_PROFILER_STATS(DiscardSuspendedThreadRunningTimes); + + // On Windows, suspending a thread makes that thread work a little bit. So we + // want to discard any added running time since the call to + // GetThreadRunningTimesDiff, which is done by overwriting the thread's + // PreviousThreadRunningTimesRef() with the current running time now. + + const mozilla::profiler::PlatformData& platformData = + aThreadData.PlatformDataCRef(); + const HANDLE profiledThread = platformData.ProfiledThread(); + + ProfiledThreadData* profiledThreadData = + aThreadData.GetProfiledThreadData(aLock); + MOZ_ASSERT(profiledThreadData); + RunningTimes& previousRunningTimes = + profiledThreadData->PreviousThreadRunningTimesRef(); + + if (ULONG64 cycles; QueryThreadCycleTime(profiledThread, &cycles) != 0) { + previousRunningTimes.ResetThreadCPUDelta(cycles); + } else { + previousRunningTimes.ClearThreadCPUDelta(); + } +} + +template <typename Func> +void Sampler::SuspendAndSampleAndResumeThread( + PSLockRef aLock, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const TimeStamp& aNow, const Func& aProcessRegs) { + HANDLE profiled_thread = aThreadData.PlatformDataCRef().ProfiledThread(); + if (profiled_thread == nullptr) { + return; + } + + // Context used for sampling the register state of the profiled thread. + CONTEXT context; + memset(&context, 0, sizeof(context)); + + //----------------------------------------------------------------// + // Suspend the samplee thread and get its context. + + static const DWORD kSuspendFailed = static_cast<DWORD>(-1); + if (SuspendThread(profiled_thread) == kSuspendFailed) { + return; + } + + // SuspendThread is asynchronous, so the thread may still be running. + // Call GetThreadContext first to ensure the thread is really suspended. + // See https://blogs.msdn.microsoft.com/oldnewthing/20150205-00/?p=44743. + + // Using only CONTEXT_CONTROL is faster but on 64-bit it causes crashes in + // RtlVirtualUnwind (see bug 1120126) so we set all the flags. +#if defined(GP_ARCH_amd64) + context.ContextFlags = CONTEXT_FULL; +#else + context.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER; +#endif + if (!GetThreadContext(profiled_thread, &context)) { + ResumeThread(profiled_thread); + return; + } + + //----------------------------------------------------------------// + // Sample the target thread. + + // WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING + // + // The profiler's "critical section" begins here. We must be very careful + // what we do here, or risk deadlock. See the corresponding comment in + // platform-linux-android.cpp for details. + + Registers regs; + PopulateRegsFromContext(regs, &context); + aProcessRegs(regs, aNow); + + //----------------------------------------------------------------// + // Resume the target thread. + + ResumeThread(profiled_thread); + + // The profiler's critical section ends here. + // + // WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING +} + +// END Sampler target specifics +//////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////// +// BEGIN SamplerThread target specifics + +static unsigned int __stdcall ThreadEntry(void* aArg) { + auto thread = static_cast<SamplerThread*>(aArg); + thread->Run(); + return 0; +} + +static unsigned int __stdcall UnregisteredThreadSpyEntry(void* aArg) { + auto thread = static_cast<SamplerThread*>(aArg); + thread->RunUnregisteredThreadSpy(); + return 0; +} + +SamplerThread::SamplerThread(PSLockRef aLock, uint32_t aActivityGeneration, + double aIntervalMilliseconds, uint32_t aFeatures) + : mSampler(aLock), + mActivityGeneration(aActivityGeneration), + mIntervalMicroseconds( + std::max(1, int(floor(aIntervalMilliseconds * 1000 + 0.5)))), + mNoTimerResolutionChange( + ProfilerFeature::HasNoTimerResolutionChange(aFeatures)) { + if ((!mNoTimerResolutionChange) && (mIntervalMicroseconds < 10 * 1000)) { + // By default the timer resolution (which tends to be 1/64Hz, around 16ms) + // is not changed. However, if the requested interval is sufficiently low, + // the resolution will be adjusted to match. Note that this affects all + // timers in Firefox, and could therefore hide issues while profiling. This + // change may be prevented with the "notimerresolutionchange" feature. + ::timeBeginPeriod(mIntervalMicroseconds / 1000); + } + + if (ProfilerFeature::HasUnregisteredThreads(aFeatures)) { + // Sampler&spy threads are not running yet, so it's safe to modify + // mSpyingState without locking the monitor. + mSpyingState = SpyingState::Spy_Initializing; + mUnregisteredThreadSpyThread = reinterpret_cast<HANDLE>( + _beginthreadex(nullptr, + /* stack_size */ 0, UnregisteredThreadSpyEntry, this, + /* initflag */ 0, nullptr)); + if (mUnregisteredThreadSpyThread == 0) { + MOZ_CRASH("_beginthreadex failed"); + } + } + + // Create a new thread. It is important to use _beginthreadex() instead of + // the Win32 function CreateThread(), because the CreateThread() does not + // initialize thread-specific structures in the C runtime library. + mThread = reinterpret_cast<HANDLE>(_beginthreadex(nullptr, + /* stack_size */ 0, + ThreadEntry, this, + /* initflag */ 0, nullptr)); + if (mThread == 0) { + MOZ_CRASH("_beginthreadex failed"); + } +} + +SamplerThread::~SamplerThread() { + if (mUnregisteredThreadSpyThread) { + { + // Make sure the spying thread is not actively working, because the win32 + // function it's using could deadlock with WaitForSingleObject below. + MonitorAutoLock spyingStateLock{mSpyingStateMonitor}; + while (mSpyingState != SpyingState::Spy_Waiting && + mSpyingState != SpyingState::SamplerToSpy_Start) { + spyingStateLock.Wait(); + } + + mSpyingState = SpyingState::MainToSpy_Shutdown; + spyingStateLock.NotifyAll(); + + do { + spyingStateLock.Wait(); + } while (mSpyingState != SpyingState::SpyToMain_ShuttingDown); + } + + WaitForSingleObject(mUnregisteredThreadSpyThread, INFINITE); + + // Close our own handle for the thread. + if (mUnregisteredThreadSpyThread != kNoThread) { + CloseHandle(mUnregisteredThreadSpyThread); + } + } + + WaitForSingleObject(mThread, INFINITE); + + // Close our own handle for the thread. + if (mThread != kNoThread) { + CloseHandle(mThread); + } + + // Just in the unlikely case some callbacks were added between the end of the + // thread and now. + InvokePostSamplingCallbacks(std::move(mPostSamplingCallbackList), + SamplingState::JustStopped); +} + +void SamplerThread::RunUnregisteredThreadSpy() { + // TODO: Consider registering this thread. + // Pros: Remove from list of unregistered threads; Not useful to profiling + // Firefox itself. + // Cons: Doesn't appear in the profile, so users may miss the expensive CPU + // cost of this work on Windows. + PR_SetCurrentThreadName("UnregisteredThreadSpy"); + + while (true) { + { + MonitorAutoLock spyingStateLock{mSpyingStateMonitor}; + // Either this is the first loop, or we're looping after working. + MOZ_ASSERT(mSpyingState == SpyingState::Spy_Initializing || + mSpyingState == SpyingState::Spy_Working); + + // Let everyone know we're waiting, and then wait. + mSpyingState = SpyingState::Spy_Waiting; + mSpyingStateMonitor.NotifyAll(); + do { + spyingStateLock.Wait(); + } while (mSpyingState == SpyingState::Spy_Waiting); + + if (mSpyingState == SpyingState::MainToSpy_Shutdown) { + mSpyingState = SpyingState::SpyToMain_ShuttingDown; + mSpyingStateMonitor.NotifyAll(); + break; + } + + MOZ_ASSERT(mSpyingState == SpyingState::SamplerToSpy_Start); + mSpyingState = SpyingState::Spy_Working; + } + + // Do the work without lock, so other threads can read the current state. + SpyOnUnregisteredThreads(); + } +} + +void SamplerThread::SleepMicro(uint32_t aMicroseconds) { + // For now, keep the old behaviour of minimum Sleep(1), even for + // smaller-than-usual sleeps after an overshoot, unless the user has + // explicitly opted into a sub-millisecond profiler interval. + if (mIntervalMicroseconds >= 1000) { + ::Sleep(std::max(1u, aMicroseconds / 1000)); + } else { + TimeStamp start = TimeStamp::Now(); + TimeStamp end = start + TimeDuration::FromMicroseconds(aMicroseconds); + + // First, sleep for as many whole milliseconds as possible. + if (aMicroseconds >= 1000) { + ::Sleep(aMicroseconds / 1000); + } + + // Then, spin until enough time has passed. + while (TimeStamp::Now() < end) { + YieldProcessor(); + } + } +} + +void SamplerThread::Stop(PSLockRef aLock) { + if ((!mNoTimerResolutionChange) && (mIntervalMicroseconds < 10 * 1000)) { + // Disable any timer resolution changes we've made. Do it now while + // gPSMutex is locked, i.e. before any other SamplerThread can be created + // and call ::timeBeginPeriod(). + // + // It's safe to do this now even though this SamplerThread is still alive, + // because the next time the main loop of Run() iterates it won't get past + // the mActivityGeneration check, and so it won't make any more ::Sleep() + // calls. + ::timeEndPeriod(mIntervalMicroseconds / 1000); + } + + mSampler.Disable(aLock); +} + +// END SamplerThread target specifics +//////////////////////////////////////////////////////////////////////// + +static void PlatformInit(PSLockRef aLock) {} + +#if defined(HAVE_NATIVE_UNWIND) +# define REGISTERS_SYNC_POPULATE(regs) \ + CONTEXT context; \ + RtlCaptureContext(&context); \ + PopulateRegsFromContext(regs, &context); +#endif diff --git a/tools/profiler/core/platform.cpp b/tools/profiler/core/platform.cpp new file mode 100644 index 0000000000..e4af84ed4c --- /dev/null +++ b/tools/profiler/core/platform.cpp @@ -0,0 +1,7246 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// There are three kinds of samples done by the profiler. +// +// - A "periodic" sample is the most complex kind. It is done in response to a +// timer while the profiler is active. It involves writing a stack trace plus +// a variety of other values (memory measurements, responsiveness +// measurements, markers, etc.) into the main ProfileBuffer. The sampling is +// done from off-thread, and so SuspendAndSampleAndResumeThread() is used to +// get the register values. +// +// - A "synchronous" sample is a simpler kind. It is done in response to an API +// call (profiler_get_backtrace()). It involves writing a stack trace and +// little else into a temporary ProfileBuffer, and wrapping that up in a +// ProfilerBacktrace that can be subsequently used in a marker. The sampling +// is done on-thread, and so REGISTERS_SYNC_POPULATE() is used to get the +// register values. +// +// - A "backtrace" sample is the simplest kind. It is done in response to an +// API call (profiler_suspend_and_sample_thread()). It involves getting a +// stack trace via a ProfilerStackCollector; it does not write to a +// ProfileBuffer. The sampling is done from off-thread, and so uses +// SuspendAndSampleAndResumeThread() to get the register values. + +#include "platform.h" + +#include "GeckoProfiler.h" +#include "GeckoProfilerReporter.h" +#include "PageInformation.h" +#include "PowerCounters.h" +#include "ProfileBuffer.h" +#include "ProfiledThreadData.h" +#include "ProfilerBacktrace.h" +#include "ProfilerChild.h" +#include "ProfilerCodeAddressService.h" +#include "ProfilerControl.h" +#include "ProfilerCPUFreq.h" +#include "ProfilerIOInterposeObserver.h" +#include "ProfilerParent.h" +#include "ProfilerRustBindings.h" +#include "mozilla/MozPromise.h" +#include "shared-libraries.h" +#include "VTuneProfiler.h" +#include "ETWTools.h" + +#include "js/ProfilingFrameIterator.h" +#include "memory_hooks.h" +#include "mozilla/ArrayUtils.h" +#include "mozilla/AutoProfilerLabel.h" +#include "mozilla/BaseAndGeckoProfilerDetail.h" +#include "mozilla/ExtensionPolicyService.h" +#include "mozilla/extensions/WebExtensionPolicy.h" +#include "mozilla/glean/GleanMetrics.h" +#include "mozilla/Monitor.h" +#include "mozilla/Preferences.h" +#include "mozilla/Printf.h" +#include "mozilla/ProcInfo.h" +#include "mozilla/ProfilerBufferSize.h" +#include "mozilla/ProfileBufferChunkManagerSingle.h" +#include "mozilla/ProfileBufferChunkManagerWithLocalLimit.h" +#include "mozilla/ProfileChunkedBuffer.h" +#include "mozilla/ProfilerBandwidthCounter.h" +#include "mozilla/SchedulerGroup.h" +#include "mozilla/Services.h" +#include "mozilla/StackWalk.h" +#include "mozilla/Try.h" +#ifdef XP_WIN +# include "mozilla/NativeNt.h" +# include "mozilla/StackWalkThread.h" +# include "mozilla/WindowsStackWalkInitialization.h" +#endif +#include "mozilla/StaticPtr.h" +#include "mozilla/ThreadLocal.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Vector.h" +#include "BaseProfiler.h" +#include "nsDirectoryServiceDefs.h" +#include "nsDirectoryServiceUtils.h" +#include "nsIDocShell.h" +#include "nsIHttpProtocolHandler.h" +#include "nsIObserverService.h" +#include "nsIPropertyBag2.h" +#include "nsIXULAppInfo.h" +#include "nsIXULRuntime.h" +#include "nsJSPrincipals.h" +#include "nsMemoryReporterManager.h" +#include "nsPIDOMWindow.h" +#include "nsProfilerStartParams.h" +#include "nsScriptSecurityManager.h" +#include "nsSystemInfo.h" +#include "nsThreadUtils.h" +#include "nsXULAppAPI.h" +#include "Tracing.h" +#include "prdtoa.h" +#include "prtime.h" + +#include <algorithm> +#include <errno.h> +#include <fstream> +#include <ostream> +#include <set> +#include <sstream> +#include <string_view> +#include <type_traits> + +#if defined(GP_OS_android) +# include "JavaExceptions.h" +# include "mozilla/java/GeckoJavaSamplerNatives.h" +# include "mozilla/jni/Refs.h" +#endif + +#if defined(GP_OS_darwin) +# include "nsCocoaFeatures.h" +#endif + +#if defined(GP_PLAT_amd64_darwin) +# include <cpuid.h> +#endif + +#if defined(GP_OS_windows) +# include <processthreadsapi.h> + +// GetThreadInformation is not available on Windows 7. +WINBASEAPI +BOOL WINAPI GetThreadInformation( + _In_ HANDLE hThread, _In_ THREAD_INFORMATION_CLASS ThreadInformationClass, + _Out_writes_bytes_(ThreadInformationSize) LPVOID ThreadInformation, + _In_ DWORD ThreadInformationSize); + +#endif + +// Win32 builds always have frame pointers, so FramePointerStackWalk() always +// works. +#if defined(GP_PLAT_x86_windows) +# define HAVE_NATIVE_UNWIND +# define USE_FRAME_POINTER_STACK_WALK +#endif + +// Win64 builds always omit frame pointers, so we use the slower +// MozStackWalk(), which works in that case. +#if defined(GP_PLAT_amd64_windows) +# define HAVE_NATIVE_UNWIND +# define USE_MOZ_STACK_WALK +#endif + +// AArch64 Win64 doesn't seem to use frame pointers, so we use the slower +// MozStackWalk(). +#if defined(GP_PLAT_arm64_windows) +# define HAVE_NATIVE_UNWIND +# define USE_MOZ_STACK_WALK +#endif + +// Mac builds use FramePointerStackWalk(). Even if we build without +// frame pointers, we'll still get useful stacks in system libraries +// because those always have frame pointers. +// We don't use MozStackWalk() on Mac. +#if defined(GP_OS_darwin) +# define HAVE_NATIVE_UNWIND +# define USE_FRAME_POINTER_STACK_WALK +#endif + +// Android builds use the ARM Exception Handling ABI to unwind. +#if defined(GP_PLAT_arm_linux) || defined(GP_PLAT_arm_android) +# define HAVE_NATIVE_UNWIND +# define USE_EHABI_STACKWALK +# include "EHABIStackWalk.h" +#endif + +// Linux/BSD builds use LUL, which uses DWARF info to unwind stacks. +#if defined(GP_PLAT_amd64_linux) || defined(GP_PLAT_x86_linux) || \ + defined(GP_PLAT_amd64_android) || defined(GP_PLAT_x86_android) || \ + defined(GP_PLAT_mips64_linux) || defined(GP_PLAT_arm64_linux) || \ + defined(GP_PLAT_arm64_android) || defined(GP_PLAT_amd64_freebsd) || \ + defined(GP_PLAT_arm64_freebsd) +# define HAVE_NATIVE_UNWIND +# define USE_LUL_STACKWALK +# include "lul/LulMain.h" +# include "lul/platform-linux-lul.h" + +// On linux we use LUL for periodic samples and synchronous samples, but we use +// FramePointerStackWalk for backtrace samples when MOZ_PROFILING is enabled. +// (See the comment at the top of the file for a definition of +// periodic/synchronous/backtrace.). +// +// FramePointerStackWalk can produce incomplete stacks when the current entry is +// in a shared library without framepointers, however LUL can take a long time +// to initialize, which is undesirable for consumers of +// profiler_suspend_and_sample_thread like the Background Hang Reporter. +# if defined(MOZ_PROFILING) +# define USE_FRAME_POINTER_STACK_WALK +# endif +#endif + +// We can only stackwalk without expensive initialization on platforms which +// support FramePointerStackWalk or MozStackWalk. LUL Stackwalking requires +// initializing LUL, and EHABIStackWalk requires initializing EHABI, both of +// which can be expensive. +#if defined(USE_FRAME_POINTER_STACK_WALK) || defined(USE_MOZ_STACK_WALK) +# define HAVE_FASTINIT_NATIVE_UNWIND +#endif + +#ifdef MOZ_VALGRIND +# include <valgrind/memcheck.h> +#else +# define VALGRIND_MAKE_MEM_DEFINED(_addr, _len) ((void)0) +#endif + +#if defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) +# include <ucontext.h> +#endif + +using namespace mozilla; +using namespace mozilla::literals::ProportionValue_literals; + +using mozilla::profiler::detail::RacyFeatures; +using ThreadRegistration = mozilla::profiler::ThreadRegistration; +using ThreadRegistrationInfo = mozilla::profiler::ThreadRegistrationInfo; +using ThreadRegistry = mozilla::profiler::ThreadRegistry; + +LazyLogModule gProfilerLog("prof"); + +ProfileChunkedBuffer& profiler_get_core_buffer() { + // Defer to the Base Profiler in mozglue to create the core buffer if needed, + // and keep a reference here, for quick access in xul. + static ProfileChunkedBuffer& sProfileChunkedBuffer = + baseprofiler::profiler_get_core_buffer(); + return sProfileChunkedBuffer; +} + +mozilla::Atomic<int, mozilla::MemoryOrdering::Relaxed> gSkipSampling; + +#if defined(GP_OS_android) +class GeckoJavaSampler + : public java::GeckoJavaSampler::Natives<GeckoJavaSampler> { + private: + GeckoJavaSampler(); + + public: + static double GetProfilerTime() { + if (!profiler_is_active()) { + return 0.0; + } + return profiler_time(); + }; + + static void JavaStringArrayToCharArray(jni::ObjectArray::Param& aJavaArray, + Vector<const char*>& aCharArray, + JNIEnv* aJni) { + int arraySize = aJavaArray->Length(); + for (int i = 0; i < arraySize; i++) { + jstring javaString = + (jstring)(aJni->GetObjectArrayElement(aJavaArray.Get(), i)); + const char* filterString = aJni->GetStringUTFChars(javaString, 0); + // FIXME. These strings are leaked. + MOZ_RELEASE_ASSERT(aCharArray.append(filterString)); + } + } + + static void StartProfiler(jni::ObjectArray::Param aFiltersArray, + jni::ObjectArray::Param aFeaturesArray) { + JNIEnv* jni = jni::GetEnvForThread(); + Vector<const char*> filtersTemp; + Vector<const char*> featureStringArray; + + JavaStringArrayToCharArray(aFiltersArray, filtersTemp, jni); + JavaStringArrayToCharArray(aFeaturesArray, featureStringArray, jni); + + uint32_t features = 0; + features = ParseFeaturesFromStringArray(featureStringArray.begin(), + featureStringArray.length()); + + // 128 * 1024 * 1024 is the entries preset that is given in + // devtools/client/performance-new/shared/background.jsm.js + profiler_start(PowerOfTwo32(128 * 1024 * 1024), 5.0, features, + filtersTemp.begin(), filtersTemp.length(), 0, Nothing()); + } + + static void StopProfiler(jni::Object::Param aGeckoResult) { + auto result = java::GeckoResult::LocalRef(aGeckoResult); + profiler_pause(); + nsCOMPtr<nsIProfiler> nsProfiler( + do_GetService("@mozilla.org/tools/profiler;1")); + nsProfiler->GetProfileDataAsGzippedArrayBufferAndroid(0)->Then( + GetMainThreadSerialEventTarget(), __func__, + [result](FallibleTArray<uint8_t> compressedProfile) { + result->Complete(jni::ByteArray::New( + reinterpret_cast<const int8_t*>(compressedProfile.Elements()), + compressedProfile.Length())); + + // Done with capturing a profile. Stop the profiler. + profiler_stop(); + }, + [result](nsresult aRv) { + char errorString[9]; + sprintf(errorString, "%08x", uint32_t(aRv)); + result->CompleteExceptionally( + mozilla::java::sdk::IllegalStateException::New(errorString) + .Cast<jni::Throwable>()); + + // Failed to capture a profile. Stop the profiler. + profiler_stop(); + }); + } +}; +#endif + +constexpr static bool ValidateFeatures() { + int expectedFeatureNumber = 0; + + // Feature numbers should start at 0 and increase by 1 each. +#define CHECK_FEATURE(n_, str_, Name_, desc_) \ + if ((n_) != expectedFeatureNumber) { \ + return false; \ + } \ + ++expectedFeatureNumber; + + PROFILER_FOR_EACH_FEATURE(CHECK_FEATURE) + +#undef CHECK_FEATURE + + return true; +} + +static_assert(ValidateFeatures(), "Feature list is invalid"); + +// Return all features that are available on this platform. +static uint32_t AvailableFeatures() { + uint32_t features = 0; + +#define ADD_FEATURE(n_, str_, Name_, desc_) \ + ProfilerFeature::Set##Name_(features); + + // Add all the possible features. + PROFILER_FOR_EACH_FEATURE(ADD_FEATURE) + +#undef ADD_FEATURE + + // Now remove features not supported on this platform/configuration. +#if !defined(GP_OS_android) + ProfilerFeature::ClearJava(features); +#endif +#if !defined(HAVE_NATIVE_UNWIND) + ProfilerFeature::ClearStackWalk(features); +#endif +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + if (getenv("XPCOM_MEM_BLOAT_LOG")) { + DEBUG_LOG("XPCOM_MEM_BLOAT_LOG is set, disabling native allocations."); + // The memory hooks are available, but the bloat log is enabled, which is + // not compatible with the native allocations tracking. See the comment in + // enable_native_allocations() (tools/profiler/core/memory_hooks.cpp) for + // more information. + ProfilerFeature::ClearNativeAllocations(features); + } +#else + // The memory hooks are not available. + ProfilerFeature::ClearNativeAllocations(features); +#endif + +#if !defined(GP_OS_windows) + ProfilerFeature::ClearNoTimerResolutionChange(features); +#endif +#if !defined(HAVE_CPU_FREQ_SUPPORT) + ProfilerFeature::ClearCPUFrequency(features); +#endif + + return features; +} + +// Default features common to all contexts (even if not available). +static constexpr uint32_t DefaultFeatures() { + return ProfilerFeature::Java | ProfilerFeature::JS | + ProfilerFeature::StackWalk | ProfilerFeature::CPUUtilization | + ProfilerFeature::Screenshots | ProfilerFeature::ProcessCPU; +} + +// Extra default features when MOZ_PROFILER_STARTUP is set (even if not +// available). +static constexpr uint32_t StartupExtraDefaultFeatures() { + // Enable file I/Os by default for startup profiles as startup is heavy on + // I/O operations. + return ProfilerFeature::FileIOAll | ProfilerFeature::IPCMessages; +} + +Json::String ToCompactString(const Json::Value& aJsonValue) { + Json::StreamWriterBuilder builder; + // No indentations, and no newlines. + builder["indentation"] = ""; + // This removes spaces after colons. + builder["enableYAMLCompatibility"] = false; + // Only 6 digits after the decimal point; timestamps in ms have ns precision. + builder["precision"] = 6; + builder["precisionType"] = "decimal"; + + return Json::writeString(builder, aJsonValue); +} + +/* static */ mozilla::baseprofiler::detail::BaseProfilerMutex + ProfilingLog::gMutex; +/* static */ mozilla::UniquePtr<Json::Value> ProfilingLog::gLog; + +/* static */ void ProfilingLog::Init() { + mozilla::baseprofiler::detail::BaseProfilerAutoLock lock{gMutex}; + MOZ_ASSERT(!gLog); + gLog = mozilla::MakeUniqueFallible<Json::Value>(Json::objectValue); + if (gLog) { + (*gLog)[Json::StaticString{"profilingLogBegin" TIMESTAMP_JSON_SUFFIX}] = + ProfilingLog::Timestamp(); + } +} + +/* static */ void ProfilingLog::Destroy() { + mozilla::baseprofiler::detail::BaseProfilerAutoLock lock{gMutex}; + MOZ_ASSERT(gLog); + gLog = nullptr; +} + +/* static */ bool ProfilingLog::IsLockedOnCurrentThread() { + return gMutex.IsLockedOnCurrentThread(); +} + +// RAII class to lock the profiler mutex. +// It provides a mechanism to determine if it is locked or not in order for +// memory hooks to avoid re-entering the profiler locked state. +// Locking order: Profiler, ThreadRegistry, ThreadRegistration. +class MOZ_RAII PSAutoLock { + public: + PSAutoLock() + : mLock([]() -> mozilla::baseprofiler::detail::BaseProfilerMutex& { + // In DEBUG builds, *before* we attempt to lock gPSMutex, we want to + // check that the ThreadRegistry, ThreadRegistration, and ProfilingLog + // mutexes are *not* locked on this thread, to avoid inversion + // deadlocks. + MOZ_ASSERT(!ThreadRegistry::IsRegistryMutexLockedOnCurrentThread()); + MOZ_ASSERT(!ThreadRegistration::IsDataMutexLockedOnCurrentThread()); + MOZ_ASSERT(!ProfilingLog::IsLockedOnCurrentThread()); + return gPSMutex; + }()) {} + + PSAutoLock(const PSAutoLock&) = delete; + void operator=(const PSAutoLock&) = delete; + + static bool IsLockedOnCurrentThread() { + return gPSMutex.IsLockedOnCurrentThread(); + } + + private: + static mozilla::baseprofiler::detail::BaseProfilerMutex gPSMutex; + mozilla::baseprofiler::detail::BaseProfilerAutoLock mLock; +}; + +/* static */ mozilla::baseprofiler::detail::BaseProfilerMutex + PSAutoLock::gPSMutex{"Gecko Profiler mutex"}; + +// Only functions that take a PSLockRef arg can access CorePS's and ActivePS's +// fields. +typedef const PSAutoLock& PSLockRef; + +#define PS_GET(type_, name_) \ + static type_ name_(PSLockRef) { \ + MOZ_ASSERT(sInstance); \ + return sInstance->m##name_; \ + } + +#define PS_GET_LOCKLESS(type_, name_) \ + static type_ name_() { \ + MOZ_ASSERT(sInstance); \ + return sInstance->m##name_; \ + } + +#define PS_GET_AND_SET(type_, name_) \ + PS_GET(type_, name_) \ + static void Set##name_(PSLockRef, type_ a##name_) { \ + MOZ_ASSERT(sInstance); \ + sInstance->m##name_ = a##name_; \ + } + +static constexpr size_t MAX_JS_FRAMES = + mozilla::profiler::ThreadRegistrationData::MAX_JS_FRAMES; +using JsFrame = mozilla::profiler::ThreadRegistrationData::JsFrame; +using JsFrameBuffer = mozilla::profiler::ThreadRegistrationData::JsFrameBuffer; + +// All functions in this file can run on multiple threads unless they have an +// NS_IsMainThread() assertion. + +// This class contains the profiler's core global state, i.e. that which is +// valid even when the profiler is not active. Most profile operations can't do +// anything useful when this class is not instantiated, so we release-assert +// its non-nullness in all such operations. +// +// Accesses to CorePS are guarded by gPSMutex. Getters and setters take a +// PSAutoLock reference as an argument as proof that the gPSMutex is currently +// locked. This makes it clear when gPSMutex is locked and helps avoid +// accidental unlocked accesses to global state. There are ways to circumvent +// this mechanism, but please don't do so without *very* good reason and a +// detailed explanation. +// +// The exceptions to this rule: +// +// - mProcessStartTime, because it's immutable; +class CorePS { + private: + CorePS() + : mProcessStartTime(TimeStamp::ProcessCreation()), + mMaybeBandwidthCounter(nullptr) +#ifdef USE_LUL_STACKWALK + , + mLul(nullptr) +#endif + { + MOZ_ASSERT(NS_IsMainThread(), + "CorePS must be created from the main thread"); + } + + ~CorePS() { +#ifdef USE_LUL_STACKWALK + delete sInstance->mLul; + delete mMaybeBandwidthCounter; +#endif + } + + public: + static void Create(PSLockRef aLock) { + MOZ_ASSERT(!sInstance); + sInstance = new CorePS(); + } + + static void Destroy(PSLockRef aLock) { + MOZ_ASSERT(sInstance); + delete sInstance; + sInstance = nullptr; + } + + // Unlike ActivePS::Exists(), CorePS::Exists() can be called without gPSMutex + // being locked. This is because CorePS is instantiated so early on the main + // thread that we don't have to worry about it being racy. + static bool Exists() { return !!sInstance; } + + static void AddSizeOf(PSLockRef, MallocSizeOf aMallocSizeOf, + size_t& aProfSize, size_t& aLulSize) { + MOZ_ASSERT(sInstance); + + aProfSize += aMallocSizeOf(sInstance); + + aProfSize += ThreadRegistry::SizeOfIncludingThis(aMallocSizeOf); + + for (auto& registeredPage : sInstance->mRegisteredPages) { + aProfSize += registeredPage->SizeOfIncludingThis(aMallocSizeOf); + } + + // Measurement of the following things may be added later if DMD finds it + // is worthwhile: + // - CorePS::mRegisteredPages itself (its elements' children are + // measured above) + +#if defined(USE_LUL_STACKWALK) + if (lul::LUL* lulPtr = sInstance->mLul; lulPtr) { + aLulSize += lulPtr->SizeOfIncludingThis(aMallocSizeOf); + } +#endif + } + + // No PSLockRef is needed for this field because it's immutable. + PS_GET_LOCKLESS(TimeStamp, ProcessStartTime) + + PS_GET(JsFrameBuffer&, JsFrames) + + PS_GET(Vector<RefPtr<PageInformation>>&, RegisteredPages) + + static void AppendRegisteredPage(PSLockRef, + RefPtr<PageInformation>&& aRegisteredPage) { + MOZ_ASSERT(sInstance); + struct RegisteredPageComparator { + PageInformation* aA; + bool operator()(PageInformation* aB) const { return aA->Equals(aB); } + }; + + auto foundPageIter = std::find_if( + sInstance->mRegisteredPages.begin(), sInstance->mRegisteredPages.end(), + RegisteredPageComparator{aRegisteredPage.get()}); + + if (foundPageIter != sInstance->mRegisteredPages.end()) { + if ((*foundPageIter)->Url().EqualsLiteral("about:blank")) { + // When a BrowsingContext is loaded, the first url loaded in it will be + // about:blank, and if the principal matches, the first document loaded + // in it will share an inner window. That's why we should delete the + // intermittent about:blank if they share the inner window. + sInstance->mRegisteredPages.erase(foundPageIter); + } else { + // Do not register the same page again. + return; + } + } + + MOZ_RELEASE_ASSERT( + sInstance->mRegisteredPages.append(std::move(aRegisteredPage))); + } + + static void RemoveRegisteredPage(PSLockRef, + uint64_t aRegisteredInnerWindowID) { + MOZ_ASSERT(sInstance); + // Remove RegisteredPage from mRegisteredPages by given inner window ID. + sInstance->mRegisteredPages.eraseIf([&](const RefPtr<PageInformation>& rd) { + return rd->InnerWindowID() == aRegisteredInnerWindowID; + }); + } + + static void ClearRegisteredPages(PSLockRef) { + MOZ_ASSERT(sInstance); + sInstance->mRegisteredPages.clear(); + } + + PS_GET(const Vector<BaseProfilerCount*>&, Counters) + + static void AppendCounter(PSLockRef, BaseProfilerCount* aCounter) { + MOZ_ASSERT(sInstance); + // we don't own the counter; they may be stored in static objects + MOZ_RELEASE_ASSERT(sInstance->mCounters.append(aCounter)); + } + + static void RemoveCounter(PSLockRef, BaseProfilerCount* aCounter) { + // we may be called to remove a counter after the profiler is stopped or + // late in shutdown. + if (sInstance) { + auto* counter = std::find(sInstance->mCounters.begin(), + sInstance->mCounters.end(), aCounter); + MOZ_RELEASE_ASSERT(counter != sInstance->mCounters.end()); + sInstance->mCounters.erase(counter); + } + } + +#ifdef USE_LUL_STACKWALK + static lul::LUL* Lul() { + MOZ_RELEASE_ASSERT(sInstance); + return sInstance->mLul; + } + static void SetLul(UniquePtr<lul::LUL> aLul) { + MOZ_RELEASE_ASSERT(sInstance); + MOZ_RELEASE_ASSERT( + sInstance->mLul.compareExchange(nullptr, aLul.release())); + } +#endif + + PS_GET_AND_SET(const nsACString&, ProcessName) + PS_GET_AND_SET(const nsACString&, ETLDplus1) + + static void SetBandwidthCounter(ProfilerBandwidthCounter* aBandwidthCounter) { + MOZ_ASSERT(sInstance); + + sInstance->mMaybeBandwidthCounter = aBandwidthCounter; + } + static ProfilerBandwidthCounter* GetBandwidthCounter() { + MOZ_ASSERT(sInstance); + + return sInstance->mMaybeBandwidthCounter; + } + + private: + // The singleton instance + static CorePS* sInstance; + + // The time that the process started. + const TimeStamp mProcessStartTime; + + // Network bandwidth counter for the Bandwidth feature. + ProfilerBandwidthCounter* mMaybeBandwidthCounter; + + // Info on all the registered pages. + // InnerWindowIDs in mRegisteredPages are unique. + Vector<RefPtr<PageInformation>> mRegisteredPages; + + // Non-owning pointers to all active counters + Vector<BaseProfilerCount*> mCounters; + +#ifdef USE_LUL_STACKWALK + // LUL's state. Null prior to the first activation, non-null thereafter. + // Owned by this CorePS. + mozilla::Atomic<lul::LUL*> mLul; +#endif + + // Process name, provided by child process initialization code. + nsAutoCString mProcessName; + // Private name, provided by child process initialization code (eTLD+1 in + // fission) + nsAutoCString mETLDplus1; + + // This memory buffer is used by the MergeStacks mechanism. Previously it was + // stack allocated, but this led to a stack overflow, as it was too much + // memory. Here the buffer can be pre-allocated, and shared with the + // MergeStacks feature as needed. MergeStacks is only run while holding the + // lock, so it is safe to have only one instance allocated for all of the + // threads. + JsFrameBuffer mJsFrames; +}; + +CorePS* CorePS::sInstance = nullptr; + +void locked_profiler_add_sampled_counter(PSLockRef aLock, + BaseProfilerCount* aCounter) { + CorePS::AppendCounter(aLock, aCounter); +} + +void locked_profiler_remove_sampled_counter(PSLockRef aLock, + BaseProfilerCount* aCounter) { + // Note: we don't enforce a final sample, though we could do so if the + // profiler was active + CorePS::RemoveCounter(aLock, aCounter); +} + +class SamplerThread; + +static SamplerThread* NewSamplerThread(PSLockRef aLock, uint32_t aGeneration, + double aInterval, uint32_t aFeatures); + +struct LiveProfiledThreadData { + UniquePtr<ProfiledThreadData> mProfiledThreadData; +}; + +// This class contains the profiler's global state that is valid only when the +// profiler is active. When not instantiated, the profiler is inactive. +// +// Accesses to ActivePS are guarded by gPSMutex, in much the same fashion as +// CorePS. +// +class ActivePS { + private: + constexpr static uint32_t ChunkSizeForEntries(uint32_t aEntries) { + return uint32_t(std::min(size_t(ClampToAllowedEntries(aEntries)) * + scBytesPerEntry / scMinimumNumberOfChunks, + size_t(scMaximumChunkSize))); + } + + static uint32_t AdjustFeatures(uint32_t aFeatures, uint32_t aFilterCount) { + // Filter out any features unavailable in this platform/configuration. + aFeatures &= AvailableFeatures(); + + // Some features imply others. + if (aFeatures & ProfilerFeature::FileIOAll) { + aFeatures |= ProfilerFeature::MainThreadIO | ProfilerFeature::FileIO; + } else if (aFeatures & ProfilerFeature::FileIO) { + aFeatures |= ProfilerFeature::MainThreadIO; + } + + if (aFeatures & ProfilerFeature::CPUAllThreads) { + aFeatures |= ProfilerFeature::CPUUtilization; + } + + return aFeatures; + } + + bool ShouldInterposeIOs() { + return ProfilerFeature::HasMainThreadIO(mFeatures) || + ProfilerFeature::HasFileIO(mFeatures) || + ProfilerFeature::HasFileIOAll(mFeatures); + } + + ActivePS( + PSLockRef aLock, const TimeStamp& aProfilingStartTime, + PowerOfTwo32 aCapacity, double aInterval, uint32_t aFeatures, + const char** aFilters, uint32_t aFilterCount, uint64_t aActiveTabID, + const Maybe<double>& aDuration, + UniquePtr<ProfileBufferChunkManagerWithLocalLimit> aChunkManagerOrNull) + : mProfilingStartTime(aProfilingStartTime), + mGeneration(sNextGeneration++), + mCapacity(aCapacity), + mDuration(aDuration), + mInterval(aInterval), + mFeatures(AdjustFeatures(aFeatures, aFilterCount)), + mActiveTabID(aActiveTabID), + mProfileBufferChunkManager( + aChunkManagerOrNull + ? std::move(aChunkManagerOrNull) + : MakeUnique<ProfileBufferChunkManagerWithLocalLimit>( + size_t(ClampToAllowedEntries(aCapacity.Value())) * + scBytesPerEntry, + ChunkSizeForEntries(aCapacity.Value()))), + mProfileBuffer([this]() -> ProfileChunkedBuffer& { + ProfileChunkedBuffer& coreBuffer = profiler_get_core_buffer(); + coreBuffer.SetChunkManagerIfDifferent(*mProfileBufferChunkManager); + return coreBuffer; + }()), + mMaybeProcessCPUCounter(ProfilerFeature::HasProcessCPU(aFeatures) + ? new ProcessCPUCounter(aLock) + : nullptr), + mMaybePowerCounters(nullptr), + mMaybeCPUFreq(nullptr), + // The new sampler thread doesn't start sampling immediately because the + // main loop within Run() is blocked until this function's caller + // unlocks gPSMutex. + mSamplerThread( + NewSamplerThread(aLock, mGeneration, aInterval, aFeatures)), + mIsPaused(false), + mIsSamplingPaused(false) { + ProfilingLog::Init(); + + // Deep copy and lower-case aFilters. + MOZ_ALWAYS_TRUE(mFilters.resize(aFilterCount)); + MOZ_ALWAYS_TRUE(mFiltersLowered.resize(aFilterCount)); + for (uint32_t i = 0; i < aFilterCount; ++i) { + mFilters[i] = aFilters[i]; + mFiltersLowered[i].reserve(mFilters[i].size()); + std::transform(mFilters[i].cbegin(), mFilters[i].cend(), + std::back_inserter(mFiltersLowered[i]), ::tolower); + } + +#if !defined(RELEASE_OR_BETA) + if (ShouldInterposeIOs()) { + // We need to register the observer on the main thread, because we want + // to observe IO that happens on the main thread. + // IOInterposer needs to be initialized before calling + // IOInterposer::Register or our observer will be silently dropped. + if (NS_IsMainThread()) { + IOInterposer::Init(); + IOInterposer::Register(IOInterposeObserver::OpAll, + &ProfilerIOInterposeObserver::GetInstance()); + } else { + NS_DispatchToMainThread( + NS_NewRunnableFunction("ActivePS::ActivePS", []() { + // Note: This could theoretically happen after ActivePS gets + // destroyed, but it's ok: + // - The Observer always checks that the profiler is (still) + // active before doing its work. + // - The destruction should happen on the same thread as this + // construction, so the un-registration will also be dispatched + // and queued on the main thread, and run after this. + IOInterposer::Init(); + IOInterposer::Register( + IOInterposeObserver::OpAll, + &ProfilerIOInterposeObserver::GetInstance()); + })); + } + } +#endif + + if (ProfilerFeature::HasPower(aFeatures)) { + mMaybePowerCounters = new PowerCounters(); + for (const auto& powerCounter : mMaybePowerCounters->GetCounters()) { + locked_profiler_add_sampled_counter(aLock, powerCounter); + } + } + + if (ProfilerFeature::HasCPUFrequency(aFeatures)) { + mMaybeCPUFreq = new ProfilerCPUFreq(); + } + } + + ~ActivePS() { + MOZ_ASSERT( + !mMaybeProcessCPUCounter, + "mMaybeProcessCPUCounter should have been deleted before ~ActivePS()"); + MOZ_ASSERT( + !mMaybePowerCounters, + "mMaybePowerCounters should have been deleted before ~ActivePS()"); + MOZ_ASSERT(!mMaybeCPUFreq, + "mMaybeCPUFreq should have been deleted before ~ActivePS()"); + +#if !defined(RELEASE_OR_BETA) + if (ShouldInterposeIOs()) { + // We need to unregister the observer on the main thread, because that's + // where we've registered it. + if (NS_IsMainThread()) { + IOInterposer::Unregister(IOInterposeObserver::OpAll, + &ProfilerIOInterposeObserver::GetInstance()); + } else { + NS_DispatchToMainThread( + NS_NewRunnableFunction("ActivePS::~ActivePS", []() { + IOInterposer::Unregister( + IOInterposeObserver::OpAll, + &ProfilerIOInterposeObserver::GetInstance()); + })); + } + } +#endif + if (mProfileBufferChunkManager) { + // We still control the chunk manager, remove it from the core buffer. + profiler_get_core_buffer().ResetChunkManager(); + } + + ProfilingLog::Destroy(); + } + + bool ThreadSelected(const char* aThreadName) { + if (mFiltersLowered.empty()) { + return true; + } + + std::string name = aThreadName; + std::transform(name.begin(), name.end(), name.begin(), ::tolower); + + for (const auto& filter : mFiltersLowered) { + if (filter == "*") { + return true; + } + + // Crude, non UTF-8 compatible, case insensitive substring search + if (name.find(filter) != std::string::npos) { + return true; + } + + // If the filter is "pid:<my pid>", profile all threads. + if (mozilla::profiler::detail::FilterHasPid(filter.c_str())) { + return true; + } + } + + return false; + } + + public: + static void Create( + PSLockRef aLock, const TimeStamp& aProfilingStartTime, + PowerOfTwo32 aCapacity, double aInterval, uint32_t aFeatures, + const char** aFilters, uint32_t aFilterCount, uint64_t aActiveTabID, + const Maybe<double>& aDuration, + UniquePtr<ProfileBufferChunkManagerWithLocalLimit> aChunkManagerOrNull) { + MOZ_ASSERT(!sInstance); + sInstance = new ActivePS(aLock, aProfilingStartTime, aCapacity, aInterval, + aFeatures, aFilters, aFilterCount, aActiveTabID, + aDuration, std::move(aChunkManagerOrNull)); + } + + [[nodiscard]] static SamplerThread* Destroy(PSLockRef aLock) { + MOZ_ASSERT(sInstance); + if (sInstance->mMaybeProcessCPUCounter) { + locked_profiler_remove_sampled_counter( + aLock, sInstance->mMaybeProcessCPUCounter); + delete sInstance->mMaybeProcessCPUCounter; + sInstance->mMaybeProcessCPUCounter = nullptr; + } + + if (sInstance->mMaybePowerCounters) { + for (const auto& powerCounter : + sInstance->mMaybePowerCounters->GetCounters()) { + locked_profiler_remove_sampled_counter(aLock, powerCounter); + } + delete sInstance->mMaybePowerCounters; + sInstance->mMaybePowerCounters = nullptr; + } + + if (sInstance->mMaybeCPUFreq) { + delete sInstance->mMaybeCPUFreq; + sInstance->mMaybeCPUFreq = nullptr; + } + + ProfilerBandwidthCounter* counter = CorePS::GetBandwidthCounter(); + if (counter && counter->IsRegistered()) { + // Because profiler_count_bandwidth_bytes does a racy + // profiler_feature_active check to avoid taking the lock, + // free'ing the memory of the counter would be crashy if the + // socket thread attempts to increment the counter while we are + // stopping the profiler. + // Instead, we keep the counter in CorePS and only mark it as + // unregistered so that the next attempt to count bytes + // will re-register it. + locked_profiler_remove_sampled_counter(aLock, counter); + counter->MarkUnregistered(); + } + + auto samplerThread = sInstance->mSamplerThread; + delete sInstance; + sInstance = nullptr; + + return samplerThread; + } + + static bool Exists(PSLockRef) { return !!sInstance; } + + static bool Equals(PSLockRef, PowerOfTwo32 aCapacity, + const Maybe<double>& aDuration, double aInterval, + uint32_t aFeatures, const char** aFilters, + uint32_t aFilterCount, uint64_t aActiveTabID) { + MOZ_ASSERT(sInstance); + if (sInstance->mCapacity != aCapacity || + sInstance->mDuration != aDuration || + sInstance->mInterval != aInterval || + sInstance->mFeatures != aFeatures || + sInstance->mFilters.length() != aFilterCount || + sInstance->mActiveTabID != aActiveTabID) { + return false; + } + + for (uint32_t i = 0; i < sInstance->mFilters.length(); ++i) { + if (strcmp(sInstance->mFilters[i].c_str(), aFilters[i]) != 0) { + return false; + } + } + return true; + } + + static size_t SizeOf(PSLockRef, MallocSizeOf aMallocSizeOf) { + MOZ_ASSERT(sInstance); + + size_t n = aMallocSizeOf(sInstance); + + n += sInstance->mProfileBuffer.SizeOfExcludingThis(aMallocSizeOf); + + // Measurement of the following members may be added later if DMD finds it + // is worthwhile: + // - mLiveProfiledThreads (both the array itself, and the contents) + // - mDeadProfiledThreads (both the array itself, and the contents) + // + + return n; + } + + static ThreadProfilingFeatures ProfilingFeaturesForThread( + PSLockRef aLock, const ThreadRegistrationInfo& aInfo) { + MOZ_ASSERT(sInstance); + if (sInstance->ThreadSelected(aInfo.Name())) { + // This thread was selected by the user, record everything. + return ThreadProfilingFeatures::Any; + } + ThreadProfilingFeatures features = ThreadProfilingFeatures::NotProfiled; + if (ActivePS::FeatureCPUAllThreads(aLock)) { + features = Combine(features, ThreadProfilingFeatures::CPUUtilization); + } + if (ActivePS::FeatureSamplingAllThreads(aLock)) { + features = Combine(features, ThreadProfilingFeatures::Sampling); + } + if (ActivePS::FeatureMarkersAllThreads(aLock)) { + features = Combine(features, ThreadProfilingFeatures::Markers); + } + return features; + } + + [[nodiscard]] static bool AppendPostSamplingCallback( + PSLockRef, PostSamplingCallback&& aCallback); + + // Writes out the current active configuration of the profile. + static void WriteActiveConfiguration( + PSLockRef aLock, JSONWriter& aWriter, + const Span<const char>& aPropertyName = MakeStringSpan("")) { + if (!sInstance) { + if (!aPropertyName.empty()) { + aWriter.NullProperty(aPropertyName); + } else { + aWriter.NullElement(); + } + return; + }; + + if (!aPropertyName.empty()) { + aWriter.StartObjectProperty(aPropertyName); + } else { + aWriter.StartObjectElement(); + } + + { + aWriter.StartArrayProperty("features"); +#define WRITE_ACTIVE_FEATURES(n_, str_, Name_, desc_) \ + if (profiler_feature_active(ProfilerFeature::Name_)) { \ + aWriter.StringElement(str_); \ + } + + PROFILER_FOR_EACH_FEATURE(WRITE_ACTIVE_FEATURES) +#undef WRITE_ACTIVE_FEATURES + aWriter.EndArray(); + } + { + aWriter.StartArrayProperty("threads"); + for (const auto& filter : sInstance->mFilters) { + aWriter.StringElement(filter); + } + aWriter.EndArray(); + } + { + // Now write all the simple values. + + // The interval is also available on profile.meta.interval + aWriter.DoubleProperty("interval", sInstance->mInterval); + aWriter.IntProperty("capacity", sInstance->mCapacity.Value()); + if (sInstance->mDuration) { + aWriter.DoubleProperty("duration", sInstance->mDuration.value()); + } + // Here, we are converting uint64_t to double. Tab IDs are + // being created using `nsContentUtils::GenerateProcessSpecificId`, which + // is specifically designed to only use 53 of the 64 bits to be lossless + // when passed into and out of JS as a double. + aWriter.DoubleProperty("activeTabID", sInstance->mActiveTabID); + } + aWriter.EndObject(); + } + + PS_GET_LOCKLESS(TimeStamp, ProfilingStartTime) + + PS_GET(uint32_t, Generation) + + PS_GET(PowerOfTwo32, Capacity) + + PS_GET(Maybe<double>, Duration) + + PS_GET(double, Interval) + + PS_GET(uint32_t, Features) + + PS_GET(uint64_t, ActiveTabID) + +#define PS_GET_FEATURE(n_, str_, Name_, desc_) \ + static bool Feature##Name_(PSLockRef) { \ + MOZ_ASSERT(sInstance); \ + return ProfilerFeature::Has##Name_(sInstance->mFeatures); \ + } + + PROFILER_FOR_EACH_FEATURE(PS_GET_FEATURE) + +#undef PS_GET_FEATURE + + static uint32_t JSFlags(PSLockRef aLock) { + uint32_t Flags = 0; + Flags |= + FeatureJS(aLock) ? uint32_t(JSInstrumentationFlags::StackSampling) : 0; + + Flags |= FeatureJSAllocations(aLock) + ? uint32_t(JSInstrumentationFlags::Allocations) + : 0; + return Flags; + } + + PS_GET(const Vector<std::string>&, Filters) + PS_GET(const Vector<std::string>&, FiltersLowered) + + // Not using PS_GET, because only the "Controlled" interface of + // `mProfileBufferChunkManager` should be exposed here. + static ProfileBufferChunkManagerWithLocalLimit& ControlledChunkManager( + PSLockRef) { + MOZ_ASSERT(sInstance); + MOZ_ASSERT(sInstance->mProfileBufferChunkManager); + return *sInstance->mProfileBufferChunkManager; + } + + static void FulfillChunkRequests(PSLockRef) { + MOZ_ASSERT(sInstance); + if (sInstance->mProfileBufferChunkManager) { + sInstance->mProfileBufferChunkManager->FulfillChunkRequests(); + } + } + + static ProfileBuffer& Buffer(PSLockRef) { + MOZ_ASSERT(sInstance); + return sInstance->mProfileBuffer; + } + + static const Vector<LiveProfiledThreadData>& LiveProfiledThreads(PSLockRef) { + MOZ_ASSERT(sInstance); + return sInstance->mLiveProfiledThreads; + } + + struct ProfiledThreadListElement { + TimeStamp mRegisterTime; + JSContext* mJSContext; // Null for unregistered threads. + ProfiledThreadData* mProfiledThreadData; + }; + using ProfiledThreadList = Vector<ProfiledThreadListElement>; + + // Returns a ProfiledThreadList with all threads that should be included in a + // profile, both for threads that are still registered, and for threads that + // have been unregistered but still have data in the buffer. + // The returned array is sorted by thread register time. + // Do not hold on to the return value past LockedRegistry. + static ProfiledThreadList ProfiledThreads( + ThreadRegistry::LockedRegistry& aLockedRegistry, PSLockRef aLock) { + MOZ_ASSERT(sInstance); + ProfiledThreadList array; + MOZ_RELEASE_ASSERT( + array.initCapacity(sInstance->mLiveProfiledThreads.length() + + sInstance->mDeadProfiledThreads.length())); + + for (ThreadRegistry::OffThreadRef offThreadRef : aLockedRegistry) { + ProfiledThreadData* profiledThreadData = + offThreadRef.UnlockedRWForLockedProfilerRef().GetProfiledThreadData( + aLock); + if (!profiledThreadData) { + // This thread was not profiled, continue with the next one. + continue; + } + ThreadRegistry::OffThreadRef::RWFromAnyThreadWithLock lockedThreadData = + offThreadRef.GetLockedRWFromAnyThread(); + MOZ_RELEASE_ASSERT(array.append(ProfiledThreadListElement{ + profiledThreadData->Info().RegisterTime(), + lockedThreadData->GetJSContext(), profiledThreadData})); + } + + for (auto& t : sInstance->mDeadProfiledThreads) { + MOZ_RELEASE_ASSERT(array.append(ProfiledThreadListElement{ + t->Info().RegisterTime(), (JSContext*)nullptr, t.get()})); + } + + std::sort(array.begin(), array.end(), + [](const ProfiledThreadListElement& a, + const ProfiledThreadListElement& b) { + return a.mRegisterTime < b.mRegisterTime; + }); + return array; + } + + static Vector<RefPtr<PageInformation>> ProfiledPages(PSLockRef aLock) { + MOZ_ASSERT(sInstance); + Vector<RefPtr<PageInformation>> array; + for (auto& d : CorePS::RegisteredPages(aLock)) { + MOZ_RELEASE_ASSERT(array.append(d)); + } + for (auto& d : sInstance->mDeadProfiledPages) { + MOZ_RELEASE_ASSERT(array.append(d)); + } + // We don't need to sort the pages like threads since we won't show them + // as a list. + return array; + } + + static ProfiledThreadData* AddLiveProfiledThread( + PSLockRef, UniquePtr<ProfiledThreadData>&& aProfiledThreadData) { + MOZ_ASSERT(sInstance); + MOZ_RELEASE_ASSERT(sInstance->mLiveProfiledThreads.append( + LiveProfiledThreadData{std::move(aProfiledThreadData)})); + + // Return a weak pointer to the ProfiledThreadData object. + return sInstance->mLiveProfiledThreads.back().mProfiledThreadData.get(); + } + + static void UnregisterThread(PSLockRef aLockRef, + ProfiledThreadData* aProfiledThreadData) { + MOZ_ASSERT(sInstance); + + DiscardExpiredDeadProfiledThreads(aLockRef); + + // Find the right entry in the mLiveProfiledThreads array and remove the + // element, moving the ProfiledThreadData object for the thread into the + // mDeadProfiledThreads array. + for (size_t i = 0; i < sInstance->mLiveProfiledThreads.length(); i++) { + LiveProfiledThreadData& thread = sInstance->mLiveProfiledThreads[i]; + if (thread.mProfiledThreadData == aProfiledThreadData) { + thread.mProfiledThreadData->NotifyUnregistered( + sInstance->mProfileBuffer.BufferRangeEnd()); + MOZ_RELEASE_ASSERT(sInstance->mDeadProfiledThreads.append( + std::move(thread.mProfiledThreadData))); + sInstance->mLiveProfiledThreads.erase( + &sInstance->mLiveProfiledThreads[i]); + return; + } + } + } + + // This is a counter to collect process CPU utilization during profiling. + // It cannot be a raw `ProfilerCounter` because we need to manually add/remove + // it while the profiler lock is already held. + class ProcessCPUCounter final : public BaseProfilerCount { + public: + explicit ProcessCPUCounter(PSLockRef aLock) + : BaseProfilerCount("processCPU", &mCounter, nullptr, "CPU", + "Process CPU utilization") { + // Adding on construction, so it's ready before the sampler starts. + locked_profiler_add_sampled_counter(aLock, this); + // Note: Removed from ActivePS::Destroy, because a lock is needed. + } + + void Add(int64_t aNumber) { mCounter += aNumber; } + + private: + ProfilerAtomicSigned mCounter; + }; + PS_GET(ProcessCPUCounter*, MaybeProcessCPUCounter); + + PS_GET(PowerCounters*, MaybePowerCounters); + + PS_GET(ProfilerCPUFreq*, MaybeCPUFreq); + + PS_GET_AND_SET(bool, IsPaused) + + // True if sampling is paused (though generic `SetIsPaused()` or specific + // `SetIsSamplingPaused()`). + static bool IsSamplingPaused(PSLockRef lock) { + MOZ_ASSERT(sInstance); + return IsPaused(lock) || sInstance->mIsSamplingPaused; + } + + static void SetIsSamplingPaused(PSLockRef, bool aIsSamplingPaused) { + MOZ_ASSERT(sInstance); + sInstance->mIsSamplingPaused = aIsSamplingPaused; + } + + static void DiscardExpiredDeadProfiledThreads(PSLockRef) { + MOZ_ASSERT(sInstance); + uint64_t bufferRangeStart = sInstance->mProfileBuffer.BufferRangeStart(); + // Discard any dead threads that were unregistered before bufferRangeStart. + sInstance->mDeadProfiledThreads.eraseIf( + [bufferRangeStart]( + const UniquePtr<ProfiledThreadData>& aProfiledThreadData) { + Maybe<uint64_t> bufferPosition = + aProfiledThreadData->BufferPositionWhenUnregistered(); + MOZ_RELEASE_ASSERT(bufferPosition, + "should have unregistered this thread"); + return *bufferPosition < bufferRangeStart; + }); + } + + static void UnregisterPage(PSLockRef aLock, + uint64_t aRegisteredInnerWindowID) { + MOZ_ASSERT(sInstance); + auto& registeredPages = CorePS::RegisteredPages(aLock); + for (size_t i = 0; i < registeredPages.length(); i++) { + RefPtr<PageInformation>& page = registeredPages[i]; + if (page->InnerWindowID() == aRegisteredInnerWindowID) { + page->NotifyUnregistered(sInstance->mProfileBuffer.BufferRangeEnd()); + MOZ_RELEASE_ASSERT( + sInstance->mDeadProfiledPages.append(std::move(page))); + registeredPages.erase(®isteredPages[i--]); + } + } + } + + static void DiscardExpiredPages(PSLockRef) { + MOZ_ASSERT(sInstance); + uint64_t bufferRangeStart = sInstance->mProfileBuffer.BufferRangeStart(); + // Discard any dead pages that were unregistered before + // bufferRangeStart. + sInstance->mDeadProfiledPages.eraseIf( + [bufferRangeStart](const RefPtr<PageInformation>& aProfiledPage) { + Maybe<uint64_t> bufferPosition = + aProfiledPage->BufferPositionWhenUnregistered(); + MOZ_RELEASE_ASSERT(bufferPosition, + "should have unregistered this page"); + return *bufferPosition < bufferRangeStart; + }); + } + + static void ClearUnregisteredPages(PSLockRef) { + MOZ_ASSERT(sInstance); + sInstance->mDeadProfiledPages.clear(); + } + + static void ClearExpiredExitProfiles(PSLockRef) { + MOZ_ASSERT(sInstance); + uint64_t bufferRangeStart = sInstance->mProfileBuffer.BufferRangeStart(); + // Discard exit profiles that were gathered before our buffer RangeStart. + // If we have started to overwrite our data from when the Base profile was + // added, we should get rid of that Base profile because it's now older than + // our oldest Gecko profile data. + // + // When adding: (In practice the starting buffer should be empty) + // v Start == End + // | <-- Buffer range, initially empty. + // ^ mGeckoIndexWhenBaseProfileAdded < Start FALSE -> keep it + // + // Later, still in range: + // v Start v End + // |=========| <-- Buffer range growing. + // ^ mGeckoIndexWhenBaseProfileAdded < Start FALSE -> keep it + // + // Even later, now out of range: + // v Start v End + // |============| <-- Buffer range full and sliding. + // ^ mGeckoIndexWhenBaseProfileAdded < Start TRUE! -> Discard it + if (sInstance->mBaseProfileThreads && + sInstance->mGeckoIndexWhenBaseProfileAdded + .ConvertToProfileBufferIndex() < + profiler_get_core_buffer().GetState().mRangeStart) { + DEBUG_LOG("ClearExpiredExitProfiles() - Discarding base profile %p", + sInstance->mBaseProfileThreads.get()); + sInstance->mBaseProfileThreads.reset(); + } + sInstance->mExitProfiles.eraseIf( + [bufferRangeStart](const ExitProfile& aExitProfile) { + return aExitProfile.mBufferPositionAtGatherTime < bufferRangeStart; + }); + } + + static void AddBaseProfileThreads(PSLockRef aLock, + UniquePtr<char[]> aBaseProfileThreads) { + MOZ_ASSERT(sInstance); + DEBUG_LOG("AddBaseProfileThreads(%p)", aBaseProfileThreads.get()); + sInstance->mBaseProfileThreads = std::move(aBaseProfileThreads); + sInstance->mGeckoIndexWhenBaseProfileAdded = + ProfileBufferBlockIndex::CreateFromProfileBufferIndex( + profiler_get_core_buffer().GetState().mRangeEnd); + } + + static UniquePtr<char[]> MoveBaseProfileThreads(PSLockRef aLock) { + MOZ_ASSERT(sInstance); + + ClearExpiredExitProfiles(aLock); + + DEBUG_LOG("MoveBaseProfileThreads() - Consuming base profile %p", + sInstance->mBaseProfileThreads.get()); + return std::move(sInstance->mBaseProfileThreads); + } + + static void AddExitProfile(PSLockRef aLock, const nsACString& aExitProfile) { + MOZ_ASSERT(sInstance); + + ClearExpiredExitProfiles(aLock); + + MOZ_RELEASE_ASSERT(sInstance->mExitProfiles.append(ExitProfile{ + nsCString(aExitProfile), sInstance->mProfileBuffer.BufferRangeEnd()})); + } + + static Vector<nsCString> MoveExitProfiles(PSLockRef aLock) { + MOZ_ASSERT(sInstance); + + ClearExpiredExitProfiles(aLock); + + Vector<nsCString> profiles; + MOZ_RELEASE_ASSERT( + profiles.initCapacity(sInstance->mExitProfiles.length())); + for (auto& profile : sInstance->mExitProfiles) { + MOZ_RELEASE_ASSERT(profiles.append(std::move(profile.mJSON))); + } + sInstance->mExitProfiles.clear(); + return profiles; + } + +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + static void SetMemoryCounter(const BaseProfilerCount* aMemoryCounter) { + MOZ_ASSERT(sInstance); + + sInstance->mMemoryCounter = aMemoryCounter; + } + + static bool IsMemoryCounter(const BaseProfilerCount* aMemoryCounter) { + MOZ_ASSERT(sInstance); + + return sInstance->mMemoryCounter == aMemoryCounter; + } +#endif + + private: + // The singleton instance. + static ActivePS* sInstance; + + const TimeStamp mProfilingStartTime; + + // We need to track activity generations. If we didn't we could have the + // following scenario. + // + // - profiler_stop() locks gPSMutex, de-instantiates ActivePS, unlocks + // gPSMutex, deletes the SamplerThread (which does a join). + // + // - profiler_start() runs on a different thread, locks gPSMutex, + // re-instantiates ActivePS, unlocks gPSMutex -- all before the join + // completes. + // + // - SamplerThread::Run() locks gPSMutex, sees that ActivePS is instantiated, + // and continues as if the start/stop pair didn't occur. Also + // profiler_stop() is stuck, unable to finish. + // + // By checking ActivePS *and* the generation, we can avoid this scenario. + // sNextGeneration is used to track the next generation number; it is static + // because it must persist across different ActivePS instantiations. + const uint32_t mGeneration; + static uint32_t sNextGeneration; + + // The maximum number of entries in mProfileBuffer. + const PowerOfTwo32 mCapacity; + + // The maximum duration of entries in mProfileBuffer, in seconds. + const Maybe<double> mDuration; + + // The interval between samples, measured in milliseconds. + const double mInterval; + + // The profile features that are enabled. + const uint32_t mFeatures; + + // Substrings of names of threads we want to profile. + Vector<std::string> mFilters; + Vector<std::string> mFiltersLowered; + + // ID of the active browser screen's active tab. + // It's being used to determine the profiled tab. It's "0" if we failed to + // get the ID. + const uint64_t mActiveTabID; + + // The chunk manager used by `mProfileBuffer` below. + // May become null if it gets transferred ouf of the Gecko Profiler. + UniquePtr<ProfileBufferChunkManagerWithLocalLimit> mProfileBufferChunkManager; + + // The buffer into which all samples are recorded. + ProfileBuffer mProfileBuffer; + + // ProfiledThreadData objects for any threads that were profiled at any point + // during this run of the profiler: + // - mLiveProfiledThreads contains all threads that are still registered, and + // - mDeadProfiledThreads contains all threads that have already been + // unregistered but for which there is still data in the profile buffer. + Vector<LiveProfiledThreadData> mLiveProfiledThreads; + Vector<UniquePtr<ProfiledThreadData>> mDeadProfiledThreads; + + // Info on all the dead pages. + // Registered pages are being moved to this array after unregistration. + // We are keeping them in case we need them in the profile data. + // We are removing them when we ensure that we won't need them anymore. + Vector<RefPtr<PageInformation>> mDeadProfiledPages; + + // Used to collect process CPU utilization values, if the feature is on. + ProcessCPUCounter* mMaybeProcessCPUCounter; + + // Used to collect power use data, if the power feature is on. + PowerCounters* mMaybePowerCounters; + + // Used to collect cpu frequency, if the CPU frequency feature is on. + ProfilerCPUFreq* mMaybeCPUFreq; + + // The current sampler thread. This class is not responsible for destroying + // the SamplerThread object; the Destroy() method returns it so the caller + // can destroy it. + SamplerThread* const mSamplerThread; + + // Is the profiler fully paused? + bool mIsPaused; + + // Is the profiler periodic sampling paused? + bool mIsSamplingPaused; + + // Optional startup profile thread array from BaseProfiler. + UniquePtr<char[]> mBaseProfileThreads; + ProfileBufferBlockIndex mGeckoIndexWhenBaseProfileAdded; + + struct ExitProfile { + nsCString mJSON; + uint64_t mBufferPositionAtGatherTime; + }; + Vector<ExitProfile> mExitProfiles; + +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + Atomic<const BaseProfilerCount*> mMemoryCounter; +#endif +}; + +ActivePS* ActivePS::sInstance = nullptr; +uint32_t ActivePS::sNextGeneration = 0; + +#undef PS_GET +#undef PS_GET_LOCKLESS +#undef PS_GET_AND_SET + +using ProfilerStateChangeMutex = + mozilla::baseprofiler::detail::BaseProfilerMutex; +using ProfilerStateChangeLock = + mozilla::baseprofiler::detail::BaseProfilerAutoLock; +static ProfilerStateChangeMutex gProfilerStateChangeMutex; + +struct IdentifiedProfilingStateChangeCallback { + ProfilingStateSet mProfilingStateSet; + ProfilingStateChangeCallback mProfilingStateChangeCallback; + uintptr_t mUniqueIdentifier; + + explicit IdentifiedProfilingStateChangeCallback( + ProfilingStateSet aProfilingStateSet, + ProfilingStateChangeCallback&& aProfilingStateChangeCallback, + uintptr_t aUniqueIdentifier) + : mProfilingStateSet(aProfilingStateSet), + mProfilingStateChangeCallback(aProfilingStateChangeCallback), + mUniqueIdentifier(aUniqueIdentifier) {} +}; +using IdentifiedProfilingStateChangeCallbackUPtr = + UniquePtr<IdentifiedProfilingStateChangeCallback>; + +static Vector<IdentifiedProfilingStateChangeCallbackUPtr> + mIdentifiedProfilingStateChangeCallbacks; + +void profiler_add_state_change_callback( + ProfilingStateSet aProfilingStateSet, + ProfilingStateChangeCallback&& aCallback, + uintptr_t aUniqueIdentifier /* = 0 */) { + MOZ_ASSERT(!PSAutoLock::IsLockedOnCurrentThread()); + ProfilerStateChangeLock lock(gProfilerStateChangeMutex); + +#ifdef DEBUG + // Check if a non-zero id is not already used. Bug forgive it in non-DEBUG + // builds; in the worst case they may get removed too early. + if (aUniqueIdentifier != 0) { + for (const IdentifiedProfilingStateChangeCallbackUPtr& idedCallback : + mIdentifiedProfilingStateChangeCallbacks) { + MOZ_ASSERT(idedCallback->mUniqueIdentifier != aUniqueIdentifier); + } + } +#endif // DEBUG + + if (aProfilingStateSet.contains(ProfilingState::AlreadyActive) && + profiler_is_active()) { + aCallback(ProfilingState::AlreadyActive); + } + + (void)mIdentifiedProfilingStateChangeCallbacks.append( + MakeUnique<IdentifiedProfilingStateChangeCallback>( + aProfilingStateSet, std::move(aCallback), aUniqueIdentifier)); +} + +// Remove the callback with the given identifier. +void profiler_remove_state_change_callback(uintptr_t aUniqueIdentifier) { + MOZ_ASSERT(aUniqueIdentifier != 0); + if (aUniqueIdentifier == 0) { + // Forgive zero in non-DEBUG builds. + return; + } + + MOZ_ASSERT(!PSAutoLock::IsLockedOnCurrentThread()); + ProfilerStateChangeLock lock(gProfilerStateChangeMutex); + + mIdentifiedProfilingStateChangeCallbacks.eraseIf( + [aUniqueIdentifier]( + const IdentifiedProfilingStateChangeCallbackUPtr& aIdedCallback) { + if (aIdedCallback->mUniqueIdentifier != aUniqueIdentifier) { + return false; + } + if (aIdedCallback->mProfilingStateSet.contains( + ProfilingState::RemovingCallback)) { + aIdedCallback->mProfilingStateChangeCallback( + ProfilingState::RemovingCallback); + } + return true; + }); +} + +static void invoke_profiler_state_change_callbacks( + ProfilingState aProfilingState) { + MOZ_ASSERT(!PSAutoLock::IsLockedOnCurrentThread()); + ProfilerStateChangeLock lock(gProfilerStateChangeMutex); + + for (const IdentifiedProfilingStateChangeCallbackUPtr& idedCallback : + mIdentifiedProfilingStateChangeCallbacks) { + if (idedCallback->mProfilingStateSet.contains(aProfilingState)) { + idedCallback->mProfilingStateChangeCallback(aProfilingState); + } + } +} + +Atomic<uint32_t, MemoryOrdering::Relaxed> RacyFeatures::sActiveAndFeatures(0); + +// The name of the main thread. +static const char* const kMainThreadName = "GeckoMain"; + +//////////////////////////////////////////////////////////////////////// +// BEGIN sampling/unwinding code + +// Additional registers that have to be saved when thread is paused. +#if defined(GP_PLAT_x86_linux) || defined(GP_PLAT_x86_android) || \ + defined(GP_ARCH_x86) +# define UNWINDING_REGS_HAVE_ECX_EDX +#elif defined(GP_PLAT_amd64_linux) || defined(GP_PLAT_amd64_android) || \ + defined(GP_PLAT_amd64_freebsd) || defined(GP_ARCH_amd64) || \ + defined(__x86_64__) +# define UNWINDING_REGS_HAVE_R10_R12 +#elif defined(GP_PLAT_arm_linux) || defined(GP_PLAT_arm_android) +# define UNWINDING_REGS_HAVE_LR_R7 +#elif defined(GP_PLAT_arm64_linux) || defined(GP_PLAT_arm64_android) || \ + defined(GP_PLAT_arm64_freebsd) || defined(GP_ARCH_arm64) || \ + defined(__aarch64__) +# define UNWINDING_REGS_HAVE_LR_R11 +#endif + +// The registers used for stack unwinding and a few other sampling purposes. +// The ctor does nothing; users are responsible for filling in the fields. +class Registers { + public: + Registers() + : mPC{nullptr}, + mSP{nullptr}, + mFP{nullptr} +#if defined(UNWINDING_REGS_HAVE_ECX_EDX) + , + mEcx{nullptr}, + mEdx{nullptr} +#elif defined(UNWINDING_REGS_HAVE_R10_R12) + , + mR10{nullptr}, + mR12{nullptr} +#elif defined(UNWINDING_REGS_HAVE_LR_R7) + , + mLR{nullptr}, + mR7{nullptr} +#elif defined(UNWINDING_REGS_HAVE_LR_R11) + , + mLR{nullptr}, + mR11{nullptr} +#endif + { + } + + void Clear() { memset(this, 0, sizeof(*this)); } + + // These fields are filled in by + // Sampler::SuspendAndSampleAndResumeThread() for periodic and backtrace + // samples, and by REGISTERS_SYNC_POPULATE for synchronous samples. + Address mPC; // Instruction pointer. + Address mSP; // Stack pointer. + Address mFP; // Frame pointer. +#if defined(UNWINDING_REGS_HAVE_ECX_EDX) + Address mEcx; // Temp for return address. + Address mEdx; // Temp for frame pointer. +#elif defined(UNWINDING_REGS_HAVE_R10_R12) + Address mR10; // Temp for return address. + Address mR12; // Temp for frame pointer. +#elif defined(UNWINDING_REGS_HAVE_LR_R7) + Address mLR; // ARM link register, or temp for return address. + Address mR7; // Temp for frame pointer. +#elif defined(UNWINDING_REGS_HAVE_LR_R11) + Address mLR; // ARM link register, or temp for return address. + Address mR11; // Temp for frame pointer. +#endif + +#if defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) + // This contains all the registers, which means it duplicates the four fields + // above. This is ok. + ucontext_t* mContext; // The context from the signal handler or below. + ucontext_t mContextSyncStorage; // Storage for sync stack unwinding. +#endif +}; + +// Setting MAX_NATIVE_FRAMES too high risks the unwinder wasting a lot of time +// looping on corrupted stacks. +static const size_t MAX_NATIVE_FRAMES = 1024; + +struct NativeStack { + void* mPCs[MAX_NATIVE_FRAMES]; + void* mSPs[MAX_NATIVE_FRAMES]; + size_t mCount; // Number of frames filled. + + NativeStack() : mPCs(), mSPs(), mCount(0) {} +}; + +Atomic<bool> WALKING_JS_STACK(false); + +struct AutoWalkJSStack { + bool walkAllowed; + + AutoWalkJSStack() : walkAllowed(false) { + walkAllowed = WALKING_JS_STACK.compareExchange(false, true); + } + + ~AutoWalkJSStack() { + if (walkAllowed) { + WALKING_JS_STACK = false; + } + } +}; + +class StackWalkControl { + public: + struct ResumePoint { + // If lost, the stack walker should resume at these values. + void* resumeSp; // If null, stop the walker here, don't resume again. + void* resumeBp; + void* resumePc; + }; + +#if ((defined(USE_MOZ_STACK_WALK) || defined(USE_FRAME_POINTER_STACK_WALK)) && \ + defined(GP_ARCH_amd64)) + public: + static constexpr bool scIsSupported = true; + + void Clear() { mResumePointCount = 0; } + + size_t ResumePointCount() const { return mResumePointCount; } + + static constexpr size_t MaxResumePointCount() { + return scMaxResumePointCount; + } + + // Add a resume point. Note that adding anything past MaxResumePointCount() + // would silently fail. In practice this means that stack walking may still + // lose native frames. + void AddResumePoint(ResumePoint&& aResumePoint) { + // If SP is null, we expect BP and PC to also be null. + MOZ_ASSERT_IF(!aResumePoint.resumeSp, !aResumePoint.resumeBp); + MOZ_ASSERT_IF(!aResumePoint.resumeSp, !aResumePoint.resumePc); + + // If BP and/or PC are not null, SP must not be null. (But we allow BP/PC to + // be null even if SP is not null.) + MOZ_ASSERT_IF(aResumePoint.resumeBp, aResumePoint.resumeSp); + MOZ_ASSERT_IF(aResumePoint.resumePc, aResumePoint.resumeSp); + + if (mResumePointCount < scMaxResumePointCount) { + mResumePoint[mResumePointCount] = std::move(aResumePoint); + ++mResumePointCount; + } + } + + // Only allow non-modifying range-for loops. + const ResumePoint* begin() const { return &mResumePoint[0]; } + const ResumePoint* end() const { return &mResumePoint[mResumePointCount]; } + + // Find the next resume point that would be a caller of the function with the + // given SP; i.e., the resume point with the closest resumeSp > aSp. + const ResumePoint* GetResumePointCallingSp(void* aSp) const { + const ResumePoint* callingResumePoint = nullptr; + for (const ResumePoint& resumePoint : *this) { + if (resumePoint.resumeSp && // This is a potential resume point. + resumePoint.resumeSp > aSp && // It is a caller of the given SP. + (!callingResumePoint || // This is the first candidate. + resumePoint.resumeSp < callingResumePoint->resumeSp) // Or better. + ) { + callingResumePoint = &resumePoint; + } + } + return callingResumePoint; + } + + private: + size_t mResumePointCount = 0; + static constexpr size_t scMaxResumePointCount = 32; + ResumePoint mResumePoint[scMaxResumePointCount]; + +#else + public: + static constexpr bool scIsSupported = false; + // Discarded constexpr-if statements are still checked during compilation, + // these declarations are necessary for that, even if not actually used. + void Clear(); + size_t ResumePointCount(); + static constexpr size_t MaxResumePointCount(); + void AddResumePoint(ResumePoint&& aResumePoint); + const ResumePoint* begin() const; + const ResumePoint* end() const; + const ResumePoint* GetResumePointCallingSp(void* aSp) const; +#endif +}; + +// Make a copy of the JS stack into a JSFrame array, and return the number of +// copied frames. +// This copy is necessary since, like the native stack, the JS stack is iterated +// youngest-to-oldest and we need to iterate oldest-to-youngest in MergeStacks. +static uint32_t ExtractJsFrames( + bool aIsSynchronous, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const Registers& aRegs, ProfilerStackCollector& aCollector, + JsFrameBuffer aJsFrames, StackWalkControl* aStackWalkControlIfSupported) { + MOZ_ASSERT(aJsFrames, + "ExtractJsFrames should only be called if there is a " + "JsFrameBuffer to fill."); + + uint32_t jsFramesCount = 0; + + // Only walk jit stack if profiling frame iterator is turned on. + JSContext* context = aThreadData.GetJSContext(); + if (context && JS::IsProfilingEnabledForContext(context)) { + AutoWalkJSStack autoWalkJSStack; + + if (autoWalkJSStack.walkAllowed) { + JS::ProfilingFrameIterator::RegisterState registerState; + registerState.pc = aRegs.mPC; + registerState.sp = aRegs.mSP; + registerState.fp = aRegs.mFP; +#if defined(UNWINDING_REGS_HAVE_ECX_EDX) + registerState.tempRA = aRegs.mEcx; + registerState.tempFP = aRegs.mEdx; +#elif defined(UNWINDING_REGS_HAVE_R10_R12) + registerState.tempRA = aRegs.mR10; + registerState.tempFP = aRegs.mR12; +#elif defined(UNWINDING_REGS_HAVE_LR_R7) + registerState.lr = aRegs.mLR; + registerState.tempFP = aRegs.mR7; +#elif defined(UNWINDING_REGS_HAVE_LR_R11) + registerState.lr = aRegs.mLR; + registerState.tempFP = aRegs.mR11; +#endif + + // Non-periodic sampling passes Nothing() as the buffer write position to + // ProfilingFrameIterator to avoid incorrectly resetting the buffer + // position of sampled JIT frames inside the JS engine. + Maybe<uint64_t> samplePosInBuffer; + if (!aIsSynchronous) { + // aCollector.SamplePositionInBuffer() will return Nothing() when + // profiler_suspend_and_sample_thread is called from the background hang + // reporter. + samplePosInBuffer = aCollector.SamplePositionInBuffer(); + } + + for (JS::ProfilingFrameIterator jsIter(context, registerState, + samplePosInBuffer); + !jsIter.done(); ++jsIter) { + if (aIsSynchronous || jsIter.isWasm()) { + jsFramesCount += + jsIter.extractStack(aJsFrames, jsFramesCount, MAX_JS_FRAMES); + if (jsFramesCount == MAX_JS_FRAMES) { + break; + } + } else { + Maybe<JS::ProfilingFrameIterator::Frame> frame = + jsIter.getPhysicalFrameWithoutLabel(); + if (frame.isSome()) { + aJsFrames[jsFramesCount++] = std::move(frame).ref(); + if (jsFramesCount == MAX_JS_FRAMES) { + break; + } + } + } + + if constexpr (StackWalkControl::scIsSupported) { + if (aStackWalkControlIfSupported) { + jsIter.getCppEntryRegisters().apply( + [&](const JS::ProfilingFrameIterator::RegisterState& + aCppEntry) { + StackWalkControl::ResumePoint resumePoint; + resumePoint.resumeSp = aCppEntry.sp; + resumePoint.resumeBp = aCppEntry.fp; + resumePoint.resumePc = aCppEntry.pc; + aStackWalkControlIfSupported->AddResumePoint( + std::move(resumePoint)); + }); + } + } else { + MOZ_ASSERT(!aStackWalkControlIfSupported, + "aStackWalkControlIfSupported should be null when " + "!StackWalkControl::scIsSupported"); + (void)aStackWalkControlIfSupported; + } + } + } + } + + return jsFramesCount; +} + +// Merges the profiling stack, native stack, and JS stack, outputting the +// details to aCollector. +static void MergeStacks( + uint32_t aFeatures, bool aIsSynchronous, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const Registers& aRegs, const NativeStack& aNativeStack, + ProfilerStackCollector& aCollector, JsFrame* aJsFrames, + uint32_t aJsFramesCount) { + // WARNING: this function runs within the profiler's "critical section". + // WARNING: this function might be called while the profiler is inactive, and + // cannot rely on ActivePS. + + MOZ_ASSERT_IF(!aJsFrames, aJsFramesCount == 0); + + const ProfilingStack& profilingStack = aThreadData.ProfilingStackCRef(); + const js::ProfilingStackFrame* profilingStackFrames = profilingStack.frames; + uint32_t profilingStackFrameCount = profilingStack.stackSize(); + + // While the profiling stack array is ordered oldest-to-youngest, the JS and + // native arrays are ordered youngest-to-oldest. We must add frames to aInfo + // oldest-to-youngest. Thus, iterate over the profiling stack forwards and JS + // and native arrays backwards. Note: this means the terminating condition + // jsIndex and nativeIndex is being < 0. + uint32_t profilingStackIndex = 0; + int32_t jsIndex = aJsFramesCount - 1; + int32_t nativeIndex = aNativeStack.mCount - 1; + + uint8_t* lastLabelFrameStackAddr = nullptr; + uint8_t* jitEndStackAddr = nullptr; + + // Iterate as long as there is at least one frame remaining. + while (profilingStackIndex != profilingStackFrameCount || jsIndex >= 0 || + nativeIndex >= 0) { + // There are 1 to 3 frames available. Find and add the oldest. + uint8_t* profilingStackAddr = nullptr; + uint8_t* jsStackAddr = nullptr; + uint8_t* nativeStackAddr = nullptr; + uint8_t* jsActivationAddr = nullptr; + + if (profilingStackIndex != profilingStackFrameCount) { + const js::ProfilingStackFrame& profilingStackFrame = + profilingStackFrames[profilingStackIndex]; + + if (profilingStackFrame.isLabelFrame() || + profilingStackFrame.isSpMarkerFrame()) { + lastLabelFrameStackAddr = (uint8_t*)profilingStackFrame.stackAddress(); + } + + // Skip any JS_OSR frames. Such frames are used when the JS interpreter + // enters a jit frame on a loop edge (via on-stack-replacement, or OSR). + // To avoid both the profiling stack frame and jit frame being recorded + // (and showing up twice), the interpreter marks the interpreter + // profiling stack frame as JS_OSR to ensure that it doesn't get counted. + if (profilingStackFrame.isOSRFrame()) { + profilingStackIndex++; + continue; + } + + MOZ_ASSERT(lastLabelFrameStackAddr); + profilingStackAddr = lastLabelFrameStackAddr; + } + + if (jsIndex >= 0) { + jsStackAddr = (uint8_t*)aJsFrames[jsIndex].stackAddress; + jsActivationAddr = (uint8_t*)aJsFrames[jsIndex].activation; + } + + if (nativeIndex >= 0) { + nativeStackAddr = (uint8_t*)aNativeStack.mSPs[nativeIndex]; + } + + // If there's a native stack frame which has the same SP as a profiling + // stack frame, pretend we didn't see the native stack frame. Ditto for a + // native stack frame which has the same SP as a JS stack frame. In effect + // this means profiling stack frames or JS frames trump conflicting native + // frames. + if (nativeStackAddr && (profilingStackAddr == nativeStackAddr || + jsStackAddr == nativeStackAddr)) { + nativeStackAddr = nullptr; + nativeIndex--; + MOZ_ASSERT(profilingStackAddr || jsStackAddr); + } + + // Sanity checks. + MOZ_ASSERT_IF(profilingStackAddr, + profilingStackAddr != jsStackAddr && + profilingStackAddr != nativeStackAddr); + MOZ_ASSERT_IF(jsStackAddr, jsStackAddr != profilingStackAddr && + jsStackAddr != nativeStackAddr); + MOZ_ASSERT_IF(nativeStackAddr, nativeStackAddr != profilingStackAddr && + nativeStackAddr != jsStackAddr); + + // Check to see if profiling stack frame is top-most. + if (profilingStackAddr > jsStackAddr && + profilingStackAddr > nativeStackAddr) { + MOZ_ASSERT(profilingStackIndex < profilingStackFrameCount); + const js::ProfilingStackFrame& profilingStackFrame = + profilingStackFrames[profilingStackIndex]; + + // Sp marker frames are just annotations and should not be recorded in + // the profile. + if (!profilingStackFrame.isSpMarkerFrame()) { + // The JIT only allows the top-most frame to have a nullptr pc. + MOZ_ASSERT_IF( + profilingStackFrame.isJsFrame() && profilingStackFrame.script() && + !profilingStackFrame.pc(), + &profilingStackFrame == + &profilingStack.frames[profilingStack.stackSize() - 1]); + if (aIsSynchronous && profilingStackFrame.categoryPair() == + JS::ProfilingCategoryPair::PROFILER) { + // For stacks captured synchronously (ie. marker stacks), stop + // walking the stack as soon as we enter the profiler category, + // to avoid showing profiler internal code in marker stacks. + return; + } + aCollector.CollectProfilingStackFrame(profilingStackFrame); + } + profilingStackIndex++; + continue; + } + + // Check to see if JS jit stack frame is top-most + if (jsStackAddr > nativeStackAddr) { + MOZ_ASSERT(jsIndex >= 0); + const JS::ProfilingFrameIterator::Frame& jsFrame = aJsFrames[jsIndex]; + jitEndStackAddr = (uint8_t*)jsFrame.endStackAddress; + // Stringifying non-wasm JIT frames is delayed until streaming time. To + // re-lookup the entry in the JitcodeGlobalTable, we need to store the + // JIT code address (OptInfoAddr) in the circular buffer. + // + // Note that we cannot do this when we are sychronously sampling the + // current thread; that is, when called from profiler_get_backtrace. The + // captured backtrace is usually externally stored for an indeterminate + // amount of time, such as in nsRefreshDriver. Problematically, the + // stored backtrace may be alive across a GC during which the profiler + // itself is disabled. In that case, the JS engine is free to discard its + // JIT code. This means that if we inserted such OptInfoAddr entries into + // the buffer, nsRefreshDriver would now be holding on to a backtrace + // with stale JIT code return addresses. + if (aIsSynchronous || + jsFrame.kind == JS::ProfilingFrameIterator::Frame_Wasm) { + aCollector.CollectWasmFrame(jsFrame.label); + } else if (jsFrame.kind == + JS::ProfilingFrameIterator::Frame_BaselineInterpreter) { + // Materialize a ProfilingStackFrame similar to the C++ Interpreter. We + // also set the IS_BLINTERP_FRAME flag to differentiate though. + JSScript* script = jsFrame.interpreterScript; + jsbytecode* pc = jsFrame.interpreterPC(); + js::ProfilingStackFrame stackFrame; + constexpr uint32_t ExtraFlags = + uint32_t(js::ProfilingStackFrame::Flags::IS_BLINTERP_FRAME); + stackFrame.initJsFrame<JS::ProfilingCategoryPair::JS_BaselineInterpret, + ExtraFlags>("", jsFrame.label, script, pc, + jsFrame.realmID); + aCollector.CollectProfilingStackFrame(stackFrame); + } else { + MOZ_ASSERT(jsFrame.kind == JS::ProfilingFrameIterator::Frame_Ion || + jsFrame.kind == JS::ProfilingFrameIterator::Frame_Baseline); + aCollector.CollectJitReturnAddr(jsFrame.returnAddress()); + } + + jsIndex--; + continue; + } + + // If we reach here, there must be a native stack frame and it must be the + // greatest frame. + if (nativeStackAddr && + // If the latest JS frame was JIT, this could be the native frame that + // corresponds to it. In that case, skip the native frame, because + // there's no need for the same frame to be present twice in the stack. + // The JS frame can be considered the symbolicated version of the native + // frame. + (!jitEndStackAddr || nativeStackAddr < jitEndStackAddr) && + // This might still be a JIT operation, check to make sure that is not + // in range of the NEXT JavaScript's stacks' activation address. + (!jsActivationAddr || nativeStackAddr > jsActivationAddr)) { + MOZ_ASSERT(nativeIndex >= 0); + void* addr = (void*)aNativeStack.mPCs[nativeIndex]; + aCollector.CollectNativeLeafAddr(addr); + } + if (nativeIndex >= 0) { + nativeIndex--; + } + } + + // Update the JS context with the current profile sample buffer generation. + // + // Only do this for periodic samples. We don't want to do this for + // synchronous samples, and we also don't want to do it for calls to + // profiler_suspend_and_sample_thread() from the background hang reporter - + // in that case, aCollector.BufferRangeStart() will return Nothing(). + if (!aIsSynchronous) { + aCollector.BufferRangeStart().apply( + [&aThreadData](uint64_t aBufferRangeStart) { + JSContext* context = aThreadData.GetJSContext(); + if (context) { + JS::SetJSContextProfilerSampleBufferRangeStart(context, + aBufferRangeStart); + } + }); + } +} + +#if defined(USE_FRAME_POINTER_STACK_WALK) || defined(USE_MOZ_STACK_WALK) +static void StackWalkCallback(uint32_t aFrameNumber, void* aPC, void* aSP, + void* aClosure) { + NativeStack* nativeStack = static_cast<NativeStack*>(aClosure); + MOZ_ASSERT(nativeStack->mCount < MAX_NATIVE_FRAMES); + nativeStack->mSPs[nativeStack->mCount] = aSP; + nativeStack->mPCs[nativeStack->mCount] = aPC; + nativeStack->mCount++; +} +#endif + +#if defined(USE_FRAME_POINTER_STACK_WALK) +static void DoFramePointerBacktrace( + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const Registers& aRegs, NativeStack& aNativeStack, + StackWalkControl* aStackWalkControlIfSupported) { + // WARNING: this function runs within the profiler's "critical section". + // WARNING: this function might be called while the profiler is inactive, and + // cannot rely on ActivePS. + + // Make a local copy of the Registers, to allow modifications. + Registers regs = aRegs; + + // Start with the current function. We use 0 as the frame number here because + // the FramePointerStackWalk() call below will use 1..N. This is a bit weird + // but it doesn't matter because StackWalkCallback() doesn't use the frame + // number argument. + StackWalkCallback(/* frameNum */ 0, regs.mPC, regs.mSP, &aNativeStack); + + const void* const stackEnd = aThreadData.StackTop(); + + // This is to check forward-progress after using a resume point. + void* previousResumeSp = nullptr; + + for (;;) { + if (!(regs.mSP && regs.mSP <= regs.mFP && regs.mFP <= stackEnd)) { + break; + } + FramePointerStackWalk(StackWalkCallback, + uint32_t(MAX_NATIVE_FRAMES - aNativeStack.mCount), + &aNativeStack, reinterpret_cast<void**>(regs.mFP), + const_cast<void*>(stackEnd)); + + if constexpr (!StackWalkControl::scIsSupported) { + break; + } else { + if (aNativeStack.mCount >= MAX_NATIVE_FRAMES) { + // No room to add more frames. + break; + } + if (!aStackWalkControlIfSupported || + aStackWalkControlIfSupported->ResumePointCount() == 0) { + // No resume information. + break; + } + void* lastSP = aNativeStack.mSPs[aNativeStack.mCount - 1]; + if (previousResumeSp && + ((uintptr_t)lastSP <= (uintptr_t)previousResumeSp)) { + // No progress after the previous resume point. + break; + } + const StackWalkControl::ResumePoint* resumePoint = + aStackWalkControlIfSupported->GetResumePointCallingSp(lastSP); + if (!resumePoint) { + break; + } + void* sp = resumePoint->resumeSp; + if (!sp) { + // Null SP in a resume point means we stop here. + break; + } + void* pc = resumePoint->resumePc; + StackWalkCallback(/* frameNum */ aNativeStack.mCount, pc, sp, + &aNativeStack); + ++aNativeStack.mCount; + if (aNativeStack.mCount >= MAX_NATIVE_FRAMES) { + break; + } + // Prepare context to resume stack walking. + regs.mPC = (Address)pc; + regs.mSP = (Address)sp; + regs.mFP = (Address)resumePoint->resumeBp; + + previousResumeSp = sp; + } + } +} +#endif + +#if defined(USE_MOZ_STACK_WALK) +static void DoMozStackWalkBacktrace( + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const Registers& aRegs, NativeStack& aNativeStack, + StackWalkControl* aStackWalkControlIfSupported) { + // WARNING: this function runs within the profiler's "critical section". + // WARNING: this function might be called while the profiler is inactive, and + // cannot rely on ActivePS. + + // Start with the current function. We use 0 as the frame number here because + // the MozStackWalkThread() call below will use 1..N. This is a bit weird but + // it doesn't matter because StackWalkCallback() doesn't use the frame number + // argument. + StackWalkCallback(/* frameNum */ 0, aRegs.mPC, aRegs.mSP, &aNativeStack); + + HANDLE thread = aThreadData.PlatformDataCRef().ProfiledThread(); + MOZ_ASSERT(thread); + + CONTEXT context_buf; + CONTEXT* context = nullptr; + if constexpr (StackWalkControl::scIsSupported) { + context = &context_buf; + memset(&context_buf, 0, sizeof(CONTEXT)); + context_buf.ContextFlags = CONTEXT_FULL; +# if defined(_M_AMD64) + context_buf.Rsp = (DWORD64)aRegs.mSP; + context_buf.Rbp = (DWORD64)aRegs.mFP; + context_buf.Rip = (DWORD64)aRegs.mPC; +# else + static_assert(!StackWalkControl::scIsSupported, + "Mismatched support between StackWalkControl and " + "DoMozStackWalkBacktrace"); +# endif + } else { + context = nullptr; + } + + // This is to check forward-progress after using a resume point. + void* previousResumeSp = nullptr; + + for (;;) { + MozStackWalkThread(StackWalkCallback, + uint32_t(MAX_NATIVE_FRAMES - aNativeStack.mCount), + &aNativeStack, thread, context); + + if constexpr (!StackWalkControl::scIsSupported) { + break; + } else { + if (aNativeStack.mCount >= MAX_NATIVE_FRAMES) { + // No room to add more frames. + break; + } + if (!aStackWalkControlIfSupported || + aStackWalkControlIfSupported->ResumePointCount() == 0) { + // No resume information. + break; + } + void* lastSP = aNativeStack.mSPs[aNativeStack.mCount - 1]; + if (previousResumeSp && + ((uintptr_t)lastSP <= (uintptr_t)previousResumeSp)) { + // No progress after the previous resume point. + break; + } + const StackWalkControl::ResumePoint* resumePoint = + aStackWalkControlIfSupported->GetResumePointCallingSp(lastSP); + if (!resumePoint) { + break; + } + void* sp = resumePoint->resumeSp; + if (!sp) { + // Null SP in a resume point means we stop here. + break; + } + void* pc = resumePoint->resumePc; + StackWalkCallback(/* frameNum */ aNativeStack.mCount, pc, sp, + &aNativeStack); + ++aNativeStack.mCount; + if (aNativeStack.mCount >= MAX_NATIVE_FRAMES) { + break; + } + // Prepare context to resume stack walking. + memset(&context_buf, 0, sizeof(CONTEXT)); + context_buf.ContextFlags = CONTEXT_FULL; +# if defined(_M_AMD64) + context_buf.Rsp = (DWORD64)sp; + context_buf.Rbp = (DWORD64)resumePoint->resumeBp; + context_buf.Rip = (DWORD64)pc; +# else + static_assert(!StackWalkControl::scIsSupported, + "Mismatched support between StackWalkControl and " + "DoMozStackWalkBacktrace"); +# endif + previousResumeSp = sp; + } + } +} +#endif + +#ifdef USE_EHABI_STACKWALK +static void DoEHABIBacktrace( + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const Registers& aRegs, NativeStack& aNativeStack, + StackWalkControl* aStackWalkControlIfSupported) { + // WARNING: this function runs within the profiler's "critical section". + // WARNING: this function might be called while the profiler is inactive, and + // cannot rely on ActivePS. + + aNativeStack.mCount = EHABIStackWalk( + aRegs.mContext->uc_mcontext, const_cast<void*>(aThreadData.StackTop()), + aNativeStack.mSPs, aNativeStack.mPCs, MAX_NATIVE_FRAMES); + (void)aStackWalkControlIfSupported; // TODO: Implement. +} +#endif + +#ifdef USE_LUL_STACKWALK + +// See the comment at the callsite for why this function is necessary. +# if defined(MOZ_HAVE_ASAN_IGNORE) +MOZ_ASAN_IGNORE static void ASAN_memcpy(void* aDst, const void* aSrc, + size_t aLen) { + // The obvious thing to do here is call memcpy(). However, although + // ASAN_memcpy() is not instrumented by ASAN, memcpy() still is, and the + // false positive still manifests! So we must implement memcpy() ourselves + // within this function. + char* dst = static_cast<char*>(aDst); + const char* src = static_cast<const char*>(aSrc); + + for (size_t i = 0; i < aLen; i++) { + dst[i] = src[i]; + } +} +# endif + +static void DoLULBacktrace( + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const Registers& aRegs, NativeStack& aNativeStack, + StackWalkControl* aStackWalkControlIfSupported) { + // WARNING: this function runs within the profiler's "critical section". + // WARNING: this function might be called while the profiler is inactive, and + // cannot rely on ActivePS. + + (void)aStackWalkControlIfSupported; // TODO: Implement. + + const mcontext_t* mc = &aRegs.mContext->uc_mcontext; + + lul::UnwindRegs startRegs; + memset(&startRegs, 0, sizeof(startRegs)); + +# if defined(GP_PLAT_amd64_linux) || defined(GP_PLAT_amd64_android) + startRegs.xip = lul::TaggedUWord(mc->gregs[REG_RIP]); + startRegs.xsp = lul::TaggedUWord(mc->gregs[REG_RSP]); + startRegs.xbp = lul::TaggedUWord(mc->gregs[REG_RBP]); +# elif defined(GP_PLAT_amd64_freebsd) + startRegs.xip = lul::TaggedUWord(mc->mc_rip); + startRegs.xsp = lul::TaggedUWord(mc->mc_rsp); + startRegs.xbp = lul::TaggedUWord(mc->mc_rbp); +# elif defined(GP_PLAT_arm_linux) || defined(GP_PLAT_arm_android) + startRegs.r15 = lul::TaggedUWord(mc->arm_pc); + startRegs.r14 = lul::TaggedUWord(mc->arm_lr); + startRegs.r13 = lul::TaggedUWord(mc->arm_sp); + startRegs.r12 = lul::TaggedUWord(mc->arm_ip); + startRegs.r11 = lul::TaggedUWord(mc->arm_fp); + startRegs.r7 = lul::TaggedUWord(mc->arm_r7); +# elif defined(GP_PLAT_arm64_linux) || defined(GP_PLAT_arm64_android) + startRegs.pc = lul::TaggedUWord(mc->pc); + startRegs.x29 = lul::TaggedUWord(mc->regs[29]); + startRegs.x30 = lul::TaggedUWord(mc->regs[30]); + startRegs.sp = lul::TaggedUWord(mc->sp); +# elif defined(GP_PLAT_arm64_freebsd) + startRegs.pc = lul::TaggedUWord(mc->mc_gpregs.gp_elr); + startRegs.x29 = lul::TaggedUWord(mc->mc_gpregs.gp_x[29]); + startRegs.x30 = lul::TaggedUWord(mc->mc_gpregs.gp_lr); + startRegs.sp = lul::TaggedUWord(mc->mc_gpregs.gp_sp); +# elif defined(GP_PLAT_x86_linux) || defined(GP_PLAT_x86_android) + startRegs.xip = lul::TaggedUWord(mc->gregs[REG_EIP]); + startRegs.xsp = lul::TaggedUWord(mc->gregs[REG_ESP]); + startRegs.xbp = lul::TaggedUWord(mc->gregs[REG_EBP]); +# elif defined(GP_PLAT_mips64_linux) + startRegs.pc = lul::TaggedUWord(mc->pc); + startRegs.sp = lul::TaggedUWord(mc->gregs[29]); + startRegs.fp = lul::TaggedUWord(mc->gregs[30]); +# else +# error "Unknown plat" +# endif + + // Copy up to N_STACK_BYTES from rsp-REDZONE upwards, but not going past the + // stack's registered top point. Do some basic validity checks too. This + // assumes that the TaggedUWord holding the stack pointer value is valid, but + // it should be, since it was constructed that way in the code just above. + + // We could construct |stackImg| so that LUL reads directly from the stack in + // question, rather than from a copy of it. That would reduce overhead and + // space use a bit. However, it gives a problem with dynamic analysis tools + // (ASan, TSan, Valgrind) which is that such tools will report invalid or + // racing memory accesses, and such accesses will be reported deep inside LUL. + // By taking a copy here, we can either sanitise the copy (for Valgrind) or + // copy it using an unchecked memcpy (for ASan, TSan). That way we don't have + // to try and suppress errors inside LUL. + // + // N_STACK_BYTES is set to 160KB. This is big enough to hold all stacks + // observed in some minutes of testing, whilst keeping the size of this + // function (DoNativeBacktrace)'s frame reasonable. Most stacks observed in + // practice are small, 4KB or less, and so the copy costs are insignificant + // compared to other profiler overhead. + // + // |stackImg| is allocated on this (the sampling thread's) stack. That + // implies that the frame for this function is at least N_STACK_BYTES large. + // In general it would be considered unacceptable to have such a large frame + // on a stack, but it only exists for the unwinder thread, and so is not + // expected to be a problem. Allocating it on the heap is troublesome because + // this function runs whilst the sampled thread is suspended, so any heap + // allocation risks deadlock. Allocating it as a global variable is not + // thread safe, which would be a problem if we ever allow multiple sampler + // threads. Hence allocating it on the stack seems to be the least-worst + // option. + + lul::StackImage stackImg; + + { +# if defined(GP_PLAT_amd64_linux) || defined(GP_PLAT_amd64_android) || \ + defined(GP_PLAT_amd64_freebsd) + uintptr_t rEDZONE_SIZE = 128; + uintptr_t start = startRegs.xsp.Value() - rEDZONE_SIZE; +# elif defined(GP_PLAT_arm_linux) || defined(GP_PLAT_arm_android) + uintptr_t rEDZONE_SIZE = 0; + uintptr_t start = startRegs.r13.Value() - rEDZONE_SIZE; +# elif defined(GP_PLAT_arm64_linux) || defined(GP_PLAT_arm64_android) || \ + defined(GP_PLAT_arm64_freebsd) + uintptr_t rEDZONE_SIZE = 0; + uintptr_t start = startRegs.sp.Value() - rEDZONE_SIZE; +# elif defined(GP_PLAT_x86_linux) || defined(GP_PLAT_x86_android) + uintptr_t rEDZONE_SIZE = 0; + uintptr_t start = startRegs.xsp.Value() - rEDZONE_SIZE; +# elif defined(GP_PLAT_mips64_linux) + uintptr_t rEDZONE_SIZE = 0; + uintptr_t start = startRegs.sp.Value() - rEDZONE_SIZE; +# else +# error "Unknown plat" +# endif + uintptr_t end = reinterpret_cast<uintptr_t>(aThreadData.StackTop()); + uintptr_t ws = sizeof(void*); + start &= ~(ws - 1); + end &= ~(ws - 1); + uintptr_t nToCopy = 0; + if (start < end) { + nToCopy = end - start; + if (nToCopy >= 1024u * 1024u) { + // start is abnormally far from end, possibly due to some special code + // that uses a separate stack elsewhere (e.g.: rr). In this case we just + // give up on this sample. + nToCopy = 0; + } else if (nToCopy > lul::N_STACK_BYTES) { + nToCopy = lul::N_STACK_BYTES; + } + } + MOZ_ASSERT(nToCopy <= lul::N_STACK_BYTES); + stackImg.mLen = nToCopy; + stackImg.mStartAvma = start; + if (nToCopy > 0) { + // If this is a vanilla memcpy(), ASAN makes the following complaint: + // + // ERROR: AddressSanitizer: stack-buffer-underflow ... + // ... + // HINT: this may be a false positive if your program uses some custom + // stack unwind mechanism or swapcontext + // + // This code is very much a custom stack unwind mechanism! So we use an + // alternative memcpy() implementation that is ignored by ASAN. +# if defined(MOZ_HAVE_ASAN_IGNORE) + ASAN_memcpy(&stackImg.mContents[0], (void*)start, nToCopy); +# else + memcpy(&stackImg.mContents[0], (void*)start, nToCopy); +# endif + (void)VALGRIND_MAKE_MEM_DEFINED(&stackImg.mContents[0], nToCopy); + } + } + + size_t framePointerFramesAcquired = 0; + lul::LUL* lul = CorePS::Lul(); + MOZ_RELEASE_ASSERT(lul); + lul->Unwind(reinterpret_cast<uintptr_t*>(aNativeStack.mPCs), + reinterpret_cast<uintptr_t*>(aNativeStack.mSPs), + &aNativeStack.mCount, &framePointerFramesAcquired, + MAX_NATIVE_FRAMES, &startRegs, &stackImg); + + // Update stats in the LUL stats object. Unfortunately this requires + // three global memory operations. + lul->mStats.mContext += 1; + lul->mStats.mCFI += aNativeStack.mCount - 1 - framePointerFramesAcquired; + lul->mStats.mFP += framePointerFramesAcquired; +} + +#endif + +#ifdef HAVE_NATIVE_UNWIND +static void DoNativeBacktrace( + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const Registers& aRegs, NativeStack& aNativeStack, + StackWalkControl* aStackWalkControlIfSupported) { + // This method determines which stackwalker is used for periodic and + // synchronous samples. (Backtrace samples are treated differently, see + // profiler_suspend_and_sample_thread() for details). The only part of the + // ordering that matters is that LUL must precede FRAME_POINTER, because on + // Linux they can both be present. +# if defined(USE_LUL_STACKWALK) + DoLULBacktrace(aThreadData, aRegs, aNativeStack, + aStackWalkControlIfSupported); +# elif defined(USE_EHABI_STACKWALK) + DoEHABIBacktrace(aThreadData, aRegs, aNativeStack, + aStackWalkControlIfSupported); +# elif defined(USE_FRAME_POINTER_STACK_WALK) + DoFramePointerBacktrace(aThreadData, aRegs, aNativeStack, + aStackWalkControlIfSupported); +# elif defined(USE_MOZ_STACK_WALK) + DoMozStackWalkBacktrace(aThreadData, aRegs, aNativeStack, + aStackWalkControlIfSupported); +# else +# error "Invalid configuration" +# endif +} +#endif + +// Writes some components shared by periodic and synchronous profiles to +// ActivePS's ProfileBuffer. (This should only be called from DoSyncSample() +// and DoPeriodicSample().) +// +// The grammar for entry sequences is in a comment above +// ProfileBuffer::StreamSamplesToJSON. +static inline void DoSharedSample( + bool aIsSynchronous, uint32_t aFeatures, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + JsFrame* aJsFrames, const Registers& aRegs, uint64_t aSamplePos, + uint64_t aBufferRangeStart, ProfileBuffer& aBuffer, + StackCaptureOptions aCaptureOptions = StackCaptureOptions::Full) { + // WARNING: this function runs within the profiler's "critical section". + + MOZ_ASSERT(!aBuffer.IsThreadSafe(), + "Mutexes cannot be used inside this critical section"); + + ProfileBufferCollector collector(aBuffer, aSamplePos, aBufferRangeStart); + StackWalkControl* stackWalkControlIfSupported = nullptr; +#if defined(HAVE_NATIVE_UNWIND) + const bool captureNative = ProfilerFeature::HasStackWalk(aFeatures) && + aCaptureOptions == StackCaptureOptions::Full; + StackWalkControl stackWalkControl; + if constexpr (StackWalkControl::scIsSupported) { + if (captureNative) { + stackWalkControlIfSupported = &stackWalkControl; + } + } +#endif // defined(HAVE_NATIVE_UNWIND) + const uint32_t jsFramesCount = + aJsFrames ? ExtractJsFrames(aIsSynchronous, aThreadData, aRegs, collector, + aJsFrames, stackWalkControlIfSupported) + : 0; + NativeStack nativeStack; +#if defined(HAVE_NATIVE_UNWIND) + if (captureNative) { + DoNativeBacktrace(aThreadData, aRegs, nativeStack, + stackWalkControlIfSupported); + + MergeStacks(aFeatures, aIsSynchronous, aThreadData, aRegs, nativeStack, + collector, aJsFrames, jsFramesCount); + } else +#endif + { + MergeStacks(aFeatures, aIsSynchronous, aThreadData, aRegs, nativeStack, + collector, aJsFrames, jsFramesCount); + + // We can't walk the whole native stack, but we can record the top frame. + if (aCaptureOptions == StackCaptureOptions::Full) { + aBuffer.AddEntry(ProfileBufferEntry::NativeLeafAddr((void*)aRegs.mPC)); + } + } +} + +// Writes the components of a synchronous sample to the given ProfileBuffer. +static void DoSyncSample( + uint32_t aFeatures, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const TimeStamp& aNow, const Registers& aRegs, ProfileBuffer& aBuffer, + StackCaptureOptions aCaptureOptions) { + // WARNING: this function runs within the profiler's "critical section". + + MOZ_ASSERT(aCaptureOptions != StackCaptureOptions::NoStack, + "DoSyncSample should not be called when no capture is needed"); + + const uint64_t bufferRangeStart = aBuffer.BufferRangeStart(); + + const uint64_t samplePos = + aBuffer.AddThreadIdEntry(aThreadData.Info().ThreadId()); + + TimeDuration delta = aNow - CorePS::ProcessStartTime(); + aBuffer.AddEntry(ProfileBufferEntry::Time(delta.ToMilliseconds())); + + if (!aThreadData.GetJSContext()) { + // No JSContext, there is no JS frame buffer (and no need for it). + DoSharedSample(/* aIsSynchronous = */ true, aFeatures, aThreadData, + /* aJsFrames = */ nullptr, aRegs, samplePos, + bufferRangeStart, aBuffer, aCaptureOptions); + } else { + // JSContext is present, we need to lock the thread data to access the JS + // frame buffer. + ThreadRegistration::WithOnThreadRef([&](ThreadRegistration::OnThreadRef + aOnThreadRef) { + aOnThreadRef.WithConstLockedRWOnThread( + [&](const ThreadRegistration::LockedRWOnThread& aLockedThreadData) { + DoSharedSample(/* aIsSynchronous = */ true, aFeatures, aThreadData, + aLockedThreadData.GetJsFrameBuffer(), aRegs, + samplePos, bufferRangeStart, aBuffer, + aCaptureOptions); + }); + }); + } +} + +// Writes the components of a periodic sample to ActivePS's ProfileBuffer. +// The ThreadId entry is already written in the main ProfileBuffer, its location +// is `aSamplePos`, we can write the rest to `aBuffer` (which may be different). +static inline void DoPeriodicSample( + PSLockRef aLock, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const Registers& aRegs, uint64_t aSamplePos, uint64_t aBufferRangeStart, + ProfileBuffer& aBuffer) { + // WARNING: this function runs within the profiler's "critical section". + + MOZ_RELEASE_ASSERT(ActivePS::Exists(aLock)); + + JsFrameBuffer& jsFrames = CorePS::JsFrames(aLock); + DoSharedSample(/* aIsSynchronous = */ false, ActivePS::Features(aLock), + aThreadData, jsFrames, aRegs, aSamplePos, aBufferRangeStart, + aBuffer); +} + +#undef UNWINDING_REGS_HAVE_ECX_EDX +#undef UNWINDING_REGS_HAVE_R10_R12 +#undef UNWINDING_REGS_HAVE_LR_R7 +#undef UNWINDING_REGS_HAVE_LR_R11 + +// END sampling/unwinding code +//////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////// +// BEGIN saving/streaming code + +const static uint64_t kJS_MAX_SAFE_UINTEGER = +9007199254740991ULL; + +static int64_t SafeJSInteger(uint64_t aValue) { + return aValue <= kJS_MAX_SAFE_UINTEGER ? int64_t(aValue) : -1; +} + +static void AddSharedLibraryInfoToStream(JSONWriter& aWriter, + const SharedLibrary& aLib) { + aWriter.StartObjectElement(); + aWriter.IntProperty("start", SafeJSInteger(aLib.GetStart())); + aWriter.IntProperty("end", SafeJSInteger(aLib.GetEnd())); + aWriter.IntProperty("offset", SafeJSInteger(aLib.GetOffset())); + aWriter.StringProperty("name", NS_ConvertUTF16toUTF8(aLib.GetModuleName())); + aWriter.StringProperty("path", NS_ConvertUTF16toUTF8(aLib.GetModulePath())); + aWriter.StringProperty("debugName", + NS_ConvertUTF16toUTF8(aLib.GetDebugName())); + aWriter.StringProperty("debugPath", + NS_ConvertUTF16toUTF8(aLib.GetDebugPath())); + aWriter.StringProperty("breakpadId", aLib.GetBreakpadId()); + aWriter.StringProperty("codeId", aLib.GetCodeId()); + aWriter.StringProperty("arch", aLib.GetArch()); + aWriter.EndObject(); +} + +void AppendSharedLibraries(JSONWriter& aWriter, + const SharedLibraryInfo& aInfo) { + for (size_t i = 0; i < aInfo.GetSize(); i++) { + AddSharedLibraryInfoToStream(aWriter, aInfo.GetEntry(i)); + } +} + +static void StreamCategories(SpliceableJSONWriter& aWriter) { + // Same order as ProfilingCategory. Format: + // [ + // { + // name: "Idle", + // color: "transparent", + // subcategories: ["Other"], + // }, + // { + // name: "Other", + // color: "grey", + // subcategories: [ + // "JSM loading", + // "Subprocess launching", + // "DLL loading" + // ] + // }, + // ... + // ] + +#define CATEGORY_JSON_BEGIN_CATEGORY(name, labelAsString, color) \ + aWriter.Start(); \ + aWriter.StringProperty("name", labelAsString); \ + aWriter.StringProperty("color", color); \ + aWriter.StartArrayProperty("subcategories"); +#define CATEGORY_JSON_SUBCATEGORY(supercategory, name, labelAsString) \ + aWriter.StringElement(labelAsString); +#define CATEGORY_JSON_END_CATEGORY \ + aWriter.EndArray(); \ + aWriter.EndObject(); + + MOZ_PROFILING_CATEGORY_LIST(CATEGORY_JSON_BEGIN_CATEGORY, + CATEGORY_JSON_SUBCATEGORY, + CATEGORY_JSON_END_CATEGORY) + +#undef CATEGORY_JSON_BEGIN_CATEGORY +#undef CATEGORY_JSON_SUBCATEGORY +#undef CATEGORY_JSON_END_CATEGORY +} + +static void StreamMarkerSchema(SpliceableJSONWriter& aWriter) { + // Get an array view with all registered marker-type-specific functions. + base_profiler_markers_detail::Streaming::LockedMarkerTypeFunctionsList + markerTypeFunctionsArray; + // List of streamed marker names, this is used to spot duplicates. + std::set<std::string> names; + // Stream the display schema for each different one. (Duplications may come + // from the same code potentially living in different libraries.) + for (const auto& markerTypeFunctions : markerTypeFunctionsArray) { + auto name = markerTypeFunctions.mMarkerTypeNameFunction(); + // std::set.insert(T&&) returns a pair, its `second` is true if the element + // was actually inserted (i.e., it was not there yet.) + const bool didInsert = + names.insert(std::string(name.data(), name.size())).second; + if (didInsert) { + markerTypeFunctions.mMarkerSchemaFunction().Stream(aWriter, name); + } + } + + // Now stream the Rust marker schemas. Passing the names set as a void pointer + // as well, so we can continue checking if the schemes are added already in + // the Rust side. + profiler::ffi::gecko_profiler_stream_marker_schemas( + &aWriter, static_cast<void*>(&names)); +} + +// Some meta information that is better recorded before streaming the profile. +// This is *not* intended to be cached, as some values could change between +// profiling sessions. +struct PreRecordedMetaInformation { + bool mAsyncStacks; + + // This struct should only live on the stack, so it's fine to use Auto + // strings. + nsAutoCString mHttpPlatform; + nsAutoCString mHttpOscpu; + nsAutoCString mHttpMisc; + + nsAutoCString mRuntimeABI; + nsAutoCString mRuntimeToolkit; + + nsAutoCString mAppInfoProduct; + nsAutoCString mAppInfoAppBuildID; + nsAutoCString mAppInfoSourceURL; + + int32_t mProcessInfoCpuCount; + int32_t mProcessInfoCpuCores; + nsAutoCString mProcessInfoCpuName; +}; + +// This function should be called out of the profiler lock. +// It gathers non-trivial data that doesn't require the profiler to stop, or for +// which the request could theoretically deadlock if the profiler is locked. +static PreRecordedMetaInformation PreRecordMetaInformation( + bool aShutdown = false) { + MOZ_ASSERT(!PSAutoLock::IsLockedOnCurrentThread()); + + PreRecordedMetaInformation info = {}; // Aggregate-init all fields. + + if (!NS_IsMainThread()) { + // Leave these properties out if we're not on the main thread. + // At the moment, the only case in which this function is called on a + // background thread is if we're in a content process and are going to + // send this profile to the parent process. In that case, the parent + // process profile's "meta" object already has the rest of the properties, + // and the parent process profile is dumped on that process's main thread. + return info; + } + + info.mAsyncStacks = + !aShutdown && Preferences::GetBool("javascript.options.asyncstack"); + + nsresult res; + + if (nsCOMPtr<nsIHttpProtocolHandler> http = + do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX "http", &res); + !NS_FAILED(res) && http) { + Unused << http->GetPlatform(info.mHttpPlatform); + +#if defined(GP_OS_darwin) + // On Mac, the http "oscpu" is capped at 10.15, so we need to get the real + // OS version directly. + int major = 0; + int minor = 0; + int bugfix = 0; + nsCocoaFeatures::GetSystemVersion(major, minor, bugfix); + if (major != 0) { + info.mHttpOscpu.AppendLiteral("macOS "); + info.mHttpOscpu.AppendInt(major); + info.mHttpOscpu.AppendLiteral("."); + info.mHttpOscpu.AppendInt(minor); + info.mHttpOscpu.AppendLiteral("."); + info.mHttpOscpu.AppendInt(bugfix); + } else +#endif +#if defined(GP_OS_windows) + // On Windows, the http "oscpu" is capped at Windows 10, so we need to get + // the real OS version directly. + OSVERSIONINFO ovi = {sizeof(OSVERSIONINFO)}; + if (GetVersionEx(&ovi)) { + info.mHttpOscpu.AppendLiteral("Windows "); + // The major version returned for Windows 11 is 10, but we can + // identify it from the build number. + info.mHttpOscpu.AppendInt( + ovi.dwBuildNumber >= 22000 ? 11 : int32_t(ovi.dwMajorVersion)); + info.mHttpOscpu.AppendLiteral("."); + info.mHttpOscpu.AppendInt(int32_t(ovi.dwMinorVersion)); +# if defined(_ARM64_) + info.mHttpOscpu.AppendLiteral(" Arm64"); +# endif + info.mHttpOscpu.AppendLiteral("; build="); + info.mHttpOscpu.AppendInt(int32_t(ovi.dwBuildNumber)); + } else +#endif + { + Unused << http->GetOscpu(info.mHttpOscpu); + } + + // Firefox version is capped to 109.0 in the http "misc" field due to some + // webcompat issues (Bug 1805967). We need to put the real version instead. + info.mHttpMisc.AssignLiteral("rv:"); + info.mHttpMisc.AppendLiteral(MOZILLA_UAVERSION); + } + + if (nsCOMPtr<nsIXULRuntime> runtime = + do_GetService("@mozilla.org/xre/runtime;1"); + runtime) { + Unused << runtime->GetXPCOMABI(info.mRuntimeABI); + Unused << runtime->GetWidgetToolkit(info.mRuntimeToolkit); + } + + if (nsCOMPtr<nsIXULAppInfo> appInfo = + do_GetService("@mozilla.org/xre/app-info;1"); + appInfo) { + Unused << appInfo->GetName(info.mAppInfoProduct); + Unused << appInfo->GetAppBuildID(info.mAppInfoAppBuildID); + Unused << appInfo->GetSourceURL(info.mAppInfoSourceURL); + } + + ProcessInfo processInfo = {}; // Aggregate-init all fields to false/zeroes. + if (NS_SUCCEEDED(CollectProcessInfo(processInfo))) { + info.mProcessInfoCpuCount = processInfo.cpuCount; + info.mProcessInfoCpuCores = processInfo.cpuCores; + info.mProcessInfoCpuName = processInfo.cpuName; + } + + return info; +} + +// Implemented in platform-specific cpps, to add object properties describing +// the units of CPU measurements in samples. +static void StreamMetaPlatformSampleUnits(PSLockRef aLock, + SpliceableJSONWriter& aWriter); + +static void StreamMetaJSCustomObject( + PSLockRef aLock, SpliceableJSONWriter& aWriter, bool aIsShuttingDown, + const PreRecordedMetaInformation& aPreRecordedMetaInformation) { + MOZ_RELEASE_ASSERT(CorePS::Exists() && ActivePS::Exists(aLock)); + + aWriter.IntProperty("version", GECKO_PROFILER_FORMAT_VERSION); + + // The "startTime" field holds the number of milliseconds since midnight + // January 1, 1970 GMT. This grotty code computes (Now - (Now - + // ProcessStartTime)) to convert CorePS::ProcessStartTime() into that form. + // Note: This is the only absolute time in the profile! All other timestamps + // are relative to this startTime. + TimeDuration delta = TimeStamp::Now() - CorePS::ProcessStartTime(); + aWriter.DoubleProperty( + "startTime", + static_cast<double>(PR_Now() / 1000.0 - delta.ToMilliseconds())); + + aWriter.DoubleProperty("profilingStartTime", (ActivePS::ProfilingStartTime() - + CorePS::ProcessStartTime()) + .ToMilliseconds()); + + if (const TimeStamp contentEarliestTime = + ActivePS::Buffer(aLock) + .UnderlyingChunkedBuffer() + .GetEarliestChunkStartTimeStamp(); + !contentEarliestTime.IsNull()) { + aWriter.DoubleProperty( + "contentEarliestTime", + (contentEarliestTime - CorePS::ProcessStartTime()).ToMilliseconds()); + } else { + aWriter.NullProperty("contentEarliestTime"); + } + + const double profilingEndTime = profiler_time(); + aWriter.DoubleProperty("profilingEndTime", profilingEndTime); + + if (aIsShuttingDown) { + aWriter.DoubleProperty("shutdownTime", profilingEndTime); + } else { + aWriter.NullProperty("shutdownTime"); + } + + aWriter.StartArrayProperty("categories"); + StreamCategories(aWriter); + aWriter.EndArray(); + + aWriter.StartArrayProperty("markerSchema"); + StreamMarkerSchema(aWriter); + aWriter.EndArray(); + + ActivePS::WriteActiveConfiguration(aLock, aWriter, + MakeStringSpan("configuration")); + + if (!NS_IsMainThread()) { + // Leave the rest of the properties out if we're not on the main thread. + // At the moment, the only case in which this function is called on a + // background thread is if we're in a content process and are going to + // send this profile to the parent process. In that case, the parent + // process profile's "meta" object already has the rest of the properties, + // and the parent process profile is dumped on that process's main thread. + return; + } + + aWriter.DoubleProperty("interval", ActivePS::Interval(aLock)); + aWriter.IntProperty("stackwalk", ActivePS::FeatureStackWalk(aLock)); + +#ifdef DEBUG + aWriter.IntProperty("debug", 1); +#else + aWriter.IntProperty("debug", 0); +#endif + + aWriter.IntProperty("gcpoison", JS::IsGCPoisoning() ? 1 : 0); + + aWriter.IntProperty("asyncstack", aPreRecordedMetaInformation.mAsyncStacks); + + aWriter.IntProperty("processType", XRE_GetProcessType()); + + aWriter.StringProperty("updateChannel", MOZ_STRINGIFY(MOZ_UPDATE_CHANNEL)); + + if (!aPreRecordedMetaInformation.mHttpPlatform.IsEmpty()) { + aWriter.StringProperty("platform", + aPreRecordedMetaInformation.mHttpPlatform); + } + if (!aPreRecordedMetaInformation.mHttpOscpu.IsEmpty()) { + aWriter.StringProperty("oscpu", aPreRecordedMetaInformation.mHttpOscpu); + } + if (!aPreRecordedMetaInformation.mHttpMisc.IsEmpty()) { + aWriter.StringProperty("misc", aPreRecordedMetaInformation.mHttpMisc); + } + + if (!aPreRecordedMetaInformation.mRuntimeABI.IsEmpty()) { + aWriter.StringProperty("abi", aPreRecordedMetaInformation.mRuntimeABI); + } + if (!aPreRecordedMetaInformation.mRuntimeToolkit.IsEmpty()) { + aWriter.StringProperty("toolkit", + aPreRecordedMetaInformation.mRuntimeToolkit); + } + + if (!aPreRecordedMetaInformation.mAppInfoProduct.IsEmpty()) { + aWriter.StringProperty("product", + aPreRecordedMetaInformation.mAppInfoProduct); + } + if (!aPreRecordedMetaInformation.mAppInfoAppBuildID.IsEmpty()) { + aWriter.StringProperty("appBuildID", + aPreRecordedMetaInformation.mAppInfoAppBuildID); + } + if (!aPreRecordedMetaInformation.mAppInfoSourceURL.IsEmpty()) { + aWriter.StringProperty("sourceURL", + aPreRecordedMetaInformation.mAppInfoSourceURL); + } + + if (!aPreRecordedMetaInformation.mProcessInfoCpuName.IsEmpty()) { + aWriter.StringProperty("CPUName", + aPreRecordedMetaInformation.mProcessInfoCpuName); + } + if (aPreRecordedMetaInformation.mProcessInfoCpuCores > 0) { + aWriter.IntProperty("physicalCPUs", + aPreRecordedMetaInformation.mProcessInfoCpuCores); + } + if (aPreRecordedMetaInformation.mProcessInfoCpuCount > 0) { + aWriter.IntProperty("logicalCPUs", + aPreRecordedMetaInformation.mProcessInfoCpuCount); + } + +#if defined(GP_OS_android) + jni::String::LocalRef deviceInformation = + java::GeckoJavaSampler::GetDeviceInformation(); + aWriter.StringProperty("device", deviceInformation->ToCString()); +#endif + + aWriter.StartObjectProperty("sampleUnits"); + { + aWriter.StringProperty("time", "ms"); + aWriter.StringProperty("eventDelay", "ms"); + StreamMetaPlatformSampleUnits(aLock, aWriter); + } + aWriter.EndObject(); + + // We should avoid collecting extension metadata for profiler when there is no + // observer service, since a ExtensionPolicyService could not be created then. + if (nsCOMPtr<nsIObserverService> os = services::GetObserverService()) { + aWriter.StartObjectProperty("extensions"); + { + { + JSONSchemaWriter schema(aWriter); + schema.WriteField("id"); + schema.WriteField("name"); + schema.WriteField("baseURL"); + } + + aWriter.StartArrayProperty("data"); + { + nsTArray<RefPtr<WebExtensionPolicy>> exts; + ExtensionPolicyService::GetSingleton().GetAll(exts); + + for (auto& ext : exts) { + aWriter.StartArrayElement(); + + nsAutoString id; + ext->GetId(id); + aWriter.StringElement(NS_ConvertUTF16toUTF8(id)); + + aWriter.StringElement(NS_ConvertUTF16toUTF8(ext->Name())); + + auto url = ext->GetURL(u""_ns); + if (url.isOk()) { + aWriter.StringElement(NS_ConvertUTF16toUTF8(url.unwrap())); + } + + aWriter.EndArray(); + } + } + aWriter.EndArray(); + } + aWriter.EndObject(); + } +} + +static void StreamPages(PSLockRef aLock, SpliceableJSONWriter& aWriter) { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + ActivePS::DiscardExpiredPages(aLock); + for (const auto& page : ActivePS::ProfiledPages(aLock)) { + page->StreamJSON(aWriter); + } +} + +#if defined(GP_OS_android) +template <int N> +static bool StartsWith(const nsACString& string, const char (&prefix)[N]) { + if (N - 1 > string.Length()) { + return false; + } + return memcmp(string.Data(), prefix, N - 1) == 0; +} + +static JS::ProfilingCategoryPair InferJavaCategory(nsACString& aName) { + if (aName.EqualsLiteral("android.os.MessageQueue.nativePollOnce()")) { + return JS::ProfilingCategoryPair::IDLE; + } + if (aName.EqualsLiteral("java.lang.Object.wait()")) { + return JS::ProfilingCategoryPair::JAVA_BLOCKED; + } + if (StartsWith(aName, "android.") || StartsWith(aName, "com.android.")) { + return JS::ProfilingCategoryPair::JAVA_ANDROID; + } + if (StartsWith(aName, "mozilla.") || StartsWith(aName, "org.mozilla.")) { + return JS::ProfilingCategoryPair::JAVA_MOZILLA; + } + if (StartsWith(aName, "java.") || StartsWith(aName, "sun.") || + StartsWith(aName, "com.sun.")) { + return JS::ProfilingCategoryPair::JAVA_LANGUAGE; + } + if (StartsWith(aName, "kotlin.") || StartsWith(aName, "kotlinx.")) { + return JS::ProfilingCategoryPair::JAVA_KOTLIN; + } + if (StartsWith(aName, "androidx.")) { + return JS::ProfilingCategoryPair::JAVA_ANDROIDX; + } + return JS::ProfilingCategoryPair::OTHER; +} + +// Marker type for Java markers without any details. +struct JavaMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("Java"); + } + static void StreamJSONMarkerData( + baseprofiler::SpliceableJSONWriter& aWriter) {} + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::TimelineOverview, MS::Location::MarkerChart, + MS::Location::MarkerTable}; + schema.SetAllLabels("{marker.name}"); + return schema; + } +}; + +// Marker type for Java markers with a detail field. +struct JavaMarkerWithDetails { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("JavaWithDetails"); + } + static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter, + const ProfilerString8View& aText) { + // This (currently) needs to be called "name" to be searchable on the + // front-end. + aWriter.StringProperty("name", aText); + } + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::TimelineOverview, MS::Location::MarkerChart, + MS::Location::MarkerTable}; + schema.SetTooltipLabel("{marker.name}"); + schema.SetChartLabel("{marker.data.name}"); + schema.SetTableLabel("{marker.name} - {marker.data.name}"); + schema.AddKeyLabelFormatSearchable("name", "Details", MS::Format::String, + MS::Searchable::Searchable); + return schema; + } +}; + +static void CollectJavaThreadProfileData( + nsTArray<java::GeckoJavaSampler::ThreadInfo::LocalRef>& javaThreads, + ProfileBuffer& aProfileBuffer) { + // Retrieve metadata about the threads. + const auto threadCount = java::GeckoJavaSampler::GetRegisteredThreadCount(); + for (int i = 0; i < threadCount; i++) { + javaThreads.AppendElement( + java::GeckoJavaSampler::GetRegisteredThreadInfo(i)); + } + + // locked_profiler_start uses sample count is 1000 for Java thread. + // This entry size is enough now, but we might have to estimate it + // if we can customize it + // Pass the samples + int sampleId = 0; + while (true) { + const auto threadId = java::GeckoJavaSampler::GetThreadId(sampleId); + double sampleTime = java::GeckoJavaSampler::GetSampleTime(sampleId); + if (threadId == 0 || sampleTime == 0.0) { + break; + } + + aProfileBuffer.AddThreadIdEntry(ProfilerThreadId::FromNumber(threadId)); + aProfileBuffer.AddEntry(ProfileBufferEntry::Time(sampleTime)); + int frameId = 0; + while (true) { + jni::String::LocalRef frameName = + java::GeckoJavaSampler::GetFrameName(sampleId, frameId++); + if (!frameName) { + break; + } + nsCString frameNameString = frameName->ToCString(); + + auto categoryPair = InferJavaCategory(frameNameString); + aProfileBuffer.CollectCodeLocation("", frameNameString.get(), 0, 0, + Nothing(), Nothing(), + Some(categoryPair)); + } + sampleId++; + } + + // Pass the markers now + while (true) { + // Gets the data from the Android UI thread only. + java::GeckoJavaSampler::Marker::LocalRef marker = + java::GeckoJavaSampler::PollNextMarker(); + if (!marker) { + // All markers are transferred. + break; + } + + // Get all the marker information from the Java thread using JNI. + const auto threadId = ProfilerThreadId::FromNumber(marker->GetThreadId()); + nsCString markerName = marker->GetMarkerName()->ToCString(); + jni::String::LocalRef text = marker->GetMarkerText(); + TimeStamp startTime = + CorePS::ProcessStartTime() + + TimeDuration::FromMilliseconds(marker->GetStartTime()); + + double endTimeMs = marker->GetEndTime(); + // A marker can be either a duration with start and end, or a point in time + // with only startTime. If endTime is 0, this means it's a point in time. + TimeStamp endTime = endTimeMs == 0 + ? startTime + : CorePS::ProcessStartTime() + + TimeDuration::FromMilliseconds(endTimeMs); + MarkerTiming timing = endTimeMs == 0 + ? MarkerTiming::InstantAt(startTime) + : MarkerTiming::Interval(startTime, endTime); + + if (!text) { + // This marker doesn't have a text. + AddMarkerToBuffer(aProfileBuffer.UnderlyingChunkedBuffer(), markerName, + geckoprofiler::category::JAVA_ANDROID, + {MarkerThreadId(threadId), std::move(timing)}, + JavaMarker{}); + } else { + // This marker has a text. + AddMarkerToBuffer(aProfileBuffer.UnderlyingChunkedBuffer(), markerName, + geckoprofiler::category::JAVA_ANDROID, + {MarkerThreadId(threadId), std::move(timing)}, + JavaMarkerWithDetails{}, text->ToCString()); + } + } +} +#endif + +UniquePtr<ProfilerCodeAddressService> +profiler_code_address_service_for_presymbolication() { + static const bool preSymbolicate = []() { + const char* symbolicate = getenv("MOZ_PROFILER_SYMBOLICATE"); + return symbolicate && symbolicate[0] != '\0'; + }(); + return preSymbolicate ? MakeUnique<ProfilerCodeAddressService>() : nullptr; +} + +static ProfilerResult<ProfileGenerationAdditionalInformation> +locked_profiler_stream_json_for_this_process( + PSLockRef aLock, SpliceableJSONWriter& aWriter, double aSinceTime, + const PreRecordedMetaInformation& aPreRecordedMetaInformation, + bool aIsShuttingDown, ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger) { + LOG("locked_profiler_stream_json_for_this_process"); + +#ifdef DEBUG + PRIntervalTime slowWithSleeps = 0; + if (!XRE_IsParentProcess()) { + for (const auto& filter : ActivePS::Filters(aLock)) { + if (filter == "test-debug-child-slow-json") { + LOG("test-debug-child-slow-json"); + // There are 10 slow-downs below, each will sleep 250ms, for a total of + // 2.5s, which should trigger the first progress request after 1s, and + // the next progress which will have advanced further, so this profile + // shouldn't get dropped. + slowWithSleeps = PR_MillisecondsToInterval(250); + } else if (filter == "test-debug-child-very-slow-json") { + LOG("test-debug-child-very-slow-json"); + // Wait for more than 2s without any progress, which should get this + // profile discarded. + PR_Sleep(PR_SecondsToInterval(5)); + } + } + } +# define SLOW_DOWN_FOR_TESTING() \ + if (slowWithSleeps != 0) { \ + DEBUG_LOG("progress=%.0f%%, sleep...", \ + aProgressLogger.GetGlobalProgress().ToDouble() * 100.0); \ + PR_Sleep(slowWithSleeps); \ + } +#else // #ifdef DEBUG +# define SLOW_DOWN_FOR_TESTING() /* No slow-downs */ +#endif // #ifdef DEBUG #else + + MOZ_RELEASE_ASSERT(CorePS::Exists() && ActivePS::Exists(aLock)); + + AUTO_PROFILER_STATS(locked_profiler_stream_json_for_this_process); + + const double collectionStartMs = profiler_time(); + + ProfileBuffer& buffer = ActivePS::Buffer(aLock); + + aProgressLogger.SetLocalProgress(1_pc, "Locked profile buffer"); + + SLOW_DOWN_FOR_TESTING(); + + // If there is a set "Window length", discard older data. + Maybe<double> durationS = ActivePS::Duration(aLock); + if (durationS.isSome()) { + const double durationStartMs = collectionStartMs - *durationS * 1000; + buffer.DiscardSamplesBeforeTime(durationStartMs); + } + aProgressLogger.SetLocalProgress(2_pc, "Discarded old data"); + + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + SLOW_DOWN_FOR_TESTING(); + +#if defined(GP_OS_android) + // Java thread profile data should be collected before serializing the meta + // object. This is because Java thread adds some markers with marker schema + // objects. And these objects should be added before the serialization of the + // `profile.meta.markerSchema` array, so these marker schema objects can also + // be serialized properly. That's why java thread profile data needs to be + // done before everything. + + // We are allocating it chunk by chunk. So this will not allocate 64 MiB + // at once. This size should be more than enough for java threads. + // This buffer is being created for each process but Android has + // relatively fewer processes compared to desktop, so it's okay here. + mozilla::ProfileBufferChunkManagerWithLocalLimit javaChunkManager( + 64 * 1024 * 1024, 1024 * 1024); + ProfileChunkedBuffer javaBufferManager( + ProfileChunkedBuffer::ThreadSafety::WithoutMutex, javaChunkManager); + ProfileBuffer javaBuffer(javaBufferManager); + + nsTArray<java::GeckoJavaSampler::ThreadInfo::LocalRef> javaThreads; + + if (ActivePS::FeatureJava(aLock)) { + CollectJavaThreadProfileData(javaThreads, javaBuffer); + aProgressLogger.SetLocalProgress(3_pc, "Collected Java thread"); + } +#endif + + // Put shared library info + aWriter.StartArrayProperty("libs"); + SharedLibraryInfo sharedLibraryInfo = SharedLibraryInfo::GetInfoForSelf(); + sharedLibraryInfo.SortByAddress(); + AppendSharedLibraries(aWriter, sharedLibraryInfo); + aWriter.EndArray(); + aProgressLogger.SetLocalProgress(4_pc, "Wrote library information"); + + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + SLOW_DOWN_FOR_TESTING(); + + // Put meta data + aWriter.StartObjectProperty("meta"); + { + StreamMetaJSCustomObject(aLock, aWriter, aIsShuttingDown, + aPreRecordedMetaInformation); + } + aWriter.EndObject(); + aProgressLogger.SetLocalProgress(5_pc, "Wrote profile metadata"); + + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + SLOW_DOWN_FOR_TESTING(); + + // Put page data + aWriter.StartArrayProperty("pages"); + { StreamPages(aLock, aWriter); } + aWriter.EndArray(); + aProgressLogger.SetLocalProgress(6_pc, "Wrote pages"); + + buffer.StreamProfilerOverheadToJSON( + aWriter, CorePS::ProcessStartTime(), aSinceTime, + aProgressLogger.CreateSubLoggerTo(10_pc, "Wrote profiler overheads")); + + buffer.StreamCountersToJSON( + aWriter, CorePS::ProcessStartTime(), aSinceTime, + aProgressLogger.CreateSubLoggerTo(14_pc, "Wrote counters")); + + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + SLOW_DOWN_FOR_TESTING(); + + // Lists the samples for each thread profile + aWriter.StartArrayProperty("threads"); + { + ActivePS::DiscardExpiredDeadProfiledThreads(aLock); + aProgressLogger.SetLocalProgress(15_pc, "Discarded expired profiles"); + + ThreadRegistry::LockedRegistry lockedRegistry; + ActivePS::ProfiledThreadList threads = + ActivePS::ProfiledThreads(lockedRegistry, aLock); + + const uint32_t threadCount = uint32_t(threads.length()); + + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + SLOW_DOWN_FOR_TESTING(); + + // Prepare the streaming context for each thread. + ProcessStreamingContext processStreamingContext( + threadCount, aWriter.SourceFailureLatch(), CorePS::ProcessStartTime(), + aSinceTime); + for (auto&& [i, progressLogger] : aProgressLogger.CreateLoopSubLoggersTo( + 20_pc, threadCount, "Preparing thread streaming contexts...")) { + ActivePS::ProfiledThreadListElement& thread = threads[i]; + MOZ_RELEASE_ASSERT(thread.mProfiledThreadData); + processStreamingContext.AddThreadStreamingContext( + *thread.mProfiledThreadData, buffer, thread.mJSContext, aService, + std::move(progressLogger)); + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + } + + SLOW_DOWN_FOR_TESTING(); + + // Read the buffer once, and extract all samples and markers that the + // context expects. + buffer.StreamSamplesAndMarkersToJSON( + processStreamingContext, aProgressLogger.CreateSubLoggerTo( + "Processing samples and markers...", 80_pc, + "Processed samples and markers")); + + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + SLOW_DOWN_FOR_TESTING(); + + // Stream each thread from the pre-filled context. + ThreadStreamingContext* const contextListBegin = + processStreamingContext.begin(); + MOZ_ASSERT(uint32_t(processStreamingContext.end() - contextListBegin) == + threadCount); + for (auto&& [i, progressLogger] : aProgressLogger.CreateLoopSubLoggersTo( + 92_pc, threadCount, "Streaming threads...")) { + ThreadStreamingContext& threadStreamingContext = contextListBegin[i]; + threadStreamingContext.FinalizeWriter(); + threadStreamingContext.mProfiledThreadData.StreamJSON( + std::move(threadStreamingContext), aWriter, + CorePS::ProcessName(aLock), CorePS::ETLDplus1(aLock), + CorePS::ProcessStartTime(), aService, std::move(progressLogger)); + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + } + aProgressLogger.SetLocalProgress(92_pc, "Wrote samples and markers"); + +#if defined(GP_OS_android) + if (ActivePS::FeatureJava(aLock)) { + for (java::GeckoJavaSampler::ThreadInfo::LocalRef& threadInfo : + javaThreads) { + ProfiledThreadData threadData(ThreadRegistrationInfo{ + threadInfo->GetName()->ToCString().BeginReading(), + ProfilerThreadId::FromNumber(threadInfo->GetId()), false, + CorePS::ProcessStartTime()}); + + threadData.StreamJSON( + javaBuffer, nullptr, aWriter, CorePS::ProcessName(aLock), + CorePS::ETLDplus1(aLock), CorePS::ProcessStartTime(), aSinceTime, + nullptr, + aProgressLogger.CreateSubLoggerTo("Streaming Java thread...", 96_pc, + "Streamed Java thread")); + } + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + } else { + aProgressLogger.SetLocalProgress(96_pc, "No Java thread"); + } +#endif + + UniquePtr<char[]> baseProfileThreads = + ActivePS::MoveBaseProfileThreads(aLock); + if (baseProfileThreads) { + aWriter.Splice(MakeStringSpan(baseProfileThreads.get())); + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + aProgressLogger.SetLocalProgress(97_pc, "Wrote baseprofiler data"); + } else { + aProgressLogger.SetLocalProgress(97_pc, "No baseprofiler data"); + } + } + aWriter.EndArray(); + + SLOW_DOWN_FOR_TESTING(); + + aWriter.StartArrayProperty("pausedRanges"); + { + buffer.StreamPausedRangesToJSON( + aWriter, aSinceTime, + aProgressLogger.CreateSubLoggerTo("Streaming pauses...", 99_pc, + "Streamed pauses")); + } + aWriter.EndArray(); + + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + + ProfilingLog::Access([&](Json::Value& aProfilingLogObject) { + aProfilingLogObject[Json::StaticString{ + "profilingLogEnd" TIMESTAMP_JSON_SUFFIX}] = ProfilingLog::Timestamp(); + + aWriter.StartObjectProperty("profilingLog"); + { + nsAutoCString pid; + pid.AppendInt(int64_t(profiler_current_process_id().ToNumber())); + Json::String logString = ToCompactString(aProfilingLogObject); + aWriter.SplicedJSONProperty(pid, logString); + } + aWriter.EndObject(); + }); + + const double collectionEndMs = profiler_time(); + + // Record timestamps for the collection into the buffer, so that consumers + // know why we didn't collect any samples for its duration. + // We put these entries into the buffer after we've collected the profile, + // so they'll be visible for the *next* profile collection (if they haven't + // been overwritten due to buffer wraparound by then). + buffer.AddEntry(ProfileBufferEntry::CollectionStart(collectionStartMs)); + buffer.AddEntry(ProfileBufferEntry::CollectionEnd(collectionEndMs)); + +#ifdef DEBUG + if (slowWithSleeps != 0) { + LOG("locked_profiler_stream_json_for_this_process done"); + } +#endif // DEBUG + + return ProfileGenerationAdditionalInformation{std::move(sharedLibraryInfo)}; +} + +// Keep this internal function non-static, so it may be used by tests. +ProfilerResult<ProfileGenerationAdditionalInformation> +do_profiler_stream_json_for_this_process( + SpliceableJSONWriter& aWriter, double aSinceTime, bool aIsShuttingDown, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger) { + LOG("profiler_stream_json_for_this_process"); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + const auto preRecordedMetaInformation = PreRecordMetaInformation(); + + aProgressLogger.SetLocalProgress(2_pc, "PreRecordMetaInformation done"); + + if (profiler_is_active()) { + invoke_profiler_state_change_callbacks(ProfilingState::GeneratingProfile); + } + + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return Err(ProfilerError::IsInactive); + } + + ProfileGenerationAdditionalInformation additionalInfo; + MOZ_TRY_VAR( + additionalInfo, + locked_profiler_stream_json_for_this_process( + lock, aWriter, aSinceTime, preRecordedMetaInformation, + aIsShuttingDown, aService, + aProgressLogger.CreateSubLoggerFromTo( + 3_pc, "locked_profiler_stream_json_for_this_process started", + 100_pc, "locked_profiler_stream_json_for_this_process done"))); + + if (aWriter.Failed()) { + return Err(ProfilerError::JsonGenerationFailed); + } + return additionalInfo; +} + +ProfilerResult<ProfileGenerationAdditionalInformation> +profiler_stream_json_for_this_process(SpliceableJSONWriter& aWriter, + double aSinceTime, bool aIsShuttingDown, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger) { + MOZ_RELEASE_ASSERT( + !XRE_IsParentProcess() || NS_IsMainThread(), + "In the parent process, profiles should only be generated from the main " + "thread, otherwise they will be incomplete."); + + ProfileGenerationAdditionalInformation additionalInfo; + MOZ_TRY_VAR(additionalInfo, do_profiler_stream_json_for_this_process( + aWriter, aSinceTime, aIsShuttingDown, + aService, std::move(aProgressLogger))); + + return additionalInfo; +} + +// END saving/streaming code +//////////////////////////////////////////////////////////////////////// + +static char FeatureCategory(uint32_t aFeature) { + if (aFeature & DefaultFeatures()) { + if (aFeature & AvailableFeatures()) { + return 'D'; + } + return 'd'; + } + + if (aFeature & StartupExtraDefaultFeatures()) { + if (aFeature & AvailableFeatures()) { + return 'S'; + } + return 's'; + } + + if (aFeature & AvailableFeatures()) { + return '-'; + } + return 'x'; +} + +static void PrintUsage() { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + + printf( + "\n" + "Profiler environment variable usage:\n" + "\n" + " MOZ_PROFILER_HELP\n" + " If set to any value, prints this message.\n" + " Use MOZ_BASE_PROFILER_HELP for BaseProfiler help.\n" + "\n" + " MOZ_LOG\n" + " Enables logging. The levels of logging available are\n" + " 'prof:3' (least verbose), 'prof:4', 'prof:5' (most verbose).\n" + "\n" + " MOZ_PROFILER_STARTUP\n" + " If set to any value other than '' or '0'/'N'/'n', starts the\n" + " profiler immediately on start-up.\n" + " Useful if you want profile code that runs very early.\n" + "\n" + " MOZ_PROFILER_STARTUP_ENTRIES=<%u..%u>\n" + " If MOZ_PROFILER_STARTUP is set, specifies the number of entries per\n" + " process in the profiler's circular buffer when the profiler is first\n" + " started.\n" + " If unset, the platform default is used:\n" + " %u entries per process, or %u when MOZ_PROFILER_STARTUP is set.\n" + " (%u bytes per entry -> %u or %u total bytes per process)\n" + " Optional units in bytes: KB, KiB, MB, MiB, GB, GiB\n" + "\n" + " MOZ_PROFILER_STARTUP_DURATION=<1..>\n" + " If MOZ_PROFILER_STARTUP is set, specifies the maximum life time of\n" + " entries in the the profiler's circular buffer when the profiler is\n" + " first started, in seconds.\n" + " If unset, the life time of the entries will only be restricted by\n" + " MOZ_PROFILER_STARTUP_ENTRIES (or its default value), and no\n" + " additional time duration restriction will be applied.\n" + "\n" + " MOZ_PROFILER_STARTUP_INTERVAL=<1..%d>\n" + " If MOZ_PROFILER_STARTUP is set, specifies the sample interval,\n" + " measured in milliseconds, when the profiler is first started.\n" + " If unset, the platform default is used.\n" + "\n" + " MOZ_PROFILER_STARTUP_FEATURES_BITFIELD=<Number>\n" + " If MOZ_PROFILER_STARTUP is set, specifies the profiling features, as\n" + " the integer value of the features bitfield.\n" + " If unset, the value from MOZ_PROFILER_STARTUP_FEATURES is used.\n" + "\n" + " MOZ_PROFILER_STARTUP_FEATURES=<Features>\n" + " If MOZ_PROFILER_STARTUP is set, specifies the profiling features, as\n" + " a comma-separated list of strings.\n" + " Ignored if MOZ_PROFILER_STARTUP_FEATURES_BITFIELD is set.\n" + " If unset, the platform default is used.\n" + "\n" + " Features: (x=unavailable, D/d=default/unavailable,\n" + " S/s=MOZ_PROFILER_STARTUP extra default/unavailable)\n", + unsigned(scMinimumBufferEntries), unsigned(scMaximumBufferEntries), + unsigned(PROFILER_DEFAULT_ENTRIES.Value()), + unsigned(PROFILER_DEFAULT_STARTUP_ENTRIES.Value()), + unsigned(scBytesPerEntry), + unsigned(PROFILER_DEFAULT_ENTRIES.Value() * scBytesPerEntry), + unsigned(PROFILER_DEFAULT_STARTUP_ENTRIES.Value() * scBytesPerEntry), + PROFILER_MAX_INTERVAL); + +#define PRINT_FEATURE(n_, str_, Name_, desc_) \ + printf(" %c %7u: \"%s\" (%s)\n", FeatureCategory(ProfilerFeature::Name_), \ + ProfilerFeature::Name_, str_, desc_); + + PROFILER_FOR_EACH_FEATURE(PRINT_FEATURE) + +#undef PRINT_FEATURE + + printf( + " - \"default\" (All above D+S defaults)\n" + "\n" + " MOZ_PROFILER_STARTUP_FILTERS=<Filters>\n" + " If MOZ_PROFILER_STARTUP is set, specifies the thread filters, as a\n" + " comma-separated list of strings. A given thread will be sampled if\n" + " any of the filters is a case-insensitive substring of the thread\n" + " name. If unset, a default is used.\n" + "\n" + " MOZ_PROFILER_STARTUP_ACTIVE_TAB_ID=<Number>\n" + " This variable is used to propagate the activeTabID of\n" + " the profiler init params to subprocesses.\n" + "\n" + " MOZ_PROFILER_SHUTDOWN=<Filename>\n" + " If set, the profiler saves a profile to the named file on shutdown.\n" + " If the Filename contains \"%%p\", this will be replaced with the'\n" + " process id of the parent process.\n" + "\n" + " MOZ_PROFILER_SYMBOLICATE\n" + " If set, the profiler will pre-symbolicate profiles.\n" + " *Note* This will add a significant pause when gathering data, and\n" + " is intended mainly for local development.\n" + "\n" + " MOZ_PROFILER_LUL_TEST\n" + " If set to any value, runs LUL unit tests at startup.\n" + "\n" + " This platform %s native unwinding.\n" + "\n", +#if defined(HAVE_NATIVE_UNWIND) + "supports" +#else + "does not support" +#endif + ); +} + +//////////////////////////////////////////////////////////////////////// +// BEGIN Sampler + +#if defined(GP_OS_linux) || defined(GP_OS_android) +struct SigHandlerCoordinator; +#endif + +// Sampler performs setup and teardown of the state required to sample with the +// profiler. Sampler may exist when ActivePS is not present. +// +// SuspendAndSampleAndResumeThread must only be called from a single thread, +// and must not sample the thread it is being called from. A separate Sampler +// instance must be used for each thread which wants to capture samples. + +// WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING +// +// With the exception of SamplerThread, all Sampler objects must be Disable-d +// before releasing the lock which was used to create them. This avoids races +// on linux with the SIGPROF signal handler. + +class Sampler { + public: + // Sets up the profiler such that it can begin sampling. + explicit Sampler(PSLockRef aLock); + + // Disable the sampler, restoring it to its previous state. This must be + // called once, and only once, before the Sampler is destroyed. + void Disable(PSLockRef aLock); + + // This method suspends and resumes the samplee thread. It calls the passed-in + // function-like object aProcessRegs (passing it a populated |const + // Registers&| arg) while the samplee thread is suspended. Note that + // the aProcessRegs function must be very careful not to do anything that + // requires a lock, since we may have interrupted the thread at any point. + // As an example, you can't call TimeStamp::Now() since on windows it + // takes a lock on the performance counter. + // + // Func must be a function-like object of type `void()`. + template <typename Func> + void SuspendAndSampleAndResumeThread( + PSLockRef aLock, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + const TimeStamp& aNow, const Func& aProcessRegs); + + private: +#if defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) + // Used to restore the SIGPROF handler when ours is removed. + struct sigaction mOldSigprofHandler; + + // This process' ID. Needed as an argument for tgkill in + // SuspendAndSampleAndResumeThread. + ProfilerProcessId mMyPid; + + // The sampler thread's ID. Used to assert that it is not sampling itself, + // which would lead to deadlock. + ProfilerThreadId mSamplerTid; + + public: + // This is the one-and-only variable used to communicate between the sampler + // thread and the samplee thread's signal handler. It's static because the + // samplee thread's signal handler is static. + static struct SigHandlerCoordinator* sSigHandlerCoordinator; +#endif +}; + +// END Sampler +//////////////////////////////////////////////////////////////////////// + +// Platform-specific function that retrieves per-thread CPU measurements. +static RunningTimes GetThreadRunningTimesDiff( + PSLockRef aLock, + ThreadRegistration::UnlockedRWForLockedProfiler& aThreadData); +// Platform-specific function that *may* discard CPU measurements since the +// previous call to GetThreadRunningTimesDiff, if the way to suspend threads on +// this platform may add running times to that thread. +// No-op otherwise, if suspending a thread doesn't make it work. +static void DiscardSuspendedThreadRunningTimes( + PSLockRef aLock, + ThreadRegistration::UnlockedRWForLockedProfiler& aThreadData); + +// Platform-specific function that retrieves process CPU measurements. +static RunningTimes GetProcessRunningTimesDiff( + PSLockRef aLock, RunningTimes& aPreviousRunningTimesToBeUpdated); + +// Template function to be used by `GetThreadRunningTimesDiff()` (unless some +// platform has a better way to achieve this). +// It help perform CPU measurements and tie them to a timestamp, such that the +// measurements and timestamp are very close together. +// This is necessary, because the relative CPU usage is computed by dividing +// consecutive CPU measurements by their timestamp difference; if there was an +// unexpected big gap, it could skew this computation and produce impossible +// spikes that would hide the rest of the data. See bug 1685938 for more info. +// Note that this may call the measurement function more than once; it is +// assumed to normally be fast. +// This was verified experimentally, but there is currently no regression +// testing for it; see follow-up bug 1687402. +template <typename GetCPURunningTimesFunction> +RunningTimes GetRunningTimesWithTightTimestamp( + GetCPURunningTimesFunction&& aGetCPURunningTimesFunction) { + // Once per process, compute a threshold over which running times and their + // timestamp is considered too far apart. + static const TimeDuration scMaxRunningTimesReadDuration = [&]() { + // Run the main CPU measurements + timestamp a number of times and capture + // their durations. + constexpr int loops = 128; + TimeDuration durations[loops]; + RunningTimes runningTimes; + TimeStamp before = TimeStamp::Now(); + for (int i = 0; i < loops; ++i) { + AUTO_PROFILER_STATS(GetRunningTimes_MaxRunningTimesReadDuration); + aGetCPURunningTimesFunction(runningTimes); + const TimeStamp after = TimeStamp::Now(); + durations[i] = after - before; + before = after; + } + // Move median duration to the middle. + std::nth_element(&durations[0], &durations[loops / 2], &durations[loops]); + // Use median*8 as cut-off point. + // Typical durations should be around a microsecond, the cut-off should then + // be around 10 microseconds, well below the expected minimum inter-sample + // interval (observed as a few milliseconds), so overall this should keep + // cpu/interval spikes + return durations[loops / 2] * 8; + }(); + + // Record CPU measurements between two timestamps. + RunningTimes runningTimes; + TimeStamp before = TimeStamp::Now(); + aGetCPURunningTimesFunction(runningTimes); + TimeStamp after = TimeStamp::Now(); + const TimeDuration duration = after - before; + + // In most cases, the above should be quick enough. But if not (e.g., because + // of an OS context switch), repeat once: + if (MOZ_UNLIKELY(duration > scMaxRunningTimesReadDuration)) { + AUTO_PROFILER_STATS(GetRunningTimes_REDO); + RunningTimes runningTimes2; + aGetCPURunningTimesFunction(runningTimes2); + TimeStamp after2 = TimeStamp::Now(); + const TimeDuration duration2 = after2 - after; + if (duration2 < duration) { + // We did it faster, use the new results. (But it could still be slower + // than expected, see note below for why it's acceptable.) + // This must stay *after* the CPU measurements. + runningTimes2.SetPostMeasurementTimeStamp(after2); + return runningTimes2; + } + // Otherwise use the initial results, they were slow, but faster than the + // second attempt. + // This means that something bad happened twice in a row on the same thread! + // So trying more times would be unlikely to get much better, and would be + // more expensive than the precision is worth. + // At worst, it means that a spike of activity may be reported in the next + // time slice. But in the end, the cumulative work is conserved, so it + // should still be visible at about the correct time in the graph. + AUTO_PROFILER_STATS(GetRunningTimes_RedoWasWorse); + } + + // This must stay *after* the CPU measurements. + runningTimes.SetPostMeasurementTimeStamp(after); + + return runningTimes; +} + +//////////////////////////////////////////////////////////////////////// +// BEGIN SamplerThread + +// The sampler thread controls sampling and runs whenever the profiler is +// active. It periodically runs through all registered threads, finds those +// that should be sampled, then pauses and samples them. + +class SamplerThread { + public: + // Creates a sampler thread, but doesn't start it. + SamplerThread(PSLockRef aLock, uint32_t aActivityGeneration, + double aIntervalMilliseconds, uint32_t aFeatures); + ~SamplerThread(); + + // This runs on (is!) the sampler thread. + void Run(); + +#if defined(GP_OS_windows) + // This runs on (is!) the thread to spy on unregistered threads. + void RunUnregisteredThreadSpy(); +#endif + + // This runs on the main thread. + void Stop(PSLockRef aLock); + + void AppendPostSamplingCallback(PSLockRef, PostSamplingCallback&& aCallback) { + // We are under lock, so it's safe to just modify the list pointer. + // Also this means the sampler has not started its run yet, so any callback + // added now will be invoked at the end of the next loop; this guarantees + // that the callback will be invoked after at least one full sampling loop. + mPostSamplingCallbackList = MakeUnique<PostSamplingCallbackListItem>( + std::move(mPostSamplingCallbackList), std::move(aCallback)); + } + + private: + void SpyOnUnregisteredThreads(); + + // Item containing a post-sampling callback, and a tail-list of more items. + // Using a linked list means no need to move items when adding more, and + // "stealing" the whole list is one pointer move. + struct PostSamplingCallbackListItem { + UniquePtr<PostSamplingCallbackListItem> mPrev; + PostSamplingCallback mCallback; + + PostSamplingCallbackListItem(UniquePtr<PostSamplingCallbackListItem> aPrev, + PostSamplingCallback&& aCallback) + : mPrev(std::move(aPrev)), mCallback(std::move(aCallback)) {} + }; + + [[nodiscard]] UniquePtr<PostSamplingCallbackListItem> + TakePostSamplingCallbacks(PSLockRef) { + return std::move(mPostSamplingCallbackList); + } + + static void InvokePostSamplingCallbacks( + UniquePtr<PostSamplingCallbackListItem> aCallbacks, + SamplingState aSamplingState) { + if (!aCallbacks) { + return; + } + // We want to drill down to the last element in this list, which is the + // oldest one, so that we invoke them in FIFO order. + // We don't expect many callbacks, so it's safe to recurse. Note that we're + // moving-from the UniquePtr, so the tail will implicitly get destroyed. + InvokePostSamplingCallbacks(std::move(aCallbacks->mPrev), aSamplingState); + // We are going to destroy this item, so we can safely move-from the + // callback before calling it (in case it has an rvalue-ref-qualified call + // operator). + std::move(aCallbacks->mCallback)(aSamplingState); + // It may be tempting for a future maintainer to change aCallbacks into an + // rvalue reference; this will remind them not to do that! + static_assert( + std::is_same_v<decltype(aCallbacks), + UniquePtr<PostSamplingCallbackListItem>>, + "We need to capture the list by-value, to implicitly destroy it"); + } + + // This suspends the calling thread for the given number of microseconds. + // Best effort timing. + void SleepMicro(uint32_t aMicroseconds); + + // The sampler used to suspend and sample threads. + Sampler mSampler; + + // The activity generation, for detecting when the sampler thread must stop. + const uint32_t mActivityGeneration; + + // The interval between samples, measured in microseconds. + const int mIntervalMicroseconds; + + // The OS-specific handle for the sampler thread. +#if defined(GP_OS_windows) + HANDLE mThread; + HANDLE mUnregisteredThreadSpyThread = nullptr; + enum class SpyingState { + NoSpying, + Spy_Initializing, + // Spy is waiting for SamplerToSpy_Start or MainToSpy_Shutdown. + Spy_Waiting, + // Sampler requests spy to start working. May be pre-empted by + // MainToSpy_Shutdown. + SamplerToSpy_Start, + // Spy is currently working, cannot be interrupted, only the spy is allowed + // to change the state again. + Spy_Working, + // Main control requests spy to shut down. + MainToSpy_Shutdown, + // Spy notified main control that it's out of the loop, about to exit. + SpyToMain_ShuttingDown + }; + SpyingState mSpyingState = SpyingState::NoSpying; + // The sampler will increment this while the spy is working, then while the + // spy is waiting the sampler will decrement it until <=0 before starting the + // spy. This will ensure that the work doesn't take more than 50% of a CPU + // core. + int mDelaySpyStart = 0; + Monitor mSpyingStateMonitor MOZ_UNANNOTATED{ + "SamplerThread::mSpyingStateMonitor"}; +#elif defined(GP_OS_darwin) || defined(GP_OS_linux) || \ + defined(GP_OS_android) || defined(GP_OS_freebsd) + pthread_t mThread; +#endif + + // Post-sampling callbacks are kept in a simple linked list, which will be + // stolen by the sampler thread at the end of its next run. + UniquePtr<PostSamplingCallbackListItem> mPostSamplingCallbackList; + +#if defined(GP_OS_windows) + bool mNoTimerResolutionChange = true; +#endif + + struct SpiedThread { + base::ProcessId mThreadId; + nsCString mName; + uint64_t mCPUTimeNs; + + SpiedThread(base::ProcessId aThreadId, const nsACString& aName, + uint64_t aCPUTimeNs) + : mThreadId(aThreadId), mName(aName), mCPUTimeNs(aCPUTimeNs) {} + + // Comparisons with just a thread id, for easy searching in an array. + friend bool operator==(const SpiedThread& aSpiedThread, + base::ProcessId aThreadId) { + return aSpiedThread.mThreadId == aThreadId; + } + friend bool operator==(base::ProcessId aThreadId, + const SpiedThread& aSpiedThread) { + return aThreadId == aSpiedThread.mThreadId; + } + }; + + // Time at which mSpiedThreads was previously updated. Null before 1st update. + TimeStamp mLastSpying; + // Unregistered threads that have been found, and are being spied on. + using SpiedThreads = AutoTArray<SpiedThread, 128>; + SpiedThreads mSpiedThreads; + + SamplerThread(const SamplerThread&) = delete; + void operator=(const SamplerThread&) = delete; +}; + +namespace geckoprofiler::markers { +struct CPUSpeedMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("CPUSpeed"); + } + static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter, + uint32_t aCPUSpeedMHz) { + aWriter.DoubleProperty("speed", double(aCPUSpeedMHz) / 1000); + } + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable}; + schema.SetTableLabel("{marker.name} Speed = {marker.data.speed}GHz"); + schema.AddKeyLabelFormat("speed", "CPU Speed (GHz)", MS::Format::String); + schema.AddChartColor("speed", MS::GraphType::Bar, MS::GraphColor::Ink); + return schema; + } +}; +} // namespace geckoprofiler::markers + +// [[nodiscard]] static +bool ActivePS::AppendPostSamplingCallback(PSLockRef aLock, + PostSamplingCallback&& aCallback) { + if (!sInstance || !sInstance->mSamplerThread) { + return false; + } + sInstance->mSamplerThread->AppendPostSamplingCallback(aLock, + std::move(aCallback)); + return true; +} + +// This function is required because we need to create a SamplerThread within +// ActivePS's constructor, but SamplerThread is defined after ActivePS. It +// could probably be removed by moving some code around. +static SamplerThread* NewSamplerThread(PSLockRef aLock, uint32_t aGeneration, + double aInterval, uint32_t aFeatures) { + return new SamplerThread(aLock, aGeneration, aInterval, aFeatures); +} + +// This function is the sampler thread. This implementation is used for all +// targets. +void SamplerThread::Run() { + NS_SetCurrentThreadName("SamplerThread"); + + // Features won't change during this SamplerThread's lifetime, so we can read + // them once and store them locally. + const uint32_t features = []() -> uint32_t { + PSAutoLock lock; + if (!ActivePS::Exists(lock)) { + // If there is no active profiler, it doesn't matter what we return, + // because this thread will exit before any feature is used. + return 0; + } + return ActivePS::Features(lock); + }(); + + // Not *no*-stack-sampling means we do want stack sampling. + const bool stackSampling = !ProfilerFeature::HasNoStackSampling(features); + + const bool cpuUtilization = ProfilerFeature::HasCPUUtilization(features); + + // Use local ProfileBuffer and underlying buffer to capture the stack. + // (This is to avoid touching the core buffer lock while a thread is + // suspended, because that thread could be working with the core buffer as + // well. + mozilla::ProfileBufferChunkManagerSingle localChunkManager( + ProfileBufferChunkManager::scExpectedMaximumStackSize); + ProfileChunkedBuffer localBuffer( + ProfileChunkedBuffer::ThreadSafety::WithoutMutex, localChunkManager); + ProfileBuffer localProfileBuffer(localBuffer); + + // Will be kept between collections, to know what each collection does. + auto previousState = localBuffer.GetState(); + + // This will be filled at every loop, to be used by the next loop to compute + // the CPU utilization between samples. + RunningTimes processRunningTimes; + + // This will be set inside the loop, from inside the lock scope, to capture + // all callbacks added before that, but none after the lock is released. + UniquePtr<PostSamplingCallbackListItem> postSamplingCallbacks; + // This will be set inside the loop, before invoking callbacks outside. + SamplingState samplingState{}; + + const TimeDuration sampleInterval = + TimeDuration::FromMicroseconds(mIntervalMicroseconds); + const uint32_t minimumIntervalSleepUs = + static_cast<uint32_t>(mIntervalMicroseconds / 4); + + // This is the scheduled time at which each sampling loop should start. + // It will determine the ideal next sampling start by adding the expected + // interval, unless when sampling runs late -- See end of while() loop. + TimeStamp scheduledSampleStart = TimeStamp::Now(); + +#if defined(HAVE_CPU_FREQ_SUPPORT) + // Used to collect CPU core frequencies, if the cpufreq feature is on. + Vector<uint32_t> CPUSpeeds; + + if (XRE_IsParentProcess() && ProfilerFeature::HasCPUFrequency(features) && + CPUSpeeds.resize(GetNumberOfProcessors())) { + { + PSAutoLock lock; + if (ProfilerCPUFreq* cpuFreq = ActivePS::MaybeCPUFreq(lock); cpuFreq) { + cpuFreq->Sample(); + for (size_t i = 0; i < CPUSpeeds.length(); ++i) { + CPUSpeeds[i] = cpuFreq->GetCPUSpeedMHz(i); + } + } + } + TimeStamp now = TimeStamp::Now(); + for (size_t i = 0; i < CPUSpeeds.length(); ++i) { + nsAutoCString name; + name.AssignLiteral("CPU "); + name.AppendInt(i); + + PROFILER_MARKER(name, OTHER, + MarkerOptions(MarkerThreadId::MainThread(), + MarkerTiming::IntervalStart(now)), + CPUSpeedMarker, CPUSpeeds[i]); + } + } +#endif + + while (true) { + const TimeStamp sampleStart = TimeStamp::Now(); + + // This scope is for |lock|. It ends before we sleep below. + { + // There should be no local callbacks left from a previous loop. + MOZ_ASSERT(!postSamplingCallbacks); + + PSAutoLock lock; + TimeStamp lockAcquired = TimeStamp::Now(); + + // Move all the post-sampling callbacks locally, so that new ones cannot + // sneak in between the end of the lock scope and the invocation after it. + postSamplingCallbacks = TakePostSamplingCallbacks(lock); + + if (!ActivePS::Exists(lock)) { + // Exit the `while` loop, including the lock scope, before invoking + // callbacks and returning. + samplingState = SamplingState::JustStopped; + break; + } + + // At this point profiler_stop() might have been called, and + // profiler_start() might have been called on another thread. If this + // happens the generation won't match. + if (ActivePS::Generation(lock) != mActivityGeneration) { + samplingState = SamplingState::JustStopped; + // Exit the `while` loop, including the lock scope, before invoking + // callbacks and returning. + break; + } + + ActivePS::ClearExpiredExitProfiles(lock); + + TimeStamp expiredMarkersCleaned = TimeStamp::Now(); + + if (int(gSkipSampling) <= 0 && !ActivePS::IsSamplingPaused(lock)) { + double sampleStartDeltaMs = + (sampleStart - CorePS::ProcessStartTime()).ToMilliseconds(); + ProfileBuffer& buffer = ActivePS::Buffer(lock); + + // Before sampling counters, update the process CPU counter if active. + if (ActivePS::ProcessCPUCounter* processCPUCounter = + ActivePS::MaybeProcessCPUCounter(lock); + processCPUCounter) { + RunningTimes processRunningTimesDiff = + GetProcessRunningTimesDiff(lock, processRunningTimes); + Maybe<uint64_t> cpu = processRunningTimesDiff.GetJsonThreadCPUDelta(); + if (cpu) { + processCPUCounter->Add(static_cast<int64_t>(*cpu)); + } + } + +#if defined(HAVE_CPU_FREQ_SUPPORT) + if (XRE_IsParentProcess() && CPUSpeeds.length() > 0) { + unsigned newSpeed[CPUSpeeds.length()]; + if (ProfilerCPUFreq* cpuFreq = ActivePS::MaybeCPUFreq(lock); + cpuFreq) { + cpuFreq->Sample(); + for (size_t i = 0; i < CPUSpeeds.length(); ++i) { + newSpeed[i] = cpuFreq->GetCPUSpeedMHz(i); + } + } + TimeStamp now = TimeStamp::Now(); + for (size_t i = 0; i < CPUSpeeds.length(); ++i) { + if (newSpeed[i] == CPUSpeeds[i]) { + continue; + } + + nsAutoCString name; + name.AssignLiteral("CPU "); + name.AppendInt(i); + + PROFILER_MARKER_UNTYPED( + name, OTHER, + MarkerOptions(MarkerThreadId::MainThread(), + MarkerTiming::IntervalEnd(now))); + PROFILER_MARKER(name, OTHER, + MarkerOptions(MarkerThreadId::MainThread(), + MarkerTiming::IntervalStart(now)), + CPUSpeedMarker, newSpeed[i]); + + CPUSpeeds[i] = newSpeed[i]; + } + } +#endif + + if (PowerCounters* powerCounters = ActivePS::MaybePowerCounters(lock); + powerCounters) { + powerCounters->Sample(); + } + + // handle per-process generic counters + double counterSampleStartDeltaMs = + (TimeStamp::Now() - CorePS::ProcessStartTime()).ToMilliseconds(); + const Vector<BaseProfilerCount*>& counters = CorePS::Counters(lock); + for (auto& counter : counters) { + if (auto sample = counter->Sample(); sample.isSampleNew) { + // create Buffer entries for each counter + buffer.AddEntry(ProfileBufferEntry::CounterId(counter)); + buffer.AddEntry( + ProfileBufferEntry::Time(counterSampleStartDeltaMs)); +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + if (ActivePS::IsMemoryCounter(counter)) { + // For the memory counter, substract the size of our buffer to + // avoid giving the misleading impression that the memory use + // keeps on growing when it's just the profiler session that's + // using a larger buffer as it gets longer. + sample.count -= static_cast<int64_t>( + ActivePS::ControlledChunkManager(lock).TotalSize()); + } +#endif + buffer.AddEntry(ProfileBufferEntry::Count(sample.count)); + if (sample.number) { + buffer.AddEntry(ProfileBufferEntry::Number(sample.number)); + } + } + } + TimeStamp countersSampled = TimeStamp::Now(); + + if (stackSampling || cpuUtilization) { + samplingState = SamplingState::SamplingCompleted; + + // Prevent threads from ending (or starting) and allow access to all + // OffThreadRef's. + ThreadRegistry::LockedRegistry lockedRegistry; + + for (ThreadRegistry::OffThreadRef offThreadRef : lockedRegistry) { + ThreadRegistration::UnlockedRWForLockedProfiler& + unlockedThreadData = + offThreadRef.UnlockedRWForLockedProfilerRef(); + ProfiledThreadData* profiledThreadData = + unlockedThreadData.GetProfiledThreadData(lock); + if (!profiledThreadData) { + // This thread is not being profiled, continue with the next one. + continue; + } + + const ThreadProfilingFeatures whatToProfile = + unlockedThreadData.ProfilingFeatures(); + const bool threadCPUUtilization = + cpuUtilization && + DoFeaturesIntersect(whatToProfile, + ThreadProfilingFeatures::CPUUtilization); + const bool threadStackSampling = + stackSampling && + DoFeaturesIntersect(whatToProfile, + ThreadProfilingFeatures::Sampling); + if (!threadCPUUtilization && !threadStackSampling) { + // Nothing to profile on this thread, continue with the next one. + continue; + } + + const ProfilerThreadId threadId = + unlockedThreadData.Info().ThreadId(); + + const RunningTimes runningTimesDiff = [&]() { + if (!threadCPUUtilization) { + // If we don't need CPU measurements, we only need a timestamp. + return RunningTimes(TimeStamp::Now()); + } + return GetThreadRunningTimesDiff(lock, unlockedThreadData); + }(); + + const TimeStamp& now = runningTimesDiff.PostMeasurementTimeStamp(); + double threadSampleDeltaMs = + (now - CorePS::ProcessStartTime()).ToMilliseconds(); + + // If the thread is asleep and has been sampled before in the same + // sleep episode, or otherwise(*) if there was zero CPU activity + // since the previous sampling, find and copy the previous sample, + // as that's cheaper than taking a new sample. + // (*) Tech note: The asleep check is done first and always, because + // it is more reliable, and knows if it's the first asleep + // sample, which cannot be duplicated; if the test was the other + // way around, it could find zero CPU and then short-circuit + // that state-changing second-asleep-check operation, which + // could result in an unneeded sample. + // However we're using current running times (instead of copying the + // old ones) because some work could have happened. + if (threadStackSampling && + (unlockedThreadData.CanDuplicateLastSampleDueToSleep() || + runningTimesDiff.GetThreadCPUDelta() == Some(uint64_t(0)))) { + const bool dup_ok = ActivePS::Buffer(lock).DuplicateLastSample( + threadId, threadSampleDeltaMs, + profiledThreadData->LastSample(), runningTimesDiff); + if (dup_ok) { + continue; + } + } + + AUTO_PROFILER_STATS(gecko_SamplerThread_Run_DoPeriodicSample); + + // Record the global profiler buffer's range start now, before + // adding the first entry for this thread's sample. + const uint64_t bufferRangeStart = buffer.BufferRangeStart(); + + // Add the thread ID now, so we know its position in the main + // buffer, which is used by some JS data. + // (DoPeriodicSample only knows about the temporary local buffer.) + const uint64_t samplePos = buffer.AddThreadIdEntry(threadId); + profiledThreadData->LastSample() = Some(samplePos); + + // Also add the time, so it's always there after the thread ID, as + // expected by the parser. (Other stack data is optional.) + buffer.AddEntry(ProfileBufferEntry::TimeBeforeCompactStack( + threadSampleDeltaMs)); + + Maybe<double> unresponsiveDuration_ms; + + // If we have RunningTimes data, store it before the CompactStack. + // Note: It is not stored inside the CompactStack so that it doesn't + // get incorrectly duplicated when the thread is sleeping. + if (!runningTimesDiff.IsEmpty()) { + profiler_get_core_buffer().PutObjects( + ProfileBufferEntry::Kind::RunningTimes, runningTimesDiff); + } + + if (threadStackSampling) { + ThreadRegistry::OffThreadRef::RWFromAnyThreadWithLock + lockedThreadData = offThreadRef.GetLockedRWFromAnyThread(); + // Suspend the thread and collect its stack data in the local + // buffer. + mSampler.SuspendAndSampleAndResumeThread( + lock, lockedThreadData.DataCRef(), now, + [&](const Registers& aRegs, const TimeStamp& aNow) { + DoPeriodicSample(lock, lockedThreadData.DataCRef(), aRegs, + samplePos, bufferRangeStart, + localProfileBuffer); + + // For "eventDelay", we want the input delay - but if + // there are no events in the input queue (or even if there + // are), we're interested in how long the delay *would* be + // for an input event now, which would be the time to finish + // the current event + the delay caused by any events + // already in the input queue (plus any High priority + // events). Events at lower priorities (in a + // PrioritizedEventQueue) than Input count for input delay + // only for the duration that they're running, since when + // they finish, any queued input event would run. + // + // Unless we record the time state of all events and queue + // states at all times, this is hard to precisely calculate, + // but we can approximate it well in post-processing with + // RunningEventDelay and RunningEventStart. + // + // RunningEventDelay is the time duration the event was + // queued before starting execution. RunningEventStart is + // the time the event started. (Note: since we care about + // Input event delays on MainThread, for + // PrioritizedEventQueues we return 0 for RunningEventDelay + // if the currently running event has a lower priority than + // Input (since Input events won't queue behind them). + // + // To directly measure this we would need to record the time + // at which the newest event currently in each queue at time + // X (the sample time) finishes running. This of course + // would require looking into the future, or recording all + // this state and then post-processing it later. If we were + // to trace every event start and end we could do this, but + // it would have significant overhead to do so (and buffer + // usage). From a recording of RunningEventDelays and + // RunningEventStarts we can infer the actual delay: + // + // clang-format off + // Event queue: <tail> D : C : B : A <head> + // Time inserted (ms): 40 : 20 : 10 : 0 + // Run Time (ms): 30 : 100 : 40 : 30 + // + // 0 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 160 170 + // [A||||||||||||] + // ----------[B|||||||||||||||||] + // -------------------------[C|||||||||||||||||||||||||||||||||||||||||||||||] + // -----------------------------------------------------------------[D|||||||||...] + // + // Calculate the delay of a new event added at time t: (run every sample) + // TimeSinceRunningEventBlockedInputEvents = RunningEventDelay + (now - RunningEventStart); + // effective_submission = now - TimeSinceRunningEventBlockedInputEvents; + // delta = (now - last_sample_time); + // last_sample_time = now; + // for (t=effective_submission to now) { + // delay[t] += delta; + // } + // + // Can be reduced in overhead by: + // TimeSinceRunningEventBlockedInputEvents = RunningEventDelay + (now - RunningEventStart); + // effective_submission = now - TimeSinceRunningEventBlockedInputEvents; + // if (effective_submission != last_submission) { + // delta = (now - last_submision); + // // this loop should be made to match each sample point in the range + // // intead of assuming 1ms sampling as this pseudocode does + // for (t=last_submission to effective_submission-1) { + // delay[t] += delta; + // delta -= 1; // assumes 1ms; adjust as needed to match for() + // } + // last_submission = effective_submission; + // } + // + // Time Head of queue Running Event RunningEventDelay Delay of Effective Started Calc (submission->now add 10ms) Final + // hypothetical Submission Running @ result + // event E + // 0 Empty A 0 30 0 0 @0=10 30 + // 10 B A 0 60 0 0 @0=20, @10=10 60 + // 20 B A 0 150 0 0 @0=30, @10=20, @20=10 150 + // 30 C B 20 140 10 30 @10=20, @20=10, @30=0 140 + // 40 C B 20 160 @10=30, @20=20... 160 + // 50 C B 20 150 150 + // 60 C B 20 140 @10=50, @20=40... 140 + // 70 D C 50 130 20 70 @20=50, @30=40... 130 + // ... + // 160 D C 50 40 @20=140, @30=130... 40 + // 170 <empty> D 140 30 40 @40=140, @50=130... (rounding) 30 + // 180 <empty> D 140 20 40 @40=150 20 + // 190 <empty> D 140 10 40 @40=160 10 + // 200 <empty> <empty> 0 0 NA 0 + // + // Function Delay(t) = the time between t and the time at which a hypothetical + // event e would start executing, if e was enqueued at time t. + // + // Delay(-1) = 0 // Before A was enqueued. No wait time, can start running + // // instantly. + // Delay(0) = 30 // The hypothetical event e got enqueued just after A got + // // enqueued. It can start running at 30, when A is done. + // Delay(5) = 25 + // Delay(10) = 60 // Can start running at 70, after both A and B are done. + // Delay(19) = 51 + // Delay(20) = 150 // Can start running at 170, after A, B & C. + // Delay(25) = 145 + // Delay(30) = 170 // Can start running at 200, after A, B, C & D. + // Delay(120) = 80 + // Delay(200) = 0 // (assuming nothing was enqueued after D) + // + // For every event that gets enqueued, the Delay time will go up by the + // event's running time at the time at which the event is enqueued. + // The Delay function will be a sawtooth of the following shape: + // + // |\ |... + // | \ | + // |\ | \ | + // | \ | \ | + // |\ | \ | \ | + // |\ | \| \| \ | + // | \| \ | + // _| \____| + // + // + // A more complex example with a PrioritizedEventQueue: + // + // Event queue: <tail> D : C : B : A <head> + // Time inserted (ms): 40 : 20 : 10 : 0 + // Run Time (ms): 30 : 100 : 40 : 30 + // Priority: Input: Norm: Norm: Norm + // + // 0 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 160 170 + // [A||||||||||||] + // ----------[B|||||||||||||||||] + // ----------------------------------------[C|||||||||||||||||||||||||||||||||||||||||||||||] + // ---------------[D||||||||||||] + // + // + // Time Head of queue Running Event RunningEventDelay Delay of Effective Started Calc (submission->now add 10ms) Final + // hypothetical Submission Running @ result + // event + // 0 Empty A 0 30 0 0 @0=10 30 + // 10 B A 0 20 0 0 @0=20, @10=10 20 + // 20 B A 0 10 0 0 @0=30, @10=20, @20=10 10 + // 30 C B 0 40 30 30 @30=10 40 + // 40 C B 0 60 30 @40=10, @30=20 60 + // 50 C B 0 50 30 @50=10, @40=20, @30=30 50 + // 60 C B 0 40 30 @60=10, @50=20, @40=30, @30=40 40 + // 70 C D 30 30 40 70 @60=20, @50=30, @40=40 30 + // 80 C D 30 20 40 70 ...@50=40, @40=50 20 + // 90 C D 30 10 40 70 ...@60=40, @50=50, @40=60 10 + // 100 <empty> C 0 100 100 100 @100=10 100 + // 110 <empty> C 0 90 100 100 @110=10, @100=20 90 + + // + // For PrioritizedEventQueue, the definition of the Delay(t) function is adjusted: the hypothetical event e has Input priority. + // Delay(-1) = 0 // Before A was enqueued. No wait time, can start running + // // instantly. + // Delay(0) = 30 // The hypothetical input event e got enqueued just after A got + // // enqueued. It can start running at 30, when A is done. + // Delay(5) = 25 + // Delay(10) = 20 + // Delay(25) = 5 // B has been queued, but e does not need to wait for B because e has Input priority and B does not. + // // So e can start running at 30, when A is done. + // Delay(30) = 40 // Can start running at 70, after B is done. + // Delay(40) = 60 // Can start at 100, after B and D are done (D is Input Priority) + // Delay(80) = 20 + // Delay(100) = 100 // Wait for C to finish + + // clang-format on + // + // Alternatively we could insert (recycled instead of + // allocated/freed) input events at every sample period + // (1ms...), and use them to back-calculate the delay. This + // might also be somewhat expensive, and would require + // guessing at the maximum delay, which would likely be in + // the seconds, and so you'd need 1000's of pre-allocated + // events per queue per thread - so there would be a memory + // impact as well. + + TimeDuration currentEventDelay; + TimeDuration currentEventRunning; + lockedThreadData->GetRunningEventDelay( + aNow, currentEventDelay, currentEventRunning); + + // Note: eventDelay is a different definition of + // responsiveness than the 16ms event injection. + + // Don't suppress 0's for now; that can be a future + // optimization. We probably want one zero to be stored + // before we start suppressing, which would be more + // complex. + unresponsiveDuration_ms = + Some(currentEventDelay.ToMilliseconds() + + currentEventRunning.ToMilliseconds()); + }); + + if (cpuUtilization) { + // Suspending the thread for sampling could have added some + // running time to it, discard any since the call to + // GetThreadRunningTimesDiff above. + DiscardSuspendedThreadRunningTimes(lock, unlockedThreadData); + } + + // If we got eventDelay data, store it before the CompactStack. + // Note: It is not stored inside the CompactStack so that it + // doesn't get incorrectly duplicated when the thread is sleeping. + if (unresponsiveDuration_ms.isSome()) { + profiler_get_core_buffer().PutObjects( + ProfileBufferEntry::Kind::UnresponsiveDurationMs, + *unresponsiveDuration_ms); + } + } + + // There *must* be a CompactStack after a TimeBeforeCompactStack; + // but note that other entries may have been concurrently inserted + // between the TimeBeforeCompactStack above and now. If the captured + // sample from `DoPeriodicSample` is complete, copy it into the + // global buffer, otherwise add an empty one to satisfy the parser + // that expects one. + auto state = localBuffer.GetState(); + if (NS_WARN_IF(state.mFailedPutBytes != + previousState.mFailedPutBytes)) { + LOG("Stack sample too big for local storage, failed to store %u " + "bytes", + unsigned(state.mFailedPutBytes - + previousState.mFailedPutBytes)); + // There *must* be a CompactStack after a TimeBeforeCompactStack, + // even an empty one. + profiler_get_core_buffer().PutObjects( + ProfileBufferEntry::Kind::CompactStack, + UniquePtr<ProfileChunkedBuffer>(nullptr)); + } else if (state.mRangeEnd - previousState.mRangeEnd >= + *profiler_get_core_buffer().BufferLength()) { + LOG("Stack sample too big for profiler storage, needed %u bytes", + unsigned(state.mRangeEnd - previousState.mRangeEnd)); + // There *must* be a CompactStack after a TimeBeforeCompactStack, + // even an empty one. + profiler_get_core_buffer().PutObjects( + ProfileBufferEntry::Kind::CompactStack, + UniquePtr<ProfileChunkedBuffer>(nullptr)); + } else { + profiler_get_core_buffer().PutObjects( + ProfileBufferEntry::Kind::CompactStack, localBuffer); + } + + // Clean up for the next run. + localBuffer.Clear(); + previousState = localBuffer.GetState(); + } + } else { + samplingState = SamplingState::NoStackSamplingCompleted; + } + +#if defined(USE_LUL_STACKWALK) + // The LUL unwind object accumulates frame statistics. Periodically we + // should poke it to give it a chance to print those statistics. This + // involves doing I/O (fprintf, __android_log_print, etc.) and so + // can't safely be done from the critical section inside + // SuspendAndSampleAndResumeThread, which is why it is done here. + lul::LUL* lul = CorePS::Lul(); + if (lul) { + lul->MaybeShowStats(); + } +#endif + TimeStamp threadsSampled = TimeStamp::Now(); + + { + AUTO_PROFILER_STATS(Sampler_FulfillChunkRequests); + ActivePS::FulfillChunkRequests(lock); + } + + buffer.CollectOverheadStats(sampleStartDeltaMs, + lockAcquired - sampleStart, + expiredMarkersCleaned - lockAcquired, + countersSampled - expiredMarkersCleaned, + threadsSampled - countersSampled); + } else { + samplingState = SamplingState::SamplingPaused; + } + } + // gPSMutex is not held after this point. + + // Invoke end-of-sampling callbacks outside of the locked scope. + InvokePostSamplingCallbacks(std::move(postSamplingCallbacks), + samplingState); + + ProfilerChild::ProcessPendingUpdate(); + + if (ProfilerFeature::HasUnregisteredThreads(features)) { +#if defined(GP_OS_windows) + { + MonitorAutoLock spyingStateLock{mSpyingStateMonitor}; + switch (mSpyingState) { + case SpyingState::SamplerToSpy_Start: + case SpyingState::Spy_Working: + // If the spy is working (or about to work), record this loop + // iteration to delay the next start. + ++mDelaySpyStart; + break; + case SpyingState::Spy_Waiting: + // The Spy is idle, waiting for instructions. Should we delay? + if (--mDelaySpyStart <= 0) { + mDelaySpyStart = 0; + mSpyingState = SpyingState::SamplerToSpy_Start; + mSpyingStateMonitor.NotifyAll(); + } + break; + default: + // Otherwise the spy should be initializing or shutting down. + MOZ_ASSERT(mSpyingState == SpyingState::Spy_Initializing || + mSpyingState == SpyingState::MainToSpy_Shutdown || + mSpyingState == SpyingState::SpyToMain_ShuttingDown); + break; + } + } +#else + // On non-Windows platforms, this is fast enough to run in this thread, + // each sampling loop. + SpyOnUnregisteredThreads(); +#endif + } + + // We expect the next sampling loop to start `sampleInterval` after this + // loop here was scheduled to start. + scheduledSampleStart += sampleInterval; + + // Try to sleep until we reach that next scheduled time. + const TimeStamp beforeSleep = TimeStamp::Now(); + if (scheduledSampleStart >= beforeSleep) { + // There is still time before the next scheduled sample time. + const uint32_t sleepTimeUs = static_cast<uint32_t>( + (scheduledSampleStart - beforeSleep).ToMicroseconds()); + if (sleepTimeUs >= minimumIntervalSleepUs) { + SleepMicro(sleepTimeUs); + } else { + // If we're too close to that time, sleep the minimum amount of time. + // Note that the next scheduled start is not shifted, so at the end of + // the next loop, sleep may again be adjusted to get closer to schedule. + SleepMicro(minimumIntervalSleepUs); + } + } else { + // This sampling loop ended after the next sampling should have started! + // There is little point to try and keep up to schedule now, it would + // require more work, while it's likely we're late because the system is + // already busy. Try and restart a normal schedule from now. + scheduledSampleStart = beforeSleep + sampleInterval; + SleepMicro(static_cast<uint32_t>(sampleInterval.ToMicroseconds())); + } + } + + // End of `while` loop. We can only be here from a `break` inside the loop. + InvokePostSamplingCallbacks(std::move(postSamplingCallbacks), samplingState); +} + +namespace geckoprofiler::markers { + +struct UnregisteredThreadLifetimeMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("UnregisteredThreadLifetime"); + } + static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter, + base::ProcessId aThreadId, + const ProfilerString8View& aName, + const ProfilerString8View& aEndEvent) { + aWriter.IntProperty("Thread Id", aThreadId); + aWriter.StringProperty("Thread Name", aName.Length() != 0 + ? aName.AsSpan() + : MakeStringSpan("~Unnamed~")); + if (aEndEvent.Length() != 0) { + aWriter.StringProperty("End Event", aEndEvent); + } + } + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable}; + schema.AddKeyFormatSearchable("Thread Id", MS::Format::Integer, + MS::Searchable::Searchable); + schema.AddKeyFormatSearchable("Thread Name", MS::Format::String, + MS::Searchable::Searchable); + schema.AddKeyFormat("End Event", MS::Format::String); + schema.AddStaticLabelValue( + "Note", + "Start and end are approximate, based on first and last appearances."); + schema.SetChartLabel( + "{marker.data.Thread Name} (tid {marker.data.Thread Id})"); + schema.SetTableLabel("{marker.name} lifetime"); + return schema; + } +}; + +struct UnregisteredThreadCPUMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("UnregisteredThreadCPU"); + } + static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter, + base::ProcessId aThreadId, + int64_t aCPUDiffNs, const TimeStamp& aStart, + const TimeStamp& aEnd) { + aWriter.IntProperty("Thread Id", aThreadId); + aWriter.IntProperty("CPU Time", aCPUDiffNs); + aWriter.DoubleProperty( + "CPU Utilization", + double(aCPUDiffNs) / ((aEnd - aStart).ToMicroseconds() * 1000.0)); + } + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable}; + schema.AddKeyFormatSearchable("Thread Id", MS::Format::Integer, + MS::Searchable::Searchable); + schema.AddKeyFormat("CPU Time", MS::Format::Nanoseconds); + schema.AddKeyFormat("CPU Utilization", MS::Format::Percentage); + schema.SetChartLabel("{marker.data.CPU Utilization}"); + schema.SetTableLabel( + "{marker.name} - Activity: {marker.data.CPU Utilization}"); + return schema; + } +}; + +} // namespace geckoprofiler::markers + +static bool IsThreadIdRegistered(ProfilerThreadId aThreadId) { + ThreadRegistry::LockedRegistry lockedRegistry; + const auto registryEnd = lockedRegistry.end(); + return std::find_if( + lockedRegistry.begin(), registryEnd, + [aThreadId](const ThreadRegistry::OffThreadRef& aOffThreadRef) { + return aOffThreadRef.UnlockedConstReaderCRef() + .Info() + .ThreadId() == aThreadId; + }) != registryEnd; +} + +static nsAutoCString MakeThreadInfoMarkerName(base::ProcessId aThreadId, + const nsACString& aName) { + nsAutoCString markerName{"tid "}; + markerName.AppendInt(int64_t(aThreadId)); + if (!aName.IsEmpty()) { + markerName.AppendLiteral(" "); + markerName.Append(aName); + } + return markerName; +} + +void SamplerThread::SpyOnUnregisteredThreads() { + const TimeStamp unregisteredThreadSearchStart = TimeStamp::Now(); + + const base::ProcessId currentProcessId = + base::ProcessId(profiler_current_process_id().ToNumber()); + nsTArray<ProcInfoRequest> request(1); + request.EmplaceBack( + /* aPid = */ currentProcessId, + /* aProcessType = */ ProcType::Unknown, + /* aOrigin = */ ""_ns, + /* aWindowInfo = */ nsTArray<WindowInfo>{}, + /* aUtilityInfo = */ nsTArray<UtilityInfo>{}, + /* aChild = */ 0 +#ifdef XP_MACOSX + , + /* aChildTask = */ MACH_PORT_NULL +#endif // XP_MACOSX + ); + + const ProcInfoPromise::ResolveOrRejectValue procInfoOrError = + GetProcInfoSync(std::move(request)); + + if (!procInfoOrError.IsResolve()) { + PROFILER_MARKER_TEXT("Failed unregistered thread search", PROFILER, + MarkerOptions(MarkerThreadId::MainThread(), + MarkerTiming::IntervalUntilNowFrom( + unregisteredThreadSearchStart)), + "Could not retrieve any process information"); + return; + } + + const auto& procInfoHashMap = procInfoOrError.ResolveValue(); + // Expecting the requested (current) process information to be present in the + // hashmap. + const auto& procInfoPtr = + procInfoHashMap.readonlyThreadsafeLookup(currentProcessId); + if (!procInfoPtr) { + PROFILER_MARKER_TEXT("Failed unregistered thread search", PROFILER, + MarkerOptions(MarkerThreadId::MainThread(), + MarkerTiming::IntervalUntilNowFrom( + unregisteredThreadSearchStart)), + "Could not retrieve information about this process"); + return; + } + + // Record the time spent so far, which is OS-bound... + PROFILER_MARKER_TEXT("Unregistered thread search", PROFILER, + MarkerOptions(MarkerThreadId::MainThread(), + MarkerTiming::IntervalUntilNowFrom( + unregisteredThreadSearchStart)), + "Work to discover threads"); + + // ... and record the time needed to process the data, which we can control. + AUTO_PROFILER_MARKER_TEXT( + "Unregistered thread search", PROFILER, + MarkerOptions(MarkerThreadId::MainThread()), + "Work to process discovered threads and record unregistered ones"_ns); + + const Span<const mozilla::ThreadInfo> threads = procInfoPtr->value().threads; + + // mLastSpying timestamp should be null only at the beginning of a session, + // when mSpiedThreads is still empty. + MOZ_ASSERT_IF(mLastSpying.IsNull(), mSpiedThreads.IsEmpty()); + + const TimeStamp previousSpying = std::exchange(mLastSpying, TimeStamp::Now()); + + // Find threads that were spied on but are not present anymore. + const auto threadsBegin = threads.begin(); + const auto threadsEnd = threads.end(); + for (size_t spiedThreadIndexPlus1 = mSpiedThreads.Length(); + spiedThreadIndexPlus1 != 0; --spiedThreadIndexPlus1) { + const SpiedThread& spiedThread = mSpiedThreads[spiedThreadIndexPlus1 - 1]; + if (std::find_if(threadsBegin, threadsEnd, + [spiedTid = spiedThread.mThreadId]( + const mozilla::ThreadInfo& aThreadInfo) { + return aThreadInfo.tid == spiedTid; + }) == threadsEnd) { + // This spied thread is gone. + PROFILER_MARKER( + MakeThreadInfoMarkerName(spiedThread.mThreadId, spiedThread.mName), + PROFILER, + MarkerOptions( + MarkerThreadId::MainThread(), + // Place the end between this update and the previous one. + MarkerTiming::IntervalEnd(previousSpying + + (mLastSpying - previousSpying) / + int64_t(2))), + UnregisteredThreadLifetimeMarker, spiedThread.mThreadId, + spiedThread.mName, "Thread disappeared"); + + // Don't spy on it anymore, assuming it won't come back. + mSpiedThreads.RemoveElementAt(spiedThreadIndexPlus1 - 1); + } + } + + for (const mozilla::ThreadInfo& threadInfo : threads) { + // Index of this encountered thread in mSpiedThreads, or NoIndex. + size_t spiedThreadIndex = mSpiedThreads.IndexOf(threadInfo.tid); + if (IsThreadIdRegistered(ProfilerThreadId::FromNumber(threadInfo.tid))) { + // This thread id is already officially registered. + if (spiedThreadIndex != SpiedThreads::NoIndex) { + // This now-registered thread was previously being spied. + SpiedThread& spiedThread = mSpiedThreads[spiedThreadIndex]; + PROFILER_MARKER( + MakeThreadInfoMarkerName(spiedThread.mThreadId, spiedThread.mName), + PROFILER, + MarkerOptions( + MarkerThreadId::MainThread(), + // Place the end between this update and the previous one. + // TODO: Find the real time from the thread registration? + MarkerTiming::IntervalEnd(previousSpying + + (mLastSpying - previousSpying) / + int64_t(2))), + UnregisteredThreadLifetimeMarker, spiedThread.mThreadId, + spiedThread.mName, "Thread registered itself"); + + // Remove from mSpiedThreads, since it can be profiled normally. + mSpiedThreads.RemoveElement(threadInfo.tid); + } + } else { + // This thread id is not registered. + if (spiedThreadIndex == SpiedThreads::NoIndex) { + // This unregistered thread has not been spied yet, store it now. + NS_ConvertUTF16toUTF8 name(threadInfo.name); + mSpiedThreads.EmplaceBack(threadInfo.tid, name, threadInfo.cpuTime); + + PROFILER_MARKER( + MakeThreadInfoMarkerName(threadInfo.tid, name), PROFILER, + MarkerOptions( + MarkerThreadId::MainThread(), + // Place the start between this update and the previous one (or + // the start of this search if it's the first one). + MarkerTiming::IntervalStart( + mLastSpying - + (mLastSpying - (previousSpying.IsNull() + ? unregisteredThreadSearchStart + : previousSpying)) / + int64_t(2))), + UnregisteredThreadLifetimeMarker, threadInfo.tid, name, + /* aEndEvent */ ""); + } else { + // This unregistered thread was already being spied, record its work. + SpiedThread& spiedThread = mSpiedThreads[spiedThreadIndex]; + int64_t diffCPUTimeNs = + int64_t(threadInfo.cpuTime) - int64_t(spiedThread.mCPUTimeNs); + spiedThread.mCPUTimeNs = threadInfo.cpuTime; + if (diffCPUTimeNs != 0) { + PROFILER_MARKER( + MakeThreadInfoMarkerName(threadInfo.tid, spiedThread.mName), + PROFILER, + MarkerOptions( + MarkerThreadId::MainThread(), + MarkerTiming::Interval(previousSpying, mLastSpying)), + UnregisteredThreadCPUMarker, threadInfo.tid, diffCPUTimeNs, + previousSpying, mLastSpying); + } + } + } + } + + PROFILER_MARKER_TEXT("Unregistered thread search", PROFILER, + MarkerOptions(MarkerThreadId::MainThread(), + MarkerTiming::IntervalUntilNowFrom( + unregisteredThreadSearchStart)), + "Work to discover and record unregistered threads"); +} + +// We #include these files directly because it means those files can use +// declarations from this file trivially. These provide target-specific +// implementations of all SamplerThread methods except Run(). +#if defined(GP_OS_windows) +# include "platform-win32.cpp" +#elif defined(GP_OS_darwin) +# include "platform-macos.cpp" +#elif defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) +# include "platform-linux-android.cpp" +#else +# error "bad platform" +#endif + +// END SamplerThread +//////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////// +// BEGIN externally visible functions + +MOZ_DEFINE_MALLOC_SIZE_OF(GeckoProfilerMallocSizeOf) + +NS_IMETHODIMP +GeckoProfilerReporter::CollectReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData, bool aAnonymize) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + + size_t profSize = 0; + size_t lulSize = 0; + + { + PSAutoLock lock; + + if (CorePS::Exists()) { + CorePS::AddSizeOf(lock, GeckoProfilerMallocSizeOf, profSize, lulSize); + } + + if (ActivePS::Exists(lock)) { + profSize += ActivePS::SizeOf(lock, GeckoProfilerMallocSizeOf); + } + } + + MOZ_COLLECT_REPORT( + "explicit/profiler/profiler-state", KIND_HEAP, UNITS_BYTES, profSize, + "Memory used by the Gecko Profiler's global state (excluding memory used " + "by LUL)."); + +#if defined(USE_LUL_STACKWALK) + MOZ_COLLECT_REPORT( + "explicit/profiler/lul", KIND_HEAP, UNITS_BYTES, lulSize, + "Memory used by LUL, a stack unwinder used by the Gecko Profiler."); +#endif + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(GeckoProfilerReporter, nsIMemoryReporter) + +static uint32_t ParseFeature(const char* aFeature, bool aIsStartup) { + if (strcmp(aFeature, "default") == 0) { + return (aIsStartup ? (DefaultFeatures() | StartupExtraDefaultFeatures()) + : DefaultFeatures()) & + AvailableFeatures(); + } + +#define PARSE_FEATURE_BIT(n_, str_, Name_, desc_) \ + if (strcmp(aFeature, str_) == 0) { \ + return ProfilerFeature::Name_; \ + } + + PROFILER_FOR_EACH_FEATURE(PARSE_FEATURE_BIT) + +#undef PARSE_FEATURE_BIT + + printf("\nUnrecognized feature \"%s\".\n\n", aFeature); + // Since we may have an old feature we don't implement anymore, don't exit. + PrintUsage(); + return 0; +} + +uint32_t ParseFeaturesFromStringArray(const char** aFeatures, + uint32_t aFeatureCount, + bool aIsStartup /* = false */) { + uint32_t features = 0; + for (size_t i = 0; i < aFeatureCount; i++) { + features |= ParseFeature(aFeatures[i], aIsStartup); + } + return features; +} + +static ProfilingStack* locked_register_thread( + PSLockRef aLock, ThreadRegistry::OffThreadRef aOffThreadRef) { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + VTUNE_REGISTER_THREAD(aOffThreadRef.UnlockedConstReaderCRef().Info().Name()); + + if (ActivePS::Exists(aLock)) { + ThreadProfilingFeatures threadProfilingFeatures = + ActivePS::ProfilingFeaturesForThread( + aLock, aOffThreadRef.UnlockedConstReaderCRef().Info()); + if (threadProfilingFeatures != ThreadProfilingFeatures::NotProfiled) { + ThreadRegistry::OffThreadRef::RWFromAnyThreadWithLock + lockedRWFromAnyThread = aOffThreadRef.GetLockedRWFromAnyThread(); + + ProfiledThreadData* profiledThreadData = ActivePS::AddLiveProfiledThread( + aLock, MakeUnique<ProfiledThreadData>( + aOffThreadRef.UnlockedConstReaderCRef().Info())); + lockedRWFromAnyThread->SetProfilingFeaturesAndData( + threadProfilingFeatures, profiledThreadData, aLock); + + if (ActivePS::FeatureJS(aLock)) { + lockedRWFromAnyThread->StartJSSampling(ActivePS::JSFlags(aLock)); + if (ThreadRegistration::LockedRWOnThread* lockedRWOnThread = + lockedRWFromAnyThread.GetLockedRWOnThread(); + lockedRWOnThread) { + // We can manually poll the current thread so it starts sampling + // immediately. + lockedRWOnThread->PollJSSampling(); + } + if (lockedRWFromAnyThread->GetJSContext()) { + profiledThreadData->NotifyReceivedJSContext( + ActivePS::Buffer(aLock).BufferRangeEnd()); + } + } + } + } + + return &aOffThreadRef.UnlockedConstReaderAndAtomicRWRef().ProfilingStackRef(); +} + +static void NotifyObservers(const char* aTopic, + nsISupports* aSubject = nullptr) { + if (!NS_IsMainThread()) { + // Dispatch a task to the main thread that notifies observers. + // If NotifyObservers is called both on and off the main thread within a + // short time, the order of the notifications can be different from the + // order of the calls to NotifyObservers. + // Getting the order 100% right isn't that important at the moment, because + // these notifications are only observed in the parent process, where the + // profiler_* functions are currently only called on the main thread. + nsCOMPtr<nsISupports> subject = aSubject; + NS_DispatchToMainThread(NS_NewRunnableFunction( + "NotifyObservers", [=] { NotifyObservers(aTopic, subject); })); + return; + } + + if (nsCOMPtr<nsIObserverService> os = services::GetObserverService()) { + os->NotifyObservers(aSubject, aTopic, nullptr); + } +} + +[[nodiscard]] static RefPtr<GenericPromise> NotifyProfilerStarted( + const PowerOfTwo32& aCapacity, const Maybe<double>& aDuration, + double aInterval, uint32_t aFeatures, const char** aFilters, + uint32_t aFilterCount, uint64_t aActiveTabID) { + nsTArray<nsCString> filtersArray; + for (size_t i = 0; i < aFilterCount; ++i) { + filtersArray.AppendElement(aFilters[i]); + } + + nsCOMPtr<nsIProfilerStartParams> params = new nsProfilerStartParams( + aCapacity.Value(), aDuration, aInterval, aFeatures, + std::move(filtersArray), aActiveTabID); + + RefPtr<GenericPromise> startPromise = ProfilerParent::ProfilerStarted(params); + NotifyObservers("profiler-started", params); + return startPromise; +} + +static void locked_profiler_start(PSLockRef aLock, PowerOfTwo32 aCapacity, + double aInterval, uint32_t aFeatures, + const char** aFilters, uint32_t aFilterCount, + uint64_t aActiveTabID, + const Maybe<double>& aDuration); + +// This basically duplicates AutoProfilerLabel's constructor. +static void* MozGlueLabelEnter(const char* aLabel, const char* aDynamicString, + void* aSp) { + ThreadRegistration::OnThreadPtr onThreadPtr = + ThreadRegistration::GetOnThreadPtr(); + if (!onThreadPtr) { + return nullptr; + } + ProfilingStack& profilingStack = + onThreadPtr->UnlockedConstReaderAndAtomicRWRef().ProfilingStackRef(); + profilingStack.pushLabelFrame(aLabel, aDynamicString, aSp, + JS::ProfilingCategoryPair::OTHER); + return &profilingStack; +} + +// This basically duplicates AutoProfilerLabel's destructor. +static void MozGlueLabelExit(void* aProfilingStack) { + if (aProfilingStack) { + reinterpret_cast<ProfilingStack*>(aProfilingStack)->pop(); + } +} + +static Vector<const char*> SplitAtCommas(const char* aString, + UniquePtr<char[]>& aStorage) { + size_t len = strlen(aString); + aStorage = MakeUnique<char[]>(len + 1); + PodCopy(aStorage.get(), aString, len + 1); + + // Iterate over all characters in aStorage and split at commas, by + // overwriting commas with the null char. + Vector<const char*> array; + size_t currentElementStart = 0; + for (size_t i = 0; i <= len; i++) { + if (aStorage[i] == ',') { + aStorage[i] = '\0'; + } + if (aStorage[i] == '\0') { + // Only add non-empty elements, otherwise ParseFeatures would later + // complain about unrecognized features. + if (currentElementStart != i) { + MOZ_RELEASE_ASSERT(array.append(&aStorage[currentElementStart])); + } + currentElementStart = i + 1; + } + } + return array; +} + +void profiler_init_threadmanager() { + LOG("profiler_init_threadmanager"); + + ThreadRegistration::WithOnThreadRef( + [](ThreadRegistration::OnThreadRef aOnThreadRef) { + aOnThreadRef.WithLockedRWOnThread( + [](ThreadRegistration::LockedRWOnThread& aThreadData) { + if (!aThreadData.GetEventTarget()) { + aThreadData.ResetMainThread(NS_GetCurrentThreadNoCreate()); + } + }); + }); +} + +static const char* get_size_suffix(const char* str) { + const char* ptr = str; + + while (isdigit(*ptr)) { + ptr++; + } + + return ptr; +} + +void profiler_init(void* aStackTop) { + LOG("profiler_init"); + + profiler_init_main_thread_id(); + + VTUNE_INIT(); + ETW::Init(); + + MOZ_RELEASE_ASSERT(!CorePS::Exists()); + + if (getenv("MOZ_PROFILER_HELP")) { + PrintUsage(); + exit(0); + } + + SharedLibraryInfo::Initialize(); + + uint32_t features = DefaultFeatures() & AvailableFeatures(); + + UniquePtr<char[]> filterStorage; + + Vector<const char*> filters; + MOZ_RELEASE_ASSERT(filters.append("GeckoMain")); + MOZ_RELEASE_ASSERT(filters.append("Compositor")); + MOZ_RELEASE_ASSERT(filters.append("Renderer")); + MOZ_RELEASE_ASSERT(filters.append("DOM Worker")); + + PowerOfTwo32 capacity = PROFILER_DEFAULT_ENTRIES; + Maybe<double> duration = Nothing(); + double interval = PROFILER_DEFAULT_INTERVAL; + uint64_t activeTabID = PROFILER_DEFAULT_ACTIVE_TAB_ID; + + ThreadRegistration::RegisterThread(kMainThreadName, aStackTop); + + { + PSAutoLock lock; + + // We've passed the possible failure point. Instantiate CorePS, which + // indicates that the profiler has initialized successfully. + CorePS::Create(lock); + + // Make sure threads already in the ThreadRegistry (like the main thread) + // get registered in CorePS as well. + { + ThreadRegistry::LockedRegistry lockedRegistry; + for (ThreadRegistry::OffThreadRef offThreadRef : lockedRegistry) { + locked_register_thread(lock, offThreadRef); + } + } + + // Platform-specific initialization. + PlatformInit(lock); + +#if defined(GP_OS_android) + if (jni::IsAvailable()) { + GeckoJavaSampler::Init(); + } +#endif + + // (Linux-only) We could create CorePS::mLul and read unwind info into it + // at this point. That would match the lifetime implied by destruction of + // it in profiler_shutdown() just below. However, that gives a big delay on + // startup, even if no profiling is actually to be done. So, instead, it is + // created on demand at the first call to PlatformStart(). + + const char* startupEnv = getenv("MOZ_PROFILER_STARTUP"); + if (!startupEnv || startupEnv[0] == '\0' || + ((startupEnv[0] == '0' || startupEnv[0] == 'N' || + startupEnv[0] == 'n') && + startupEnv[1] == '\0')) { + return; + } + + LOG("- MOZ_PROFILER_STARTUP is set"); + + // Startup default capacity may be different. + capacity = PROFILER_DEFAULT_STARTUP_ENTRIES; + + const char* startupCapacity = getenv("MOZ_PROFILER_STARTUP_ENTRIES"); + if (startupCapacity && startupCapacity[0] != '\0') { + errno = 0; + long capacityLong = strtol(startupCapacity, nullptr, 10); + std::string_view sizeSuffix = get_size_suffix(startupCapacity); + + if (sizeSuffix == "KB") { + capacityLong *= 1000 / scBytesPerEntry; + } else if (sizeSuffix == "KiB") { + capacityLong *= 1024 / scBytesPerEntry; + } else if (sizeSuffix == "MB") { + capacityLong *= (1000 * 1000) / scBytesPerEntry; + } else if (sizeSuffix == "MiB") { + capacityLong *= (1024 * 1024) / scBytesPerEntry; + } else if (sizeSuffix == "GB") { + capacityLong *= (1000 * 1000 * 1000) / scBytesPerEntry; + } else if (sizeSuffix == "GiB") { + capacityLong *= (1024 * 1024 * 1024) / scBytesPerEntry; + } else if (!sizeSuffix.empty()) { + LOG("- MOZ_PROFILER_STARTUP_ENTRIES unit must be one of the " + "following: KB, KiB, MB, MiB, GB, GiB"); + PrintUsage(); + exit(1); + } + + // `long` could be 32 or 64 bits, so we force a 64-bit comparison with + // the maximum 32-bit signed number (as more than that is clamped down to + // 2^31 anyway). + if (errno == 0 && capacityLong > 0 && + static_cast<uint64_t>(capacityLong) <= + static_cast<uint64_t>(INT32_MAX)) { + capacity = PowerOfTwo32( + ClampToAllowedEntries(static_cast<uint32_t>(capacityLong))); + LOG("- MOZ_PROFILER_STARTUP_ENTRIES = %u", unsigned(capacity.Value())); + } else { + LOG("- MOZ_PROFILER_STARTUP_ENTRIES not a valid integer: %s", + startupCapacity); + PrintUsage(); + exit(1); + } + } + + const char* startupDuration = getenv("MOZ_PROFILER_STARTUP_DURATION"); + if (startupDuration && startupDuration[0] != '\0') { + errno = 0; + double durationVal = PR_strtod(startupDuration, nullptr); + if (errno == 0 && durationVal >= 0.0) { + if (durationVal > 0.0) { + duration = Some(durationVal); + } + LOG("- MOZ_PROFILER_STARTUP_DURATION = %f", durationVal); + } else { + LOG("- MOZ_PROFILER_STARTUP_DURATION not a valid float: %s", + startupDuration); + PrintUsage(); + exit(1); + } + } + + const char* startupInterval = getenv("MOZ_PROFILER_STARTUP_INTERVAL"); + if (startupInterval && startupInterval[0] != '\0') { + errno = 0; + interval = PR_strtod(startupInterval, nullptr); + if (errno == 0 && interval > 0.0 && interval <= PROFILER_MAX_INTERVAL) { + LOG("- MOZ_PROFILER_STARTUP_INTERVAL = %f", interval); + } else { + LOG("- MOZ_PROFILER_STARTUP_INTERVAL not a valid float: %s", + startupInterval); + PrintUsage(); + exit(1); + } + } + + features |= StartupExtraDefaultFeatures() & AvailableFeatures(); + + const char* startupFeaturesBitfield = + getenv("MOZ_PROFILER_STARTUP_FEATURES_BITFIELD"); + if (startupFeaturesBitfield && startupFeaturesBitfield[0] != '\0') { + errno = 0; + features = strtol(startupFeaturesBitfield, nullptr, 10); + if (errno == 0) { + LOG("- MOZ_PROFILER_STARTUP_FEATURES_BITFIELD = %d", features); + } else { + LOG("- MOZ_PROFILER_STARTUP_FEATURES_BITFIELD not a valid integer: %s", + startupFeaturesBitfield); + PrintUsage(); + exit(1); + } + } else { + const char* startupFeatures = getenv("MOZ_PROFILER_STARTUP_FEATURES"); + if (startupFeatures) { + // Interpret startupFeatures as a list of feature strings, separated by + // commas. + UniquePtr<char[]> featureStringStorage; + Vector<const char*> featureStringArray = + SplitAtCommas(startupFeatures, featureStringStorage); + features = ParseFeaturesFromStringArray(featureStringArray.begin(), + featureStringArray.length(), + /* aIsStartup */ true); + LOG("- MOZ_PROFILER_STARTUP_FEATURES = %d", features); + } + } + + const char* startupFilters = getenv("MOZ_PROFILER_STARTUP_FILTERS"); + if (startupFilters && startupFilters[0] != '\0') { + filters = SplitAtCommas(startupFilters, filterStorage); + LOG("- MOZ_PROFILER_STARTUP_FILTERS = %s", startupFilters); + + if (mozilla::profiler::detail::FiltersExcludePid(filters)) { + LOG(" -> This process is excluded and won't be profiled"); + return; + } + } + + const char* startupActiveTabID = + getenv("MOZ_PROFILER_STARTUP_ACTIVE_TAB_ID"); + if (startupActiveTabID && startupActiveTabID[0] != '\0') { + std::istringstream iss(startupActiveTabID); + iss >> activeTabID; + if (!iss.fail()) { + LOG("- MOZ_PROFILER_STARTUP_ACTIVE_TAB_ID = %" PRIu64, activeTabID); + } else { + LOG("- MOZ_PROFILER_STARTUP_ACTIVE_TAB_ID not a valid " + "uint64_t: %s", + startupActiveTabID); + PrintUsage(); + exit(1); + } + } + + locked_profiler_start(lock, capacity, interval, features, filters.begin(), + filters.length(), activeTabID, duration); + } + + // The GeckoMain thread registration happened too early to record a marker, + // so let's record it again now. + profiler_mark_thread_awake(); + +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + // Start counting memory allocations (outside of lock because this may call + // profiler_add_sampled_counter which would attempt to take the lock.) + ActivePS::SetMemoryCounter(mozilla::profiler::install_memory_hooks()); +#endif + + invoke_profiler_state_change_callbacks(ProfilingState::Started); + + // We do this with gPSMutex unlocked. The comment in profiler_stop() explains + // why. + Unused << NotifyProfilerStarted(capacity, duration, interval, features, + filters.begin(), filters.length(), 0); +} + +static void locked_profiler_save_profile_to_file( + PSLockRef aLock, const char* aFilename, + const PreRecordedMetaInformation& aPreRecordedMetaInformation, + bool aIsShuttingDown); + +static SamplerThread* locked_profiler_stop(PSLockRef aLock); + +void profiler_shutdown(IsFastShutdown aIsFastShutdown) { + LOG("profiler_shutdown"); + + VTUNE_SHUTDOWN(); + ETW::Shutdown(); + + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + if (profiler_is_active()) { + invoke_profiler_state_change_callbacks(ProfilingState::Stopping); + } + invoke_profiler_state_change_callbacks(ProfilingState::ShuttingDown); + + const auto preRecordedMetaInformation = + PreRecordMetaInformation(/* aShutdown = */ true); + + ProfilerParent::ProfilerWillStopIfStarted(); + + // If the profiler is active we must get a handle to the SamplerThread before + // ActivePS is destroyed, in order to delete it. + SamplerThread* samplerThread = nullptr; + { + PSAutoLock lock; + + // Save the profile on shutdown if requested. + if (ActivePS::Exists(lock)) { + const char* filename = getenv("MOZ_PROFILER_SHUTDOWN"); + if (filename && filename[0] != '\0') { + locked_profiler_save_profile_to_file(lock, filename, + preRecordedMetaInformation, + /* aIsShuttingDown */ true); + } + if (aIsFastShutdown == IsFastShutdown::Yes) { + return; + } + + samplerThread = locked_profiler_stop(lock); + } else if (aIsFastShutdown == IsFastShutdown::Yes) { + return; + } + + CorePS::Destroy(lock); + } + + // We do these operations with gPSMutex unlocked. The comments in + // profiler_stop() explain why. + if (samplerThread) { + Unused << ProfilerParent::ProfilerStopped(); + NotifyObservers("profiler-stopped"); + delete samplerThread; + } + + // Reverse the registration done in profiler_init. + ThreadRegistration::UnregisterThread(); +} + +static bool WriteProfileToJSONWriter(SpliceableChunkedJSONWriter& aWriter, + double aSinceTime, bool aIsShuttingDown, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger) { + LOG("WriteProfileToJSONWriter"); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + aWriter.Start(); + { + auto rv = profiler_stream_json_for_this_process( + aWriter, aSinceTime, aIsShuttingDown, aService, + aProgressLogger.CreateSubLoggerFromTo( + 0_pc, + "WriteProfileToJSONWriter: " + "profiler_stream_json_for_this_process started", + 100_pc, + "WriteProfileToJSONWriter: " + "profiler_stream_json_for_this_process done")); + + if (rv.isErr()) { + return false; + } + + // Don't include profiles from other processes because this is a + // synchronous function. + aWriter.StartArrayProperty("processes"); + aWriter.EndArray(); + } + aWriter.End(); + return !aWriter.Failed(); +} + +void profiler_set_process_name(const nsACString& aProcessName, + const nsACString* aETLDplus1) { + LOG("profiler_set_process_name(\"%s\", \"%s\")", aProcessName.Data(), + aETLDplus1 ? aETLDplus1->Data() : "<none>"); + PSAutoLock lock; + CorePS::SetProcessName(lock, aProcessName); + if (aETLDplus1) { + CorePS::SetETLDplus1(lock, *aETLDplus1); + } +} + +UniquePtr<char[]> profiler_get_profile(double aSinceTime, + bool aIsShuttingDown) { + LOG("profiler_get_profile"); + + UniquePtr<ProfilerCodeAddressService> service = + profiler_code_address_service_for_presymbolication(); + + FailureLatchSource failureLatch; + SpliceableChunkedJSONWriter b{failureLatch}; + if (!WriteProfileToJSONWriter(b, aSinceTime, aIsShuttingDown, service.get(), + ProgressLogger{})) { + return nullptr; + } + return b.ChunkedWriteFunc().CopyData(); +} + +[[nodiscard]] bool profiler_get_profile_json( + SpliceableChunkedJSONWriter& aSpliceableChunkedJSONWriter, + double aSinceTime, bool aIsShuttingDown, + mozilla::ProgressLogger aProgressLogger) { + LOG("profiler_get_profile_json"); + + UniquePtr<ProfilerCodeAddressService> service = + profiler_code_address_service_for_presymbolication(); + + return WriteProfileToJSONWriter( + aSpliceableChunkedJSONWriter, aSinceTime, aIsShuttingDown, service.get(), + aProgressLogger.CreateSubLoggerFromTo( + 0.1_pc, "profiler_get_profile_json: WriteProfileToJSONWriter started", + 99.9_pc, "profiler_get_profile_json: WriteProfileToJSONWriter done")); +} + +void profiler_get_start_params(int* aCapacity, Maybe<double>* aDuration, + double* aInterval, uint32_t* aFeatures, + Vector<const char*>* aFilters, + uint64_t* aActiveTabID) { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + if (NS_WARN_IF(!aCapacity) || NS_WARN_IF(!aDuration) || + NS_WARN_IF(!aInterval) || NS_WARN_IF(!aFeatures) || + NS_WARN_IF(!aFilters)) { + return; + } + + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + *aCapacity = 0; + *aDuration = Nothing(); + *aInterval = 0; + *aFeatures = 0; + *aActiveTabID = 0; + aFilters->clear(); + return; + } + + *aCapacity = ActivePS::Capacity(lock).Value(); + *aDuration = ActivePS::Duration(lock); + *aInterval = ActivePS::Interval(lock); + *aFeatures = ActivePS::Features(lock); + *aActiveTabID = ActivePS::ActiveTabID(lock); + + const Vector<std::string>& filters = ActivePS::Filters(lock); + MOZ_ALWAYS_TRUE(aFilters->resize(filters.length())); + for (uint32_t i = 0; i < filters.length(); ++i) { + (*aFilters)[i] = filters[i].c_str(); + } +} + +ProfileBufferControlledChunkManager* profiler_get_controlled_chunk_manager() { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + PSAutoLock lock; + if (NS_WARN_IF(!ActivePS::Exists(lock))) { + return nullptr; + } + return &ActivePS::ControlledChunkManager(lock); +} + +namespace mozilla { + +void GetProfilerEnvVarsForChildProcess( + std::function<void(const char* key, const char* value)>&& aSetEnv) { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + aSetEnv("MOZ_PROFILER_STARTUP", ""); + return; + } + + aSetEnv("MOZ_PROFILER_STARTUP", "1"); + + // If MOZ_PROFILER_SHUTDOWN is defined, make sure it's empty in children, so + // that they don't attempt to write over that file. + if (getenv("MOZ_PROFILER_SHUTDOWN")) { + aSetEnv("MOZ_PROFILER_SHUTDOWN", ""); + } + + // Hidden option to stop Base Profiler, mostly due to Talos intermittents, + // see https://bugzilla.mozilla.org/show_bug.cgi?id=1638851#c3 + // TODO: Investigate root cause and remove this in bugs 1648324 and 1648325. + if (getenv("MOZ_PROFILER_STARTUP_NO_BASE")) { + aSetEnv("MOZ_PROFILER_STARTUP_NO_BASE", "1"); + } + + auto capacityString = + Smprintf("%u", unsigned(ActivePS::Capacity(lock).Value())); + aSetEnv("MOZ_PROFILER_STARTUP_ENTRIES", capacityString.get()); + + // Use AppendFloat instead of Smprintf with %f because the decimal + // separator used by %f is locale-dependent. But the string we produce needs + // to be parseable by strtod, which only accepts the period character as a + // decimal separator. AppendFloat always uses the period character. + nsCString intervalString; + intervalString.AppendFloat(ActivePS::Interval(lock)); + aSetEnv("MOZ_PROFILER_STARTUP_INTERVAL", intervalString.get()); + + auto featuresString = Smprintf("%d", ActivePS::Features(lock)); + aSetEnv("MOZ_PROFILER_STARTUP_FEATURES_BITFIELD", featuresString.get()); + + std::string filtersString; + const Vector<std::string>& filters = ActivePS::Filters(lock); + for (uint32_t i = 0; i < filters.length(); ++i) { + if (i != 0) { + filtersString += ","; + } + filtersString += filters[i]; + } + aSetEnv("MOZ_PROFILER_STARTUP_FILTERS", filtersString.c_str()); + + auto activeTabIDString = Smprintf("%" PRIu64, ActivePS::ActiveTabID(lock)); + aSetEnv("MOZ_PROFILER_STARTUP_ACTIVE_TAB_ID", activeTabIDString.get()); +} + +} // namespace mozilla + +void profiler_received_exit_profile(const nsACString& aExitProfile) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + MOZ_RELEASE_ASSERT(CorePS::Exists()); + PSAutoLock lock; + if (!ActivePS::Exists(lock)) { + return; + } + ActivePS::AddExitProfile(lock, aExitProfile); +} + +Vector<nsCString> profiler_move_exit_profiles() { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + PSAutoLock lock; + Vector<nsCString> profiles; + if (ActivePS::Exists(lock)) { + profiles = ActivePS::MoveExitProfiles(lock); + } + return profiles; +} + +static void locked_profiler_save_profile_to_file( + PSLockRef aLock, const char* aFilename, + const PreRecordedMetaInformation& aPreRecordedMetaInformation, + bool aIsShuttingDown = false) { + nsAutoCString processedFilename(aFilename); + const auto processInsertionIndex = processedFilename.Find("%p"); + if (processInsertionIndex != kNotFound) { + // Replace "%p" with the process id. + nsAutoCString process; + process.AppendInt(profiler_current_process_id().ToNumber()); + processedFilename.Replace(processInsertionIndex, 2, process); + LOG("locked_profiler_save_profile_to_file(\"%s\" -> \"%s\")", aFilename, + processedFilename.get()); + } else { + LOG("locked_profiler_save_profile_to_file(\"%s\")", aFilename); + } + + MOZ_RELEASE_ASSERT(CorePS::Exists() && ActivePS::Exists(aLock)); + + std::ofstream stream; + stream.open(processedFilename.get()); + if (stream.is_open()) { + OStreamJSONWriteFunc sw(stream); + SpliceableJSONWriter w(sw, FailureLatchInfallibleSource::Singleton()); + w.Start(); + { + Unused << locked_profiler_stream_json_for_this_process( + aLock, w, /* sinceTime */ 0, aPreRecordedMetaInformation, + aIsShuttingDown, nullptr, ProgressLogger{}); + + w.StartArrayProperty("processes"); + Vector<nsCString> exitProfiles = ActivePS::MoveExitProfiles(aLock); + for (auto& exitProfile : exitProfiles) { + if (!exitProfile.IsEmpty() && exitProfile[0] != '*') { + w.Splice(exitProfile); + } + } + w.EndArray(); + } + w.End(); + + stream.close(); + } +} + +void profiler_save_profile_to_file(const char* aFilename) { + LOG("profiler_save_profile_to_file(%s)", aFilename); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + const auto preRecordedMetaInformation = PreRecordMetaInformation(); + + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return; + } + + locked_profiler_save_profile_to_file(lock, aFilename, + preRecordedMetaInformation); +} + +uint32_t profiler_get_available_features() { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + return AvailableFeatures(); +} + +Maybe<ProfilerBufferInfo> profiler_get_buffer_info() { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return Nothing(); + } + + return Some(ActivePS::Buffer(lock).GetProfilerBufferInfo()); +} + +static void PollJSSamplingForCurrentThread() { + ThreadRegistration::WithOnThreadRef( + [](ThreadRegistration::OnThreadRef aOnThreadRef) { + aOnThreadRef.WithLockedRWOnThread( + [](ThreadRegistration::LockedRWOnThread& aThreadData) { + aThreadData.PollJSSampling(); + }); + }); +} + +// When the profiler is started on a background thread, we can't synchronously +// call PollJSSampling on the main thread's ThreadInfo. And the next regular +// call to PollJSSampling on the main thread would only happen once the main +// thread triggers a JS interrupt callback. +// This means that all the JS execution between profiler_start() and the first +// JS interrupt would happen with JS sampling disabled, and we wouldn't get any +// JS function information for that period of time. +// So in order to start JS sampling as soon as possible, we dispatch a runnable +// to the main thread which manually calls PollJSSamplingForCurrentThread(). +// In some cases this runnable will lose the race with the next JS interrupt. +// That's fine; PollJSSamplingForCurrentThread() is immune to redundant calls. +static void TriggerPollJSSamplingOnMainThread() { + nsCOMPtr<nsIThread> mainThread; + nsresult rv = NS_GetMainThread(getter_AddRefs(mainThread)); + if (NS_SUCCEEDED(rv) && mainThread) { + nsCOMPtr<nsIRunnable> task = + NS_NewRunnableFunction("TriggerPollJSSamplingOnMainThread", + []() { PollJSSamplingForCurrentThread(); }); + SchedulerGroup::Dispatch(task.forget()); + } +} + +static void locked_profiler_start(PSLockRef aLock, PowerOfTwo32 aCapacity, + double aInterval, uint32_t aFeatures, + const char** aFilters, uint32_t aFilterCount, + uint64_t aActiveTabID, + const Maybe<double>& aDuration) { + TimeStamp profilingStartTime = TimeStamp::Now(); + + if (LOG_TEST) { + LOG("locked_profiler_start"); + LOG("- capacity = %u", unsigned(aCapacity.Value())); + LOG("- duration = %.2f", aDuration ? *aDuration : -1); + LOG("- interval = %.2f", aInterval); + LOG("- tab ID = %" PRIu64, aActiveTabID); + +#define LOG_FEATURE(n_, str_, Name_, desc_) \ + if (ProfilerFeature::Has##Name_(aFeatures)) { \ + LOG("- feature = %s", str_); \ + } + + PROFILER_FOR_EACH_FEATURE(LOG_FEATURE) + +#undef LOG_FEATURE + + for (uint32_t i = 0; i < aFilterCount; i++) { + LOG("- threads = %s", aFilters[i]); + } + } + + MOZ_RELEASE_ASSERT(CorePS::Exists() && !ActivePS::Exists(aLock)); + + // Do this before the Base Profiler is stopped, to keep the existing buffer + // (if any) alive for our use. + if (NS_IsMainThread()) { + mozilla::base_profiler_markers_detail::EnsureBufferForMainThreadAddMarker(); + } else { + NS_DispatchToMainThread( + NS_NewRunnableFunction("EnsureBufferForMainThreadAddMarker", + &mozilla::base_profiler_markers_detail:: + EnsureBufferForMainThreadAddMarker)); + } + + UniquePtr<ProfileBufferChunkManagerWithLocalLimit> baseChunkManager; + bool profilersHandOver = false; + if (baseprofiler::profiler_is_active()) { + // Note that we still hold the lock, so the sampler cannot run yet and + // interact negatively with the still-active BaseProfiler sampler. + // Assume that Base Profiler is active because of MOZ_PROFILER_STARTUP. + + // Take ownership of the chunk manager from the Base Profiler, to extend its + // lifetime during the new Gecko Profiler session. Since we're using the + // same core buffer, all the base profiler data remains. + baseChunkManager = baseprofiler::detail::ExtractBaseProfilerChunkManager(); + + if (baseChunkManager) { + profilersHandOver = true; + if (const TimeStamp baseProfilingStartTime = + baseprofiler::detail::GetProfilingStartTime(); + !baseProfilingStartTime.IsNull()) { + profilingStartTime = baseProfilingStartTime; + } + + BASE_PROFILER_MARKER_TEXT( + "Profilers handover", PROFILER, MarkerTiming::IntervalStart(), + "Transition from Base to Gecko Profiler, some data may be missing"); + } + + // Now stop Base Profiler (BP), as further recording will be ignored anyway, + // and so that it won't clash with Gecko Profiler (GP) sampling starting + // after the lock is dropped. + // On Linux this is especially important to do before creating the GP + // sampler, because the BP sampler may send a signal (to stop threads to be + // sampled), which the GP would intercept before its own initialization is + // complete and ready to handle such signals. + // Note that even though `profiler_stop()` doesn't immediately destroy and + // join the sampler thread, it safely deactivates it in such a way that the + // thread will soon exit without doing any actual work. + // TODO: Allow non-sampling profiling to continue. + // TODO: Re-start BP after GP shutdown, to capture post-XPCOM shutdown. + baseprofiler::profiler_stop(); + } + +#if defined(GP_PLAT_amd64_windows) || defined(GP_PLAT_arm64_windows) + mozilla::WindowsStackWalkInitialization(); +#endif + + // Fall back to the default values if the passed-in values are unreasonable. + // We want to be able to store at least one full stack. + PowerOfTwo32 capacity = + (aCapacity.Value() >= + ProfileBufferChunkManager::scExpectedMaximumStackSize / scBytesPerEntry) + ? aCapacity + : PROFILER_DEFAULT_ENTRIES; + Maybe<double> duration = aDuration; + + if (aDuration && *aDuration <= 0) { + duration = Nothing(); + } + + double interval = aInterval > 0 ? aInterval : PROFILER_DEFAULT_INTERVAL; + + ActivePS::Create(aLock, profilingStartTime, capacity, interval, aFeatures, + aFilters, aFilterCount, aActiveTabID, duration, + std::move(baseChunkManager)); + + // ActivePS::Create can only succeed or crash. + MOZ_ASSERT(ActivePS::Exists(aLock)); + + // Set up profiling for each registered thread, if appropriate. +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + bool isMainThreadBeingProfiled = false; +#endif + ThreadRegistry::LockedRegistry lockedRegistry; + for (ThreadRegistry::OffThreadRef offThreadRef : lockedRegistry) { + const ThreadRegistrationInfo& info = + offThreadRef.UnlockedConstReaderCRef().Info(); + + ThreadProfilingFeatures threadProfilingFeatures = + ActivePS::ProfilingFeaturesForThread(aLock, info); + if (threadProfilingFeatures != ThreadProfilingFeatures::NotProfiled) { + ThreadRegistry::OffThreadRef::RWFromAnyThreadWithLock lockedThreadData = + offThreadRef.GetLockedRWFromAnyThread(); + ProfiledThreadData* profiledThreadData = ActivePS::AddLiveProfiledThread( + aLock, MakeUnique<ProfiledThreadData>(info)); + lockedThreadData->SetProfilingFeaturesAndData(threadProfilingFeatures, + profiledThreadData, aLock); + lockedThreadData->GetNewCpuTimeInNs(); + if (ActivePS::FeatureJS(aLock)) { + lockedThreadData->StartJSSampling(ActivePS::JSFlags(aLock)); + if (ThreadRegistration::LockedRWOnThread* lockedRWOnThread = + lockedThreadData.GetLockedRWOnThread(); + lockedRWOnThread) { + // We can manually poll the current thread so it starts sampling + // immediately. + lockedRWOnThread->PollJSSampling(); + } else if (info.IsMainThread()) { + // Dispatch a runnable to the main thread to call + // PollJSSampling(), so that we don't have wait for the next JS + // interrupt callback in order to start profiling JS. + TriggerPollJSSamplingOnMainThread(); + } + } +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + if (info.IsMainThread()) { + isMainThreadBeingProfiled = true; + } +#endif + lockedThreadData->ReinitializeOnResume(); + if (ActivePS::FeatureJS(aLock) && lockedThreadData->GetJSContext()) { + profiledThreadData->NotifyReceivedJSContext(0); + } + } + } + + // Setup support for pushing/popping labels in mozglue. + RegisterProfilerLabelEnterExit(MozGlueLabelEnter, MozGlueLabelExit); + +#if defined(GP_OS_android) + if (ActivePS::FeatureJava(aLock)) { + int javaInterval = interval; + // Java sampling doesn't accurately keep up with the sampling rate that is + // lower than 1ms. + if (javaInterval < 1) { + javaInterval = 1; + } + + JNIEnv* env = jni::GetEnvForThread(); + const auto& filters = ActivePS::Filters(aLock); + jni::ObjectArray::LocalRef javaFilters = + jni::ObjectArray::New<jni::String>(filters.length()); + for (size_t i = 0; i < filters.length(); i++) { + javaFilters->SetElement(i, jni::StringParam(filters[i].data(), env)); + } + + // Send the interval-relative entry count, but we have 100000 hard cap in + // the java code, it can't be more than that. + java::GeckoJavaSampler::Start( + javaFilters, javaInterval, + std::round((double)(capacity.Value()) * interval / + (double)(javaInterval))); + } +#endif + +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + if (ActivePS::FeatureNativeAllocations(aLock)) { + if (isMainThreadBeingProfiled) { + mozilla::profiler::enable_native_allocations(); + } else { + NS_WARNING( + "The nativeallocations feature is turned on, but the main thread is " + "not being profiled. The allocations are only stored on the main " + "thread."); + } + } +#endif + + if (ProfilerFeature::HasAudioCallbackTracing(aFeatures)) { + StartAudioCallbackTracing(); + } + + // At the very end, set up RacyFeatures. + RacyFeatures::SetActive(ActivePS::Features(aLock)); + + if (profilersHandOver) { + PROFILER_MARKER_UNTYPED("Profilers handover", PROFILER, + MarkerTiming::IntervalEnd()); + } +} + +RefPtr<GenericPromise> profiler_start(PowerOfTwo32 aCapacity, double aInterval, + uint32_t aFeatures, const char** aFilters, + uint32_t aFilterCount, + uint64_t aActiveTabID, + const Maybe<double>& aDuration) { + LOG("profiler_start"); + + ProfilerParent::ProfilerWillStopIfStarted(); + + SamplerThread* samplerThread = nullptr; + { + PSAutoLock lock; + + // Initialize if necessary. + if (!CorePS::Exists()) { + profiler_init(nullptr); + } + + // Reset the current state if the profiler is running. + if (ActivePS::Exists(lock)) { + // Note: Not invoking callbacks with ProfilingState::Stopping, because + // we're under lock, and also it would not be useful: Any profiling data + // will be discarded, and we're immediately restarting the profiler below + // and then notifying ProfilingState::Started. + samplerThread = locked_profiler_stop(lock); + } + + locked_profiler_start(lock, aCapacity, aInterval, aFeatures, aFilters, + aFilterCount, aActiveTabID, aDuration); + } + +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + // Start counting memory allocations (outside of lock because this may call + // profiler_add_sampled_counter which would attempt to take the lock.) + ActivePS::SetMemoryCounter(mozilla::profiler::install_memory_hooks()); +#endif + + invoke_profiler_state_change_callbacks(ProfilingState::Started); + + // We do these operations with gPSMutex unlocked. The comments in + // profiler_stop() explain why. + if (samplerThread) { + Unused << ProfilerParent::ProfilerStopped(); + NotifyObservers("profiler-stopped"); + delete samplerThread; + } + return NotifyProfilerStarted(aCapacity, aDuration, aInterval, aFeatures, + aFilters, aFilterCount, aActiveTabID); +} + +void profiler_ensure_started(PowerOfTwo32 aCapacity, double aInterval, + uint32_t aFeatures, const char** aFilters, + uint32_t aFilterCount, uint64_t aActiveTabID, + const Maybe<double>& aDuration) { + LOG("profiler_ensure_started"); + + ProfilerParent::ProfilerWillStopIfStarted(); + + bool startedProfiler = false; + SamplerThread* samplerThread = nullptr; + { + PSAutoLock lock; + + // Initialize if necessary. + if (!CorePS::Exists()) { + profiler_init(nullptr); + } + + if (ActivePS::Exists(lock)) { + // The profiler is active. + if (!ActivePS::Equals(lock, aCapacity, aDuration, aInterval, aFeatures, + aFilters, aFilterCount, aActiveTabID)) { + // Stop and restart with different settings. + // Note: Not invoking callbacks with ProfilingState::Stopping, because + // we're under lock, and also it would not be useful: Any profiling data + // will be discarded, and we're immediately restarting the profiler + // below and then notifying ProfilingState::Started. + samplerThread = locked_profiler_stop(lock); + locked_profiler_start(lock, aCapacity, aInterval, aFeatures, aFilters, + aFilterCount, aActiveTabID, aDuration); + startedProfiler = true; + } + } else { + // The profiler is stopped. + locked_profiler_start(lock, aCapacity, aInterval, aFeatures, aFilters, + aFilterCount, aActiveTabID, aDuration); + startedProfiler = true; + } + } + + // We do these operations with gPSMutex unlocked. The comments in + // profiler_stop() explain why. + if (samplerThread) { + Unused << ProfilerParent::ProfilerStopped(); + NotifyObservers("profiler-stopped"); + delete samplerThread; + } + + if (startedProfiler) { + invoke_profiler_state_change_callbacks(ProfilingState::Started); + + Unused << NotifyProfilerStarted(aCapacity, aDuration, aInterval, aFeatures, + aFilters, aFilterCount, aActiveTabID); + } +} + +[[nodiscard]] static SamplerThread* locked_profiler_stop(PSLockRef aLock) { + LOG("locked_profiler_stop"); + + MOZ_RELEASE_ASSERT(CorePS::Exists() && ActivePS::Exists(aLock)); + + // At the very start, clear RacyFeatures. + RacyFeatures::SetInactive(); + + if (ActivePS::FeatureAudioCallbackTracing(aLock)) { + StopAudioCallbackTracing(); + } + +#if defined(GP_OS_android) + if (ActivePS::FeatureJava(aLock)) { + java::GeckoJavaSampler::Stop(); + } +#endif + + // Remove support for pushing/popping labels in mozglue. + RegisterProfilerLabelEnterExit(nullptr, nullptr); + + // Stop sampling live threads. + ThreadRegistry::LockedRegistry lockedRegistry; + for (ThreadRegistry::OffThreadRef offThreadRef : lockedRegistry) { + if (offThreadRef.UnlockedRWForLockedProfilerRef().ProfilingFeatures() == + ThreadProfilingFeatures::NotProfiled) { + continue; + } + + ThreadRegistry::OffThreadRef::RWFromAnyThreadWithLock lockedThreadData = + offThreadRef.GetLockedRWFromAnyThread(); + + lockedThreadData->ClearProfilingFeaturesAndData(aLock); + + if (ActivePS::FeatureJS(aLock)) { + lockedThreadData->StopJSSampling(); + if (ThreadRegistration::LockedRWOnThread* lockedRWOnThread = + lockedThreadData.GetLockedRWOnThread(); + lockedRWOnThread) { + // We are on the thread, we can manually poll the current thread so it + // stops profiling immediately. + lockedRWOnThread->PollJSSampling(); + } else if (lockedThreadData->Info().IsMainThread()) { + // Dispatch a runnable to the main thread to call PollJSSampling(), + // so that we don't have wait for the next JS interrupt callback in + // order to start profiling JS. + TriggerPollJSSamplingOnMainThread(); + } + } + } + +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + if (ActivePS::FeatureNativeAllocations(aLock)) { + mozilla::profiler::disable_native_allocations(); + } +#endif + + // The Stop() call doesn't actually stop Run(); that happens in this + // function's caller when the sampler thread is destroyed. Stop() just gives + // the SamplerThread a chance to do some cleanup with gPSMutex locked. + SamplerThread* samplerThread = ActivePS::Destroy(aLock); + samplerThread->Stop(aLock); + + if (NS_IsMainThread()) { + mozilla::base_profiler_markers_detail:: + ReleaseBufferForMainThreadAddMarker(); + } else { + NS_DispatchToMainThread( + NS_NewRunnableFunction("ReleaseBufferForMainThreadAddMarker", + &mozilla::base_profiler_markers_detail:: + ReleaseBufferForMainThreadAddMarker)); + } + + return samplerThread; +} + +RefPtr<GenericPromise> profiler_stop() { + LOG("profiler_stop"); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + if (profiler_is_active()) { + invoke_profiler_state_change_callbacks(ProfilingState::Stopping); + } + + ProfilerParent::ProfilerWillStopIfStarted(); + +#if defined(MOZ_REPLACE_MALLOC) && defined(MOZ_PROFILER_MEMORY) + // Remove the hooks early, as native allocations (if they are on) can be + // quite expensive. + mozilla::profiler::remove_memory_hooks(); +#endif + + SamplerThread* samplerThread; + { + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return GenericPromise::CreateAndResolve(/* unused */ true, __func__); + } + + samplerThread = locked_profiler_stop(lock); + } + + // We notify observers with gPSMutex unlocked. Otherwise we might get a + // deadlock, if code run by these functions calls a profiler function that + // locks gPSMutex, for example when it wants to insert a marker. + // (This has been seen in practise in bug 1346356, when we were still firing + // these notifications synchronously.) + RefPtr<GenericPromise> promise = ProfilerParent::ProfilerStopped(); + NotifyObservers("profiler-stopped"); + + // We delete with gPSMutex unlocked. Otherwise we would get a deadlock: we + // would be waiting here with gPSMutex locked for SamplerThread::Run() to + // return so the join operation within the destructor can complete, but Run() + // needs to lock gPSMutex to return. + // + // Because this call occurs with gPSMutex unlocked, it -- including the final + // iteration of Run()'s loop -- must be able detect deactivation and return + // in a way that's safe with respect to other gPSMutex-locking operations + // that may have occurred in the meantime. + delete samplerThread; + + return promise; +} + +bool profiler_is_paused() { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return false; + } + + return ActivePS::IsPaused(lock); +} + +/* [[nodiscard]] */ bool profiler_callback_after_sampling( + PostSamplingCallback&& aCallback) { + LOG("profiler_callback_after_sampling"); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + PSAutoLock lock; + + return ActivePS::AppendPostSamplingCallback(lock, std::move(aCallback)); +} + +RefPtr<GenericPromise> profiler_pause() { + LOG("profiler_pause"); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + invoke_profiler_state_change_callbacks(ProfilingState::Pausing); + + { + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return GenericPromise::CreateAndResolve(/* unused */ true, __func__); + } + +#if defined(GP_OS_android) + if (ActivePS::FeatureJava(lock) && !ActivePS::IsSamplingPaused(lock)) { + // Not paused yet, so this is the first pause, let Java know. + // TODO: Distinguish Pause and PauseSampling in Java. + java::GeckoJavaSampler::PauseSampling(); + } +#endif + + RacyFeatures::SetPaused(); + ActivePS::SetIsPaused(lock, true); + ActivePS::Buffer(lock).AddEntry(ProfileBufferEntry::Pause(profiler_time())); + } + + // gPSMutex must be unlocked when we notify, to avoid potential deadlocks. + RefPtr<GenericPromise> promise = ProfilerParent::ProfilerPaused(); + NotifyObservers("profiler-paused"); + return promise; +} + +RefPtr<GenericPromise> profiler_resume() { + LOG("profiler_resume"); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + { + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return GenericPromise::CreateAndResolve(/* unused */ true, __func__); + } + + ActivePS::Buffer(lock).AddEntry( + ProfileBufferEntry::Resume(profiler_time())); + ActivePS::SetIsPaused(lock, false); + RacyFeatures::SetUnpaused(); + +#if defined(GP_OS_android) + if (ActivePS::FeatureJava(lock) && !ActivePS::IsSamplingPaused(lock)) { + // Not paused anymore, so this is the last unpause, let Java know. + // TODO: Distinguish Unpause and UnpauseSampling in Java. + java::GeckoJavaSampler::UnpauseSampling(); + } +#endif + } + + // gPSMutex must be unlocked when we notify, to avoid potential deadlocks. + RefPtr<GenericPromise> promise = ProfilerParent::ProfilerResumed(); + NotifyObservers("profiler-resumed"); + + invoke_profiler_state_change_callbacks(ProfilingState::Resumed); + + return promise; +} + +bool profiler_is_sampling_paused() { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return false; + } + + return ActivePS::IsSamplingPaused(lock); +} + +RefPtr<GenericPromise> profiler_pause_sampling() { + LOG("profiler_pause_sampling"); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + { + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return GenericPromise::CreateAndResolve(/* unused */ true, __func__); + } + +#if defined(GP_OS_android) + if (ActivePS::FeatureJava(lock) && !ActivePS::IsSamplingPaused(lock)) { + // Not paused yet, so this is the first pause, let Java know. + // TODO: Distinguish Pause and PauseSampling in Java. + java::GeckoJavaSampler::PauseSampling(); + } +#endif + + RacyFeatures::SetSamplingPaused(); + ActivePS::SetIsSamplingPaused(lock, true); + ActivePS::Buffer(lock).AddEntry( + ProfileBufferEntry::PauseSampling(profiler_time())); + } + + // gPSMutex must be unlocked when we notify, to avoid potential deadlocks. + RefPtr<GenericPromise> promise = ProfilerParent::ProfilerPausedSampling(); + NotifyObservers("profiler-paused-sampling"); + return promise; +} + +RefPtr<GenericPromise> profiler_resume_sampling() { + LOG("profiler_resume_sampling"); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + { + PSAutoLock lock; + + if (!ActivePS::Exists(lock)) { + return GenericPromise::CreateAndResolve(/* unused */ true, __func__); + } + + ActivePS::Buffer(lock).AddEntry( + ProfileBufferEntry::ResumeSampling(profiler_time())); + ActivePS::SetIsSamplingPaused(lock, false); + RacyFeatures::SetSamplingUnpaused(); + +#if defined(GP_OS_android) + if (ActivePS::FeatureJava(lock) && !ActivePS::IsSamplingPaused(lock)) { + // Not paused anymore, so this is the last unpause, let Java know. + // TODO: Distinguish Unpause and UnpauseSampling in Java. + java::GeckoJavaSampler::UnpauseSampling(); + } +#endif + } + + // gPSMutex must be unlocked when we notify, to avoid potential deadlocks. + RefPtr<GenericPromise> promise = ProfilerParent::ProfilerResumedSampling(); + NotifyObservers("profiler-resumed-sampling"); + return promise; +} + +bool profiler_feature_active(uint32_t aFeature) { + // This function runs both on and off the main thread. + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + // This function is hot enough that we use RacyFeatures, not ActivePS. + return RacyFeatures::IsActiveWithFeature(aFeature); +} + +bool profiler_active_without_feature(uint32_t aFeature) { + // This function runs both on and off the main thread. + + // This function is hot enough that we use RacyFeatures, not ActivePS. + return RacyFeatures::IsActiveWithoutFeature(aFeature); +} + +void profiler_write_active_configuration(JSONWriter& aWriter) { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + PSAutoLock lock; + ActivePS::WriteActiveConfiguration(lock, aWriter); +} + +void profiler_add_sampled_counter(BaseProfilerCount* aCounter) { + DEBUG_LOG("profiler_add_sampled_counter(%s)", aCounter->mLabel); + PSAutoLock lock; + locked_profiler_add_sampled_counter(lock, aCounter); +} + +void profiler_remove_sampled_counter(BaseProfilerCount* aCounter) { + DEBUG_LOG("profiler_remove_sampled_counter(%s)", aCounter->mLabel); + PSAutoLock lock; + locked_profiler_remove_sampled_counter(lock, aCounter); +} + +void profiler_count_bandwidth_bytes(int64_t aCount) { + NS_ASSERTION(profiler_feature_active(ProfilerFeature::Bandwidth), + "Should not call profiler_count_bandwidth_bytes when the " + "Bandwidth feature is not set"); + + ProfilerBandwidthCounter* counter = CorePS::GetBandwidthCounter(); + if (MOZ_UNLIKELY(!counter)) { + counter = new ProfilerBandwidthCounter(); + CorePS::SetBandwidthCounter(counter); + } + + counter->Add(aCount); +} + +ProfilingStack* profiler_register_thread(const char* aName, + void* aGuessStackTop) { + DEBUG_LOG("profiler_register_thread(%s)", aName); + + // This will call `ThreadRegistry::Register()` (see below). + return ThreadRegistration::RegisterThread(aName, aGuessStackTop); +} + +/* static */ +void ThreadRegistry::Register(ThreadRegistration::OnThreadRef aOnThreadRef) { + // Set the thread name (except for the main thread, which is controlled + // elsewhere, and influences the process name on some systems like Linux). + if (!aOnThreadRef.UnlockedConstReaderCRef().Info().IsMainThread()) { + // Make sure we have a nsThread wrapper for the current thread, and that + // NSPR knows its name. + (void)NS_GetCurrentThread(); + NS_SetCurrentThreadName( + aOnThreadRef.UnlockedConstReaderCRef().Info().Name()); + } + + PSAutoLock lock; + + { + RegistryLockExclusive lock{sRegistryMutex}; + MOZ_RELEASE_ASSERT(sRegistryContainer.append(OffThreadRef{aOnThreadRef})); + } + + if (!CorePS::Exists()) { + // CorePS has not been created yet. + // If&when that happens, it will handle already-registered threads then. + return; + } + + (void)locked_register_thread(lock, OffThreadRef{aOnThreadRef}); +} + +void profiler_unregister_thread() { + // This will call `ThreadRegistry::Unregister()` (see below). + ThreadRegistration::UnregisterThread(); +} + +static void locked_unregister_thread( + PSLockRef lock, ThreadRegistration::OnThreadRef aOnThreadRef) { + if (!CorePS::Exists()) { + // This function can be called after the main thread has already shut + // down. + return; + } + + // We don't call StopJSSampling() here; there's no point doing that for a JS + // thread that is in the process of disappearing. + + ThreadRegistration::OnThreadRef::RWOnThreadWithLock lockedThreadData = + aOnThreadRef.GetLockedRWOnThread(); + + ProfiledThreadData* profiledThreadData = + lockedThreadData->GetProfiledThreadData(lock); + lockedThreadData->ClearProfilingFeaturesAndData(lock); + + MOZ_RELEASE_ASSERT( + lockedThreadData->Info().ThreadId() == profiler_current_thread_id(), + "Thread being unregistered has changed its TID"); + + DEBUG_LOG("profiler_unregister_thread: %s", lockedThreadData->Info().Name()); + + if (profiledThreadData && ActivePS::Exists(lock)) { + ActivePS::UnregisterThread(lock, profiledThreadData); + } +} + +/* static */ +void ThreadRegistry::Unregister(ThreadRegistration::OnThreadRef aOnThreadRef) { + PSAutoLock psLock; + locked_unregister_thread(psLock, aOnThreadRef); + + RegistryLockExclusive lock{sRegistryMutex}; + for (OffThreadRef& thread : sRegistryContainer) { + if (thread.IsPointingAt(*aOnThreadRef.mThreadRegistration)) { + sRegistryContainer.erase(&thread); + break; + } + } +} + +void profiler_register_page(uint64_t aTabID, uint64_t aInnerWindowID, + const nsCString& aUrl, + uint64_t aEmbedderInnerWindowID, + bool aIsPrivateBrowsing) { + DEBUG_LOG("profiler_register_page(%" PRIu64 ", %" PRIu64 ", %s, %" PRIu64 + ", %s)", + aTabID, aInnerWindowID, aUrl.get(), aEmbedderInnerWindowID, + aIsPrivateBrowsing ? "true" : "false"); + + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + PSAutoLock lock; + + // When a Browsing context is first loaded, the first url loaded in it will be + // about:blank. Because of that, this call keeps the first non-about:blank + // registration of window and discards the previous one. + RefPtr<PageInformation> pageInfo = new PageInformation( + aTabID, aInnerWindowID, aUrl, aEmbedderInnerWindowID, aIsPrivateBrowsing); + CorePS::AppendRegisteredPage(lock, std::move(pageInfo)); + + // After appending the given page to CorePS, look for the expired + // pages and remove them if there are any. + if (ActivePS::Exists(lock)) { + ActivePS::DiscardExpiredPages(lock); + } +} + +void profiler_unregister_page(uint64_t aRegisteredInnerWindowID) { + PSAutoLock lock; + + if (!CorePS::Exists()) { + // This function can be called after the main thread has already shut down. + return; + } + + // During unregistration, if the profiler is active, we have to keep the + // page information since there may be some markers associated with the given + // page. But if profiler is not active. we have no reason to keep the + // page information here because there can't be any marker associated with it. + if (ActivePS::Exists(lock)) { + ActivePS::UnregisterPage(lock, aRegisteredInnerWindowID); + } else { + CorePS::RemoveRegisteredPage(lock, aRegisteredInnerWindowID); + } +} + +void profiler_clear_all_pages() { + { + PSAutoLock lock; + + if (!CorePS::Exists()) { + // This function can be called after the main thread has already shut + // down. + return; + } + + CorePS::ClearRegisteredPages(lock); + if (ActivePS::Exists(lock)) { + ActivePS::ClearUnregisteredPages(lock); + } + } + + // gPSMutex must be unlocked when we notify, to avoid potential deadlocks. + ProfilerParent::ClearAllPages(); +} + +namespace geckoprofiler::markers::detail { + +Maybe<uint64_t> profiler_get_inner_window_id_from_docshell( + nsIDocShell* aDocshell) { + Maybe<uint64_t> innerWindowID = Nothing(); + if (aDocshell) { + auto outerWindow = aDocshell->GetWindow(); + if (outerWindow) { + auto innerWindow = outerWindow->GetCurrentInnerWindow(); + if (innerWindow) { + innerWindowID = Some(innerWindow->WindowID()); + } + } + } + return innerWindowID; +} + +} // namespace geckoprofiler::markers::detail + +namespace geckoprofiler::markers { + +struct CPUAwakeMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("Awake"); + } + static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter, + int64_t aCPUTimeNs, int64_t aCPUId +#ifdef GP_OS_darwin + , + uint32_t aQoS +#endif +#ifdef GP_OS_windows + , + int32_t aAbsolutePriority, + int32_t aRelativePriority, + int32_t aCurrentPriority +#endif + ) { + if (aCPUTimeNs) { + constexpr double NS_PER_MS = 1'000'000; + aWriter.DoubleProperty("CPU Time", double(aCPUTimeNs) / NS_PER_MS); + // CPU Time is only provided for the end marker, the other fields are for + // the start marker. + return; + } + +#ifndef GP_PLAT_arm64_darwin + aWriter.IntProperty("CPU Id", aCPUId); +#endif +#ifdef GP_OS_windows + if (aAbsolutePriority) { + aWriter.IntProperty("absPriority", aAbsolutePriority); + } + if (aCurrentPriority) { + aWriter.IntProperty("curPriority", aCurrentPriority); + } + aWriter.IntProperty("priority", aRelativePriority); +#endif +#ifdef GP_OS_darwin + const char* QoS = ""; + switch (aQoS) { + case QOS_CLASS_USER_INTERACTIVE: + QoS = "User Interactive"; + break; + case QOS_CLASS_USER_INITIATED: + QoS = "User Initiated"; + break; + case QOS_CLASS_DEFAULT: + QoS = "Default"; + break; + case QOS_CLASS_UTILITY: + QoS = "Utility"; + break; + case QOS_CLASS_BACKGROUND: + QoS = "Background"; + break; + default: + QoS = "Unspecified"; + } + + aWriter.StringProperty("QoS", + ProfilerString8View::WrapNullTerminatedString(QoS)); +#endif + } + + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable}; + schema.AddKeyFormat("CPU Time", MS::Format::Duration); +#ifndef GP_PLAT_arm64_darwin + schema.AddKeyFormat("CPU Id", MS::Format::Integer); + schema.SetTableLabel("Awake - CPU Id = {marker.data.CPU Id}"); +#endif +#ifdef GP_OS_windows + schema.AddKeyLabelFormat("priority", "Relative Thread Priority", + MS::Format::Integer); + schema.AddKeyLabelFormat("absPriority", "Base Thread Priority", + MS::Format::Integer); + schema.AddKeyLabelFormat("curPriority", "Current Thread Priority", + MS::Format::Integer); +#endif +#ifdef GP_OS_darwin + schema.AddKeyLabelFormat("QoS", "Quality of Service", MS::Format::String); +#endif + return schema; + } +}; + +} // namespace geckoprofiler::markers + +void profiler_mark_thread_asleep() { + if (!profiler_thread_is_being_profiled_for_markers()) { + return; + } + + uint64_t cpuTimeNs = ThreadRegistration::WithOnThreadRefOr( + [](ThreadRegistration::OnThreadRef aOnThreadRef) { + return aOnThreadRef.UnlockedConstReaderAndAtomicRWRef() + .GetNewCpuTimeInNs(); + }, + 0); + PROFILER_MARKER("Awake", OTHER, MarkerTiming::IntervalEnd(), CPUAwakeMarker, + cpuTimeNs, 0 /* cpuId */ +#if defined(GP_OS_darwin) + , + 0 /* qos_class */ +#endif +#if defined(GP_OS_windows) + , + 0 /* priority */, 0 /* thread priority */, + 0 /* current priority */ +#endif + ); +} + +void profiler_thread_sleep() { + profiler_mark_thread_asleep(); + ThreadRegistration::WithOnThreadRef( + [](ThreadRegistration::OnThreadRef aOnThreadRef) { + aOnThreadRef.UnlockedConstReaderAndAtomicRWRef().SetSleeping(); + }); +} + +#if defined(GP_OS_windows) +# if !defined(__MINGW32__) +enum { + ThreadBasicInformation, +}; +# endif + +struct THREAD_BASIC_INFORMATION { + NTSTATUS ExitStatus; + PVOID TebBaseAddress; + CLIENT_ID ClientId; + KAFFINITY AffMask; + DWORD Priority; + DWORD BasePriority; +}; +#endif + +static mozilla::Atomic<uint64_t, mozilla::MemoryOrdering::Relaxed> gWakeCount( + 0); + +namespace geckoprofiler::markers { +struct WakeUpCountMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("WakeUpCount"); + } + static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter, + int32_t aCount, + const ProfilerString8View& aType) { + aWriter.IntProperty("Count", aCount); + aWriter.StringProperty("label", aType); + } + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable}; + schema.AddKeyFormat("Count", MS::Format::Integer); + schema.SetTooltipLabel("{marker.name} - {marker.data.label}"); + schema.SetTableLabel( + "{marker.name} - {marker.data.label}: {marker.data.count}"); + return schema; + } +}; +} // namespace geckoprofiler::markers + +void profiler_record_wakeup_count(const nsACString& aProcessType) { + static uint64_t previousThreadWakeCount = 0; + + uint64_t newWakeups = gWakeCount - previousThreadWakeCount; + if (newWakeups > 0) { + if (newWakeups < std::numeric_limits<int32_t>::max()) { + int32_t newWakeups32 = int32_t(newWakeups); + mozilla::glean::power::total_thread_wakeups.Add(newWakeups32); + mozilla::glean::power::wakeups_per_process_type.Get(aProcessType) + .Add(newWakeups32); + PROFILER_MARKER("Thread Wake-ups", OTHER, {}, WakeUpCountMarker, + newWakeups32, aProcessType); + } + + previousThreadWakeCount += newWakeups; + } + +#ifdef NIGHTLY_BUILD + ThreadRegistry::LockedRegistry lockedRegistry; + for (ThreadRegistry::OffThreadRef offThreadRef : lockedRegistry) { + const ThreadRegistry::UnlockedConstReaderAndAtomicRW& threadData = + offThreadRef.UnlockedConstReaderAndAtomicRWRef(); + threadData.RecordWakeCount(); + } +#endif +} + +void profiler_mark_thread_awake() { + ++gWakeCount; + if (!profiler_thread_is_being_profiled_for_markers()) { + return; + } + + int64_t cpuId = 0; +#if defined(GP_OS_windows) + cpuId = GetCurrentProcessorNumber(); +#elif defined(GP_OS_darwin) +# ifdef GP_PLAT_amd64_darwin + unsigned int eax, ebx, ecx, edx; + __cpuid_count(1, 0, eax, ebx, ecx, edx); + // Check if we have an APIC. + if ((edx & (1 << 9))) { + // APIC ID is bits 24-31 of EBX + cpuId = ebx >> 24; + } +# endif +#else + cpuId = sched_getcpu(); +#endif + +#if defined(GP_OS_windows) + LONG priority; + static const auto get_thread_information_fn = + reinterpret_cast<decltype(&::GetThreadInformation)>(::GetProcAddress( + ::GetModuleHandle(L"Kernel32.dll"), "GetThreadInformation")); + + if (!get_thread_information_fn || + !get_thread_information_fn(GetCurrentThread(), ThreadAbsoluteCpuPriority, + &priority, sizeof(priority))) { + priority = 0; + } + + static const auto nt_query_information_thread_fn = + reinterpret_cast<decltype(&::NtQueryInformationThread)>(::GetProcAddress( + ::GetModuleHandle(L"ntdll.dll"), "NtQueryInformationThread")); + + LONG currentPriority = 0; + if (nt_query_information_thread_fn) { + THREAD_BASIC_INFORMATION threadInfo; + auto status = (*nt_query_information_thread_fn)( + GetCurrentThread(), (THREADINFOCLASS)ThreadBasicInformation, + &threadInfo, sizeof(threadInfo), NULL); + if (NT_SUCCESS(status)) { + currentPriority = threadInfo.Priority; + } + } +#endif + PROFILER_MARKER("Awake", OTHER, MarkerTiming::IntervalStart(), CPUAwakeMarker, + 0 /* CPU time */, cpuId +#if defined(GP_OS_darwin) + , + qos_class_self() +#endif +#if defined(GP_OS_windows) + , + priority, GetThreadPriority(GetCurrentThread()), + currentPriority +#endif + ); +} + +void profiler_thread_wake() { + profiler_mark_thread_awake(); + ThreadRegistration::WithOnThreadRef( + [](ThreadRegistration::OnThreadRef aOnThreadRef) { + aOnThreadRef.UnlockedConstReaderAndAtomicRWRef().SetAwake(); + }); +} + +void profiler_js_interrupt_callback() { + // This function runs on JS threads being sampled. + PollJSSamplingForCurrentThread(); +} + +double profiler_time() { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + TimeDuration delta = TimeStamp::Now() - CorePS::ProcessStartTime(); + return delta.ToMilliseconds(); +} + +bool profiler_capture_backtrace_into(ProfileChunkedBuffer& aChunkedBuffer, + StackCaptureOptions aCaptureOptions) { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + if (!profiler_is_active() || + aCaptureOptions == StackCaptureOptions::NoStack) { + return false; + } + + return ThreadRegistration::WithOnThreadRefOr( + [&](ThreadRegistration::OnThreadRef aOnThreadRef) { + mozilla::Maybe<uint32_t> maybeFeatures = + RacyFeatures::FeaturesIfActiveAndUnpaused(); + if (!maybeFeatures) { + return false; + } + + ProfileBuffer profileBuffer(aChunkedBuffer); + + Registers regs; +#if defined(HAVE_NATIVE_UNWIND) + REGISTERS_SYNC_POPULATE(regs); +#else + regs.Clear(); +#endif + + DoSyncSample(*maybeFeatures, + aOnThreadRef.UnlockedReaderAndAtomicRWOnThreadCRef(), + TimeStamp::Now(), regs, profileBuffer, aCaptureOptions); + + return true; + }, + // If this was called from a non-registered thread, return false and do no + // more work. This can happen from a memory hook. + false); +} + +UniquePtr<ProfileChunkedBuffer> profiler_capture_backtrace() { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + AUTO_PROFILER_LABEL_HOT("profiler_capture_backtrace", PROFILER); + + // Quick is-active and feature check before allocating a buffer. + // If NoMarkerStacks is set, we don't want to capture a backtrace. + if (!profiler_active_without_feature(ProfilerFeature::NoMarkerStacks)) { + return nullptr; + } + + auto buffer = MakeUnique<ProfileChunkedBuffer>( + ProfileChunkedBuffer::ThreadSafety::WithoutMutex, + MakeUnique<ProfileBufferChunkManagerSingle>( + ProfileBufferChunkManager::scExpectedMaximumStackSize)); + + if (!profiler_capture_backtrace_into(*buffer, StackCaptureOptions::Full)) { + return nullptr; + } + + return buffer; +} + +UniqueProfilerBacktrace profiler_get_backtrace() { + UniquePtr<ProfileChunkedBuffer> buffer = profiler_capture_backtrace(); + + if (!buffer) { + return nullptr; + } + + return UniqueProfilerBacktrace( + new ProfilerBacktrace("SyncProfile", std::move(buffer))); +} + +void ProfilerBacktraceDestructor::operator()(ProfilerBacktrace* aBacktrace) { + delete aBacktrace; +} + +bool profiler_is_locked_on_current_thread() { + // This function is used to help users avoid calling `profiler_...` functions + // when the profiler may already have a lock in place, which would prevent a + // 2nd recursive lock (resulting in a crash or a never-ending wait), or a + // deadlock between any two mutexes. So we must return `true` for any of: + // - The main profiler mutex, used by most functions, and/or + // - The buffer mutex, used directly in some functions without locking the + // main mutex, e.g., marker-related functions. + // - The ProfilerParent or ProfilerChild mutex, used to store and process + // buffer chunk updates. + return PSAutoLock::IsLockedOnCurrentThread() || + ThreadRegistry::IsRegistryMutexLockedOnCurrentThread() || + ThreadRegistration::IsDataMutexLockedOnCurrentThread() || + profiler_get_core_buffer().IsThreadSafeAndLockedOnCurrentThread() || + ProfilerParent::IsLockedOnCurrentThread() || + ProfilerChild::IsLockedOnCurrentThread(); +} + +void profiler_set_js_context(JSContext* aCx) { + MOZ_ASSERT(aCx); + ThreadRegistration::WithOnThreadRef( + [&](ThreadRegistration::OnThreadRef aOnThreadRef) { + // The profiler mutex must be locked before the ThreadRegistration's. + PSAutoLock lock; + aOnThreadRef.WithLockedRWOnThread( + [&](ThreadRegistration::LockedRWOnThread& aThreadData) { + aThreadData.SetJSContext(aCx); + + if (!ActivePS::Exists(lock) || !ActivePS::FeatureJS(lock)) { + return; + } + + // This call is on-thread, so we can call PollJSSampling() to + // start JS sampling immediately. + aThreadData.PollJSSampling(); + + if (ProfiledThreadData* profiledThreadData = + aThreadData.GetProfiledThreadData(lock); + profiledThreadData) { + profiledThreadData->NotifyReceivedJSContext( + ActivePS::Buffer(lock).BufferRangeEnd()); + } + }); + }); +} + +void profiler_clear_js_context() { + MOZ_RELEASE_ASSERT(CorePS::Exists()); + + ThreadRegistration::WithOnThreadRef( + [](ThreadRegistration::OnThreadRef aOnThreadRef) { + JSContext* cx = + aOnThreadRef.UnlockedReaderAndAtomicRWOnThreadCRef().GetJSContext(); + if (!cx) { + return; + } + + // The profiler mutex must be locked before the ThreadRegistration's. + PSAutoLock lock; + ThreadRegistration::OnThreadRef::RWOnThreadWithLock lockedThreadData = + aOnThreadRef.GetLockedRWOnThread(); + + if (ProfiledThreadData* profiledThreadData = + lockedThreadData->GetProfiledThreadData(lock); + profiledThreadData && ActivePS::Exists(lock) && + ActivePS::FeatureJS(lock)) { + profiledThreadData->NotifyAboutToLoseJSContext( + cx, CorePS::ProcessStartTime(), ActivePS::Buffer(lock)); + + // Notify the JS context that profiling for this context has + // stopped. Do this by calling StopJSSampling and PollJSSampling + // before nulling out the JSContext. + lockedThreadData->StopJSSampling(); + lockedThreadData->PollJSSampling(); + + lockedThreadData->ClearJSContext(); + + // Tell the thread that we'd like to have JS sampling on this + // thread again, once it gets a new JSContext (if ever). + lockedThreadData->StartJSSampling(ActivePS::JSFlags(lock)); + } else { + // This thread is not being profiled or JS profiling is off, we only + // need to clear the context pointer. + lockedThreadData->ClearJSContext(); + } + }); +} + +static void profiler_suspend_and_sample_thread( + const PSAutoLock* aLockIfAsynchronousSampling, + const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aThreadData, + JsFrame* aJsFrames, uint32_t aFeatures, ProfilerStackCollector& aCollector, + bool aSampleNative) { + const ThreadRegistrationInfo& info = aThreadData.Info(); + + if (info.IsMainThread()) { + aCollector.SetIsMainThread(); + } + + // Allocate the space for the native stack + NativeStack nativeStack; + + auto collectStack = [&](const Registers& aRegs, const TimeStamp& aNow) { + // The target thread is now suspended. Collect a native backtrace, + // and call the callback. + StackWalkControl* stackWalkControlIfSupported = nullptr; +#if defined(HAVE_FASTINIT_NATIVE_UNWIND) + StackWalkControl stackWalkControl; + if constexpr (StackWalkControl::scIsSupported) { + if (aSampleNative) { + stackWalkControlIfSupported = &stackWalkControl; + } + } +#endif + const uint32_t jsFramesCount = + aJsFrames ? ExtractJsFrames(!aLockIfAsynchronousSampling, aThreadData, + aRegs, aCollector, aJsFrames, + stackWalkControlIfSupported) + : 0; + +#if defined(HAVE_FASTINIT_NATIVE_UNWIND) + if (aSampleNative) { + // We can only use FramePointerStackWalk or MozStackWalk from + // suspend_and_sample_thread as other stackwalking methods may not be + // initialized. +# if defined(USE_FRAME_POINTER_STACK_WALK) + DoFramePointerBacktrace(aThreadData, aRegs, nativeStack, + stackWalkControlIfSupported); +# elif defined(USE_MOZ_STACK_WALK) + DoMozStackWalkBacktrace(aThreadData, aRegs, nativeStack, + stackWalkControlIfSupported); +# else +# error "Invalid configuration" +# endif + + MergeStacks(aFeatures, !aLockIfAsynchronousSampling, aThreadData, aRegs, + nativeStack, aCollector, aJsFrames, jsFramesCount); + } else +#endif + { + MergeStacks(aFeatures, !aLockIfAsynchronousSampling, aThreadData, aRegs, + nativeStack, aCollector, aJsFrames, jsFramesCount); + + aCollector.CollectNativeLeafAddr((void*)aRegs.mPC); + } + }; + + if (!aLockIfAsynchronousSampling) { + // Sampling the current thread, do NOT suspend it! + Registers regs; +#if defined(HAVE_NATIVE_UNWIND) + REGISTERS_SYNC_POPULATE(regs); +#else + regs.Clear(); +#endif + collectStack(regs, TimeStamp::Now()); + } else { + // Suspend, sample, and then resume the target thread. + Sampler sampler(*aLockIfAsynchronousSampling); + TimeStamp now = TimeStamp::Now(); + sampler.SuspendAndSampleAndResumeThread(*aLockIfAsynchronousSampling, + aThreadData, now, collectStack); + + // NOTE: Make sure to disable the sampler before it is destroyed, in + // case the profiler is running at the same time. + sampler.Disable(*aLockIfAsynchronousSampling); + } +} + +// NOTE: aCollector's methods will be called while the target thread is paused. +// Doing things in those methods like allocating -- which may try to claim +// locks -- is a surefire way to deadlock. +void profiler_suspend_and_sample_thread(ProfilerThreadId aThreadId, + uint32_t aFeatures, + ProfilerStackCollector& aCollector, + bool aSampleNative /* = true */) { + if (!aThreadId.IsSpecified() || aThreadId == profiler_current_thread_id()) { + // Sampling the current thread. Get its information from the TLS (no locking + // required.) + ThreadRegistration::WithOnThreadRef( + [&](ThreadRegistration::OnThreadRef aOnThreadRef) { + aOnThreadRef.WithUnlockedReaderAndAtomicRWOnThread( + [&](const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& + aThreadData) { + if (!aThreadData.GetJSContext()) { + // No JSContext, there is no JS frame buffer (and no need for + // it). + profiler_suspend_and_sample_thread( + /* aLockIfAsynchronousSampling = */ nullptr, aThreadData, + /* aJsFrames = */ nullptr, aFeatures, aCollector, + aSampleNative); + } else { + // JSContext is present, we need to lock the thread data to + // access the JS frame buffer. + aOnThreadRef.WithConstLockedRWOnThread( + [&](const ThreadRegistration::LockedRWOnThread& + aLockedThreadData) { + profiler_suspend_and_sample_thread( + /* aLockIfAsynchronousSampling = */ nullptr, + aThreadData, aLockedThreadData.GetJsFrameBuffer(), + aFeatures, aCollector, aSampleNative); + }); + } + }); + }); + } else { + // Lock the profiler before accessing the ThreadRegistry. + PSAutoLock lock; + ThreadRegistry::WithOffThreadRef( + aThreadId, [&](ThreadRegistry::OffThreadRef aOffThreadRef) { + aOffThreadRef.WithLockedRWFromAnyThread( + [&](const ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& + aThreadData) { + JsFrameBuffer& jsFrames = CorePS::JsFrames(lock); + profiler_suspend_and_sample_thread(&lock, aThreadData, jsFrames, + aFeatures, aCollector, + aSampleNative); + }); + }); + } +} + +// END externally visible functions +//////////////////////////////////////////////////////////////////////// diff --git a/tools/profiler/core/platform.h b/tools/profiler/core/platform.h new file mode 100644 index 0000000000..59d2c7ff42 --- /dev/null +++ b/tools/profiler/core/platform.h @@ -0,0 +1,381 @@ +// Copyright (c) 2006-2011 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in +// the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google, Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +// AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +// SUCH DAMAGE. + +#ifndef TOOLS_PLATFORM_H_ +#define TOOLS_PLATFORM_H_ + +#include "PlatformMacros.h" + +#include "json/json.h" +#include "mozilla/Atomics.h" +#include "mozilla/BaseProfilerDetail.h" +#include "mozilla/Logging.h" +#include "mozilla/MathAlgorithms.h" +#include "mozilla/ProfileBufferEntrySerialization.h" +#include "mozilla/ProfileJSONWriter.h" +#include "mozilla/ProfilerUtils.h" +#include "mozilla/ProgressLogger.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Vector.h" +#include "nsString.h" +#include "shared-libraries.h" + +#include <cstddef> +#include <cstdint> +#include <functional> + +class ProfilerCodeAddressService; + +namespace mozilla { +struct SymbolTable; +} + +extern mozilla::LazyLogModule gProfilerLog; + +// These are for MOZ_LOG="prof:3" or higher. It's the default logging level for +// the profiler, and should be used sparingly. +#define LOG_TEST MOZ_LOG_TEST(gProfilerLog, mozilla::LogLevel::Info) +#define LOG(arg, ...) \ + MOZ_LOG(gProfilerLog, mozilla::LogLevel::Info, \ + ("[%" PRIu64 "] " arg, \ + uint64_t(profiler_current_process_id().ToNumber()), ##__VA_ARGS__)) + +// These are for MOZ_LOG="prof:4" or higher. It should be used for logging that +// is somewhat more verbose than LOG. +#define DEBUG_LOG_TEST MOZ_LOG_TEST(gProfilerLog, mozilla::LogLevel::Debug) +#define DEBUG_LOG(arg, ...) \ + MOZ_LOG(gProfilerLog, mozilla::LogLevel::Debug, \ + ("[%" PRIu64 "] " arg, \ + uint64_t(profiler_current_process_id().ToNumber()), ##__VA_ARGS__)) + +typedef uint8_t* Address; + +// Stringify the given JSON value, in the most compact format. +// Note: Numbers are limited to a precision of 6 decimal digits, so that +// timestamps in ms have a precision in ns. +Json::String ToCompactString(const Json::Value& aJsonValue); + +// Profiling log stored in a Json::Value. The actual log only exists while the +// profiler is running, and will be inserted at the end of the JSON profile. +class ProfilingLog { + public: + // These will be called by ActivePS when the profiler starts/stops. + static void Init(); + static void Destroy(); + + // Access the profiling log JSON object, in order to modify it. + // Only calls the given function if the profiler is active. + // Thread-safe. But `aF` must not call other locking profiler functions. + // This is intended to capture some internal logging that doesn't belong in + // other places like markers. The log is accessible through the JS console on + // profiler.firefox.com, in the `profile.profilingLog` object; the data format + // is intentionally not defined, and not intended to be shown in the + // front-end. + // Please use caution not to output too much data. + template <typename F> + static void Access(F&& aF) { + mozilla::baseprofiler::detail::BaseProfilerAutoLock lock{gMutex}; + if (gLog) { + std::forward<F>(aF)(*gLog); + } + } + +#define DURATION_JSON_SUFFIX "_ms" + + // Convert a TimeDuration to the value to be stored in the log. + // Use DURATION_JSON_SUFFIX as suffix in the property name. + static Json::Value Duration(const mozilla::TimeDuration& aDuration) { + return Json::Value{aDuration.ToMilliseconds()}; + } + +#define TIMESTAMP_JSON_SUFFIX "_TSms" + + // Convert a TimeStamp to the value to be stored in the log. + // Use TIMESTAMP_JSON_SUFFIX as suffix in the property name. + static Json::Value Timestamp( + const mozilla::TimeStamp& aTimestamp = mozilla::TimeStamp::Now()) { + if (aTimestamp.IsNull()) { + return Json::Value{0.0}; + } + return Duration(aTimestamp - mozilla::TimeStamp::ProcessCreation()); + } + + static bool IsLockedOnCurrentThread(); + + private: + static mozilla::baseprofiler::detail::BaseProfilerMutex gMutex; + static mozilla::UniquePtr<Json::Value> gLog; +}; + +// ---------------------------------------------------------------------------- +// Miscellaneous + +// If positive, skip stack-sampling in the sampler thread loop. +// Users should increment it atomically when samplings should be avoided, and +// later decrement it back. Multiple uses can overlap. +// There could be a sampling in progress when this is first incremented, so if +// it is critical to prevent any sampling, lock the profiler mutex instead. +// Relaxed ordering, because it's used to request that the profiler pause +// future sampling; this is not time critical, nor dependent on anything else. +extern mozilla::Atomic<int, mozilla::MemoryOrdering::Relaxed> gSkipSampling; + +void AppendSharedLibraries(mozilla::JSONWriter& aWriter, + const SharedLibraryInfo& aInfo); + +// Convert the array of strings to a bitfield. +uint32_t ParseFeaturesFromStringArray(const char** aFeatures, + uint32_t aFeatureCount, + bool aIsStartup = false); + +// Add the begin/end 'Awake' markers for the thread. +void profiler_mark_thread_awake(); + +void profiler_mark_thread_asleep(); + +[[nodiscard]] bool profiler_get_profile_json( + SpliceableChunkedJSONWriter& aSpliceableChunkedJSONWriter, + double aSinceTime, bool aIsShuttingDown, + mozilla::ProgressLogger aProgressLogger); + +// Flags to conveniently track various JS instrumentations. +enum class JSInstrumentationFlags { + StackSampling = 0x1, + Allocations = 0x2, +}; + +// Write out the information of the active profiling configuration. +void profiler_write_active_configuration(mozilla::JSONWriter& aWriter); + +// Extract all received exit profiles that have not yet expired (i.e., they +// still intersect with this process' buffer range). +mozilla::Vector<nsCString> profiler_move_exit_profiles(); + +// If the "MOZ_PROFILER_SYMBOLICATE" env-var is set, we return a new +// ProfilerCodeAddressService object to use for local symbolication of profiles. +// This is off by default, and mainly intended for local development. +mozilla::UniquePtr<ProfilerCodeAddressService> +profiler_code_address_service_for_presymbolication(); + +extern "C" { +// This function is defined in the profiler rust module at +// tools/profiler/rust-helper. mozilla::SymbolTable and CompactSymbolTable +// have identical memory layout. +bool profiler_get_symbol_table(const char* debug_path, const char* breakpad_id, + mozilla::SymbolTable* symbol_table); + +bool profiler_demangle_rust(const char* mangled, char* buffer, size_t len); +} + +// For each running times value, call MACRO(index, name, unit, jsonProperty) +#define PROFILER_FOR_EACH_RUNNING_TIME(MACRO) \ + MACRO(0, ThreadCPU, Delta, threadCPUDelta) + +// This class contains all "running times" such as CPU usage measurements. +// All measurements are listed in `PROFILER_FOR_EACH_RUNNING_TIME` above. +// Each measurement is optional and only takes a value when explicitly set. +// Two RunningTimes object may be subtracted, to get the difference between +// known values. +class RunningTimes { + public: + constexpr RunningTimes() = default; + + // Constructor with only a timestamp, useful when no measurements will be + // taken. + constexpr explicit RunningTimes(const mozilla::TimeStamp& aTimeStamp) + : mPostMeasurementTimeStamp(aTimeStamp) {} + + constexpr void Clear() { *this = RunningTimes{}; } + + constexpr bool IsEmpty() const { return mKnownBits == 0; } + + // This should be called right after CPU measurements have been taken. + void SetPostMeasurementTimeStamp(const mozilla::TimeStamp& aTimeStamp) { + mPostMeasurementTimeStamp = aTimeStamp; + } + + const mozilla::TimeStamp& PostMeasurementTimeStamp() const { + return mPostMeasurementTimeStamp; + } + + // Should be filled for any registered thread. + +#define RUNNING_TIME_MEMBER(index, name, unit, jsonProperty) \ + constexpr bool Is##name##unit##Known() const { \ + return (mKnownBits & mGot##name##unit) != 0; \ + } \ + \ + constexpr void Clear##name##unit() { \ + m##name##unit = 0; \ + mKnownBits &= ~mGot##name##unit; \ + } \ + \ + constexpr void Reset##name##unit(uint64_t a##name##unit) { \ + m##name##unit = a##name##unit; \ + mKnownBits |= mGot##name##unit; \ + } \ + \ + constexpr void Set##name##unit(uint64_t a##name##unit) { \ + MOZ_ASSERT(!Is##name##unit##Known(), #name #unit " already set"); \ + Reset##name##unit(a##name##unit); \ + } \ + \ + constexpr mozilla::Maybe<uint64_t> Get##name##unit() const { \ + if (Is##name##unit##Known()) { \ + return mozilla::Some(m##name##unit); \ + } \ + return mozilla::Nothing{}; \ + } \ + \ + constexpr mozilla::Maybe<uint64_t> GetJson##name##unit() const { \ + if (Is##name##unit##Known()) { \ + return mozilla::Some(ConvertRawToJson(m##name##unit)); \ + } \ + return mozilla::Nothing{}; \ + } + + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_MEMBER) + +#undef RUNNING_TIME_MEMBER + + // Take values from another RunningTimes. + RunningTimes& TakeFrom(RunningTimes& aOther) { + if (!aOther.IsEmpty()) { +#define RUNNING_TIME_TAKE(index, name, unit, jsonProperty) \ + if (aOther.Is##name##unit##Known()) { \ + Set##name##unit(std::exchange(aOther.m##name##unit, 0)); \ + } + + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_TAKE) + +#undef RUNNING_TIME_TAKE + + aOther.mKnownBits = 0; + } + return *this; + } + + // Difference from `aBefore` to `this`. Any unknown makes the result unknown. + // PostMeasurementTimeStamp set to `this` PostMeasurementTimeStamp, to keep + // the most recent timestamp associated with the end of the interval over + // which the difference applies. + RunningTimes operator-(const RunningTimes& aBefore) const { + RunningTimes diff; + diff.mPostMeasurementTimeStamp = mPostMeasurementTimeStamp; +#define RUNNING_TIME_SUB(index, name, unit, jsonProperty) \ + if (Is##name##unit##Known() && aBefore.Is##name##unit##Known()) { \ + diff.Set##name##unit(m##name##unit - aBefore.m##name##unit); \ + } + + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_SUB) + +#undef RUNNING_TIME_SUB + return diff; + } + + private: + friend mozilla::ProfileBufferEntryWriter::Serializer<RunningTimes>; + friend mozilla::ProfileBufferEntryReader::Deserializer<RunningTimes>; + + // Platform-dependent. + static uint64_t ConvertRawToJson(uint64_t aRawValue); + + mozilla::TimeStamp mPostMeasurementTimeStamp; + + uint32_t mKnownBits = 0u; + +#define RUNNING_TIME_MEMBER(index, name, unit, jsonProperty) \ + static constexpr uint32_t mGot##name##unit = 1u << index; \ + uint64_t m##name##unit = 0; + + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_MEMBER) + +#undef RUNNING_TIME_MEMBER +}; + +template <> +struct mozilla::ProfileBufferEntryWriter::Serializer<RunningTimes> { + static Length Bytes(const RunningTimes& aRunningTimes) { + Length bytes = 0; + +#define RUNNING_TIME_SERIALIZATION_BYTES(index, name, unit, jsonProperty) \ + if (aRunningTimes.Is##name##unit##Known()) { \ + bytes += ULEB128Size(aRunningTimes.m##name##unit); \ + } + + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_SERIALIZATION_BYTES) + +#undef RUNNING_TIME_SERIALIZATION_BYTES + return ULEB128Size(aRunningTimes.mKnownBits) + bytes; + } + + static void Write(ProfileBufferEntryWriter& aEW, + const RunningTimes& aRunningTimes) { + aEW.WriteULEB128(aRunningTimes.mKnownBits); + +#define RUNNING_TIME_SERIALIZE(index, name, unit, jsonProperty) \ + if (aRunningTimes.Is##name##unit##Known()) { \ + aEW.WriteULEB128(aRunningTimes.m##name##unit); \ + } + + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_SERIALIZE) + +#undef RUNNING_TIME_SERIALIZE + } +}; + +template <> +struct mozilla::ProfileBufferEntryReader::Deserializer<RunningTimes> { + static void ReadInto(ProfileBufferEntryReader& aER, + RunningTimes& aRunningTimes) { + aRunningTimes = Read(aER); + } + + static RunningTimes Read(ProfileBufferEntryReader& aER) { + // Start with empty running times, everything is cleared. + RunningTimes times; + + // This sets all the bits into mKnownBits, we don't need to modify it + // further. + times.mKnownBits = aER.ReadULEB128<uint32_t>(); + + // For each member that should be known, read its value. +#define RUNNING_TIME_DESERIALIZE(index, name, unit, jsonProperty) \ + if (times.Is##name##unit##Known()) { \ + times.m##name##unit = aER.ReadULEB128<decltype(times.m##name##unit)>(); \ + } + + PROFILER_FOR_EACH_RUNNING_TIME(RUNNING_TIME_DESERIALIZE) + +#undef RUNNING_TIME_DESERIALIZE + + return times; + } +}; + +#endif /* ndef TOOLS_PLATFORM_H_ */ diff --git a/tools/profiler/core/shared-libraries-linux.cc b/tools/profiler/core/shared-libraries-linux.cc new file mode 100644 index 0000000000..fce1ecfa5c --- /dev/null +++ b/tools/profiler/core/shared-libraries-linux.cc @@ -0,0 +1,281 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "shared-libraries.h" + +#define PATH_MAX_TOSTRING(x) #x +#define PATH_MAX_STRING(x) PATH_MAX_TOSTRING(x) +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <limits.h> +#include <unistd.h> +#include <fstream> +#include "platform.h" +#include "shared-libraries.h" +#include "GeckoProfiler.h" +#include "mozilla/Sprintf.h" +#include "mozilla/Unused.h" +#include "nsDebug.h" +#include "nsNativeCharsetUtils.h" +#include <nsTArray.h> + +#include "common/linux/file_id.h" +#include <algorithm> +#include <dlfcn.h> +#if defined(GP_OS_linux) || defined(GP_OS_android) +# include <features.h> +#endif +#include <sys/types.h> + +#if defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) +# include <link.h> // dl_phdr_info +#else +# error "Unexpected configuration" +#endif + +#if defined(GP_OS_android) +extern "C" MOZ_EXPORT __attribute__((weak)) int dl_iterate_phdr( + int (*callback)(struct dl_phdr_info* info, size_t size, void* data), + void* data); +#endif + +struct LoadedLibraryInfo { + LoadedLibraryInfo(const char* aName, unsigned long aBaseAddress, + unsigned long aFirstMappingStart, + unsigned long aLastMappingEnd) + : mName(aName), + mBaseAddress(aBaseAddress), + mFirstMappingStart(aFirstMappingStart), + mLastMappingEnd(aLastMappingEnd) {} + + nsCString mName; + unsigned long mBaseAddress; + unsigned long mFirstMappingStart; + unsigned long mLastMappingEnd; +}; + +static nsCString IDtoUUIDString( + const google_breakpad::wasteful_vector<uint8_t>& aIdentifier) { + using namespace google_breakpad; + + nsCString uuid; + const std::string str = FileID::ConvertIdentifierToUUIDString(aIdentifier); + uuid.Append(str.c_str(), str.size()); + // This is '0', not '\0', since it represents the breakpad id age. + uuid.Append('0'); + return uuid; +} + +// Return raw Build ID in hex. +static nsCString IDtoString( + const google_breakpad::wasteful_vector<uint8_t>& aIdentifier) { + using namespace google_breakpad; + + nsCString uuid; + const std::string str = FileID::ConvertIdentifierToString(aIdentifier); + uuid.Append(str.c_str(), str.size()); + return uuid; +} + +// Get the breakpad Id for the binary file pointed by bin_name +static nsCString getBreakpadId(const char* bin_name) { + using namespace google_breakpad; + + PageAllocator allocator; + auto_wasteful_vector<uint8_t, kDefaultBuildIdSize> identifier(&allocator); + + FileID file_id(bin_name); + if (file_id.ElfFileIdentifier(identifier)) { + return IDtoUUIDString(identifier); + } + + return ""_ns; +} + +// Get the code Id for the binary file pointed by bin_name +static nsCString getCodeId(const char* bin_name) { + using namespace google_breakpad; + + PageAllocator allocator; + auto_wasteful_vector<uint8_t, kDefaultBuildIdSize> identifier(&allocator); + + FileID file_id(bin_name); + if (file_id.ElfFileIdentifier(identifier)) { + return IDtoString(identifier); + } + + return ""_ns; +} + +static SharedLibrary SharedLibraryAtPath(const char* path, + unsigned long libStart, + unsigned long libEnd, + unsigned long offset = 0) { + nsAutoString pathStr; + mozilla::Unused << NS_WARN_IF( + NS_FAILED(NS_CopyNativeToUnicode(nsDependentCString(path), pathStr))); + + nsAutoString nameStr = pathStr; + int32_t pos = nameStr.RFindChar('/'); + if (pos != kNotFound) { + nameStr.Cut(0, pos + 1); + } + + return SharedLibrary(libStart, libEnd, offset, getBreakpadId(path), + getCodeId(path), nameStr, pathStr, nameStr, pathStr, + ""_ns, ""); +} + +static int dl_iterate_callback(struct dl_phdr_info* dl_info, size_t size, + void* data) { + auto libInfoList = reinterpret_cast<nsTArray<LoadedLibraryInfo>*>(data); + + if (dl_info->dlpi_phnum <= 0) return 0; + + unsigned long baseAddress = dl_info->dlpi_addr; + unsigned long firstMappingStart = -1; + unsigned long lastMappingEnd = 0; + + for (size_t i = 0; i < dl_info->dlpi_phnum; i++) { + if (dl_info->dlpi_phdr[i].p_type != PT_LOAD) { + continue; + } + unsigned long start = dl_info->dlpi_addr + dl_info->dlpi_phdr[i].p_vaddr; + unsigned long end = start + dl_info->dlpi_phdr[i].p_memsz; + if (start < firstMappingStart) { + firstMappingStart = start; + } + if (end > lastMappingEnd) { + lastMappingEnd = end; + } + } + + libInfoList->AppendElement(LoadedLibraryInfo( + dl_info->dlpi_name, baseAddress, firstMappingStart, lastMappingEnd)); + + return 0; +} + +SharedLibraryInfo SharedLibraryInfo::GetInfoForSelf() { + SharedLibraryInfo info; + +#if defined(GP_OS_linux) + // We need to find the name of the executable (exeName, exeNameLen) and the + // address of its executable section (exeExeAddr) in the running image. + char exeName[PATH_MAX]; + memset(exeName, 0, sizeof(exeName)); + + ssize_t exeNameLen = readlink("/proc/self/exe", exeName, sizeof(exeName) - 1); + if (exeNameLen == -1) { + // readlink failed for whatever reason. Note this, but keep going. + exeName[0] = '\0'; + exeNameLen = 0; + LOG("SharedLibraryInfo::GetInfoForSelf(): readlink failed"); + } else { + // Assert no buffer overflow. + MOZ_RELEASE_ASSERT(exeNameLen >= 0 && + exeNameLen < static_cast<ssize_t>(sizeof(exeName))); + } + + unsigned long exeExeAddr = 0; +#endif + +#if defined(GP_OS_android) + // If dl_iterate_phdr doesn't exist, we give up immediately. + if (!dl_iterate_phdr) { + // On ARM Android, dl_iterate_phdr is provided by the custom linker. + // So if libxul was loaded by the system linker (e.g. as part of + // xpcshell when running tests), it won't be available and we should + // not call it. + return info; + } +#endif + +#if defined(GP_OS_linux) || defined(GP_OS_android) + // Read info from /proc/self/maps. We ignore most of it. + pid_t pid = profiler_current_process_id().ToNumber(); + char path[PATH_MAX]; + char modulePath[PATH_MAX + 1]; + SprintfLiteral(path, "/proc/%d/maps", pid); + std::ifstream maps(path); + std::string line; + while (std::getline(maps, line)) { + int ret; + unsigned long start; + unsigned long end; + char perm[6 + 1] = ""; + unsigned long offset; + modulePath[0] = 0; + ret = sscanf(line.c_str(), + "%lx-%lx %6s %lx %*s %*x %" PATH_MAX_STRING(PATH_MAX) "s\n", + &start, &end, perm, &offset, modulePath); + if (!strchr(perm, 'x')) { + // Ignore non executable entries + continue; + } + if (ret != 5 && ret != 4) { + LOG("SharedLibraryInfo::GetInfoForSelf(): " + "reading /proc/self/maps failed"); + continue; + } + +# if defined(GP_OS_linux) + // Try to establish the main executable's load address. + if (exeNameLen > 0 && strcmp(modulePath, exeName) == 0) { + exeExeAddr = start; + } +# elif defined(GP_OS_android) + // Use /proc/pid/maps to get the dalvik-jit section since it has no + // associated phdrs. + if (0 == strcmp(modulePath, "/dev/ashmem/dalvik-jit-code-cache")) { + info.AddSharedLibrary( + SharedLibraryAtPath(modulePath, start, end, offset)); + if (info.GetSize() > 10000) { + LOG("SharedLibraryInfo::GetInfoForSelf(): " + "implausibly large number of mappings acquired"); + break; + } + } +# endif + } +#endif + + nsTArray<LoadedLibraryInfo> libInfoList; + + // We collect the bulk of the library info using dl_iterate_phdr. + dl_iterate_phdr(dl_iterate_callback, &libInfoList); + + for (const auto& libInfo : libInfoList) { + info.AddSharedLibrary( + SharedLibraryAtPath(libInfo.mName.get(), libInfo.mFirstMappingStart, + libInfo.mLastMappingEnd, + libInfo.mFirstMappingStart - libInfo.mBaseAddress)); + } + +#if defined(GP_OS_linux) + // Make another pass over the information we just harvested from + // dl_iterate_phdr. If we see a nameless object mapped at what we earlier + // established to be the main executable's load address, attach the + // executable's name to that entry. + for (size_t i = 0; i < info.GetSize(); i++) { + SharedLibrary& lib = info.GetMutableEntry(i); + if (lib.GetStart() <= exeExeAddr && exeExeAddr <= lib.GetEnd() && + lib.GetNativeDebugPath().empty()) { + lib = SharedLibraryAtPath(exeName, lib.GetStart(), lib.GetEnd(), + lib.GetOffset()); + + // We only expect to see one such entry. + break; + } + } +#endif + + return info; +} + +void SharedLibraryInfo::Initialize() { /* do nothing */ +} diff --git a/tools/profiler/core/shared-libraries-macos.cc b/tools/profiler/core/shared-libraries-macos.cc new file mode 100644 index 0000000000..415fda3633 --- /dev/null +++ b/tools/profiler/core/shared-libraries-macos.cc @@ -0,0 +1,211 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "shared-libraries.h" + +#include "ClearOnShutdown.h" +#include "mozilla/StaticMutex.h" +#include "mozilla/Unused.h" +#include "nsNativeCharsetUtils.h" +#include <AvailabilityMacros.h> + +#include <dlfcn.h> +#include <mach-o/arch.h> +#include <mach-o/dyld_images.h> +#include <mach-o/dyld.h> +#include <mach-o/loader.h> +#include <mach/mach_init.h> +#include <mach/mach_traps.h> +#include <mach/task_info.h> +#include <mach/task.h> +#include <sstream> +#include <stdlib.h> +#include <string.h> +#include <vector> + +// Architecture specific abstraction. +#if defined(GP_ARCH_x86) +typedef mach_header platform_mach_header; +typedef segment_command mach_segment_command_type; +# define MACHO_MAGIC_NUMBER MH_MAGIC +# define CMD_SEGMENT LC_SEGMENT +# define seg_size uint32_t +#else +typedef mach_header_64 platform_mach_header; +typedef segment_command_64 mach_segment_command_type; +# define MACHO_MAGIC_NUMBER MH_MAGIC_64 +# define CMD_SEGMENT LC_SEGMENT_64 +# define seg_size uint64_t +#endif + +struct NativeSharedLibrary { + const platform_mach_header* header; + std::string path; +}; +static std::vector<NativeSharedLibrary>* sSharedLibrariesList = nullptr; +static mozilla::StaticMutex sSharedLibrariesMutex MOZ_UNANNOTATED; + +static void SharedLibraryAddImage(const struct mach_header* mh, + intptr_t vmaddr_slide) { + // NOTE: Presumably for backwards-compatibility reasons, this function accepts + // a mach_header even on 64-bit where it ought to be a mach_header_64. We cast + // it to the right type here. + auto header = reinterpret_cast<const platform_mach_header*>(mh); + + Dl_info info; + if (!dladdr(header, &info)) { + return; + } + + mozilla::StaticMutexAutoLock lock(sSharedLibrariesMutex); + if (!sSharedLibrariesList) { + return; + } + + NativeSharedLibrary lib = {header, info.dli_fname}; + sSharedLibrariesList->push_back(lib); +} + +static void SharedLibraryRemoveImage(const struct mach_header* mh, + intptr_t vmaddr_slide) { + // NOTE: Presumably for backwards-compatibility reasons, this function accepts + // a mach_header even on 64-bit where it ought to be a mach_header_64. We cast + // it to the right type here. + auto header = reinterpret_cast<const platform_mach_header*>(mh); + + mozilla::StaticMutexAutoLock lock(sSharedLibrariesMutex); + if (!sSharedLibrariesList) { + return; + } + + uint32_t count = sSharedLibrariesList->size(); + for (uint32_t i = 0; i < count; ++i) { + if ((*sSharedLibrariesList)[i].header == header) { + sSharedLibrariesList->erase(sSharedLibrariesList->begin() + i); + return; + } + } +} + +void SharedLibraryInfo::Initialize() { + // NOTE: We intentionally leak this memory here. We're allocating dynamically + // in order to avoid static initializers. + sSharedLibrariesList = new std::vector<NativeSharedLibrary>(); + + _dyld_register_func_for_add_image(SharedLibraryAddImage); + _dyld_register_func_for_remove_image(SharedLibraryRemoveImage); +} + +static void addSharedLibrary(const platform_mach_header* header, + const char* path, SharedLibraryInfo& info) { + const struct load_command* cmd = + reinterpret_cast<const struct load_command*>(header + 1); + + seg_size size = 0; + unsigned long long start = reinterpret_cast<unsigned long long>(header); + // Find the cmd segment in the macho image. It will contain the offset we care + // about. + const uint8_t* uuid_bytes = nullptr; + for (unsigned int i = 0; + cmd && (i < header->ncmds) && (uuid_bytes == nullptr || size == 0); + ++i) { + if (cmd->cmd == CMD_SEGMENT) { + const mach_segment_command_type* seg = + reinterpret_cast<const mach_segment_command_type*>(cmd); + + if (!strcmp(seg->segname, "__TEXT")) { + size = seg->vmsize; + } + } else if (cmd->cmd == LC_UUID) { + const uuid_command* ucmd = reinterpret_cast<const uuid_command*>(cmd); + uuid_bytes = ucmd->uuid; + } + + cmd = reinterpret_cast<const struct load_command*>( + reinterpret_cast<const char*>(cmd) + cmd->cmdsize); + } + + nsAutoCString uuid; + nsAutoCString breakpadId; + if (uuid_bytes != nullptr) { + uuid.AppendPrintf( + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X" + "%02X", + uuid_bytes[0], uuid_bytes[1], uuid_bytes[2], uuid_bytes[3], + uuid_bytes[4], uuid_bytes[5], uuid_bytes[6], uuid_bytes[7], + uuid_bytes[8], uuid_bytes[9], uuid_bytes[10], uuid_bytes[11], + uuid_bytes[12], uuid_bytes[13], uuid_bytes[14], uuid_bytes[15]); + + // Breakpad id is the same as the uuid but with the additional trailing 0 + // for the breakpad id age. + breakpadId.AppendPrintf( + "%s" + "0" /* breakpad id age */, + uuid.get()); + } + + nsAutoString pathStr; + mozilla::Unused << NS_WARN_IF( + NS_FAILED(NS_CopyNativeToUnicode(nsDependentCString(path), pathStr))); + + nsAutoString nameStr = pathStr; + int32_t pos = nameStr.RFindChar('/'); + if (pos != kNotFound) { + nameStr.Cut(0, pos + 1); + } + + const NXArchInfo* archInfo = + NXGetArchInfoFromCpuType(header->cputype, header->cpusubtype); + + info.AddSharedLibrary(SharedLibrary(start, start + size, 0, breakpadId, uuid, + nameStr, pathStr, nameStr, pathStr, ""_ns, + archInfo ? archInfo->name : "")); +} + +// Translate the statically stored sSharedLibrariesList information into a +// SharedLibraryInfo object. +SharedLibraryInfo SharedLibraryInfo::GetInfoForSelf() { + mozilla::StaticMutexAutoLock lock(sSharedLibrariesMutex); + SharedLibraryInfo sharedLibraryInfo; + + for (auto& info : *sSharedLibrariesList) { + addSharedLibrary(info.header, info.path.c_str(), sharedLibraryInfo); + } + + // Add the entry for dyld itself. + // We only support macOS 10.12+, which corresponds to dyld version 15+. + // dyld version 15 added the dyldPath property. + task_dyld_info_data_t task_dyld_info; + mach_msg_type_number_t count = TASK_DYLD_INFO_COUNT; + if (task_info(mach_task_self(), TASK_DYLD_INFO, (task_info_t)&task_dyld_info, + &count) != KERN_SUCCESS) { + return sharedLibraryInfo; + } + + struct dyld_all_image_infos* aii = + (struct dyld_all_image_infos*)task_dyld_info.all_image_info_addr; + if (aii->version >= 15) { + const platform_mach_header* header = + reinterpret_cast<const platform_mach_header*>( + aii->dyldImageLoadAddress); + addSharedLibrary(header, aii->dyldPath, sharedLibraryInfo); + } + + return sharedLibraryInfo; +} diff --git a/tools/profiler/core/shared-libraries-win32.cc b/tools/profiler/core/shared-libraries-win32.cc new file mode 100644 index 0000000000..1dc0d06836 --- /dev/null +++ b/tools/profiler/core/shared-libraries-win32.cc @@ -0,0 +1,146 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <windows.h> + +#include "shared-libraries.h" +#include "nsWindowsHelpers.h" +#include "mozilla/NativeNt.h" +#include "mozilla/WindowsEnumProcessModules.h" +#include "mozilla/WindowsProcessMitigations.h" +#include "nsPrintfCString.h" + +static bool IsModuleUnsafeToLoad(const nsAString& aModuleName) { + // Hackaround for Bug 1723868. There is no safe way to prevent the module + // Microsoft's VP9 Video Decoder from being unloaded because mfplat.dll may + // have posted more than one task to unload the module in the work queue + // without calling LoadLibrary. + if (aModuleName.LowerCaseEqualsLiteral("msvp9dec_store.dll")) { + return true; + } + + return false; +} + +void AddSharedLibraryFromModuleInfo(SharedLibraryInfo& sharedLibraryInfo, + const wchar_t* aModulePath, + mozilla::Maybe<HMODULE> aModule) { + nsDependentSubstring moduleNameStr( + mozilla::nt::GetLeafName(nsDependentString(aModulePath))); + + // If the module is unsafe to call LoadLibraryEx for, we skip. + if (IsModuleUnsafeToLoad(moduleNameStr)) { + return; + } + + // If EAF+ is enabled, parsing ntdll's PE header causes a crash. + if (mozilla::IsEafPlusEnabled() && + moduleNameStr.LowerCaseEqualsLiteral("ntdll.dll")) { + return; + } + + // Load the module again - to make sure that its handle will remain valid as + // we attempt to read the PDB information from it - or for the first time if + // we only have a path. We want to load the DLL without running the newly + // loaded module's DllMain function, but not as a data file because we want + // to be able to do RVA computations easily. Hence, we use the flag + // LOAD_LIBRARY_AS_IMAGE_RESOURCE which ensures that the sections (not PE + // headers) will be relocated by the loader. Otherwise GetPdbInfo() and/or + // GetVersionInfo() can cause a crash. If the original handle |aModule| is + // valid, LoadLibraryEx just increments its refcount. + nsModuleHandle handleLock( + ::LoadLibraryExW(aModulePath, NULL, LOAD_LIBRARY_AS_IMAGE_RESOURCE)); + if (!handleLock) { + return; + } + + mozilla::nt::PEHeaders headers(handleLock.get()); + if (!headers) { + return; + } + + mozilla::Maybe<mozilla::Range<const uint8_t>> bounds = headers.GetBounds(); + if (!bounds) { + return; + } + + // Put the original |aModule| into SharedLibrary, but we get debug info + // from |handleLock| as |aModule| might be inaccessible. + const uintptr_t modStart = + aModule.isSome() ? reinterpret_cast<uintptr_t>(*aModule) + : reinterpret_cast<uintptr_t>(handleLock.get()); + const uintptr_t modEnd = modStart + bounds->length(); + + nsAutoCString breakpadId; + nsAutoString pdbPathStr; + if (const auto* debugInfo = headers.GetPdbInfo()) { + MOZ_ASSERT(breakpadId.IsEmpty()); + const GUID& pdbSig = debugInfo->pdbSignature; + breakpadId.AppendPrintf( + "%08lX" // m0 + "%04X%04X" // m1,m2 + "%02X%02X%02X%02X%02X%02X%02X%02X" // m3 + "%X", // pdbAge + pdbSig.Data1, pdbSig.Data2, pdbSig.Data3, pdbSig.Data4[0], + pdbSig.Data4[1], pdbSig.Data4[2], pdbSig.Data4[3], pdbSig.Data4[4], + pdbSig.Data4[5], pdbSig.Data4[6], pdbSig.Data4[7], debugInfo->pdbAge); + + // The PDB file name could be different from module filename, + // so report both + // e.g. The PDB for C:\Windows\SysWOW64\ntdll.dll is wntdll.pdb + pdbPathStr = NS_ConvertUTF8toUTF16(debugInfo->pdbFileName); + } + + nsAutoCString codeId; + DWORD timestamp; + DWORD imageSize; + if (headers.GetTimeStamp(timestamp) && headers.GetImageSize(imageSize)) { + codeId.AppendPrintf( + "%08lX" // Uppercase 8 digits of hex timestamp with leading zeroes. + "%lx", // Lowercase hex image size + timestamp, imageSize); + } + + nsAutoCString versionStr; + uint64_t version; + if (headers.GetVersionInfo(version)) { + versionStr.AppendPrintf("%u.%u.%u.%u", + static_cast<uint32_t>((version >> 48) & 0xFFFFu), + static_cast<uint32_t>((version >> 32) & 0xFFFFu), + static_cast<uint32_t>((version >> 16) & 0xFFFFu), + static_cast<uint32_t>(version & 0xFFFFu)); + } + + const nsString& pdbNameStr = + PromiseFlatString(mozilla::nt::GetLeafName(pdbPathStr)); + SharedLibrary shlib(modStart, modEnd, + 0, // DLLs are always mapped at offset 0 on Windows + breakpadId, codeId, PromiseFlatString(moduleNameStr), + nsDependentString(aModulePath), pdbNameStr, pdbPathStr, + versionStr, ""); + sharedLibraryInfo.AddSharedLibrary(shlib); +} + +SharedLibraryInfo SharedLibraryInfo::GetInfoForSelf() { + SharedLibraryInfo sharedLibraryInfo; + + auto addSharedLibraryFromModuleInfo = + [&sharedLibraryInfo](const wchar_t* aModulePath, HMODULE aModule) { + AddSharedLibraryFromModuleInfo(sharedLibraryInfo, aModulePath, + mozilla::Some(aModule)); + }; + + mozilla::EnumerateProcessModules(addSharedLibraryFromModuleInfo); + return sharedLibraryInfo; +} + +SharedLibraryInfo SharedLibraryInfo::GetInfoFromPath(const wchar_t* aPath) { + SharedLibraryInfo sharedLibraryInfo; + AddSharedLibraryFromModuleInfo(sharedLibraryInfo, aPath, mozilla::Nothing()); + return sharedLibraryInfo; +} + +void SharedLibraryInfo::Initialize() { /* do nothing */ +} diff --git a/tools/profiler/core/vtune/ittnotify.h b/tools/profiler/core/vtune/ittnotify.h new file mode 100644 index 0000000000..f1d65b3328 --- /dev/null +++ b/tools/profiler/core/vtune/ittnotify.h @@ -0,0 +1,4123 @@ +/* <copyright> + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + The full GNU General Public License is included in this distribution + in the file called LICENSE.GPL. + + Contact Information: + http://software.intel.com/en-us/articles/intel-vtune-amplifier-xe/ + + BSD LICENSE + + Copyright (c) 2005-2014 Intel Corporation. All rights reserved. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +</copyright> */ +#ifndef _ITTNOTIFY_H_ +#define _ITTNOTIFY_H_ + +/** +@file +@brief Public User API functions and types +@mainpage + +The ITT API is used to annotate a user's program with additional information +that can be used by correctness and performance tools. The user inserts +calls in their program. Those calls generate information that is collected +at runtime, and used by Intel(R) Threading Tools. + +@section API Concepts +The following general concepts are used throughout the API. + +@subsection Unicode Support +Many API functions take character string arguments. On Windows, there +are two versions of each such function. The function name is suffixed +by W if Unicode support is enabled, and by A otherwise. Any API function +that takes a character string argument adheres to this convention. + +@subsection Conditional Compilation +Many users prefer having an option to modify ITT API code when linking it +inside their runtimes. ITT API header file provides a mechanism to replace +ITT API function names inside your code with empty strings. To do this, +define the macros INTEL_NO_ITTNOTIFY_API during compilation and remove the +static library from the linker script. + +@subsection Domains +[see domains] +Domains provide a way to separate notification for different modules or +libraries in a program. Domains are specified by dotted character strings, +e.g. TBB.Internal.Control. + +A mechanism (to be specified) is provided to enable and disable +domains. By default, all domains are enabled. +@subsection Named Entities and Instances +Named entities (frames, regions, tasks, and markers) communicate +information about the program to the analysis tools. A named entity often +refers to a section of program code, or to some set of logical concepts +that the programmer wants to group together. + +Named entities relate to the programmer's static view of the program. When +the program actually executes, many instances of a given named entity +may be created. + +The API annotations denote instances of named entities. The actual +named entities are displayed using the analysis tools. In other words, +the named entities come into existence when instances are created. + +Instances of named entities may have instance identifiers (IDs). Some +API calls use instance identifiers to create relationships between +different instances of named entities. Other API calls associate data +with instances of named entities. + +Some named entities must always have instance IDs. In particular, regions +and frames always have IDs. Task and markers need IDs only if the ID is +needed in another API call (such as adding a relation or metadata). + +The lifetime of instance IDs is distinct from the lifetime of +instances. This allows various relationships to be specified separate +from the actual execution of instances. This flexibility comes at the +expense of extra API calls. + +The same ID may not be reused for different instances, unless a previous +[ref] __itt_id_destroy call for that ID has been issued. +*/ + +/** @cond exclude_from_documentation */ +#ifndef ITT_OS_WIN +# define ITT_OS_WIN 1 +#endif /* ITT_OS_WIN */ + +#ifndef ITT_OS_LINUX +# define ITT_OS_LINUX 2 +#endif /* ITT_OS_LINUX */ + +#ifndef ITT_OS_MAC +# define ITT_OS_MAC 3 +#endif /* ITT_OS_MAC */ + +#ifndef ITT_OS_FREEBSD +# define ITT_OS_FREEBSD 4 +#endif /* ITT_OS_FREEBSD */ + +#ifndef ITT_OS +# if defined WIN32 || defined _WIN32 +# define ITT_OS ITT_OS_WIN +# elif defined( __APPLE__ ) && defined( __MACH__ ) +# define ITT_OS ITT_OS_MAC +# elif defined( __FreeBSD__ ) +# define ITT_OS ITT_OS_FREEBSD +# else +# define ITT_OS ITT_OS_LINUX +# endif +#endif /* ITT_OS */ + +#ifndef ITT_PLATFORM_WIN +# define ITT_PLATFORM_WIN 1 +#endif /* ITT_PLATFORM_WIN */ + +#ifndef ITT_PLATFORM_POSIX +# define ITT_PLATFORM_POSIX 2 +#endif /* ITT_PLATFORM_POSIX */ + +#ifndef ITT_PLATFORM_MAC +# define ITT_PLATFORM_MAC 3 +#endif /* ITT_PLATFORM_MAC */ + +#ifndef ITT_PLATFORM_FREEBSD +# define ITT_PLATFORM_FREEBSD 4 +#endif /* ITT_PLATFORM_FREEBSD */ + +#ifndef ITT_PLATFORM +# if ITT_OS==ITT_OS_WIN +# define ITT_PLATFORM ITT_PLATFORM_WIN +# elif ITT_OS==ITT_OS_MAC +# define ITT_PLATFORM ITT_PLATFORM_MAC +# elif ITT_OS==ITT_OS_FREEBSD +# define ITT_PLATFORM ITT_PLATFORM_FREEBSD +# else +# define ITT_PLATFORM ITT_PLATFORM_POSIX +# endif +#endif /* ITT_PLATFORM */ + +#if defined(_UNICODE) && !defined(UNICODE) +#define UNICODE +#endif + +#include <stddef.h> +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#include <tchar.h> +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#include <stdint.h> +#if defined(UNICODE) || defined(_UNICODE) +#include <wchar.h> +#endif /* UNICODE || _UNICODE */ +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef ITTAPI_CDECL +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# define ITTAPI_CDECL __cdecl +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# if defined _M_IX86 || defined __i386__ +# define ITTAPI_CDECL __attribute__ ((cdecl)) +# else /* _M_IX86 || __i386__ */ +# define ITTAPI_CDECL /* actual only on x86 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* ITTAPI_CDECL */ + +#ifndef STDCALL +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# define STDCALL __stdcall +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# if defined _M_IX86 || defined __i386__ +# define STDCALL __attribute__ ((stdcall)) +# else /* _M_IX86 || __i386__ */ +# define STDCALL /* supported only on x86 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* STDCALL */ + +#define ITTAPI ITTAPI_CDECL +#define LIBITTAPI ITTAPI_CDECL + +/* TODO: Temporary for compatibility! */ +#define ITTAPI_CALL ITTAPI_CDECL +#define LIBITTAPI_CALL ITTAPI_CDECL + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +/* use __forceinline (VC++ specific) */ +#define ITT_INLINE __forceinline +#define ITT_INLINE_ATTRIBUTE /* nothing */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +/* + * Generally, functions are not inlined unless optimization is specified. + * For functions declared inline, this attribute inlines the function even + * if no optimization level was specified. + */ +#ifdef __STRICT_ANSI__ +#define ITT_INLINE static +#define ITT_INLINE_ATTRIBUTE __attribute__((unused)) +#else /* __STRICT_ANSI__ */ +#define ITT_INLINE static inline +#define ITT_INLINE_ATTRIBUTE __attribute__((always_inline, unused)) +#endif /* __STRICT_ANSI__ */ +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +/** @endcond */ + +#ifdef INTEL_ITTNOTIFY_ENABLE_LEGACY +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# pragma message("WARNING!!! Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro") +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# warning "Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro" +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# include "vtune/legacy/ittnotify.h" +#endif /* INTEL_ITTNOTIFY_ENABLE_LEGACY */ + +/** @cond exclude_from_documentation */ +/* Helper macro for joining tokens */ +#define ITT_JOIN_AUX(p,n) p##n +#define ITT_JOIN(p,n) ITT_JOIN_AUX(p,n) + +#ifdef ITT_MAJOR +#undef ITT_MAJOR +#endif +#ifdef ITT_MINOR +#undef ITT_MINOR +#endif +#define ITT_MAJOR 3 +#define ITT_MINOR 0 + +/* Standard versioning of a token with major and minor version numbers */ +#define ITT_VERSIONIZE(x) \ + ITT_JOIN(x, \ + ITT_JOIN(_, \ + ITT_JOIN(ITT_MAJOR, \ + ITT_JOIN(_, ITT_MINOR)))) + +#ifndef INTEL_ITTNOTIFY_PREFIX +# define INTEL_ITTNOTIFY_PREFIX __itt_ +#endif /* INTEL_ITTNOTIFY_PREFIX */ +#ifndef INTEL_ITTNOTIFY_POSTFIX +# define INTEL_ITTNOTIFY_POSTFIX _ptr_ +#endif /* INTEL_ITTNOTIFY_POSTFIX */ + +#define ITTNOTIFY_NAME_AUX(n) ITT_JOIN(INTEL_ITTNOTIFY_PREFIX,n) +#define ITTNOTIFY_NAME(n) ITT_VERSIONIZE(ITTNOTIFY_NAME_AUX(ITT_JOIN(n,INTEL_ITTNOTIFY_POSTFIX))) + +#define ITTNOTIFY_VOID(n) (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n) +#define ITTNOTIFY_DATA(n) (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n) + +#define ITTNOTIFY_VOID_D0(n,d) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_VOID_D1(n,d,x) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_VOID_D2(n,d,x,y) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_VOID_D3(n,d,x,y,z) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_VOID_D4(n,d,x,y,z,a) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_VOID_D5(n,d,x,y,z,a,b) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_VOID_D6(n,d,x,y,z,a,b,c) (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) +#define ITTNOTIFY_DATA_D0(n,d) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_DATA_D1(n,d,x) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_DATA_D2(n,d,x,y) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_DATA_D3(n,d,x,y,z) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_DATA_D4(n,d,x,y,z,a) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_DATA_D5(n,d,x,y,z,a,b) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_DATA_D6(n,d,x,y,z,a,b,c) (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) + +#ifdef ITT_STUB +#undef ITT_STUB +#endif +#ifdef ITT_STUBV +#undef ITT_STUBV +#endif +#define ITT_STUBV(api,type,name,args) \ + typedef type (api* ITT_JOIN(ITTNOTIFY_NAME(name),_t)) args; \ + extern ITT_JOIN(ITTNOTIFY_NAME(name),_t) ITTNOTIFY_NAME(name); +#define ITT_STUB ITT_STUBV +/** @endcond */ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** @cond exclude_from_gpa_documentation */ +/** + * @defgroup public Public API + * @{ + * @} + */ + +/** + * @defgroup control Collection Control + * @ingroup public + * General behavior: application continues to run, but no profiling information is being collected + * + * Pausing occurs not only for the current thread but for all process as well as spawned processes + * - Intel(R) Parallel Inspector and Intel(R) Inspector XE: + * - Does not analyze or report errors that involve memory access. + * - Other errors are reported as usual. Pausing data collection in + * Intel(R) Parallel Inspector and Intel(R) Inspector XE + * only pauses tracing and analyzing memory access. + * It does not pause tracing or analyzing threading APIs. + * . + * - Intel(R) Parallel Amplifier and Intel(R) VTune(TM) Amplifier XE: + * - Does continue to record when new threads are started. + * . + * - Other effects: + * - Possible reduction of runtime overhead. + * . + * @{ + */ +/** @brief Pause collection */ +void ITTAPI __itt_pause(void); +/** @brief Resume collection */ +void ITTAPI __itt_resume(void); +/** @brief Detach collection */ +void ITTAPI __itt_detach(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, pause, (void)) +ITT_STUBV(ITTAPI, void, resume, (void)) +ITT_STUBV(ITTAPI, void, detach, (void)) +#define __itt_pause ITTNOTIFY_VOID(pause) +#define __itt_pause_ptr ITTNOTIFY_NAME(pause) +#define __itt_resume ITTNOTIFY_VOID(resume) +#define __itt_resume_ptr ITTNOTIFY_NAME(resume) +#define __itt_detach ITTNOTIFY_VOID(detach) +#define __itt_detach_ptr ITTNOTIFY_NAME(detach) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_pause() +#define __itt_pause_ptr 0 +#define __itt_resume() +#define __itt_resume_ptr 0 +#define __itt_detach() +#define __itt_detach_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_pause_ptr 0 +#define __itt_resume_ptr 0 +#define __itt_detach_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} control group */ +/** @endcond */ + +/** + * @defgroup threads Threads + * @ingroup public + * Give names to threads + * @{ + */ +/** + * @brief Sets thread name of calling thread + * @param[in] name - name of thread + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_thread_set_nameA(const char *name); +void ITTAPI __itt_thread_set_nameW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_thread_set_name __itt_thread_set_nameW +# define __itt_thread_set_name_ptr __itt_thread_set_nameW_ptr +#else /* UNICODE */ +# define __itt_thread_set_name __itt_thread_set_nameA +# define __itt_thread_set_name_ptr __itt_thread_set_nameA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_thread_set_name(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, thread_set_nameA, (const char *name)) +ITT_STUBV(ITTAPI, void, thread_set_nameW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, thread_set_name, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_thread_set_nameA ITTNOTIFY_VOID(thread_set_nameA) +#define __itt_thread_set_nameA_ptr ITTNOTIFY_NAME(thread_set_nameA) +#define __itt_thread_set_nameW ITTNOTIFY_VOID(thread_set_nameW) +#define __itt_thread_set_nameW_ptr ITTNOTIFY_NAME(thread_set_nameW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_thread_set_name ITTNOTIFY_VOID(thread_set_name) +#define __itt_thread_set_name_ptr ITTNOTIFY_NAME(thread_set_name) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_thread_set_nameA(name) +#define __itt_thread_set_nameA_ptr 0 +#define __itt_thread_set_nameW(name) +#define __itt_thread_set_nameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_thread_set_name(name) +#define __itt_thread_set_name_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_thread_set_nameA_ptr 0 +#define __itt_thread_set_nameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_thread_set_name_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** + * @brief Mark current thread as ignored from this point on, for the duration of its existence. + */ +void ITTAPI __itt_thread_ignore(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, thread_ignore, (void)) +#define __itt_thread_ignore ITTNOTIFY_VOID(thread_ignore) +#define __itt_thread_ignore_ptr ITTNOTIFY_NAME(thread_ignore) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_thread_ignore() +#define __itt_thread_ignore_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_thread_ignore_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} threads group */ + +/** + * @defgroup suppress Error suppression + * @ingroup public + * General behavior: application continues to run, but errors are suppressed + * + * @{ + */ + +/*****************************************************************//** + * @name group of functions used for error suppression in correctness tools + *********************************************************************/ +/** @{ */ +/** + * @hideinitializer + * @brief possible value for suppression mask + */ +#define __itt_suppress_all_errors 0x7fffffff + +/** + * @hideinitializer + * @brief possible value for suppression mask (suppresses errors from threading analysis) + */ +#define __itt_suppress_threading_errors 0x000000ff + +/** + * @hideinitializer + * @brief possible value for suppression mask (suppresses errors from memory analysis) + */ +#define __itt_suppress_memory_errors 0x0000ff00 + +/** + * @brief Start suppressing errors identified in mask on this thread + */ +void ITTAPI __itt_suppress_push(unsigned int mask); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_push, (unsigned int mask)) +#define __itt_suppress_push ITTNOTIFY_VOID(suppress_push) +#define __itt_suppress_push_ptr ITTNOTIFY_NAME(suppress_push) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_push(mask) +#define __itt_suppress_push_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_push_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Undo the effects of the matching call to __itt_suppress_push + */ +void ITTAPI __itt_suppress_pop(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_pop, (void)) +#define __itt_suppress_pop ITTNOTIFY_VOID(suppress_pop) +#define __itt_suppress_pop_ptr ITTNOTIFY_NAME(suppress_pop) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_pop() +#define __itt_suppress_pop_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_pop_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @enum __itt_model_disable + * @brief Enumerator for the disable methods + */ +typedef enum __itt_suppress_mode { + __itt_unsuppress_range, + __itt_suppress_range +} __itt_suppress_mode_t; + +/** + * @brief Mark a range of memory for error suppression or unsuppression for error types included in mask + */ +void ITTAPI __itt_suppress_mark_range(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_mark_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size)) +#define __itt_suppress_mark_range ITTNOTIFY_VOID(suppress_mark_range) +#define __itt_suppress_mark_range_ptr ITTNOTIFY_NAME(suppress_mark_range) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_mark_range(mask) +#define __itt_suppress_mark_range_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_mark_range_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Undo the effect of a matching call to __itt_suppress_mark_range. If not matching + * call is found, nothing is changed. + */ +void ITTAPI __itt_suppress_clear_range(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_clear_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size)) +#define __itt_suppress_clear_range ITTNOTIFY_VOID(suppress_clear_range) +#define __itt_suppress_clear_range_ptr ITTNOTIFY_NAME(suppress_clear_range) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_clear_range(mask) +#define __itt_suppress_clear_range_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_clear_range_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} */ +/** @} suppress group */ + +/** + * @defgroup sync Synchronization + * @ingroup public + * Indicate user-written synchronization code + * @{ + */ +/** + * @hideinitializer + * @brief possible value of attribute argument for sync object type + */ +#define __itt_attr_barrier 1 + +/** + * @hideinitializer + * @brief possible value of attribute argument for sync object type + */ +#define __itt_attr_mutex 2 + +/** +@brief Name a synchronization object +@param[in] addr Handle for the synchronization object. You should +use a real address to uniquely identify the synchronization object. +@param[in] objtype null-terminated object type string. If NULL is +passed, the name will be "User Synchronization". +@param[in] objname null-terminated object name string. If NULL, +no name will be assigned to the object. +@param[in] attribute one of [#__itt_attr_barrier, #__itt_attr_mutex] + */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_sync_createA(void *addr, const char *objtype, const char *objname, int attribute); +void ITTAPI __itt_sync_createW(void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_sync_create __itt_sync_createW +# define __itt_sync_create_ptr __itt_sync_createW_ptr +#else /* UNICODE */ +# define __itt_sync_create __itt_sync_createA +# define __itt_sync_create_ptr __itt_sync_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_sync_create (void *addr, const char *objtype, const char *objname, int attribute); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, sync_createA, (void *addr, const char *objtype, const char *objname, int attribute)) +ITT_STUBV(ITTAPI, void, sync_createW, (void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, sync_create, (void *addr, const char* objtype, const char* objname, int attribute)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_createA ITTNOTIFY_VOID(sync_createA) +#define __itt_sync_createA_ptr ITTNOTIFY_NAME(sync_createA) +#define __itt_sync_createW ITTNOTIFY_VOID(sync_createW) +#define __itt_sync_createW_ptr ITTNOTIFY_NAME(sync_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_create ITTNOTIFY_VOID(sync_create) +#define __itt_sync_create_ptr ITTNOTIFY_NAME(sync_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_createA(addr, objtype, objname, attribute) +#define __itt_sync_createA_ptr 0 +#define __itt_sync_createW(addr, objtype, objname, attribute) +#define __itt_sync_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_create(addr, objtype, objname, attribute) +#define __itt_sync_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_createA_ptr 0 +#define __itt_sync_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** +@brief Rename a synchronization object + +You can use the rename call to assign or reassign a name to a given +synchronization object. +@param[in] addr handle for the synchronization object. +@param[in] name null-terminated object name string. +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_sync_renameA(void *addr, const char *name); +void ITTAPI __itt_sync_renameW(void *addr, const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_sync_rename __itt_sync_renameW +# define __itt_sync_rename_ptr __itt_sync_renameW_ptr +#else /* UNICODE */ +# define __itt_sync_rename __itt_sync_renameA +# define __itt_sync_rename_ptr __itt_sync_renameA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_sync_rename(void *addr, const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, sync_renameA, (void *addr, const char *name)) +ITT_STUBV(ITTAPI, void, sync_renameW, (void *addr, const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, sync_rename, (void *addr, const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_renameA ITTNOTIFY_VOID(sync_renameA) +#define __itt_sync_renameA_ptr ITTNOTIFY_NAME(sync_renameA) +#define __itt_sync_renameW ITTNOTIFY_VOID(sync_renameW) +#define __itt_sync_renameW_ptr ITTNOTIFY_NAME(sync_renameW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_rename ITTNOTIFY_VOID(sync_rename) +#define __itt_sync_rename_ptr ITTNOTIFY_NAME(sync_rename) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_renameA(addr, name) +#define __itt_sync_renameA_ptr 0 +#define __itt_sync_renameW(addr, name) +#define __itt_sync_renameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_rename(addr, name) +#define __itt_sync_rename_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_renameA_ptr 0 +#define __itt_sync_renameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_rename_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + @brief Destroy a synchronization object. + @param addr Handle for the synchronization object. + */ +void ITTAPI __itt_sync_destroy(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_destroy, (void *addr)) +#define __itt_sync_destroy ITTNOTIFY_VOID(sync_destroy) +#define __itt_sync_destroy_ptr ITTNOTIFY_NAME(sync_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_destroy(addr) +#define __itt_sync_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/*****************************************************************//** + * @name group of functions is used for performance measurement tools + *********************************************************************/ +/** @{ */ +/** + * @brief Enter spin loop on user-defined sync object + */ +void ITTAPI __itt_sync_prepare(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_prepare, (void *addr)) +#define __itt_sync_prepare ITTNOTIFY_VOID(sync_prepare) +#define __itt_sync_prepare_ptr ITTNOTIFY_NAME(sync_prepare) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_prepare(addr) +#define __itt_sync_prepare_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_prepare_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Quit spin loop without acquiring spin object + */ +void ITTAPI __itt_sync_cancel(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_cancel, (void *addr)) +#define __itt_sync_cancel ITTNOTIFY_VOID(sync_cancel) +#define __itt_sync_cancel_ptr ITTNOTIFY_NAME(sync_cancel) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_cancel(addr) +#define __itt_sync_cancel_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_cancel_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Successful spin loop completion (sync object acquired) + */ +void ITTAPI __itt_sync_acquired(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_acquired, (void *addr)) +#define __itt_sync_acquired ITTNOTIFY_VOID(sync_acquired) +#define __itt_sync_acquired_ptr ITTNOTIFY_NAME(sync_acquired) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_acquired(addr) +#define __itt_sync_acquired_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_acquired_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Start sync object releasing code. Is called before the lock release call. + */ +void ITTAPI __itt_sync_releasing(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_releasing, (void *addr)) +#define __itt_sync_releasing ITTNOTIFY_VOID(sync_releasing) +#define __itt_sync_releasing_ptr ITTNOTIFY_NAME(sync_releasing) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_releasing(addr) +#define __itt_sync_releasing_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_releasing_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} */ + +/** @} sync group */ + +/**************************************************************//** + * @name group of functions is used for correctness checking tools + ******************************************************************/ +/** @{ */ +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_prepare(void* addr); + */ +void ITTAPI __itt_fsync_prepare(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_prepare, (void *addr)) +#define __itt_fsync_prepare ITTNOTIFY_VOID(fsync_prepare) +#define __itt_fsync_prepare_ptr ITTNOTIFY_NAME(fsync_prepare) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_prepare(addr) +#define __itt_fsync_prepare_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_prepare_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_cancel(void *addr); + */ +void ITTAPI __itt_fsync_cancel(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_cancel, (void *addr)) +#define __itt_fsync_cancel ITTNOTIFY_VOID(fsync_cancel) +#define __itt_fsync_cancel_ptr ITTNOTIFY_NAME(fsync_cancel) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_cancel(addr) +#define __itt_fsync_cancel_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_cancel_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_acquired(void *addr); + */ +void ITTAPI __itt_fsync_acquired(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_acquired, (void *addr)) +#define __itt_fsync_acquired ITTNOTIFY_VOID(fsync_acquired) +#define __itt_fsync_acquired_ptr ITTNOTIFY_NAME(fsync_acquired) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_acquired(addr) +#define __itt_fsync_acquired_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_acquired_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_releasing(void* addr); + */ +void ITTAPI __itt_fsync_releasing(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_releasing, (void *addr)) +#define __itt_fsync_releasing ITTNOTIFY_VOID(fsync_releasing) +#define __itt_fsync_releasing_ptr ITTNOTIFY_NAME(fsync_releasing) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_releasing(addr) +#define __itt_fsync_releasing_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_releasing_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} */ + +/** + * @defgroup model Modeling by Intel(R) Parallel Advisor + * @ingroup public + * This is the subset of itt used for modeling by Intel(R) Parallel Advisor. + * This API is called ONLY using annotate.h, by "Annotation" macros + * the user places in their sources during the parallelism modeling steps. + * + * site_begin/end and task_begin/end take the address of handle variables, + * which are writeable by the API. Handles must be 0 initialized prior + * to the first call to begin, or may cause a run-time failure. + * The handles are initialized in a multi-thread safe way by the API if + * the handle is 0. The commonly expected idiom is one static handle to + * identify a site or task. If a site or task of the same name has already + * been started during this collection, the same handle MAY be returned, + * but is not required to be - it is unspecified if data merging is done + * based on name. These routines also take an instance variable. Like + * the lexical instance, these must be 0 initialized. Unlike the lexical + * instance, this is used to track a single dynamic instance. + * + * API used by the Intel(R) Parallel Advisor to describe potential concurrency + * and related activities. User-added source annotations expand to calls + * to these procedures to enable modeling of a hypothetical concurrent + * execution serially. + * @{ + */ +#if !defined(_ADVISOR_ANNOTATE_H_) || defined(ANNOTATE_EXPAND_NULL) + +typedef void* __itt_model_site; /*!< @brief handle for lexical site */ +typedef void* __itt_model_site_instance; /*!< @brief handle for dynamic instance */ +typedef void* __itt_model_task; /*!< @brief handle for lexical site */ +typedef void* __itt_model_task_instance; /*!< @brief handle for dynamic instance */ + +/** + * @enum __itt_model_disable + * @brief Enumerator for the disable methods + */ +typedef enum { + __itt_model_disable_observation, + __itt_model_disable_collection +} __itt_model_disable; + +#endif /* !_ADVISOR_ANNOTATE_H_ || ANNOTATE_EXPAND_NULL */ + +/** + * @brief ANNOTATE_SITE_BEGIN/ANNOTATE_SITE_END support. + * + * site_begin/end model a potential concurrency site. + * site instances may be recursively nested with themselves. + * site_end exits the most recently started but unended site for the current + * thread. The handle passed to end may be used to validate structure. + * Instances of a site encountered on different threads concurrently + * are considered completely distinct. If the site name for two different + * lexical sites match, it is unspecified whether they are treated as the + * same or different for data presentation. + */ +void ITTAPI __itt_model_site_begin(__itt_model_site *site, __itt_model_site_instance *instance, const char *name); +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_model_site_beginW(const wchar_t *name); +#endif +void ITTAPI __itt_model_site_beginA(const char *name); +void ITTAPI __itt_model_site_beginAL(const char *name, size_t siteNameLen); +void ITTAPI __itt_model_site_end (__itt_model_site *site, __itt_model_site_instance *instance); +void ITTAPI __itt_model_site_end_2(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_site_begin, (__itt_model_site *site, __itt_model_site_instance *instance, const char *name)) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, model_site_beginW, (const wchar_t *name)) +#endif +ITT_STUBV(ITTAPI, void, model_site_beginA, (const char *name)) +ITT_STUBV(ITTAPI, void, model_site_beginAL, (const char *name, size_t siteNameLen)) +ITT_STUBV(ITTAPI, void, model_site_end, (__itt_model_site *site, __itt_model_site_instance *instance)) +ITT_STUBV(ITTAPI, void, model_site_end_2, (void)) +#define __itt_model_site_begin ITTNOTIFY_VOID(model_site_begin) +#define __itt_model_site_begin_ptr ITTNOTIFY_NAME(model_site_begin) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_site_beginW ITTNOTIFY_VOID(model_site_beginW) +#define __itt_model_site_beginW_ptr ITTNOTIFY_NAME(model_site_beginW) +#endif +#define __itt_model_site_beginA ITTNOTIFY_VOID(model_site_beginA) +#define __itt_model_site_beginA_ptr ITTNOTIFY_NAME(model_site_beginA) +#define __itt_model_site_beginAL ITTNOTIFY_VOID(model_site_beginAL) +#define __itt_model_site_beginAL_ptr ITTNOTIFY_NAME(model_site_beginAL) +#define __itt_model_site_end ITTNOTIFY_VOID(model_site_end) +#define __itt_model_site_end_ptr ITTNOTIFY_NAME(model_site_end) +#define __itt_model_site_end_2 ITTNOTIFY_VOID(model_site_end_2) +#define __itt_model_site_end_2_ptr ITTNOTIFY_NAME(model_site_end_2) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_site_begin(site, instance, name) +#define __itt_model_site_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_site_beginW(name) +#define __itt_model_site_beginW_ptr 0 +#endif +#define __itt_model_site_beginA(name) +#define __itt_model_site_beginA_ptr 0 +#define __itt_model_site_beginAL(name, siteNameLen) +#define __itt_model_site_beginAL_ptr 0 +#define __itt_model_site_end(site, instance) +#define __itt_model_site_end_ptr 0 +#define __itt_model_site_end_2() +#define __itt_model_site_end_2_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_site_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_site_beginW_ptr 0 +#endif +#define __itt_model_site_beginA_ptr 0 +#define __itt_model_site_beginAL_ptr 0 +#define __itt_model_site_end_ptr 0 +#define __itt_model_site_end_2_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_TASK_BEGIN/ANNOTATE_TASK_END support + * + * task_begin/end model a potential task, which is contained within the most + * closely enclosing dynamic site. task_end exits the most recently started + * but unended task. The handle passed to end may be used to validate + * structure. It is unspecified if bad dynamic nesting is detected. If it + * is, it should be encoded in the resulting data collection. The collector + * should not fail due to construct nesting issues, nor attempt to directly + * indicate the problem. + */ +void ITTAPI __itt_model_task_begin(__itt_model_task *task, __itt_model_task_instance *instance, const char *name); +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_model_task_beginW(const wchar_t *name); +void ITTAPI __itt_model_iteration_taskW(const wchar_t *name); +#endif +void ITTAPI __itt_model_task_beginA(const char *name); +void ITTAPI __itt_model_task_beginAL(const char *name, size_t taskNameLen); +void ITTAPI __itt_model_iteration_taskA(const char *name); +void ITTAPI __itt_model_iteration_taskAL(const char *name, size_t taskNameLen); +void ITTAPI __itt_model_task_end (__itt_model_task *task, __itt_model_task_instance *instance); +void ITTAPI __itt_model_task_end_2(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_task_begin, (__itt_model_task *task, __itt_model_task_instance *instance, const char *name)) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, model_task_beginW, (const wchar_t *name)) +ITT_STUBV(ITTAPI, void, model_iteration_taskW, (const wchar_t *name)) +#endif +ITT_STUBV(ITTAPI, void, model_task_beginA, (const char *name)) +ITT_STUBV(ITTAPI, void, model_task_beginAL, (const char *name, size_t taskNameLen)) +ITT_STUBV(ITTAPI, void, model_iteration_taskA, (const char *name)) +ITT_STUBV(ITTAPI, void, model_iteration_taskAL, (const char *name, size_t taskNameLen)) +ITT_STUBV(ITTAPI, void, model_task_end, (__itt_model_task *task, __itt_model_task_instance *instance)) +ITT_STUBV(ITTAPI, void, model_task_end_2, (void)) +#define __itt_model_task_begin ITTNOTIFY_VOID(model_task_begin) +#define __itt_model_task_begin_ptr ITTNOTIFY_NAME(model_task_begin) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_task_beginW ITTNOTIFY_VOID(model_task_beginW) +#define __itt_model_task_beginW_ptr ITTNOTIFY_NAME(model_task_beginW) +#define __itt_model_iteration_taskW ITTNOTIFY_VOID(model_iteration_taskW) +#define __itt_model_iteration_taskW_ptr ITTNOTIFY_NAME(model_iteration_taskW) +#endif +#define __itt_model_task_beginA ITTNOTIFY_VOID(model_task_beginA) +#define __itt_model_task_beginA_ptr ITTNOTIFY_NAME(model_task_beginA) +#define __itt_model_task_beginAL ITTNOTIFY_VOID(model_task_beginAL) +#define __itt_model_task_beginAL_ptr ITTNOTIFY_NAME(model_task_beginAL) +#define __itt_model_iteration_taskA ITTNOTIFY_VOID(model_iteration_taskA) +#define __itt_model_iteration_taskA_ptr ITTNOTIFY_NAME(model_iteration_taskA) +#define __itt_model_iteration_taskAL ITTNOTIFY_VOID(model_iteration_taskAL) +#define __itt_model_iteration_taskAL_ptr ITTNOTIFY_NAME(model_iteration_taskAL) +#define __itt_model_task_end ITTNOTIFY_VOID(model_task_end) +#define __itt_model_task_end_ptr ITTNOTIFY_NAME(model_task_end) +#define __itt_model_task_end_2 ITTNOTIFY_VOID(model_task_end_2) +#define __itt_model_task_end_2_ptr ITTNOTIFY_NAME(model_task_end_2) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_task_begin(task, instance, name) +#define __itt_model_task_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_task_beginW(name) +#define __itt_model_task_beginW_ptr 0 +#endif +#define __itt_model_task_beginA(name) +#define __itt_model_task_beginA_ptr 0 +#define __itt_model_task_beginAL(name, siteNameLen) +#define __itt_model_task_beginAL_ptr 0 +#define __itt_model_iteration_taskA(name) +#define __itt_model_iteration_taskA_ptr 0 +#define __itt_model_iteration_taskAL(name, siteNameLen) +#define __itt_model_iteration_taskAL_ptr 0 +#define __itt_model_task_end(task, instance) +#define __itt_model_task_end_ptr 0 +#define __itt_model_task_end_2() +#define __itt_model_task_end_2_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_task_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_task_beginW_ptr 0 +#endif +#define __itt_model_task_beginA_ptr 0 +#define __itt_model_task_beginAL_ptr 0 +#define __itt_model_iteration_taskA_ptr 0 +#define __itt_model_iteration_taskAL_ptr 0 +#define __itt_model_task_end_ptr 0 +#define __itt_model_task_end_2_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_LOCK_ACQUIRE/ANNOTATE_LOCK_RELEASE support + * + * lock_acquire/release model a potential lock for both lockset and + * performance modeling. Each unique address is modeled as a separate + * lock, with invalid addresses being valid lock IDs. Specifically: + * no storage is accessed by the API at the specified address - it is only + * used for lock identification. Lock acquires may be self-nested and are + * unlocked by a corresponding number of releases. + * (These closely correspond to __itt_sync_acquired/__itt_sync_releasing, + * but may not have identical semantics.) + */ +void ITTAPI __itt_model_lock_acquire(void *lock); +void ITTAPI __itt_model_lock_acquire_2(void *lock); +void ITTAPI __itt_model_lock_release(void *lock); +void ITTAPI __itt_model_lock_release_2(void *lock); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_lock_acquire, (void *lock)) +ITT_STUBV(ITTAPI, void, model_lock_acquire_2, (void *lock)) +ITT_STUBV(ITTAPI, void, model_lock_release, (void *lock)) +ITT_STUBV(ITTAPI, void, model_lock_release_2, (void *lock)) +#define __itt_model_lock_acquire ITTNOTIFY_VOID(model_lock_acquire) +#define __itt_model_lock_acquire_ptr ITTNOTIFY_NAME(model_lock_acquire) +#define __itt_model_lock_acquire_2 ITTNOTIFY_VOID(model_lock_acquire_2) +#define __itt_model_lock_acquire_2_ptr ITTNOTIFY_NAME(model_lock_acquire_2) +#define __itt_model_lock_release ITTNOTIFY_VOID(model_lock_release) +#define __itt_model_lock_release_ptr ITTNOTIFY_NAME(model_lock_release) +#define __itt_model_lock_release_2 ITTNOTIFY_VOID(model_lock_release_2) +#define __itt_model_lock_release_2_ptr ITTNOTIFY_NAME(model_lock_release_2) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_lock_acquire(lock) +#define __itt_model_lock_acquire_ptr 0 +#define __itt_model_lock_acquire_2(lock) +#define __itt_model_lock_acquire_2_ptr 0 +#define __itt_model_lock_release(lock) +#define __itt_model_lock_release_ptr 0 +#define __itt_model_lock_release_2(lock) +#define __itt_model_lock_release_2_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_lock_acquire_ptr 0 +#define __itt_model_lock_acquire_2_ptr 0 +#define __itt_model_lock_release_ptr 0 +#define __itt_model_lock_release_2_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_RECORD_ALLOCATION/ANNOTATE_RECORD_DEALLOCATION support + * + * record_allocation/deallocation describe user-defined memory allocator + * behavior, which may be required for correctness modeling to understand + * when storage is not expected to be actually reused across threads. + */ +void ITTAPI __itt_model_record_allocation (void *addr, size_t size); +void ITTAPI __itt_model_record_deallocation(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_record_allocation, (void *addr, size_t size)) +ITT_STUBV(ITTAPI, void, model_record_deallocation, (void *addr)) +#define __itt_model_record_allocation ITTNOTIFY_VOID(model_record_allocation) +#define __itt_model_record_allocation_ptr ITTNOTIFY_NAME(model_record_allocation) +#define __itt_model_record_deallocation ITTNOTIFY_VOID(model_record_deallocation) +#define __itt_model_record_deallocation_ptr ITTNOTIFY_NAME(model_record_deallocation) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_record_allocation(addr, size) +#define __itt_model_record_allocation_ptr 0 +#define __itt_model_record_deallocation(addr) +#define __itt_model_record_deallocation_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_record_allocation_ptr 0 +#define __itt_model_record_deallocation_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_INDUCTION_USES support + * + * Note particular storage is inductive through the end of the current site + */ +void ITTAPI __itt_model_induction_uses(void* addr, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_induction_uses, (void *addr, size_t size)) +#define __itt_model_induction_uses ITTNOTIFY_VOID(model_induction_uses) +#define __itt_model_induction_uses_ptr ITTNOTIFY_NAME(model_induction_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_induction_uses(addr, size) +#define __itt_model_induction_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_induction_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_REDUCTION_USES support + * + * Note particular storage is used for reduction through the end + * of the current site + */ +void ITTAPI __itt_model_reduction_uses(void* addr, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_reduction_uses, (void *addr, size_t size)) +#define __itt_model_reduction_uses ITTNOTIFY_VOID(model_reduction_uses) +#define __itt_model_reduction_uses_ptr ITTNOTIFY_NAME(model_reduction_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_reduction_uses(addr, size) +#define __itt_model_reduction_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_reduction_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_OBSERVE_USES support + * + * Have correctness modeling record observations about uses of storage + * through the end of the current site + */ +void ITTAPI __itt_model_observe_uses(void* addr, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_observe_uses, (void *addr, size_t size)) +#define __itt_model_observe_uses ITTNOTIFY_VOID(model_observe_uses) +#define __itt_model_observe_uses_ptr ITTNOTIFY_NAME(model_observe_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_observe_uses(addr, size) +#define __itt_model_observe_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_observe_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_CLEAR_USES support + * + * Clear the special handling of a piece of storage related to induction, + * reduction or observe_uses + */ +void ITTAPI __itt_model_clear_uses(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_clear_uses, (void *addr)) +#define __itt_model_clear_uses ITTNOTIFY_VOID(model_clear_uses) +#define __itt_model_clear_uses_ptr ITTNOTIFY_NAME(model_clear_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_clear_uses(addr) +#define __itt_model_clear_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_clear_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_DISABLE_*_PUSH/ANNOTATE_DISABLE_*_POP support + * + * disable_push/disable_pop push and pop disabling based on a parameter. + * Disabling observations stops processing of memory references during + * correctness modeling, and all annotations that occur in the disabled + * region. This allows description of code that is expected to be handled + * specially during conversion to parallelism or that is not recognized + * by tools (e.g. some kinds of synchronization operations.) + * This mechanism causes all annotations in the disabled region, other + * than disable_push and disable_pop, to be ignored. (For example, this + * might validly be used to disable an entire parallel site and the contained + * tasks and locking in it for data collection purposes.) + * The disable for collection is a more expensive operation, but reduces + * collector overhead significantly. This applies to BOTH correctness data + * collection and performance data collection. For example, a site + * containing a task might only enable data collection for the first 10 + * iterations. Both performance and correctness data should reflect this, + * and the program should run as close to full speed as possible when + * collection is disabled. + */ +void ITTAPI __itt_model_disable_push(__itt_model_disable x); +void ITTAPI __itt_model_disable_pop(void); +void ITTAPI __itt_model_aggregate_task(size_t x); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_disable_push, (__itt_model_disable x)) +ITT_STUBV(ITTAPI, void, model_disable_pop, (void)) +ITT_STUBV(ITTAPI, void, model_aggregate_task, (size_t x)) +#define __itt_model_disable_push ITTNOTIFY_VOID(model_disable_push) +#define __itt_model_disable_push_ptr ITTNOTIFY_NAME(model_disable_push) +#define __itt_model_disable_pop ITTNOTIFY_VOID(model_disable_pop) +#define __itt_model_disable_pop_ptr ITTNOTIFY_NAME(model_disable_pop) +#define __itt_model_aggregate_task ITTNOTIFY_VOID(model_aggregate_task) +#define __itt_model_aggregate_task_ptr ITTNOTIFY_NAME(model_aggregate_task) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_disable_push(x) +#define __itt_model_disable_push_ptr 0 +#define __itt_model_disable_pop() +#define __itt_model_disable_pop_ptr 0 +#define __itt_model_aggregate_task(x) +#define __itt_model_aggregate_task_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_disable_push_ptr 0 +#define __itt_model_disable_pop_ptr 0 +#define __itt_model_aggregate_task_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} model group */ + +/** + * @defgroup heap Heap + * @ingroup public + * Heap group + * @{ + */ + +typedef void* __itt_heap_function; + +/** + * @brief Create an identification for heap function + * @return non-zero identifier or NULL + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_heap_function ITTAPI __itt_heap_function_createA(const char* name, const char* domain); +__itt_heap_function ITTAPI __itt_heap_function_createW(const wchar_t* name, const wchar_t* domain); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_heap_function_create __itt_heap_function_createW +# define __itt_heap_function_create_ptr __itt_heap_function_createW_ptr +#else +# define __itt_heap_function_create __itt_heap_function_createA +# define __itt_heap_function_create_ptr __itt_heap_function_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_heap_function ITTAPI __itt_heap_function_create(const char* name, const char* domain); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_heap_function, heap_function_createA, (const char* name, const char* domain)) +ITT_STUB(ITTAPI, __itt_heap_function, heap_function_createW, (const wchar_t* name, const wchar_t* domain)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_heap_function, heap_function_create, (const char* name, const char* domain)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_heap_function_createA ITTNOTIFY_DATA(heap_function_createA) +#define __itt_heap_function_createA_ptr ITTNOTIFY_NAME(heap_function_createA) +#define __itt_heap_function_createW ITTNOTIFY_DATA(heap_function_createW) +#define __itt_heap_function_createW_ptr ITTNOTIFY_NAME(heap_function_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_heap_function_create ITTNOTIFY_DATA(heap_function_create) +#define __itt_heap_function_create_ptr ITTNOTIFY_NAME(heap_function_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_heap_function_createA(name, domain) (__itt_heap_function)0 +#define __itt_heap_function_createA_ptr 0 +#define __itt_heap_function_createW(name, domain) (__itt_heap_function)0 +#define __itt_heap_function_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_heap_function_create(name, domain) (__itt_heap_function)0 +#define __itt_heap_function_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_heap_function_createA_ptr 0 +#define __itt_heap_function_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_heap_function_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an allocation begin occurrence. + */ +void ITTAPI __itt_heap_allocate_begin(__itt_heap_function h, size_t size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_allocate_begin, (__itt_heap_function h, size_t size, int initialized)) +#define __itt_heap_allocate_begin ITTNOTIFY_VOID(heap_allocate_begin) +#define __itt_heap_allocate_begin_ptr ITTNOTIFY_NAME(heap_allocate_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_allocate_begin(h, size, initialized) +#define __itt_heap_allocate_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_allocate_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an allocation end occurrence. + */ +void ITTAPI __itt_heap_allocate_end(__itt_heap_function h, void** addr, size_t size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_allocate_end, (__itt_heap_function h, void** addr, size_t size, int initialized)) +#define __itt_heap_allocate_end ITTNOTIFY_VOID(heap_allocate_end) +#define __itt_heap_allocate_end_ptr ITTNOTIFY_NAME(heap_allocate_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_allocate_end(h, addr, size, initialized) +#define __itt_heap_allocate_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_allocate_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an free begin occurrence. + */ +void ITTAPI __itt_heap_free_begin(__itt_heap_function h, void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_free_begin, (__itt_heap_function h, void* addr)) +#define __itt_heap_free_begin ITTNOTIFY_VOID(heap_free_begin) +#define __itt_heap_free_begin_ptr ITTNOTIFY_NAME(heap_free_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_free_begin(h, addr) +#define __itt_heap_free_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_free_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an free end occurrence. + */ +void ITTAPI __itt_heap_free_end(__itt_heap_function h, void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_free_end, (__itt_heap_function h, void* addr)) +#define __itt_heap_free_end ITTNOTIFY_VOID(heap_free_end) +#define __itt_heap_free_end_ptr ITTNOTIFY_NAME(heap_free_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_free_end(h, addr) +#define __itt_heap_free_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_free_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an reallocation begin occurrence. + */ +void ITTAPI __itt_heap_reallocate_begin(__itt_heap_function h, void* addr, size_t new_size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_reallocate_begin, (__itt_heap_function h, void* addr, size_t new_size, int initialized)) +#define __itt_heap_reallocate_begin ITTNOTIFY_VOID(heap_reallocate_begin) +#define __itt_heap_reallocate_begin_ptr ITTNOTIFY_NAME(heap_reallocate_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_reallocate_begin(h, addr, new_size, initialized) +#define __itt_heap_reallocate_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_reallocate_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an reallocation end occurrence. + */ +void ITTAPI __itt_heap_reallocate_end(__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_reallocate_end, (__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized)) +#define __itt_heap_reallocate_end ITTNOTIFY_VOID(heap_reallocate_end) +#define __itt_heap_reallocate_end_ptr ITTNOTIFY_NAME(heap_reallocate_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_reallocate_end(h, addr, new_addr, new_size, initialized) +#define __itt_heap_reallocate_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_reallocate_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief internal access begin */ +void ITTAPI __itt_heap_internal_access_begin(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_internal_access_begin, (void)) +#define __itt_heap_internal_access_begin ITTNOTIFY_VOID(heap_internal_access_begin) +#define __itt_heap_internal_access_begin_ptr ITTNOTIFY_NAME(heap_internal_access_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_internal_access_begin() +#define __itt_heap_internal_access_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_internal_access_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief internal access end */ +void ITTAPI __itt_heap_internal_access_end(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_internal_access_end, (void)) +#define __itt_heap_internal_access_end ITTNOTIFY_VOID(heap_internal_access_end) +#define __itt_heap_internal_access_end_ptr ITTNOTIFY_NAME(heap_internal_access_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_internal_access_end() +#define __itt_heap_internal_access_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_internal_access_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief record memory growth begin */ +void ITTAPI __itt_heap_record_memory_growth_begin(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_record_memory_growth_begin, (void)) +#define __itt_heap_record_memory_growth_begin ITTNOTIFY_VOID(heap_record_memory_growth_begin) +#define __itt_heap_record_memory_growth_begin_ptr ITTNOTIFY_NAME(heap_record_memory_growth_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_record_memory_growth_begin() +#define __itt_heap_record_memory_growth_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_record_memory_growth_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief record memory growth end */ +void ITTAPI __itt_heap_record_memory_growth_end(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_record_memory_growth_end, (void)) +#define __itt_heap_record_memory_growth_end ITTNOTIFY_VOID(heap_record_memory_growth_end) +#define __itt_heap_record_memory_growth_end_ptr ITTNOTIFY_NAME(heap_record_memory_growth_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_record_memory_growth_end() +#define __itt_heap_record_memory_growth_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_record_memory_growth_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Specify the type of heap detection/reporting to modify. + */ +/** + * @hideinitializer + * @brief Report on memory leaks. + */ +#define __itt_heap_leaks 0x00000001 + +/** + * @hideinitializer + * @brief Report on memory growth. + */ +#define __itt_heap_growth 0x00000002 + + +/** @brief heap reset detection */ +void ITTAPI __itt_heap_reset_detection(unsigned int reset_mask); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_reset_detection, (unsigned int reset_mask)) +#define __itt_heap_reset_detection ITTNOTIFY_VOID(heap_reset_detection) +#define __itt_heap_reset_detection_ptr ITTNOTIFY_NAME(heap_reset_detection) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_reset_detection() +#define __itt_heap_reset_detection_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_reset_detection_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief report */ +void ITTAPI __itt_heap_record(unsigned int record_mask); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_record, (unsigned int record_mask)) +#define __itt_heap_record ITTNOTIFY_VOID(heap_record) +#define __itt_heap_record_ptr ITTNOTIFY_NAME(heap_record) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_record() +#define __itt_heap_record_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_record_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} heap group */ +/** @endcond */ +/* ========================================================================== */ + +/** + * @defgroup domains Domains + * @ingroup public + * Domains group + * @{ + */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_domain +{ + volatile int flags; /*!< Zero if disabled, non-zero if enabled. The meaning of different non-zero values is reserved to the runtime */ + const char* nameA; /*!< Copy of original name in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* nameW; /*!< Copy of original name in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* nameW; +#endif /* UNICODE || _UNICODE */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_domain* next; +} __itt_domain; + +#pragma pack(pop) +/** @endcond */ + +/** + * @ingroup domains + * @brief Create a domain. + * Create domain using some domain name: the URI naming style is recommended. + * Because the set of domains is expected to be static over the application's + * execution time, there is no mechanism to destroy a domain. + * Any domain can be accessed by any thread in the process, regardless of + * which thread created the domain. This call is thread-safe. + * @param[in] name name of domain + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_domain* ITTAPI __itt_domain_createA(const char *name); +__itt_domain* ITTAPI __itt_domain_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_domain_create __itt_domain_createW +# define __itt_domain_create_ptr __itt_domain_createW_ptr +#else /* UNICODE */ +# define __itt_domain_create __itt_domain_createA +# define __itt_domain_create_ptr __itt_domain_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_domain* ITTAPI __itt_domain_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_domain*, domain_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_domain*, domain_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_domain*, domain_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_domain_createA ITTNOTIFY_DATA(domain_createA) +#define __itt_domain_createA_ptr ITTNOTIFY_NAME(domain_createA) +#define __itt_domain_createW ITTNOTIFY_DATA(domain_createW) +#define __itt_domain_createW_ptr ITTNOTIFY_NAME(domain_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_domain_create ITTNOTIFY_DATA(domain_create) +#define __itt_domain_create_ptr ITTNOTIFY_NAME(domain_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_domain_createA(name) (__itt_domain*)0 +#define __itt_domain_createA_ptr 0 +#define __itt_domain_createW(name) (__itt_domain*)0 +#define __itt_domain_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_domain_create(name) (__itt_domain*)0 +#define __itt_domain_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_domain_createA_ptr 0 +#define __itt_domain_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_domain_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} domains group */ + +/** + * @defgroup ids IDs + * @ingroup public + * IDs group + * @{ + */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_id +{ + unsigned long long d1, d2, d3; +} __itt_id; + +#pragma pack(pop) +/** @endcond */ + +const __itt_id __itt_null = { 0, 0, 0 }; + +/** + * @ingroup ids + * @brief A convenience function is provided to create an ID without domain control. + * @brief This is a convenience function to initialize an __itt_id structure. This function + * does not affect the collector runtime in any way. After you make the ID with this + * function, you still must create it with the __itt_id_create function before using the ID + * to identify a named entity. + * @param[in] addr The address of object; high QWORD of the ID value. + * @param[in] extra The extra data to unique identify object; low QWORD of the ID value. + */ + +ITT_INLINE __itt_id ITTAPI __itt_id_make(void* addr, unsigned long long extra) ITT_INLINE_ATTRIBUTE; +ITT_INLINE __itt_id ITTAPI __itt_id_make(void* addr, unsigned long long extra) +{ + __itt_id id = __itt_null; + id.d1 = (unsigned long long)((uintptr_t)addr); + id.d2 = (unsigned long long)extra; + id.d3 = (unsigned long long)0; /* Reserved. Must be zero */ + return id; +} + +/** + * @ingroup ids + * @brief Create an instance of identifier. + * This establishes the beginning of the lifetime of an instance of + * the given ID in the trace. Once this lifetime starts, the ID + * can be used to tag named entity instances in calls such as + * __itt_task_begin, and to specify relationships among + * identified named entity instances, using the \ref relations APIs. + * Instance IDs are not domain specific! + * @param[in] domain The domain controlling the execution of this call. + * @param[in] id The ID to create. + */ +void ITTAPI __itt_id_create(const __itt_domain *domain, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, id_create, (const __itt_domain *domain, __itt_id id)) +#define __itt_id_create(d,x) ITTNOTIFY_VOID_D1(id_create,d,x) +#define __itt_id_create_ptr ITTNOTIFY_NAME(id_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_id_create(domain,id) +#define __itt_id_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_id_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup ids + * @brief Destroy an instance of identifier. + * This ends the lifetime of the current instance of the given ID value in the trace. + * Any relationships that are established after this lifetime ends are invalid. + * This call must be performed before the given ID value can be reused for a different + * named entity instance. + * @param[in] domain The domain controlling the execution of this call. + * @param[in] id The ID to destroy. + */ +void ITTAPI __itt_id_destroy(const __itt_domain *domain, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, id_destroy, (const __itt_domain *domain, __itt_id id)) +#define __itt_id_destroy(d,x) ITTNOTIFY_VOID_D1(id_destroy,d,x) +#define __itt_id_destroy_ptr ITTNOTIFY_NAME(id_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_id_destroy(domain,id) +#define __itt_id_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_id_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} ids group */ + +/** + * @defgroup handless String Handles + * @ingroup public + * String Handles group + * @{ + */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_string_handle +{ + const char* strA; /*!< Copy of original string in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* strW; /*!< Copy of original string in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* strW; +#endif /* UNICODE || _UNICODE */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_string_handle* next; +} __itt_string_handle; + +#pragma pack(pop) +/** @endcond */ + +/** + * @ingroup handles + * @brief Create a string handle. + * Create and return handle value that can be associated with a string. + * Consecutive calls to __itt_string_handle_create with the same name + * return the same value. Because the set of string handles is expected to remain + * static during the application's execution time, there is no mechanism to destroy a string handle. + * Any string handle can be accessed by any thread in the process, regardless of which thread created + * the string handle. This call is thread-safe. + * @param[in] name The input string + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_string_handle* ITTAPI __itt_string_handle_createA(const char *name); +__itt_string_handle* ITTAPI __itt_string_handle_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_string_handle_create __itt_string_handle_createW +# define __itt_string_handle_create_ptr __itt_string_handle_createW_ptr +#else /* UNICODE */ +# define __itt_string_handle_create __itt_string_handle_createA +# define __itt_string_handle_create_ptr __itt_string_handle_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_string_handle* ITTAPI __itt_string_handle_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_string_handle_createA ITTNOTIFY_DATA(string_handle_createA) +#define __itt_string_handle_createA_ptr ITTNOTIFY_NAME(string_handle_createA) +#define __itt_string_handle_createW ITTNOTIFY_DATA(string_handle_createW) +#define __itt_string_handle_createW_ptr ITTNOTIFY_NAME(string_handle_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_string_handle_create ITTNOTIFY_DATA(string_handle_create) +#define __itt_string_handle_create_ptr ITTNOTIFY_NAME(string_handle_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_string_handle_createA(name) (__itt_string_handle*)0 +#define __itt_string_handle_createA_ptr 0 +#define __itt_string_handle_createW(name) (__itt_string_handle*)0 +#define __itt_string_handle_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_string_handle_create(name) (__itt_string_handle*)0 +#define __itt_string_handle_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_string_handle_createA_ptr 0 +#define __itt_string_handle_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_string_handle_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} handles group */ + +/** @cond exclude_from_documentation */ +typedef unsigned long long __itt_timestamp; +/** @endcond */ + +#define __itt_timestamp_none ((__itt_timestamp)-1LL) + +/** @cond exclude_from_gpa_documentation */ + +/** + * @ingroup timestamps + * @brief Return timestamp corresponding to the current moment. + * This returns the timestamp in the format that is the most relevant for the current + * host or platform (RDTSC, QPC, and others). You can use the "<" operator to + * compare __itt_timestamp values. + */ +__itt_timestamp ITTAPI __itt_get_timestamp(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_timestamp, get_timestamp, (void)) +#define __itt_get_timestamp ITTNOTIFY_DATA(get_timestamp) +#define __itt_get_timestamp_ptr ITTNOTIFY_NAME(get_timestamp) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_get_timestamp() +#define __itt_get_timestamp_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_get_timestamp_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} timestamps */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** + * @defgroup regions Regions + * @ingroup public + * Regions group + * @{ + */ +/** + * @ingroup regions + * @brief Begin of region instance. + * Successive calls to __itt_region_begin with the same ID are ignored + * until a call to __itt_region_end with the same ID + * @param[in] domain The domain for this region instance + * @param[in] id The instance ID for this region instance. Must not be __itt_null + * @param[in] parentid The instance ID for the parent of this region instance, or __itt_null + * @param[in] name The name of this region + */ +void ITTAPI __itt_region_begin(const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name); + +/** + * @ingroup regions + * @brief End of region instance. + * The first call to __itt_region_end with a given ID ends the + * region. Successive calls with the same ID are ignored, as are + * calls that do not have a matching __itt_region_begin call. + * @param[in] domain The domain for this region instance + * @param[in] id The instance ID for this region instance + */ +void ITTAPI __itt_region_end(const __itt_domain *domain, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, region_begin, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, region_end, (const __itt_domain *domain, __itt_id id)) +#define __itt_region_begin(d,x,y,z) ITTNOTIFY_VOID_D3(region_begin,d,x,y,z) +#define __itt_region_begin_ptr ITTNOTIFY_NAME(region_begin) +#define __itt_region_end(d,x) ITTNOTIFY_VOID_D1(region_end,d,x) +#define __itt_region_end_ptr ITTNOTIFY_NAME(region_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_region_begin(d,x,y,z) +#define __itt_region_begin_ptr 0 +#define __itt_region_end(d,x) +#define __itt_region_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_region_begin_ptr 0 +#define __itt_region_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} regions group */ + +/** + * @defgroup frames Frames + * @ingroup public + * Frames are similar to regions, but are intended to be easier to use and to implement. + * In particular: + * - Frames always represent periods of elapsed time + * - By default, frames have no nesting relationships + * @{ + */ + +/** + * @ingroup frames + * @brief Begin a frame instance. + * Successive calls to __itt_frame_begin with the + * same ID are ignored until a call to __itt_frame_end with the same ID. + * @param[in] domain The domain for this frame instance + * @param[in] id The instance ID for this frame instance or NULL + */ +void ITTAPI __itt_frame_begin_v3(const __itt_domain *domain, __itt_id *id); + +/** + * @ingroup frames + * @brief End a frame instance. + * The first call to __itt_frame_end with a given ID + * ends the frame. Successive calls with the same ID are ignored, as are + * calls that do not have a matching __itt_frame_begin call. + * @param[in] domain The domain for this frame instance + * @param[in] id The instance ID for this frame instance or NULL for current + */ +void ITTAPI __itt_frame_end_v3(const __itt_domain *domain, __itt_id *id); + +/** + * @ingroup frames + * @brief Submits a frame instance. + * Successive calls to __itt_frame_begin or __itt_frame_submit with the + * same ID are ignored until a call to __itt_frame_end or __itt_frame_submit + * with the same ID. + * Passing special __itt_timestamp_none value as "end" argument means + * take the current timestamp as the end timestamp. + * @param[in] domain The domain for this frame instance + * @param[in] id The instance ID for this frame instance or NULL + * @param[in] begin Timestamp of the beginning of the frame + * @param[in] end Timestamp of the end of the frame + */ +void ITTAPI __itt_frame_submit_v3(const __itt_domain *domain, __itt_id *id, + __itt_timestamp begin, __itt_timestamp end); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, frame_begin_v3, (const __itt_domain *domain, __itt_id *id)) +ITT_STUBV(ITTAPI, void, frame_end_v3, (const __itt_domain *domain, __itt_id *id)) +ITT_STUBV(ITTAPI, void, frame_submit_v3, (const __itt_domain *domain, __itt_id *id, __itt_timestamp begin, __itt_timestamp end)) +#define __itt_frame_begin_v3(d,x) ITTNOTIFY_VOID_D1(frame_begin_v3,d,x) +#define __itt_frame_begin_v3_ptr ITTNOTIFY_NAME(frame_begin_v3) +#define __itt_frame_end_v3(d,x) ITTNOTIFY_VOID_D1(frame_end_v3,d,x) +#define __itt_frame_end_v3_ptr ITTNOTIFY_NAME(frame_end_v3) +#define __itt_frame_submit_v3(d,x,b,e) ITTNOTIFY_VOID_D3(frame_submit_v3,d,x,b,e) +#define __itt_frame_submit_v3_ptr ITTNOTIFY_NAME(frame_submit_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_frame_begin_v3(domain,id) +#define __itt_frame_begin_v3_ptr 0 +#define __itt_frame_end_v3(domain,id) +#define __itt_frame_end_v3_ptr 0 +#define __itt_frame_submit_v3(domain,id,begin,end) +#define __itt_frame_submit_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_frame_begin_v3_ptr 0 +#define __itt_frame_end_v3_ptr 0 +#define __itt_frame_submit_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} frames group */ +/** @endcond */ + +/** + * @defgroup taskgroup Task Group + * @ingroup public + * Task Group + * @{ + */ +/** + * @ingroup task_groups + * @brief Denotes a task_group instance. + * Successive calls to __itt_task_group with the same ID are ignored. + * @param[in] domain The domain for this task_group instance + * @param[in] id The instance ID for this task_group instance. Must not be __itt_null. + * @param[in] parentid The instance ID for the parent of this task_group instance, or __itt_null. + * @param[in] name The name of this task_group + */ +void ITTAPI __itt_task_group(const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_group, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +#define __itt_task_group(d,x,y,z) ITTNOTIFY_VOID_D3(task_group,d,x,y,z) +#define __itt_task_group_ptr ITTNOTIFY_NAME(task_group) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_group(d,x,y,z) +#define __itt_task_group_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_group_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} taskgroup group */ + +/** + * @defgroup tasks Tasks + * @ingroup public + * A task instance represents a piece of work performed by a particular + * thread for a period of time. A call to __itt_task_begin creates a + * task instance. This becomes the current instance for that task on that + * thread. A following call to __itt_task_end on the same thread ends the + * instance. There may be multiple simultaneous instances of tasks with the + * same name on different threads. If an ID is specified, the task instance + * receives that ID. Nested tasks are allowed. + * + * Note: The task is defined by the bracketing of __itt_task_begin and + * __itt_task_end on the same thread. If some scheduling mechanism causes + * task switching (the thread executes a different user task) or task + * switching (the user task switches to a different thread) then this breaks + * the notion of current instance. Additional API calls are required to + * deal with that possibility. + * @{ + */ + +/** + * @ingroup tasks + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] taskid The instance ID for this task instance, or __itt_null + * @param[in] parentid The parent instance to which this task instance belongs, or __itt_null + * @param[in] name The name of this task + */ +void ITTAPI __itt_task_begin(const __itt_domain *domain, __itt_id taskid, __itt_id parentid, __itt_string_handle *name); + +/** + * @ingroup tasks + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] taskid The identifier for this task instance (may be 0) + * @param[in] parentid The parent of this task (may be 0) + * @param[in] fn The pointer to the function you are tracing + */ +void ITTAPI __itt_task_begin_fn(const __itt_domain *domain, __itt_id taskid, __itt_id parentid, void* fn); + +/** + * @ingroup tasks + * @brief End the current task instance. + * @param[in] domain The domain for this task + */ +void ITTAPI __itt_task_end(const __itt_domain *domain); + +/** + * @ingroup tasks + * @brief Begin an overlapped task instance. + * @param[in] domain The domain for this task. + * @param[in] taskid The identifier for this task instance, *cannot* be __itt_null. + * @param[in] parentid The parent of this task, or __itt_null. + * @param[in] name The name of this task. + */ +void ITTAPI __itt_task_begin_overlapped(const __itt_domain* domain, __itt_id taskid, __itt_id parentid, __itt_string_handle* name); + +/** + * @ingroup tasks + * @brief End an overlapped task instance. + * @param[in] domain The domain for this task + * @param[in] taskid Explicit ID of finished task + */ +void ITTAPI __itt_task_end_overlapped(const __itt_domain *domain, __itt_id taskid); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_begin, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, task_begin_fn, (const __itt_domain *domain, __itt_id id, __itt_id parentid, void* fn)) +ITT_STUBV(ITTAPI, void, task_end, (const __itt_domain *domain)) +ITT_STUBV(ITTAPI, void, task_begin_overlapped, (const __itt_domain *domain, __itt_id taskid, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, task_end_overlapped, (const __itt_domain *domain, __itt_id taskid)) +#define __itt_task_begin(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin,d,x,y,z) +#define __itt_task_begin_ptr ITTNOTIFY_NAME(task_begin) +#define __itt_task_begin_fn(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin_fn,d,x,y,z) +#define __itt_task_begin_fn_ptr ITTNOTIFY_NAME(task_begin_fn) +#define __itt_task_end(d) ITTNOTIFY_VOID_D0(task_end,d) +#define __itt_task_end_ptr ITTNOTIFY_NAME(task_end) +#define __itt_task_begin_overlapped(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin_overlapped,d,x,y,z) +#define __itt_task_begin_overlapped_ptr ITTNOTIFY_NAME(task_begin_overlapped) +#define __itt_task_end_overlapped(d,x) ITTNOTIFY_VOID_D1(task_end_overlapped,d,x) +#define __itt_task_end_overlapped_ptr ITTNOTIFY_NAME(task_end_overlapped) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_begin(domain,id,parentid,name) +#define __itt_task_begin_ptr 0 +#define __itt_task_begin_fn(domain,id,parentid,fn) +#define __itt_task_begin_fn_ptr 0 +#define __itt_task_end(domain) +#define __itt_task_end_ptr 0 +#define __itt_task_begin_overlapped(domain,taskid,parentid,name) +#define __itt_task_begin_overlapped_ptr 0 +#define __itt_task_end_overlapped(domain,taskid) +#define __itt_task_end_overlapped_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_begin_ptr 0 +#define __itt_task_begin_fn_ptr 0 +#define __itt_task_end_ptr 0 +#define __itt_task_begin_overlapped_ptr 0 +#define __itt_task_end_overlapped_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} tasks group */ + + +/** + * @defgroup markers Markers + * Markers represent a single discreet event in time. Markers have a scope, + * described by an enumerated type __itt_scope. Markers are created by + * the API call __itt_marker. A marker instance can be given an ID for use in + * adding metadata. + * @{ + */ + +/** + * @brief Describes the scope of an event object in the trace. + */ +typedef enum +{ + __itt_scope_unknown = 0, + __itt_scope_global, + __itt_scope_track_group, + __itt_scope_track, + __itt_scope_task, + __itt_scope_marker +} __itt_scope; + +/** @cond exclude_from_documentation */ +#define __itt_marker_scope_unknown __itt_scope_unknown +#define __itt_marker_scope_global __itt_scope_global +#define __itt_marker_scope_process __itt_scope_track_group +#define __itt_marker_scope_thread __itt_scope_track +#define __itt_marker_scope_task __itt_scope_task +/** @endcond */ + +/** + * @ingroup markers + * @brief Create a marker instance + * @param[in] domain The domain for this marker + * @param[in] id The instance ID for this marker or __itt_null + * @param[in] name The name for this marker + * @param[in] scope The scope for this marker + */ +void ITTAPI __itt_marker(const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, marker, (const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope)) +#define __itt_marker(d,x,y,z) ITTNOTIFY_VOID_D3(marker,d,x,y,z) +#define __itt_marker_ptr ITTNOTIFY_NAME(marker) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_marker(domain,id,name,scope) +#define __itt_marker_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_marker_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} markers group */ + +/** + * @defgroup metadata Metadata + * The metadata API is used to attach extra information to named + * entities. Metadata can be attached to an identified named entity by ID, + * or to the current entity (which is always a task). + * + * Conceptually metadata has a type (what kind of metadata), a key (the + * name of the metadata), and a value (the actual data). The encoding of + * the value depends on the type of the metadata. + * + * The type of metadata is specified by an enumerated type __itt_metdata_type. + * @{ + */ + +/** + * @ingroup parameters + * @brief describes the type of metadata + */ +typedef enum { + __itt_metadata_unknown = 0, + __itt_metadata_u64, /**< Unsigned 64-bit integer */ + __itt_metadata_s64, /**< Signed 64-bit integer */ + __itt_metadata_u32, /**< Unsigned 32-bit integer */ + __itt_metadata_s32, /**< Signed 32-bit integer */ + __itt_metadata_u16, /**< Unsigned 16-bit integer */ + __itt_metadata_s16, /**< Signed 16-bit integer */ + __itt_metadata_float, /**< Signed 32-bit floating-point */ + __itt_metadata_double /**< SIgned 64-bit floating-point */ +} __itt_metadata_type; + +/** + * @ingroup parameters + * @brief Add metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + * @param[in] key The name of the metadata + * @param[in] type The type of the metadata + * @param[in] count The number of elements of the given type. If count == 0, no metadata will be added. + * @param[in] data The metadata itself +*/ +void ITTAPI __itt_metadata_add(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, metadata_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data)) +#define __itt_metadata_add(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(metadata_add,d,x,y,z,a,b) +#define __itt_metadata_add_ptr ITTNOTIFY_NAME(metadata_add) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_metadata_add(d,x,y,z,a,b) +#define __itt_metadata_add_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_metadata_add_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup parameters + * @brief Add string metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + * @param[in] key The name of the metadata + * @param[in] data The metadata itself + * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_metadata_str_addA(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length); +void ITTAPI __itt_metadata_str_addW(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t *data, size_t length); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_metadata_str_add __itt_metadata_str_addW +# define __itt_metadata_str_add_ptr __itt_metadata_str_addW_ptr +#else /* UNICODE */ +# define __itt_metadata_str_add __itt_metadata_str_addA +# define __itt_metadata_str_add_ptr __itt_metadata_str_addA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_metadata_str_add(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length); +#endif + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, metadata_str_addA, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length)) +ITT_STUBV(ITTAPI, void, metadata_str_addW, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t *data, size_t length)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, metadata_str_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_addA(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_addA,d,x,y,z,a) +#define __itt_metadata_str_addA_ptr ITTNOTIFY_NAME(metadata_str_addA) +#define __itt_metadata_str_addW(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_addW,d,x,y,z,a) +#define __itt_metadata_str_addW_ptr ITTNOTIFY_NAME(metadata_str_addW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add,d,x,y,z,a) +#define __itt_metadata_str_add_ptr ITTNOTIFY_NAME(metadata_str_add) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_addA(d,x,y,z,a) +#define __itt_metadata_str_addA_ptr 0 +#define __itt_metadata_str_addW(d,x,y,z,a) +#define __itt_metadata_str_addW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add(d,x,y,z,a) +#define __itt_metadata_str_add_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_addA_ptr 0 +#define __itt_metadata_str_addW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup parameters + * @brief Add metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] scope The scope of the instance to which the metadata is to be added + + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + + * @param[in] key The name of the metadata + * @param[in] type The type of the metadata + * @param[in] count The number of elements of the given type. If count == 0, no metadata will be added. + * @param[in] data The metadata itself +*/ +void ITTAPI __itt_metadata_add_with_scope(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, metadata_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data)) +#define __itt_metadata_add_with_scope(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(metadata_add_with_scope,d,x,y,z,a,b) +#define __itt_metadata_add_with_scope_ptr ITTNOTIFY_NAME(metadata_add_with_scope) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_metadata_add_with_scope(d,x,y,z,a,b) +#define __itt_metadata_add_with_scope_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_metadata_add_with_scope_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup parameters + * @brief Add string metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] scope The scope of the instance to which the metadata is to be added + + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + + * @param[in] key The name of the metadata + * @param[in] data The metadata itself + * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_metadata_str_add_with_scopeA(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length); +void ITTAPI __itt_metadata_str_add_with_scopeW(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_metadata_str_add_with_scope __itt_metadata_str_add_with_scopeW +# define __itt_metadata_str_add_with_scope_ptr __itt_metadata_str_add_with_scopeW_ptr +#else /* UNICODE */ +# define __itt_metadata_str_add_with_scope __itt_metadata_str_add_with_scopeA +# define __itt_metadata_str_add_with_scope_ptr __itt_metadata_str_add_with_scopeA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_metadata_str_add_with_scope(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length); +#endif + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeA, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length)) +ITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeW, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, metadata_str_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scopeA,d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeA_ptr ITTNOTIFY_NAME(metadata_str_add_with_scopeA) +#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scopeW,d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeW_ptr ITTNOTIFY_NAME(metadata_str_add_with_scopeW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_with_scope(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scope,d,x,y,z,a) +#define __itt_metadata_str_add_with_scope_ptr ITTNOTIFY_NAME(metadata_str_add_with_scope) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeA_ptr 0 +#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_with_scope(d,x,y,z,a) +#define __itt_metadata_str_add_with_scope_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_add_with_scopeA_ptr 0 +#define __itt_metadata_str_add_with_scopeW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_with_scope_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} metadata group */ + +/** + * @defgroup relations Relations + * Instances of named entities can be explicitly associated with other + * instances using instance IDs and the relationship API calls. + * + * @{ + */ + +/** + * @ingroup relations + * @brief The kind of relation between two instances is specified by the enumerated type __itt_relation. + * Relations between instances can be added with an API call. The relation + * API uses instance IDs. Relations can be added before or after the actual + * instances are created and persist independently of the instances. This + * is the motivation for having different lifetimes for instance IDs and + * the actual instances. + */ +typedef enum +{ + __itt_relation_is_unknown = 0, + __itt_relation_is_dependent_on, /**< "A is dependent on B" means that A cannot start until B completes */ + __itt_relation_is_sibling_of, /**< "A is sibling of B" means that A and B were created as a group */ + __itt_relation_is_parent_of, /**< "A is parent of B" means that A created B */ + __itt_relation_is_continuation_of, /**< "A is continuation of B" means that A assumes the dependencies of B */ + __itt_relation_is_child_of, /**< "A is child of B" means that A was created by B (inverse of is_parent_of) */ + __itt_relation_is_continued_by, /**< "A is continued by B" means that B assumes the dependencies of A (inverse of is_continuation_of) */ + __itt_relation_is_predecessor_to /**< "A is predecessor to B" means that B cannot start until A completes (inverse of is_dependent_on) */ +} __itt_relation; + +/** + * @ingroup relations + * @brief Add a relation to the current task instance. + * The current task instance is the head of the relation. + * @param[in] domain The domain controlling this call + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_to_current(const __itt_domain *domain, __itt_relation relation, __itt_id tail); + +/** + * @ingroup relations + * @brief Add a relation between two instance identifiers. + * @param[in] domain The domain controlling this call + * @param[in] head The ID for the head of the relation + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add(const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, relation_add_to_current, (const __itt_domain *domain, __itt_relation relation, __itt_id tail)) +ITT_STUBV(ITTAPI, void, relation_add, (const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail)) +#define __itt_relation_add_to_current(d,x,y) ITTNOTIFY_VOID_D2(relation_add_to_current,d,x,y) +#define __itt_relation_add_to_current_ptr ITTNOTIFY_NAME(relation_add_to_current) +#define __itt_relation_add(d,x,y,z) ITTNOTIFY_VOID_D3(relation_add,d,x,y,z) +#define __itt_relation_add_ptr ITTNOTIFY_NAME(relation_add) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_relation_add_to_current(d,x,y) +#define __itt_relation_add_to_current_ptr 0 +#define __itt_relation_add(d,x,y,z) +#define __itt_relation_add_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_relation_add_to_current_ptr 0 +#define __itt_relation_add_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} relations group */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_clock_info +{ + unsigned long long clock_freq; /*!< Clock domain frequency */ + unsigned long long clock_base; /*!< Clock domain base timestamp */ +} __itt_clock_info; + +#pragma pack(pop) +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef void (ITTAPI *__itt_get_clock_info_fn)(__itt_clock_info* clock_info, void* data); +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_clock_domain +{ + __itt_clock_info info; /*!< Most recent clock domain info */ + __itt_get_clock_info_fn fn; /*!< Callback function pointer */ + void* fn_data; /*!< Input argument for the callback function */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_clock_domain* next; +} __itt_clock_domain; + +#pragma pack(pop) +/** @endcond */ + +/** + * @ingroup clockdomains + * @brief Create a clock domain. + * Certain applications require the capability to trace their application using + * a clock domain different than the CPU, for instance the instrumentation of events + * that occur on a GPU. + * Because the set of domains is expected to be static over the application's execution time, + * there is no mechanism to destroy a domain. + * Any domain can be accessed by any thread in the process, regardless of which thread created + * the domain. This call is thread-safe. + * @param[in] fn A pointer to a callback function which retrieves alternative CPU timestamps + * @param[in] fn_data Argument for a callback function; may be NULL + */ +__itt_clock_domain* ITTAPI __itt_clock_domain_create(__itt_get_clock_info_fn fn, void* fn_data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_clock_domain*, clock_domain_create, (__itt_get_clock_info_fn fn, void* fn_data)) +#define __itt_clock_domain_create ITTNOTIFY_DATA(clock_domain_create) +#define __itt_clock_domain_create_ptr ITTNOTIFY_NAME(clock_domain_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_clock_domain_create(fn,fn_data) (__itt_clock_domain*)0 +#define __itt_clock_domain_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_clock_domain_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomains + * @brief Recalculate clock domains frequences and clock base timestamps. + */ +void ITTAPI __itt_clock_domain_reset(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, clock_domain_reset, (void)) +#define __itt_clock_domain_reset ITTNOTIFY_VOID(clock_domain_reset) +#define __itt_clock_domain_reset_ptr ITTNOTIFY_NAME(clock_domain_reset) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_clock_domain_reset() +#define __itt_clock_domain_reset_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_clock_domain_reset_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Create an instance of identifier. This establishes the beginning of the lifetime of + * an instance of the given ID in the trace. Once this lifetime starts, the ID can be used to + * tag named entity instances in calls such as __itt_task_begin, and to specify relationships among + * identified named entity instances, using the \ref relations APIs. + * @param[in] domain The domain controlling the execution of this call. + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The ID to create. + */ +void ITTAPI __itt_id_create_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id); + +/** + * @ingroup clockdomain + * @brief Destroy an instance of identifier. This ends the lifetime of the current instance of the + * given ID value in the trace. Any relationships that are established after this lifetime ends are + * invalid. This call must be performed before the given ID value can be reused for a different + * named entity instance. + * @param[in] domain The domain controlling the execution of this call. + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The ID to destroy. + */ +void ITTAPI __itt_id_destroy_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, id_create_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id)) +ITT_STUBV(ITTAPI, void, id_destroy_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id)) +#define __itt_id_create_ex(d,x,y,z) ITTNOTIFY_VOID_D3(id_create_ex,d,x,y,z) +#define __itt_id_create_ex_ptr ITTNOTIFY_NAME(id_create_ex) +#define __itt_id_destroy_ex(d,x,y,z) ITTNOTIFY_VOID_D3(id_destroy_ex,d,x,y,z) +#define __itt_id_destroy_ex_ptr ITTNOTIFY_NAME(id_destroy_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_id_create_ex(domain,clock_domain,timestamp,id) +#define __itt_id_create_ex_ptr 0 +#define __itt_id_destroy_ex(domain,clock_domain,timestamp,id) +#define __itt_id_destroy_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_id_create_ex_ptr 0 +#define __itt_id_destroy_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid The instance ID for this task instance, or __itt_null + * @param[in] parentid The parent instance to which this task instance belongs, or __itt_null + * @param[in] name The name of this task + */ +void ITTAPI __itt_task_begin_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name); + +/** + * @ingroup clockdomain + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid The identifier for this task instance, or __itt_null + * @param[in] parentid The parent of this task, or __itt_null + * @param[in] fn The pointer to the function you are tracing + */ +void ITTAPI __itt_task_begin_fn_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, void* fn); + +/** + * @ingroup clockdomain + * @brief End the current task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + */ +void ITTAPI __itt_task_end_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_begin_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, task_begin_fn_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, void* fn)) +ITT_STUBV(ITTAPI, void, task_end_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp)) +#define __itt_task_begin_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_ex,d,x,y,z,a,b) +#define __itt_task_begin_ex_ptr ITTNOTIFY_NAME(task_begin_ex) +#define __itt_task_begin_fn_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_fn_ex,d,x,y,z,a,b) +#define __itt_task_begin_fn_ex_ptr ITTNOTIFY_NAME(task_begin_fn_ex) +#define __itt_task_end_ex(d,x,y) ITTNOTIFY_VOID_D2(task_end_ex,d,x,y) +#define __itt_task_end_ex_ptr ITTNOTIFY_NAME(task_end_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_begin_ex(domain,clock_domain,timestamp,id,parentid,name) +#define __itt_task_begin_ex_ptr 0 +#define __itt_task_begin_fn_ex(domain,clock_domain,timestamp,id,parentid,fn) +#define __itt_task_begin_fn_ex_ptr 0 +#define __itt_task_end_ex(domain,clock_domain,timestamp) +#define __itt_task_end_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_begin_ex_ptr 0 +#define __itt_task_begin_fn_ex_ptr 0 +#define __itt_task_end_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @defgroup counters Counters + * @ingroup public + * Counters are user-defined objects with a monotonically increasing + * value. Counter values are 64-bit unsigned integers. + * Counters have names that can be displayed in + * the tools. + * @{ + */ + +/** + * @brief opaque structure for counter identification + */ +/** @cond exclude_from_documentation */ + +typedef struct ___itt_counter* __itt_counter; + +/** + * @brief Create an unsigned 64 bits integer counter with given name/domain + * + * After __itt_counter_create() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta), + * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) + * can be used to change the value of the counter, where value_ptr is a pointer to an unsigned 64 bits integer + * + * The call is equal to __itt_counter_create_typed(name, domain, __itt_metadata_u64) + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_createA(const char *name, const char *domain); +__itt_counter ITTAPI __itt_counter_createW(const wchar_t *name, const wchar_t *domain); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create __itt_counter_createW +# define __itt_counter_create_ptr __itt_counter_createW_ptr +#else /* UNICODE */ +# define __itt_counter_create __itt_counter_createA +# define __itt_counter_create_ptr __itt_counter_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create(const char *name, const char *domain); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char *name, const char *domain)) +ITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create, (const char *name, const char *domain)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA ITTNOTIFY_DATA(counter_createA) +#define __itt_counter_createA_ptr ITTNOTIFY_NAME(counter_createA) +#define __itt_counter_createW ITTNOTIFY_DATA(counter_createW) +#define __itt_counter_createW_ptr ITTNOTIFY_NAME(counter_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create ITTNOTIFY_DATA(counter_create) +#define __itt_counter_create_ptr ITTNOTIFY_NAME(counter_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA(name, domain) +#define __itt_counter_createA_ptr 0 +#define __itt_counter_createW(name, domain) +#define __itt_counter_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create(name, domain) +#define __itt_counter_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_ptr 0 +#define __itt_counter_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Increment the unsigned 64 bits integer counter value + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_inc(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc, (__itt_counter id)) +#define __itt_counter_inc ITTNOTIFY_VOID(counter_inc) +#define __itt_counter_inc_ptr ITTNOTIFY_NAME(counter_inc) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc(id) +#define __itt_counter_inc_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** + * @brief Increment the unsigned 64 bits integer counter value with x + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_inc_delta(__itt_counter id, unsigned long long value); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc_delta, (__itt_counter id, unsigned long long value)) +#define __itt_counter_inc_delta ITTNOTIFY_VOID(counter_inc_delta) +#define __itt_counter_inc_delta_ptr ITTNOTIFY_NAME(counter_inc_delta) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc_delta(id, value) +#define __itt_counter_inc_delta_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_delta_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Decrement the unsigned 64 bits integer counter value + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_dec(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec, (__itt_counter id)) +#define __itt_counter_dec ITTNOTIFY_VOID(counter_dec) +#define __itt_counter_dec_ptr ITTNOTIFY_NAME(counter_dec) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec(id) +#define __itt_counter_dec_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** + * @brief Decrement the unsigned 64 bits integer counter value with x + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_dec_delta(__itt_counter id, unsigned long long value); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec_delta, (__itt_counter id, unsigned long long value)) +#define __itt_counter_dec_delta ITTNOTIFY_VOID(counter_dec_delta) +#define __itt_counter_dec_delta_ptr ITTNOTIFY_NAME(counter_dec_delta) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec_delta(id, value) +#define __itt_counter_dec_delta_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_delta_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup counters + * @brief Increment a counter by one. + * The first call with a given name creates a counter by that name and sets its + * value to zero. Successive calls increment the counter value. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + */ +void ITTAPI __itt_counter_inc_v3(const __itt_domain *domain, __itt_string_handle *name); + +/** + * @ingroup counters + * @brief Increment a counter by the value specified in delta. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + * @param[in] delta The amount by which to increment the counter + */ +void ITTAPI __itt_counter_inc_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc_v3, (const __itt_domain *domain, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, counter_inc_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta)) +#define __itt_counter_inc_v3(d,x) ITTNOTIFY_VOID_D1(counter_inc_v3,d,x) +#define __itt_counter_inc_v3_ptr ITTNOTIFY_NAME(counter_inc_v3) +#define __itt_counter_inc_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_inc_delta_v3,d,x,y) +#define __itt_counter_inc_delta_v3_ptr ITTNOTIFY_NAME(counter_inc_delta_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc_v3(domain,name) +#define __itt_counter_inc_v3_ptr 0 +#define __itt_counter_inc_delta_v3(domain,name,delta) +#define __itt_counter_inc_delta_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_v3_ptr 0 +#define __itt_counter_inc_delta_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + + +/** + * @ingroup counters + * @brief Decrement a counter by one. + * The first call with a given name creates a counter by that name and sets its + * value to zero. Successive calls decrement the counter value. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + */ +void ITTAPI __itt_counter_dec_v3(const __itt_domain *domain, __itt_string_handle *name); + +/** + * @ingroup counters + * @brief Decrement a counter by the value specified in delta. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + * @param[in] delta The amount by which to decrement the counter + */ +void ITTAPI __itt_counter_dec_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec_v3, (const __itt_domain *domain, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, counter_dec_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta)) +#define __itt_counter_dec_v3(d,x) ITTNOTIFY_VOID_D1(counter_dec_v3,d,x) +#define __itt_counter_dec_v3_ptr ITTNOTIFY_NAME(counter_dec_v3) +#define __itt_counter_dec_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_dec_delta_v3,d,x,y) +#define __itt_counter_dec_delta_v3_ptr ITTNOTIFY_NAME(counter_dec_delta_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec_v3(domain,name) +#define __itt_counter_dec_v3_ptr 0 +#define __itt_counter_dec_delta_v3(domain,name,delta) +#define __itt_counter_dec_delta_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_v3_ptr 0 +#define __itt_counter_dec_delta_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} counters group */ + + +/** + * @brief Set the counter value + */ +void ITTAPI __itt_counter_set_value(__itt_counter id, void *value_ptr); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value, (__itt_counter id, void *value_ptr)) +#define __itt_counter_set_value ITTNOTIFY_VOID(counter_set_value) +#define __itt_counter_set_value_ptr ITTNOTIFY_NAME(counter_set_value) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value(id, value_ptr) +#define __itt_counter_set_value_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Set the counter value + */ +void ITTAPI __itt_counter_set_value_ex(__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value_ex, (__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr)) +#define __itt_counter_set_value_ex ITTNOTIFY_VOID(counter_set_value_ex) +#define __itt_counter_set_value_ex_ptr ITTNOTIFY_NAME(counter_set_value_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) +#define __itt_counter_set_value_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Create a typed counter with given name/domain + * + * After __itt_counter_create_typed() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta), + * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) + * can be used to change the value of the counter + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_create_typedA(const char *name, const char *domain, __itt_metadata_type type); +__itt_counter ITTAPI __itt_counter_create_typedW(const wchar_t *name, const wchar_t *domain, __itt_metadata_type type); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create_typed __itt_counter_create_typedW +# define __itt_counter_create_typed_ptr __itt_counter_create_typedW_ptr +#else /* UNICODE */ +# define __itt_counter_create_typed __itt_counter_create_typedA +# define __itt_counter_create_typed_ptr __itt_counter_create_typedA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create_typed(const char *name, const char *domain, __itt_metadata_type type); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedA, (const char *name, const char *domain, __itt_metadata_type type)) +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedW, (const wchar_t *name, const wchar_t *domain, __itt_metadata_type type)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create_typed, (const char *name, const char *domain, __itt_metadata_type type)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA ITTNOTIFY_DATA(counter_create_typedA) +#define __itt_counter_create_typedA_ptr ITTNOTIFY_NAME(counter_create_typedA) +#define __itt_counter_create_typedW ITTNOTIFY_DATA(counter_create_typedW) +#define __itt_counter_create_typedW_ptr ITTNOTIFY_NAME(counter_create_typedW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed ITTNOTIFY_DATA(counter_create_typed) +#define __itt_counter_create_typed_ptr ITTNOTIFY_NAME(counter_create_typed) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA(name, domain, type) +#define __itt_counter_create_typedA_ptr 0 +#define __itt_counter_create_typedW(name, domain, type) +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed(name, domain, type) +#define __itt_counter_create_typed_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA_ptr 0 +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Destroy the counter identified by the pointer previously returned by __itt_counter_create() or + * __itt_counter_create_typed() + */ +void ITTAPI __itt_counter_destroy(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_destroy, (__itt_counter id)) +#define __itt_counter_destroy ITTNOTIFY_VOID(counter_destroy) +#define __itt_counter_destroy_ptr ITTNOTIFY_NAME(counter_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_destroy(id) +#define __itt_counter_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} counters group */ + +/** + * @ingroup markers + * @brief Create a marker instance. + * @param[in] domain The domain for this marker + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The instance ID for this marker, or __itt_null + * @param[in] name The name for this marker + * @param[in] scope The scope for this marker + */ +void ITTAPI __itt_marker_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, marker_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope)) +#define __itt_marker_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(marker_ex,d,x,y,z,a,b) +#define __itt_marker_ex_ptr ITTNOTIFY_NAME(marker_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_marker_ex(domain,clock_domain,timestamp,id,name,scope) +#define __itt_marker_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_marker_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Add a relation to the current task instance. + * The current task instance is the head of the relation. + * @param[in] domain The domain controlling this call + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_to_current_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail); + +/** + * @ingroup clockdomain + * @brief Add a relation between two instance identifiers. + * @param[in] domain The domain controlling this call + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] head The ID for the head of the relation + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, relation_add_to_current_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail)) +ITT_STUBV(ITTAPI, void, relation_add_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail)) +#define __itt_relation_add_to_current_ex(d,x,y,z,a) ITTNOTIFY_VOID_D4(relation_add_to_current_ex,d,x,y,z,a) +#define __itt_relation_add_to_current_ex_ptr ITTNOTIFY_NAME(relation_add_to_current_ex) +#define __itt_relation_add_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(relation_add_ex,d,x,y,z,a,b) +#define __itt_relation_add_ex_ptr ITTNOTIFY_NAME(relation_add_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_relation_add_to_current_ex(domain,clock_domain,timestame,relation,tail) +#define __itt_relation_add_to_current_ex_ptr 0 +#define __itt_relation_add_ex(domain,clock_domain,timestamp,head,relation,tail) +#define __itt_relation_add_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_relation_add_to_current_ex_ptr 0 +#define __itt_relation_add_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum ___itt_track_group_type +{ + __itt_track_group_type_normal = 0 +} __itt_track_group_type; +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_track_group +{ + __itt_string_handle* name; /*!< Name of the track group */ + struct ___itt_track* track; /*!< List of child tracks */ + __itt_track_group_type tgtype; /*!< Type of the track group */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_track_group* next; +} __itt_track_group; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Placeholder for custom track types. Currently, "normal" custom track + * is the only available track type. + */ +typedef enum ___itt_track_type +{ + __itt_track_type_normal = 0 +#ifdef INTEL_ITTNOTIFY_API_PRIVATE + , __itt_track_type_queue +#endif /* INTEL_ITTNOTIFY_API_PRIVATE */ +} __itt_track_type; + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_track +{ + __itt_string_handle* name; /*!< Name of the track group */ + __itt_track_group* group; /*!< Parent group to a track */ + __itt_track_type ttype; /*!< Type of the track */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_track* next; +} __itt_track; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Create logical track group. + */ +__itt_track_group* ITTAPI __itt_track_group_create(__itt_string_handle* name, __itt_track_group_type track_group_type); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_track_group*, track_group_create, (__itt_string_handle* name, __itt_track_group_type track_group_type)) +#define __itt_track_group_create ITTNOTIFY_DATA(track_group_create) +#define __itt_track_group_create_ptr ITTNOTIFY_NAME(track_group_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_track_group_create(name) (__itt_track_group*)0 +#define __itt_track_group_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_track_group_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Create logical track. + */ +__itt_track* ITTAPI __itt_track_create(__itt_track_group* track_group, __itt_string_handle* name, __itt_track_type track_type); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_track*, track_create, (__itt_track_group* track_group,__itt_string_handle* name, __itt_track_type track_type)) +#define __itt_track_create ITTNOTIFY_DATA(track_create) +#define __itt_track_create_ptr ITTNOTIFY_NAME(track_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_track_create(track_group,name,track_type) (__itt_track*)0 +#define __itt_track_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_track_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Set the logical track. + */ +void ITTAPI __itt_set_track(__itt_track* track); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, set_track, (__itt_track *track)) +#define __itt_set_track ITTNOTIFY_VOID(set_track) +#define __itt_set_track_ptr ITTNOTIFY_NAME(set_track) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_set_track(track) +#define __itt_set_track_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_set_track_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/* ========================================================================== */ +/** @cond exclude_from_gpa_documentation */ +/** + * @defgroup events Events + * @ingroup public + * Events group + * @{ + */ +/** @brief user event type */ +typedef int __itt_event; + +/** + * @brief Create an event notification + * @note name or namelen being null/name and namelen not matching, user event feature not enabled + * @return non-zero event identifier upon success and __itt_err otherwise + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_event LIBITTAPI __itt_event_createA(const char *name, int namelen); +__itt_event LIBITTAPI __itt_event_createW(const wchar_t *name, int namelen); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_event_create __itt_event_createW +# define __itt_event_create_ptr __itt_event_createW_ptr +#else +# define __itt_event_create __itt_event_createA +# define __itt_event_create_ptr __itt_event_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_event LIBITTAPI __itt_event_create(const char *name, int namelen); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(LIBITTAPI, __itt_event, event_createA, (const char *name, int namelen)) +ITT_STUB(LIBITTAPI, __itt_event, event_createW, (const wchar_t *name, int namelen)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(LIBITTAPI, __itt_event, event_create, (const char *name, int namelen)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_event_createA ITTNOTIFY_DATA(event_createA) +#define __itt_event_createA_ptr ITTNOTIFY_NAME(event_createA) +#define __itt_event_createW ITTNOTIFY_DATA(event_createW) +#define __itt_event_createW_ptr ITTNOTIFY_NAME(event_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_event_create ITTNOTIFY_DATA(event_create) +#define __itt_event_create_ptr ITTNOTIFY_NAME(event_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_event_createA(name, namelen) (__itt_event)0 +#define __itt_event_createA_ptr 0 +#define __itt_event_createW(name, namelen) (__itt_event)0 +#define __itt_event_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_event_create(name, namelen) (__itt_event)0 +#define __itt_event_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_event_createA_ptr 0 +#define __itt_event_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_event_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an event occurrence. + * @return __itt_err upon failure (invalid event id/user event feature not enabled) + */ +int LIBITTAPI __itt_event_start(__itt_event event); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(LIBITTAPI, int, event_start, (__itt_event event)) +#define __itt_event_start ITTNOTIFY_DATA(event_start) +#define __itt_event_start_ptr ITTNOTIFY_NAME(event_start) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_event_start(event) (int)0 +#define __itt_event_start_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_event_start_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an event end occurrence. + * @note It is optional if events do not have durations. + * @return __itt_err upon failure (invalid event id/user event feature not enabled) + */ +int LIBITTAPI __itt_event_end(__itt_event event); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(LIBITTAPI, int, event_end, (__itt_event event)) +#define __itt_event_end ITTNOTIFY_DATA(event_end) +#define __itt_event_end_ptr ITTNOTIFY_NAME(event_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_event_end(event) (int)0 +#define __itt_event_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_event_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} events group */ + + +/** + * @defgroup arrays Arrays Visualizer + * @ingroup public + * Visualize arrays + * @{ + */ + +/** + * @enum __itt_av_data_type + * @brief Defines types of arrays data (for C/C++ intrinsic types) + */ +typedef enum +{ + __itt_e_first = 0, + __itt_e_char = 0, /* 1-byte integer */ + __itt_e_uchar, /* 1-byte unsigned integer */ + __itt_e_int16, /* 2-byte integer */ + __itt_e_uint16, /* 2-byte unsigned integer */ + __itt_e_int32, /* 4-byte integer */ + __itt_e_uint32, /* 4-byte unsigned integer */ + __itt_e_int64, /* 8-byte integer */ + __itt_e_uint64, /* 8-byte unsigned integer */ + __itt_e_float, /* 4-byte floating */ + __itt_e_double, /* 8-byte floating */ + __itt_e_last = __itt_e_double +} __itt_av_data_type; + +/** + * @brief Save an array data to a file. + * Output format is defined by the file extension. The csv and bmp formats are supported (bmp - for 2-dimensional array only). + * @param[in] data - pointer to the array data + * @param[in] rank - the rank of the array + * @param[in] dimensions - pointer to an array of integers, which specifies the array dimensions. + * The size of dimensions must be equal to the rank + * @param[in] type - the type of the array, specified as one of the __itt_av_data_type values (for intrinsic types) + * @param[in] filePath - the file path; the output format is defined by the file extension + * @param[in] columnOrder - defines how the array is stored in the linear memory. + * It should be 1 for column-major order (e.g. in FORTRAN) or 0 - for row-major order (e.g. in C). + */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_av_saveA(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); +int ITTAPI __itt_av_saveW(void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_av_save __itt_av_saveW +# define __itt_av_save_ptr __itt_av_saveW_ptr +#else /* UNICODE */ +# define __itt_av_save __itt_av_saveA +# define __itt_av_save_ptr __itt_av_saveA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_av_save(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, av_saveA, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) +ITT_STUB(ITTAPI, int, av_saveW, (void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, av_save, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA ITTNOTIFY_DATA(av_saveA) +#define __itt_av_saveA_ptr ITTNOTIFY_NAME(av_saveA) +#define __itt_av_saveW ITTNOTIFY_DATA(av_saveW) +#define __itt_av_saveW_ptr ITTNOTIFY_NAME(av_saveW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save ITTNOTIFY_DATA(av_save) +#define __itt_av_save_ptr ITTNOTIFY_NAME(av_save) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA(name) +#define __itt_av_saveA_ptr 0 +#define __itt_av_saveW(name) +#define __itt_av_saveW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save(name) +#define __itt_av_save_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA_ptr 0 +#define __itt_av_saveW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +void ITTAPI __itt_enable_attach(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, enable_attach, (void)) +#define __itt_enable_attach ITTNOTIFY_VOID(enable_attach) +#define __itt_enable_attach_ptr ITTNOTIFY_NAME(enable_attach) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_enable_attach() +#define __itt_enable_attach_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_enable_attach_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** @} arrays group */ + +/** @endcond */ + +/** + * @brief Module load info + * This API is used to report necessary information in case of module relocation + * @param[in] start_addr - relocated module start address + * @param[in] end_addr - relocated module end address + * @param[in] path - file system path to the module + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_module_loadA(void *start_addr, void *end_addr, const char *path); +void ITTAPI __itt_module_loadW(void *start_addr, void *end_addr, const wchar_t *path); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_module_load __itt_module_loadW +# define __itt_module_load_ptr __itt_module_loadW_ptr +#else /* UNICODE */ +# define __itt_module_load __itt_module_loadA +# define __itt_module_load_ptr __itt_module_loadA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_module_load(void *start_addr, void *end_addr, const char *path); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, void, module_loadA, (void *start_addr, void *end_addr, const char *path)) +ITT_STUB(ITTAPI, void, module_loadW, (void *start_addr, void *end_addr, const wchar_t *path)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, void, module_load, (void *start_addr, void *end_addr, const char *path)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA ITTNOTIFY_VOID(module_loadA) +#define __itt_module_loadA_ptr ITTNOTIFY_NAME(module_loadA) +#define __itt_module_loadW ITTNOTIFY_VOID(module_loadW) +#define __itt_module_loadW_ptr ITTNOTIFY_NAME(module_loadW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load ITTNOTIFY_VOID(module_load) +#define __itt_module_load_ptr ITTNOTIFY_NAME(module_load) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA(start_addr, end_addr, path) +#define __itt_module_loadA_ptr 0 +#define __itt_module_loadW(start_addr, end_addr, path) +#define __itt_module_loadW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load(start_addr, end_addr, path) +#define __itt_module_load_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA_ptr 0 +#define __itt_module_loadW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + + + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* _ITTNOTIFY_H_ */ + +#ifdef INTEL_ITTNOTIFY_API_PRIVATE + +#ifndef _ITTNOTIFY_PRIVATE_ +#define _ITTNOTIFY_PRIVATE_ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @ingroup clockdomain + * @brief Begin an overlapped task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid The identifier for this task instance, *cannot* be __itt_null. + * @param[in] parentid The parent of this task, or __itt_null. + * @param[in] name The name of this task. + */ +void ITTAPI __itt_task_begin_overlapped_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name); + +/** + * @ingroup clockdomain + * @brief End an overlapped task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid Explicit ID of finished task + */ +void ITTAPI __itt_task_end_overlapped_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_begin_overlapped_ex, (const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name)) +ITT_STUBV(ITTAPI, void, task_end_overlapped_ex, (const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid)) +#define __itt_task_begin_overlapped_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_overlapped_ex,d,x,y,z,a,b) +#define __itt_task_begin_overlapped_ex_ptr ITTNOTIFY_NAME(task_begin_overlapped_ex) +#define __itt_task_end_overlapped_ex(d,x,y,z) ITTNOTIFY_VOID_D3(task_end_overlapped_ex,d,x,y,z) +#define __itt_task_end_overlapped_ex_ptr ITTNOTIFY_NAME(task_end_overlapped_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_begin_overlapped_ex(domain,clock_domain,timestamp,taskid,parentid,name) +#define __itt_task_begin_overlapped_ex_ptr 0 +#define __itt_task_end_overlapped_ex(domain,clock_domain,timestamp,taskid) +#define __itt_task_end_overlapped_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_begin_overlapped_ex_ptr 0 +#define __itt_task_end_overlapped_ptr 0 +#define __itt_task_end_overlapped_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @defgroup makrs_internal Marks + * @ingroup internal + * Marks group + * @warning Internal API: + * - It is not shipped to outside of Intel + * - It is delivered to internal Intel teams using e-mail or SVN access only + * @{ + */ +/** @brief user mark type */ +typedef int __itt_mark_type; + +/** + * @brief Creates a user mark type with the specified name using char or Unicode string. + * @param[in] name - name of mark to create + * @return Returns a handle to the mark type + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_mark_type ITTAPI __itt_mark_createA(const char *name); +__itt_mark_type ITTAPI __itt_mark_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_mark_create __itt_mark_createW +# define __itt_mark_create_ptr __itt_mark_createW_ptr +#else /* UNICODE */ +# define __itt_mark_create __itt_mark_createA +# define __itt_mark_create_ptr __itt_mark_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_mark_type ITTAPI __itt_mark_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_mark_type, mark_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_mark_type, mark_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_mark_type, mark_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_createA ITTNOTIFY_DATA(mark_createA) +#define __itt_mark_createA_ptr ITTNOTIFY_NAME(mark_createA) +#define __itt_mark_createW ITTNOTIFY_DATA(mark_createW) +#define __itt_mark_createW_ptr ITTNOTIFY_NAME(mark_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_create ITTNOTIFY_DATA(mark_create) +#define __itt_mark_create_ptr ITTNOTIFY_NAME(mark_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_createA(name) (__itt_mark_type)0 +#define __itt_mark_createA_ptr 0 +#define __itt_mark_createW(name) (__itt_mark_type)0 +#define __itt_mark_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_create(name) (__itt_mark_type)0 +#define __itt_mark_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_createA_ptr 0 +#define __itt_mark_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Creates a "discrete" user mark type of the specified type and an optional parameter using char or Unicode string. + * + * - The mark of "discrete" type is placed to collection results in case of success. It appears in overtime view(s) as a special tick sign. + * - The call is "synchronous" - function returns after mark is actually added to results. + * - This function is useful, for example, to mark different phases of application + * (beginning of the next mark automatically meand end of current region). + * - Can be used together with "continuous" marks (see below) at the same collection session + * @param[in] mt - mark, created by __itt_mark_create(const char* name) function + * @param[in] parameter - string parameter of mark + * @return Returns zero value in case of success, non-zero value otherwise. + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_markA(__itt_mark_type mt, const char *parameter); +int ITTAPI __itt_markW(__itt_mark_type mt, const wchar_t *parameter); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_mark __itt_markW +# define __itt_mark_ptr __itt_markW_ptr +#else /* UNICODE */ +# define __itt_mark __itt_markA +# define __itt_mark_ptr __itt_markA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_mark(__itt_mark_type mt, const char *parameter); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, markA, (__itt_mark_type mt, const char *parameter)) +ITT_STUB(ITTAPI, int, markW, (__itt_mark_type mt, const wchar_t *parameter)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, mark, (__itt_mark_type mt, const char *parameter)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_markA ITTNOTIFY_DATA(markA) +#define __itt_markA_ptr ITTNOTIFY_NAME(markA) +#define __itt_markW ITTNOTIFY_DATA(markW) +#define __itt_markW_ptr ITTNOTIFY_NAME(markW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark ITTNOTIFY_DATA(mark) +#define __itt_mark_ptr ITTNOTIFY_NAME(mark) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_markA(mt, parameter) (int)0 +#define __itt_markA_ptr 0 +#define __itt_markW(mt, parameter) (int)0 +#define __itt_markW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark(mt, parameter) (int)0 +#define __itt_mark_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_markA_ptr 0 +#define __itt_markW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Use this if necessary to create a "discrete" user event type (mark) for process + * rather then for one thread + * @see int __itt_mark(__itt_mark_type mt, const char* parameter); + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_mark_globalA(__itt_mark_type mt, const char *parameter); +int ITTAPI __itt_mark_globalW(__itt_mark_type mt, const wchar_t *parameter); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_mark_global __itt_mark_globalW +# define __itt_mark_global_ptr __itt_mark_globalW_ptr +#else /* UNICODE */ +# define __itt_mark_global __itt_mark_globalA +# define __itt_mark_global_ptr __itt_mark_globalA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_mark_global(__itt_mark_type mt, const char *parameter); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, mark_globalA, (__itt_mark_type mt, const char *parameter)) +ITT_STUB(ITTAPI, int, mark_globalW, (__itt_mark_type mt, const wchar_t *parameter)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, mark_global, (__itt_mark_type mt, const char *parameter)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_globalA ITTNOTIFY_DATA(mark_globalA) +#define __itt_mark_globalA_ptr ITTNOTIFY_NAME(mark_globalA) +#define __itt_mark_globalW ITTNOTIFY_DATA(mark_globalW) +#define __itt_mark_globalW_ptr ITTNOTIFY_NAME(mark_globalW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_global ITTNOTIFY_DATA(mark_global) +#define __itt_mark_global_ptr ITTNOTIFY_NAME(mark_global) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_globalA(mt, parameter) (int)0 +#define __itt_mark_globalA_ptr 0 +#define __itt_mark_globalW(mt, parameter) (int)0 +#define __itt_mark_globalW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_global(mt, parameter) (int)0 +#define __itt_mark_global_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_globalA_ptr 0 +#define __itt_mark_globalW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_global_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Creates an "end" point for "continuous" mark with specified name. + * + * - Returns zero value in case of success, non-zero value otherwise. + * Also returns non-zero value when preceding "begin" point for the + * mark with the same name failed to be created or not created. + * - The mark of "continuous" type is placed to collection results in + * case of success. It appears in overtime view(s) as a special tick + * sign (different from "discrete" mark) together with line from + * corresponding "begin" mark to "end" mark. + * @note Continuous marks can overlap and be nested inside each other. + * Discrete mark can be nested inside marked region + * @param[in] mt - mark, created by __itt_mark_create(const char* name) function + * @return Returns zero value in case of success, non-zero value otherwise. + */ +int ITTAPI __itt_mark_off(__itt_mark_type mt); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, int, mark_off, (__itt_mark_type mt)) +#define __itt_mark_off ITTNOTIFY_DATA(mark_off) +#define __itt_mark_off_ptr ITTNOTIFY_NAME(mark_off) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_mark_off(mt) (int)0 +#define __itt_mark_off_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_mark_off_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Use this if necessary to create an "end" point for mark of process + * @see int __itt_mark_off(__itt_mark_type mt); + */ +int ITTAPI __itt_mark_global_off(__itt_mark_type mt); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, int, mark_global_off, (__itt_mark_type mt)) +#define __itt_mark_global_off ITTNOTIFY_DATA(mark_global_off) +#define __itt_mark_global_off_ptr ITTNOTIFY_NAME(mark_global_off) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_mark_global_off(mt) (int)0 +#define __itt_mark_global_off_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_mark_global_off_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} marks group */ + +/** + * @defgroup counters_internal Counters + * @ingroup internal + * Counters group + * @{ + */ + + +/** + * @defgroup stitch Stack Stitching + * @ingroup internal + * Stack Stitching group + * @{ + */ +/** + * @brief opaque structure for counter identification + */ +typedef struct ___itt_caller *__itt_caller; + +/** + * @brief Create the stitch point e.g. a point in call stack where other stacks should be stitched to. + * The function returns a unique identifier which is used to match the cut points with corresponding stitch points. + */ +__itt_caller ITTAPI __itt_stack_caller_create(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_caller, stack_caller_create, (void)) +#define __itt_stack_caller_create ITTNOTIFY_DATA(stack_caller_create) +#define __itt_stack_caller_create_ptr ITTNOTIFY_NAME(stack_caller_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_caller_create() (__itt_caller)0 +#define __itt_stack_caller_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_caller_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Destroy the inforamtion about stitch point identified by the pointer previously returned by __itt_stack_caller_create() + */ +void ITTAPI __itt_stack_caller_destroy(__itt_caller id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, stack_caller_destroy, (__itt_caller id)) +#define __itt_stack_caller_destroy ITTNOTIFY_VOID(stack_caller_destroy) +#define __itt_stack_caller_destroy_ptr ITTNOTIFY_NAME(stack_caller_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_caller_destroy(id) +#define __itt_stack_caller_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_caller_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Sets the cut point. Stack from each event which occurs after this call will be cut + * at the same stack level the function was called and stitched to the corresponding stitch point. + */ +void ITTAPI __itt_stack_callee_enter(__itt_caller id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, stack_callee_enter, (__itt_caller id)) +#define __itt_stack_callee_enter ITTNOTIFY_VOID(stack_callee_enter) +#define __itt_stack_callee_enter_ptr ITTNOTIFY_NAME(stack_callee_enter) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_callee_enter(id) +#define __itt_stack_callee_enter_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_callee_enter_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief This function eliminates the cut point which was set by latest __itt_stack_callee_enter(). + */ +void ITTAPI __itt_stack_callee_leave(__itt_caller id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, stack_callee_leave, (__itt_caller id)) +#define __itt_stack_callee_leave ITTNOTIFY_VOID(stack_callee_leave) +#define __itt_stack_callee_leave_ptr ITTNOTIFY_NAME(stack_callee_leave) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_callee_leave(id) +#define __itt_stack_callee_leave_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_callee_leave_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} stitch group */ + +/* ***************************************************************************************************************************** */ + +#include <stdarg.h> + +/** @cond exclude_from_documentation */ +typedef enum __itt_error_code +{ + __itt_error_success = 0, /*!< no error */ + __itt_error_no_module = 1, /*!< module can't be loaded */ + /* %1$s -- library name; win: %2$d -- system error code; unx: %2$s -- system error message. */ + __itt_error_no_symbol = 2, /*!< symbol not found */ + /* %1$s -- library name, %2$s -- symbol name. */ + __itt_error_unknown_group = 3, /*!< unknown group specified */ + /* %1$s -- env var name, %2$s -- group name. */ + __itt_error_cant_read_env = 4, /*!< GetEnvironmentVariable() failed */ + /* %1$s -- env var name, %2$d -- system error. */ + __itt_error_env_too_long = 5, /*!< variable value too long */ + /* %1$s -- env var name, %2$d -- actual length of the var, %3$d -- max allowed length. */ + __itt_error_system = 6 /*!< pthread_mutexattr_init or pthread_mutex_init failed */ + /* %1$s -- function name, %2$d -- errno. */ +} __itt_error_code; + +typedef void (__itt_error_handler_t)(__itt_error_code code, va_list); +__itt_error_handler_t* __itt_set_error_handler(__itt_error_handler_t*); + +const char* ITTAPI __itt_api_version(void); +/** @endcond */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#define __itt_error_handler ITT_JOIN(INTEL_ITTNOTIFY_PREFIX, error_handler) +void __itt_error_handler(__itt_error_code code, va_list args); +extern const int ITTNOTIFY_NAME(err); +#define __itt_err ITTNOTIFY_NAME(err) +ITT_STUB(ITTAPI, const char*, api_version, (void)) +#define __itt_api_version ITTNOTIFY_DATA(api_version) +#define __itt_api_version_ptr ITTNOTIFY_NAME(api_version) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_api_version() (const char*)0 +#define __itt_api_version_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_api_version_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* _ITTNOTIFY_PRIVATE_ */ + +#endif /* INTEL_ITTNOTIFY_API_PRIVATE */ diff --git a/tools/profiler/docs/buffer.rst b/tools/profiler/docs/buffer.rst new file mode 100644 index 0000000000..a97e52136e --- /dev/null +++ b/tools/profiler/docs/buffer.rst @@ -0,0 +1,70 @@ +Buffers and Memory Management +============================= + +In a post-Fission world, precise memory management across many threads and processes is +especially important. In order for the profiler to achieve this, it uses a chunked buffer +strategy. + +The `ProfileBuffer`_ is the overall buffer class that controls the memory and storage +for the profile, it allows allocating objects into it. This can be used freely +by things like markers and samples to store data as entries, without needing to know +about the general strategy for how the memory is managed. + +The `ProfileBuffer`_ is then backed by the `ProfileChunkedBuffer`_. This specialized +buffer grows incrementally, by allocating additional `ProfileBufferChunk`_ objects. +More and more chunks will be allocated until a memory limit is reached, where they will +be released. After releasing, the chunk will either be recycled or freed. + +The limiting of memory usage is coordinated by the `ProfilerParent`_ in the parent +process. The `ProfilerParent`_ and `ProfilerChild`_ exchange IPC messages with information +about how much memory is being used. When the maximum byte threshold is passed, +the ProfileChunkManager in the parent process removes the oldest chunk, and then the +`ProfilerParent`_ sends a `DestroyReleasedChunksAtOrBefore`_ message to all of child +processes so that the oldest chunks in the profile are released. This helps long profiles +to keep having data in a similar time frame. + +Profile Buffer Terminology +########################## + +ProfilerParent + The main profiler machinery is installed in the parent process. It uses IPC to + communicate to the child processes. The PProfiler is the actor which is used + to communicate across processes to coordinate things. See `ProfilerParent.h`_. The + ProfilerParent uses the DestroyReleasedChunksAtOrBefore message to control the + overall chunk limit. + +ProfilerChild + ProfilerChild is installed in every child process, it will receive requests from + DestroyReleasedChunksAtOrBefore. + +Entry + This is an individual entry in the `ProfileBuffer.h`_,. These entry sizes are not + related to the chunks sizes. An individual entry can straddle two different chunks. + An entry can contain various pieces of data, like markers, samples, and stacks. + +Chunk + An arbitrary sized chunk of memory, managed by the `ProfileChunkedBuffer`_, and + IPC calls from the ProfilerParent. + +Unreleased Chunk + This chunk is currently being used to write entries into. + +Released chunk + This chunk is full of data. When memory limits happen, it can either be recycled + or freed. + +Recycled chunk + This is a chunk that was previously written into, and full. When memory limits occur, + rather than freeing the memory, it is re-used as the next chunk. + +.. _ProfileChunkedBuffer: https://searchfox.org/mozilla-central/search?q=ProfileChunkedBuffer&path=&case=true®exp=false +.. _ProfileChunkManager: https://searchfox.org/mozilla-central/search?q=ProfileBufferChunkManager.h&path=&case=true®exp=false +.. _ProfileBufferChunk: https://searchfox.org/mozilla-central/search?q=ProfileBufferChunk&path=&case=true®exp=false +.. _ProfileBufferChunkManagerWithLocalLimit: https://searchfox.org/mozilla-central/search?q=ProfileBufferChunkManagerWithLocalLimit&case=true&path= +.. _ProfilerParent.h: https://searchfox.org/mozilla-central/source/tools/profiler/public/ProfilerParent.h +.. _ProfilerChild.h: https://searchfox.org/mozilla-central/source/tools/profiler/public/ProfilerChild.h +.. _ProfileBuffer.h: https://searchfox.org/mozilla-central/source/tools/profiler/core/ProfileBuffer.h +.. _ProfileBuffer: https://searchfox.org/mozilla-central/search?q=ProfileBuffer&path=&case=true®exp=false +.. _ProfilerParent: https://searchfox.org/mozilla-central/search?q=ProfilerParent&path=&case=true®exp=false +.. _ProfilerChild: https://searchfox.org/mozilla-central/search?q=ProfilerChild&path=&case=true®exp=false +.. _DestroyReleasedChunksAtOrBefore: https://searchfox.org/mozilla-central/search?q=DestroyReleasedChunksAtOrBefore&path=&case=true®exp=false diff --git a/tools/profiler/docs/code-overview.rst b/tools/profiler/docs/code-overview.rst new file mode 100644 index 0000000000..3ca662e141 --- /dev/null +++ b/tools/profiler/docs/code-overview.rst @@ -0,0 +1,1494 @@ +Profiler Code Overview +###################### + +This is an overview of the code that implements the Profiler inside Firefox +with dome details around tricky subjects, or pointers to more detailed +documentation and/or source code. + +It assumes familiarity with Firefox development, including Mercurial (hg), mach, +moz.build files, Try, Phabricator, etc. + +It also assumes knowledge of the user-visible part of the Firefox Profiler, that +is: How to use the Firefox Profiler, and what profiles contain that is shown +when capturing a profile. See the main website https://profiler.firefox.com, and +its `documentation <https://profiler.firefox.com/docs/>`_. + +For just an "overview", it may look like a huge amount of information, but the +Profiler code is indeed quite expansive, so it takes a lot of words to explain +even just a high-level view of it! For on-the-spot needs, it should be possible +to search for some terms here and follow the clues. But for long-term +maintainers, it would be worth skimming this whole document to get a grasp of +the domain, and return to get some more detailed information before diving into +the code. + +WIP note: This document should be correct at the time it is written, but the +profiler code constantly evolves to respond to bugs or to provide new exciting +features, so this document could become obsolete in parts! It should still be +useful as an overview, but its correctness should be verified by looking at the +actual code. If you notice any significant discrepancy or broken links, please +help by +`filing a bug <https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=Gecko+Profiler>`_. + +***** +Terms +***** + +This is the common usage for some frequently-used terms, as understood by the +Dev Tools team. But incorrect usage can sometimes happen, context is key! + +* **profiler** (a): Generic name for software that enables the profiling of + code. (`"Profiling" on Wikipedia <https://en.wikipedia.org/wiki/Profiling_(computer_programming)>`_) +* **Profiler** (the): All parts of the profiler code inside Firefox. +* **Base Profiler** (the): Parts of the Profiler that live in + mozglue/baseprofiler, and can be used from anywhere, but has limited + functionality. +* **Gecko Profiler** (the): Parts of the Profiler that live in tools/profiler, + and can only be used from other code in the XUL library. +* **Profilers** (the): Both the Base Profiler and the Gecko Profiler. +* **profiling session**: This is the time during which the profiler is running + and collecting data. +* **profile** (a): The output from a profiling session, either as a file, or a + shared viewable profile on https://profiler.firefox.com +* **Profiler back-end** (the): Other name for the Profiler code inside Firefox, + to distinguish it from... +* **Profiler front-end** (the): The website https://profiler.firefox.com that + displays profiles captured by the back-end. +* **Firefox Profiler** (the): The whole suite comprised of the back-end and front-end. + +****************** +Guiding Principles +****************** + +When working on the profiler, here are some guiding principles to keep in mind: + +* Low profiling overhead in cpu and memory. For the Profiler to provide the best + value, it should stay out of the way and consume as few resources (in time and + memory) as possible, so as not to skew the actual Firefox code too much. + +* Common data structures and code should be in the Base Profiler when possible. + + WIP note: Deduplication is slowly happening, see + `meta bug 1557566 <https://bugzilla.mozilla.org/show_bug.cgi?id=1557566>`_. + This document focuses on the Profiler back-end, and mainly the Gecko Profiler + (because this is where most of the code lives, the Base Profiler is mostly a + subset, originally just a cut-down version of the Gecko Profiler); so unless + specified, descriptions below are about the Gecko Profiler, but know that + there may be some equivalent code in the Base Profiler as well. + +* Use appropriate programming-language features where possible to reduce coding + errors in both our code, and our users' usage of it. In C++, this can be done + by using a specific class/struct types for a given usage, to avoid misuse + (e.g., an generic integer representing a **process** could be incorrectly + given to a function expecting a **thread**; we have specific types for these + instead, more below.) + +* Follow the + `Coding Style <https://firefox-source-docs.mozilla.org/code-quality/coding-style/index.html>`_. + +* Whenever possible, write tests (if not present already) for code you add or + modify -- but this may be too difficult in some case, use good judgement and + at least test manually instead. + +****************** +Profiler Lifecycle +****************** + +Here is a high-level view of the Base **or** Gecko Profiler lifecycle, as part +of a Firefox run. The following sections will go into much more details. + +* Profiler initialization, preparing some common data. +* Threads de/register themselves as they start and stop. +* During each User/test-controlled profiling session: + + * Profiler start, preparing data structures that will store the profiling data. + * Periodic sampling from a separate thread, happening at a user-selected + frequency (usually once every 1-2 ms), and recording snapshots of what + Firefox is doing: + + * CPU sampling, measuring how much time each thread has spent actually + running on the CPU. + * Stack sampling, capturing a stack of functions calls from whichever leaf + function the program is in at this point in time, up to the top-most + caller (i.e., at least the ``main()`` function, or its callers if any). + Note that unlike most external profilers, the Firefox Profiler back-end + is capable or getting more useful information than just native functions + calls (compiled from C++ or Rust): + + * Labels added by Firefox developers along the stack, usually to identify + regions of code that perform "interesting" operations (like layout, file + I/Os, etc.). + * JavaScript function calls, including the level of optimization applied. + * Java function calls. + * At any time, Markers may record more specific details of what is happening, + e.g.: User operations, page rendering steps, garbage collection, etc. + * Optional profiler pause, which stops most recording, usually near the end of + a session so that no data gets recorded past this point. + * Profile JSON output, generated from all the recorded profiling data. + * Profiler stop, tearing down profiling session objects. +* Profiler shutdown. + +Note that the Base Profiler can start earlier, and then the data collected so +far, as well as the responsibility for periodic sampling, is handed over to the +Gecko Profiler: + +#. (Firefox starts) +#. Base Profiler init +#. Base Profiler start +#. (Firefox loads the libxul library and initializes XPCOM) +#. Gecko Profiler init +#. Gecko Profiler start +#. Handover from Base to Gecko +#. Base Profiler stop +#. (Bulk of the profiling session) +#. JSON generation +#. Gecko Profiler stop +#. Gecko Profiler shutdown +#. (Firefox ends XPCOM) +#. Base Profiler shutdown +#. (Firefox exits) + +Base Profiler functions that add data (mostly markers and labels) may be called +from anywhere, and will be recorded by either Profiler. The corresponding +functions in Gecko Profiler can only be called from other libxul code, and can +only be recorded by the Gecko Profiler. + +Whenever possible, Gecko Profiler functions should be preferred if accessible, +as they may provide extended functionality (e.g., better stacks with JS in +markers). Otherwise fallback on Base Profiler functions. + +*********** +Directories +*********** + +* Non-Profiler supporting code + + * `mfbt <https://searchfox.org/mozilla-central/source/mfbt>`_ - Mostly + replacements for C++ std library facilities. + + * `mozglue/misc <https://searchfox.org/mozilla-central/source/mozglue/misc>`_ + + * `PlatformMutex.h <https://searchfox.org/mozilla-central/source/mozglue/misc/PlatformMutex.h>`_ - + Mutex base classes. + * `StackWalk.h <https://searchfox.org/mozilla-central/source/mozglue/misc/StackWalk.h>`_ - + Stack-walking functions. + * `TimeStamp.h <https://searchfox.org/mozilla-central/source/mozglue/misc/TimeStamp.h>`_ - + Timestamps and time durations. + + * `xpcom <https://searchfox.org/mozilla-central/source/xpcom>`_ + + * `ds <https://searchfox.org/mozilla-central/source/xpcom/ds>`_ - + Data structures like arrays, strings. + + * `threads <https://searchfox.org/mozilla-central/source/xpcom/threads>`_ - + Threading functions. + +* Profiler back-end + + * `mozglue/baseprofiler <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler>`_ - + Base Profiler code, usable from anywhere in Firefox. Because it lives in + mozglue, it's loaded right at the beginning, so it's possible to start the + profiler very early, even before Firefox loads its big&heavy "xul" library. + + * `baseprofiler's public <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/public>`_ - + Public headers, may be #included from anywhere. + * `baseprofiler's core <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/core>`_ - + Main implementation code. + * `baseprofiler's lul <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/lul>`_ - + Special stack-walking code for Linux. + * `../tests/TestBaseProfiler.cpp <https://searchfox.org/mozilla-central/source/mozglue/tests/TestBaseProfiler.cpp>`_ - + Unit tests. + + * `tools/profiler <https://searchfox.org/mozilla-central/source/tools/profiler>`_ - + Gecko Profiler code, only usable from the xul library. That library is + loaded a short time after Firefox starts, so the Gecko Profiler is not able + to profile the early phase of the application, Base Profiler handles that, + and can pass its collected data to the Gecko Profiler when the latter + starts. + + * `public <https://searchfox.org/mozilla-central/source/tools/profiler/public>`_ - + Public headers, may be #included from most libxul code. + * `core <https://searchfox.org/mozilla-central/source/tools/profiler/core>`_ - + Main implementation code. + * `gecko <https://searchfox.org/mozilla-central/source/tools/profiler/gecko>`_ - + Control from JS, and multi-process/IPC code. + * `lul <https://searchfox.org/mozilla-central/source/tools/profiler/lul>`_ - + Special stack-walking code for Linux. + * `rust-api <https://searchfox.org/mozilla-central/source/tools/profiler/rust-api>`_, + `rust-helper <https://searchfox.org/mozilla-central/source/tools/profiler/rust-helper>`_ + * `tests <https://searchfox.org/mozilla-central/source/tools/profiler/tests>`_ + + * `devtools/client/performance-new <https://searchfox.org/mozilla-central/source/devtools/client/performance-new>`_, + `devtools/shared/performance-new <https://searchfox.org/mozilla-central/source/devtools/shared/performance-new>`_ - + Middleware code for about:profiling and devtools panel functionality. + + * js, starting with + `js/src/vm/GeckoProfiler.h <https://searchfox.org/mozilla-central/source/js/src/vm/GeckoProfiler.h>`_ - + JavaScript engine support, mostly to capture JS stacks. + + * `toolkit/components/extensions/schemas/geckoProfiler.json <https://searchfox.org/mozilla-central/source/toolkit/components/extensions/schemas/geckoProfiler.json>`_ - + File that needs to be updated when Profiler features change. + +* Profiler front-end + + * Out of scope for this document, but its code and bug repository can be found at: + https://github.com/firefox-devtools/profiler . Sometimes work needs to be + done on both the back-end of the front-end, especially when modifying the + back-end's JSON output format. + +******* +Headers +******* + +The most central public header is +`GeckoProfiler.h <https://searchfox.org/mozilla-central/source/tools/profiler/public/GeckoProfiler.h>`_, +from which almost everything else can be found, it can be a good starting point +for exploration. +It includes other headers, which together contain important top-level macros and +functions. + +WIP note: GeckoProfiler.h used to be the header that contained everything! +To better separate areas of functionality, and to hopefully reduce compilation +times, parts of it have been split into smaller headers, and this work will +continue, see `bug 1681416 <https://bugzilla.mozilla.org/show_bug.cgi?id=1681416>`_. + +MOZ_GECKO_PROFILER and Macros +============================= + +Mozilla officially supports the Profiler on `tier-1 platforms +<https://firefox-source-docs.mozilla.org/contributing/build/supported.html>`_: +Windows, macos, Linux and Android. +There is also some code running on tier 2-3 platforms (e.g., for FreeBSD), but +the team at Mozilla is not obligated to maintain it; we do try to keep it +running, and some external contributors are keeping an eye on it and provide +patches when things do break. + +To reduce the burden on unsupported platforms, a lot of the Profilers code is +only compiled when ``MOZ_GECKO_PROFILER`` is #defined. This means that some +public functions may not always be declared or implemented, and should be +surrounded by guards like ``#ifdef MOZ_GECKO_PROFILER``. + +Some commonly-used functions offer an empty definition in the +non-``MOZ_GECKO_PROFILER`` case, so these functions may be called from anywhere +without guard. + +Other functions have associated macros that can always be used, and resolve to +nothing on unsupported platforms. E.g., +``PROFILER_REGISTER_THREAD`` calls ``profiler_register_thread`` where supported, +otherwise does nothing. + +WIP note: There is an effort to eventually get rid of ``MOZ_GECKO_PROFILER`` and +its associated macros, see +`bug 1635350 <https://bugzilla.mozilla.org/show_bug.cgi?id=1635350>`_. + +RAII "Auto" macros and classes +============================== +A number of functions are intended to be called in pairs, usually to start and +then end some operation. To ease their use, and ensure that both functions are +always called together, they usually have an associated class and/or macro that +may be called only once. This pattern of using an object's destructor to ensure +that some action always eventually happens, is called +`RAII <https://en.cppreference.com/w/cpp/language/raii>`_ in C++, with the +common prefix "auto". + +E.g.: In ``MOZ_GECKO_PROFILER`` builds, +`AUTO_PROFILER_INIT <https://searchfox.org/mozilla-central/search?q=AUTO_PROFILER_INIT>`_ +instantiates an +`AutoProfilerInit <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AAutoProfilerInit>`_ +object, which calls ``profiler_init`` when constructed, and +``profiler_shutdown`` when destroyed. + +********************* +Platform Abstractions +********************* + +This section describes some platform abstractions that are used throughout the +Profilers. (Other platform abstractions will be described where they are used.) + +Process and Thread IDs +====================== + +The Profiler back-end often uses process and thread IDs (aka "pid" and "tid"), +which are commonly just a number. +For better code correctness, and to hide specific platform details, they are +encapsulated in opaque types +`BaseProfilerProcessId <https://searchfox.org/mozilla-central/search?q=BaseProfilerProcessId>`_ +and +`BaseProfilerThreadId <https://searchfox.org/mozilla-central/search?q=BaseProfilerThreadId>`_. +These types should be used wherever possible. +When interfacing with other code, they may be converted using the member +functions ``FromNumber`` and ``ToNumber``. + +To find the current process or thread ID, use +`profiler_current_process_id <https://searchfox.org/mozilla-central/search?q=profiler_current_process_id>`_ +or +`profiler_current_thread_id <https://searchfox.org/mozilla-central/search?q=profiler_current_thread_id>`_. + +The main thread ID is available through +`profiler_main_thread_id <https://searchfox.org/mozilla-central/search?q=profiler_main_thread_id>`_ +(assuming +`profiler_init_main_thread_id <https://searchfox.org/mozilla-central/search?q=profiler_init_main_thread_id>`_ +was called when the application started -- especially important in stand-alone +test programs.) +And +`profiler_is_main_thread <https://searchfox.org/mozilla-central/search?q=profiler_is_main_thread>`_ +is a quick way to find out if the current thread is the main thread. + +Locking +======= +The locking primitives in PlatformMutex.h are not supposed to be used as-is, but +through a user-accessible implementation. For the Profilers, this is in +`BaseProfilerDetail.h <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/public/BaseProfilerDetail.h>`_. + +In addition to the usual ``Lock``, ``TryLock``, and ``Unlock`` functions, +`BaseProfilerMutex <https://searchfox.org/mozilla-central/search?q=BaseProfilerMutex>`_ +objects have a name (which may be helpful when debugging), +they record the thread on which they are locked (making it possible to know if +the mutex is locked on the current thread), and in ``DEBUG`` builds there are +assertions verifying that the mutex is not incorrectly used recursively, to +verify the correct ordering of different Profiler mutexes, and that it is +unlocked before destruction. + +Mutexes should preferably be locked within C++ block scopes, or as class +members, by using +`BaseProfilerAutoLock <https://searchfox.org/mozilla-central/search?q=BaseProfilerAutoLock>`_. + +Some classes give the option to use a mutex or not (so that single-threaded code +can more efficiently bypass locking operations), for these we have +`BaseProfilerMaybeMutex <https://searchfox.org/mozilla-central/search?q=BaseProfilerMaybeMutex>`_ +and +`BaseProfilerMaybeAutoLock <https://searchfox.org/mozilla-central/search?q=BaseProfilerMaybeAutoLock>`_. + +There is also a special type of shared lock (aka RWLock, see +`RWLock on wikipedia <https://en.wikipedia.org/wiki/Readers%E2%80%93writer_lock>`_), +which may be locked in multiple threads (through ``LockShared`` or preferably +`BaseProfilerAutoLockShared <https://searchfox.org/mozilla-central/search?q=BaseProfilerAutoLockShared>`_), +or locked exclusively, preventing any other locking (through ``LockExclusive`` or preferably +`BaseProfilerAutoLockExclusive <https://searchfox.org/mozilla-central/search?q=BaseProfilerAutoLockExclusive>`_). + +********************* +Main Profiler Classes +********************* + +Diagram showing the most important Profiler classes, see details in the +following sections: + +(As noted, the "RegisteredThread" classes are now obsolete in the Gecko +Profiler, see the "Thread Registration" section below for an updated diagram and +description.) + +.. image:: profilerclasses-20220913.png + +*********************** +Profiler Initialization +*********************** + +`profiler_init <https://searchfox.org/mozilla-central/search?q=symbol:_Z13profiler_initPv>`_ +and +`baseprofiler::profiler_init <https://searchfox.org/mozilla-central/search?q=symbol:_ZN7mozilla12baseprofiler13profiler_initEPv>`_ +must be called from the main thread, and are used to prepare important aspects +of the profiler, including: + +* Making sure the main thread ID is recorded. +* Handling ``MOZ_PROFILER_HELP=1 ./mach run`` to display the command-line help. +* Creating the ``CorePS`` instance -- more details below. +* Registering the main thread. +* Initializing some platform-specific code. +* Handling other environment variables that are used to immediately start the + profiler, with optional settings provided in other env-vars. + +CorePS +====== + +The `CorePS class <https://searchfox.org/mozilla-central/search?q=symbol:T_CorePS>`_ +has a single instance that should live for the duration of the Firefox +application, and contains important information that could be needed even when +the Profiler is not running. + +It includes: + +* A static pointer to its single instance. +* The process start time. +* JavaScript-specific data structures. +* A list of registered + `PageInformations <https://searchfox.org/mozilla-central/search?q=symbol:T_PageInformation>`_, + used to keep track of the tabs that this process handles. +* A list of + `BaseProfilerCounts <https://searchfox.org/mozilla-central/search?q=symbol:T_BaseProfilerCount>`_, + used to record things like the process memory usage. +* The process name, and optionally the "eTLD+1" (roughly sub-domain) that this + process handles. +* In the Base Profiler only, a list of + `RegisteredThreads <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%253A%253Abaseprofiler%253A%253ARegisteredThread>`_. + WIP note: This storage has been reworked in the Gecko Profiler (more below), + and in practice the Base Profiler only registers the main thread. This should + eventually disappear as part of the de-duplication work + (`bug 1557566 <https://bugzilla.mozilla.org/show_bug.cgi?id=1557566>`_). + +******************* +Thread Registration +******************* + +Threads need to register themselves in order to get fully profiled. +This section describes the main data structures that record the list of +registered threads and their data. + +WIP note: There is some work happening to add limited profiling of unregistered +threads, with the hope that more and more functionality could be added to +eventually use the same registration data structures. + +Diagram showing the relevant classes, see details in the following sub-sections: + +.. image:: profilerthreadregistration-20220913.png + +ProfilerThreadRegistry +====================== + +The +`static ProfilerThreadRegistry object <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3Aprofiler%3A%3AThreadRegistry>`_ +contains a list of ``OffThreadRef`` objects. + +Each ``OffThreadRef`` points to a ``ProfilerThreadRegistration``, and restricts +access to a safe subset of the thread data, and forces a mutex lock if necessary +(more information under ProfilerThreadRegistrationData below). + +ProfilerThreadRegistration +========================== + +A +`ProfilerThreadRegistration object <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3Aprofiler%3A%3AThreadRegistration>`_ +contains a lot of information relevant to its thread, to help with profiling it. + +This data is accessible from the thread itself through an ``OnThreadRef`` +object, which points to the ``ThreadRegistration``, and restricts access to a +safe subset of thread data, and forces a mutex lock if necessary (more +information under ProfilerThreadRegistrationData below). + +ThreadRegistrationData and accessors +==================================== + +`The ProfilerThreadRegistrationData.h header <https://searchfox.org/mozilla-central/source/tools/profiler/public/ProfilerThreadRegistrationData.h>`_ +contains a hierarchy of classes that encapsulate all the thread-related data. + +``ThreadRegistrationData`` contains all the actual data members, including: + +* Some long-lived + `ThreadRegistrationInfo <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%253A%253Aprofiler%253A%253AThreadRegistrationInfo>`_, + containing the thread name, its registration time, the thread ID, and whether + it's the main thread. +* A ``ProfilingStack`` that gathers developer-provided pseudo-frames, and JS + frames. +* Some platform-specific ``PlatformData`` (usually required to actually record + profiling measurements for that thread). +* A pointer to the top of the stack. +* A shared pointer to the thread's ``nsIThread``. +* A pointer to the ``JSContext``. +* An optional pre-allocated ``JsFrame`` buffer used during stack-sampling. +* Some JS flags. +* Sleep-related data (to avoid costly sampling while the thread is known to not + be doing anything). +* The current ``ThreadProfilingFeatures``, to know what kind of data to record. +* When profiling, a pointer to a ``ProfiledThreadData``, which contains some + more data needed during and just after profiling. + +As described in their respective code comments, each data member is supposed to +be accessed in certain ways, e.g., the ``JSContext`` should only be "written +from thread, read from thread and suspended thread". To enforce these rules, +data members can only be accessed through certain classes, which themselves can +only be instantiated in the correct conditions. + +The accessor classes are, from base to most-derived: + +* ``ThreadRegistrationData``, not an accessor itself, but it's the base class + with all the ``protected`` data. +* ``ThreadRegistrationUnlockedConstReader``, giving unlocked ``const`` access to + the ``ThreadRegistrationInfo``, ``PlatformData``, and stack top. +* ``ThreadRegistrationUnlockedConstReaderAndAtomicRW``, giving unlocked + access to the atomic data members: ``ProfilingStack``, sleep-related data, + ``ThreadProfilingFeatures``. +* ``ThreadRegistrationUnlockedRWForLockedProfiler``, giving access that's + protected by the Profiler's main lock, but doesn't require a + ``ThreadRegistration`` lock, to the ``ProfiledThreadData`` +* ``ThreadRegistrationUnlockedReaderAndAtomicRWOnThread``, giving unlocked + mutable access, but only on the thread itself, to the ``JSContext``. +* ``ThreadRegistrationLockedRWFromAnyThread``, giving locked access from any + thread to mutex-protected data: ``ThreadProfilingFeatures``, ``JsFrame``, + ``nsIThread``, and the JS flags. +* ``ThreadRegistrationLockedRWOnThread``, giving locked access, but only from + the thread itself, to the ``JSContext`` and a JS flag-related operation. +* ``ThreadRegistration::EmbeddedData``, containing all of the above, and stored + as a data member in each ``ThreadRegistration``. + +To recapitulate, if some code needs some data on the thread, it can use +``ThreadRegistration`` functions to request access (with the required rights, +like a mutex lock). +To access data about another thread, use similar functions from +``ThreadRegistry`` instead. +You may find some examples in the implementations of the functions in +ProfilerThreadState.h (see the following section). + +ProfilerThreadState.h functions +=============================== + +The +`ProfilerThreadState.h <https://searchfox.org/mozilla-central/source/tools/profiler/public/ProfilerThreadState.h>`_ +header provides a few helpful functions related to threads, including: + +* ``profiler_is_active_and_thread_is_registered`` +* ``profiler_thread_is_being_profiled`` (for the current thread or another + thread, and for a given set of features) +* ``profiler_thread_is_sleeping`` + +************** +Profiler Start +************** + +There are multiple ways to start the profiler, through command line env-vars, +and programmatically in C++ and JS. + +The main public C++ function is +`profiler_start <https://searchfox.org/mozilla-central/search?q=symbol:_Z14profiler_startN7mozilla10PowerOfTwoIjEEdjPPKcjyRKNS_5MaybeIdEE%2C_Z14profiler_startN7mozilla10PowerOfTwoIjEEdjPPKcjmRKNS_5MaybeIdEE>`_. +It takes all the features specifications, and returns a promise that gets +resolved when the Profiler has fully started in all processes (multi-process +profiling is described later in this document, for now the focus will be on each +process running its instance of the Profiler). It first calls ``profiler_init`` +if needed, and also ``profiler_stop`` if the profiler was already running. + +The main implementation, which can be called from multiple sources, is +`locked_profiler_start <https://searchfox.org/mozilla-central/search?q=locked_profiler_start>`_. +It performs a number of operations to start the profiling session, including: + +* Record the session start time. +* Pre-allocate some work buffer to capture stacks for markers on the main thread. +* In the Gecko Profiler only: If the Base Profiler was running, take ownership + of the data collected so far, and stop the Base Profiler (we don't want both + trying to collect the same data at the same time!) +* Create the ActivePS, which keeps track of most of the profiling session + information, more about it below. +* For each registered thread found in the ``ThreadRegistry``, check if it's one + of the threads to profile, and if yes set the appropriate data into the + corresponding ``ThreadRegistrationData`` (including informing the JS engine to + start recording profiling data). +* On Android, start the Java sampler. +* If native allocations are to be profiled, setup the appropriate hooks. +* Start the audio callback tracing if requested. +* Set the public shared "active" state, used by many functions to quickly assess + whether to actually record profiling data. + +ActivePS +======== + +The `ActivePS class <https://searchfox.org/mozilla-central/search?q=symbol:T_ActivePS>`_ +has a single instance at a time, that should live for the length of the +profiling session. + +It includes: + +* The session start time. +* A way to track "generations" (in case an old ActivePS still lives when the + next one starts, so that in-flight data goes to the correct place.) +* Requested features: Buffer capacity, periodic sampling interval, feature set, + list of threads to profile, optional: specific tab to profile. +* The profile data storage buffer and its chunk manager (see "Storage" section + below for details.) +* More data about live and dead profiled threads. +* Optional counters for per-process CPU usage, and power usage. +* A pointer to the ``SamplerThread`` object (see "Periodic Sampling" section + below for details.) + +******* +Storage +******* + +During a session, the profiling data is serialized into a buffer, which is made +of "chunks", each of which contains "blocks", which have a size and the "entry" +data. + +During a profiling session, there is one main profile buffer, which may be +started by the Base Profiler, and then handed over to the Gecko Profiler when +the latter starts. + +The buffer is divided in chunks of equal size, which are allocated before they +are needed. When the data reaches a user-set limit, the oldest chunk is +recycled. This means that for long-enough profiling sessions, only the most +recent data (that could fit under the limit) is kept. + +Each chunk stores a sequence of blocks of variable length. The chunk itself +only knows where the first full block starts, and where the last block ends, +which is where the next block will be reserved. + +To add an entry to the buffer, a block is reserved, the size is written first +(so that readers can find the start of the next block), and then the entry bytes +are written. + +The following sessions give more technical details. + +leb128iterator.h +================ + +`This utility header <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/public/leb128iterator.h>`_ +contains some functions to read and write unsigned "LEB128" numbers +(`LEB128 on wikipedia <https://en.wikipedia.org/wiki/LEB128>`_). + +They are an efficient way to serialize numbers that are usually small, e.g., +numbers up to 127 only take one byte, two bytes up to 16,383, etc. + +ProfileBufferBlockIndex +======================= + +`A ProfileBufferBlockIndex object <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferBlockIndex>`_ +encapsulates a block index that is known to be the valid start of a block. It is +created when a block is reserved, or when trusted code computes the start of a +block in a chunk. + +The more generic +`ProfileBufferIndex <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferIndex>`_ +type is used when working inside blocks. + +ProfileBufferChunk +================== + +`A ProfileBufferChunk <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferChunk>`_ +is a variable-sized object. It contains: + +* A public copyable header, itself containing: + + * The local offset to the first full block (a chunk may start with the end of + a block that was started at the end of the previous chunk). That offset in + the very first chunk is the natural start to read all the data in the + buffer. + * The local offset past the last reserved block. This is where the next block + should be reserved, unless it points past the end of this chunk size. + * The timestamp when the chunk was first used. + * The timestamp when the chunk became full. + * The number of bytes that may be stored in this chunk. + * The number of reserved blocks. + * The global index where this chunk starts. + * The process ID writing into this chunk. + +* An owning unique pointer to the next chunk. It may be null for the last chunk + in a chain. + +* In ``DEBUG`` builds, a state variable, which is used to ensure that the chunk + goes through a known sequence of states (e.g., Created, then InUse, then + Done, etc.) See the sequence diagram + `where the member variable is defined <https://searchfox.org/mozilla-central/search?q=symbol:F_%3CT_mozilla%3A%3AProfileBufferChunk%3A%3AInternalHeader%3E_mState>`_. + +* The actual buffer data. + +Because a ProfileBufferChunk is variable-size, it must be created through its +static ``Create`` function, which takes care of allocating the correct amount +of bytes, at the correct alignment. + +Chunk Managers +============== + +ProfilerBufferChunkManager +-------------------------- + +`The ProfileBufferChunkManager abstract class <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferChunkManager>`_ +defines the interface of classes that manage chunks. + +Concrete implementations are responsible for: +* Creating chunks for their user, with a mechanism to pre-allocate chunks before they are actually needed. +* Taking back and owning chunks when they are "released" (usually when full). +* Automatically destroying or recycling the oldest released chunks. +* Giving temporary access to extant released chunks. + +ProfileBufferChunkManagerSingle +------------------------------- + +`A ProfileBufferChunkManagerSingle object <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferChunkManagerSingle>`_ +manages a single chunk. + +That chunk is always the same, it is never destroyed. The user may use it and +optionally release it. The manager can then be reset, and that one chunk will +be available again for use. + +A request for a second chunk would always fail. + +This manager is short-lived and not thread-safe. It is useful when there is some +limited data that needs to be captured without blocking the global profiling +buffer, usually one stack sample. This data may then be extracted and quickly +added to the global buffer. + +ProfileBufferChunkManagerWithLocalLimit +--------------------------------------- + +`A ProfileBufferChunkManagerWithLocalLimit object <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferChunkManagerSingle>`_ +implements the ``ProfileBufferChunkManager`` interface fully, managing a number +of chunks, and making sure their total combined size stays under a given limit. +This is the main chunk manager user during a profiling session. + +Note: It also implements the ``ProfileBufferControlledChunkManager`` interface, +this is explained in the later section "Multi-Process Profiling". + +It is thread-safe, and one instance is shared by both Profilers. + +ProfileChunkedBuffer +==================== + +`A ProfileChunkedBuffer object <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileChunkedBuffer>`_ +uses a ``ProfilerBufferChunkManager`` to store data, and handles the different +C++ types of data that the Profilers want to read/write as entries in buffer +chunks. + +Its main function is ``ReserveAndPut``: + +* It takes an invocable object (like a lambda) that should return the size of + the entry to store, this is to potentially avoid costly operations just to + compute a size, when the profiler may not be running. +* It attempts to reserve the space in its chunks, requesting a new chunk if + necessary. +* It then calls a provided invocable object with a + `ProfileBufferEntryWriter <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferEntryWriter>`_, + which offers a range of functions to help serialize C++ objects. The + de/serialization functions are found in specializations of + `ProfileBufferEntryWriter::Serializer <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferEntryWriter%3A%3ASerializer>`_ + and + `ProfileBufferEntryReader::Deserializer <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferEntryReader%3A%3ADeserializer>`_. + +More "put" functions use ``ReserveAndPut`` to more easily serialize blocks of +memory, or C++ objects. + +``ProfileChunkedBuffer`` is optionally thread-safe, using a +``BaseProfilerMaybeMutex``. + +WIP note: Using a mutex makes this storage too noisy for profiling some +real-time (like audio processing). +`Bug 1697953 <https://bugzilla.mozilla.org/show_bug.cgi?id=1697953>`_ will look +at switching to using atomic variables instead. +An alternative would be to use a totally separate non-thread-safe buffers for +each real-time thread that requires it (see +`bug 1754889 <https://bugzilla.mozilla.org/show_bug.cgi?id=1754889>`_). + +ProfileBuffer +============= + +`A ProfileBuffer object <https://searchfox.org/mozilla-central/search?q=symbol:T_ProfileBuffer>`_ +uses a ``ProfileChunkedBuffer`` to store data, and handles the different kinds +of entries that the Profilers want to read/write. + +Each entry starts with a tag identifying a kind. These kinds can be found in +`ProfileBufferEntryKinds.h <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/public/ProfileBufferEntryKinds.h>`_. + +There are "legacy" kinds, which are small fixed-length entries, such as: +Categories, labels, frame information, counters, etc. These can be stored in +`ProfileBufferEntry objects <https://searchfox.org/mozilla-central/search?q=symbol:T_ProfileBufferEntry>`_ + +And there are "modern" kinds, which have variable sizes, such as: Markers, CPU +running times, full stacks, etc. These are more directly handled by code that +can access the underlying ``ProfileChunkedBuffer``. + +The other major responsibility of a ``ProfileChunkedBuffer`` is to read back all +this data, sometimes during profiling (e.g., to duplicate a stack), but mainly +at the end of a session when generating the output JSON profile. + +***************** +Periodic Sampling +***************** + +Probably the most important job of the Profiler is to sample stacks of a number +of running threads, to help developers know which functions get used a lot when +performing some operation on Firefox. + +This is accomplished from a special thread, which regularly springs into action +and captures all this data. + +SamplerThread +============= + +`The SamplerThread object <https://searchfox.org/mozilla-central/search?q=symbol:T_SamplerThread>`_ +manages the information needed during sampling. It is created when the profiler +starts, and is stored inside the ``ActivePS``, see above for details. + +It includes: + +* A ``Sampler`` object that contains platform-specific details, which are + implemented in separate files like platform-win32.cpp, etc. +* The same generation index as its owning ``ActivePS``. +* The requested interval between samples. +* A handle to the thread where the sampling happens, its main function is + `Run() function <https://searchfox.org/mozilla-central/search?q=symbol:_ZN13SamplerThread3RunEv>`_. +* A list of callbacks to invoke after the next sampling. These may be used by + tests to wait for sampling to actually happen. +* The unregistered-thread-spy data, and an optional handle on another thread + that takes care of "spying" on unregistered thread (on platforms where that + operation is too expensive to run directly on the sampling thread). + +The ``Run()`` function takes care of performing the periodic sampling work: +(more details in the following sections) + +* Retrieve the sampling parameters. +* Instantiate a ``ProfileBuffer`` on the stack, to capture samples from other threads. +* Loop until a ``break``: + + * Lock the main profiler mutex, and do: + + * Check if sampling should stop, and break from the loop. + * Clean-up exit profiles (these are profiles sent from dying sub-processes, + and are kept for as long as they overlap with this process' own buffer range). + * Record the CPU utilization of the whole process. + * Record the power consumption. + * Sample each registered counter, including the memory counter. + * For each registered thread to be profiled: + + * Record the CPU utilization. + * If the thread is marked as "still sleeping", record a "same as before" + sample, otherwise suspend the thread and take a full stack sample. + * On some threads, record the event delay to compute the + (un)responsiveness. WIP note: This implementation may change. + + * Record profiling overhead durations. + + * Unlock the main profiler mutex. + * Invoke registered post-sampling callbacks. + * Spy on unregistered threads. + * Based on the requested sampling interval, and how much time this loop took, + compute when the next sampling loop should start, and make the thread sleep + for the appropriate amount of time. The goal is to be as regular as + possible, but if some/all loops take too much time, don't try too hard to + catch up, because the system is probably under stress already. + * Go back to the top of the loop. + +* If we're here, we hit a loop ``break`` above. +* Invoke registered post-sampling callbacks, to let them know that sampling + stopped. + +CPU Utilization +=============== + +CPU Utilization is stored as a number of milliseconds that a thread or process +has spent running on the CPU since the previous sampling. + +Implementations are platform-dependent, and can be found in +`the GetThreadRunningTimesDiff function <https://searchfox.org/mozilla-central/search?q=symbol:_ZL25GetThreadRunningTimesDiffRK10PSAutoLockRN7mozilla8profiler45ThreadRegistrationUnlockedRWForLockedProfilerE>`_ +and +`the GetProcessRunningTimesDiff function <https://searchfox.org/mozilla-central/search?q=symbol:_ZL26GetProcessRunningTimesDiffRK10PSAutoLockR12RunningTimes>`_. + +Power Consumption +================= + +Energy probes added in 2022. + +Stacks +====== + +Stacks are the sequence of calls going from the entry point in the program +(generally ``main()`` and some OS-specific functions above), down to the +function where code is currently being executed. + +Native Frames +------------- + +Compiled code, from C++ and Rust source. + +Label Frames +------------ + +Pseudo-frames with arbitrary text, added from any language, mostly C++. + +JS, Wasm Frames +--------------- + +Frames corresponding to JavaScript functions. + +Java Frames +----------- + +Recorded by the JavaSampler. + +Stack Merging +------------- + +The above types of frames are all captured in different ways, and when finally +taking an actual stack sample (apart from Java), they get merged into one stack. + +All frames have an associated address in the call stack, and can therefore be +merged mostly by ordering them by this stack address. See +`MergeStacks <https://searchfox.org/mozilla-central/search?q=symbol:_ZL11MergeStacksjbRKN7mozilla8profiler51ThreadRegistrationUnlockedReaderAndAtomicRWOnThreadERK9RegistersRK11NativeStackR22ProfilerStackCollectorPN2JS22ProfilingFrameIterator5FrameEj>`_ +for the implementation details. + +Counters +======== + +Counters are a special kind of probe, which can be continuously updated during +profiling, and the ``SamplerThread`` will sample their value at every loop. + +Memory Counter +-------------- + +This is the main counter. During a profiling session, hooks into the memory +manager keep track of each de/allocation, so at each sampling we know how many +operations were performed, and what is the current memory usage compared to the +previous sampling. + +Profiling Overhead +================== + +The ``SamplerThread`` records timestamps between parts of its sampling loop, and +records this as the sampling overhead. This may be useful to determine if the +profiler itself may have used too much of the computer resources, which could +skew the profile and give wrong impressions. + +Unregistered Thread Profiling +============================= + +At some intervals (not necessarily every sampling loop, depending on the OS), +the profiler may attempt to find unregistered threads, and record some +information about them. + +WIP note: This feature is experimental, and data is captured in markers on the +main thread. More work is needed to put this data in tracks like regular +registered threads, and capture more data like stack samples and markers. + +******* +Markers +******* + +Markers are events with a precise timestamp or time range, they have a name, a +category, options (out of a few choices), and optional marker-type-specific +payload data. + +Before describing the implementation, it is useful to be familiar with how +markers are natively added from C++, because this drives how the implementation +takes all this information and eventually outputs it in the final JSON profile. + +Adding Markers from C++ +======================= + +See https://firefox-source-docs.mozilla.org/tools/profiler/markers-guide.html + +Implementation +============== + +The main function that records markers is +`profiler_add_marker <https://searchfox.org/mozilla-central/search?q=symbol:_Z19profiler_add_markerRKN7mozilla18ProfilerStringViewIcEERKNS_14MarkerCategoryEONS_13MarkerOptionsET_DpRKT0_>`_. +It's a variadic templated function that takes the different the expected +arguments, first checks if the marker should actually be recorded (the profiler +should be running, and the target thread should be profiled), and then calls +into the deeper implementation function ``AddMarkerToBuffer`` with a reference +to the main profiler buffer. + +`AddMarkerToBuffer <https://searchfox.org/mozilla-central/search?q=symbol:_Z17AddMarkerToBufferRN7mozilla20ProfileChunkedBufferERKNS_18ProfilerStringViewIcEERKNS_14MarkerCategoryEONS_13MarkerOptionsET_DpRKT0_>`_ +takes the marker type as an object, removes it from the function parameter list, +and calls the next function with the marker type as an explicit template +parameter, and also a pointer to the function that can capture the stack +(because it is different between Base and Gecko Profilers, in particular the +latter one knows about JS). + +From here, we enter the land of +`BaseProfilerMarkersDetail.h <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/public/BaseProfilerMarkersDetail.h>`_, +which employs some heavy template techniques, in order to most efficiently +serialize the given marker payload arguments, in order to make them +deserializable when outputting the final JSON. In previous implementations, for +each new marker type, a new C++ class derived from a payload abstract class was +required, that had to implement all the constructors and virtual functions to: + +* Create the payload object. +* Serialize the payload into the profile buffer. +* Deserialize from the profile buffer to a new payload object. +* Convert the payload into the final output JSON. + +Now, the templated functions automatically take care of serializing all given +function call arguments directly (instead of storing them somewhere first), and +preparing a deserialization function that will recreate them on the stack and +directly call the user-provided JSONification function with these arguments. + +Continuing from the public ``AddMarkerToBuffer``, +`mozilla::base_profiler_markers_detail::AddMarkerToBuffer <https://searchfox.org/mozilla-central/search?q=symbol:_ZN7mozilla28base_profiler_markers_detail17AddMarkerToBufferERNS_20ProfileChunkedBufferERKNS_18ProfilerStringViewIcEERKNS_14MarkerCategoryEONS_13MarkerOptionsEPFbS2_NS_19StackCaptureOptionsEEDpRKT0_>`_ +sets some defaults if not specified by the caller: Target the current thread, +use the current time. + +Then if a stack capture was requested, attempt to do it in +the most efficient way, using a pre-allocated buffer if possible. + +WIP note: This potential allocation should be avoided in time-critical thread. +There is already a buffer for the main thread (because it's the busiest thread), +but there could be more pre-allocated threads, for specific real-time thread +that need it, or picked from a pool of pre-allocated buffers. See +`bug 1578792 <https://bugzilla.mozilla.org/show_bug.cgi?id=1578792>`_. + +From there, `AddMarkerWithOptionalStackToBuffer <https://searchfox.org/mozilla-central/search?q=AddMarkerWithOptionalStackToBuffer>`_ +handles ``NoPayload`` markers (usually added with ``PROFILER_MARKER_UNTYPED``) +in a special way, mostly to avoid the extra work associated with handling +payloads. Otherwise it continues with the following function. + +`MarkerTypeSerialization<MarkerType>::Serialize <symbol:_ZN7mozilla28base_profiler_markers_detail23MarkerTypeSerialization9SerializeERNS_20ProfileChunkedBufferERKNS_18ProfilerStringViewIcEERKNS_14MarkerCategoryEONS_13MarkerOptionsEDpRKTL0__>`_ +retrieves the deserialization tag associated with the marker type. If it's the +first time this marker type is used, +`Streaming::TagForMarkerTypeFunctions <symbol:_ZN7mozilla28base_profiler_markers_detail9Streaming25TagForMarkerTypeFunctionsEPFvRNS_24ProfileBufferEntryReaderERNS_12baseprofiler20SpliceableJSONWriterEEPFNS_4SpanIKcLy18446744073709551615EEEvEPFNS_12MarkerSchemaEvE,_ZN7mozilla28base_profiler_markers_detail9Streaming25TagForMarkerTypeFunctionsEPFvRNS_24ProfileBufferEntryReaderERNS_12baseprofiler20SpliceableJSONWriterEEPFNS_4SpanIKcLm18446744073709551615EEEvEPFNS_12MarkerSchemaEvE,_ZN7mozilla28base_profiler_markers_detail9Streaming25TagForMarkerTypeFunctionsEPFvRNS_24ProfileBufferEntryReaderERNS_12baseprofiler20SpliceableJSONWriterEEPFNS_4SpanIKcLj4294967295EEEvEPFNS_12MarkerSchemaEvE>`_ +adds it to the global list (which stores some function pointers used during +deserialization). + +Then the main serialization happens in +`StreamFunctionTypeHelper<decltype(MarkerType::StreamJSONMarkerData)>::Serialize <symbol:_ZN7mozilla28base_profiler_markers_detail24StreamFunctionTypeHelperIFT_RNS_12baseprofiler20SpliceableJSONWriterEDpT0_EE9SerializeERNS_20ProfileChunkedBufferERKNS_18ProfilerStringViewIcEERKNS_14MarkerCategoryEONS_13MarkerOptionsEhDpRKS6_>`_. +Deconstructing this mouthful of an template: + +* ``MarkerType::StreamJSONMarkerData`` is the user-provided function that will + eventually produce the final JSON, but here it's only used to know the + parameter types that it expects. +* ``StreamFunctionTypeHelper`` takes that function prototype, and can extract + its argument by specializing on ```R(SpliceableJSONWriter&, As...)``, now + ``As...`` is a parameter pack matching the function parameters. +* Note that ``Serialize`` also takes a parameter pack, which contains all the + referenced arguments given to the top ``AddBufferToMarker`` call. These two + packs are supposed to match, at least the given arguments should be + convertible to the target pack parameter types. +* That specialization's ``Serialize`` function calls the buffer's ``PutObjects`` + variadic function to write all the marker data, that is: + + * The entry kind that must be at the beginning of every buffer entry, in this + case `ProfileBufferEntryKind::Marker <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/public/ProfileBufferEntryKinds.h#78>`_. + * The common marker data (options first, name, category, deserialization tag). + * Then all the marker-type-specific arguments. Note that the C++ types + are those extracted from the deserialization function, so we know that + whatever is serialized here can be later deserialized using those same + types. + +The deserialization side is described in the later section "JSON output of +Markers". + +Adding Markers from Rust +======================== + +See https://firefox-source-docs.mozilla.org/tools/profiler/instrumenting-rust.html#adding-markers + +Adding Markers from JS +====================== + +See https://firefox-source-docs.mozilla.org/tools/profiler/instrumenting-javascript.html + +Adding Markers from Java +======================== + +See https://searchfox.org/mozilla-central/source/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ProfilerController.java + +************* +Profiling Log +************* + +During a profiling session, some profiler-related events may be recorded using +`ProfilingLog::Access <https://searchfox.org/mozilla-central/search?q=symbol:_ZN12ProfilingLog6AccessEOT_>`_. + +The resulting JSON object is added near the end of the process' JSON generation, +in a top-level property named "profilingLog". This object is free-form, and is +not intended to be displayed, or even read by most people. But it may include +interesting information for advanced users, or could be an early temporary +prototyping ground for new features. + +See "profileGatheringLog" for another log related to late events. + +WIP note: This was introduced shortly before this documentation, so at this time +it doesn't do much at all. + +*************** +Profile Capture +*************** + +Usually at the end of a profiling session, a profile is "captured", and either +saved to disk, or sent to the front-end https://profiler.firefox.com for +analysis. This section describes how the captured data is converted to the +Gecko Profiler JSON format. + +FailureLatch +============ + +`The FailureLatch interface <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AFailureLatch>`_ +is used during the JSON generation, in order to catch any unrecoverable error +(such as running Out Of Memory), to exit the process early, and to forward the +error to callers. + +There are two main implementations, suffixed "source" as they are the one source +of failure-handling, which is passed as ``FailureLatch&`` throughout the code: + +* `FailureLatchInfallibleSource <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AFailureLatchInfallibleSource>`_ + is an "infallible" latch, meaning that it doesn't expect any failure. So if + a failure actually happened, the program would immediately terminate! (This + was the default behavior prior to introducing these latches.) +* `FailureLatchSource <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AFailureLatchSource>`_ + is a "fallible" latch, it will record the first failure that happens, and + "latch" into the failure state. The code should regularly examine this state, + and return early when possible. Eventually this failure state may be exposed + to end users. + +ProgressLogger, ProportionValue +=============================== + +`A ProgressLogger object <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProgressLogger>`_ +is used to track the progress of a long operation, in this case the JSON +generation process. + +To match how the JSON generation code works (as a tree of C++ functions calls), +each ``ProgressLogger`` in a function usually records progress from 0 to 100% +locally inside that function. If that function calls a sub-function, it gives it +a sub-logger, which in the caller function is set to represent a local sub-range +(like 20% to 40%), but to the called function it will look like its own local +``ProgressLogger`` that goes from 0 to 100%. The very top ``ProgressLogger`` +converts the deepest local progress value to the corresponding global progress. + +Progress values are recorded in +`ProportionValue objects <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProportionValue>`_, +which effectively record fractional value with no loss of precision. + +This progress is most useful when the parent process is waiting for child +processes to do their work, to make sure progress does happen, otherwise to stop +waiting for frozen processes. More about that in the "Multi-Process Profiling" +section below. + +JSONWriter +========== + +`A JSONWriter object <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AJSONWriter>`_ +offers a simple way to create a JSON stream (start/end collections, add +elements, etc.), and calls back into a provided +`JSONWriteFunc interface <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AJSONWriteFunc>`_ +to output characters. + +While these classes live outside of the Profiler directories, it may sometimes be +worth maintaining and/or modifying them to better serve the Profiler's needs. +But there are other users, so be careful not to break other things! + +SpliceableJSONWriter and SpliceableChunkedJSONWriter +==================================================== + +Because the Profiler deals with large amounts of data (big profiles can take +tens to hundreds of megabytes!), some specialized wrappers add better handling +of these large JSON streams. + +`SpliceableJSONWriter <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3Abaseprofiler%3A%3ASpliceableJSONWriter>`_ +is a subclass of ``JSONWriter``, and allows the "splicing" of JSON strings, +i.e., being able to take a whole well-formed JSON string, and directly inserting +it as a JSON object in the target JSON being streamed. + +It also offers some functions that are often useful for the Profiler, such as: +* Converting a timestamp into a JSON object in the stream, taking care of keeping a nanosecond precision, without unwanted zeroes or nines at the end. +* Adding a number of null elements. +* Adding a unique string index, and add that string to a provided unique-string list if necessary. (More about UniqueStrings below.) + +`SpliceableChunkedJSONWriter <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3Abaseprofiler%3A%3ASpliceableChunkedJSONWriter>`_ +is a subclass of ``SpliceableJSONWriter``. Its main attribute is that it provides its own writer +(`ChunkedJSONWriteFunc <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3Abaseprofiler%3A%3AChunkedJSONWriteFunc>`_), +which stores the stream as a sequence of "chunks" (heap-allocated buffers). +It starts with a chunk of a default size, and writes incoming data into it, +later allocating more chunks as needed. This avoids having massive buffers being +resized all the time. + +It also offers the same splicing abilities as its parent class, but in case an +incoming JSON string comes from another ``SpliceableChunkedJSONWriter``, it's +able to just steal the chunks and add them to its list, thereby avoiding +expensive allocations and copies and destructions. + +UniqueStrings +============= + +Because a lot of strings would be repeated in profiles (e.g., frequent marker +names), such strings are stored in a separate JSON array of strings, and an +index into this list is used instead of that full string object. + +Note that these unique-string indices are currently only located in specific +spots in the JSON tree, they cannot be used just anywhere strings are accepted. + +`The UniqueJSONStrings class <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3Abaseprofiler%3A%3AUniqueJSONStrings>`_ +stores this list of unique strings in a ``SpliceableChunkedJSONWriter``. +Given a string, it takes care of storing it if encountered for the first time, +and inserts the index into a target ``SpliceableJSONWriter``. + +JSON Generation +=============== + +The "Gecko Profile Format" can be found at +https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md . + +The implementation in the back-end is +`locked_profiler_stream_json_for_this_process <https://searchfox.org/mozilla-central/search?q=locked_profiler_stream_json_for_this_process>`_. +It outputs each JSON top-level JSON object, mostly in sequence. See the code for +how each object is output. Note that there is special handling for samples and +markers, as explained in the following section. + +ProcessStreamingContext and ThreadStreamingContext +-------------------------------------------------- + +In JSON profiles, samples and markers are separated by thread and by +samples/markers. Because there are potentially tens to a hundred threads, it +would be very costly to read the full profile buffer once for each of these +groups. So instead the buffer is read once, and all samples and markers are +handled as they are read, and their JSON output is sent to separate JSON +writers. + +`A ProcessStreamingContext object <https://searchfox.org/mozilla-central/search?q=symbol:T_ProcessStreamingContext>`_ +contains all the information to facilitate this output, including a list of +`ThreadStreamingContext's <https://searchfox.org/mozilla-central/search?q=symbol:T_ThreadStreamingContext>`_, +which each contain one ``SpliceableChunkedJSONWriter`` for the samples, and one +for the markers in this thread. + +When reading entries from the profile buffer, samples and markers are found by +their ``ProfileBufferEntryKind``, and as part of deserializing either kind (more +about each below), the thread ID is read, and determines which +``ThreadStreamingContext`` will receive the JSON output. + +At the end of this process, all ``SpliceableChunkedJSONWriters`` are efficiently +spliced (mainly a pointer move) into the final JSON output. + +JSON output of Samples +---------------------- + +This work is done in +`ProfileBuffer::DoStreamSamplesAndMarkersToJSON <https://searchfox.org/mozilla-central/search?q=DoStreamSamplesAndMarkersToJSON>`_. + +From the main ``ProfileChunkedBuffer``, each entry is visited, its +``ProfileBufferEntryKind`` is read first, and for samples all frames from +captured stack are converted to the appropriate JSON. + +`A UniqueStacks object <https://searchfox.org/mozilla-central/search?q=symbol:T_UniqueStacks>`_ +is used to de-duplicate frames and even sub-stacks: + +* Each unique frame string is written into a JSON array inside a + ``SpliceableChunkedJSONWriter``, and its index is the frame identifier. +* Each stack level is also de-duplicated, and identifies the associated frame + string, and points at the calling stack level (i.e., closer to the root). +* Finally, the identifier for the top of the stack is stored, along with a + timestamp (and potentially some more information) as the sample. + +For example, if we have collected the following samples: + +#. A -> B -> C +#. A -> B +#. A -> B -> D + +The frame table would contain each frame name, something like: +``["A", "B", "C", "D"]``. So the frame containing "A" has index 0, "B" is at 1, +etc. + +The stack table would contain each stack level, something like: +``[[0, null], [1, 0], [2, 1], [3, 1]]``. ``[0, null]`` means the frame is 0 +("A"), and it has no caller, it's the root frame. ``[1, 0]`` means the frame is +1 ("B"), and its caller is stack 0, which is just the previous one in this +example. + +And the three samples stored in the thread data would be therefore be: 2, 1, 3 +(E.g.: "2" points in the stack table at the frame [2,1] with "C", and from them +down to "B", then "A"). + +All this contains all the information needed to reconstruct all full stack +samples. + +JSON output of Markers +---------------------- + +This also happens +`inside ProfileBuffer::DoStreamSamplesAndMarkersToJSON <https://searchfox.org/mozilla-central/search?q=DoStreamSamplesAndMarkersToJSON>`_. + +When a ``ProfileBufferEntryKind::Marker`` is encountered, +`the DeserializeAfterKindAndStream function <https://searchfox.org/mozilla-central/search?q=DeserializeAfterKindAndStream>`_ +reads the ``MarkerOptions`` (stored as explained above), which include the +thread ID, identifying which ``ThreadStreamingContext``'s +``SpliceableChunkedJSONWriter`` to use. + +After that, the common marker data (timing, category, etc.) is output. + +Then the ``Streaming::DeserializerTag`` identifies which type of marker this is. +The special case of ``0`` (no payload) means nothing more is output. + +Otherwise some more common data is output as part of the payload if present, in +particular the "inner window id" (used to match markers with specific html +frames), and stack. + +WIP note: Some of these may move around in the future, see +`bug 1774326 <https://bugzilla.mozilla.org/show_bug.cgi?id=1774326>`_, +`bug 1774328 <https://bugzilla.mozilla.org/show_bug.cgi?id=1774328>`_, and +others. + +In case of a C++-written payload, the ``DeserializerTag`` identifies the +``MarkerDataDeserializer`` function to use. This is part of the heavy templated +code in BaseProfilerMarkersDetail.h, the function is defined as +`MarkerTypeSerialization<MarkerType>::Deserialize <https://searchfox.org/mozilla-central/search?q=symbol:_ZN7mozilla28base_profiler_markers_detail23MarkerTypeSerialization11DeserializeERNS_24ProfileBufferEntryReaderERNS_12baseprofiler20SpliceableJSONWriterE>`_, +which outputs the marker type name, and then each marker payload argument. The +latter is done by using the user-defined ``MarkerType::StreamJSONMarkerData`` +parameter list, and recursively deserializing each parameter from the profile +buffer into an on-stack variable of a corresponding type, at the end of which +``MarkerType::StreamJSONMarkerData`` can be called with all of these arguments +at it expects, and that function does the actual JSON streaming as the user +programmed. + +************* +Profiler Stop +************* + +See "Profiler Start" and do the reverse! + +There is some special handling of the ``SampleThread`` object, just to ensure +that it gets deleted outside of the main profiler mutex being locked, otherwise +this could result in a deadlock (because it needs to take the lock before being +able to check the state variable indicating that the sampling loop and thread +should end). + +***************** +Profiler Shutdown +***************** + +See "Profiler Initialization" and do the reverse! + +One additional action is handling the optional ``MOZ_PROFILER_SHUTDOWN`` +environment variable, to output a profile if the profiler was running. + +*********************** +Multi-Process Profiling +*********************** + +All of the above explanations focused on what the profiler is doing is each +process: Starting, running and collecting samples, markers, and more data, +outputting JSON profiles, and stopping. + +But Firefox is a multi-process program, since +`Electrolysis aka e10s <https://wiki.mozilla.org/Electrolysis>`_ introduce child +processes to handle web content and extensions, and especially since +`Fission <https://wiki.mozilla.org/Project_Fission>`_ forced even parts of the +same webpage to run in separate processes, mainly for added security. Since then +Firefox can spawn many processes, sometimes 10 to 20 when visiting busy sites. + +The following sections explains how profiling Firefox as a whole works. + +IPC (Inter-Process Communication) +================================= + +See https://firefox-source-docs.mozilla.org/ipc/. + +As a quick summary, some message-passing function-like declarations live in +`PProfiler.ipdl <https://searchfox.org/mozilla-central/source/tools/profiler/gecko/PProfiler.ipdl>`_, +and corresponding ``SendX`` and ``RecvX`` C++ functions are respectively +generated in +`PProfilerParent.h <https://searchfox.org/mozilla-central/source/__GENERATED__/ipc/ipdl/_ipdlheaders/mozilla/PProfilerParent.h>`_, +and virtually declared (for user implementation) in +`PProfilerChild.h <https://searchfox.org/mozilla-central/source/__GENERATED__/ipc/ipdl/_ipdlheaders/mozilla/PProfilerChild.h>`_. + +During Profiling +================ + +Exit profiles +------------- + +One IPC message that is not in PProfiler.ipdl, is +`ShutdownProfile <https://searchfox.org/mozilla-central/search?q=ShutdownProfile%28&path=&case=false®exp=false>`_ +in +`PContent.ipdl <https://searchfox.org/mozilla-central/source/dom/ipc/PContent.ipdl>`_. + +It's called from +`ContentChild::ShutdownInternal <https://searchfox.org/mozilla-central/search?q=symbol:_ZN7mozilla3dom12ContentChild16ShutdownInternalEv>`_, +just before a child process ends, and if the profiler was running, to ensure +that the profile data is collected and sent to the parent, for storage in its +``ActivePS``. + +See +`ActivePS::AddExitProfile <https://searchfox.org/mozilla-central/search?q=symbol:_ZN8ActivePS14AddExitProfileERK10PSAutoLockRK12nsTSubstringIcE>`_ +for details. Note that the current "buffer position at gathering time" (which is +effectively the largest ``ProfileBufferBlockIndex`` that is present in the +global profile buffer) is recorded. Later, +`ClearExpiredExitProfiles <https://searchfox.org/mozilla-central/search?q=ClearExpiredExitProfiles>`_ +looks at the **smallest** ``ProfileBufferBlockIndex`` still present in the +buffer (because early chunks may have been discarded to limit memory usage), and +discards exit profiles that were recorded before, because their data is now +older than anything stored in the parent. + +Profile Buffer Global Memory Control +------------------------------------ + +Each process runs its own profiler, with each its own profile chunked buffer. To +keep the overall memory usage of all these buffers under the user-picked limit, +processes work together, with the parent process overseeing things. + +Diagram showing the relevant classes, see details in the following sub-sections: + +.. image:: fissionprofiler-20200424.png + +ProfileBufferControlledChunkManager +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`The ProfileBufferControlledChunkManager interface <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferControlledChunkManager>`_ +allows a controller to get notified about all chunk updates, and to force the +destruction/recycling of old chunks. +`The ProfileBufferChunkManagerWithLocalLimit class <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferChunkManagerWithLocalLimit>`_ +implements it. + +`An Update object <https://searchfox.org/mozilla-central/search?q=symbol:T_mozilla%3A%3AProfileBufferControlledChunkManager%3A%3AUpdate>`_ +contains all information related to chunk changes: How much memory is currently +used by the local chunk manager, how much has been "released" (and therefore +could be destroyed/recycled), and a list of all chunks that were released since +the previous update; it also has a special state meaning that the child is +shutting down so there won't be updates anymore. An ``Update`` may be "folded" +into a previous one, to create a combined update equivalent to the two separate +ones one after the other. + +Update Handling in the ProfilerChild +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When the profiler starts in a child process, the ``ProfilerChild`` +`starts to listen for updates <https://searchfox.org/mozilla-central/search?q=symbol:_ZN7mozilla13ProfilerChild17SetupChunkManagerEv>`_. + +These updates are stored and folded into previous ones (if any). At some point, +`an AwaitNextChunkManagerUpdate message <https://searchfox.org/mozilla-central/search?q=RecvAwaitNextChunkManagerUpdate>`_ +will be received, and any update can be forwarded to the parent. The local +update is cleared, ready to store future updates. + +Update Handling in the ProfilerParent +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When the profiler starts AND when there are child processes, the +`ProfilerParent's ProfilerParentTracker <https://searchfox.org/mozilla-central/search?q=ProfilerParentTracker>`_ +creates +`a ProfileBufferGlobalController <https://searchfox.org/mozilla-central/search?q=ProfileBufferGlobalController>`_, +which starts to listen for updates from the local chunk manager. + +The ``ProfilerParentTracker`` is also responsible for keeping track of child +processes, and to regularly +`send them AwaitNextChunkManagerUpdate messages <https://searchfox.org/mozilla-central/search?q=SendAwaitNextChunkManagerUpdate>`_, +that the child's ``ProfilerChild`` answers to with updates. The update may +indicate that the child is shutting down, in which case the tracker will stop +tracking it. + +All these updates (from the local chunk manager, and from child processes' own +chunk managers) are processed in +`ProfileBufferGlobalController::HandleChunkManagerNonFinalUpdate <https://searchfox.org/mozilla-central/search?q=HandleChunkManagerNonFinalUpdate>`_. +Based on this stream of updates, it is possible to calculate the total memory +used by all profile buffers in all processes, and to keep track of all chunks +that have been "released" (i.e., are full, and can be destroyed). When the total +memory usage reaches the user-selected limit, the controller can lookup the +oldest chunk, and get it destroyed (either a local call for parent chunks, or by +sending +`a DestroyReleasedChunksAtOrBefore message <https://searchfox.org/mozilla-central/search?q=DestroyReleasedChunksAtOrBefore>`_ +to the owning child). + +Historical note: Prior to Fission, the Profiler used to keep one fixed-size +circular buffer in each process, but as Fission made the possible number of +processes unlimited, the memory consumption grew too fast, and required the +implementation of the above system. But there may still be mentions of +"circular buffers" in the code or documents; these have effectively been +replaced by chunked buffers, with centralized chunk control. + +Gathering Child Profiles +======================== + +When it's time to capture a full profile, the parent process performs its own +JSON generation (as described above), and sends +`a GatherProfile message <https://searchfox.org/mozilla-central/search?q=GatherProfile%28>`_ +to all child processes, which will make them generate their JSON profile and +send it back to the parent. + +All child profiles, including the exit profiles collected during profiling, are +stored as elements of a top-level array with property name "processes". + +During the gathering phase, while the parent is waiting for child responses, it +regularly sends +`GetGatherProfileProgress messages <https://searchfox.org/mozilla-central/search?q=GetGatherProfileProgress>`_ +to all child processes that have not sent their profile yet, and the parent +expects responses within a short timeframe. The response carries a progress +value. If at some point two messages went with no progress was made anywhere +(either there was no response, or the progress value didn't change), the parent +assumes that remaining child processes may be frozen indefinitely, stops the +gathering and considers the JSON generation complete. + +During all of the above work, events are logged (especially issues with child +processes), and are added at the end of the JSON profile, in a top-level object +with property name "profileGatheringLog". This object is free-form, and is not +intended to be displayed, or even read by most people. But it may include +interesting information for advanced users regarding the profile-gathering +phase. diff --git a/tools/profiler/docs/fissionprofiler-20200424.png b/tools/profiler/docs/fissionprofiler-20200424.png Binary files differnew file mode 100644 index 0000000000..1602877a5b --- /dev/null +++ b/tools/profiler/docs/fissionprofiler-20200424.png diff --git a/tools/profiler/docs/fissionprofiler.umlet.uxf b/tools/profiler/docs/fissionprofiler.umlet.uxf new file mode 100644 index 0000000000..3325294e25 --- /dev/null +++ b/tools/profiler/docs/fissionprofiler.umlet.uxf @@ -0,0 +1,546 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<diagram program="umlet" version="14.3.0">
+ <zoom_level>10</zoom_level>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>70</x>
+ <y>110</y>
+ <w>300</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>/PProfilerParent/
+bg=light_gray
+--
+*+SendAwaitNextChunkManagerUpdate()*
+*+SendDestroyReleasedChunksAtOrBefore()*</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>470</x>
+ <y>20</y>
+ <w>210</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>*ProfileBufferChunkMetadata*
+bg=light_gray
+--
++doneTimeStamp
++bufferBytes
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>780</x>
+ <y>110</y>
+ <w>330</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>/PProfilerChild/
+bg=light_gray
+--
+*/+RecvAwaitNextChunkManagerUpdate() = 0/*
+*/+RecvDestroyReleasedChunksAtOrBefore() = 0/*
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>110</x>
+ <y>260</y>
+ <w>220</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>ProfilerParent
+--
+*-processId*
+--
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>210</x>
+ <y>170</y>
+ <w>30</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>740</x>
+ <y>250</y>
+ <w>410</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>ProfilerChild
+--
+-UpdateStorage: unreleased bytes, released: {pid, rangeStart[ ]}
+--
+*+RecvAwaitNextChunkUpdate()*
+*+RecvDestroyReleasedChunksAtOrBefore()*
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>930</x>
+ <y>170</y>
+ <w>30</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>110</x>
+ <y>400</y>
+ <w>220</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>ProfilerParentTracker
+--
+_+Enumerate()_
+_*+ForChild()*_</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>210</x>
+ <y>320</y>
+ <w>190</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=<-
+m1=0..n
+nsTArray<ProfilerParent*></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>80</x>
+ <y>1070</y>
+ <w>150</w>
+ <h>30</h>
+ </coordinates>
+ <panel_attributes>ProfileBufferChunk</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>380</x>
+ <y>1070</y>
+ <w>210</w>
+ <h>30</h>
+ </coordinates>
+ <panel_attributes>/ProfileBufferChunkManager/</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>180</x>
+ <y>900</y>
+ <w>700</w>
+ <h>50</h>
+ </coordinates>
+ <panel_attributes>ProfileBufferChunkManagerWithLocalLimit
+--
+-mUpdateCallback</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>480</x>
+ <y>940</y>
+ <w>30</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;130.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>380</x>
+ <y>1200</y>
+ <w>210</w>
+ <h>30</h>
+ </coordinates>
+ <panel_attributes>ProfileChunkedBuffer</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>410</x>
+ <y>1090</y>
+ <w>140</w>
+ <h>130</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+mChunkManager</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;110.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>960</x>
+ <y>1200</y>
+ <w>100</w>
+ <h>30</h>
+ </coordinates>
+ <panel_attributes>CorePS</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>960</x>
+ <y>1040</y>
+ <w>100</w>
+ <h>30</h>
+ </coordinates>
+ <panel_attributes>ActivePS</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>580</x>
+ <y>1200</y>
+ <w>400</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+mCoreBuffer</panel_attributes>
+ <additional_attributes>10.0;20.0;380.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>870</x>
+ <y>940</y>
+ <w>250</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+mProfileBufferChunkManager</panel_attributes>
+ <additional_attributes>10.0;10.0;90.0;100.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>830</x>
+ <y>1140</y>
+ <w>100</w>
+ <h>30</h>
+ </coordinates>
+ <panel_attributes>ProfileBuffer</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>920</x>
+ <y>1060</y>
+ <w>130</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+mProfileBuffer</panel_attributes>
+ <additional_attributes>10.0;90.0;40.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>580</x>
+ <y>1160</y>
+ <w>270</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+mEntries</panel_attributes>
+ <additional_attributes>10.0;50.0;250.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>90</x>
+ <y>1090</y>
+ <w>310</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+m1=0..1
+mCurrentChunk: UniquePtr<></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;130.0;290.0;130.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>210</x>
+ <y>1080</y>
+ <w>200</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+m1=0..N
+mNextChunks: UniquePtr<></panel_attributes>
+ <additional_attributes>20.0;10.0;170.0;130.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>200</x>
+ <y>940</y>
+ <w>230</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+m1=0..N
+mReleasedChunks: UniquePtr<></panel_attributes>
+ <additional_attributes>10.0;130.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>530</x>
+ <y>1090</y>
+ <w>270</w>
+ <h>130</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+mOwnedChunkManager: UniquePtr<></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;110.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>480</x>
+ <y>390</y>
+ <w>550</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>*ProfileBufferGlobalController*
+--
+-mMaximumBytes
+-mCurrentUnreleasedBytesTotal
+-mCurrentUnreleasedBytes: {pid, unreleased bytes}[ ] sorted by pid
+-mCurrentReleasedBytes
+-mReleasedChunks: {doneTimeStamp, bytes, pid}[ ] sorted by timestamp
+-mDestructionCallback: function<void(pid, rangeStart)>
+--
++Update(pid, unreleased bytes, released: ProfileBufferChunkMetadata[ ])</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>320</x>
+ <y>420</y>
+ <w>180</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+mController</panel_attributes>
+ <additional_attributes>160.0;20.0;10.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>20</x>
+ <y>400</y>
+ <w>110</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+_sInstance_</panel_attributes>
+ <additional_attributes>90.0;60.0;10.0;60.0;10.0;10.0;90.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>480</x>
+ <y>250</y>
+ <w>220</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>The controller is only needed
+if there *are* child processes,
+so we can create it with the first
+child (at which point the tracker
+can register itself with the local
+profiler), and destroyed with the
+last child.
+bg=blue</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>690</x>
+ <y>330</y>
+ <w>100</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes/>
+ <additional_attributes>10.0;10.0;80.0;60.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>130</x>
+ <y>460</y>
+ <w>200</w>
+ <h>380</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+mParentChunkManager</panel_attributes>
+ <additional_attributes>180.0;360.0;10.0;360.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>740</x>
+ <y>330</y>
+ <w>350</w>
+ <h>510</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+mLocalBufferChunkManager</panel_attributes>
+ <additional_attributes>10.0;490.0;330.0;490.0;330.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>470</x>
+ <y>650</y>
+ <w>400</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>*ProfileBufferControlledChunkManager::Update*
+--
+-mUnreleasedBytes
+-mReleasedBytes
+-mOldestDoneTimeStamp
+-mNewReleasedChunks: ChunkMetadata[ ]</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>470</x>
+ <y>560</y>
+ <w>400</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>*ProfileBufferControlledChunkManager::ChunkMetadata*
+--
+-mDoneTimeStamp
+-mBufferBytes</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>670</x>
+ <y>610</y>
+ <w>30</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>lt=<.</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;40.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>670</x>
+ <y>740</y>
+ <w>30</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>lt=<.</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;40.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>670</x>
+ <y>50</y>
+ <w>130</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=<.</panel_attributes>
+ <additional_attributes>10.0;10.0;110.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>360</x>
+ <y>50</y>
+ <w>130</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=<.</panel_attributes>
+ <additional_attributes>110.0;10.0;10.0;90.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>400</x>
+ <y>130</y>
+ <w>350</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>*ProfileBufferChunkManagerUpdate*
+bg=light_gray
+--
+-unreleasedBytes
+-releasedBytes
+-oldestDoneTimeStamp
+-newlyReleasedChunks: ProfileBufferChunkMetadata[ ]</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>310</x>
+ <y>780</y>
+ <w>440</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>*ProfileBufferControlledChunkManager*
+--
+*/+SetUpdateCallback(function<void(update: Update&&)>)/*
+*/+DestroyChunksAtOrBefore(timeStamp)/*</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>480</x>
+ <y>840</y>
+ <w>30</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;60.0</additional_attributes>
+ </element>
+</diagram>
diff --git a/tools/profiler/docs/index.rst b/tools/profiler/docs/index.rst new file mode 100644 index 0000000000..53920e7d2f --- /dev/null +++ b/tools/profiler/docs/index.rst @@ -0,0 +1,37 @@ +Gecko Profiler +============== + +The Firefox Profiler is the collection of tools used to profile Firefox. This is backed +by the Gecko Profiler, which is the primarily C++ component that instruments Gecko. It +is configurable, and supports a variety of data sources and recording modes. Primarily, +it is used as a statistical profiler, where the execution of threads that have been +registered with the profile is paused, and a sample is taken. Generally, this includes +a stackwalk with combined native stack frame, JavaScript stack frames, and custom stack +frame labels. + +In addition to the sampling, the profiler can collect markers, which are collected +deterministically (as opposed to statistically, like samples). These include some +kind of text description, and optionally a payload with more information. + +This documentation serves to document the Gecko Profiler and Base Profiler components, +while the profiler.firefox.com interface is documented at `profiler.firefox.com/docs/ <https://profiler.firefox.com/docs/>`_ + +.. toctree:: + :maxdepth: 1 + + code-overview + buffer + instrumenting-javascript + instrumenting-rust + markers-guide + memory + +The following areas still need documentation: + + * LUL + * Instrumenting Java + * Registering Threads + * Samples and Stack Walking + * Triggering Gecko Profiles in Automation + * JS Tracer + * Serialization diff --git a/tools/profiler/docs/instrumenting-javascript.rst b/tools/profiler/docs/instrumenting-javascript.rst new file mode 100644 index 0000000000..928d94781e --- /dev/null +++ b/tools/profiler/docs/instrumenting-javascript.rst @@ -0,0 +1,60 @@ +Instrumenting JavaScript +======================== + +There are multiple ways to use the profiler with JavaScript. There is the "JavaScript" +profiler feature (via about:profiling), which enables stack walking for JavaScript code. +This is most likely turned on already for every profiler preset. + +In addition, markers can be created to specifically marker an instant in time, or a +duration. This can be helpful to make sense of a particular piece of the front-end, +or record events that normally wouldn't show up in samples. + +.. note:: + This guide explains JavaScript markers in depth. To learn more about how to add a + marker in C++ or Rust, please take a look at their documentation + in :doc:`markers-guide` or :doc:`instrumenting-rust` respectively. + +Markers in Browser Chrome +************************* + +If you have access to ChromeUtils, then adding a marker is relatively easily. + +.. code-block:: javascript + + // Add an instant marker, representing a single point in time + ChromeUtils.addProfilerMarker("MarkerName"); + + // Add a duration marker, representing a span of time. + const startTime = Cu.now(); + doWork(); + ChromeUtils.addProfilerMarker("MarkerName", startTime); + + // Add a duration marker, representing a span of time, with some additional tex + const startTime = Cu.now(); + doWork(); + ChromeUtils.addProfilerMarker("MarkerName", startTime, "Details about this event"); + + // Add an instant marker, with some additional tex + const startTime = Cu.now(); + doWork(); + ChromeUtils.addProfilerMarker("MarkerName", undefined, "Details about this event"); + +Markers in Content Code +*********************** + +If instrumenting content code, then the `UserTiming`_ API is the best bet. +:code:`performance.mark` will create an instant marker, and a :code:`performance.measure` +will create a duration marker. These markers will show up under UserTiming in +the profiler UI. + +.. code-block:: javascript + + // Create an instant marker. + performance.mark("InstantMarkerName"); + + doWork(); + + // Measuring with the performance API will also create duration markers. + performance.measure("DurationMarkerName", "InstantMarkerName"); + +.. _UserTiming: https://developer.mozilla.org/en-US/docs/Web/API/User_Timing_API diff --git a/tools/profiler/docs/instrumenting-rust.rst b/tools/profiler/docs/instrumenting-rust.rst new file mode 100644 index 0000000000..c3e12f42dc --- /dev/null +++ b/tools/profiler/docs/instrumenting-rust.rst @@ -0,0 +1,433 @@ +Instrumenting Rust +================== + +There are multiple ways to use the profiler with Rust. Native stack sampling already +includes the Rust frames without special handling. There is the "Native Stacks" +profiler feature (via about:profiling), which enables stack walking for native code. +This is most likely turned on already for every profiler presets. + +In addition to that, there is a profiler Rust API to instrument the Rust code +and add more information to the profile data. There are three main functionalities +to use: + +1. Register Rust threads with the profiler, so the profiler can record these threads. +2. Add stack frame labels to annotate and categorize a part of the stack. +3. Add markers to specifically mark instants in time, or durations. This can be + helpful to make sense of a particular piece of the code, or record events that + normally wouldn't show up in samples. + +Crate to Include as a Dependency +-------------------------------- + +Profiler Rust API is located inside the ``gecko-profiler`` crate. This needs to +be included in the project dependencies before the following functionalities can +be used. + +To be able to include it, a new dependency entry needs to be added to the project's +``Cargo.toml`` file like this: + +.. code-block:: toml + + [dependencies] + gecko-profiler = { path = "../../tools/profiler/rust-api" } + +Note that the relative path needs to be updated depending on the project's location +in mozilla-central. + +Registering Threads +------------------- + +To be able to see the threads in the profile data, they need to be registered +with the profiler. Also, they need to be unregistered when they are exiting. +It's important to give a unique name to the thread, so they can be filtered easily. + +Registering and unregistering a thread is straightforward: + +.. code-block:: rust + + // Register it with a given name. + gecko_profiler::register_thread("Thread Name"); + // After doing some work, and right before exiting the thread, unregister it. + gecko_profiler::unregister_thread(); + +For example, here's how to register and unregister a simple thread: + +.. code-block:: rust + + let thread_name = "New Thread"; + std::thread::Builder::new() + .name(thread_name.into()) + .spawn(move || { + gecko_profiler::register_thread(thread_name); + // DO SOME WORK + gecko_profiler::unregister_thread(); + }) + .unwrap(); + +Or with a thread pool: + +.. code-block:: rust + + let worker = rayon::ThreadPoolBuilder::new() + .thread_name(move |idx| format!("Worker#{}", idx)) + .start_handler(move |idx| { + gecko_profiler::register_thread(&format!("Worker#{}", idx)); + }) + .exit_handler(|_idx| { + gecko_profiler::unregister_thread(); + }) + .build(); + +.. note:: + Registering a thread only will not make it appear in the profile data. In + addition, it needs to be added to the "Threads" filter in about:profiling. + This filter input is a comma-separated list. It matches partial names and + supports the wildcard ``*``. + +Adding Stack Frame Labels +------------------------- + +Stack frame labels are useful for annotating a part of the call stack with a +category. The category will appear in the various places on the Firefox Profiler +analysis page like timeline, call tree tab, flame graph tab, etc. + +``gecko_profiler_label!`` macro is used to add a new label frame. The added label +frame will exist between the call of this macro and the end of the current scope. + +Adding a stack frame label: + +.. code-block:: rust + + // Marking the stack as "Layout" category, no subcategory provided. + gecko_profiler_label!(Layout); + // Marking the stack as "JavaScript" category and "Parsing" subcategory. + gecko_profiler_label!(JavaScript, Parsing); + + // Or the entire function scope can be marked with a procedural macro. This is + // essentially a syntactical sugar and it expands into a function with a + // gecko_profiler_label! call at the very start: + #[gecko_profiler_fn_label(DOM)] + fn foo(bar: u32) -> u32 { + bar + } + +See the list of all profiling categories in the `profiling_categories.yaml`_ file. + +Adding Markers +-------------- + +Markers are packets of arbitrary data that are added to a profile by the Firefox code, +usually to indicate something important happening at a point in time, or during an interval of time. + +Each marker has a name, a category, some common optional information (timing, backtrace, etc.), +and an optional payload of a specific type (containing arbitrary data relevant to that type). + +.. note:: + This guide explains Rust markers in depth. To learn more about how to add a + marker in C++ or JavaScript, please take a look at their documentation + in :doc:`markers-guide` or :doc:`instrumenting-javascript` respectively. + +Examples +^^^^^^^^ + +Short examples, details are below. + +.. code-block:: rust + + // Record a simple marker with the category of Graphics, DisplayListBuilding. + gecko_profiler::add_untyped_marker( + // Name of the marker as a string. + "Marker Name", + // Category with an optional sub-category. + gecko_profiler_category!(Graphics, DisplayListBuilding), + // MarkerOptions that keeps options like marker timing and marker stack. + // It will be a point in type by default. + Default::default(), + ); + +.. code-block:: rust + + // Create a marker with some additional text information. + let info = "info about this marker"; + gecko_profiler::add_text_marker( + // Name of the marker as a string. + "Marker Name", + // Category with an optional sub-category. + gecko_profiler_category!(DOM), + // MarkerOptions that keeps options like marker timing and marker stack. + MarkerOptions { + timing: MarkerTiming::instant_now(), + ..Default::default() + }, + // Additional information as a string. + info, + ); + +.. code-block:: rust + + // Record a custom marker of type `ExampleNumberMarker` (see definition below). + gecko_profiler::add_marker( + // Name of the marker as a string. + "Marker Name", + // Category with an optional sub-category. + gecko_profiler_category!(Graphics, DisplayListBuilding), + // MarkerOptions that keeps options like marker timing and marker stack. + Default::default(), + // Marker payload. + ExampleNumberMarker { number: 5 }, + ); + + .... + + // Marker type definition. It needs to derive Serialize, Deserialize. + #[derive(Serialize, Deserialize, Debug)] + pub struct ExampleNumberMarker { + number: i32, + } + + // Marker payload needs to implement the ProfilerMarker trait. + impl gecko_profiler::ProfilerMarker for ExampleNumberMarker { + // Unique marker type name. + fn marker_type_name() -> &'static str { + "example number" + } + // Data specific to this marker type, serialized to JSON for profiler.firefox.com. + fn stream_json_marker_data(&self, json_writer: &mut gecko_profiler::JSONWriter) { + json_writer.int_property("number", self.number.into()); + } + // Where and how to display the marker and its data. + fn marker_type_display() -> gecko_profiler::MarkerSchema { + use gecko_profiler::marker::schema::*; + let mut schema = MarkerSchema::new(&[Location::MarkerChart]); + schema.set_chart_label("Name: {marker.name}"); + schema.add_key_label_format("number", "Number", Format::Integer); + schema + } + } + +Untyped Markers +^^^^^^^^^^^^^^^ + +Untyped markers don't carry any information apart from common marker data: +Name, category, options. + +.. code-block:: rust + + gecko_profiler::add_untyped_marker( + // Name of the marker as a string. + "Marker Name", + // Category with an optional sub-category. + gecko_profiler_category!(Graphics, DisplayListBuilding), + // MarkerOptions that keeps options like marker timing and marker stack. + MarkerOptions { + timing: MarkerTiming::instant_now(), + ..Default::default() + }, + ); + +1. Marker name + The first argument is the name of this marker. This will be displayed in most places + the marker is shown. It can be a literal string, or any dynamic string. +2. `Profiling category pair`_ + A category + subcategory pair from the `the list of categories`_. + ``gecko_profiler_category!`` macro should be used to create a profiling category + pair since it's easier to use, e.g. ``gecko_profiler_category!(JavaScript, Parsing)``. + Second parameter can be omitted to use the default subcategory directly. + ``gecko_profiler_category!`` macro is encouraged to use, but ``ProfilingCategoryPair`` + enum can also be used if needed. +3. `MarkerOptions`_ + See the options below. It can be omitted if there are no arguments with ``Default::default()``. + Some options can also be omitted, ``MarkerOptions {<options>, ..Default::default()}``, + with one or more of the following options types: + + * `MarkerTiming`_ + This specifies an instant or interval of time. It defaults to the current instant if + left unspecified. Otherwise use ``MarkerTiming::instant_at(ProfilerTime)`` or + ``MarkerTiming::interval(pt1, pt2)``; timestamps are usually captured with + ``ProfilerTime::Now()``. It is also possible to record only the start or the end of an + interval, pairs of start/end markers will be matched by their name. + * `MarkerStack`_ + By default, markers do not record a "stack" (or "backtrace"). To record a stack at + this point, in the most efficient manner, specify ``MarkerStack::Full``. To + capture a stack without native frames for reduced overhead, specify + ``MarkerStack::NonNative``. + + *Note: Currently, all C++ marker options are not present in the Rust side. They will + be added in the future.* + +Text Markers +^^^^^^^^^^^^ + +Text markers are very common, they carry an extra text as a fourth argument, in addition to +the marker name. Use the following macro: + +.. code-block:: rust + + let info = "info about this marker"; + gecko_profiler::add_text_marker( + // Name of the marker as a string. + "Marker Name", + // Category with an optional sub-category. + gecko_profiler_category!(DOM), + // MarkerOptions that keeps options like marker timing and marker stack. + MarkerOptions { + stack: MarkerStack::Full, + ..Default::default() + }, + // Additional information as a string. + info, + ); + +As useful as it is, using an expensive ``format!`` operation to generate a complex text +comes with a variety of issues. It can leak potentially sensitive information +such as URLs during the profile sharing step. profiler.firefox.com cannot +access the information programmatically. It won't get the formatting benefits of the +built-in marker schema. Please consider using a custom marker type to separate and +better present the data. + +Other Typed Markers +^^^^^^^^^^^^^^^^^^^ + +From Rust code, a marker of some type ``YourMarker`` (details about type definition follow) can be +recorded like this: + +.. code-block:: rust + + gecko_profiler::add_marker( + // Name of the marker as a string. + "Marker Name", + // Category with an optional sub-category. + gecko_profiler_category!(JavaScript), + // MarkerOptions that keeps options like marker timing and marker stack. + Default::default(), + // Marker payload. + YourMarker { number: 5, text: "some string".to_string() }, + ); + +After the first three common arguments (like in ``gecko_profiler::add_untyped_marker``), +there is a marker payload struct and it needs to be defined. Let's take a look at +how to define it. + +How to Define New Marker Types +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each marker type must be defined once and only once. +The definition is a Rust ``struct``, it's constructed when recording markers of +that type in Rust. Each marker struct holds the data that is required for them +to show in the profiler.firefox.com. +By convention, the suffix "Marker" is recommended to better distinguish them +from non-profiler entities in the source. + +Each marker payload must derive ``serde::Serialize`` and ``serde::Deserialize``. +They are also exported from ``gecko-profiler`` crate if a project doesn't have it. +Each marker payload should include its data as its fields like this: + +.. code-block:: rust + + #[derive(Serialize, Deserialize, Debug)] + pub struct YourMarker { + number: i32, + text: String, + } + +Each marker struct must also implement the `ProfilerMarker`_ trait. + +``ProfilerMarker`` trait +************************ + +`ProfilerMarker`_ trait must be implemented for all marker types. Its methods are +similar to C++ counterparts, please refer to :ref:`the C++ markers guide to learn +more about them <how-to-define-new-marker-types>`. It includes three methods that +needs to be implemented: + +1. ``marker_type_name() -> &'static str``: + A marker type must have a unique name, it is used to keep track of the type of + markers in the profiler storage, and to identify them uniquely on profiler.firefox.com. + (It does not need to be the same as the struct's name.) + + E.g.: + + .. code-block:: rust + + fn marker_type_name() -> &'static str { + "your marker type" + } + +2. ``stream_json_marker_data(&self, json_writer: &mut JSONWriter)`` + All markers of any type have some common data: A name, a category, options like + timing, etc. as previously explained. + + In addition, a certain marker type may carry zero of more arbitrary pieces of + information, and they are always the same for all markers of that type. + + These are defined in a special static member function ``stream_json_marker_data``. + + It's a member method and takes a ``&mut JSONWriter`` as a parameter, + it will be used to stream the data as JSON, to later be read by + profiler.firefox.com. See `JSONWriter object and its methods`_. + + E.g.: + + .. code-block:: rust + + fn stream_json_marker_data(&self, json_writer: &mut JSONWriter) { + json_writer.int_property("number", self.number.into()); + json_writer.string_property("text", &self.text); + } + +3. ``marker_type_display() -> schema::MarkerSchema`` + Now that how to stream type-specific data (from Firefox to + profiler.firefox.com) is defined, it needs to be described where and how this + data will be displayed on profiler.firefox.com. + + The static member function ``marker_type_display`` returns an opaque ``MarkerSchema`` + object, which will be forwarded to profiler.firefox.com. + + See the `MarkerSchema::Location enumeration for the full list`_. Also see the + `MarkerSchema struct for its possible methods`_. + + E.g.: + + .. code-block:: rust + + fn marker_type_display() -> schema::MarkerSchema { + // Import MarkerSchema related types for easier use. + use crate::marker::schema::*; + // Create a MarkerSchema struct with a list of locations provided. + // One or more constructor arguments determine where this marker will be displayed in + // the profiler.firefox.com UI. + let mut schema = MarkerSchema::new(&[Location::MarkerChart]); + + // Some labels can optionally be specified, to display certain information in different + // locations: set_chart_label, set_tooltip_label, and set_table_label``; or + // set_all_labels to define all of them the same way. + schema.set_all_labels("{marker.name} - {marker.data.number}); + + // Next, define the main display of marker data, which will appear in the Marker Chart + // tooltips and the Marker Table sidebar. + schema.add_key_label_format("number", "Number", Format::Number); + schema.add_key_label_format("text", "Text", Format::String); + schema.add_static_label_value("Help", "This is my own marker type"); + + // Lastly, return the created schema. + schema + } + + Note that the strings in ``set_all_labels`` may refer to marker data within braces: + + * ``{marker.name}``: Marker name. + * ``{marker.data.X}``: Type-specific data, as streamed with property name "X" + from ``stream_json_marker_data``. + + :ref:`See the C++ markers guide for more details about it <marker-type-display-schema>`. + +.. _profiling_categories.yaml: https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/build/profiling_categories.yaml +.. _Profiling category pair: https://searchfox.org/mozilla-central/source/__GENERATED__/tools/profiler/rust-api/src/gecko_bindings/profiling_categories.rs +.. _the list of categories: https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/build/profiling_categories.yaml +.. _MarkerOptions: https://searchfox.org/mozilla-central/define?q=rust_analyzer::cargo::gecko_profiler::0_1_0::options::marker::MarkerOptions +.. _MarkerTiming: https://searchfox.org/mozilla-central/define?q=rust_analyzer::cargo::gecko_profiler::0_1_0::options::marker::MarkerTiming +.. _MarkerStack: https://searchfox.org/mozilla-central/define?q=rust_analyzer::cargo::gecko_profiler::0_1_0::options::marker::[MarkerStack] +.. _ProfilerMarker: https://searchfox.org/mozilla-central/define?q=rust_analyzer::cargo::gecko_profiler::0_1_0::marker::ProfilerMarker +.. _MarkerSchema::Location enumeration for the full list: https://searchfox.org/mozilla-central/define?q=T_mozilla%3A%3AMarkerSchema%3A%3ALocation +.. _JSONWriter object and its methods: https://searchfox.org/mozilla-central/define?q=rust_analyzer::cargo::gecko_profiler::0_1_0::json_writer::JSONWriter +.. _MarkerSchema struct for its possible methods: https://searchfox.org/mozilla-central/define?q=rust_analyzer::cargo::gecko_profiler::0_1_0::schema::marker::MarkerSchema diff --git a/tools/profiler/docs/markers-guide.rst b/tools/profiler/docs/markers-guide.rst new file mode 100644 index 0000000000..86744c523c --- /dev/null +++ b/tools/profiler/docs/markers-guide.rst @@ -0,0 +1,551 @@ +Markers +======= + +Markers are packets of arbitrary data that are added to a profile by the Firefox code, usually to +indicate something important happening at a point in time, or during an interval of time. + +Each marker has a name, a category, some common optional information (timing, backtrace, etc.), +and an optional payload of a specific type (containing arbitrary data relevant to that type). + +.. note:: + This guide explains C++ markers in depth. To learn more about how to add a + marker in JavaScript or Rust, please take a look at their documentation + in :doc:`instrumenting-javascript` or :doc:`instrumenting-rust` respectively. + +Example +------- + +Short example, details below. + +Note: Most marker-related identifiers are in the ``mozilla`` namespace, to be added where necessary. + +.. code-block:: cpp + + // Record a simple marker with the category of DOM. + PROFILER_MARKER_UNTYPED("Marker Name", DOM); + + // Create a marker with some additional text information. (Be wary of printf!) + PROFILER_MARKER_TEXT("Marker Name", JS, MarkerOptions{}, "Additional text information."); + + // Record a custom marker of type `ExampleNumberMarker` (see definition below). + PROFILER_MARKER("Number", OTHER, MarkerOptions{}, ExampleNumberMarker, 42); + +.. code-block:: cpp + + // Marker type definition. + struct ExampleNumberMarker : public BaseMarkerType<ExampleNumberMarker> { + // Unique marker type name. + static constexpr const char* Name = "number"; + // Marker description. + static constexpr const char* Description = "This is a number marker."; + + // For convenience. + using MS = MarkerSchema; + // Fields of payload for the marker. + static constexpr MS::PayloadField PayloadFields[] = { + {"number", MS::InputType::Uint32t, "Number", MS::Format::Integer}}; + + // Locations this marker should be displayed. + static constexpr MS::Location Locations[] = {MS::Location::MarkerChart, + MS::Location::MarkerTable}; + // Location specific label for this marker. + static constexpr const char* ChartLabel = "Number: {marker.data.number}"; + + // Data specific to this marker type, as passed to PROFILER_MARKER/profiler_add_marker. + static void StreamJSONMarkerData(SpliceableJSONWriter& aWriter, uint32_t a number) { + // Custom writer for marker fields, or using the default parent + // implementation if the function arguments match the schema. + StreamJSONMarkerDataImpl(aWriter, a number); + } + }; + +When adding a marker whose arguments differ from the schema, a translator +function and a custom implementation of StreamJSONMarkerData can be used. + +.. code-block:: c++ + + // Marker type definition. + struct ExampleBooleanMarker : public BaseMarkerType<ExampleBooleanMarker> { + // Unique marker type name. + static constexpr const char* Name = "boolean"; + // Marker description. + static constexpr const char* Description = "This is a boolean marker."; + + // For convenience. + using MS = MarkerSchema; + // Fields of payload for the marker. + static constexpr MS::PayloadField PayloadFields[] = { + {"boolean", MS::InputType::CString, "Boolean"}}; + + // Locations this marker should be displayed. + static constexpr MS::Location Locations[] = {MS::Location::MarkerChart, + MS::Location::MarkerTable}; + // Location specific label for this marker. + static constexpr const char* ChartLabel = "Boolean: {marker.data.boolean}"; + + // Data specific to this marker type, as passed to PROFILER_MARKER/profiler_add_marker. + static void StreamJSONMarkerData(SpliceableJSONWriter& aWriter, bool aBoolean) { + // Note the schema expects a string, we cannot use the default implementation. + if (aBoolean) { + aWriter.StringProperty("boolean", "true"); + } else { + aWriter.StringProperty("boolean", "false"); + } + } + + // The translation to the schema must also be defined in a translator function. + // The argument list should match that to PROFILER_MARKER/profiler_add_marker. + static void TranslateMarkerInputToSchema(void* aContext, bool aBoolean) { + // This should call ETW::OutputMarkerSchema with an argument list matching the schema. + if (aIsStart) { + ETW::OutputMarkerSchema(aContext, ExampleBooleanMarker{}, ProfilerStringView("true")); + } else { + ETW::OutputMarkerSchema(aContext, ExampleBooleanMarker{}, ProfilerStringView("false")); + } + } + }; + +A more detailed description is offered below. + + +How to Record Markers +--------------------- + +Header to Include +^^^^^^^^^^^^^^^^^ + +If the compilation unit only defines and records untyped, text, and/or its own markers, include +`the main profiler markers header <https://searchfox.org/mozilla-central/source/tools/profiler/public/ProfilerMarkers.h>`_: + +.. code-block:: cpp + + #include "mozilla/ProfilerMarkers.h" + +If it also records one of the other common markers defined in +`ProfilerMarkerTypes.h <https://searchfox.org/mozilla-central/source/tools/profiler/public/ProfilerMarkerTypes.h>`_, +include that one instead: + +.. code-block:: cpp + + #include "mozilla/ProfilerMarkerTypes.h" + +And if it uses any other profiler functions (e.g., labels), use +`the main Gecko Profiler header <https://searchfox.org/mozilla-central/source/tools/profiler/public/GeckoProfiler.h>`_ +instead: + +.. code-block:: cpp + + #include "GeckoProfiler.h" + +The above works from source files that end up in libxul, which is true for the majority +of Firefox source code. But some files live outside of libxul, such as mfbt, in which +case the advice is the same but the equivalent headers are from the Base Profiler instead: + +.. code-block:: cpp + + #include "mozilla/BaseProfilerMarkers.h" // Only own/untyped/text markers + #include "mozilla/BaseProfilerMarkerTypes.h" // Only common markers + #include "BaseProfiler.h" // Markers and other profiler functions + +Untyped Markers +^^^^^^^^^^^^^^^ + +Untyped markers don't carry any information apart from common marker data: +Name, category, options. + +.. code-block:: cpp + + PROFILER_MARKER_UNTYPED( + // Name, and category pair. + "Marker Name", OTHER, + // Marker options, may be omitted if all defaults are acceptable. + MarkerOptions(MarkerStack::Capture(), ...)); + +``PROFILER_MARKER_UNTYPED`` is a macro that simplifies the use of the main +``profiler_add_marker`` function, by adding the appropriate namespaces, and a surrounding +``#ifdef MOZ_GECKO_PROFILER`` guard. + +1. Marker name + The first argument is the name of this marker. This will be displayed in most places + the marker is shown. It can be a literal C string, or any dynamic string object. +2. `Category pair name <https://searchfox.org/mozilla-central/source/__GENERATED__/mozglue/baseprofiler/public/ProfilingCategoryList.h>`_ + Choose a category + subcategory from the `the list of categories <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/build/profiling_categories.yaml>`_. + This is the second parameter of each ``SUBCATEGORY`` line, for instance ``LAYOUT_Reflow``. + (Internally, this is really a `MarkerCategory <https://searchfox.org/mozilla-central/define?q=T_mozilla%3A%3AMarkerCategory>`_ + object, in case you need to construct it elsewhere.) +3. `MarkerOptions <https://searchfox.org/mozilla-central/define?q=T_mozilla%3A%3AMarkerOptions>`_ + See the options below. It can be omitted if there are no other arguments, ``{}``, or + ``MarkerOptions()`` (no specified options); only one of the following option types + alone; or ``MarkerOptions(...)`` with one or more of the following options types: + + * `MarkerThreadId <https://searchfox.org/mozilla-central/define?q=T_mozilla%3A%3AMarkerThreadId>`_ + Rarely used, as it defaults to the current thread. Otherwise it specifies the target + "thread id" (aka "track") where the marker should appear; This may be useful when + referring to something that happened on another thread (use ``profiler_current_thread_id()`` + from the original thread to get its id); or for some important markers, they may be + sent to the "main thread", which can be specified with ``MarkerThreadId::MainThread()``. + * `MarkerTiming <https://searchfox.org/mozilla-central/define?q=T_mozilla%3A%3AMarkerTiming>`_ + This specifies an instant or interval of time. It defaults to the current instant if + left unspecified. Otherwise use ``MarkerTiming::InstantAt(timestamp)`` or + ``MarkerTiming::Interval(ts1, ts2)``; timestamps are usually captured with + ``TimeStamp::Now()``. It is also possible to record only the start or the end of an + interval, pairs of start/end markers will be matched by their name. *Note: The + upcoming "marker sets" feature will make this pairing more reliable, and also + allow more than two markers to be connected*. + * `MarkerStack <https://searchfox.org/mozilla-central/define?q=T_mozilla%3A%3AMarkerStack>`_ + By default, markers do not record a "stack" (or "backtrace"). To record a stack at + this point, in the most efficient manner, specify ``MarkerStack::Capture()``. To + record a previously captured stack, first store a stack into a + ``UniquePtr<ProfileChunkedBuffer>`` with ``profiler_capture_backtrace()``, then pass + it to the marker with ``MarkerStack::TakeBacktrace(std::move(stack))``. + * `MarkerInnerWindowId <https://searchfox.org/mozilla-central/define?q=T_mozilla%3A%3AMarkerInnerWindowId>`_ + If you have access to an "inner window id", consider specifying it as an option, to + help profiler.firefox.com to classify them by tab. + +"Auto" Scoped Interval Markers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To capture time intervals around some important operations, it is common to store a timestamp, do the work, +and then record a marker, e.g.: + +.. code-block:: cpp + + void DoTimedWork() { + TimeStamp start = TimeStamp::Now(); + DoWork(); + PROFILER_MARKER_TEXT("Timed work", OTHER, MarkerTiming::IntervalUntilNowFrom(start), "Details"); + } + +`RAII <https://en.cppreference.com/w/cpp/language/raii>`_ objects automate this, by recording the time +when the object is constructed, and later recording the marker when the object is destroyed at the end +of its C++ scope. +This is especially useful if there are multiple scope exit points. + +``AUTO_PROFILER_MARKER_TEXT`` is `the only one implemented <https://searchfox.org/mozilla-central/search?q=id%3AAUTO_PROFILER_MARKER_TEXT`_ at this time. + +.. code-block:: cpp + + void MaybeDoTimedWork(bool aDoIt) { + AUTO_PROFILER_MARKER_TEXT("Timed work", OTHER, "Details"); + if (!aDoIt) { /* Marker recorded here... */ return; } + DoWork(); + /* ... or here. */ + } + +Note that these RAII objects only record one marker. In some situation, a very long +operation could be missed if it hasn't completed by the end of the profiling session. +In this case, consider recording two distinct markers, using +``MarkerTiming::IntervalStart()`` and ``MarkerTiming::IntervalEnd()``. + +Text Markers +^^^^^^^^^^^^ + +Text markers are very common, they carry an extra text as a fourth argument, in addition to +the marker name. Use the following macro: + +.. code-block:: cpp + + PROFILER_MARKER_TEXT( + // Name, category pair, options. + "Marker Name", OTHER, {}, + // Text string. + "Here are some more details." + ); + +As useful as it is, using an expensive ``printf`` operation to generate a complex text +comes with a variety of issues string. It can leak potentially sensitive information +such as URLs can be leaked during the profile sharing step. profiler.firefox.com cannot +access the information programmatically. It won't get the formatting benefits of the +built-in marker schema. Please consider using a custom marker type to separate and +better present the data. + +Other Typed Markers +^^^^^^^^^^^^^^^^^^^ + +From C++ code, a marker of some type ``YourMarker`` (details about type definition follow) can be +recorded like this: + +.. code-block:: cpp + + PROFILER_MARKER( + "YourMarker name", OTHER, + MarkerOptions(MarkerTiming::IntervalUntilNowFrom(someStartTimestamp), + MarkerInnerWindowId(innerWindowId))), + YourMarker, "some string", 12345, "http://example.com", someTimeStamp); + +After the first three common arguments (like in ``PROFILER_MARKER_UNTYPED``), there are: + +4. The marker type, which is the name of the C++ ``struct`` that defines that type. +5. A variadic list of type-specific argument. They must match the number of, and must + be convertible to the types defined in the schema. If they are not, they must match + the number of and be convertible to the types in ``StreamJSONMarkerData`` and + ``TranslateMarkerInputToSchema``. + +Where to Define New Marker Types +-------------------------------- + +The first step is to determine the location of the marker type definition: + +* If this type is only used in one function, or a component, it can be defined in a + local common place relative to its use. +* For a more common type that could be used from multiple locations: + + * If there is no dependency on XUL, it can be defined in the Base Profiler, which can + be used in most locations in the codebase: + `mozglue/baseprofiler/public/BaseProfilerMarkerTypes.h <https://searchfox.org/mozilla-central/source/mozglue/baseprofiler/public/BaseProfilerMarkerTypes.h>`__ + + * However, if there is a XUL dependency, then it needs to be defined in the Gecko Profiler: + `tools/profiler/public/ProfilerMarkerTypes.h <https://searchfox.org/mozilla-central/source/tools/profiler/public/ProfilerMarkerTypes.h>`__ + +.. _how-to-define-new-marker-types: + +How to Define New Marker Types +------------------------------ + +Each marker type must be defined once and only once. +The definition is a C++ ``struct``, that inherits from ``BaseMarkerType``, its identifier is used when recording +markers of that type in C++. +By convention, the suffix "Marker" is recommended to better distinguish them +from non-profiler entities in the source. + +.. code-block:: cpp + + struct YourMarker : BaseMarkerType<YourMarker> { + +Marker Type Name & Description +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A marker type must have a unique name, it is used to keep track of the type of +markers in the profiler storage, and to identify them uniquely on profiler.firefox.com. +(It does not need to be the same as the ``struct``'s name.) + +This name is defined in a special static data member ``Name``: + +.. code-block:: cpp + + // … + static constexpr const char* Name = "YourMarker"; + +In addition you must add a description of your marker in a special static data member ``Description``: + +.. code-block:: cpp + + // … + static constexpr const char* Description = "This is my marker!"; + +Marker Type Data +^^^^^^^^^^^^^^^^ + +All markers of any type have some common data: A name, a category, options like +timing, etc. as previously explained. + +In addition, a certain marker type may carry zero of more arbitrary pieces of +information, and they are always the same for all markers of that type. + +These are defined in a special static member data array of ``PayloadField`` s. +Each payload field specifies a key, a C++ type description, a label, a format, +and optionally some additional options (see the ``PayloadField`` type). The +most important fields are: + +* Key: Element property name as streamed in ``StreamJSONMarkerData``. +* Type: An enum value describing the C++ type specified to PROFILER_MARKER/profiler_add_marker. +* Label: Prefix to display to label the field. +* Format: How to format the data element value, see `MarkerSchema::Format for details <https://searchfox.org/mozilla-central/define?q=T_mozilla%3A%3AMarkerSchema%3A%3AFormat>`_. + +.. code-block:: cpp + + // … + // This will be used repeatedly and is done for convenience. + using MS = MarkerSchema; + static constexpr MS::PayloadField PayloadFields[] = { + {"number", MS::InputType::Uint32t, "Number", MS::Format::Integer}}; + +In addition, a ``StreamJSONMarkerData`` function must be defined that matches +the C++ argument types to PROFILER_MARKER. + +The first function parameters is always ``SpliceableJSONWriter& aWriter``, +it will be used to stream the data as JSON, to later be read by +profiler.firefox.com. + +.. code-block:: cpp + + // … + static void StreamJSONMarkerData(SpliceableJSONWriter& aWriter, + +The following function parameters is how the data is received as C++ objects +from the call sites. + +* Most C/C++ `POD (Plain Old Data) <https://en.cppreference.com/w/cpp/named_req/PODType>`_ + and `trivially-copyable <https://en.cppreference.com/w/cpp/named_req/TriviallyCopyable>`_ + types should work as-is, including ``TimeStamp``. +* Character strings should be passed using ``const ProfilerString8View&`` (this handles + literal strings, and various ``std::string`` and ``nsCString`` types, and spans with or + without null terminator). Use ``const ProfilerString16View&`` for 16-bit strings such as + ``nsString``. +* Other types can be used if they define specializations for ``ProfileBufferEntryWriter::Serializer`` + and ``ProfileBufferEntryReader::Deserializer``. You should rarely need to define new + ones, but if needed see how existing specializations are written, or contact the + `perf-tools team for help <https://chat.mozilla.org/#/room/#profiler:mozilla.org>`_. + +Passing by value or by reference-to-const is recommended, because arguments are serialized +in binary form (i.e., there are no optimizable ``move`` operations). + +For example, here's how to handle a string, a 64-bit number, another string, and +a timestamp: + +.. code-block:: cpp + + // … + const ProfilerString8View& aString, + const int64_t aBytes, + const ProfilerString8View& aURL, + const TimeStamp& aTime) { + +Then the body of the function turns these parameters into a JSON stream. + +If these parameter types match the types specified in the schema, both in order +and number. It can simply call the default implementation. + +.. code-block:: cpp + + // … + static void StreamJSONMarkerData(SpliceableJSONWriter& aWriter, + const ProfilerString8View& aString, + const int64_t aBytes, + const ProfilerString8View& aURL, + const TimeStamp& aTime) { + StreamJSONMarkerDataImpl(aWrite, aString, aBytes, aURL, aTime); + } + + +If the parameters passed to PROFILER_MARKER do not match the schema, some +additional work is required. + +When this function is called, the writer has just started a JSON object, so +everything that is written should be a named object property. Use +``SpliceableJSONWriter`` functions, in most cases ``...Property`` functions +from its parent class ``JSONWriter``: ``NullProperty``, ``BoolProperty``, +``IntProperty``, ``DoubleProperty``, ``StringProperty``. (Other nested JSON +types like arrays or objects are not supported by the profiler.) + +As a special case, ``TimeStamps`` must be streamed using ``aWriter.TimeProperty(timestamp)``. + +The property names will be used to identify where each piece of data is stored and +how it should be displayed on profiler.firefox.com (see next section). + +Suppose our marker schema defines a string for a boolean, here is how that could be streamed. + +.. code-block:: cpp + + // … + + static void StreamJSONMarkerData(SpliceableJSONWriter& aWriter, + bool aBoolean) { + aWriter.StringProperty("myBoolean", aBoolean ? "true" : "false"); + } + +In addition, a ``TranslateMarkerInputToSchema`` function must be added to +ensure correct output to ETW. + +.. code-block:: c++ + + // The translation to the schema must also be defined in a translator function. + // The argument list should match that to PROFILER_MARKER/profiler_add_marker. + static void TranslateMarkerInputToSchema(void* aContext, bool aBoolean) { + // This should call ETW::OutputMarkerSchema with an argument list matching the schema. + if (aIsStart) { + ETW::OutputMarkerSchema(aContext, YourMarker{}, ProfilerStringView("true")); + } else { + ETW::OutputMarkerSchema(aContext, YourMarker{}, ProfilerStringView("false")); + } + } + +.. _marker-type-display-schema: + +Marker Type Display Schema +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now that we have defined how to stream type-specific data (from Firefox to +profiler.firefox.com), we need to describe where and how this data will be +displayed on profiler.firefox.com. + +The location data member determines where this marker will be displayed in +the profiler.firefox.com UI. See the `MarkerSchema::Location enumeration for the +full list <https://searchfox.org/mozilla-central/define?q=T_mozilla%3A%3AMarkerSchema%3A%3ALocation>`_. + +Here is the most common set of locations, showing markers of that type in both the +Marker Chart and the Marker Table panels: + +.. code-block:: cpp + + // … + static constexpr MS::Location Locations[] = {MS::Location::MarkerChart, + MS::Location::MarkerTable}; + +Some labels can optionally be specified, to display certain information in different +locations: ``ChartLabel``, ``TooltipLabel``, and ``TableLabel``; or ``AllLabels`` to +define all of them the same way. + +The arguments is a string that may refer to marker data within braces: + +* ``{marker.name}``: Marker name. +* ``{marker.data.X}``: Type-specific data, as streamed with property name "X" from ``StreamJSONMarkerData`` (e.g., ``aWriter.IntProperty("X", a number);`` + +For example, here's how to set the Marker Chart label to show the marker name and the +``myBytes`` number of bytes: + +.. code-block:: cpp + + // … + static constexpr const char* ChartLabel = "{marker.name} – {marker.data.myBytes}"; + +profiler.firefox.com will apply the label with the data in a consistent manner. For +example, with this label definition, it could display marker information like the +following in the Firefox Profiler's Marker Chart: + + * "Marker Name – 10B" + * "Marker Name – 25.204KB" + * "Marker Name – 512.54MB" + +For implementation details on this processing, see `src/profiler-logic/marker-schema.js <https://github.com/firefox-devtools/profiler/blob/main/src/profile-logic/marker-schema.js>`_ +in the profiler's front-end. + +Any other ``struct`` member function is ignored. There could be utility functions used by the above +compulsory functions, to make the code clearer. + +And that is the end of the marker definition ``struct``. + +.. code-block:: cpp + + // … + }; + +Performance Considerations +-------------------------- + +During profiling, it is best to reduce the amount of work spent doing profiler +operations, as they can influence the performance of the code that you want to profile. + +Whenever possible, consider passing simple types to marker functions, such that +``StreamJSONMarkerData`` will do the minimum amount of work necessary to serialize +the marker type-specific arguments to its internal buffer representation. POD types +(numbers) and strings are the easiest and cheapest to serialize. Look at the +corresponding ``ProfileBufferEntryWriter::Serializer`` specializations if you +want to better understand the work done. + +Avoid doing expensive operations when recording markers. E.g.: ``printf`` of +different things into a string, or complex computations; instead pass the +``printf``/computation arguments straight through to the marker function, so that +``StreamJSONMarkerData`` can do the expensive work at the end of the profiling session. + +Marker Architecture Description +------------------------------- + +The above sections should give all the information needed for adding your own marker +types. However, if you are wanting to work on the marker architecture itself, this +section will describe how the system works. + +TODO: + * Briefly describe the buffer and serialization. + * Describe the template strategy for generating marker types + * Describe the serialization and link to profiler front-end docs on marker processing (if they exist) diff --git a/tools/profiler/docs/memory.rst b/tools/profiler/docs/memory.rst new file mode 100644 index 0000000000..347a91f9e7 --- /dev/null +++ b/tools/profiler/docs/memory.rst @@ -0,0 +1,46 @@ +Profiling Memory +================ + +Sampling stacks from native allocations +--------------------------------------- + +The profiler can sample allocations and de-allocations from malloc using the +"Native Allocations" feature. This can be enabled by going to `about:profiling` and +enabling the "Native Allocations" checkbox. It is only available in Nightly, as it +uses a technique of hooking into malloc that could be a little more risky to apply to +the broader population of Firefox users. + +This implementation is located in: `tools/profiler/core/memory_hooks.cpp +<https://searchfox.org/mozilla-central/source/tools/profiler/core/memory_hooks.cpp>`_ + +It works by hooking into all of the malloc calls. When the profiler is running, it +performs a `Bernoulli trial`_, that will pass for a given probability of per-byte +allocated. What this means is that larger allocations have a higher chance of being +recorded compared to smaller allocations. Currently, there is no way to configure +the per-byte probability. This means that sampled allocation sizes will be closer +to the actual allocated bytes. + +This infrastructure is quite similar to DMD, but with the additional motiviations of +making it easy to turn on and use with the profiler. The overhead is quite high, +especially on systems with more expensive stack walking, like Linux. Turning off +thee "Native Stacks" feature can help lower overhead, but will give less information. + +For more information on analyzing these profiles, see the `Firefox Profiler docs`_. + +Memory counters +--------------- + +Similar to the Native Allocations feature, memory counters use the malloc memory hook +that is only available in Nightly. When it's available, the memory counters are always +turned on. This is a lightweight way to count in a very granular fashion how much +memory is being allocated and deallocated during the profiling session. + +This information is then visualized in the `Firefox Profiler memory track`_. + +This feature uses the `Profiler Counters`_, which can be used to create other types +of cheap counting instrumentation. + +.. _Bernoulli trial: https://en.wikipedia.org/wiki/Bernoulli_trial +.. _Firefox Profiler docs: https://profiler.firefox.com/docs/#/./memory-allocations +.. _Firefox Profiler memory track: https://profiler.firefox.com/docs/#/./memory-allocations?id=memory-track +.. _Profiler Counters: https://searchfox.org/mozilla-central/source/tools/profiler/public/ProfilerCounts.h diff --git a/tools/profiler/docs/profilerclasses-20220913.png b/tools/profiler/docs/profilerclasses-20220913.png Binary files differnew file mode 100644 index 0000000000..a5ba265407 --- /dev/null +++ b/tools/profiler/docs/profilerclasses-20220913.png diff --git a/tools/profiler/docs/profilerclasses.umlet.uxf b/tools/profiler/docs/profilerclasses.umlet.uxf new file mode 100644 index 0000000000..c807853401 --- /dev/null +++ b/tools/profiler/docs/profilerclasses.umlet.uxf @@ -0,0 +1,811 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<diagram program="umlet" version="15.0.0">
+ <zoom_level>10</zoom_level>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>80</x>
+ <y>370</y>
+ <w>340</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>ThreadInfo
+--
+-mName: nsCString
+-mRegisterTime: TimeStamp
+-mThreadId: int
+-mIsMainThread: bool
+--
+NS_INLINE_DECL_THREADSAFE_REFCOUNTING
++Name()
++RegisterTime()
++ThreadId()
++IsMainThread()
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>470</x>
+ <y>300</y>
+ <w>600</w>
+ <h>260</h>
+ </coordinates>
+ <panel_attributes>RacyRegisteredThread
+--
+-mProfilingStackOwner: NotNull<RefPtr<ProfilingStackOwner>>
+-mThreadId
+-mSleep: Atomic<int> /* AWAKE, SLEEPING_NOT_OBSERVED, SLEEPING_OBSERVED */
+-mIsBeingProfiled: Atomic<bool, Relaxed>
+--
++SetIsBeingProfiled()
++IsBeingProfiled()
++ReinitializeOnResume()
++CanDuplicateLastSampleDueToSleep()
++SetSleeping()
++SetAwake()
++IsSleeping()
++ThreadId()
++ProfilingStack()
++ProfilingStackOwner()</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>470</x>
+ <y>650</y>
+ <w>350</w>
+ <h>360</h>
+ </coordinates>
+ <panel_attributes>RegisteredThread
+--
+-mPlatformData: UniquePlatformData
+-mStackTop: const void*
+-mThread: nsCOMPtr<nsIThread>
+-mContext: JSContext*
+-mJSSampling: enum {INACTIVE, ACTIVE_REQUESTED, ACTIVE, INACTIVE_REQUESTED}
+-mmJSFlags: uint32_t
+--
++RacyRegisteredThread()
++GetPlatformData()
++StackTop()
++GetRunningEventDelay()
++SizeOfIncludingThis()
++SetJSContext()
++ClearJSContext()
++GetJSContext()
++Info(): RefPtr<ThreadInfo>
++GetEventTarget(): nsCOMPtr<nsIEventTarget>
++ResetMainThread(nsIThread*)
++StartJSSampling()
++StopJSSampling()
++PollJSSampling()
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>750</x>
+ <y>550</y>
+ <w>180</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+mRacyRegisteredThread</panel_attributes>
+ <additional_attributes>10.0;100.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>290</x>
+ <y>550</y>
+ <w>230</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mThreadInfo: RefPtr<></panel_attributes>
+ <additional_attributes>210.0;100.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>70</x>
+ <y>660</y>
+ <w>340</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>PageInformation
+--
+-mBrowsingContextID: uint64_t
+-mInnerWindowID: uint64_t
+-mUrl: nsCString
+-mEmbedderInnerWindowID: uint64_t
+--
+NS_INLINE_DECL_THREADSAFE_REFCOUNTING
++SizeOfIncludingThis(MallocSizeOf)
++Equals(PageInformation*)
++StreamJSON(SpliceableJSONWriter&)
++InnerWindowID()
++BrowsingContextID()
++Url()
++EmbedderInnerWindowID()
++BufferPositionWhenUnregistered(): Maybe<uint64_t>
++NotifyUnregistered(aBufferPosition: uint64_t)</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>760</x>
+ <y>1890</y>
+ <w>570</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>ProfilerBacktrace
+--
+-mName: UniqueFreePtr<char>
+-mThreadId: int
+-mProfileChunkedBuffer: UniquePtr<ProfileChunkedBuffer>
+-mProfileBuffer: UniquePtr<ProfileBuffer>
+--
++StreamJSON(SpliceableJSONWriter&, aProcessStartTime: TimeStamp, UniqueStacks&)
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>20</x>
+ <y>2140</y>
+ <w>620</w>
+ <h>580</h>
+ </coordinates>
+ <panel_attributes>ProfileChunkedBuffer
+--
+-mMutex: BaseProfilerMaybeMutex
+-mChunkManager: ProfileBufferChunkManager*
+-mOwnedChunkManager: UniquePtr<ProfileBufferChunkManager>
+-mCurrentChunk: UniquePtr<ProfileBufferChunk>
+-mNextChunks: UniquePtr<ProfileBufferChunk>
+-mRequestedChunkHolder: RefPtr<RequestedChunkRefCountedHolder>
+-mNextChunkRangeStart: ProfileBufferIndex
+-mRangeStart: Atomic<ProfileBufferIndex, ReleaseAcquire>
+-mRangeEnd: ProfileBufferIndex
+-mPushedBlockCount: uint64_t
+-mClearedBlockCount: Atomic<uint64_t, ReleaseAcquire>
+--
++Byte = ProfileBufferChunk::Byte
++Length = ProfileBufferChunk::Length
++IsThreadSafe()
++IsInSession()
++ResetChunkManager()
++SetChunkManager()
++Clear()
++BufferLength(): Maybe<size_t>
++SizeOfExcludingThis(MallocSizeOf)
++SizeOfIncludingThis(MallocSizeOf)
++GetState()
++IsThreadSafeAndLockedOnCurrentThread(): bool
++LockAndRun(Callback&&)
++ReserveAndPut(CallbackEntryBytes&&, Callback<auto(Maybe<ProfileBufferEntryWriter>&)>&&)
++Put(aEntryBytes: Length, Callback<auto(Maybe<ProfileBufferEntryWriter>&)>&&)
++PutFrom(const void*, Length)
++PutObjects(const Ts&...)
++PutObject(const T&)
++GetAllChunks()
++Read(Callback<void(Reader&)>&&): bool
++ReadEach(Callback<void(ProfileBufferEntryReader& [, ProfileBufferBlockIndex])>&&)
++ReadAt(ProfileBufferBlockIndex, Callback<void(Maybe<ProfileBufferEntryReader>&&)>&&)
++AppendContents</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>810</x>
+ <y>2100</y>
+ <w>500</w>
+ <h>620</h>
+ </coordinates>
+ <panel_attributes>ProfileBufferChunk
+--
++Header: {
+ mOffsetFirstBlock; mOffsetPastLastBlock; mDoneTimeStamp;
+ mBufferBytes; mBlockCount; mRangeStart; mProcessId;
+ }
+-InternalHeader: { mHeader: Header; mNext: UniquePtr<ProfileBufferChunk>; }
+--
+-mInternalHeader: InternalHeader
+-mBuffer: Byte /* First byte */
+--
++Byte = uint8_t
++Length = uint32_t
++SpanOfBytes = Span<Byte>
+/+Create(aMinBufferBytes: Length): UniquePtr<ProfileBufferChunk>/
++ReserveInitialBlockAsTail(Length): SpanOfBytes
++ReserveBlock(Length): { SpanOfBytes, ProfileBufferBlockIndex }
++MarkDone()
++MarkRecycled()
++ChunkHeader()
++BufferBytes()
++ChunkBytes()
++SizeOfExcludingThis(MallocSizeOf)
++SizeOfIncludingThis(MallocSizeOf)
++RemainingBytes(): Length
++OffsetFirstBlock(): Length
++OffsetPastLastBlock(): Length
++BlockCount(): Length
++ProcessId(): int
++SetProcessId(int)
++RangeStart(): ProfileBufferIndex
++SetRangeStart(ProfileBufferIndex)
++BufferSpan(): Span<const Byte>
++ByteAt(aOffset: Length)
++GetNext(): maybe-const ProfileBufferChunk*
++ReleaseNext(): UniquePtr<ProfileBufferChunk>
++InsertNext(UniquePtr<ProfileBufferChunk>&&)
++Last(): const ProfileBufferChunk*
++SetLast(UniquePtr<ProfileBufferChunk>&&)
+/+Join(UniquePtr<ProfileBufferChunk>&&, UniquePtr<ProfileBufferChunk>&&)/
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>120</x>
+ <y>2850</y>
+ <w>570</w>
+ <h>350</h>
+ </coordinates>
+ <panel_attributes>ProfileBufferEntryReader
+--
+-mCurrentSpan: SpanOfConstBytes
+-mNextSpanOrEmpty: SpanOfConstBytes
+-mCurrentBlockIndex: ProfileBufferBlockIndex
+-mNextBlockIndex: ProfileBufferBlockIndex
+--
++RemainingBytes(): Length
++SetRemainingBytes(Length)
++CurrentBlockIndex(): ProfileBufferBlockIndex
++NextBlockIndex(): ProfileBufferBlockIndex
++EmptyIteratorAtOffset(Length): ProfileBufferEntryReader
++operator*(): const Byte&
++operator++(): ProfileBufferEntryReader&
++operator+=(Length): ProfileBufferEntryReader&
++operator==(const ProfileBufferEntryReader&)
++operator!=(const ProfileBufferEntryReader&)
++ReadULEB128<T>(): T
++ReadBytes(void*, Length)
++ReadIntoObject(T&)
++ReadIntoObjects(Ts&...)
++ReadObject<T>(): T</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>740</x>
+ <y>2850</y>
+ <w>570</w>
+ <h>300</h>
+ </coordinates>
+ <panel_attributes>ProfileBufferEntryWriter
+--
+-mCurrentSpan: SpanOfBytes
+-mNextSpanOrEmpty: SpanOfBytes
+-mCurrentBlockIndex: ProfileBufferBlockIndex
+-mNextBlockIndex: ProfileBufferBlockIndex
+--
++RemainingBytes(): Length
++CurrentBlockIndex(): ProfileBufferBlockIndex
++NextBlockIndex(): ProfileBufferBlockIndex
++operator*(): Byte&
++operator++(): ProfileBufferEntryReader&
++operator+=(Length): ProfileBufferEntryReader&
+/+ULEB128Size(T): unsigned/
++WriteULEB128(T)
+/+SumBytes(const Ts&...): Length/
++WriteFromReader(ProfileBufferEntryReader&, Length)
++WriteObject(const T&)
++WriteObjects(const T&)</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>120</x>
+ <y>3270</y>
+ <w>570</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>ProfileBufferEntryReader::Deserializer<T>
+/to be specialized for all types read from ProfileBufferEntryReader/
+--
+/+ReadInto(ProfileBufferEntryReader&, T&)/
+/+Read<T>(ProfileBufferEntryReader&): T/</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>740</x>
+ <y>3270</y>
+ <w>570</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>ProfileBufferEntryWriter::Serializer<T>
+/to be specialized for all types written into ProfileBufferEntryWriter/
+--
+/+Bytes(const T&): Length/
+/+Write(ProfileBufferEntryWriter&, const T&)/</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>330</x>
+ <y>2710</y>
+ <w>110</w>
+ <h>160</h>
+ </coordinates>
+ <panel_attributes>lt=.>
+<<creates>></panel_attributes>
+ <additional_attributes>10.0;10.0;60.0;140.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>430</x>
+ <y>2710</y>
+ <w>360</w>
+ <h>160</h>
+ </coordinates>
+ <panel_attributes>lt=.>
+<<creates>></panel_attributes>
+ <additional_attributes>10.0;10.0;340.0;140.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>660</x>
+ <y>2710</y>
+ <w>260</w>
+ <h>160</h>
+ </coordinates>
+ <panel_attributes>lt=.>
+<<points into>></panel_attributes>
+ <additional_attributes>10.0;140.0;240.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>870</x>
+ <y>2710</y>
+ <w>140</w>
+ <h>160</h>
+ </coordinates>
+ <panel_attributes>lt=.>
+<<points into>></panel_attributes>
+ <additional_attributes>10.0;140.0;80.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>630</x>
+ <y>2170</y>
+ <w>200</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mCurrentChunk</panel_attributes>
+ <additional_attributes>10.0;20.0;180.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>630</x>
+ <y>2230</y>
+ <w>200</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mNextChunks</panel_attributes>
+ <additional_attributes>10.0;20.0;180.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1100</x>
+ <y>2030</y>
+ <w>170</w>
+ <h>90</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mInternalHeader.mNext</panel_attributes>
+ <additional_attributes>10.0;70.0;10.0;20.0;150.0;20.0;150.0;70.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>490</x>
+ <y>3190</y>
+ <w>70</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=.>
+<<uses>></panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>580</x>
+ <y>3190</y>
+ <w>230</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=.>
+<<uses>></panel_attributes>
+ <additional_attributes>10.0;10.0;210.0;80.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>50</x>
+ <y>1620</y>
+ <w>570</w>
+ <h>410</h>
+ </coordinates>
+ <panel_attributes>ProfileBuffer
+--
+-mFirstSamplingTimeNs: double
+-mLastSamplingTimeNs: double
+-mIntervalNs, etc.: ProfilerStats
+--
++IsThreadSafe(): bool
++AddEntry(const ProfileBufferEntry&): uint64_t
++AddThreadIdEntry(int): uint64_t
++PutObjects(Kind, const Ts&...): ProfileBufferBlockIndex
++CollectCodeLocation(...)
++AddJITInfoForRange(...)
++StreamSamplesToJSON(SpliceableJSONWriter&, aThreadId: int, aSinceTime: double, UniqueStacks&)
++StreamMarkersToJSON(SpliceableJSONWriter&, ...)
++StreamPausedRangesToJSON(SpliceableJSONWriter&, aSinceTime: double)
++StreamProfilerOverheadToJSON(SpliceableJSONWriter&, ...)
++StreamCountersToJSON(SpliceableJSONWriter&, ...)
++DuplicateLsstSample
++DiscardSamplesBeforeTime(aTime: double)
++GetEntry(aPosition: uint64_t): ProfileBufferEntry
++SizeOfExcludingThis(MallocSizeOf)
++SizeOfIncludingThis(MallocSizeOf)
++CollectOverheadStats(...)
++GetProfilerBufferInfo(): ProfilerBufferInfo
++BufferRangeStart(): uint64_t
++BufferRangeEnd(): uint64_t</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>690</x>
+ <y>1620</y>
+ <w>230</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>ProfileBufferEntry
+--
++mKind: Kind
++mStorage: uint8_t[kNumChars=8]</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>930</x>
+ <y>1620</y>
+ <w>440</w>
+ <h>130</h>
+ </coordinates>
+ <panel_attributes>UniqueJSONStrings
+--
+-mStringTableWriter: SpliceableChunkedJSONWriter
+-mStringHashToIndexMap: HashMap<HashNumber, uint32_t>
+--
++SpliceStringTableElements(SpliceableJSONWriter&)
++WriteProperty(JSONWriter&, aName: const char*, aStr: const char*)
++WriteElement(JSONWriter&, aStr: const char*)
++GetOrAddIndex(const char*): uint32_t</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>680</x>
+ <y>1760</y>
+ <w>470</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>UniqueStack
+--
+-mFrameTableWriter: SpliceableChunkedJSONWriter
+-mFrameToIndexMap: HashMap<FrameKey, uint32_t, FrameKeyHasher>
+-mStackTableWriter: SpliceableChunkedJSONWriter
+-mStackToIndexMap: HashMap<StackKey, uint32_t, StackKeyHasher>
+-mJITInfoRanges: Vector<JITFrameInfoForBufferRange></panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>320</x>
+ <y>2020</y>
+ <w>230</w>
+ <h>140</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mEntries: ProfileChunkedBuffer&</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;120.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>1640</y>
+ <w>100</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=.>
+<<uses>></panel_attributes>
+ <additional_attributes>10.0;20.0;80.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>1710</y>
+ <w>340</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=.>
+<<uses>></panel_attributes>
+ <additional_attributes>10.0;20.0;320.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>1800</y>
+ <w>90</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=.>
+<<uses>></panel_attributes>
+ <additional_attributes>10.0;20.0;70.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>1900</y>
+ <w>170</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mProfileBuffer</panel_attributes>
+ <additional_attributes>150.0;20.0;10.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>590</x>
+ <y>1940</y>
+ <w>250</w>
+ <h>220</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mProfileChunkedBuffer</panel_attributes>
+ <additional_attributes>170.0;10.0;10.0;200.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>20</x>
+ <y>1030</y>
+ <w>490</w>
+ <h>550</h>
+ </coordinates>
+ <panel_attributes>CorePS
+--
+/-sInstance: CorePS*/
+-mMainThreadId: int
+-mProcessStartTime: TimeStamp
+-mCoreBuffer: ProfileChunkedBuffer
+-mRegisteredThreads: Vector<UniquePtr<RegisteredThread>>
+-mRegisteredPages: Vector<RefPtr<PageInformation>>
+-mCounters: Vector<BaseProfilerCount*>
+-mLul: UniquePtr<lul::LUL> /* linux only */
+-mProcessName: nsAutoCString
+-mJsFrames: JsFrameBuffer
+--
++Create
++Destroy
++Exists(): bool
++AddSizeOf(...)
++MainThreadId()
++ProcessStartTime()
++CoreBuffer()
++RegisteredThreads(PSLockRef)
++JsFrames(PSLockRef)
+/+AppendRegisteredThread(PSLockRef, UniquePtr<RegisteredThread>)/
+/+RemoveRegisteredThread(PSLockRef, RegisteredThread*)/
++RegisteredPages(PSLockRef)
+/+AppendRegisteredPage(PSLockRef, RefPtr<PageInformation>)/
+/+RemoveRegisteredPage(PSLockRef, aRegisteredInnerWindowID: uint64_t)/
+/+ClearRegisteredPages(PSLockRef)/
++Counters(PSLockRef)
++AppendCounter
++RemoveCounter
++Lul(PSLockRef)
++SetLul(PSLockRef, UniquePtr<lul::LUL>)
++ProcessName(PSLockRef)
++SetProcessName(PSLockRef, const nsACString&)
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>20</x>
+ <y>1570</y>
+ <w>110</w>
+ <h>590</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+mCoreBuffer</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;570.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>160</x>
+ <y>840</y>
+ <w>150</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mRegisteredPages</panel_attributes>
+ <additional_attributes>10.0;190.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>250</x>
+ <y>840</y>
+ <w>240</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mRegisteredThreads</panel_attributes>
+ <additional_attributes>10.0;190.0;220.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>920</x>
+ <y>860</y>
+ <w>340</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>SamplerThread
+--
+-mSampler: Sampler
+-mActivityGeneration: uint32_t
+-mIntervalMicroseconds: int
+-mThread /* OS-specific */
+-mPostSamplingCallbackList: UniquePtr<PostSamplingCallbackListItem>
+--
++Run()
++Stop(PSLockRef)
++AppendPostSamplingCallback(PSLockRef, PostSamplingCallback&&)</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1060</x>
+ <y>600</y>
+ <w>340</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>Sampler
+--
+-mOldSigprofHandler: sigaction
+-mMyPid: int
+-mSamplerTid: int
++sSigHandlerCoordinator
+--
++Disable(PSLockRef)
++SuspendAndSampleAndResumeThread(PSLockRef, const RegisteredThread&, aNow: TimeStamp, const Func&)
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1190</x>
+ <y>780</y>
+ <w>90</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<<-
+mSampler</panel_attributes>
+ <additional_attributes>10.0;80.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>610</x>
+ <y>1130</y>
+ <w>470</w>
+ <h>400</h>
+ </coordinates>
+ <panel_attributes>ActivePS
+--
+/-sInstance: ActivePS*/
+-mGeneration: const uint32_t
+/-sNextGeneration: uint32_t/
+-mCapacity: const PowerOfTwo
+-mDuration: const Maybe<double>
+-mInterval: const double /* milliseconds */
+-mFeatures: const uint32_t
+-mFilters: Vector<std::string>
+-mActiveBrowsingContextID: uint64_t
+-mProfileBufferChunkManager: ProfileBufferChunkManagerWithLocalLimit
+-mProfileBuffer: ProfileBuffer
+-mLiveProfiledThreads: Vector<LiveProfiledThreadData>
+-mDeadProfiledThreads: Vector<UniquePtr<ProfiledThreadData>>
+-mDeadProfiledPages: Vector<RefPtr<PageInformation>>
+-mSamplerThread: SamplerThread* const
+-mInterposeObserver: RefPtr<ProfilerIOInterposeObserver>
+-mPaused: bool
+-mWasPaused: bool /* linux */
+-mBaseProfileThreads: UniquePtr<char[]>
+-mGeckoIndexWhenBaseProfileAdded: ProfileBufferBlockIndex
+-mExitProfiles: Vector<ExitProfile>
+--
++</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>970</x>
+ <y>1040</y>
+ <w>140</w>
+ <h>110</h>
+ </coordinates>
+ <panel_attributes>lt=<<<<-
+mSamplerThread</panel_attributes>
+ <additional_attributes>10.0;90.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLNote</id>
+ <coordinates>
+ <x>500</x>
+ <y>160</y>
+ <w>510</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>bg=red
+This document pre-dates the generated image profilerclasses-20220913.png!
+Unfortunately, the changes to make the image were lost.
+
+This previous version may still be useful to start reconstructing the image,
+if there is a need to update it.</panel_attributes>
+ <additional_attributes/>
+ </element>
+</diagram>
diff --git a/tools/profiler/docs/profilerthreadregistration-20220913.png b/tools/profiler/docs/profilerthreadregistration-20220913.png Binary files differnew file mode 100644 index 0000000000..8f7049d743 --- /dev/null +++ b/tools/profiler/docs/profilerthreadregistration-20220913.png diff --git a/tools/profiler/docs/profilerthreadregistration.umlet.uxf b/tools/profiler/docs/profilerthreadregistration.umlet.uxf new file mode 100644 index 0000000000..3e07215db4 --- /dev/null +++ b/tools/profiler/docs/profilerthreadregistration.umlet.uxf @@ -0,0 +1,710 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<diagram program="umlet" version="15.0.0">
+ <zoom_level>10</zoom_level>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>200</x>
+ <y>330</y>
+ <w>370</w>
+ <h>250</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistry::OffThreadRef
+--
++UnlockedConstReaderCRef() const
++WithUnlockedConstReader(F&& aF) const
++UnlockedConstReaderAndAtomicRWCRef() const
++WithUnlockedConstReaderAndAtomicRW(F&& aF) const
++UnlockedConstReaderAndAtomicRWRef()
++WithUnlockedConstReaderAndAtomicRW(F&& aF)
++UnlockedRWForLockedProfilerCRef()
++WithUnlockedRWForLockedProfiler(F&& aF)
++UnlockedRWForLockedProfilerRef()
++WithUnlockedRWForLockedProfiler(F&& aF)
++ConstLockedRWFromAnyThread()
++WithConstLockedRWFromAnyThread(F&& aF)
++LockedRWFromAnyThread()
++WithLockedRWFromAnyThread(F&& aF)</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>310</x>
+ <y>80</y>
+ <w>560</w>
+ <h>160</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistry
+--
+-sRegistryMutex: RegistryMutex (aka BaseProfilerSharedMutex)
+/exclusive lock used during un/registration, shared lock for other accesses/
+--
+friend class ThreadRegistration
+-Register(ThreadRegistration::OnThreadRef)
+-Unregister(ThreadRegistration::OnThreadRef)
+--
++WithOffThreadRef(ProfilerThreadId, auto&& aF) static
++WithOffThreadRefOr(ProfilerThreadId, auto&& aF, auto&& aFallbackReturn) static: auto</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>310</x>
+ <y>630</y>
+ <w>530</w>
+ <h>260</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistration
+--
+-mDataMutex: DataMutex (aka BaseProfilerMutex)
+-mIsOnHeap: bool
+-mIsRegistryLockedSharedOnThisThread: bool
+-tlsThreadRegistration: MOZ_THREAD_LOCAL(ThreadRegistration*)
+-GetTLS() static: tlsThreadRegistration*
+-GetFromTLS() static: ThreadRegistration*
+--
++ThreadRegistration(const char* aName, const void* aStackTop)
++~ThreadRegistration()
++RegisterThread(const char* aName, const void* aStackTop) static: ProfilingStack*
++UnregisterThread() static
++IsRegistered() static: bool
++GetOnThreadPtr() static OnThreadPtr
++WithOnThreadRefOr(auto&& aF, auto&& aFallbackReturn) static: auto
++IsDataMutexLockedOnCurrentThread() static: bool</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>880</x>
+ <y>620</y>
+ <w>450</w>
+ <h>290</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistration::OnThreadRef
+--
++UnlockedConstReaderCRef() const
++WithUnlockedConstReader(auto&& aF) const: auto
++UnlockedConstReaderAndAtomicRWCRef() const
++WithUnlockedConstReaderAndAtomicRW(auto&& aF) const: auto
++UnlockedConstReaderAndAtomicRWRef()
++WithUnlockedConstReaderAndAtomicRW(auto&& aF): auto
++UnlockedRWForLockedProfilerCRef() const
++WithUnlockedRWForLockedProfiler(auto&& aF) const: auto
++UnlockedRWForLockedProfilerRef()
++WithUnlockedRWForLockedProfiler(auto&& aF): auto
++UnlockedReaderAndAtomicRWOnThreadCRef() const
++WithUnlockedReaderAndAtomicRWOnThread(auto&& aF) const: auto
++UnlockedReaderAndAtomicRWOnThreadRef()
++WithUnlockedReaderAndAtomicRWOnThread(auto&& aF): auto
++RWOnThreadWithLock LockedRWOnThread()
++WithLockedRWOnThread(auto&& aF): auto</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>1040</x>
+ <y>440</y>
+ <w>230</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistration::OnThreadPtr
+--
++operator*(): OnThreadRef
++operator->(): OnThreadRef</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>450</x>
+ <y>940</y>
+ <w>350</w>
+ <h>240</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistrationData
+--
+-mProfilingStack: ProfilingStack
+-mStackTop: const void* const
+-mThread: nsCOMPtr<nsIThread>
+-mJSContext: JSContext*
+-mJsFrameBuffer: JsFrame*
+-mJSFlags: uint32_t
+-Sleep: Atomic<int>
+-mThreadCpuTimeInNsAtLastSleep: Atomic<uint64_t>
+-mWakeCount: Atomic<uint64_t, Relaxed>
+-mRecordWakeCountMutex: BaseProfilerMutex
+-mAlreadyRecordedWakeCount: uint64_t
+-mAlreadyRecordedCpuTimeInMs: uin64_t
+-mThreadProfilingFeatures: ThreadProfilingFeatures</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>460</x>
+ <y>1220</y>
+ <w>330</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistrationUnlockedConstReader
+--
++Info() const: const ThreadRegistrationInfo&
++PlatformDataCRef() const: const PlatformData&
++StackTop() const: const void*</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>440</x>
+ <y>1340</y>
+ <w>370</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistrationUnlockedConstReaderAndAtomicRW
+--
++ProfilingStackCRef() const: const ProfilingStack&
++ProfilingStackRef(): ProfilingStack&
++ProfilingFeatures() const: ThreadProfilingFeatures
++SetSleeping()
++SetAwake()
++GetNewCpuTimeInNs(): uint64_t
++RecordWakeCount() const
++ReinitializeOnResume()
++CanDuplicateLastSampleDueToSleep(): bool
++IsSleeping(): bool</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>460</x>
+ <y>1570</y>
+ <w>330</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistrationUnlockedRWForLockedProfiler
+--
++GetProfiledThreadData(): const ProfiledThreadData*
++GetProfiliedThreadData(): ProfiledThreadData*</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>430</x>
+ <y>1670</y>
+ <w>390</w>
+ <h>50</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistrationUnlockedReaderAndAtomicRWOnThread
+--
++GetJSContext(): JSContext*</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>380</x>
+ <y>1840</y>
+ <w>490</w>
+ <h>190</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistrationLockedRWFromAnyThread
+--
++SetProfilingFeaturesAndData(
+ ThreadProfilingFeatures, ProfiledThreadData*, const PSAutoLock&)
++ClearProfilingFeaturesAndData(const PSAutoLock&)
++GetJsFrameBuffer() const JsFrame*
++GetEventTarget() const: const nsCOMPtr<nsIEventTarget>
++ResetMainThread()
++GetRunningEventDelay(const TimeStamp&, TimeDuration&, TimeDuration&)
++StartJSSampling(uint32_t)
++StopJSSampling()</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>490</x>
+ <y>2070</y>
+ <w>260</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistrationLockedRWOnThread
+--
++SetJSContext(JSContext*)
++ClearJSContext()
++PollJSSampling()</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>1170</y>
+ <w>30</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;50.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>500</x>
+ <y>2190</y>
+ <w>240</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistration::EmbeddedData
+--</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>1290</y>
+ <w>30</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;50.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>1520</y>
+ <w>30</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;50.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>1620</y>
+ <w>30</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;50.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>650</x>
+ <y>1710</y>
+ <w>30</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;130.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>2020</y>
+ <w>30</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;50.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>610</x>
+ <y>2140</y>
+ <w>30</w>
+ <h>70</h>
+ </coordinates>
+ <panel_attributes>lt=<<-</panel_attributes>
+ <additional_attributes>10.0;10.0;10.0;50.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>340</x>
+ <y>880</y>
+ <w>180</w>
+ <h>1370</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+mData</panel_attributes>
+ <additional_attributes>160.0;1350.0;10.0;1350.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>990</x>
+ <y>930</y>
+ <w>210</w>
+ <h>100</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistrationInfo
+--
++Name(): const char*
++RegisterTime(): const TimeStamp&
++ThreadId(): ProfilerThreadId
++IsMainThread(): bool</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>790</x>
+ <y>980</y>
+ <w>220</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+mInfo</panel_attributes>
+ <additional_attributes>200.0;20.0;10.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>990</x>
+ <y>1040</y>
+ <w>210</w>
+ <h>50</h>
+ </coordinates>
+ <panel_attributes>PlatformData
+--
+</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>790</x>
+ <y>1040</y>
+ <w>220</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>>
+mPlatformData</panel_attributes>
+ <additional_attributes>200.0;20.0;10.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>990</x>
+ <y>1100</y>
+ <w>210</w>
+ <h>60</h>
+ </coordinates>
+ <panel_attributes>ProfiledThreadData
+--</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>790</x>
+ <y>1100</y>
+ <w>220</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+mProfiledThreadData: *</panel_attributes>
+ <additional_attributes>200.0;20.0;10.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>710</x>
+ <y>480</y>
+ <w>350</w>
+ <h>170</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+m1=0..1
+mThreadRegistration: *</panel_attributes>
+ <additional_attributes>10.0;150.0;330.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>830</x>
+ <y>580</y>
+ <w>260</w>
+ <h>130</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+m1=1
+mThreadRegistration: *</panel_attributes>
+ <additional_attributes>10.0;110.0;40.0;20.0;220.0;20.0;240.0;40.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1140</x>
+ <y>500</y>
+ <w>90</w>
+ <h>140</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<creates></panel_attributes>
+ <additional_attributes>10.0;120.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>780</x>
+ <y>900</y>
+ <w>450</w>
+ <h>380</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<accesses></panel_attributes>
+ <additional_attributes>10.0;360.0;430.0;360.0;430.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>800</x>
+ <y>900</y>
+ <w>510</w>
+ <h>560</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<accesses></panel_attributes>
+ <additional_attributes>10.0;540.0;420.0;540.0;420.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>780</x>
+ <y>900</y>
+ <w>540</w>
+ <h>720</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<accesses></panel_attributes>
+ <additional_attributes>10.0;700.0;450.0;700.0;450.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>810</x>
+ <y>900</y>
+ <w>520</w>
+ <h>820</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<accesses></panel_attributes>
+ <additional_attributes>10.0;800.0;430.0;800.0;430.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>900</x>
+ <y>2070</y>
+ <w>410</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistration::OnThreadRef::ConstRWOnThreadWithLock
+--
+-mDataLock: BaseProfilerAutoLock
+--
++DataCRef() const: ThreadRegistrationLockedRWOnThread&
++operator->() const: ThreadRegistrationLockedRWOnThread&</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>740</x>
+ <y>2100</y>
+ <w>180</w>
+ <h>40</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+mLockedRWOnThread</panel_attributes>
+ <additional_attributes>10.0;20.0;160.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>1250</x>
+ <y>900</y>
+ <w>90</w>
+ <h>1190</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<creates></panel_attributes>
+ <additional_attributes>10.0;1170.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>660</x>
+ <y>440</y>
+ <w>400</w>
+ <h>210</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<creates></panel_attributes>
+ <additional_attributes>380.0;10.0;10.0;190.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>740</x>
+ <y>880</y>
+ <w>160</w>
+ <h>50</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<creates></panel_attributes>
+ <additional_attributes>140.0;30.0;50.0;30.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>460</x>
+ <y>230</y>
+ <w>150</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+m1=0..N
+sRegistryContainer:
+static Vector<></panel_attributes>
+ <additional_attributes>10.0;100.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>800</x>
+ <y>250</y>
+ <w>470</w>
+ <h>150</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistry::LockedRegistry
+--
+-mRegistryLock: RegistryLockShared (aka BaseProfilerAutoLockShared)
+--
++LockedRegistry()
++~LockedRegistry()
++begin() const: const OffThreadRef*
++end() const: const OffThreadRef*
++begin(): OffThreadRef*
++end(): OffThreadRef*</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>560</x>
+ <y>350</y>
+ <w>260</w>
+ <h>50</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<accesses with
+shared lock></panel_attributes>
+ <additional_attributes>10.0;20.0;240.0;20.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>550</x>
+ <y>390</y>
+ <w>330</w>
+ <h>260</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<updates
+mIsRegistryLockedSharedOnThisThread></panel_attributes>
+ <additional_attributes>10.0;240.0;310.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>330</x>
+ <y>570</y>
+ <w>170</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+m1=1
+mThreadRegistration: *</panel_attributes>
+ <additional_attributes>120.0;60.0;40.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>280</x>
+ <y>570</y>
+ <w>200</w>
+ <h>710</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<accesses></panel_attributes>
+ <additional_attributes>180.0;690.0;10.0;690.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>270</x>
+ <y>570</y>
+ <w>190</w>
+ <h>890</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<accesses></panel_attributes>
+ <additional_attributes>170.0;870.0;10.0;870.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>UMLClass</id>
+ <coordinates>
+ <x>200</x>
+ <y>1740</y>
+ <w>440</w>
+ <h>80</h>
+ </coordinates>
+ <panel_attributes>ThreadRegistry::OffThreadRef::{,Const}RWFromAnyThreadWithLock
+--
+-mDataLock: BaseProfilerAutoLock
+--
++DataCRef() {,const}: ThreadRegistrationLockedRWOnThread&
++operator->() {,const}: ThreadRegistrationLockedRWOnThread&</panel_attributes>
+ <additional_attributes/>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>250</x>
+ <y>570</y>
+ <w>90</w>
+ <h>1190</h>
+ </coordinates>
+ <panel_attributes>lt=<.
+<creates></panel_attributes>
+ <additional_attributes>10.0;1170.0;10.0;10.0</additional_attributes>
+ </element>
+ <element>
+ <id>Relation</id>
+ <coordinates>
+ <x>180</x>
+ <y>1810</y>
+ <w>220</w>
+ <h>120</h>
+ </coordinates>
+ <panel_attributes>lt=->>>>
+mLockedRWFromAnyThread</panel_attributes>
+ <additional_attributes>200.0;100.0;80.0;100.0;80.0;10.0</additional_attributes>
+ </element>
+</diagram>
diff --git a/tools/profiler/gecko/ChildProfilerController.cpp b/tools/profiler/gecko/ChildProfilerController.cpp new file mode 100644 index 0000000000..f51cb9437d --- /dev/null +++ b/tools/profiler/gecko/ChildProfilerController.cpp @@ -0,0 +1,170 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ChildProfilerController.h" + +#include "ProfilerChild.h" + +#include "mozilla/ProfilerState.h" +#include "mozilla/ipc/Endpoint.h" +#include "nsExceptionHandler.h" +#include "nsIThread.h" +#include "nsThreadUtils.h" + +using namespace mozilla::ipc; + +namespace mozilla { + +/* static */ +already_AddRefed<ChildProfilerController> ChildProfilerController::Create( + mozilla::ipc::Endpoint<PProfilerChild>&& aEndpoint) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + RefPtr<ChildProfilerController> cpc = new ChildProfilerController(); + cpc->Init(std::move(aEndpoint)); + return cpc.forget(); +} + +ChildProfilerController::ChildProfilerController() + : mThread(nullptr, "ChildProfilerController::mThread") { + MOZ_COUNT_CTOR(ChildProfilerController); +} + +void ChildProfilerController::Init(Endpoint<PProfilerChild>&& aEndpoint) { + RefPtr<nsIThread> newProfilerChildThread; + if (NS_SUCCEEDED(NS_NewNamedThread("ProfilerChild", + getter_AddRefs(newProfilerChildThread)))) { + { + auto lock = mThread.Lock(); + RefPtr<nsIThread>& lockedmThread = lock.ref(); + MOZ_ASSERT(!lockedmThread, "There is already a ProfilerChild thread"); + // Copy ref'd ptr into mThread. Don't move/swap, so that + // newProfilerChildThread can be used below. + lockedmThread = newProfilerChildThread; + } + // Now that mThread has been set, run SetupProfilerChild on the thread. + newProfilerChildThread->Dispatch( + NewRunnableMethod<Endpoint<PProfilerChild>&&>( + "ChildProfilerController::SetupProfilerChild", this, + &ChildProfilerController::SetupProfilerChild, std::move(aEndpoint)), + NS_DISPATCH_NORMAL); + } +} + +ProfileAndAdditionalInformation +ChildProfilerController::GrabShutdownProfileAndShutdown() { + ProfileAndAdditionalInformation profileAndAdditionalInformation; + ShutdownAndMaybeGrabShutdownProfileFirst(&profileAndAdditionalInformation); + return profileAndAdditionalInformation; +} + +void ChildProfilerController::Shutdown() { + ShutdownAndMaybeGrabShutdownProfileFirst(nullptr); +} + +void ChildProfilerController::ShutdownAndMaybeGrabShutdownProfileFirst( + ProfileAndAdditionalInformation* aOutShutdownProfileInformation) { + // First, get the owning reference out of mThread, so it cannot be used in + // ChildProfilerController after this (including re-entrantly during the + // profilerChildThread->Shutdown() inner event loop below). + RefPtr<nsIThread> profilerChildThread; + { + auto lock = mThread.Lock(); + RefPtr<nsIThread>& lockedmThread = lock.ref(); + lockedmThread.swap(profilerChildThread); + } + if (profilerChildThread) { + if (profiler_is_active()) { + CrashReporter::AnnotateCrashReport( + CrashReporter::Annotation::ProfilerChildShutdownPhase, + "Profiling - Dispatching ShutdownProfilerChild"_ns); + profilerChildThread->Dispatch( + NewRunnableMethod<ProfileAndAdditionalInformation*>( + "ChildProfilerController::ShutdownProfilerChild", this, + &ChildProfilerController::ShutdownProfilerChild, + aOutShutdownProfileInformation), + NS_DISPATCH_NORMAL); + // Shut down the thread. This call will spin until all runnables + // (including the ShutdownProfilerChild runnable) have been processed. + profilerChildThread->Shutdown(); + } else { + CrashReporter::AnnotateCrashReport( + CrashReporter::Annotation::ProfilerChildShutdownPhase, + "Not profiling - Running ShutdownProfilerChild"_ns); + // If we're not profiling, this operation will be very quick, so it can be + // done synchronously. This avoids having to manually shutdown the thread, + // which runs a risky inner event loop, see bug 1613798. + NS_DispatchAndSpinEventLoopUntilComplete( + "ChildProfilerController::ShutdownProfilerChild SYNC"_ns, + profilerChildThread, + NewRunnableMethod<ProfileAndAdditionalInformation*>( + "ChildProfilerController::ShutdownProfilerChild SYNC", this, + &ChildProfilerController::ShutdownProfilerChild, nullptr)); + } + // At this point, `profilerChildThread` should be the last reference to the + // thread, so it will now get destroyed. + } +} + +ChildProfilerController::~ChildProfilerController() { + MOZ_COUNT_DTOR(ChildProfilerController); + +#ifdef DEBUG + { + auto lock = mThread.Lock(); + RefPtr<nsIThread>& lockedmThread = lock.ref(); + MOZ_ASSERT( + !lockedmThread, + "Please call Shutdown before destroying ChildProfilerController"); + } +#endif + MOZ_ASSERT(!mProfilerChild); +} + +void ChildProfilerController::SetupProfilerChild( + Endpoint<PProfilerChild>&& aEndpoint) { + { + auto lock = mThread.Lock(); + RefPtr<nsIThread>& lockedmThread = lock.ref(); + // We should be on the ProfilerChild thread. In rare cases, we could already + // be in shutdown, in which case mThread is null; we still need to continue, + // so that ShutdownProfilerChild can work on a valid mProfilerChild. + MOZ_RELEASE_ASSERT(!lockedmThread || + lockedmThread == NS_GetCurrentThread()); + } + MOZ_ASSERT(aEndpoint.IsValid()); + + mProfilerChild = new ProfilerChild(); + Endpoint<PProfilerChild> endpoint = std::move(aEndpoint); + + if (!endpoint.Bind(mProfilerChild)) { + MOZ_CRASH("Failed to bind ProfilerChild!"); + } +} + +void ChildProfilerController::ShutdownProfilerChild( + ProfileAndAdditionalInformation* aOutShutdownProfileInformation) { + const bool isProfiling = profiler_is_active(); + if (aOutShutdownProfileInformation) { + CrashReporter::AnnotateCrashReport( + CrashReporter::Annotation::ProfilerChildShutdownPhase, + isProfiling ? "Profiling - GrabShutdownProfile"_ns + : "Not profiling - GrabShutdownProfile"_ns); + *aOutShutdownProfileInformation = mProfilerChild->GrabShutdownProfile(); + } + CrashReporter::AnnotateCrashReport( + CrashReporter::Annotation::ProfilerChildShutdownPhase, + isProfiling ? "Profiling - Destroying ProfilerChild"_ns + : "Not profiling - Destroying ProfilerChild"_ns); + mProfilerChild->Destroy(); + mProfilerChild = nullptr; + CrashReporter::AnnotateCrashReport( + CrashReporter::Annotation::ProfilerChildShutdownPhase, + isProfiling + ? "Profiling - ShutdownProfilerChild complete, waiting for thread shutdown"_ns + : "Not Profiling - ShutdownProfilerChild complete, waiting for thread shutdown"_ns); +} + +} // namespace mozilla diff --git a/tools/profiler/gecko/PProfiler.ipdl b/tools/profiler/gecko/PProfiler.ipdl new file mode 100644 index 0000000000..0dda8d3209 --- /dev/null +++ b/tools/profiler/gecko/PProfiler.ipdl @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include ProfilerTypes; + +namespace mozilla { + +// PProfiler is a top-level protocol. It is used to let the main process +// control the Gecko Profiler in other processes, and request profiles from +// those processes. +// It is a top-level protocol so that its child endpoint can be on a +// background thread, so that profiles can be gathered even if the main thread +// is unresponsive. +[ChildImpl=virtual, ParentImpl=virtual, ChildProc=anychild] +async protocol PProfiler +{ +child: + // The unused returned value is to have a promise we can await. + async Start(ProfilerInitParams params) returns (bool unused); + async EnsureStarted(ProfilerInitParams params) returns (bool unused); + async Stop() returns (bool unused); + async Pause() returns (bool unused); + async Resume() returns (bool unused); + async PauseSampling() returns (bool unused); + async ResumeSampling() returns (bool unused); + + async WaitOnePeriodicSampling() returns (bool sampled); + + async AwaitNextChunkManagerUpdate() returns (ProfileBufferChunkManagerUpdate update); + async DestroyReleasedChunksAtOrBefore(TimeStamp timeStamp); + + // The returned shmem may contain an empty string (unavailable), an error + // message starting with '*', or a profile as a stringified JSON object. + async GatherProfile() returns (IPCProfileAndAdditionalInformation profileAndAdditionalInformation); + async GetGatherProfileProgress() returns (GatherProfileProgress progress); + + async ClearAllPages(); +}; + +} // namespace mozilla + diff --git a/tools/profiler/gecko/ProfilerChild.cpp b/tools/profiler/gecko/ProfilerChild.cpp new file mode 100644 index 0000000000..db7ef99423 --- /dev/null +++ b/tools/profiler/gecko/ProfilerChild.cpp @@ -0,0 +1,565 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfilerChild.h" + +#include "GeckoProfiler.h" +#include "platform.h" +#include "ProfilerCodeAddressService.h" +#include "ProfilerControl.h" +#include "ProfilerParent.h" + +#include "chrome/common/ipc_channel.h" +#include "nsPrintfCString.h" +#include "nsThreadUtils.h" + +#include <memory> + +namespace mozilla { + +/* static */ DataMutexBase<ProfilerChild::ProfilerChildAndUpdate, + baseprofiler::detail::BaseProfilerMutex> + ProfilerChild::sPendingChunkManagerUpdate{ + "ProfilerChild::sPendingChunkManagerUpdate"}; + +ProfilerChild::ProfilerChild() + : mThread(NS_GetCurrentThread()), mDestroyed(false) { + MOZ_COUNT_CTOR(ProfilerChild); +} + +ProfilerChild::~ProfilerChild() { MOZ_COUNT_DTOR(ProfilerChild); } + +void ProfilerChild::ResolveChunkUpdate( + PProfilerChild::AwaitNextChunkManagerUpdateResolver& aResolve) { + MOZ_ASSERT(!!aResolve, + "ResolveChunkUpdate should only be called when there's a pending " + "resolver"); + MOZ_ASSERT( + !mChunkManagerUpdate.IsNotUpdate(), + "ResolveChunkUpdate should only be called with a real or final update"); + MOZ_ASSERT( + !mDestroyed, + "ResolveChunkUpdate should not be called if the actor was destroyed"); + if (mChunkManagerUpdate.IsFinal()) { + // Final update, send a special "unreleased value", but don't clear the + // local copy so we know we got the final update. + std::move(aResolve)(ProfilerParent::MakeFinalUpdate()); + } else { + // Optimization note: The ProfileBufferChunkManagerUpdate constructor takes + // the newly-released chunks nsTArray by reference-to-const, therefore + // constructing and then moving the array here would make a copy. So instead + // we first give it an empty array, and then we can write the data directly + // into the update's array. + ProfileBufferChunkManagerUpdate update{ + mChunkManagerUpdate.UnreleasedBytes(), + mChunkManagerUpdate.ReleasedBytes(), + mChunkManagerUpdate.OldestDoneTimeStamp(), + {}}; + update.newlyReleasedChunks().SetCapacity( + mChunkManagerUpdate.NewlyReleasedChunksRef().size()); + for (const ProfileBufferControlledChunkManager::ChunkMetadata& chunk : + mChunkManagerUpdate.NewlyReleasedChunksRef()) { + update.newlyReleasedChunks().EmplaceBack(chunk.mDoneTimeStamp, + chunk.mBufferBytes); + } + + std::move(aResolve)(update); + + // Clear the update we just sent, so it's ready for later updates to be + // folded into it. + mChunkManagerUpdate.Clear(); + } + + // Discard the resolver, so it's empty next time there's a new request. + aResolve = nullptr; +} + +void ProfilerChild::ProcessChunkManagerUpdate( + ProfileBufferControlledChunkManager::Update&& aUpdate) { + if (mDestroyed) { + return; + } + // Always store the data, it could be the final update. + mChunkManagerUpdate.Fold(std::move(aUpdate)); + if (mAwaitNextChunkManagerUpdateResolver) { + // There is already a pending resolver, give it the info now. + ResolveChunkUpdate(mAwaitNextChunkManagerUpdateResolver); + } +} + +/* static */ void ProfilerChild::ProcessPendingUpdate() { + auto lockedUpdate = sPendingChunkManagerUpdate.Lock(); + if (!lockedUpdate->mProfilerChild || lockedUpdate->mUpdate.IsNotUpdate()) { + return; + } + lockedUpdate->mProfilerChild->mThread->Dispatch(NS_NewRunnableFunction( + "ProfilerChild::ProcessPendingUpdate", []() mutable { + auto lockedUpdate = sPendingChunkManagerUpdate.Lock(); + if (!lockedUpdate->mProfilerChild || + lockedUpdate->mUpdate.IsNotUpdate()) { + return; + } + lockedUpdate->mProfilerChild->ProcessChunkManagerUpdate( + std::move(lockedUpdate->mUpdate)); + lockedUpdate->mUpdate.Clear(); + })); +} + +/* static */ bool ProfilerChild::IsLockedOnCurrentThread() { + return sPendingChunkManagerUpdate.Mutex().IsLockedOnCurrentThread(); +} + +void ProfilerChild::SetupChunkManager() { + mChunkManager = profiler_get_controlled_chunk_manager(); + if (NS_WARN_IF(!mChunkManager)) { + return; + } + + // Make sure there are no updates (from a previous run). + mChunkManagerUpdate.Clear(); + { + auto lockedUpdate = sPendingChunkManagerUpdate.Lock(); + lockedUpdate->mProfilerChild = this; + lockedUpdate->mUpdate.Clear(); + } + + mChunkManager->SetUpdateCallback( + [](ProfileBufferControlledChunkManager::Update&& aUpdate) { + // Updates from the chunk manager are stored for later processing. + // We avoid dispatching a task, as this could deadlock (if the queueing + // mutex is held elsewhere). + auto lockedUpdate = sPendingChunkManagerUpdate.Lock(); + if (!lockedUpdate->mProfilerChild) { + return; + } + lockedUpdate->mUpdate.Fold(std::move(aUpdate)); + }); +} + +void ProfilerChild::ResetChunkManager() { + if (!mChunkManager) { + return; + } + + // We have a chunk manager, reset the callback, which will add a final + // pending update. + mChunkManager->SetUpdateCallback({}); + + // Clear the pending update. + auto lockedUpdate = sPendingChunkManagerUpdate.Lock(); + lockedUpdate->mProfilerChild = nullptr; + lockedUpdate->mUpdate.Clear(); + // And process a final update right now. + ProcessChunkManagerUpdate( + ProfileBufferControlledChunkManager::Update(nullptr)); + + mChunkManager = nullptr; + mAwaitNextChunkManagerUpdateResolver = nullptr; +} + +mozilla::ipc::IPCResult ProfilerChild::RecvStart( + const ProfilerInitParams& params, StartResolver&& aResolve) { + nsTArray<const char*> filterArray; + for (size_t i = 0; i < params.filters().Length(); ++i) { + filterArray.AppendElement(params.filters()[i].get()); + } + + profiler_start(PowerOfTwo32(params.entries()), params.interval(), + params.features(), filterArray.Elements(), + filterArray.Length(), params.activeTabID(), params.duration()); + + SetupChunkManager(); + + aResolve(/* unused */ true); + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvEnsureStarted( + const ProfilerInitParams& params, EnsureStartedResolver&& aResolve) { + nsTArray<const char*> filterArray; + for (size_t i = 0; i < params.filters().Length(); ++i) { + filterArray.AppendElement(params.filters()[i].get()); + } + + profiler_ensure_started(PowerOfTwo32(params.entries()), params.interval(), + params.features(), filterArray.Elements(), + filterArray.Length(), params.activeTabID(), + params.duration()); + + SetupChunkManager(); + + aResolve(/* unused */ true); + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvStop(StopResolver&& aResolve) { + ResetChunkManager(); + profiler_stop(); + aResolve(/* unused */ true); + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvPause(PauseResolver&& aResolve) { + profiler_pause(); + aResolve(/* unused */ true); + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvResume(ResumeResolver&& aResolve) { + profiler_resume(); + aResolve(/* unused */ true); + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvPauseSampling( + PauseSamplingResolver&& aResolve) { + profiler_pause_sampling(); + aResolve(/* unused */ true); + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvResumeSampling( + ResumeSamplingResolver&& aResolve) { + profiler_resume_sampling(); + aResolve(/* unused */ true); + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvWaitOnePeriodicSampling( + WaitOnePeriodicSamplingResolver&& aResolve) { + std::shared_ptr<WaitOnePeriodicSamplingResolver> resolve = + std::make_shared<WaitOnePeriodicSamplingResolver>(std::move(aResolve)); + if (!profiler_callback_after_sampling( + [self = RefPtr(this), resolve](SamplingState aSamplingState) mutable { + if (self->mDestroyed) { + return; + } + MOZ_RELEASE_ASSERT(self->mThread); + self->mThread->Dispatch(NS_NewRunnableFunction( + "nsProfiler::WaitOnePeriodicSampling result on main thread", + [resolve = std::move(resolve), aSamplingState]() { + (*resolve)(aSamplingState == + SamplingState::SamplingCompleted || + aSamplingState == + SamplingState::NoStackSamplingCompleted); + })); + })) { + // Callback was not added (e.g., profiler is not running) and will never be + // invoked, so we need to resolve the promise here. + (*resolve)(false); + } + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvClearAllPages() { + profiler_clear_all_pages(); + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvAwaitNextChunkManagerUpdate( + AwaitNextChunkManagerUpdateResolver&& aResolve) { + MOZ_ASSERT(!mDestroyed, + "Recv... should not be called if the actor was destroyed"); + // Pick up pending updates if any. + { + auto lockedUpdate = sPendingChunkManagerUpdate.Lock(); + if (lockedUpdate->mProfilerChild && !lockedUpdate->mUpdate.IsNotUpdate()) { + mChunkManagerUpdate.Fold(std::move(lockedUpdate->mUpdate)); + lockedUpdate->mUpdate.Clear(); + } + } + if (mChunkManagerUpdate.IsNotUpdate()) { + // No data yet, store the resolver for later. + mAwaitNextChunkManagerUpdateResolver = std::move(aResolve); + } else { + // We have data, send it now. + ResolveChunkUpdate(aResolve); + } + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvDestroyReleasedChunksAtOrBefore( + const TimeStamp& aTimeStamp) { + if (mChunkManager) { + mChunkManager->DestroyChunksAtOrBefore(aTimeStamp); + } + return IPC_OK(); +} + +struct GatherProfileThreadParameters + : public external::AtomicRefCounted<GatherProfileThreadParameters> { + MOZ_DECLARE_REFCOUNTED_TYPENAME(GatherProfileThreadParameters) + + GatherProfileThreadParameters( + RefPtr<ProfilerChild> aProfilerChild, + RefPtr<ProgressLogger::SharedProgress> aProgress, + ProfilerChild::GatherProfileResolver&& aResolver) + : profilerChild(std::move(aProfilerChild)), + progress(std::move(aProgress)), + resolver(std::move(aResolver)) {} + + RefPtr<ProfilerChild> profilerChild; + + FailureLatchSource failureLatchSource; + + // Separate RefPtr used when working on separate thread. This way, if the + // "ProfilerChild" thread decides to overwrite its mGatherProfileProgress with + // a new one, the work done here will still only use the old one. + RefPtr<ProgressLogger::SharedProgress> progress; + + // Resolver for the GatherProfile promise. Must only be called on the + // "ProfilerChild" thread. + ProfilerChild::GatherProfileResolver resolver; +}; + +/* static */ +void ProfilerChild::GatherProfileThreadFunction( + void* already_AddRefedParameters) { + PR_SetCurrentThreadName("GatherProfileThread"); + + RefPtr<GatherProfileThreadParameters> parameters = + already_AddRefed<GatherProfileThreadParameters>{ + static_cast<GatherProfileThreadParameters*>( + already_AddRefedParameters)}; + + ProgressLogger progressLogger( + parameters->progress, "Gather-profile thread started", "Profile sent"); + using namespace mozilla::literals::ProportionValue_literals; // For `1_pc`. + + auto writer = + MakeUnique<SpliceableChunkedJSONWriter>(parameters->failureLatchSource); + if (!profiler_get_profile_json( + *writer, + /* aSinceTime */ 0, + /* aIsShuttingDown */ false, + progressLogger.CreateSubLoggerFromTo( + 1_pc, "profiler_get_profile_json started", 99_pc, + "profiler_get_profile_json done"))) { + // Failed to get a profile, reset the writer pointer, so that we'll send a + // failure message. + writer.reset(); + } + + if (NS_WARN_IF(NS_FAILED( + parameters->profilerChild->mThread->Dispatch(NS_NewRunnableFunction( + "ProfilerChild::ProcessPendingUpdate", + [parameters, + // Forward progress logger to on-ProfilerChild-thread task, so + // that it doesn't get marked as 100% done when this off-thread + // function ends. + progressLogger = std::move(progressLogger), + writer = std::move(writer)]() mutable { + // We are now on the ProfilerChild thread, about to send the + // completed profile. Any incoming progress request will now be + // handled after this task ends, so updating the progress is now + // useless and we can just get rid of the progress storage. + if (parameters->profilerChild->mGatherProfileProgress == + parameters->progress) { + // The ProfilerChild progress is still the one we know. + parameters->profilerChild->mGatherProfileProgress = nullptr; + } + + // Shmem allocation and promise resolution must be made on the + // ProfilerChild thread, that's why this task was needed here. + mozilla::ipc::Shmem shmem; + if (writer) { + if (const size_t len = writer->ChunkedWriteFunc().Length(); + len < UINT32_MAX) { + bool shmemSuccess = true; + const bool copySuccess = + writer->ChunkedWriteFunc() + .CopyDataIntoLazilyAllocatedBuffer( + [&](size_t allocationSize) -> char* { + MOZ_ASSERT(allocationSize == len + 1); + if (parameters->profilerChild->AllocShmem( + allocationSize, &shmem)) { + return shmem.get<char>(); + } + shmemSuccess = false; + return nullptr; + }); + if (!shmemSuccess || !copySuccess) { + const nsPrintfCString message( + (!shmemSuccess) + ? "*Could not create shmem for profile from pid " + "%u (%zu B)" + : "*Could not write profile from pid %u (%zu B)", + unsigned(profiler_current_process_id().ToNumber()), + len); + if (parameters->profilerChild->AllocShmem( + message.Length() + 1, &shmem)) { + strcpy(shmem.get<char>(), message.Data()); + } + } + } else { + const nsPrintfCString message( + "*Profile from pid %u bigger (%zu) than shmem max " + "(%zu)", + unsigned(profiler_current_process_id().ToNumber()), len, + size_t(UINT32_MAX)); + if (parameters->profilerChild->AllocShmem( + message.Length() + 1, &shmem)) { + strcpy(shmem.get<char>(), message.Data()); + } + } + writer = nullptr; + } else { + // No profile. + const char* failure = + parameters->failureLatchSource.GetFailure(); + const nsPrintfCString message( + "*Could not generate profile from pid %u%s%s", + unsigned(profiler_current_process_id().ToNumber()), + failure ? ", failure: " : "", failure ? failure : ""); + if (parameters->profilerChild->AllocShmem( + message.Length() + 1, &shmem)) { + strcpy(shmem.get<char>(), message.Data()); + } + } + + SharedLibraryInfo sharedLibraryInfo = + SharedLibraryInfo::GetInfoForSelf(); + parameters->resolver(IPCProfileAndAdditionalInformation{ + shmem, Some(ProfileGenerationAdditionalInformation{ + std::move(sharedLibraryInfo)})}); + }))))) { + // Failed to dispatch the task to the ProfilerChild thread. The IPC cannot + // be resolved on this thread, so it will never be resolved! + // And it would be unsafe to modify mGatherProfileProgress; But the parent + // should notice that's it's not advancing anymore. + } +} + +mozilla::ipc::IPCResult ProfilerChild::RecvGatherProfile( + GatherProfileResolver&& aResolve) { + mGatherProfileProgress = MakeRefPtr<ProgressLogger::SharedProgress>(); + mGatherProfileProgress->SetProgress(ProportionValue{0.0}, + "Received gather-profile request"); + + auto parameters = MakeRefPtr<GatherProfileThreadParameters>( + this, mGatherProfileProgress, std::move(aResolve)); + + // The GatherProfileThreadFunction thread function will cast its void* + // argument to already_AddRefed<GatherProfileThreadParameters>. + parameters.get()->AddRef(); + PRThread* gatherProfileThread = PR_CreateThread( + PR_SYSTEM_THREAD, GatherProfileThreadFunction, parameters.get(), + PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, PR_UNJOINABLE_THREAD, 0); + + if (!gatherProfileThread) { + // Failed to create and start worker thread, resolve with an empty profile. + mozilla::ipc::Shmem shmem; + if (AllocShmem(1, &shmem)) { + shmem.get<char>()[0] = '\0'; + } + parameters->resolver(IPCProfileAndAdditionalInformation{shmem, Nothing()}); + // And clean up. + parameters.get()->Release(); + mGatherProfileProgress = nullptr; + } + + return IPC_OK(); +} + +mozilla::ipc::IPCResult ProfilerChild::RecvGetGatherProfileProgress( + GetGatherProfileProgressResolver&& aResolve) { + if (mGatherProfileProgress) { + aResolve(GatherProfileProgress{ + mGatherProfileProgress->Progress().ToUnderlyingType(), + nsCString(mGatherProfileProgress->LastLocation())}); + } else { + aResolve( + GatherProfileProgress{ProportionValue::MakeInvalid().ToUnderlyingType(), + nsCString("No gather-profile in progress")}); + } + return IPC_OK(); +} + +void ProfilerChild::ActorDestroy(ActorDestroyReason aActorDestroyReason) { + mDestroyed = true; +} + +void ProfilerChild::Destroy() { + ResetChunkManager(); + if (!mDestroyed) { + Close(); + } +} + +ProfileAndAdditionalInformation ProfilerChild::GrabShutdownProfile() { + LOG("GrabShutdownProfile"); + + UniquePtr<ProfilerCodeAddressService> service = + profiler_code_address_service_for_presymbolication(); + FailureLatchSource failureLatch; + SpliceableChunkedJSONWriter writer{failureLatch}; + writer.Start(); + auto rv = profiler_stream_json_for_this_process( + writer, /* aSinceTime */ 0, + /* aIsShuttingDown */ true, service.get(), ProgressLogger{}); + if (rv.isErr()) { + const char* failure = writer.GetFailure(); + return ProfileAndAdditionalInformation( + nsPrintfCString("*Profile unavailable for pid %u%s%s", + unsigned(profiler_current_process_id().ToNumber()), + failure ? ", failure: " : "", failure ? failure : "")); + } + + auto additionalInfo = rv.unwrap(); + + writer.StartArrayProperty("processes"); + writer.EndArray(); + writer.End(); + + const size_t len = writer.ChunkedWriteFunc().Length(); + // This string and information are destined to be sent as a shutdown profile, + // which is limited by the maximum IPC message size. + // TODO: IPC to change to shmem (bug 1780330), raising this limit to + // JS::MaxStringLength. + if (len + additionalInfo.SizeOf() >= + size_t(IPC::Channel::kMaximumMessageSize)) { + return ProfileAndAdditionalInformation( + nsPrintfCString("*Profile from pid %u bigger (%zu) than IPC max (%zu)", + unsigned(profiler_current_process_id().ToNumber()), len, + size_t(IPC::Channel::kMaximumMessageSize))); + } + + nsCString profileCString; + if (!profileCString.SetLength(len, fallible)) { + return ProfileAndAdditionalInformation(nsPrintfCString( + "*Could not allocate %zu bytes for profile from pid %u", len, + unsigned(profiler_current_process_id().ToNumber()))); + } + MOZ_ASSERT(*(profileCString.Data() + len) == '\0', + "We expected a null at the end of the string buffer, to be " + "rewritten by CopyDataIntoLazilyAllocatedBuffer"); + + char* const profileBeginWriting = profileCString.BeginWriting(); + if (!profileBeginWriting) { + return ProfileAndAdditionalInformation( + nsPrintfCString("*Could not write profile from pid %u", + unsigned(profiler_current_process_id().ToNumber()))); + } + + // Here, we have enough space reserved in `profileCString`, starting at + // `profileBeginWriting`, copy the JSON profile there. + if (!writer.ChunkedWriteFunc().CopyDataIntoLazilyAllocatedBuffer( + [&](size_t aBufferLen) -> char* { + MOZ_RELEASE_ASSERT(aBufferLen == len + 1); + return profileBeginWriting; + })) { + return ProfileAndAdditionalInformation( + nsPrintfCString("*Could not copy profile from pid %u", + unsigned(profiler_current_process_id().ToNumber()))); + } + MOZ_ASSERT(*(profileCString.Data() + len) == '\0', + "We still expected a null at the end of the string buffer"); + + return ProfileAndAdditionalInformation{std::move(profileCString), + std::move(additionalInfo)}; +} + +} // namespace mozilla diff --git a/tools/profiler/gecko/ProfilerIOInterposeObserver.cpp b/tools/profiler/gecko/ProfilerIOInterposeObserver.cpp new file mode 100644 index 0000000000..cf33789f69 --- /dev/null +++ b/tools/profiler/gecko/ProfilerIOInterposeObserver.cpp @@ -0,0 +1,216 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfilerIOInterposeObserver.h" +#include "GeckoProfiler.h" + +using namespace mozilla; + +/* static */ +ProfilerIOInterposeObserver& ProfilerIOInterposeObserver::GetInstance() { + static ProfilerIOInterposeObserver sProfilerIOInterposeObserver; + return sProfilerIOInterposeObserver; +} + +namespace geckoprofiler::markers { +struct FileIOMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("FileIO"); + } + static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter, + const ProfilerString8View& aOperation, + const ProfilerString8View& aSource, + const ProfilerString8View& aFilename, + MarkerThreadId aOperationThreadId) { + aWriter.StringProperty("operation", aOperation); + aWriter.StringProperty("source", aSource); + if (aFilename.Length() != 0) { + aWriter.StringProperty("filename", aFilename); + } + if (!aOperationThreadId.IsUnspecified()) { + // Tech note: If `ToNumber()` returns a uint64_t, the conversion to + // int64_t is "implementation-defined" before C++20. This is acceptable + // here, because this is a one-way conversion to a unique identifier + // that's used to visually separate data by thread on the front-end. + aWriter.IntProperty( + "threadId", + static_cast<int64_t>(aOperationThreadId.ThreadId().ToNumber())); + } + } + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable, + MS::Location::TimelineFileIO}; + schema.AddKeyLabelFormatSearchable("operation", "Operation", + MS::Format::String, + MS::Searchable::Searchable); + schema.AddKeyLabelFormatSearchable("source", "Source", MS::Format::String, + MS::Searchable::Searchable); + schema.AddKeyLabelFormatSearchable("filename", "Filename", + MS::Format::FilePath, + MS::Searchable::Searchable); + schema.AddKeyLabelFormatSearchable("threadId", "Thread ID", + MS::Format::String, + MS::Searchable::Searchable); + return schema; + } +}; +} // namespace geckoprofiler::markers + +static auto GetFilename(IOInterposeObserver::Observation& aObservation) { + AUTO_PROFILER_STATS(IO_filename); + constexpr size_t scExpectedMaxFilename = 512; + nsAutoStringN<scExpectedMaxFilename> filename16; + aObservation.Filename(filename16); + nsAutoCStringN<scExpectedMaxFilename> filename8; + if (!filename16.IsEmpty()) { + CopyUTF16toUTF8(filename16, filename8); + } + return filename8; +} + +void ProfilerIOInterposeObserver::Observe(Observation& aObservation) { + if (profiler_is_locked_on_current_thread()) { + // Don't observe I/Os originating from the profiler itself (when internally + // locked) to avoid deadlocks when calling profiler functions. + AUTO_PROFILER_STATS(IO_profiler_locked); + return; + } + + Maybe<uint32_t> maybeFeatures = profiler_features_if_active_and_unpaused(); + if (maybeFeatures.isNothing()) { + return; + } + uint32_t features = *maybeFeatures; + + if (!profiler_thread_is_being_profiled_for_markers( + profiler_main_thread_id()) && + !profiler_thread_is_being_profiled_for_markers()) { + return; + } + + AUTO_PROFILER_LABEL("ProfilerIOInterposeObserver", PROFILER); + if (IsMainThread()) { + // This is the main thread. + // Capture a marker if any "IO" feature is on. + // If it's not being profiled, we have nowhere to store FileIO markers. + if (!profiler_thread_is_being_profiled_for_markers() || + !(features & ProfilerFeature::MainThreadIO)) { + return; + } + AUTO_PROFILER_STATS(IO_MT); + nsAutoCString type{aObservation.FileType()}; + type.AppendLiteral("IO"); + + // Store the marker in the current thread. + PROFILER_MARKER( + type, OTHER, + MarkerOptions( + MarkerTiming::Interval(aObservation.Start(), aObservation.End()), + MarkerStack::Capture()), + FileIOMarker, + // aOperation + ProfilerString8View::WrapNullTerminatedString( + aObservation.ObservedOperationString()), + // aSource + ProfilerString8View::WrapNullTerminatedString(aObservation.Reference()), + // aFilename + GetFilename(aObservation), + // aOperationThreadId - Do not include a thread ID, as it's the same as + // the markers. Only include this field when the marker is being sent + // from another thread. + MarkerThreadId{}); + + } else if (profiler_thread_is_being_profiled_for_markers()) { + // This is a non-main thread that is being profiled. + if (!(features & ProfilerFeature::FileIO)) { + return; + } + AUTO_PROFILER_STATS(IO_off_MT); + + nsAutoCString type{aObservation.FileType()}; + type.AppendLiteral("IO"); + + // Share a backtrace between the marker on this thread, and the marker on + // the main thread. + UniquePtr<ProfileChunkedBuffer> backtrace = profiler_capture_backtrace(); + + // Store the marker in the current thread. + PROFILER_MARKER( + type, OTHER, + MarkerOptions( + MarkerTiming::Interval(aObservation.Start(), aObservation.End()), + backtrace ? MarkerStack::UseBacktrace(*backtrace) + : MarkerStack::NoStack()), + FileIOMarker, + // aOperation + ProfilerString8View::WrapNullTerminatedString( + aObservation.ObservedOperationString()), + // aSource + ProfilerString8View::WrapNullTerminatedString(aObservation.Reference()), + // aFilename + GetFilename(aObservation), + // aOperationThreadId - Do not include a thread ID, as it's the same as + // the markers. Only include this field when the marker is being sent + // from another thread. + MarkerThreadId{}); + + // Store the marker in the main thread as well, with a distinct marker name + // and thread id. + type.AppendLiteral(" (non-main thread)"); + PROFILER_MARKER( + type, OTHER, + MarkerOptions( + MarkerTiming::Interval(aObservation.Start(), aObservation.End()), + backtrace ? MarkerStack::UseBacktrace(*backtrace) + : MarkerStack::NoStack(), + // This is the important piece that changed. + // It will send a marker to the main thread. + MarkerThreadId::MainThread()), + FileIOMarker, + // aOperation + ProfilerString8View::WrapNullTerminatedString( + aObservation.ObservedOperationString()), + // aSource + ProfilerString8View::WrapNullTerminatedString(aObservation.Reference()), + // aFilename + GetFilename(aObservation), + // aOperationThreadId - Include the thread ID in the payload. + MarkerThreadId::CurrentThread()); + + } else { + // This is a thread that is not being profiled. We still want to capture + // file I/Os (to the main thread) if the "FileIOAll" feature is on. + if (!(features & ProfilerFeature::FileIOAll)) { + return; + } + AUTO_PROFILER_STATS(IO_other); + nsAutoCString type{aObservation.FileType()}; + if (profiler_is_active_and_thread_is_registered()) { + type.AppendLiteral("IO (non-profiled thread)"); + } else { + type.AppendLiteral("IO (unregistered thread)"); + } + + // Only store this marker on the main thread, as this thread was not being + // profiled. + PROFILER_MARKER( + type, OTHER, + MarkerOptions( + MarkerTiming::Interval(aObservation.Start(), aObservation.End()), + MarkerStack::Capture(), + // Store this marker on the main thread. + MarkerThreadId::MainThread()), + FileIOMarker, + // aOperation + ProfilerString8View::WrapNullTerminatedString( + aObservation.ObservedOperationString()), + // aSource + ProfilerString8View::WrapNullTerminatedString(aObservation.Reference()), + // aFilename + GetFilename(aObservation), + // aOperationThreadId - Note which thread this marker is coming from. + MarkerThreadId::CurrentThread()); + } +} diff --git a/tools/profiler/gecko/ProfilerIOInterposeObserver.h b/tools/profiler/gecko/ProfilerIOInterposeObserver.h new file mode 100644 index 0000000000..9e22a34f15 --- /dev/null +++ b/tools/profiler/gecko/ProfilerIOInterposeObserver.h @@ -0,0 +1,32 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef PROFILERIOINTERPOSEOBSERVER_H +#define PROFILERIOINTERPOSEOBSERVER_H + +#include "mozilla/IOInterposer.h" +#include "nsISupportsImpl.h" + +namespace mozilla { + +/** + * This class is the observer that calls into the profiler whenever + * main thread I/O occurs. + */ +class ProfilerIOInterposeObserver final : public IOInterposeObserver { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ProfilerIOInterposeObserver) + + public: + static ProfilerIOInterposeObserver& GetInstance(); + + virtual void Observe(Observation& aObservation) override; + + private: + ProfilerIOInterposeObserver() = default; + virtual ~ProfilerIOInterposeObserver() {} +}; + +} // namespace mozilla + +#endif // PROFILERIOINTERPOSEOBSERVER_H diff --git a/tools/profiler/gecko/ProfilerParent.cpp b/tools/profiler/gecko/ProfilerParent.cpp new file mode 100644 index 0000000000..403a4c2d62 --- /dev/null +++ b/tools/profiler/gecko/ProfilerParent.cpp @@ -0,0 +1,1005 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ProfilerParent.h" + +#ifdef MOZ_GECKO_PROFILER +# include "nsProfiler.h" +# include "platform.h" +#endif + +#include "GeckoProfiler.h" +#include "ProfilerControl.h" +#include "mozilla/BaseAndGeckoProfilerDetail.h" +#include "mozilla/BaseProfilerDetail.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/DataMutex.h" +#include "mozilla/IOInterposer.h" +#include "mozilla/ipc/Endpoint.h" +#include "mozilla/Maybe.h" +#include "mozilla/ProfileBufferControlledChunkManager.h" +#include "mozilla/ProfilerBufferSize.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Unused.h" +#include "nsTArray.h" +#include "nsThreadUtils.h" + +#include <utility> + +namespace mozilla { + +using namespace ipc; + +/* static */ +Endpoint<PProfilerChild> ProfilerParent::CreateForProcess( + base::ProcessId aOtherPid) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + Endpoint<PProfilerChild> child; +#ifdef MOZ_GECKO_PROFILER + Endpoint<PProfilerParent> parent; + nsresult rv = PProfiler::CreateEndpoints(&parent, &child); + + if (NS_FAILED(rv)) { + MOZ_CRASH("Failed to create top level actor for PProfiler!"); + } + + RefPtr<ProfilerParent> actor = new ProfilerParent(aOtherPid); + if (!parent.Bind(actor)) { + MOZ_CRASH("Failed to bind parent actor for PProfiler!"); + } + + actor->Init(); +#endif + + return child; +} + +#ifdef MOZ_GECKO_PROFILER + +class ProfilerParentTracker; + +// This class is responsible for gathering updates from chunk managers in +// different process, and request for the oldest chunks to be destroyed whenever +// the given memory limit is reached. +class ProfileBufferGlobalController final { + public: + explicit ProfileBufferGlobalController(size_t aMaximumBytes); + + ~ProfileBufferGlobalController(); + + void HandleChildChunkManagerUpdate( + base::ProcessId aProcessId, + ProfileBufferControlledChunkManager::Update&& aUpdate); + + static bool IsLockedOnCurrentThread(); + + private: + // Calls aF(Json::Value&). + template <typename F> + void Log(F&& aF); + + static void LogUpdateChunks(Json::Value& updates, base::ProcessId aProcessId, + const TimeStamp& aTimeStamp, int aChunkDiff); + void LogUpdate(base::ProcessId aProcessId, + const ProfileBufferControlledChunkManager::Update& aUpdate); + void LogDeletion(base::ProcessId aProcessId, const TimeStamp& aTimeStamp); + + void HandleChunkManagerNonFinalUpdate( + base::ProcessId aProcessId, + ProfileBufferControlledChunkManager::Update&& aUpdate, + ProfileBufferControlledChunkManager& aParentChunkManager); + + const size_t mMaximumBytes; + + const base::ProcessId mParentProcessId = base::GetCurrentProcId(); + + struct ParentChunkManagerAndPendingUpdate { + ProfileBufferControlledChunkManager* mChunkManager = nullptr; + ProfileBufferControlledChunkManager::Update mPendingUpdate; + }; + + static DataMutexBase<ParentChunkManagerAndPendingUpdate, + baseprofiler::detail::BaseProfilerMutex> + sParentChunkManagerAndPendingUpdate; + + size_t mUnreleasedTotalBytes = 0; + + struct PidAndBytes { + base::ProcessId mProcessId; + size_t mBytes; + + // For searching and sorting. + bool operator==(base::ProcessId aSearchedProcessId) const { + return mProcessId == aSearchedProcessId; + } + bool operator==(const PidAndBytes& aOther) const { + return mProcessId == aOther.mProcessId; + } + bool operator<(base::ProcessId aSearchedProcessId) const { + return mProcessId < aSearchedProcessId; + } + bool operator<(const PidAndBytes& aOther) const { + return mProcessId < aOther.mProcessId; + } + }; + using PidAndBytesArray = nsTArray<PidAndBytes>; + PidAndBytesArray mUnreleasedBytesByPid; + + size_t mReleasedTotalBytes = 0; + + struct TimeStampAndBytesAndPid { + TimeStamp mTimeStamp; + size_t mBytes; + base::ProcessId mProcessId; + + // For searching and sorting. + bool operator==(const TimeStampAndBytesAndPid& aOther) const { + // Sort first by timestamps, and then by pid in rare cases with the same + // timestamps. + return mTimeStamp == aOther.mTimeStamp && mProcessId == aOther.mProcessId; + } + bool operator<(const TimeStampAndBytesAndPid& aOther) const { + // Sort first by timestamps, and then by pid in rare cases with the same + // timestamps. + return mTimeStamp < aOther.mTimeStamp || + (MOZ_UNLIKELY(mTimeStamp == aOther.mTimeStamp) && + mProcessId < aOther.mProcessId); + } + }; + using TimeStampAndBytesAndPidArray = nsTArray<TimeStampAndBytesAndPid>; + TimeStampAndBytesAndPidArray mReleasedChunksByTime; +}; + +/* static */ +DataMutexBase<ProfileBufferGlobalController::ParentChunkManagerAndPendingUpdate, + baseprofiler::detail::BaseProfilerMutex> + ProfileBufferGlobalController::sParentChunkManagerAndPendingUpdate{ + "ProfileBufferGlobalController::sParentChunkManagerAndPendingUpdate"}; + +// This singleton class tracks live ProfilerParent's (meaning there's a current +// connection with a child process). +// It also knows when the local profiler is running. +// And when both the profiler is running and at least one child is present, it +// creates a ProfileBufferGlobalController and forwards chunk updates to it. +class ProfilerParentTracker final { + public: + static void StartTracking(ProfilerParent* aParent); + static void StopTracking(ProfilerParent* aParent); + + static void ProfilerStarted(uint32_t aEntries); + static void ProfilerWillStopIfStarted(); + + // Number of non-destroyed tracked ProfilerParents. + static size_t ProfilerParentCount(); + + template <typename FuncType> + static void Enumerate(FuncType&& aIterFunc); + + template <typename FuncType> + static void ForChild(base::ProcessId aChildPid, FuncType&& aIterFunc); + + static void ForwardChildChunkManagerUpdate( + base::ProcessId aProcessId, + ProfileBufferControlledChunkManager::Update&& aUpdate); + + ProfilerParentTracker(); + ~ProfilerParentTracker(); + + private: + // Get the singleton instance; Create one on the first request, unless we are + // past XPCOMShutdownThreads, which is when it should get destroyed. + static ProfilerParentTracker* GetInstance(); + + // List of parents for currently-connected child processes. + nsTArray<ProfilerParent*> mProfilerParents; + + // If non-0, the parent profiler is running, with this limit (in number of + // entries.) This is needed here, because the parent profiler may start + // running before child processes are known (e.g., startup profiling). + uint32_t mEntries = 0; + + // When the profiler is running and there is at least one parent-child + // connection, this is the controller that should receive chunk updates. + Maybe<ProfileBufferGlobalController> mMaybeController; +}; + +static const Json::StaticString logRoot{"bufferGlobalController"}; + +template <typename F> +void ProfileBufferGlobalController::Log(F&& aF) { + ProfilingLog::Access([&](Json::Value& aLog) { + Json::Value& root = aLog[logRoot]; + if (!root.isObject()) { + root = Json::Value(Json::objectValue); + root[Json::StaticString{"logBegin" TIMESTAMP_JSON_SUFFIX}] = + ProfilingLog::Timestamp(); + } + std::forward<F>(aF)(root); + }); +} + +/* static */ +void ProfileBufferGlobalController::LogUpdateChunks(Json::Value& updates, + base::ProcessId aProcessId, + const TimeStamp& aTimeStamp, + int aChunkDiff) { + MOZ_ASSERT(updates.isArray()); + Json::Value row{Json::arrayValue}; + row.append(Json::Value{Json::UInt64(aProcessId)}); + row.append(ProfilingLog::Timestamp(aTimeStamp)); + row.append(Json::Value{Json::Int(aChunkDiff)}); + updates.append(std::move(row)); +} + +void ProfileBufferGlobalController::LogUpdate( + base::ProcessId aProcessId, + const ProfileBufferControlledChunkManager::Update& aUpdate) { + Log([&](Json::Value& aRoot) { + Json::Value& updates = aRoot[Json::StaticString{"updates"}]; + if (!updates.isArray()) { + aRoot[Json::StaticString{"updatesSchema"}] = + Json::StaticString{"0: pid, 1: chunkRelease_TSms, 3: chunkDiff"}; + updates = Json::Value{Json::arrayValue}; + } + if (aUpdate.IsFinal()) { + LogUpdateChunks(updates, aProcessId, TimeStamp{}, 0); + } else if (!aUpdate.IsNotUpdate()) { + for (const auto& chunk : aUpdate.NewlyReleasedChunksRef()) { + LogUpdateChunks(updates, aProcessId, chunk.mDoneTimeStamp, 1); + } + } + }); +} + +void ProfileBufferGlobalController::LogDeletion(base::ProcessId aProcessId, + const TimeStamp& aTimeStamp) { + Log([&](Json::Value& aRoot) { + Json::Value& updates = aRoot[Json::StaticString{"updates"}]; + if (!updates.isArray()) { + updates = Json::Value{Json::arrayValue}; + } + LogUpdateChunks(updates, aProcessId, aTimeStamp, -1); + }); +} + +ProfileBufferGlobalController::ProfileBufferGlobalController( + size_t aMaximumBytes) + : mMaximumBytes(aMaximumBytes) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + + Log([](Json::Value& aRoot) { + aRoot[Json::StaticString{"controllerCreationTime" TIMESTAMP_JSON_SUFFIX}] = + ProfilingLog::Timestamp(); + }); + + // This is the local chunk manager for this parent process, so updates can be + // handled here. + ProfileBufferControlledChunkManager* parentChunkManager = + profiler_get_controlled_chunk_manager(); + + if (NS_WARN_IF(!parentChunkManager)) { + Log([](Json::Value& aRoot) { + aRoot[Json::StaticString{"controllerCreationFailureReason"}] = + "No parent chunk manager"; + }); + return; + } + + { + auto lockedParentChunkManagerAndPendingUpdate = + sParentChunkManagerAndPendingUpdate.Lock(); + lockedParentChunkManagerAndPendingUpdate->mChunkManager = + parentChunkManager; + } + + parentChunkManager->SetUpdateCallback( + [this](ProfileBufferControlledChunkManager::Update&& aUpdate) { + MOZ_ASSERT(!aUpdate.IsNotUpdate(), + "Update callback should never be given a non-update"); + auto lockedParentChunkManagerAndPendingUpdate = + sParentChunkManagerAndPendingUpdate.Lock(); + if (aUpdate.IsFinal()) { + // Final update of the parent. + // We cannot keep the chunk manager, and there's no point handling + // updates anymore. Do some cleanup now, to free resources before + // we're destroyed. + lockedParentChunkManagerAndPendingUpdate->mChunkManager = nullptr; + lockedParentChunkManagerAndPendingUpdate->mPendingUpdate.Clear(); + mUnreleasedTotalBytes = 0; + mUnreleasedBytesByPid.Clear(); + mReleasedTotalBytes = 0; + mReleasedChunksByTime.Clear(); + return; + } + if (!lockedParentChunkManagerAndPendingUpdate->mChunkManager) { + // No chunk manager, ignore updates. + return; + } + // Special handling of parent non-final updates: + // These updates are coming from *this* process, and may originate from + // scopes in any thread where any lock is held, so using other locks (to + // e.g., dispatch tasks or send IPCs) could trigger a deadlock. Instead, + // parent updates are stored locally and handled when the next + // non-parent update needs handling, see HandleChildChunkManagerUpdate. + lockedParentChunkManagerAndPendingUpdate->mPendingUpdate.Fold( + std::move(aUpdate)); + }); +} + +ProfileBufferGlobalController ::~ProfileBufferGlobalController() { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + // Extract the parent chunk manager (if still set). + // This means any update after this will be ignored. + ProfileBufferControlledChunkManager* parentChunkManager = []() { + auto lockedParentChunkManagerAndPendingUpdate = + sParentChunkManagerAndPendingUpdate.Lock(); + lockedParentChunkManagerAndPendingUpdate->mPendingUpdate.Clear(); + return std::exchange( + lockedParentChunkManagerAndPendingUpdate->mChunkManager, nullptr); + }(); + if (parentChunkManager) { + // We had not received a final update yet, so the chunk manager is still + // valid. Reset the callback in the chunk manager, this will immediately + // invoke the callback with the final empty update; see handling above. + parentChunkManager->SetUpdateCallback({}); + } +} + +void ProfileBufferGlobalController::HandleChildChunkManagerUpdate( + base::ProcessId aProcessId, + ProfileBufferControlledChunkManager::Update&& aUpdate) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + + MOZ_ASSERT(aProcessId != mParentProcessId); + + MOZ_ASSERT(!aUpdate.IsNotUpdate(), + "HandleChildChunkManagerUpdate should not be given a non-update"); + + auto lockedParentChunkManagerAndPendingUpdate = + sParentChunkManagerAndPendingUpdate.Lock(); + if (!lockedParentChunkManagerAndPendingUpdate->mChunkManager) { + // No chunk manager, ignore updates. + return; + } + + if (aUpdate.IsFinal()) { + // Final update in a child process, remove all traces of that process. + LogUpdate(aProcessId, aUpdate); + size_t index = mUnreleasedBytesByPid.BinaryIndexOf(aProcessId); + if (index != PidAndBytesArray::NoIndex) { + // We already have a value for this pid. + PidAndBytes& pidAndBytes = mUnreleasedBytesByPid[index]; + mUnreleasedTotalBytes -= pidAndBytes.mBytes; + mUnreleasedBytesByPid.RemoveElementAt(index); + } + + size_t released = 0; + mReleasedChunksByTime.RemoveElementsBy( + [&released, aProcessId](const auto& chunk) { + const bool match = chunk.mProcessId == aProcessId; + if (match) { + released += chunk.mBytes; + } + return match; + }); + if (released != 0) { + mReleasedTotalBytes -= released; + } + + // Total can only have gone down, so there's no need to check the limit. + return; + } + + // Non-final update in child process. + + // Before handling the child update, we may have pending updates from the + // parent, which can be processed now since we're in an IPC callback outside + // of any profiler-related scope. + if (!lockedParentChunkManagerAndPendingUpdate->mPendingUpdate.IsNotUpdate()) { + MOZ_ASSERT( + !lockedParentChunkManagerAndPendingUpdate->mPendingUpdate.IsFinal()); + HandleChunkManagerNonFinalUpdate( + mParentProcessId, + std::move(lockedParentChunkManagerAndPendingUpdate->mPendingUpdate), + *lockedParentChunkManagerAndPendingUpdate->mChunkManager); + lockedParentChunkManagerAndPendingUpdate->mPendingUpdate.Clear(); + } + + HandleChunkManagerNonFinalUpdate( + aProcessId, std::move(aUpdate), + *lockedParentChunkManagerAndPendingUpdate->mChunkManager); +} + +/* static */ +bool ProfileBufferGlobalController::IsLockedOnCurrentThread() { + return sParentChunkManagerAndPendingUpdate.Mutex().IsLockedOnCurrentThread(); +} + +void ProfileBufferGlobalController::HandleChunkManagerNonFinalUpdate( + base::ProcessId aProcessId, + ProfileBufferControlledChunkManager::Update&& aUpdate, + ProfileBufferControlledChunkManager& aParentChunkManager) { + MOZ_ASSERT(!aUpdate.IsFinal()); + LogUpdate(aProcessId, aUpdate); + + size_t index = mUnreleasedBytesByPid.BinaryIndexOf(aProcessId); + if (index != PidAndBytesArray::NoIndex) { + // We already have a value for this pid. + PidAndBytes& pidAndBytes = mUnreleasedBytesByPid[index]; + mUnreleasedTotalBytes = + mUnreleasedTotalBytes - pidAndBytes.mBytes + aUpdate.UnreleasedBytes(); + pidAndBytes.mBytes = aUpdate.UnreleasedBytes(); + } else { + // New pid. + mUnreleasedBytesByPid.InsertElementSorted( + PidAndBytes{aProcessId, aUpdate.UnreleasedBytes()}); + mUnreleasedTotalBytes += aUpdate.UnreleasedBytes(); + } + + size_t destroyedReleased = 0; + if (!aUpdate.OldestDoneTimeStamp().IsNull()) { + size_t i = 0; + for (; i < mReleasedChunksByTime.Length(); ++i) { + if (mReleasedChunksByTime[i].mTimeStamp >= + aUpdate.OldestDoneTimeStamp()) { + break; + } + } + // Here, i is the index of the first item that's at or after + // aUpdate.mOldestDoneTimeStamp, so chunks from aProcessId before that have + // been destroyed. + while (i != 0) { + --i; + const TimeStampAndBytesAndPid& item = mReleasedChunksByTime[i]; + if (item.mProcessId == aProcessId) { + destroyedReleased += item.mBytes; + mReleasedChunksByTime.RemoveElementAt(i); + } + } + } + + size_t newlyReleased = 0; + for (const ProfileBufferControlledChunkManager::ChunkMetadata& chunk : + aUpdate.NewlyReleasedChunksRef()) { + newlyReleased += chunk.mBufferBytes; + mReleasedChunksByTime.InsertElementSorted(TimeStampAndBytesAndPid{ + chunk.mDoneTimeStamp, chunk.mBufferBytes, aProcessId}); + } + + mReleasedTotalBytes = mReleasedTotalBytes - destroyedReleased + newlyReleased; + +# ifdef DEBUG + size_t totalReleased = 0; + for (const TimeStampAndBytesAndPid& item : mReleasedChunksByTime) { + totalReleased += item.mBytes; + } + MOZ_ASSERT(mReleasedTotalBytes == totalReleased); +# endif // DEBUG + + std::vector<ProfileBufferControlledChunkManager::ChunkMetadata> toDestroy; + while (mUnreleasedTotalBytes + mReleasedTotalBytes > mMaximumBytes && + !mReleasedChunksByTime.IsEmpty()) { + // We have reached the global memory limit, and there *are* released chunks + // that can be destroyed. Start with the first one, which is the oldest. + const TimeStampAndBytesAndPid& oldest = mReleasedChunksByTime[0]; + LogDeletion(oldest.mProcessId, oldest.mTimeStamp); + mReleasedTotalBytes -= oldest.mBytes; + if (oldest.mProcessId == mParentProcessId) { + aParentChunkManager.DestroyChunksAtOrBefore(oldest.mTimeStamp); + } else { + ProfilerParentTracker::ForChild( + oldest.mProcessId, + [timestamp = oldest.mTimeStamp](ProfilerParent* profilerParent) { + Unused << profilerParent->SendDestroyReleasedChunksAtOrBefore( + timestamp); + }); + } + mReleasedChunksByTime.RemoveElementAt(0); + } +} + +/* static */ +ProfilerParentTracker* ProfilerParentTracker::GetInstance() { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + + // The main instance pointer, it will be initialized at most once, before + // XPCOMShutdownThreads. + static UniquePtr<ProfilerParentTracker> instance = nullptr; + if (MOZ_UNLIKELY(!instance)) { + if (PastShutdownPhase(ShutdownPhase::XPCOMShutdownThreads)) { + return nullptr; + } + + instance = MakeUnique<ProfilerParentTracker>(); + + // The tracker should get destroyed before threads are shutdown, because its + // destruction closes extant channels, which could trigger promise + // rejections that need to be dispatched to other threads. + ClearOnShutdown(&instance, ShutdownPhase::XPCOMShutdownThreads); + } + + return instance.get(); +} + +/* static */ +void ProfilerParentTracker::StartTracking(ProfilerParent* aProfilerParent) { + ProfilerParentTracker* tracker = GetInstance(); + if (!tracker) { + return; + } + + if (tracker->mMaybeController.isNothing() && tracker->mEntries != 0) { + // There is no controller yet, but the profiler has started. + // Since we're adding a ProfilerParent, it's a good time to start + // controlling the global memory usage of the profiler. + // (And this helps delay the Controller startup, because the parent profiler + // can start *very* early in the process, when some resources like threads + // are not ready yet.) + tracker->mMaybeController.emplace(size_t(tracker->mEntries) * + scBytesPerEntry); + } + + tracker->mProfilerParents.AppendElement(aProfilerParent); +} + +/* static */ +void ProfilerParentTracker::StopTracking(ProfilerParent* aParent) { + ProfilerParentTracker* tracker = GetInstance(); + if (!tracker) { + return; + } + + tracker->mProfilerParents.RemoveElement(aParent); +} + +/* static */ +void ProfilerParentTracker::ProfilerStarted(uint32_t aEntries) { + ProfilerParentTracker* tracker = GetInstance(); + if (!tracker) { + return; + } + + tracker->mEntries = ClampToAllowedEntries(aEntries); + + if (tracker->mMaybeController.isNothing() && + !tracker->mProfilerParents.IsEmpty()) { + // We are already tracking child processes, so it's a good time to start + // controlling the global memory usage of the profiler. + tracker->mMaybeController.emplace(size_t(tracker->mEntries) * + scBytesPerEntry); + } +} + +/* static */ +void ProfilerParentTracker::ProfilerWillStopIfStarted() { + ProfilerParentTracker* tracker = GetInstance(); + if (!tracker) { + return; + } + + tracker->mEntries = 0; + tracker->mMaybeController = Nothing{}; +} + +/* static */ +size_t ProfilerParentTracker::ProfilerParentCount() { + size_t count = 0; + ProfilerParentTracker* tracker = GetInstance(); + if (tracker) { + for (ProfilerParent* profilerParent : tracker->mProfilerParents) { + if (!profilerParent->mDestroyed) { + ++count; + } + } + } + return count; +} + +template <typename FuncType> +/* static */ +void ProfilerParentTracker::Enumerate(FuncType&& aIterFunc) { + ProfilerParentTracker* tracker = GetInstance(); + if (!tracker) { + return; + } + + for (ProfilerParent* profilerParent : tracker->mProfilerParents) { + if (!profilerParent->mDestroyed) { + aIterFunc(profilerParent); + } + } +} + +template <typename FuncType> +/* static */ +void ProfilerParentTracker::ForChild(base::ProcessId aChildPid, + FuncType&& aIterFunc) { + ProfilerParentTracker* tracker = GetInstance(); + if (!tracker) { + return; + } + + for (ProfilerParent* profilerParent : tracker->mProfilerParents) { + if (profilerParent->mChildPid == aChildPid) { + if (!profilerParent->mDestroyed) { + std::forward<FuncType>(aIterFunc)(profilerParent); + } + return; + } + } +} + +/* static */ +void ProfilerParentTracker::ForwardChildChunkManagerUpdate( + base::ProcessId aProcessId, + ProfileBufferControlledChunkManager::Update&& aUpdate) { + ProfilerParentTracker* tracker = GetInstance(); + if (!tracker || tracker->mMaybeController.isNothing()) { + return; + } + + MOZ_ASSERT(!aUpdate.IsNotUpdate(), + "No process should ever send a non-update"); + tracker->mMaybeController->HandleChildChunkManagerUpdate(aProcessId, + std::move(aUpdate)); +} + +ProfilerParentTracker::ProfilerParentTracker() { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + MOZ_COUNT_CTOR(ProfilerParentTracker); +} + +ProfilerParentTracker::~ProfilerParentTracker() { + // This destructor should only be called on the main thread. + MOZ_RELEASE_ASSERT(NS_IsMainThread() || + // OR we're not on the main thread (including if we are + // past the end of `main()`), which is fine *if* there are + // no ProfilerParent's still registered, in which case + // nothing else will happen in this destructor anyway. + // See bug 1713971 for more information. + mProfilerParents.IsEmpty()); + MOZ_COUNT_DTOR(ProfilerParentTracker); + + // Close the channels of any profiler parents that haven't been destroyed. + for (ProfilerParent* profilerParent : mProfilerParents.Clone()) { + if (!profilerParent->mDestroyed) { + // Keep the object alive until the call to Close() has completed. + // Close() will trigger a call to DeallocPProfilerParent. + RefPtr<ProfilerParent> actor = profilerParent; + actor->Close(); + } + } +} + +ProfilerParent::ProfilerParent(base::ProcessId aChildPid) + : mChildPid(aChildPid), mDestroyed(false) { + MOZ_COUNT_CTOR(ProfilerParent); + + MOZ_RELEASE_ASSERT(NS_IsMainThread()); +} + +void ProfilerParent::Init() { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + + ProfilerParentTracker::StartTracking(this); + + // We propagated the profiler state from the parent process to the child + // process through MOZ_PROFILER_STARTUP* environment variables. + // However, the profiler state might have changed in this process since then, + // and now that an active communication channel has been established with the + // child process, it's a good time to sync up the two profilers again. + + int entries = 0; + Maybe<double> duration = Nothing(); + double interval = 0; + mozilla::Vector<const char*> filters; + uint32_t features; + uint64_t activeTabID; + profiler_get_start_params(&entries, &duration, &interval, &features, &filters, + &activeTabID); + + if (entries != 0) { + ProfilerInitParams ipcParams; + ipcParams.enabled() = true; + ipcParams.entries() = entries; + ipcParams.duration() = duration; + ipcParams.interval() = interval; + ipcParams.features() = features; + ipcParams.activeTabID() = activeTabID; + + // If the filters exclude our pid, make sure it's stopped, otherwise + // continue with starting it. + if (!profiler::detail::FiltersExcludePid( + filters, ProfilerProcessId::FromNumber(mChildPid))) { + ipcParams.filters().SetCapacity(filters.length()); + for (const char* filter : filters) { + ipcParams.filters().AppendElement(filter); + } + + Unused << SendEnsureStarted(ipcParams); + RequestChunkManagerUpdate(); + return; + } + } + + Unused << SendStop(); +} +#endif // MOZ_GECKO_PROFILER + +ProfilerParent::~ProfilerParent() { + MOZ_COUNT_DTOR(ProfilerParent); + + MOZ_RELEASE_ASSERT(NS_IsMainThread()); +#ifdef MOZ_GECKO_PROFILER + ProfilerParentTracker::StopTracking(this); +#endif +} + +#ifdef MOZ_GECKO_PROFILER +/* static */ +nsTArray<ProfilerParent::SingleProcessProfilePromiseAndChildPid> +ProfilerParent::GatherProfiles() { + nsTArray<SingleProcessProfilePromiseAndChildPid> results; + if (!NS_IsMainThread()) { + return results; + } + + results.SetCapacity(ProfilerParentTracker::ProfilerParentCount()); + ProfilerParentTracker::Enumerate([&](ProfilerParent* profilerParent) { + results.AppendElement(SingleProcessProfilePromiseAndChildPid{ + profilerParent->SendGatherProfile(), profilerParent->mChildPid}); + }); + return results; +} + +/* static */ +RefPtr<ProfilerParent::SingleProcessProgressPromise> +ProfilerParent::RequestGatherProfileProgress(base::ProcessId aChildPid) { + RefPtr<SingleProcessProgressPromise> promise; + ProfilerParentTracker::ForChild( + aChildPid, [&promise](ProfilerParent* profilerParent) { + promise = profilerParent->SendGetGatherProfileProgress(); + }); + return promise; +} + +// Magic value for ProfileBufferChunkManagerUpdate::unreleasedBytes meaning +// that this is a final update from a child. +constexpr static uint64_t scUpdateUnreleasedBytesFINAL = uint64_t(-1); + +/* static */ +ProfileBufferChunkManagerUpdate ProfilerParent::MakeFinalUpdate() { + return ProfileBufferChunkManagerUpdate{ + uint64_t(scUpdateUnreleasedBytesFINAL), 0, TimeStamp{}, + nsTArray<ProfileBufferChunkMetadata>{}}; +} + +/* static */ +bool ProfilerParent::IsLockedOnCurrentThread() { + return ProfileBufferGlobalController::IsLockedOnCurrentThread(); +} + +void ProfilerParent::RequestChunkManagerUpdate() { + if (mDestroyed) { + return; + } + + RefPtr<AwaitNextChunkManagerUpdatePromise> updatePromise = + SendAwaitNextChunkManagerUpdate(); + updatePromise->Then( + GetMainThreadSerialEventTarget(), __func__, + [self = RefPtr<ProfilerParent>(this)]( + const ProfileBufferChunkManagerUpdate& aUpdate) { + if (aUpdate.unreleasedBytes() == scUpdateUnreleasedBytesFINAL) { + // Special value meaning it's the final update from that child. + ProfilerParentTracker::ForwardChildChunkManagerUpdate( + self->mChildPid, + ProfileBufferControlledChunkManager::Update(nullptr)); + } else { + // Not the final update, translate it. + std::vector<ProfileBufferControlledChunkManager::ChunkMetadata> + chunks; + if (!aUpdate.newlyReleasedChunks().IsEmpty()) { + chunks.reserve(aUpdate.newlyReleasedChunks().Length()); + for (const ProfileBufferChunkMetadata& chunk : + aUpdate.newlyReleasedChunks()) { + chunks.emplace_back(chunk.doneTimeStamp(), chunk.bufferBytes()); + } + } + // Let the tracker handle it. + ProfilerParentTracker::ForwardChildChunkManagerUpdate( + self->mChildPid, + ProfileBufferControlledChunkManager::Update( + aUpdate.unreleasedBytes(), aUpdate.releasedBytes(), + aUpdate.oldestDoneTimeStamp(), std::move(chunks))); + // This was not a final update, so start a new request. + self->RequestChunkManagerUpdate(); + } + }, + [self = RefPtr<ProfilerParent>(this)]( + mozilla::ipc::ResponseRejectReason aReason) { + // Rejection could be for a number of reasons, assume the child will + // not respond anymore, so we pretend we received a final update. + ProfilerParentTracker::ForwardChildChunkManagerUpdate( + self->mChildPid, + ProfileBufferControlledChunkManager::Update(nullptr)); + }); +} + +// Ref-counted class that resolves a promise on destruction. +// Usage: +// RefPtr<GenericPromise> f() { +// return PromiseResolverOnDestruction::RunTask( +// [](RefPtr<PromiseResolverOnDestruction> aPromiseResolver){ +// // Give *copies* of aPromiseResolver to asynchronous sub-tasks, the +// // last remaining RefPtr destruction will resolve the promise. +// }); +// } +class PromiseResolverOnDestruction { + public: + NS_INLINE_DECL_REFCOUNTING(PromiseResolverOnDestruction) + + template <typename TaskFunction> + static RefPtr<GenericPromise> RunTask(TaskFunction&& aTaskFunction) { + RefPtr<PromiseResolverOnDestruction> promiseResolver = + new PromiseResolverOnDestruction(); + RefPtr<GenericPromise> promise = + promiseResolver->mPromiseHolder.Ensure(__func__); + std::forward<TaskFunction>(aTaskFunction)(std::move(promiseResolver)); + return promise; + } + + private: + PromiseResolverOnDestruction() = default; + + ~PromiseResolverOnDestruction() { + mPromiseHolder.ResolveIfExists(/* unused */ true, __func__); + } + + MozPromiseHolder<GenericPromise> mPromiseHolder; +}; + +// Given a ProfilerParentSendFunction: (ProfilerParent*) -> some MozPromise, +// run the function on all live ProfilerParents and return a GenericPromise, and +// when their promise gets resolve, resolve our Generic promise. +template <typename ProfilerParentSendFunction> +static RefPtr<GenericPromise> SendAndConvertPromise( + ProfilerParentSendFunction&& aProfilerParentSendFunction) { + if (!NS_IsMainThread()) { + return GenericPromise::CreateAndResolve(/* unused */ true, __func__); + } + + return PromiseResolverOnDestruction::RunTask( + [&](RefPtr<PromiseResolverOnDestruction> aPromiseResolver) { + ProfilerParentTracker::Enumerate([&](ProfilerParent* profilerParent) { + std::forward<ProfilerParentSendFunction>(aProfilerParentSendFunction)( + profilerParent) + ->Then(GetMainThreadSerialEventTarget(), __func__, + [aPromiseResolver]( + typename std::remove_reference_t< + decltype(*std::forward<ProfilerParentSendFunction>( + aProfilerParentSendFunction)( + profilerParent))>::ResolveOrRejectValue&&) { + // Whatever the resolution/rejection is, do nothing. + // The lambda aPromiseResolver ref-count will decrease. + }); + }); + }); +} + +/* static */ +RefPtr<GenericPromise> ProfilerParent::ProfilerStarted( + nsIProfilerStartParams* aParams) { + if (!NS_IsMainThread()) { + return GenericPromise::CreateAndResolve(/* unused */ true, __func__); + } + + ProfilerInitParams ipcParams; + double duration; + ipcParams.enabled() = true; + aParams->GetEntries(&ipcParams.entries()); + aParams->GetDuration(&duration); + if (duration > 0.0) { + ipcParams.duration() = Some(duration); + } else { + ipcParams.duration() = Nothing(); + } + aParams->GetInterval(&ipcParams.interval()); + aParams->GetFeatures(&ipcParams.features()); + ipcParams.filters() = aParams->GetFilters().Clone(); + // We need filters as a Span<const char*> to test pids in the lambda below. + auto filtersCStrings = nsTArray<const char*>{aParams->GetFilters().Length()}; + for (const auto& filter : aParams->GetFilters()) { + filtersCStrings.AppendElement(filter.Data()); + } + aParams->GetActiveTabID(&ipcParams.activeTabID()); + + ProfilerParentTracker::ProfilerStarted(ipcParams.entries()); + + return SendAndConvertPromise([&](ProfilerParent* profilerParent) { + if (profiler::detail::FiltersExcludePid( + filtersCStrings, + ProfilerProcessId::FromNumber(profilerParent->mChildPid))) { + // This pid is excluded, don't start the profiler at all. + return PProfilerParent::StartPromise::CreateAndResolve(/* unused */ true, + __func__); + } + auto promise = profilerParent->SendStart(ipcParams); + profilerParent->RequestChunkManagerUpdate(); + return promise; + }); +} + +/* static */ +void ProfilerParent::ProfilerWillStopIfStarted() { + if (!NS_IsMainThread()) { + return; + } + + ProfilerParentTracker::ProfilerWillStopIfStarted(); +} + +/* static */ +RefPtr<GenericPromise> ProfilerParent::ProfilerStopped() { + return SendAndConvertPromise([](ProfilerParent* profilerParent) { + return profilerParent->SendStop(); + }); +} + +/* static */ +RefPtr<GenericPromise> ProfilerParent::ProfilerPaused() { + return SendAndConvertPromise([](ProfilerParent* profilerParent) { + return profilerParent->SendPause(); + }); +} + +/* static */ +RefPtr<GenericPromise> ProfilerParent::ProfilerResumed() { + return SendAndConvertPromise([](ProfilerParent* profilerParent) { + return profilerParent->SendResume(); + }); +} + +/* static */ +RefPtr<GenericPromise> ProfilerParent::ProfilerPausedSampling() { + return SendAndConvertPromise([](ProfilerParent* profilerParent) { + return profilerParent->SendPauseSampling(); + }); +} + +/* static */ +RefPtr<GenericPromise> ProfilerParent::ProfilerResumedSampling() { + return SendAndConvertPromise([](ProfilerParent* profilerParent) { + return profilerParent->SendResumeSampling(); + }); +} + +/* static */ +void ProfilerParent::ClearAllPages() { + if (!NS_IsMainThread()) { + return; + } + + ProfilerParentTracker::Enumerate([](ProfilerParent* profilerParent) { + Unused << profilerParent->SendClearAllPages(); + }); +} + +/* static */ +RefPtr<GenericPromise> ProfilerParent::WaitOnePeriodicSampling() { + return SendAndConvertPromise([](ProfilerParent* profilerParent) { + return profilerParent->SendWaitOnePeriodicSampling(); + }); +} + +void ProfilerParent::ActorDestroy(ActorDestroyReason aActorDestroyReason) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + mDestroyed = true; +} + +#endif + +} // namespace mozilla diff --git a/tools/profiler/gecko/ProfilerTypes.ipdlh b/tools/profiler/gecko/ProfilerTypes.ipdlh new file mode 100644 index 0000000000..6255d47db0 --- /dev/null +++ b/tools/profiler/gecko/ProfilerTypes.ipdlh @@ -0,0 +1,43 @@ +/* -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil; tab-width: 8 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +using class mozilla::TimeStamp from "mozilla/TimeStamp.h"; +using struct mozilla::ProfileGenerationAdditionalInformation from "ProfileAdditionalInformation.h"; + +namespace mozilla { + +struct ProfilerInitParams { + bool enabled; + uint32_t entries; + double? duration; + double interval; + uint32_t features; + uint64_t activeTabID; + nsCString[] filters; +}; + +struct ProfileBufferChunkMetadata { + TimeStamp doneTimeStamp; + uint32_t bufferBytes; +}; + +struct ProfileBufferChunkManagerUpdate { + uint64_t unreleasedBytes; + uint64_t releasedBytes; + TimeStamp oldestDoneTimeStamp; + ProfileBufferChunkMetadata[] newlyReleasedChunks; +}; + +struct GatherProfileProgress { + uint32_t progressProportionValueUnderlyingType; + nsCString progressLocation; +}; + +struct IPCProfileAndAdditionalInformation { + Shmem profileShmem; + ProfileGenerationAdditionalInformation? additionalInformation; +}; + +} // namespace mozilla diff --git a/tools/profiler/gecko/components.conf b/tools/profiler/gecko/components.conf new file mode 100644 index 0000000000..b1775c37ab --- /dev/null +++ b/tools/profiler/gecko/components.conf @@ -0,0 +1,17 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + 'js_name': 'profiler', + 'cid': '{25db9b8e-8123-4de1-b66d-8bbbedf2cdf4}', + 'contract_ids': ['@mozilla.org/tools/profiler;1'], + 'interfaces': ['nsIProfiler'], + 'type': 'nsProfiler', + 'headers': ['/tools/profiler/gecko/nsProfiler.h'], + 'init_method': 'Init', + }, +] diff --git a/tools/profiler/gecko/nsIProfiler.idl b/tools/profiler/gecko/nsIProfiler.idl new file mode 100644 index 0000000000..8b501d4b9f --- /dev/null +++ b/tools/profiler/gecko/nsIProfiler.idl @@ -0,0 +1,208 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +%{C++ +#include "mozilla/Maybe.h" +#include "nsTArrayForwardDeclare.h" +#include "nsStringFwd.h" +#include "mozilla/MozPromise.h" +%} + +[ref] native nsCString(const nsCString); +[ref] native StringArrayRef(const nsTArray<nsCString>); +native ProfileDataBufferMozPromise(RefPtr<mozilla::MozPromise<FallibleTArray<uint8_t>, nsresult, true>>); + +/** + * Start-up parameters for subprocesses are passed through nsIObserverService, + * which, unfortunately, means we need to implement nsISupports in order to + * go through it. + */ +[scriptable, builtinclass, uuid(0a175ba7-8fcf-4ce9-9c4b-ccc6272f4425)] +interface nsIProfilerStartParams : nsISupports +{ + readonly attribute uint32_t entries; + readonly attribute double duration; + readonly attribute double interval; + readonly attribute uint32_t features; + readonly attribute uint64_t activeTabID; + + [noscript, notxpcom, nostdcall] StringArrayRef getFilters(); +}; + +[scriptable, builtinclass, uuid(ead3f75c-0e0e-4fbb-901c-1e5392ef5b2a)] +interface nsIProfiler : nsISupports +{ + /* + * Control functions return as soon as this process' profiler has done its + * work. The returned promise gets resolved when sub-processes have completed + * their operation, or immediately if there are no sub-processes. + */ + [implicit_jscontext] + Promise StartProfiler(in uint32_t aEntries, in double aInterval, + in Array<AUTF8String> aFeatures, + [optional] in Array<AUTF8String> aFilters, + [optional] in uint64_t aActiveTabID, + [optional] in double aDuration); + [implicit_jscontext] + Promise StopProfiler(); + boolean IsPaused(); + [implicit_jscontext] + Promise Pause(); + [implicit_jscontext] + Promise Resume(); + boolean IsSamplingPaused(); + [implicit_jscontext] + Promise PauseSampling(); + [implicit_jscontext] + Promise ResumeSampling(); + + /* + * Resolves the returned promise after at least one full periodic sampling in + * each process. + * Rejects the promise if sampler is not running (yet, or anymore, or paused) + * in the parent process. + * This is mainly useful in tests, to wait just long enough to guarantee that + * at least one sample was taken in each process. + */ + [implicit_jscontext] + Promise waitOnePeriodicSampling(); + + /* + * Returns the JSON string of the profile. If aSinceTime is passed, only + * report samples taken at >= aSinceTime. + */ + string GetProfile([optional] in double aSinceTime); + + /* + * Returns a JS object of the profile. If aSinceTime is passed, only report + * samples taken at >= aSinceTime. + */ + [implicit_jscontext] + jsval getProfileData([optional] in double aSinceTime); + + [implicit_jscontext] + Promise getProfileDataAsync([optional] in double aSinceTime); + + [implicit_jscontext] + Promise getProfileDataAsArrayBuffer([optional] in double aSinceTime); + + [implicit_jscontext] + Promise getProfileDataAsGzippedArrayBuffer([optional] in double aSinceTime); + + /** + * Asynchronously dump the profile collected so far to a file. + * Returns a promise that resolves once the file has been written, with data + * from all responsive Firefox processes. Note: This blocks the parent process + * while collecting its own data, then unblocks while child processes data is + * being collected. + * `aFilename` may be a full path, or a path relative to where Firefox was + * launched. The target directory must already exist. + */ + [implicit_jscontext] + Promise dumpProfileToFileAsync(in ACString aFilename, + [optional] in double aSinceTime); + + /** + * Synchronously dump the profile collected so far in this process to a file. + * This profile will only contain data from the parent process, and from child + * processes that have ended during the session; other currently-live + * processes are ignored. + * `aFilename` may be a full path, or a path relative to where Firefox was + * launched. The target directory must already exist. + */ + void dumpProfileToFile(in string aFilename); + + boolean IsActive(); + + /** + * Clear all registered and unregistered page information in prifiler. + */ + void ClearAllPages(); + + /** + * Returns an array of the features that are supported in this build. + * Features may vary depending on platform and build flags. + */ + Array<AUTF8String> GetFeatures(); + + /** + * Returns a JavaScript object that contains a description of the currently configured + * state of the profiler when the profiler is active. This can be useful to assert + * the UI of the profiler's recording panel in tests. It returns null when the profiler + * is not active. + */ + [implicit_jscontext] + readonly attribute jsval activeConfiguration; + + /** + * Returns an array of all features that are supported by the profiler. + * The array may contain features that are not supported in this build. + */ + Array<AUTF8String> GetAllFeatures(); + + void GetBufferInfo(out uint32_t aCurrentPosition, out uint32_t aTotalSize, + out uint32_t aGeneration); + + /** + * Returns the elapsed time, in milliseconds, since the profiler's epoch. + * The epoch is guaranteed to be constant for the duration of the + * process, but is otherwise arbitrary. + */ + double getElapsedTime(); + + /** + * Contains an array of shared library objects. + * Every object has the properties: + * - start: The start address of the memory region occupied by this library. + * - end: The end address of the memory region occupied by this library. + * - offset: Usually zero, except on Linux / Android if the first mapped + * section of the library has been mapped to an address that's + * different from the library's base address. + * Then offset = start - baseAddress. + * - name: The name (file basename) of the binary. + * - path: The full absolute path to the binary. + * - debugName: On Windows, the name of the pdb file for the binary. On other + * platforms, the same as |name|. + * - debugPath: On Windows, the full absolute path of the pdb file for the + * binary. On other platforms, the same as |path|. + * - arch: On Mac, the name of the architecture that identifies the right + * binary image of a fat binary. Example values are "i386", "x86_64", + * and "x86_64h". (x86_64h is used for binaries that contain + * instructions that are specific to the Intel Haswell microarchitecture.) + * On non-Mac platforms, arch is "". + * - breakpadId: A unique identifier string for this library, as used by breakpad. + */ + [implicit_jscontext] + readonly attribute jsval sharedLibraries; + + /** + * Returns a promise that resolves to a SymbolTableAsTuple for the binary at + * the given path. + * + * SymbolTable as tuple: [addrs, index, buffer] + * Contains a symbol table, which can be used to map addresses to strings. + * + * The first element of this tuple, commonly named "addrs", is a sorted array of + * symbol addresses, as library-relative offsets in bytes, in ascending order. + * The third element of this tuple, commonly named "buffer", is a buffer of + * bytes that contains all strings from this symbol table, in the order of the + * addresses they correspond to, in utf-8 encoded form, all concatenated + * together. + * The second element of this tuple, commonly named "index", contains positions + * into "buffer". For every address, that position is where the string for that + * address starts in the buffer. + * index.length == addrs.length + 1. + * index[addrs.length] is the end position of the last string in the buffer. + * + * The string for the address addrs[i] is + * (new TextDecoder()).decode(buffer.subarray(index[i], index[i + 1])) + */ + [implicit_jscontext] + Promise getSymbolTable(in ACString aDebugPath, in ACString aBreakpadID); + + [notxpcom, nostdcall] ProfileDataBufferMozPromise getProfileDataAsGzippedArrayBufferAndroid(in double aSinceTime); +}; diff --git a/tools/profiler/gecko/nsProfiler.cpp b/tools/profiler/gecko/nsProfiler.cpp new file mode 100644 index 0000000000..0d989d1bef --- /dev/null +++ b/tools/profiler/gecko/nsProfiler.cpp @@ -0,0 +1,1491 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsProfiler.h" + +#include <fstream> +#include <limits> +#include <sstream> +#include <string> +#include <utility> + +#include "GeckoProfiler.h" +#include "ProfilerControl.h" +#include "ProfilerParent.h" +#include "js/Array.h" // JS::NewArrayObject +#include "js/JSON.h" +#include "js/PropertyAndElement.h" // JS_SetElement +#include "js/Value.h" +#include "json/json.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/JSONStringWriteFuncs.h" +#include "mozilla/SchedulerGroup.h" +#include "mozilla/Services.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/TypedArray.h" +#include "mozilla/Preferences.h" +#include "nsComponentManagerUtils.h" +#include "nsIInterfaceRequestor.h" +#include "nsIInterfaceRequestorUtils.h" +#include "nsILoadContext.h" +#include "nsIWebNavigation.h" +#include "nsProfilerStartParams.h" +#include "nsProxyRelease.h" +#include "nsString.h" +#include "nsThreadUtils.h" +#include "platform.h" +#include "shared-libraries.h" +#include "zlib.h" + +#ifndef ANDROID +# include <cstdio> +#else +# include <android/log.h> +#endif + +using namespace mozilla; + +using dom::AutoJSAPI; +using dom::Promise; +using std::string; + +static constexpr size_t scLengthMax = size_t(JS::MaxStringLength); +// Used when trying to add more JSON data, to account for the extra space needed +// for the log and to close the profile. +static constexpr size_t scLengthAccumulationThreshold = scLengthMax - 16 * 1024; + +NS_IMPL_ISUPPORTS(nsProfiler, nsIProfiler) + +nsProfiler::nsProfiler() : mGathering(false) {} + +nsProfiler::~nsProfiler() { + if (mSymbolTableThread) { + mSymbolTableThread->Shutdown(); + } + ResetGathering(NS_ERROR_ILLEGAL_DURING_SHUTDOWN); +} + +nsresult nsProfiler::Init() { return NS_OK; } + +template <typename JsonLogObjectUpdater> +void nsProfiler::Log(JsonLogObjectUpdater&& aJsonLogObjectUpdater) { + if (mGatheringLog) { + MOZ_ASSERT(mGatheringLog->isObject()); + std::forward<JsonLogObjectUpdater>(aJsonLogObjectUpdater)(*mGatheringLog); + MOZ_ASSERT(mGatheringLog->isObject()); + } +} + +template <typename JsonArrayAppender> +void nsProfiler::LogEvent(JsonArrayAppender&& aJsonArrayAppender) { + Log([&](Json::Value& aRoot) { + Json::Value& events = aRoot[Json::StaticString{"events"}]; + if (!events.isArray()) { + events = Json::Value{Json::arrayValue}; + } + Json::Value newEvent{Json::arrayValue}; + newEvent.append(ProfilingLog::Timestamp()); + std::forward<JsonArrayAppender>(aJsonArrayAppender)(newEvent); + MOZ_ASSERT(newEvent.isArray()); + events.append(std::move(newEvent)); + }); +} + +void nsProfiler::LogEventLiteralString(const char* aEventString) { + LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{aEventString}); + }); +} + +static nsresult FillVectorFromStringArray(Vector<const char*>& aVector, + const nsTArray<nsCString>& aArray) { + if (NS_WARN_IF(!aVector.reserve(aArray.Length()))) { + return NS_ERROR_OUT_OF_MEMORY; + } + for (auto& entry : aArray) { + aVector.infallibleAppend(entry.get()); + } + return NS_OK; +} + +// Given a PromiseReturningFunction: () -> GenericPromise, +// run the function, and return a JS Promise (through aPromise) that will be +// resolved when the function's GenericPromise gets resolved. +template <typename PromiseReturningFunction> +static nsresult RunFunctionAndConvertPromise( + JSContext* aCx, Promise** aPromise, + PromiseReturningFunction&& aPromiseReturningFunction) { + MOZ_ASSERT(NS_IsMainThread()); + + if (NS_WARN_IF(!aCx)) { + return NS_ERROR_FAILURE; + } + + nsIGlobalObject* globalObject = xpc::CurrentNativeGlobal(aCx); + if (NS_WARN_IF(!globalObject)) { + return NS_ERROR_FAILURE; + } + + ErrorResult result; + RefPtr<Promise> promise = Promise::Create(globalObject, result); + if (NS_WARN_IF(result.Failed())) { + return result.StealNSResult(); + } + + std::forward<PromiseReturningFunction>(aPromiseReturningFunction)()->Then( + GetMainThreadSerialEventTarget(), __func__, + [promise](GenericPromise::ResolveOrRejectValue&&) { + promise->MaybeResolveWithUndefined(); + }); + + promise.forget(aPromise); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::StartProfiler(uint32_t aEntries, double aInterval, + const nsTArray<nsCString>& aFeatures, + const nsTArray<nsCString>& aFilters, + uint64_t aActiveTabID, double aDuration, + JSContext* aCx, Promise** aPromise) { + ResetGathering(NS_ERROR_DOM_ABORT_ERR); + + Vector<const char*> featureStringVector; + nsresult rv = FillVectorFromStringArray(featureStringVector, aFeatures); + if (NS_FAILED(rv)) { + return rv; + } + uint32_t features = ParseFeaturesFromStringArray( + featureStringVector.begin(), featureStringVector.length()); + Maybe<double> duration = aDuration > 0.0 ? Some(aDuration) : Nothing(); + + Vector<const char*> filterStringVector; + rv = FillVectorFromStringArray(filterStringVector, aFilters); + if (NS_FAILED(rv)) { + return rv; + } + + return RunFunctionAndConvertPromise(aCx, aPromise, [&]() { + return profiler_start(PowerOfTwo32(aEntries), aInterval, features, + filterStringVector.begin(), + filterStringVector.length(), aActiveTabID, duration); + }); +} + +NS_IMETHODIMP +nsProfiler::StopProfiler(JSContext* aCx, Promise** aPromise) { + ResetGathering(NS_ERROR_DOM_ABORT_ERR); + return RunFunctionAndConvertPromise(aCx, aPromise, + []() { return profiler_stop(); }); +} + +NS_IMETHODIMP +nsProfiler::IsPaused(bool* aIsPaused) { + *aIsPaused = profiler_is_paused(); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::Pause(JSContext* aCx, Promise** aPromise) { + return RunFunctionAndConvertPromise(aCx, aPromise, + []() { return profiler_pause(); }); +} + +NS_IMETHODIMP +nsProfiler::Resume(JSContext* aCx, Promise** aPromise) { + return RunFunctionAndConvertPromise(aCx, aPromise, + []() { return profiler_resume(); }); +} + +NS_IMETHODIMP +nsProfiler::IsSamplingPaused(bool* aIsSamplingPaused) { + *aIsSamplingPaused = profiler_is_sampling_paused(); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::PauseSampling(JSContext* aCx, Promise** aPromise) { + return RunFunctionAndConvertPromise( + aCx, aPromise, []() { return profiler_pause_sampling(); }); +} + +NS_IMETHODIMP +nsProfiler::ResumeSampling(JSContext* aCx, Promise** aPromise) { + return RunFunctionAndConvertPromise( + aCx, aPromise, []() { return profiler_resume_sampling(); }); +} + +NS_IMETHODIMP +nsProfiler::ClearAllPages() { + profiler_clear_all_pages(); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::WaitOnePeriodicSampling(JSContext* aCx, Promise** aPromise) { + MOZ_ASSERT(NS_IsMainThread()); + + if (NS_WARN_IF(!aCx)) { + return NS_ERROR_FAILURE; + } + + nsIGlobalObject* globalObject = xpc::CurrentNativeGlobal(aCx); + if (NS_WARN_IF(!globalObject)) { + return NS_ERROR_FAILURE; + } + + ErrorResult result; + RefPtr<Promise> promise = Promise::Create(globalObject, result); + if (NS_WARN_IF(result.Failed())) { + return result.StealNSResult(); + } + + // The callback cannot officially own the promise RefPtr directly, because + // `Promise` doesn't support multi-threading, and the callback could destroy + // the promise in the sampler thread. + // `nsMainThreadPtrHandle` ensures that the promise can only be destroyed on + // the main thread. And the invocation from the Sampler thread immediately + // dispatches a task back to the main thread, to resolve/reject the promise. + // The lambda needs to be `mutable`, to allow moving-from + // `promiseHandleInSampler`. + if (!profiler_callback_after_sampling( + [promiseHandleInSampler = nsMainThreadPtrHandle<Promise>( + new nsMainThreadPtrHolder<Promise>( + "WaitOnePeriodicSampling promise for Sampler", promise))]( + SamplingState aSamplingState) mutable { + SchedulerGroup::Dispatch(NS_NewRunnableFunction( + "nsProfiler::WaitOnePeriodicSampling result on main thread", + [promiseHandleInMT = std::move(promiseHandleInSampler), + aSamplingState]() mutable { + switch (aSamplingState) { + case SamplingState::JustStopped: + case SamplingState::SamplingPaused: + promiseHandleInMT->MaybeReject(NS_ERROR_FAILURE); + break; + + case SamplingState::NoStackSamplingCompleted: + case SamplingState::SamplingCompleted: + // The parent process has succesfully done a sampling, + // check the child processes (if any). + ProfilerParent::WaitOnePeriodicSampling()->Then( + GetMainThreadSerialEventTarget(), __func__, + [promiseHandleInMT = std::move(promiseHandleInMT)]( + GenericPromise::ResolveOrRejectValue&&) { + promiseHandleInMT->MaybeResolveWithUndefined(); + }); + break; + + default: + MOZ_ASSERT(false, "Unexpected SamplingState value"); + promiseHandleInMT->MaybeReject(NS_ERROR_DOM_UNKNOWN_ERR); + break; + } + })); + })) { + // Callback was not added (e.g., profiler is not running) and will never be + // invoked, so we need to resolve the promise here. + promise->MaybeReject(NS_ERROR_DOM_UNKNOWN_ERR); + } + + promise.forget(aPromise); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetProfile(double aSinceTime, char** aProfile) { + mozilla::UniquePtr<char[]> profile = profiler_get_profile(aSinceTime); + *aProfile = profile.release(); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetSharedLibraries(JSContext* aCx, + JS::MutableHandle<JS::Value> aResult) { + JS::Rooted<JS::Value> val(aCx); + { + JSONStringWriteFunc<nsCString> buffer; + JSONWriter w(buffer, JSONWriter::SingleLineStyle); + w.StartArrayElement(); + SharedLibraryInfo sharedLibraryInfo = SharedLibraryInfo::GetInfoForSelf(); + sharedLibraryInfo.SortByAddress(); + AppendSharedLibraries(w, sharedLibraryInfo); + w.EndArray(); + NS_ConvertUTF8toUTF16 buffer16(buffer.StringCRef()); + MOZ_ALWAYS_TRUE(JS_ParseJSON(aCx, + static_cast<const char16_t*>(buffer16.get()), + buffer16.Length(), &val)); + } + JS::Rooted<JSObject*> obj(aCx, &val.toObject()); + if (!obj) { + return NS_ERROR_FAILURE; + } + aResult.setObject(*obj); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetActiveConfiguration(JSContext* aCx, + JS::MutableHandle<JS::Value> aResult) { + JS::Rooted<JS::Value> jsValue(aCx); + { + JSONStringWriteFunc<nsCString> buffer; + JSONWriter writer(buffer, JSONWriter::SingleLineStyle); + profiler_write_active_configuration(writer); + NS_ConvertUTF8toUTF16 buffer16(buffer.StringCRef()); + MOZ_ALWAYS_TRUE(JS_ParseJSON(aCx, + static_cast<const char16_t*>(buffer16.get()), + buffer16.Length(), &jsValue)); + } + if (jsValue.isNull()) { + aResult.setNull(); + } else { + JS::Rooted<JSObject*> obj(aCx, &jsValue.toObject()); + if (!obj) { + return NS_ERROR_FAILURE; + } + aResult.setObject(*obj); + } + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::DumpProfileToFile(const char* aFilename) { + profiler_save_profile_to_file(aFilename); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetProfileData(double aSinceTime, JSContext* aCx, + JS::MutableHandle<JS::Value> aResult) { + mozilla::UniquePtr<char[]> profile = profiler_get_profile(aSinceTime); + if (!profile) { + return NS_ERROR_FAILURE; + } + + NS_ConvertUTF8toUTF16 js_string(nsDependentCString(profile.get())); + auto profile16 = static_cast<const char16_t*>(js_string.get()); + + JS::Rooted<JS::Value> val(aCx); + MOZ_ALWAYS_TRUE(JS_ParseJSON(aCx, profile16, js_string.Length(), &val)); + + aResult.set(val); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetProfileDataAsync(double aSinceTime, JSContext* aCx, + Promise** aPromise) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!profiler_is_active()) { + return NS_ERROR_FAILURE; + } + + if (NS_WARN_IF(!aCx)) { + return NS_ERROR_FAILURE; + } + + nsIGlobalObject* globalObject = xpc::CurrentNativeGlobal(aCx); + if (NS_WARN_IF(!globalObject)) { + return NS_ERROR_FAILURE; + } + + ErrorResult result; + RefPtr<Promise> promise = Promise::Create(globalObject, result); + if (NS_WARN_IF(result.Failed())) { + return result.StealNSResult(); + } + + StartGathering(aSinceTime) + ->Then( + GetMainThreadSerialEventTarget(), __func__, + [promise](const mozilla::ProfileAndAdditionalInformation& aResult) { + AutoJSAPI jsapi; + if (NS_WARN_IF(!jsapi.Init(promise->GetGlobalObject()))) { + // We're really hosed if we can't get a JS context for some + // reason. + promise->MaybeReject(NS_ERROR_DOM_UNKNOWN_ERR); + return; + } + + JSContext* cx = jsapi.cx(); + + // Now parse the JSON so that we resolve with a JS Object. + JS::Rooted<JS::Value> val(cx); + { + NS_ConvertUTF8toUTF16 js_string(aResult.mProfile); + if (!JS_ParseJSON(cx, + static_cast<const char16_t*>(js_string.get()), + js_string.Length(), &val)) { + if (!jsapi.HasException()) { + promise->MaybeReject(NS_ERROR_DOM_UNKNOWN_ERR); + } else { + JS::Rooted<JS::Value> exn(cx); + DebugOnly<bool> gotException = jsapi.StealException(&exn); + MOZ_ASSERT(gotException); + + jsapi.ClearException(); + promise->MaybeReject(exn); + } + } else { + promise->MaybeResolve(val); + } + } + }, + [promise](nsresult aRv) { promise->MaybeReject(aRv); }); + + promise.forget(aPromise); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetProfileDataAsArrayBuffer(double aSinceTime, JSContext* aCx, + Promise** aPromise) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!profiler_is_active()) { + return NS_ERROR_FAILURE; + } + + if (NS_WARN_IF(!aCx)) { + return NS_ERROR_FAILURE; + } + + nsIGlobalObject* globalObject = xpc::CurrentNativeGlobal(aCx); + if (NS_WARN_IF(!globalObject)) { + return NS_ERROR_FAILURE; + } + + ErrorResult result; + RefPtr<Promise> promise = Promise::Create(globalObject, result); + if (NS_WARN_IF(result.Failed())) { + return result.StealNSResult(); + } + + StartGathering(aSinceTime) + ->Then( + GetMainThreadSerialEventTarget(), __func__, + [promise](const mozilla::ProfileAndAdditionalInformation& aResult) { + AutoJSAPI jsapi; + if (NS_WARN_IF(!jsapi.Init(promise->GetGlobalObject()))) { + // We're really hosed if we can't get a JS context for some + // reason. + promise->MaybeReject(NS_ERROR_DOM_UNKNOWN_ERR); + return; + } + + JSContext* cx = jsapi.cx(); + ErrorResult error; + JSObject* typedArray = + dom::ArrayBuffer::Create(cx, aResult.mProfile, error); + if (!error.Failed()) { + JS::Rooted<JS::Value> val(cx, JS::ObjectValue(*typedArray)); + promise->MaybeResolve(val); + } else { + promise->MaybeReject(std::move(error)); + } + }, + [promise](nsresult aRv) { promise->MaybeReject(aRv); }); + + promise.forget(aPromise); + return NS_OK; +} + +nsresult CompressString(const nsCString& aString, + FallibleTArray<uint8_t>& aOutBuff) { + // Compress a buffer via zlib (as with `compress()`), but emit a + // gzip header as well. Like `compress()`, this is limited to 4GB in + // size, but that shouldn't be an issue for our purposes. + uLongf outSize = compressBound(aString.Length()); + if (!aOutBuff.SetLength(outSize, fallible)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + int zerr; + z_stream stream; + stream.zalloc = nullptr; + stream.zfree = nullptr; + stream.opaque = nullptr; + stream.next_out = (Bytef*)aOutBuff.Elements(); + stream.avail_out = aOutBuff.Length(); + stream.next_in = (z_const Bytef*)aString.Data(); + stream.avail_in = aString.Length(); + + // A windowBits of 31 is the default (15) plus 16 for emitting a + // gzip header; a memLevel of 8 is the default. + zerr = + deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, + /* windowBits */ 31, /* memLevel */ 8, Z_DEFAULT_STRATEGY); + if (zerr != Z_OK) { + return NS_ERROR_FAILURE; + } + + zerr = deflate(&stream, Z_FINISH); + outSize = stream.total_out; + deflateEnd(&stream); + + if (zerr != Z_STREAM_END) { + return NS_ERROR_FAILURE; + } + + aOutBuff.TruncateLength(outSize); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetProfileDataAsGzippedArrayBuffer(double aSinceTime, + JSContext* aCx, + Promise** aPromise) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!profiler_is_active()) { + return NS_ERROR_FAILURE; + } + + if (NS_WARN_IF(!aCx)) { + return NS_ERROR_FAILURE; + } + + nsIGlobalObject* globalObject = xpc::CurrentNativeGlobal(aCx); + if (NS_WARN_IF(!globalObject)) { + return NS_ERROR_FAILURE; + } + + ErrorResult result; + RefPtr<Promise> promise = Promise::Create(globalObject, result); + if (NS_WARN_IF(result.Failed())) { + return result.StealNSResult(); + } + + StartGathering(aSinceTime) + ->Then( + GetMainThreadSerialEventTarget(), __func__, + [promise](const mozilla::ProfileAndAdditionalInformation& aResult) { + AutoJSAPI jsapi; + if (NS_WARN_IF(!jsapi.Init(promise->GetGlobalObject()))) { + // We're really hosed if we can't get a JS context for some + // reason. + promise->MaybeReject(NS_ERROR_DOM_UNKNOWN_ERR); + return; + } + + FallibleTArray<uint8_t> outBuff; + nsresult result = CompressString(aResult.mProfile, outBuff); + + if (result != NS_OK) { + promise->MaybeReject(result); + return; + } + + JSContext* cx = jsapi.cx(); + // Get the profile typedArray. + ErrorResult error; + JSObject* typedArray = dom::ArrayBuffer::Create(cx, outBuff, error); + if (error.Failed()) { + promise->MaybeReject(std::move(error)); + return; + } + JS::Rooted<JS::Value> typedArrayValue(cx, + JS::ObjectValue(*typedArray)); + // Get the additional information object. + JS::Rooted<JS::Value> additionalInfoVal(cx); + if (aResult.mAdditionalInformation.isSome()) { + aResult.mAdditionalInformation->ToJSValue(cx, &additionalInfoVal); + } else { + additionalInfoVal.setUndefined(); + } + + // Create the return object. + JS::Rooted<JSObject*> resultObj(cx, JS_NewPlainObject(cx)); + JS_SetProperty(cx, resultObj, "profile", typedArrayValue); + JS_SetProperty(cx, resultObj, "additionalInformation", + additionalInfoVal); + promise->MaybeResolve(resultObj); + }, + [promise](nsresult aRv) { promise->MaybeReject(aRv); }); + + promise.forget(aPromise); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::DumpProfileToFileAsync(const nsACString& aFilename, + double aSinceTime, JSContext* aCx, + Promise** aPromise) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!profiler_is_active()) { + return NS_ERROR_FAILURE; + } + + if (NS_WARN_IF(!aCx)) { + return NS_ERROR_FAILURE; + } + + nsIGlobalObject* globalObject = xpc::CurrentNativeGlobal(aCx); + if (NS_WARN_IF(!globalObject)) { + return NS_ERROR_FAILURE; + } + + ErrorResult result; + RefPtr<Promise> promise = Promise::Create(globalObject, result); + if (NS_WARN_IF(result.Failed())) { + return result.StealNSResult(); + } + + nsCString filename(aFilename); + + StartGathering(aSinceTime) + ->Then( + GetMainThreadSerialEventTarget(), __func__, + [filename, + promise](const mozilla::ProfileAndAdditionalInformation& aResult) { + if (aResult.mProfile.Length() >= + size_t(std::numeric_limits<std::streamsize>::max())) { + promise->MaybeReject(NS_ERROR_FILE_TOO_BIG); + return; + } + + std::ofstream stream; + stream.open(filename.get()); + if (!stream.is_open()) { + promise->MaybeReject(NS_ERROR_FILE_UNRECOGNIZED_PATH); + return; + } + + stream.write(aResult.mProfile.get(), + std::streamsize(aResult.mProfile.Length())); + stream.close(); + + promise->MaybeResolveWithUndefined(); + }, + [promise](nsresult aRv) { promise->MaybeReject(aRv); }); + + promise.forget(aPromise); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetSymbolTable(const nsACString& aDebugPath, + const nsACString& aBreakpadID, JSContext* aCx, + Promise** aPromise) { + MOZ_ASSERT(NS_IsMainThread()); + + if (NS_WARN_IF(!aCx)) { + return NS_ERROR_FAILURE; + } + + nsIGlobalObject* globalObject = + xpc::NativeGlobal(JS::CurrentGlobalOrNull(aCx)); + + if (NS_WARN_IF(!globalObject)) { + return NS_ERROR_FAILURE; + } + + ErrorResult result; + RefPtr<Promise> promise = Promise::Create(globalObject, result); + if (NS_WARN_IF(result.Failed())) { + return result.StealNSResult(); + } + + GetSymbolTableMozPromise(aDebugPath, aBreakpadID) + ->Then( + GetMainThreadSerialEventTarget(), __func__, + [promise](const SymbolTable& aSymbolTable) { + AutoJSAPI jsapi; + if (NS_WARN_IF(!jsapi.Init(promise->GetGlobalObject()))) { + // We're really hosed if we can't get a JS context for some + // reason. + promise->MaybeReject(NS_ERROR_DOM_UNKNOWN_ERR); + return; + } + + JSContext* cx = jsapi.cx(); + + ErrorResult error; + JS::Rooted<JSObject*> addrsArray( + cx, dom::Uint32Array::Create(cx, aSymbolTable.mAddrs, error)); + if (error.Failed()) { + promise->MaybeReject(std::move(error)); + return; + } + + JS::Rooted<JSObject*> indexArray( + cx, dom::Uint32Array::Create(cx, aSymbolTable.mIndex, error)); + if (error.Failed()) { + promise->MaybeReject(std::move(error)); + return; + } + + JS::Rooted<JSObject*> bufferArray( + cx, dom::Uint8Array::Create(cx, aSymbolTable.mBuffer, error)); + if (error.Failed()) { + promise->MaybeReject(std::move(error)); + return; + } + + JS::Rooted<JSObject*> tuple(cx, JS::NewArrayObject(cx, 3)); + JS_SetElement(cx, tuple, 0, addrsArray); + JS_SetElement(cx, tuple, 1, indexArray); + JS_SetElement(cx, tuple, 2, bufferArray); + promise->MaybeResolve(tuple); + }, + [promise](nsresult aRv) { promise->MaybeReject(aRv); }); + + promise.forget(aPromise); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetElapsedTime(double* aElapsedTime) { + *aElapsedTime = profiler_time(); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::IsActive(bool* aIsActive) { + *aIsActive = profiler_is_active(); + return NS_OK; +} + +static void GetArrayOfStringsForFeatures(uint32_t aFeatures, + nsTArray<nsCString>& aFeatureList) { +#define COUNT_IF_SET(n_, str_, Name_, desc_) \ + if (ProfilerFeature::Has##Name_(aFeatures)) { \ + len++; \ + } + + // Count the number of features in use. + uint32_t len = 0; + PROFILER_FOR_EACH_FEATURE(COUNT_IF_SET) + +#undef COUNT_IF_SET + + aFeatureList.SetCapacity(len); + +#define DUP_IF_SET(n_, str_, Name_, desc_) \ + if (ProfilerFeature::Has##Name_(aFeatures)) { \ + aFeatureList.AppendElement(str_); \ + } + + // Insert the strings for the features in use. + PROFILER_FOR_EACH_FEATURE(DUP_IF_SET) + +#undef DUP_IF_SET +} + +NS_IMETHODIMP +nsProfiler::GetFeatures(nsTArray<nsCString>& aFeatureList) { + uint32_t features = profiler_get_available_features(); + GetArrayOfStringsForFeatures(features, aFeatureList); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetAllFeatures(nsTArray<nsCString>& aFeatureList) { + GetArrayOfStringsForFeatures((uint32_t)-1, aFeatureList); + return NS_OK; +} + +NS_IMETHODIMP +nsProfiler::GetBufferInfo(uint32_t* aCurrentPosition, uint32_t* aTotalSize, + uint32_t* aGeneration) { + MOZ_ASSERT(aCurrentPosition); + MOZ_ASSERT(aTotalSize); + MOZ_ASSERT(aGeneration); + Maybe<ProfilerBufferInfo> info = profiler_get_buffer_info(); + if (info) { + *aCurrentPosition = info->mRangeEnd % info->mEntryCount; + *aTotalSize = info->mEntryCount; + *aGeneration = info->mRangeEnd / info->mEntryCount; + } else { + *aCurrentPosition = 0; + *aTotalSize = 0; + *aGeneration = 0; + } + return NS_OK; +} + +bool nsProfiler::SendProgressRequest(PendingProfile& aPendingProfile) { + RefPtr<ProfilerParent::SingleProcessProgressPromise> progressPromise = + ProfilerParent::RequestGatherProfileProgress(aPendingProfile.childPid); + if (!progressPromise) { + LOG("RequestGatherProfileProgress(%u) -> null!", + unsigned(aPendingProfile.childPid)); + LogEvent([&](Json::Value& aEvent) { + aEvent.append( + Json::StaticString{"Failed to send progress request to pid:"}); + aEvent.append(Json::Value::UInt64(aPendingProfile.childPid)); + }); + // Failed to send request. + return false; + } + + DEBUG_LOG("RequestGatherProfileProgress(%u) sent...", + unsigned(aPendingProfile.childPid)); + LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{"Requested progress from pid:"}); + aEvent.append(Json::Value::UInt64(aPendingProfile.childPid)); + }); + aPendingProfile.lastProgressRequest = TimeStamp::Now(); + progressPromise->Then( + GetMainThreadSerialEventTarget(), __func__, + [self = RefPtr<nsProfiler>(this), + childPid = aPendingProfile.childPid](GatherProfileProgress&& aResult) { + if (!self->mGathering) { + return; + } + PendingProfile* pendingProfile = self->GetPendingProfile(childPid); + DEBUG_LOG( + "RequestGatherProfileProgress(%u) response: %.2f '%s' " + "(%u were pending, %s %u)", + unsigned(childPid), + ProportionValue::FromUnderlyingType( + aResult.progressProportionValueUnderlyingType()) + .ToDouble() * + 100.0, + aResult.progressLocation().Data(), + unsigned(self->mPendingProfiles.length()), + pendingProfile ? "including" : "excluding", unsigned(childPid)); + self->LogEvent([&](Json::Value& aEvent) { + aEvent.append( + Json::StaticString{"Got response from pid, with progress:"}); + aEvent.append(Json::Value::UInt64(childPid)); + aEvent.append( + Json::Value{ProportionValue::FromUnderlyingType( + aResult.progressProportionValueUnderlyingType()) + .ToDouble() * + 100.0}); + }); + if (pendingProfile) { + // We have a progress report for a still-pending profile. + pendingProfile->lastProgressResponse = TimeStamp::Now(); + // Has it actually made progress? + if (aResult.progressProportionValueUnderlyingType() != + pendingProfile->progressProportion.ToUnderlyingType()) { + pendingProfile->lastProgressChange = + pendingProfile->lastProgressResponse; + pendingProfile->progressProportion = + ProportionValue::FromUnderlyingType( + aResult.progressProportionValueUnderlyingType()); + pendingProfile->progressLocation = aResult.progressLocation(); + self->RestartGatheringTimer(); + } + } + }, + [self = RefPtr<nsProfiler>(this), childPid = aPendingProfile.childPid]( + ipc::ResponseRejectReason&& aReason) { + if (!self->mGathering) { + return; + } + PendingProfile* pendingProfile = self->GetPendingProfile(childPid); + LOG("RequestGatherProfileProgress(%u) rejection: %d " + "(%u were pending, %s %u)", + unsigned(childPid), (int)aReason, + unsigned(self->mPendingProfiles.length()), + pendingProfile ? "including" : "excluding", unsigned(childPid)); + self->LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{ + "Got progress request rejection from pid, with reason:"}); + aEvent.append(Json::Value::UInt64(childPid)); + aEvent.append(Json::Value::UInt{static_cast<unsigned>(aReason)}); + }); + if (pendingProfile) { + // Failure response, assume the child process is gone. + MOZ_ASSERT(self->mPendingProfiles.begin() <= pendingProfile && + pendingProfile < self->mPendingProfiles.end()); + self->mPendingProfiles.erase(pendingProfile); + if (self->mPendingProfiles.empty()) { + // We've got all of the async profiles now. Let's finish off the + // profile and resolve the Promise. + self->FinishGathering(); + } + } + }); + return true; +} + +/* static */ void nsProfiler::GatheringTimerCallback(nsITimer* aTimer, + void* aClosure) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + nsCOMPtr<nsIProfiler> profiler( + do_GetService("@mozilla.org/tools/profiler;1")); + if (!profiler) { + // No (more) profiler service. + return; + } + nsProfiler* self = static_cast<nsProfiler*>(profiler.get()); + if (self != aClosure) { + // Different service object!? + return; + } + if (aTimer != self->mGatheringTimer) { + // This timer was cancelled after this callback was queued. + return; + } + + bool progressWasMade = false; + + // Going backwards, it's easier and cheaper to erase elements if needed. + for (auto iPlus1 = self->mPendingProfiles.length(); iPlus1 != 0; --iPlus1) { + PendingProfile& pendingProfile = self->mPendingProfiles[iPlus1 - 1]; + + bool needToSendProgressRequest = false; + if (pendingProfile.lastProgressRequest.IsNull()) { + DEBUG_LOG("GatheringTimerCallback() - child %u: No data yet", + unsigned(pendingProfile.childPid)); + // First time going through the list, send an initial progress request. + needToSendProgressRequest = true; + // We pretend that progress was made, so we don't give up yet. + progressWasMade = true; + } else if (pendingProfile.lastProgressResponse.IsNull()) { + LOG("GatheringTimerCallback() - child %u: Waiting for first response", + unsigned(pendingProfile.childPid)); + // Still waiting for the first response, no progress made here, don't send + // another request. + } else if (pendingProfile.lastProgressResponse <= + pendingProfile.lastProgressRequest) { + LOG("GatheringTimerCallback() - child %u: Waiting for response", + unsigned(pendingProfile.childPid)); + // Still waiting for a response to the last request, no progress made + // here, don't send another request. + } else if (pendingProfile.lastProgressChange.IsNull()) { + LOG("GatheringTimerCallback() - child %u: Still waiting for first change", + unsigned(pendingProfile.childPid)); + // Still waiting for the first change, no progress made here, but send a + // new request. + needToSendProgressRequest = true; + } else if (pendingProfile.lastProgressRequest < + pendingProfile.lastProgressChange) { + DEBUG_LOG("GatheringTimerCallback() - child %u: Recent change", + unsigned(pendingProfile.childPid)); + // We have a recent change, progress was made. + needToSendProgressRequest = true; + progressWasMade = true; + } else { + LOG("GatheringTimerCallback() - child %u: No recent change", + unsigned(pendingProfile.childPid)); + needToSendProgressRequest = true; + } + + // And send a new progress request. + if (needToSendProgressRequest) { + if (!self->SendProgressRequest(pendingProfile)) { + // Failed to even send the request, consider this process gone. + self->mPendingProfiles.erase(&pendingProfile); + LOG("... Failed to send progress request"); + } else { + DEBUG_LOG("... Sent progress request"); + } + } else { + DEBUG_LOG("... No progress request"); + } + } + + if (self->mPendingProfiles.empty()) { + // We've got all of the async profiles now. Let's finish off the profile + // and resolve the Promise. + self->FinishGathering(); + return; + } + + // Not finished yet. + + if (progressWasMade) { + // We made some progress, just restart the timer. + DEBUG_LOG("GatheringTimerCallback() - Progress made, restart timer"); + self->RestartGatheringTimer(); + return; + } + + DEBUG_LOG("GatheringTimerCallback() - Timeout!"); + self->mGatheringTimer = nullptr; + if (!profiler_is_active() || !self->mGathering) { + // Not gathering anymore. + return; + } + self->LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{ + "No progress made recently, giving up; pending pids:"}); + for (const PendingProfile& pendingProfile : self->mPendingProfiles) { + aEvent.append(Json::Value::UInt64(pendingProfile.childPid)); + } + }); + NS_WARNING("Profiler failed to gather profiles from all sub-processes"); + // We have really reached a timeout while gathering, finish now. + // TODO: Add information about missing processes. + self->FinishGathering(); +} + +void nsProfiler::RestartGatheringTimer() { + if (mGatheringTimer) { + uint32_t delayMs = 0; + const nsresult r = mGatheringTimer->GetDelay(&delayMs); + mGatheringTimer->Cancel(); + if (NS_FAILED(r) || delayMs == 0 || + NS_FAILED(mGatheringTimer->InitWithNamedFuncCallback( + GatheringTimerCallback, this, delayMs, + nsITimer::TYPE_ONE_SHOT_LOW_PRIORITY, + "nsProfilerGatheringTimer"))) { + // Can't restart the timer, so we can't wait any longer. + FinishGathering(); + } + } +} + +nsProfiler::PendingProfile* nsProfiler::GetPendingProfile( + base::ProcessId aChildPid) { + for (PendingProfile& pendingProfile : mPendingProfiles) { + if (pendingProfile.childPid == aChildPid) { + return &pendingProfile; + } + } + return nullptr; +} + +void nsProfiler::GatheredOOPProfile( + base::ProcessId aChildPid, const nsACString& aProfile, + mozilla::Maybe<ProfileGenerationAdditionalInformation>&& + aAdditionalInformation) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + + if (!profiler_is_active()) { + return; + } + + if (!mGathering) { + // If we're not actively gathering, then we don't actually care that we + // gathered a profile here. This can happen for processes that exit while + // profiling. + return; + } + + MOZ_RELEASE_ASSERT(mWriter.isSome(), + "Should always have a writer if mGathering is true"); + + // Combine all the additional information into a single struct. + if (aAdditionalInformation.isSome()) { + mProfileGenerationAdditionalInformation->Append( + std::move(*aAdditionalInformation)); + } + + if (!aProfile.IsEmpty()) { + if (mWriter->ChunkedWriteFunc().Length() + aProfile.Length() < + scLengthAccumulationThreshold) { + // TODO: Remove PromiseFlatCString, see bug 1657033. + mWriter->Splice(PromiseFlatCString(aProfile)); + } else { + LogEvent([&](Json::Value& aEvent) { + aEvent.append( + Json::StaticString{"Discarded child profile that would make the " + "full profile too big, pid and size:"}); + aEvent.append(Json::Value::UInt64(aChildPid)); + aEvent.append(Json::Value::UInt64{aProfile.Length()}); + }); + } + } + + if (PendingProfile* pendingProfile = GetPendingProfile(aChildPid); + pendingProfile) { + mPendingProfiles.erase(pendingProfile); + + if (mPendingProfiles.empty()) { + // We've got all of the async profiles now. Let's finish off the profile + // and resolve the Promise. + FinishGathering(); + } + } + + // Not finished yet, restart the timer to let any remaining child enough time + // to do their profile-streaming. + RestartGatheringTimer(); +} + +RefPtr<nsProfiler::GatheringPromiseAndroid> +nsProfiler::GetProfileDataAsGzippedArrayBufferAndroid(double aSinceTime) { + MOZ_ASSERT(NS_IsMainThread()); + + if (!profiler_is_active()) { + return GatheringPromiseAndroid::CreateAndReject(NS_ERROR_FAILURE, __func__); + } + + return StartGathering(aSinceTime) + ->Then( + GetMainThreadSerialEventTarget(), __func__, + [](const mozilla::ProfileAndAdditionalInformation& aResult) { + FallibleTArray<uint8_t> outBuff; + nsresult result = CompressString(aResult.mProfile, outBuff); + if (result != NS_OK) { + return GatheringPromiseAndroid::CreateAndReject(result, __func__); + } + return GatheringPromiseAndroid::CreateAndResolve(std::move(outBuff), + __func__); + }, + [](nsresult aRv) { + return GatheringPromiseAndroid::CreateAndReject(aRv, __func__); + }); +} + +RefPtr<nsProfiler::GatheringPromise> nsProfiler::StartGathering( + double aSinceTime) { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + + if (mGathering) { + // If we're already gathering, return a rejected promise - this isn't + // going to end well. + return GatheringPromise::CreateAndReject(NS_ERROR_NOT_AVAILABLE, __func__); + } + + mGathering = true; + mGatheringLog = mozilla::MakeUnique<Json::Value>(Json::objectValue); + (*mGatheringLog)[Json::StaticString{ + "profileGatheringLogBegin" TIMESTAMP_JSON_SUFFIX}] = + ProfilingLog::Timestamp(); + + if (mGatheringTimer) { + mGatheringTimer->Cancel(); + mGatheringTimer = nullptr; + } + + // Start building shared library info starting from the current process. + mProfileGenerationAdditionalInformation.emplace( + SharedLibraryInfo::GetInfoForSelf()); + + // Request profiles from the other processes. This will trigger asynchronous + // calls to ProfileGatherer::GatheredOOPProfile as the profiles arrive. + // + // Do this before the call to profiler_stream_json_for_this_process() because + // that call is slow and we want to let the other processes grab their + // profiles as soon as possible. + nsTArray<ProfilerParent::SingleProcessProfilePromiseAndChildPid> profiles = + ProfilerParent::GatherProfiles(); + + MOZ_ASSERT(mPendingProfiles.empty()); + if (!mPendingProfiles.reserve(profiles.Length())) { + ResetGathering(NS_ERROR_OUT_OF_MEMORY); + return GatheringPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__); + } + + mFailureLatchSource.emplace(); + mWriter.emplace(*mFailureLatchSource); + + UniquePtr<ProfilerCodeAddressService> service = + profiler_code_address_service_for_presymbolication(); + + // Start building up the JSON result and grab the profile from this process. + mWriter->Start(); + auto rv = profiler_stream_json_for_this_process(*mWriter, aSinceTime, + /* aIsShuttingDown */ false, + service.get()); + if (rv.isErr()) { + // The profiler is inactive. This either means that it was inactive even + // at the time that ProfileGatherer::Start() was called, or that it was + // stopped on a different thread since that call. Either way, we need to + // reject the promise and stop gathering. + ResetGathering(NS_ERROR_NOT_AVAILABLE); + return GatheringPromise::CreateAndReject(NS_ERROR_NOT_AVAILABLE, __func__); + } + + LogEvent([&](Json::Value& aEvent) { + aEvent.append( + Json::StaticString{"Generated parent process profile, size:"}); + aEvent.append(Json::Value::UInt64{mWriter->ChunkedWriteFunc().Length()}); + }); + + mWriter->StartArrayProperty("processes"); + + // If we have any process exit profiles, add them immediately. + if (Vector<nsCString> exitProfiles = profiler_move_exit_profiles(); + !exitProfiles.empty()) { + for (auto& exitProfile : exitProfiles) { + if (!exitProfile.IsEmpty()) { + if (exitProfile[0] == '*') { + LogEvent([&](Json::Value& aEvent) { + aEvent.append( + Json::StaticString{"Exit non-profile with error message:"}); + aEvent.append(exitProfile.Data() + 1); + }); + } else if (mWriter->ChunkedWriteFunc().Length() + exitProfile.Length() < + scLengthAccumulationThreshold) { + mWriter->Splice(exitProfile); + LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{"Added exit profile with size:"}); + aEvent.append(Json::Value::UInt64{exitProfile.Length()}); + }); + } else { + LogEvent([&](Json::Value& aEvent) { + aEvent.append( + Json::StaticString{"Discarded an exit profile that would make " + "the full profile too big, size:"}); + aEvent.append(Json::Value::UInt64{exitProfile.Length()}); + }); + } + } + } + + LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{ + "Processed all exit profiles, total size so far:"}); + aEvent.append(Json::Value::UInt64{mWriter->ChunkedWriteFunc().Length()}); + }); + } else { + // There are no pending profiles, we're already done. + LogEventLiteralString("No exit profiles."); + } + + mPromiseHolder.emplace(); + RefPtr<GatheringPromise> promise = mPromiseHolder->Ensure(__func__); + + // Keep the array property "processes" and the root object in mWriter open + // until FinishGathering() is called. As profiles from the other processes + // come in, they will be inserted and end up in the right spot. + // FinishGathering() will close the array and the root object. + + if (!profiles.IsEmpty()) { + // There *are* pending profiles, let's add handlers for their promises. + + // This timeout value is used to monitor progress while gathering child + // profiles. The timer will be restarted after we receive a response with + // any progress. + constexpr uint32_t cMinChildTimeoutS = 1u; // 1 second minimum and default. + constexpr uint32_t cMaxChildTimeoutS = 60u; // 1 minute max. + uint32_t childTimeoutS = Preferences::GetUint( + "devtools.performance.recording.child.timeout_s", cMinChildTimeoutS); + if (childTimeoutS < cMinChildTimeoutS) { + childTimeoutS = cMinChildTimeoutS; + } else if (childTimeoutS > cMaxChildTimeoutS) { + childTimeoutS = cMaxChildTimeoutS; + } + const uint32_t childTimeoutMs = childTimeoutS * PR_MSEC_PER_SEC; + Unused << NS_NewTimerWithFuncCallback( + getter_AddRefs(mGatheringTimer), GatheringTimerCallback, this, + childTimeoutMs, nsITimer::TYPE_ONE_SHOT_LOW_PRIORITY, + "nsProfilerGatheringTimer", GetMainThreadSerialEventTarget()); + + MOZ_ASSERT(mPendingProfiles.capacity() >= profiles.Length()); + for (const auto& profile : profiles) { + mPendingProfiles.infallibleAppend(PendingProfile{profile.childPid}); + LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{"Waiting for pending profile, pid:"}); + aEvent.append(Json::Value::UInt64(profile.childPid)); + }); + profile.profilePromise->Then( + GetMainThreadSerialEventTarget(), __func__, + [self = RefPtr<nsProfiler>(this), childPid = profile.childPid]( + IPCProfileAndAdditionalInformation&& aResult) { + PendingProfile* pendingProfile = self->GetPendingProfile(childPid); + mozilla::ipc::Shmem profileShmem = aResult.profileShmem(); + LOG("GatherProfile(%u) response: %u bytes (%u were pending, %s %u)", + unsigned(childPid), unsigned(profileShmem.Size<char>()), + unsigned(self->mPendingProfiles.length()), + pendingProfile ? "including" : "excluding", unsigned(childPid)); + if (profileShmem.IsReadable()) { + self->LogEvent([&](Json::Value& aEvent) { + aEvent.append( + Json::StaticString{"Got profile from pid, with size:"}); + aEvent.append(Json::Value::UInt64(childPid)); + aEvent.append(Json::Value::UInt64{profileShmem.Size<char>()}); + }); + const nsDependentCSubstring profileString( + profileShmem.get<char>(), profileShmem.Size<char>() - 1); + if (profileString.IsEmpty() || profileString[0] != '*') { + self->GatheredOOPProfile( + childPid, profileString, + std::move(aResult.additionalInformation())); + } else { + self->LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{ + "Child non-profile from pid, with error message:"}); + aEvent.append(Json::Value::UInt64(childPid)); + aEvent.append(profileString.Data() + 1); + }); + self->GatheredOOPProfile(childPid, ""_ns, Nothing()); + } + } else { + // This can happen if the child failed to allocate + // the Shmem (or maliciously sent an invalid Shmem). + self->LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{"Got failure from pid:"}); + aEvent.append(Json::Value::UInt64(childPid)); + }); + self->GatheredOOPProfile(childPid, ""_ns, Nothing()); + } + }, + [self = RefPtr<nsProfiler>(this), + childPid = profile.childPid](ipc::ResponseRejectReason&& aReason) { + PendingProfile* pendingProfile = self->GetPendingProfile(childPid); + LOG("GatherProfile(%u) rejection: %d (%u were pending, %s %u)", + unsigned(childPid), (int)aReason, + unsigned(self->mPendingProfiles.length()), + pendingProfile ? "including" : "excluding", unsigned(childPid)); + self->LogEvent([&](Json::Value& aEvent) { + aEvent.append( + Json::StaticString{"Got rejection from pid, with reason:"}); + aEvent.append(Json::Value::UInt64(childPid)); + aEvent.append(Json::Value::UInt{static_cast<unsigned>(aReason)}); + }); + self->GatheredOOPProfile(childPid, ""_ns, Nothing()); + }); + } + } else { + // There are no pending profiles, we're already done. + LogEventLiteralString("No pending child profiles."); + FinishGathering(); + } + + return promise; +} + +RefPtr<nsProfiler::SymbolTablePromise> nsProfiler::GetSymbolTableMozPromise( + const nsACString& aDebugPath, const nsACString& aBreakpadID) { + MozPromiseHolder<SymbolTablePromise> promiseHolder; + RefPtr<SymbolTablePromise> promise = promiseHolder.Ensure(__func__); + + if (!mSymbolTableThread) { + nsresult rv = NS_NewNamedThread("ProfSymbolTable", + getter_AddRefs(mSymbolTableThread)); + if (NS_WARN_IF(NS_FAILED(rv))) { + promiseHolder.Reject(NS_ERROR_FAILURE, __func__); + return promise; + } + } + + nsresult rv = mSymbolTableThread->Dispatch(NS_NewRunnableFunction( + "nsProfiler::GetSymbolTableMozPromise runnable on ProfSymbolTable thread", + [promiseHolder = std::move(promiseHolder), + debugPath = nsCString(aDebugPath), + breakpadID = nsCString(aBreakpadID)]() mutable { + AUTO_PROFILER_LABEL_DYNAMIC_NSCSTRING("profiler_get_symbol_table", + OTHER, debugPath); + SymbolTable symbolTable; + bool succeeded = profiler_get_symbol_table( + debugPath.get(), breakpadID.get(), &symbolTable); + if (succeeded) { + promiseHolder.Resolve(std::move(symbolTable), __func__); + } else { + promiseHolder.Reject(NS_ERROR_FAILURE, __func__); + } + })); + + if (NS_WARN_IF(NS_FAILED(rv))) { + // Get-symbol task was not dispatched and therefore won't fulfill the + // promise, we must reject the promise now. + promiseHolder.Reject(NS_ERROR_FAILURE, __func__); + } + + return promise; +} + +void nsProfiler::FinishGathering() { + MOZ_RELEASE_ASSERT(NS_IsMainThread()); + MOZ_RELEASE_ASSERT(mWriter.isSome()); + MOZ_RELEASE_ASSERT(mPromiseHolder.isSome()); + MOZ_RELEASE_ASSERT(mProfileGenerationAdditionalInformation.isSome()); + + // Close the "processes" array property. + mWriter->EndArray(); + + if (mGatheringLog) { + LogEvent([&](Json::Value& aEvent) { + aEvent.append(Json::StaticString{"Finished gathering, total size:"}); + aEvent.append(Json::Value::UInt64{mWriter->ChunkedWriteFunc().Length()}); + }); + (*mGatheringLog)[Json::StaticString{ + "profileGatheringLogEnd" TIMESTAMP_JSON_SUFFIX}] = + ProfilingLog::Timestamp(); + mWriter->StartObjectProperty("profileGatheringLog"); + { + nsAutoCString pid; + pid.AppendInt(int64_t(profiler_current_process_id().ToNumber())); + Json::String logString = ToCompactString(*mGatheringLog); + mGatheringLog = nullptr; + mWriter->SplicedJSONProperty(pid, logString); + } + mWriter->EndObject(); + } + + // Close the root object of the generated JSON. + mWriter->End(); + + if (const char* failure = mWriter->GetFailure(); failure) { +#ifndef ANDROID + fprintf(stderr, "JSON generation failure: %s", failure); +#else + __android_log_print(ANDROID_LOG_INFO, "GeckoProfiler", + "JSON generation failure: %s", failure); +#endif + NS_WARNING("Error during JSON generation, probably OOM."); + ResetGathering(NS_ERROR_OUT_OF_MEMORY); + return; + } + + // And try to resolve the promise with the profile JSON. + const size_t len = mWriter->ChunkedWriteFunc().Length(); + if (len >= scLengthMax) { + NS_WARNING("Profile JSON is too big to fit in a string."); + ResetGathering(NS_ERROR_FILE_TOO_BIG); + return; + } + + nsCString result; + if (!result.SetLength(len, fallible)) { + NS_WARNING("Cannot allocate a string for the Profile JSON."); + ResetGathering(NS_ERROR_OUT_OF_MEMORY); + return; + } + MOZ_ASSERT(*(result.Data() + len) == '\0', + "We expected a null at the end of the string buffer, to be " + "rewritten by CopyDataIntoLazilyAllocatedBuffer"); + + char* const resultBeginWriting = result.BeginWriting(); + if (!resultBeginWriting) { + NS_WARNING("Cannot access the string to write the Profile JSON."); + ResetGathering(NS_ERROR_CACHE_WRITE_ACCESS_DENIED); + return; + } + + // Here, we have enough space reserved in `result`, starting at + // `resultBeginWriting`, copy the JSON profile there. + if (!mWriter->ChunkedWriteFunc().CopyDataIntoLazilyAllocatedBuffer( + [&](size_t aBufferLen) -> char* { + MOZ_RELEASE_ASSERT(aBufferLen == len + 1); + return resultBeginWriting; + })) { + NS_WARNING("Could not copy profile JSON, probably OOM."); + ResetGathering(NS_ERROR_FILE_TOO_BIG); + return; + } + MOZ_ASSERT(*(result.Data() + len) == '\0', + "We still expected a null at the end of the string buffer"); + + mProfileGenerationAdditionalInformation->FinishGathering(); + mPromiseHolder->Resolve( + ProfileAndAdditionalInformation{ + std::move(result), + std::move(*mProfileGenerationAdditionalInformation)}, + __func__); + + ResetGathering(NS_ERROR_UNEXPECTED); +} + +void nsProfiler::ResetGathering(nsresult aPromiseRejectionIfPending) { + // If we have an unfulfilled Promise in flight, we should reject it before + // destroying the promise holder. + if (mPromiseHolder.isSome()) { + mPromiseHolder->RejectIfExists(aPromiseRejectionIfPending, __func__); + mPromiseHolder.reset(); + } + mPendingProfiles.clearAndFree(); + mGathering = false; + mGatheringLog = nullptr; + if (mGatheringTimer) { + mGatheringTimer->Cancel(); + mGatheringTimer = nullptr; + } + mWriter.reset(); + mFailureLatchSource.reset(); + mProfileGenerationAdditionalInformation.reset(); +} diff --git a/tools/profiler/gecko/nsProfiler.h b/tools/profiler/gecko/nsProfiler.h new file mode 100644 index 0000000000..3757df3079 --- /dev/null +++ b/tools/profiler/gecko/nsProfiler.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsProfiler_h +#define nsProfiler_h + +#include "base/process.h" +#include "mozilla/Attributes.h" +#include "mozilla/Maybe.h" +#include "mozilla/MozPromise.h" +#include "mozilla/ProfileJSONWriter.h" +#include "mozilla/ProportionValue.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Vector.h" +#include "nsIProfiler.h" +#include "nsITimer.h" +#include "nsServiceManagerUtils.h" +#include "ProfilerCodeAddressService.h" +#include "ProfileAdditionalInformation.h" + +namespace Json { +class Value; +} // namespace Json + +class nsProfiler final : public nsIProfiler { + public: + nsProfiler(); + + NS_DECL_ISUPPORTS + NS_DECL_NSIPROFILER + + nsresult Init(); + + static nsProfiler* GetOrCreate() { + nsCOMPtr<nsIProfiler> iprofiler = + do_GetService("@mozilla.org/tools/profiler;1"); + return static_cast<nsProfiler*>(iprofiler.get()); + } + + private: + ~nsProfiler(); + + using GatheringPromiseAndroid = + mozilla::MozPromise<FallibleTArray<uint8_t>, nsresult, true>; + using GatheringPromise = + mozilla::MozPromise<mozilla::ProfileAndAdditionalInformation, nsresult, + false>; + using SymbolTablePromise = + mozilla::MozPromise<mozilla::SymbolTable, nsresult, true>; + + RefPtr<GatheringPromise> StartGathering(double aSinceTime); + void GatheredOOPProfile( + base::ProcessId aChildPid, const nsACString& aProfile, + mozilla::Maybe<mozilla::ProfileGenerationAdditionalInformation>&& + aAdditionalInformation); + void FinishGathering(); + void ResetGathering(nsresult aPromiseRejectionIfPending); + static void GatheringTimerCallback(nsITimer* aTimer, void* aClosure); + void RestartGatheringTimer(); + + RefPtr<SymbolTablePromise> GetSymbolTableMozPromise( + const nsACString& aDebugPath, const nsACString& aBreakpadID); + + struct ExitProfile { + nsCString mJSON; + uint64_t mBufferPositionAtGatherTime; + }; + + struct PendingProfile { + base::ProcessId childPid; + + mozilla::ProportionValue progressProportion; + nsCString progressLocation; + + mozilla::TimeStamp lastProgressRequest; + mozilla::TimeStamp lastProgressResponse; + mozilla::TimeStamp lastProgressChange; + + explicit PendingProfile(base::ProcessId aChildPid) : childPid(aChildPid) {} + }; + + PendingProfile* GetPendingProfile(base::ProcessId aChildPid); + // Returns false if the request could not be sent. + bool SendProgressRequest(PendingProfile& aPendingProfile); + + // If the log is active, call aJsonLogObjectUpdater(Json::Value&) on the log's + // root object. + template <typename JsonLogObjectUpdater> + void Log(JsonLogObjectUpdater&& aJsonLogObjectUpdater); + // If the log is active, call aJsonArrayAppender(Json::Value&) on a Json + // array that already contains a timestamp, and to which event-related + // elements may be appended. + template <typename JsonArrayAppender> + void LogEvent(JsonArrayAppender&& aJsonArrayAppender); + void LogEventLiteralString(const char* aEventString); + + // These fields are all related to profile gathering. + mozilla::Vector<ExitProfile> mExitProfiles; + mozilla::Maybe<mozilla::MozPromiseHolder<GatheringPromise>> mPromiseHolder; + nsCOMPtr<nsIThread> mSymbolTableThread; + mozilla::Maybe<mozilla::FailureLatchSource> mFailureLatchSource; + mozilla::Maybe<SpliceableChunkedJSONWriter> mWriter; + mozilla::Maybe<mozilla::ProfileGenerationAdditionalInformation> + mProfileGenerationAdditionalInformation; + mozilla::Vector<PendingProfile> mPendingProfiles; + bool mGathering; + nsCOMPtr<nsITimer> mGatheringTimer; + // Supplemental log to the profiler's "profilingLog" (which has already been + // completed in JSON profiles that are gathered). + mozilla::UniquePtr<Json::Value> mGatheringLog; +}; + +#endif // nsProfiler_h diff --git a/tools/profiler/gecko/nsProfilerCIID.h b/tools/profiler/gecko/nsProfilerCIID.h new file mode 100644 index 0000000000..3df44596b1 --- /dev/null +++ b/tools/profiler/gecko/nsProfilerCIID.h @@ -0,0 +1,16 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsProfilerCIID_h__ +#define nsProfilerCIID_h__ + +#define NS_PROFILER_CID \ + { \ + 0x25db9b8e, 0x8123, 0x4de1, { \ + 0xb6, 0x6d, 0x8b, 0xbb, 0xed, 0xf2, 0xcd, 0xf4 \ + } \ + } + +#endif diff --git a/tools/profiler/gecko/nsProfilerStartParams.cpp b/tools/profiler/gecko/nsProfilerStartParams.cpp new file mode 100644 index 0000000000..dd7c3f4ab7 --- /dev/null +++ b/tools/profiler/gecko/nsProfilerStartParams.cpp @@ -0,0 +1,65 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsProfilerStartParams.h" +#include "ipc/IPCMessageUtils.h" + +NS_IMPL_ISUPPORTS(nsProfilerStartParams, nsIProfilerStartParams) + +nsProfilerStartParams::nsProfilerStartParams( + uint32_t aEntries, const mozilla::Maybe<double>& aDuration, + double aInterval, uint32_t aFeatures, nsTArray<nsCString>&& aFilters, + uint64_t aActiveTabID) + : mEntries(aEntries), + mDuration(aDuration), + mInterval(aInterval), + mFeatures(aFeatures), + mFilters(std::move(aFilters)), + mActiveTabID(aActiveTabID) {} + +nsProfilerStartParams::~nsProfilerStartParams() {} + +NS_IMETHODIMP +nsProfilerStartParams::GetEntries(uint32_t* aEntries) { + NS_ENSURE_ARG_POINTER(aEntries); + *aEntries = mEntries; + return NS_OK; +} + +NS_IMETHODIMP +nsProfilerStartParams::GetDuration(double* aDuration) { + NS_ENSURE_ARG_POINTER(aDuration); + if (mDuration) { + *aDuration = *mDuration; + } else { + *aDuration = 0; + } + return NS_OK; +} + +NS_IMETHODIMP +nsProfilerStartParams::GetInterval(double* aInterval) { + NS_ENSURE_ARG_POINTER(aInterval); + *aInterval = mInterval; + return NS_OK; +} + +NS_IMETHODIMP +nsProfilerStartParams::GetFeatures(uint32_t* aFeatures) { + NS_ENSURE_ARG_POINTER(aFeatures); + *aFeatures = mFeatures; + return NS_OK; +} + +const nsTArray<nsCString>& nsProfilerStartParams::GetFilters() { + return mFilters; +} + +NS_IMETHODIMP +nsProfilerStartParams::GetActiveTabID(uint64_t* aActiveTabID) { + NS_ENSURE_ARG_POINTER(aActiveTabID); + *aActiveTabID = mActiveTabID; + return NS_OK; +} diff --git a/tools/profiler/gecko/nsProfilerStartParams.h b/tools/profiler/gecko/nsProfilerStartParams.h new file mode 100644 index 0000000000..25c2b5082f --- /dev/null +++ b/tools/profiler/gecko/nsProfilerStartParams.h @@ -0,0 +1,36 @@ +/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _NSPROFILERSTARTPARAMS_H_ +#define _NSPROFILERSTARTPARAMS_H_ + +#include "nsIProfiler.h" +#include "nsString.h" +#include "nsTArray.h" + +class nsProfilerStartParams : public nsIProfilerStartParams { + public: + // This class can be used on multiple threads. For example, it's used for the + // observer notification from profiler_start, which can run on any thread but + // posts the notification to the main thread. + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIPROFILERSTARTPARAMS + + nsProfilerStartParams(uint32_t aEntries, + const mozilla::Maybe<double>& aDuration, + double aInterval, uint32_t aFeatures, + nsTArray<nsCString>&& aFilters, uint64_t aActiveTabID); + + private: + virtual ~nsProfilerStartParams(); + uint32_t mEntries; + mozilla::Maybe<double> mDuration; + double mInterval; + uint32_t mFeatures; + nsTArray<nsCString> mFilters; + uint64_t mActiveTabID; +}; + +#endif diff --git a/tools/profiler/lul/AutoObjectMapper.cpp b/tools/profiler/lul/AutoObjectMapper.cpp new file mode 100644 index 0000000000..f7489fbfee --- /dev/null +++ b/tools/profiler/lul/AutoObjectMapper.cpp @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <sys/mman.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include "mozilla/Assertions.h" +#include "mozilla/Sprintf.h" + +#include "PlatformMacros.h" +#include "AutoObjectMapper.h" + +// A helper function for creating failure error messages in +// AutoObjectMapper*::Map. +static void failedToMessage(void (*aLog)(const char*), const char* aHowFailed, + std::string aFileName) { + char buf[300]; + SprintfLiteral(buf, "AutoObjectMapper::Map: Failed to %s \'%s\'", aHowFailed, + aFileName.c_str()); + buf[sizeof(buf) - 1] = 0; + aLog(buf); +} + +AutoObjectMapperPOSIX::AutoObjectMapperPOSIX(void (*aLog)(const char*)) + : mImage(nullptr), mSize(0), mLog(aLog), mIsMapped(false) {} + +AutoObjectMapperPOSIX::~AutoObjectMapperPOSIX() { + if (!mIsMapped) { + // There's nothing to do. + MOZ_ASSERT(!mImage); + MOZ_ASSERT(mSize == 0); + return; + } + MOZ_ASSERT(mSize > 0); + // The following assertion doesn't necessarily have to be true, + // but we assume (reasonably enough) that no mmap facility would + // be crazy enough to map anything at page zero. + MOZ_ASSERT(mImage); + munmap(mImage, mSize); +} + +bool AutoObjectMapperPOSIX::Map(/*OUT*/ void** start, /*OUT*/ size_t* length, + std::string fileName) { + MOZ_ASSERT(!mIsMapped); + + int fd = open(fileName.c_str(), O_RDONLY); + if (fd == -1) { + failedToMessage(mLog, "open", fileName); + return false; + } + + struct stat st; + int err = fstat(fd, &st); + size_t sz = (err == 0) ? st.st_size : 0; + if (err != 0 || sz == 0) { + failedToMessage(mLog, "fstat", fileName); + close(fd); + return false; + } + + void* image = mmap(nullptr, sz, PROT_READ, MAP_SHARED, fd, 0); + if (image == MAP_FAILED) { + failedToMessage(mLog, "mmap", fileName); + close(fd); + return false; + } + + close(fd); + mIsMapped = true; + mImage = *start = image; + mSize = *length = sz; + return true; +} diff --git a/tools/profiler/lul/AutoObjectMapper.h b/tools/profiler/lul/AutoObjectMapper.h new file mode 100644 index 0000000000..f63aa43e0e --- /dev/null +++ b/tools/profiler/lul/AutoObjectMapper.h @@ -0,0 +1,64 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef AutoObjectMapper_h +#define AutoObjectMapper_h + +#include <string> + +#include "mozilla/Attributes.h" +#include "PlatformMacros.h" + +// A (nearly-) RAII class that maps an object in and then unmaps it on +// destruction. This base class version uses the "normal" POSIX +// functions: open, fstat, close, mmap, munmap. + +class MOZ_STACK_CLASS AutoObjectMapperPOSIX { + public: + // The constructor does not attempt to map the file, because that + // might fail. Instead, once the object has been constructed, + // call Map() to attempt the mapping. There is no corresponding + // Unmap() since the unmapping is done in the destructor. Failure + // messages are sent to |aLog|. + explicit AutoObjectMapperPOSIX(void (*aLog)(const char*)); + + // Unmap the file on destruction of this object. + ~AutoObjectMapperPOSIX(); + + // Map |fileName| into the address space and return the mapping + // extents. If the file is zero sized this will fail. The file is + // mapped read-only and private. Returns true iff the mapping + // succeeded, in which case *start and *length hold its extent. + // Once a call to Map succeeds, all subsequent calls to it will + // fail. + bool Map(/*OUT*/ void** start, /*OUT*/ size_t* length, std::string fileName); + + protected: + // If we are currently holding a mapped object, these record the + // mapped address range. + void* mImage; + size_t mSize; + + // A logging sink, for complaining about mapping failures. + void (*mLog)(const char*); + + private: + // Are we currently holding a mapped object? This is private to + // the base class. Derived classes need to have their own way to + // track whether they are holding a mapped object. + bool mIsMapped; + + // Disable copying and assignment. + AutoObjectMapperPOSIX(const AutoObjectMapperPOSIX&); + AutoObjectMapperPOSIX& operator=(const AutoObjectMapperPOSIX&); + // Disable heap allocation of this class. + void* operator new(size_t); + void* operator new[](size_t); + void operator delete(void*); + void operator delete[](void*); +}; + +#endif // AutoObjectMapper_h diff --git a/tools/profiler/lul/LulCommon.cpp b/tools/profiler/lul/LulCommon.cpp new file mode 100644 index 0000000000..428f102c42 --- /dev/null +++ b/tools/profiler/lul/LulCommon.cpp @@ -0,0 +1,100 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ + +// Copyright (c) 2011, 2013 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Original author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/module.cc +// src/common/unique_string.cc + +// There's no internal-only interface for LulCommon. Hence include +// the external interface directly. +#include "LulCommonExt.h" + +#include <stdlib.h> +#include <string.h> + +#include <string> +#include <map> + +namespace lul { + +using std::string; + +//////////////////////////////////////////////////////////////// +// Module +// +Module::Module(const string& name, const string& os, const string& architecture, + const string& id) + : name_(name), os_(os), architecture_(architecture), id_(id) {} + +Module::~Module() {} + +//////////////////////////////////////////////////////////////// +// UniqueString +// +class UniqueString { + public: + explicit UniqueString(string str) { str_ = strdup(str.c_str()); } + ~UniqueString() { free(reinterpret_cast<void*>(const_cast<char*>(str_))); } + const char* str_; +}; + +const char* FromUniqueString(const UniqueString* ustr) { return ustr->str_; } + +bool IsEmptyUniqueString(const UniqueString* ustr) { + return (ustr->str_)[0] == '\0'; +} + +//////////////////////////////////////////////////////////////// +// UniqueStringUniverse +// +UniqueStringUniverse::~UniqueStringUniverse() { + for (std::map<string, UniqueString*>::iterator it = map_.begin(); + it != map_.end(); it++) { + delete it->second; + } +} + +const UniqueString* UniqueStringUniverse::ToUniqueString(string str) { + std::map<string, UniqueString*>::iterator it = map_.find(str); + if (it == map_.end()) { + UniqueString* ustr = new UniqueString(str); + map_[str] = ustr; + return ustr; + } else { + return it->second; + } +} + +} // namespace lul diff --git a/tools/profiler/lul/LulCommonExt.h b/tools/profiler/lul/LulCommonExt.h new file mode 100644 index 0000000000..b20a7321ff --- /dev/null +++ b/tools/profiler/lul/LulCommonExt.h @@ -0,0 +1,509 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ + +// Copyright (c) 2006, 2010, 2012, 2013 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Original author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// module.h: Define google_breakpad::Module. A Module holds debugging +// information, and can write that information out as a Breakpad +// symbol file. + +// (C) Copyright Greg Colvin and Beman Dawes 1998, 1999. +// Copyright (c) 2001, 2002 Peter Dimov +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// +// See http://www.boost.org/libs/smart_ptr/scoped_ptr.htm for documentation. +// + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/unique_string.h +// src/common/scoped_ptr.h +// src/common/module.h + +// External interface for the "Common" component of LUL. + +#ifndef LulCommonExt_h +#define LulCommonExt_h + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> + +#include <string> +#include <map> +#include <vector> +#include <cstddef> // for std::ptrdiff_t + +#include "mozilla/Assertions.h" + +namespace lul { + +using std::map; +using std::string; + +//////////////////////////////////////////////////////////////// +// UniqueString +// + +// Abstract type +class UniqueString; + +// Get the contained C string (debugging only) +const char* FromUniqueString(const UniqueString*); + +// Is the given string empty (that is, "") ? +bool IsEmptyUniqueString(const UniqueString*); + +//////////////////////////////////////////////////////////////// +// UniqueStringUniverse +// + +// All UniqueStrings live in some specific UniqueStringUniverse. +class UniqueStringUniverse { + public: + UniqueStringUniverse() {} + ~UniqueStringUniverse(); + // Convert a |string| to a UniqueString, that lives in this universe. + const UniqueString* ToUniqueString(string str); + + private: + map<string, UniqueString*> map_; +}; + +//////////////////////////////////////////////////////////////// +// GUID +// + +typedef struct { + uint32_t data1; + uint16_t data2; + uint16_t data3; + uint8_t data4[8]; +} MDGUID; // GUID + +typedef MDGUID GUID; + +//////////////////////////////////////////////////////////////// +// scoped_ptr +// + +// scoped_ptr mimics a built-in pointer except that it guarantees deletion +// of the object pointed to, either on destruction of the scoped_ptr or via +// an explicit reset(). scoped_ptr is a simple solution for simple needs; +// use shared_ptr or std::auto_ptr if your needs are more complex. + +// *** NOTE *** +// If your scoped_ptr is a class member of class FOO pointing to a +// forward declared type BAR (as shown below), then you MUST use a non-inlined +// version of the destructor. The destructor of a scoped_ptr (called from +// FOO's destructor) must have a complete definition of BAR in order to +// destroy it. Example: +// +// -- foo.h -- +// class BAR; +// +// class FOO { +// public: +// FOO(); +// ~FOO(); // Required for sources that instantiate class FOO to compile! +// +// private: +// scoped_ptr<BAR> bar_; +// }; +// +// -- foo.cc -- +// #include "foo.h" +// FOO::~FOO() {} // Empty, but must be non-inlined to FOO's class definition. + +// scoped_ptr_malloc added by Google +// When one of these goes out of scope, instead of doing a delete or +// delete[], it calls free(). scoped_ptr_malloc<char> is likely to see +// much more use than any other specializations. + +// release() added by Google +// Use this to conditionally transfer ownership of a heap-allocated object +// to the caller, usually on method success. + +template <typename T> +class scoped_ptr { + private: + T* ptr; + + scoped_ptr(scoped_ptr const&); + scoped_ptr& operator=(scoped_ptr const&); + + public: + typedef T element_type; + + explicit scoped_ptr(T* p = 0) : ptr(p) {} + + ~scoped_ptr() { delete ptr; } + + void reset(T* p = 0) { + if (ptr != p) { + delete ptr; + ptr = p; + } + } + + T& operator*() const { + MOZ_ASSERT(ptr != 0); + return *ptr; + } + + T* operator->() const { + MOZ_ASSERT(ptr != 0); + return ptr; + } + + bool operator==(T* p) const { return ptr == p; } + + bool operator!=(T* p) const { return ptr != p; } + + T* get() const { return ptr; } + + void swap(scoped_ptr& b) { + T* tmp = b.ptr; + b.ptr = ptr; + ptr = tmp; + } + + T* release() { + T* tmp = ptr; + ptr = 0; + return tmp; + } + + private: + // no reason to use these: each scoped_ptr should have its own object + template <typename U> + bool operator==(scoped_ptr<U> const& p) const; + template <typename U> + bool operator!=(scoped_ptr<U> const& p) const; +}; + +template <typename T> +inline void swap(scoped_ptr<T>& a, scoped_ptr<T>& b) { + a.swap(b); +} + +template <typename T> +inline bool operator==(T* p, const scoped_ptr<T>& b) { + return p == b.get(); +} + +template <typename T> +inline bool operator!=(T* p, const scoped_ptr<T>& b) { + return p != b.get(); +} + +// scoped_array extends scoped_ptr to arrays. Deletion of the array pointed to +// is guaranteed, either on destruction of the scoped_array or via an explicit +// reset(). Use shared_array or std::vector if your needs are more complex. + +template <typename T> +class scoped_array { + private: + T* ptr; + + scoped_array(scoped_array const&); + scoped_array& operator=(scoped_array const&); + + public: + typedef T element_type; + + explicit scoped_array(T* p = 0) : ptr(p) {} + + ~scoped_array() { delete[] ptr; } + + void reset(T* p = 0) { + if (ptr != p) { + delete[] ptr; + ptr = p; + } + } + + T& operator[](std::ptrdiff_t i) const { + MOZ_ASSERT(ptr != 0); + MOZ_ASSERT(i >= 0); + return ptr[i]; + } + + bool operator==(T* p) const { return ptr == p; } + + bool operator!=(T* p) const { return ptr != p; } + + T* get() const { return ptr; } + + void swap(scoped_array& b) { + T* tmp = b.ptr; + b.ptr = ptr; + ptr = tmp; + } + + T* release() { + T* tmp = ptr; + ptr = 0; + return tmp; + } + + private: + // no reason to use these: each scoped_array should have its own object + template <typename U> + bool operator==(scoped_array<U> const& p) const; + template <typename U> + bool operator!=(scoped_array<U> const& p) const; +}; + +template <class T> +inline void swap(scoped_array<T>& a, scoped_array<T>& b) { + a.swap(b); +} + +template <typename T> +inline bool operator==(T* p, const scoped_array<T>& b) { + return p == b.get(); +} + +template <typename T> +inline bool operator!=(T* p, const scoped_array<T>& b) { + return p != b.get(); +} + +// This class wraps the c library function free() in a class that can be +// passed as a template argument to scoped_ptr_malloc below. +class ScopedPtrMallocFree { + public: + inline void operator()(void* x) const { free(x); } +}; + +// scoped_ptr_malloc<> is similar to scoped_ptr<>, but it accepts a +// second template argument, the functor used to free the object. + +template <typename T, typename FreeProc = ScopedPtrMallocFree> +class scoped_ptr_malloc { + private: + T* ptr; + + scoped_ptr_malloc(scoped_ptr_malloc const&); + scoped_ptr_malloc& operator=(scoped_ptr_malloc const&); + + public: + typedef T element_type; + + explicit scoped_ptr_malloc(T* p = 0) : ptr(p) {} + + ~scoped_ptr_malloc() { free_((void*)ptr); } + + void reset(T* p = 0) { + if (ptr != p) { + free_((void*)ptr); + ptr = p; + } + } + + T& operator*() const { + MOZ_ASSERT(ptr != 0); + return *ptr; + } + + T* operator->() const { + MOZ_ASSERT(ptr != 0); + return ptr; + } + + bool operator==(T* p) const { return ptr == p; } + + bool operator!=(T* p) const { return ptr != p; } + + T* get() const { return ptr; } + + void swap(scoped_ptr_malloc& b) { + T* tmp = b.ptr; + b.ptr = ptr; + ptr = tmp; + } + + T* release() { + T* tmp = ptr; + ptr = 0; + return tmp; + } + + private: + // no reason to use these: each scoped_ptr_malloc should have its own object + template <typename U, typename GP> + bool operator==(scoped_ptr_malloc<U, GP> const& p) const; + template <typename U, typename GP> + bool operator!=(scoped_ptr_malloc<U, GP> const& p) const; + + static FreeProc const free_; +}; + +template <typename T, typename FP> +FP const scoped_ptr_malloc<T, FP>::free_ = FP(); + +template <typename T, typename FP> +inline void swap(scoped_ptr_malloc<T, FP>& a, scoped_ptr_malloc<T, FP>& b) { + a.swap(b); +} + +template <typename T, typename FP> +inline bool operator==(T* p, const scoped_ptr_malloc<T, FP>& b) { + return p == b.get(); +} + +template <typename T, typename FP> +inline bool operator!=(T* p, const scoped_ptr_malloc<T, FP>& b) { + return p != b.get(); +} + +//////////////////////////////////////////////////////////////// +// Module +// + +// A Module represents the contents of a module, and supports methods +// for adding information produced by parsing STABS or DWARF data +// --- possibly both from the same file --- and then writing out the +// unified contents as a Breakpad-format symbol file. +class Module { + public: + // The type of addresses and sizes in a symbol table. + typedef uint64_t Address; + + // Representation of an expression. This can either be a postfix + // expression, in which case it is stored as a string, or a simple + // expression of the form (identifier + imm) or *(identifier + imm). + // It can also be invalid (denoting "no value"). + enum ExprHow { kExprInvalid = 1, kExprPostfix, kExprSimple, kExprSimpleMem }; + + struct Expr { + // Construct a simple-form expression + Expr(const UniqueString* ident, long offset, bool deref) { + if (IsEmptyUniqueString(ident)) { + Expr(); + } else { + postfix_ = ""; + ident_ = ident; + offset_ = offset; + how_ = deref ? kExprSimpleMem : kExprSimple; + } + } + + // Construct an invalid expression + Expr() { + postfix_ = ""; + ident_ = nullptr; + offset_ = 0; + how_ = kExprInvalid; + } + + // Return the postfix expression string, either directly, + // if this is a postfix expression, or by synthesising it + // for a simple expression. + std::string getExprPostfix() const { + switch (how_) { + case kExprPostfix: + return postfix_; + case kExprSimple: + case kExprSimpleMem: { + char buf[40]; + sprintf(buf, " %ld %c%s", labs(offset_), offset_ < 0 ? '-' : '+', + how_ == kExprSimple ? "" : " ^"); + return std::string(FromUniqueString(ident_)) + std::string(buf); + } + case kExprInvalid: + default: + MOZ_ASSERT(0 && "getExprPostfix: invalid Module::Expr type"); + return "Expr::genExprPostfix: kExprInvalid"; + } + } + + // The identifier that gives the starting value for simple expressions. + const UniqueString* ident_; + // The offset to add for simple expressions. + long offset_; + // The Postfix expression string to evaluate for non-simple expressions. + std::string postfix_; + // The operation expressed by this expression. + ExprHow how_; + }; + + // A map from register names to expressions that recover + // their values. This can represent a complete set of rules to + // follow at some address, or a set of changes to be applied to an + // extant set of rules. + // NOTE! there are two completely different types called RuleMap. This + // is one of them. + typedef std::map<const UniqueString*, Expr> RuleMap; + + // A map from addresses to RuleMaps, representing changes that take + // effect at given addresses. + typedef std::map<Address, RuleMap> RuleChangeMap; + + // A range of 'STACK CFI' stack walking information. An instance of + // this structure corresponds to a 'STACK CFI INIT' record and the + // subsequent 'STACK CFI' records that fall within its range. + struct StackFrameEntry { + // The starting address and number of bytes of machine code this + // entry covers. + Address address, size; + + // The initial register recovery rules, in force at the starting + // address. + RuleMap initial_rules; + + // A map from addresses to rule changes. To find the rules in + // force at a given address, start with initial_rules, and then + // apply the changes given in this map for all addresses up to and + // including the address you're interested in. + RuleChangeMap rule_changes; + }; + + // Create a new module with the given name, operating system, + // architecture, and ID string. + Module(const std::string& name, const std::string& os, + const std::string& architecture, const std::string& id); + ~Module(); + + private: + // Module header entries. + std::string name_, os_, architecture_, id_; +}; + +} // namespace lul + +#endif // LulCommonExt_h diff --git a/tools/profiler/lul/LulDwarf.cpp b/tools/profiler/lul/LulDwarf.cpp new file mode 100644 index 0000000000..ea38ce50ea --- /dev/null +++ b/tools/profiler/lul/LulDwarf.cpp @@ -0,0 +1,2538 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ + +// Copyright (c) 2010 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// CFI reader author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> +// Original author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// Implementation of dwarf2reader::LineInfo, dwarf2reader::CompilationUnit, +// and dwarf2reader::CallFrameInfo. See dwarf2reader.h for details. + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/dwarf/bytereader.cc +// src/common/dwarf/dwarf2reader.cc +// src/common/dwarf_cfi_to_module.cc + +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <stdlib.h> + +#include <stack> +#include <string> + +#include "mozilla/Assertions.h" +#include "mozilla/Attributes.h" +#include "mozilla/Sprintf.h" +#include "mozilla/Vector.h" + +#include "LulCommonExt.h" +#include "LulDwarfInt.h" + +// Set this to 1 for verbose logging +#define DEBUG_DWARF 0 + +namespace lul { + +using std::pair; +using std::string; + +ByteReader::ByteReader(enum Endianness endian) + : offset_reader_(NULL), + address_reader_(NULL), + endian_(endian), + address_size_(0), + offset_size_(0), + have_section_base_(), + have_text_base_(), + have_data_base_(), + have_function_base_() {} + +ByteReader::~ByteReader() {} + +void ByteReader::SetOffsetSize(uint8 size) { + offset_size_ = size; + MOZ_ASSERT(size == 4 || size == 8); + if (size == 4) { + this->offset_reader_ = &ByteReader::ReadFourBytes; + } else { + this->offset_reader_ = &ByteReader::ReadEightBytes; + } +} + +void ByteReader::SetAddressSize(uint8 size) { + address_size_ = size; + MOZ_ASSERT(size == 4 || size == 8); + if (size == 4) { + this->address_reader_ = &ByteReader::ReadFourBytes; + } else { + this->address_reader_ = &ByteReader::ReadEightBytes; + } +} + +uint64 ByteReader::ReadInitialLength(const char* start, size_t* len) { + const uint64 initial_length = ReadFourBytes(start); + start += 4; + + // In DWARF2/3, if the initial length is all 1 bits, then the offset + // size is 8 and we need to read the next 8 bytes for the real length. + if (initial_length == 0xffffffff) { + SetOffsetSize(8); + *len = 12; + return ReadOffset(start); + } else { + SetOffsetSize(4); + *len = 4; + } + return initial_length; +} + +bool ByteReader::ValidEncoding(DwarfPointerEncoding encoding) const { + if (encoding == DW_EH_PE_omit) return true; + if (encoding == DW_EH_PE_aligned) return true; + if ((encoding & 0x7) > DW_EH_PE_udata8) return false; + if ((encoding & 0x70) > DW_EH_PE_funcrel) return false; + return true; +} + +bool ByteReader::UsableEncoding(DwarfPointerEncoding encoding) const { + switch (encoding & 0x70) { + case DW_EH_PE_absptr: + return true; + case DW_EH_PE_pcrel: + return have_section_base_; + case DW_EH_PE_textrel: + return have_text_base_; + case DW_EH_PE_datarel: + return have_data_base_; + case DW_EH_PE_funcrel: + return have_function_base_; + default: + return false; + } +} + +uint64 ByteReader::ReadEncodedPointer(const char* buffer, + DwarfPointerEncoding encoding, + size_t* len) const { + // UsableEncoding doesn't approve of DW_EH_PE_omit, so we shouldn't + // see it here. + MOZ_ASSERT(encoding != DW_EH_PE_omit); + + // The Linux Standards Base 4.0 does not make this clear, but the + // GNU tools (gcc/unwind-pe.h; readelf/dwarf.c; gdb/dwarf2-frame.c) + // agree that aligned pointers are always absolute, machine-sized, + // machine-signed pointers. + if (encoding == DW_EH_PE_aligned) { + MOZ_ASSERT(have_section_base_); + + // We don't need to align BUFFER in *our* address space. Rather, we + // need to find the next position in our buffer that would be aligned + // when the .eh_frame section the buffer contains is loaded into the + // program's memory. So align assuming that buffer_base_ gets loaded at + // address section_base_, where section_base_ itself may or may not be + // aligned. + + // First, find the offset to START from the closest prior aligned + // address. + uint64 skew = section_base_ & (AddressSize() - 1); + // Now find the offset from that aligned address to buffer. + uint64 offset = skew + (buffer - buffer_base_); + // Round up to the next boundary. + uint64 aligned = (offset + AddressSize() - 1) & -AddressSize(); + // Convert back to a pointer. + const char* aligned_buffer = buffer_base_ + (aligned - skew); + // Finally, store the length and actually fetch the pointer. + *len = aligned_buffer - buffer + AddressSize(); + return ReadAddress(aligned_buffer); + } + + // Extract the value first, ignoring whether it's a pointer or an + // offset relative to some base. + uint64 offset; + switch (encoding & 0x0f) { + case DW_EH_PE_absptr: + // DW_EH_PE_absptr is weird, as it is used as a meaningful value for + // both the high and low nybble of encoding bytes. When it appears in + // the high nybble, it means that the pointer is absolute, not an + // offset from some base address. When it appears in the low nybble, + // as here, it means that the pointer is stored as a normal + // machine-sized and machine-signed address. A low nybble of + // DW_EH_PE_absptr does not imply that the pointer is absolute; it is + // correct for us to treat the value as an offset from a base address + // if the upper nybble is not DW_EH_PE_absptr. + offset = ReadAddress(buffer); + *len = AddressSize(); + break; + + case DW_EH_PE_uleb128: + offset = ReadUnsignedLEB128(buffer, len); + break; + + case DW_EH_PE_udata2: + offset = ReadTwoBytes(buffer); + *len = 2; + break; + + case DW_EH_PE_udata4: + offset = ReadFourBytes(buffer); + *len = 4; + break; + + case DW_EH_PE_udata8: + offset = ReadEightBytes(buffer); + *len = 8; + break; + + case DW_EH_PE_sleb128: + offset = ReadSignedLEB128(buffer, len); + break; + + case DW_EH_PE_sdata2: + offset = ReadTwoBytes(buffer); + // Sign-extend from 16 bits. + offset = (offset ^ 0x8000) - 0x8000; + *len = 2; + break; + + case DW_EH_PE_sdata4: + offset = ReadFourBytes(buffer); + // Sign-extend from 32 bits. + offset = (offset ^ 0x80000000ULL) - 0x80000000ULL; + *len = 4; + break; + + case DW_EH_PE_sdata8: + // No need to sign-extend; this is the full width of our type. + offset = ReadEightBytes(buffer); + *len = 8; + break; + + default: + abort(); + } + + // Find the appropriate base address. + uint64 base; + switch (encoding & 0x70) { + case DW_EH_PE_absptr: + base = 0; + break; + + case DW_EH_PE_pcrel: + MOZ_ASSERT(have_section_base_); + base = section_base_ + (buffer - buffer_base_); + break; + + case DW_EH_PE_textrel: + MOZ_ASSERT(have_text_base_); + base = text_base_; + break; + + case DW_EH_PE_datarel: + MOZ_ASSERT(have_data_base_); + base = data_base_; + break; + + case DW_EH_PE_funcrel: + MOZ_ASSERT(have_function_base_); + base = function_base_; + break; + + default: + abort(); + } + + uint64 pointer = base + offset; + + // Remove inappropriate upper bits. + if (AddressSize() == 4) + pointer = pointer & 0xffffffff; + else + MOZ_ASSERT(AddressSize() == sizeof(uint64)); + + return pointer; +} + +// A DWARF rule for recovering the address or value of a register, or +// computing the canonical frame address. This is an 8-way sum-of-products +// type. Excluding the INVALID variant, there is one subclass of this for +// each '*Rule' member function in CallFrameInfo::Handler. +// +// This could logically be nested within State, but then the qualified names +// get horrendous. + +class CallFrameInfo::Rule final { + public: + enum Tag { + INVALID, + Undefined, + SameValue, + Offset, + ValOffset, + Register, + Expression, + ValExpression + }; + + private: + // tag_ (below) indicates the form of the expression. There are 7 forms + // plus INVALID. All non-INVALID expressions denote a machine-word-sized + // value at unwind time. The description below assumes the presence of, at + // unwind time: + // + // * a function R, which takes a Dwarf register number and returns its value + // in the callee frame (the one we are unwinding out of). + // + // * a function EvalDwarfExpr, which evaluates a Dwarf expression. + // + // Register numbers are encoded using the target ABI's Dwarf + // register-numbering conventions. Except where otherwise noted, a register + // value may also be the special value CallFrameInfo::Handler::kCFARegister + // ("the CFA"). + // + // The expression forms are represented using tag_, word1_ and word2_. The + // forms and denoted values are as follows: + // + // * INVALID: not a valid expression. + // valid fields: (none) + // denotes: no value + // + // * Undefined: denotes no value. This is used for a register whose value + // cannot be recovered. + // valid fields: (none) + // denotes: no value + // + // * SameValue: the register's value is the same as in the callee. + // valid fields: (none) + // denotes: R(the register that this Rule is associated with, + // not stored here) + // + // * Offset: the register's value is in memory at word2_ bytes away from + // Dwarf register number word1_. word2_ is interpreted as a *signed* + // offset. + // valid fields: word1_=DwarfReg, word2=Offset + // denotes: *(R(word1_) + word2_) + // + // * ValOffset: same as Offset, without the dereference. + // valid fields: word1_=DwarfReg, word2=Offset + // denotes: R(word1_) + word2_ + // + // * Register: the register's value is in some other register, + // which may not be the CFA. + // valid fields: word1_=DwarfReg + // denotes: R(word1_) + // + // * Expression: the register's value is in memory at a location that can be + // computed from the Dwarf expression contained in the word2_ bytes + // starting at word1_. Note these locations are into the area of the .so + // temporarily mmaped info for debuginfo reading and have no validity once + // debuginfo reading has finished. + // valid fields: ExprStart=word1_, ExprLen=word2_ + // denotes: *(EvalDwarfExpr(word1_, word2_)) + // + // * ValExpression: same as Expression, without the dereference. + // valid fields: ExprStart=word1_, ExprLen=word2_ + // denotes: EvalDwarfExpr(word1_, word2_) + // + + // 3 words (or less) for representation. Unused word1_/word2_ fields must + // be set to zero. + Tag tag_; + uintptr_t word1_; + uintptr_t word2_; + + // To ensure that word1_ can hold a pointer to an expression string. + static_assert(sizeof(const char*) <= sizeof(word1_)); + // To ensure that word2_ can hold any string length or memory offset. + static_assert(sizeof(size_t) <= sizeof(word2_)); + + // This class denotes an 8-way sum-of-product type, and accessing invalid + // fields is meaningless. The accessors and constructors below enforce + // that. + bool isCanonical() const { + switch (tag_) { + case Tag::INVALID: + case Tag::Undefined: + case Tag::SameValue: + return word1_ == 0 && word2_ == 0; + case Tag::Offset: + case Tag::ValOffset: + return true; + case Tag::Register: + return word2_ == 0; + case Tag::Expression: + case Tag::ValExpression: + return true; + default: + MOZ_CRASH(); + } + } + + public: + Tag tag() const { return tag_; } + int dwreg() const { + switch (tag_) { + case Tag::Offset: + case Tag::ValOffset: + case Tag::Register: + return (int)word1_; + default: + MOZ_CRASH(); + } + } + intptr_t offset() const { + switch (tag_) { + case Tag::Offset: + case Tag::ValOffset: + return (intptr_t)word2_; + default: + MOZ_CRASH(); + } + } + ImageSlice expr() const { + switch (tag_) { + case Tag::Expression: + case Tag::ValExpression: + return ImageSlice((const char*)word1_, (size_t)word2_); + default: + MOZ_CRASH(); + } + } + + // Constructor-y stuff + Rule() { + tag_ = Tag::INVALID; + word1_ = 0; + word2_ = 0; + } + + static Rule mkINVALID() { + Rule r; // is initialised by Rule() + return r; + } + static Rule mkUndefinedRule() { + Rule r; + r.tag_ = Tag::Undefined; + r.word1_ = 0; + r.word2_ = 0; + return r; + } + static Rule mkSameValueRule() { + Rule r; + r.tag_ = Tag::SameValue; + r.word1_ = 0; + r.word2_ = 0; + return r; + } + static Rule mkOffsetRule(int dwreg, intptr_t offset) { + Rule r; + r.tag_ = Tag::Offset; + r.word1_ = (uintptr_t)dwreg; + r.word2_ = (uintptr_t)offset; + return r; + } + static Rule mkValOffsetRule(int dwreg, intptr_t offset) { + Rule r; + r.tag_ = Tag::ValOffset; + r.word1_ = (uintptr_t)dwreg; + r.word2_ = (uintptr_t)offset; + return r; + } + static Rule mkRegisterRule(int dwreg) { + Rule r; + r.tag_ = Tag::Register; + r.word1_ = (uintptr_t)dwreg; + r.word2_ = 0; + return r; + } + static Rule mkExpressionRule(ImageSlice expr) { + Rule r; + r.tag_ = Tag::Expression; + r.word1_ = (uintptr_t)expr.start_; + r.word2_ = (uintptr_t)expr.length_; + return r; + } + static Rule mkValExpressionRule(ImageSlice expr) { + Rule r; + r.tag_ = Tag::ValExpression; + r.word1_ = (uintptr_t)expr.start_; + r.word2_ = (uintptr_t)expr.length_; + return r; + } + + // Misc + inline bool isVALID() const { return tag_ != Tag::INVALID; } + + bool operator==(const Rule& rhs) const { + MOZ_ASSERT(isVALID() && rhs.isVALID()); + MOZ_ASSERT(isCanonical()); + MOZ_ASSERT(rhs.isCanonical()); + if (tag_ != rhs.tag_) { + return false; + } + switch (tag_) { + case Tag::INVALID: + MOZ_CRASH(); + case Tag::Undefined: + case Tag::SameValue: + return true; + case Tag::Offset: + case Tag::ValOffset: + return word1_ == rhs.word1_ && word2_ == rhs.word2_; + case Tag::Register: + return word1_ == rhs.word1_; + case Tag::Expression: + case Tag::ValExpression: + return expr() == rhs.expr(); + default: + MOZ_CRASH(); + } + } + + bool operator!=(const Rule& rhs) const { return !(*this == rhs); } + + // Tell HANDLER that, at ADDRESS in the program, REG can be + // recovered using this rule. If REG is kCFARegister, then this rule + // describes how to compute the canonical frame address. Return what the + // HANDLER member function returned. + bool Handle(Handler* handler, uint64 address, int reg) const { + MOZ_ASSERT(isVALID()); + MOZ_ASSERT(isCanonical()); + switch (tag_) { + case Tag::Undefined: + return handler->UndefinedRule(address, reg); + case Tag::SameValue: + return handler->SameValueRule(address, reg); + case Tag::Offset: + return handler->OffsetRule(address, reg, word1_, word2_); + case Tag::ValOffset: + return handler->ValOffsetRule(address, reg, word1_, word2_); + case Tag::Register: + return handler->RegisterRule(address, reg, word1_); + case Tag::Expression: + return handler->ExpressionRule( + address, reg, ImageSlice((const char*)word1_, (size_t)word2_)); + case Tag::ValExpression: + return handler->ValExpressionRule( + address, reg, ImageSlice((const char*)word1_, (size_t)word2_)); + default: + MOZ_CRASH(); + } + } + + void SetBaseRegister(unsigned reg) { + MOZ_ASSERT(isVALID()); + MOZ_ASSERT(isCanonical()); + switch (tag_) { + case Tag::ValOffset: + word1_ = reg; + break; + case Tag::Offset: + // We don't actually need SetBaseRegister or SetOffset here, since they + // are only ever applied to CFA rules, for DW_CFA_def_cfa_offset, and it + // doesn't make sense to use OffsetRule for computing the CFA: it + // computes the address at which a register is saved, not a value. + // (fallthrough) + case Tag::Undefined: + case Tag::SameValue: + case Tag::Register: + case Tag::Expression: + case Tag::ValExpression: + // Do nothing + break; + default: + MOZ_CRASH(); + } + } + + void SetOffset(long long offset) { + MOZ_ASSERT(isVALID()); + MOZ_ASSERT(isCanonical()); + switch (tag_) { + case Tag::ValOffset: + word2_ = offset; + break; + case Tag::Offset: + // Same comment as in SetBaseRegister applies + // (fallthrough) + case Tag::Undefined: + case Tag::SameValue: + case Tag::Register: + case Tag::Expression: + case Tag::ValExpression: + // Do nothing + break; + default: + MOZ_CRASH(); + } + } + + // For debugging only + string show() const { + char buf[100]; + string s = ""; + switch (tag_) { + case Tag::INVALID: + s = "INVALID"; + break; + case Tag::Undefined: + s = "Undefined"; + break; + case Tag::SameValue: + s = "SameValue"; + break; + case Tag::Offset: + s = "Offset{..}"; + break; + case Tag::ValOffset: + sprintf(buf, "ValOffset{reg=%d offs=%lld}", (int)word1_, + (long long int)word2_); + s = string(buf); + break; + case Tag::Register: + s = "Register{..}"; + break; + case Tag::Expression: + s = "Expression{..}"; + break; + case Tag::ValExpression: + s = "ValExpression{..}"; + break; + default: + MOZ_CRASH(); + } + return s; + } +}; + +// `RuleMapLowLevel` is a simple class that maps from `int` (register numbers) +// to `Rule`. This is implemented as a vector of `<int, Rule>` pairs, with a +// 12-element inline capacity. From a big-O perspective this is obviously a +// terrible way to implement an associative map. This workload is however +// quite special in that the maximum number of elements is normally 7 (on +// x86_64-linux), and so this implementation is much faster than one based on +// std::map with its attendant R-B-tree node allocation and balancing +// overheads. +// +// An iterator that enumerates the mapping in increasing order of the `int` +// keys is provided. This ordered iteration facility is required by +// CallFrameInfo::RuleMap::HandleTransitionTo, which needs to iterate through +// two such maps simultaneously and in-order so as to compare them. + +// All `Rule`s in the map must satisfy `isVALID()`. That conveniently means +// that `Rule::mkINVALID()` can be used to indicate "not found` in `get()`. + +class CallFrameInfo::RuleMapLowLevel { + using Entry = pair<int, Rule>; + + // The inline capacity of 12 is carefully chosen. It would be wise to make + // careful measurements of time, instruction count, allocation count and + // allocated bytes before changing it. For x86_64-linux, a value of 8 is + // marginally better; using 12 increases the total heap bytes allocated by + // around 20%. For arm64-linux, a value of 24 is better; using 12 increases + // the total blocks allocated by around 20%. But it's a not bad tradeoff + // for both targets, and in any case is vastly superior to the previous + // scheme of using `std::map`. + mozilla::Vector<Entry, 12> entries_; + + public: + void clear() { entries_.clear(); } + + RuleMapLowLevel() { clear(); } + + RuleMapLowLevel& operator=(const RuleMapLowLevel& rhs) { + entries_.clear(); + for (size_t i = 0; i < rhs.entries_.length(); i++) { + bool ok = entries_.append(rhs.entries_[i]); + MOZ_RELEASE_ASSERT(ok); + } + return *this; + } + + void set(int reg, Rule rule) { + MOZ_ASSERT(rule.isVALID()); + // Find the place where it should go, if any + size_t i = 0; + size_t nEnt = entries_.length(); + while (i < nEnt && entries_[i].first < reg) { + i++; + } + if (i == nEnt) { + // No entry exists, and all the existing ones are for lower register + // numbers. So just add it at the end. + bool ok = entries_.append(Entry(reg, rule)); + MOZ_RELEASE_ASSERT(ok); + } else { + // It needs to live at location `i`, and .. + MOZ_ASSERT(i < nEnt); + if (entries_[i].first == reg) { + // .. there's already an old entry, so just update it. + entries_[i].second = rule; + } else { + // .. there's no previous entry, so shift `i` and all those following + // it one place to the right, and put the new entry at `i`. Doing it + // manually is measurably cheaper than using `Vector::insert`. + MOZ_ASSERT(entries_[i].first > reg); + bool ok = entries_.append(Entry(999999, Rule::mkINVALID())); + MOZ_RELEASE_ASSERT(ok); + for (size_t j = nEnt; j >= i + 1; j--) { + entries_[j] = entries_[j - 1]; + } + entries_[i] = Entry(reg, rule); + } + } + // Check in-order-ness and validity. + for (size_t i = 0; i < entries_.length(); i++) { + MOZ_ASSERT(entries_[i].second.isVALID()); + MOZ_ASSERT_IF(i > 0, entries_[i - 1].first < entries_[i].first); + } + MOZ_ASSERT(get(reg).isVALID()); + } + + // Find the entry for `reg`, or return `Rule::mkINVALID()` if not found. + Rule get(int reg) const { + size_t nEnt = entries_.length(); + // "early exit" in the case where `entries_[i].first > reg` was tested on + // x86_64 and found to be slightly slower than just testing all entries, + // presumably because the reduced amount of searching was not offset by + // the cost of an extra test per iteration. + for (size_t i = 0; i < nEnt; i++) { + if (entries_[i].first == reg) { + CallFrameInfo::Rule ret = entries_[i].second; + MOZ_ASSERT(ret.isVALID()); + return ret; + } + } + return CallFrameInfo::Rule::mkINVALID(); + } + + // A very simple in-order iteration facility. + class Iter { + const RuleMapLowLevel* rmll_; + size_t nextIx_; + + public: + explicit Iter(const RuleMapLowLevel* rmll) : rmll_(rmll), nextIx_(0) {} + bool avail() const { return nextIx_ < rmll_->entries_.length(); } + bool finished() const { return !avail(); } + // Move the iterator to the next entry. + void step() { + MOZ_RELEASE_ASSERT(nextIx_ < rmll_->entries_.length()); + nextIx_++; + } + // Get the value at the current iteration point, but don't advance to the + // next entry. + pair<int, Rule> peek() { + MOZ_RELEASE_ASSERT(nextIx_ < rmll_->entries_.length()); + return rmll_->entries_[nextIx_]; + } + }; +}; + +// A map from register numbers to rules. This is a wrapper around +// `RuleMapLowLevel`, with added logic for dealing with the "special" CFA +// rule, and with `HandleTransitionTo`, which effectively computes the +// difference between two `RuleMaps`. + +class CallFrameInfo::RuleMap { + public: + RuleMap() : cfa_rule_(Rule::mkINVALID()) {} + RuleMap(const RuleMap& rhs) : cfa_rule_(Rule::mkINVALID()) { *this = rhs; } + ~RuleMap() { Clear(); } + + RuleMap& operator=(const RuleMap& rhs); + + // Set the rule for computing the CFA to RULE. + void SetCFARule(Rule rule) { cfa_rule_ = rule; } + + // Return the current CFA rule. Be careful not to modify it -- it's returned + // by value. If you want to modify the CFA rule, use CFARuleRef() instead. + // We use these two for DW_CFA_def_cfa_offset and DW_CFA_def_cfa_register, + // and for detecting references to the CFA before a rule for it has been + // established. + Rule CFARule() const { return cfa_rule_; } + Rule* CFARuleRef() { return &cfa_rule_; } + + // Return the rule for REG, or the INVALID rule if there is none. + Rule RegisterRule(int reg) const; + + // Set the rule for computing REG to RULE. + void SetRegisterRule(int reg, Rule rule); + + // Make all the appropriate calls to HANDLER as if we were changing from + // this RuleMap to NEW_RULES at ADDRESS. We use this to implement + // DW_CFA_restore_state, where lots of rules can change simultaneously. + // Return true if all handlers returned true; otherwise, return false. + bool HandleTransitionTo(Handler* handler, uint64 address, + const RuleMap& new_rules) const; + + private: + // Remove all register rules and clear cfa_rule_. + void Clear(); + + // The rule for computing the canonical frame address. + Rule cfa_rule_; + + // A map from register numbers to postfix expressions to recover + // their values. + RuleMapLowLevel registers_; +}; + +CallFrameInfo::RuleMap& CallFrameInfo::RuleMap::operator=(const RuleMap& rhs) { + Clear(); + if (rhs.cfa_rule_.isVALID()) cfa_rule_ = rhs.cfa_rule_; + registers_ = rhs.registers_; + return *this; +} + +CallFrameInfo::Rule CallFrameInfo::RuleMap::RegisterRule(int reg) const { + MOZ_ASSERT(reg != Handler::kCFARegister); + return registers_.get(reg); +} + +void CallFrameInfo::RuleMap::SetRegisterRule(int reg, Rule rule) { + MOZ_ASSERT(reg != Handler::kCFARegister); + MOZ_ASSERT(rule.isVALID()); + registers_.set(reg, rule); +} + +bool CallFrameInfo::RuleMap::HandleTransitionTo( + Handler* handler, uint64 address, const RuleMap& new_rules) const { + // Transition from cfa_rule_ to new_rules.cfa_rule_. + if (cfa_rule_.isVALID() && new_rules.cfa_rule_.isVALID()) { + if (cfa_rule_ != new_rules.cfa_rule_ && + !new_rules.cfa_rule_.Handle(handler, address, Handler::kCFARegister)) { + return false; + } + } else if (cfa_rule_.isVALID()) { + // this RuleMap has a CFA rule but new_rules doesn't. + // CallFrameInfo::Handler has no way to handle this --- and shouldn't; + // it's garbage input. The instruction interpreter should have + // detected this and warned, so take no action here. + } else if (new_rules.cfa_rule_.isVALID()) { + // This shouldn't be possible: NEW_RULES is some prior state, and + // there's no way to remove entries. + MOZ_ASSERT(0); + } else { + // Both CFA rules are empty. No action needed. + } + + // Traverse the two maps in order by register number, and report + // whatever differences we find. + RuleMapLowLevel::Iter old_it(®isters_); + RuleMapLowLevel::Iter new_it(&new_rules.registers_); + while (!old_it.finished() && !new_it.finished()) { + pair<int, Rule> old_pair = old_it.peek(); + pair<int, Rule> new_pair = new_it.peek(); + if (old_pair.first < new_pair.first) { + // This RuleMap has an entry for old.first, but NEW_RULES doesn't. + // + // This isn't really the right thing to do, but since CFI generally + // only mentions callee-saves registers, and GCC's convention for + // callee-saves registers is that they are unchanged, it's a good + // approximation. + if (!handler->SameValueRule(address, old_pair.first)) { + return false; + } + old_it.step(); + } else if (old_pair.first > new_pair.first) { + // NEW_RULES has an entry for new_pair.first, but this RuleMap + // doesn't. This shouldn't be possible: NEW_RULES is some prior + // state, and there's no way to remove entries. + MOZ_ASSERT(0); + } else { + // Both maps have an entry for this register. Report the new + // rule if it is different. + if (old_pair.second != new_pair.second && + !new_pair.second.Handle(handler, address, new_pair.first)) { + return false; + } + new_it.step(); + old_it.step(); + } + } + // Finish off entries from this RuleMap with no counterparts in new_rules. + while (!old_it.finished()) { + pair<int, Rule> old_pair = old_it.peek(); + if (!handler->SameValueRule(address, old_pair.first)) return false; + old_it.step(); + } + // Since we only make transitions from a rule set to some previously + // saved rule set, and we can only add rules to the map, NEW_RULES + // must have fewer rules than *this. + MOZ_ASSERT(new_it.finished()); + + return true; +} + +// Remove all register rules and clear cfa_rule_. +void CallFrameInfo::RuleMap::Clear() { + cfa_rule_ = Rule::mkINVALID(); + registers_.clear(); +} + +// The state of the call frame information interpreter as it processes +// instructions from a CIE and FDE. +class CallFrameInfo::State { + public: + // Create a call frame information interpreter state with the given + // reporter, reader, handler, and initial call frame info address. + State(ByteReader* reader, Handler* handler, Reporter* reporter, + uint64 address) + : reader_(reader), + handler_(handler), + reporter_(reporter), + address_(address), + entry_(NULL), + cursor_(NULL), + saved_rules_(NULL) {} + + ~State() { + if (saved_rules_) delete saved_rules_; + } + + // Interpret instructions from CIE, save the resulting rule set for + // DW_CFA_restore instructions, and return true. On error, report + // the problem to reporter_ and return false. + bool InterpretCIE(const CIE& cie); + + // Interpret instructions from FDE, and return true. On error, + // report the problem to reporter_ and return false. + bool InterpretFDE(const FDE& fde); + + private: + // The operands of a CFI instruction, for ParseOperands. + struct Operands { + unsigned register_number; // A register number. + uint64 offset; // An offset or address. + long signed_offset; // A signed offset. + ImageSlice expression; // A DWARF expression. + }; + + // Parse CFI instruction operands from STATE's instruction stream as + // described by FORMAT. On success, populate OPERANDS with the + // results, and return true. On failure, report the problem and + // return false. + // + // Each character of FORMAT should be one of the following: + // + // 'r' unsigned LEB128 register number (OPERANDS->register_number) + // 'o' unsigned LEB128 offset (OPERANDS->offset) + // 's' signed LEB128 offset (OPERANDS->signed_offset) + // 'a' machine-size address (OPERANDS->offset) + // (If the CIE has a 'z' augmentation string, 'a' uses the + // encoding specified by the 'R' argument.) + // '1' a one-byte offset (OPERANDS->offset) + // '2' a two-byte offset (OPERANDS->offset) + // '4' a four-byte offset (OPERANDS->offset) + // '8' an eight-byte offset (OPERANDS->offset) + // 'e' a DW_FORM_block holding a (OPERANDS->expression) + // DWARF expression + bool ParseOperands(const char* format, Operands* operands); + + // Interpret one CFI instruction from STATE's instruction stream, update + // STATE, report any rule changes to handler_, and return true. On + // failure, report the problem and return false. + MOZ_ALWAYS_INLINE bool DoInstruction(); + + // Repeatedly call `DoInstruction`, until either: + // * it returns `false`, which indicates some kind of failure, + // in which case return `false` from here too, or + // * we've run out of instructions (that is, `cursor_ >= entry_->end`), + // in which case return `true`. + // This is marked as never-inline because it is the only place that + // `DoInstruction` is called from, and we want to maximise the chances that + // `DoInstruction` is inlined into this routine. + MOZ_NEVER_INLINE bool DoInstructions(); + + // The following Do* member functions are subroutines of DoInstruction, + // factoring out the actual work of operations that have several + // different encodings. + + // Set the CFA rule to be the value of BASE_REGISTER plus OFFSET, and + // return true. On failure, report and return false. (Used for + // DW_CFA_def_cfa and DW_CFA_def_cfa_sf.) + bool DoDefCFA(unsigned base_register, long offset); + + // Change the offset of the CFA rule to OFFSET, and return true. On + // failure, report and return false. (Subroutine for + // DW_CFA_def_cfa_offset and DW_CFA_def_cfa_offset_sf.) + bool DoDefCFAOffset(long offset); + + // Specify that REG can be recovered using RULE, and return true. On + // failure, report and return false. + bool DoRule(unsigned reg, Rule rule); + + // Specify that REG can be found at OFFSET from the CFA, and return true. + // On failure, report and return false. (Subroutine for DW_CFA_offset, + // DW_CFA_offset_extended, and DW_CFA_offset_extended_sf.) + bool DoOffset(unsigned reg, long offset); + + // Specify that the caller's value for REG is the CFA plus OFFSET, + // and return true. On failure, report and return false. (Subroutine + // for DW_CFA_val_offset and DW_CFA_val_offset_sf.) + bool DoValOffset(unsigned reg, long offset); + + // Restore REG to the rule established in the CIE, and return true. On + // failure, report and return false. (Subroutine for DW_CFA_restore and + // DW_CFA_restore_extended.) + bool DoRestore(unsigned reg); + + // Return the section offset of the instruction at cursor. For use + // in error messages. + uint64 CursorOffset() { return entry_->offset + (cursor_ - entry_->start); } + + // Report that entry_ is incomplete, and return false. For brevity. + bool ReportIncomplete() { + reporter_->Incomplete(entry_->offset, entry_->kind); + return false; + } + + // For reading multi-byte values with the appropriate endianness. + ByteReader* reader_; + + // The handler to which we should report the data we find. + Handler* handler_; + + // For reporting problems in the info we're parsing. + Reporter* reporter_; + + // The code address to which the next instruction in the stream applies. + uint64 address_; + + // The entry whose instructions we are currently processing. This is + // first a CIE, and then an FDE. + const Entry* entry_; + + // The next instruction to process. + const char* cursor_; + + // The current set of rules. + RuleMap rules_; + + // The set of rules established by the CIE, used by DW_CFA_restore + // and DW_CFA_restore_extended. We set this after interpreting the + // CIE's instructions. + RuleMap cie_rules_; + + // A stack of saved states, for DW_CFA_remember_state and + // DW_CFA_restore_state. + std::stack<RuleMap>* saved_rules_; +}; + +bool CallFrameInfo::State::InterpretCIE(const CIE& cie) { + entry_ = &cie; + cursor_ = entry_->instructions; + if (!DoInstructions()) { + return false; + } + // Note the rules established by the CIE, for use by DW_CFA_restore + // and DW_CFA_restore_extended. + cie_rules_ = rules_; + return true; +} + +bool CallFrameInfo::State::InterpretFDE(const FDE& fde) { + entry_ = &fde; + cursor_ = entry_->instructions; + return DoInstructions(); +} + +bool CallFrameInfo::State::ParseOperands(const char* format, + Operands* operands) { + size_t len; + const char* operand; + + for (operand = format; *operand; operand++) { + size_t bytes_left = entry_->end - cursor_; + switch (*operand) { + case 'r': + operands->register_number = reader_->ReadUnsignedLEB128(cursor_, &len); + if (len > bytes_left) return ReportIncomplete(); + cursor_ += len; + break; + + case 'o': + operands->offset = reader_->ReadUnsignedLEB128(cursor_, &len); + if (len > bytes_left) return ReportIncomplete(); + cursor_ += len; + break; + + case 's': + operands->signed_offset = reader_->ReadSignedLEB128(cursor_, &len); + if (len > bytes_left) return ReportIncomplete(); + cursor_ += len; + break; + + case 'a': + operands->offset = reader_->ReadEncodedPointer( + cursor_, entry_->cie->pointer_encoding, &len); + if (len > bytes_left) return ReportIncomplete(); + cursor_ += len; + break; + + case '1': + if (1 > bytes_left) return ReportIncomplete(); + operands->offset = static_cast<unsigned char>(*cursor_++); + break; + + case '2': + if (2 > bytes_left) return ReportIncomplete(); + operands->offset = reader_->ReadTwoBytes(cursor_); + cursor_ += 2; + break; + + case '4': + if (4 > bytes_left) return ReportIncomplete(); + operands->offset = reader_->ReadFourBytes(cursor_); + cursor_ += 4; + break; + + case '8': + if (8 > bytes_left) return ReportIncomplete(); + operands->offset = reader_->ReadEightBytes(cursor_); + cursor_ += 8; + break; + + case 'e': { + size_t expression_length = reader_->ReadUnsignedLEB128(cursor_, &len); + if (len > bytes_left || expression_length > bytes_left - len) + return ReportIncomplete(); + cursor_ += len; + operands->expression = ImageSlice(cursor_, expression_length); + cursor_ += expression_length; + break; + } + + default: + MOZ_ASSERT(0); + } + } + + return true; +} + +MOZ_ALWAYS_INLINE +bool CallFrameInfo::State::DoInstruction() { + CIE* cie = entry_->cie; + Operands ops; + + // Our entry's kind should have been set by now. + MOZ_ASSERT(entry_->kind != kUnknown); + + // We shouldn't have been invoked unless there were more + // instructions to parse. + MOZ_ASSERT(cursor_ < entry_->end); + + unsigned opcode = *cursor_++; + if ((opcode & 0xc0) != 0) { + switch (opcode & 0xc0) { + // Advance the address. + case DW_CFA_advance_loc: { + size_t code_offset = opcode & 0x3f; + address_ += code_offset * cie->code_alignment_factor; + break; + } + + // Find a register at an offset from the CFA. + case DW_CFA_offset: + if (!ParseOperands("o", &ops) || + !DoOffset(opcode & 0x3f, ops.offset * cie->data_alignment_factor)) + return false; + break; + + // Restore the rule established for a register by the CIE. + case DW_CFA_restore: + if (!DoRestore(opcode & 0x3f)) return false; + break; + + // The 'if' above should have excluded this possibility. + default: + MOZ_ASSERT(0); + } + + // Return here, so the big switch below won't be indented. + return true; + } + + switch (opcode) { + // Set the address. + case DW_CFA_set_loc: + if (!ParseOperands("a", &ops)) return false; + address_ = ops.offset; + break; + + // Advance the address. + case DW_CFA_advance_loc1: + if (!ParseOperands("1", &ops)) return false; + address_ += ops.offset * cie->code_alignment_factor; + break; + + // Advance the address. + case DW_CFA_advance_loc2: + if (!ParseOperands("2", &ops)) return false; + address_ += ops.offset * cie->code_alignment_factor; + break; + + // Advance the address. + case DW_CFA_advance_loc4: + if (!ParseOperands("4", &ops)) return false; + address_ += ops.offset * cie->code_alignment_factor; + break; + + // Advance the address. + case DW_CFA_MIPS_advance_loc8: + if (!ParseOperands("8", &ops)) return false; + address_ += ops.offset * cie->code_alignment_factor; + break; + + // Compute the CFA by adding an offset to a register. + case DW_CFA_def_cfa: + if (!ParseOperands("ro", &ops) || + !DoDefCFA(ops.register_number, ops.offset)) + return false; + break; + + // Compute the CFA by adding an offset to a register. + case DW_CFA_def_cfa_sf: + if (!ParseOperands("rs", &ops) || + !DoDefCFA(ops.register_number, + ops.signed_offset * cie->data_alignment_factor)) + return false; + break; + + // Change the base register used to compute the CFA. + case DW_CFA_def_cfa_register: { + Rule* cfa_rule = rules_.CFARuleRef(); + if (!cfa_rule->isVALID()) { + reporter_->NoCFARule(entry_->offset, entry_->kind, CursorOffset()); + return false; + } + if (!ParseOperands("r", &ops)) return false; + cfa_rule->SetBaseRegister(ops.register_number); + if (!cfa_rule->Handle(handler_, address_, Handler::kCFARegister)) + return false; + break; + } + + // Change the offset used to compute the CFA. + case DW_CFA_def_cfa_offset: + if (!ParseOperands("o", &ops) || !DoDefCFAOffset(ops.offset)) + return false; + break; + + // Change the offset used to compute the CFA. + case DW_CFA_def_cfa_offset_sf: + if (!ParseOperands("s", &ops) || + !DoDefCFAOffset(ops.signed_offset * cie->data_alignment_factor)) + return false; + break; + + // Specify an expression whose value is the CFA. + case DW_CFA_def_cfa_expression: { + if (!ParseOperands("e", &ops)) return false; + Rule rule = Rule::mkValExpressionRule(ops.expression); + rules_.SetCFARule(rule); + if (!rule.Handle(handler_, address_, Handler::kCFARegister)) return false; + break; + } + + // The register's value cannot be recovered. + case DW_CFA_undefined: { + if (!ParseOperands("r", &ops) || + !DoRule(ops.register_number, Rule::mkUndefinedRule())) + return false; + break; + } + + // The register's value is unchanged from its value in the caller. + case DW_CFA_same_value: { + if (!ParseOperands("r", &ops) || + !DoRule(ops.register_number, Rule::mkSameValueRule())) + return false; + break; + } + + // Find a register at an offset from the CFA. + case DW_CFA_offset_extended: + if (!ParseOperands("ro", &ops) || + !DoOffset(ops.register_number, + ops.offset * cie->data_alignment_factor)) + return false; + break; + + // The register is saved at an offset from the CFA. + case DW_CFA_offset_extended_sf: + if (!ParseOperands("rs", &ops) || + !DoOffset(ops.register_number, + ops.signed_offset * cie->data_alignment_factor)) + return false; + break; + + // The register is saved at an offset from the CFA. + case DW_CFA_GNU_negative_offset_extended: + if (!ParseOperands("ro", &ops) || + !DoOffset(ops.register_number, + -ops.offset * cie->data_alignment_factor)) + return false; + break; + + // The register's value is the sum of the CFA plus an offset. + case DW_CFA_val_offset: + if (!ParseOperands("ro", &ops) || + !DoValOffset(ops.register_number, + ops.offset * cie->data_alignment_factor)) + return false; + break; + + // The register's value is the sum of the CFA plus an offset. + case DW_CFA_val_offset_sf: + if (!ParseOperands("rs", &ops) || + !DoValOffset(ops.register_number, + ops.signed_offset * cie->data_alignment_factor)) + return false; + break; + + // The register has been saved in another register. + case DW_CFA_register: { + if (!ParseOperands("ro", &ops) || + !DoRule(ops.register_number, Rule::mkRegisterRule(ops.offset))) + return false; + break; + } + + // An expression yields the address at which the register is saved. + case DW_CFA_expression: { + if (!ParseOperands("re", &ops) || + !DoRule(ops.register_number, Rule::mkExpressionRule(ops.expression))) + return false; + break; + } + + // An expression yields the caller's value for the register. + case DW_CFA_val_expression: { + if (!ParseOperands("re", &ops) || + !DoRule(ops.register_number, + Rule::mkValExpressionRule(ops.expression))) + return false; + break; + } + + // Restore the rule established for a register by the CIE. + case DW_CFA_restore_extended: + if (!ParseOperands("r", &ops) || !DoRestore(ops.register_number)) + return false; + break; + + // Save the current set of rules on a stack. + case DW_CFA_remember_state: + if (!saved_rules_) { + saved_rules_ = new std::stack<RuleMap>(); + } + saved_rules_->push(rules_); + break; + + // Pop the current set of rules off the stack. + case DW_CFA_restore_state: { + if (!saved_rules_ || saved_rules_->empty()) { + reporter_->EmptyStateStack(entry_->offset, entry_->kind, + CursorOffset()); + return false; + } + const RuleMap& new_rules = saved_rules_->top(); + if (rules_.CFARule().isVALID() && !new_rules.CFARule().isVALID()) { + reporter_->ClearingCFARule(entry_->offset, entry_->kind, + CursorOffset()); + return false; + } + rules_.HandleTransitionTo(handler_, address_, new_rules); + rules_ = new_rules; + saved_rules_->pop(); + break; + } + + // No operation. (Padding instruction.) + case DW_CFA_nop: + break; + + // A SPARC register window save: Registers 8 through 15 (%o0-%o7) + // are saved in registers 24 through 31 (%i0-%i7), and registers + // 16 through 31 (%l0-%l7 and %i0-%i7) are saved at CFA offsets + // (0-15 * the register size). The register numbers must be + // hard-coded. A GNU extension, and not a pretty one. + case DW_CFA_GNU_window_save: { + // Save %o0-%o7 in %i0-%i7. + for (int i = 8; i < 16; i++) + if (!DoRule(i, Rule::mkRegisterRule(i + 16))) return false; + // Save %l0-%l7 and %i0-%i7 at the CFA. + for (int i = 16; i < 32; i++) + // Assume that the byte reader's address size is the same as + // the architecture's register size. !@#%*^ hilarious. + if (!DoRule(i, Rule::mkOffsetRule(Handler::kCFARegister, + (i - 16) * reader_->AddressSize()))) + return false; + break; + } + + // I'm not sure what this is. GDB doesn't use it for unwinding. + case DW_CFA_GNU_args_size: + if (!ParseOperands("o", &ops)) return false; + break; + + // An opcode we don't recognize. + default: { + reporter_->BadInstruction(entry_->offset, entry_->kind, CursorOffset()); + return false; + } + } + + return true; +} + +// See declaration above for rationale re the no-inline directive. +MOZ_NEVER_INLINE +bool CallFrameInfo::State::DoInstructions() { + while (cursor_ < entry_->end) { + if (!DoInstruction()) { + return false; + } + } + return true; +} + +bool CallFrameInfo::State::DoDefCFA(unsigned base_register, long offset) { + Rule rule = Rule::mkValOffsetRule(base_register, offset); + rules_.SetCFARule(rule); + return rule.Handle(handler_, address_, Handler::kCFARegister); +} + +bool CallFrameInfo::State::DoDefCFAOffset(long offset) { + Rule* cfa_rule = rules_.CFARuleRef(); + if (!cfa_rule->isVALID()) { + reporter_->NoCFARule(entry_->offset, entry_->kind, CursorOffset()); + return false; + } + cfa_rule->SetOffset(offset); + return cfa_rule->Handle(handler_, address_, Handler::kCFARegister); +} + +bool CallFrameInfo::State::DoRule(unsigned reg, Rule rule) { + rules_.SetRegisterRule(reg, rule); + return rule.Handle(handler_, address_, reg); +} + +bool CallFrameInfo::State::DoOffset(unsigned reg, long offset) { + if (!rules_.CFARule().isVALID()) { + reporter_->NoCFARule(entry_->offset, entry_->kind, CursorOffset()); + return false; + } + Rule rule = Rule::mkOffsetRule(Handler::kCFARegister, offset); + return DoRule(reg, rule); +} + +bool CallFrameInfo::State::DoValOffset(unsigned reg, long offset) { + if (!rules_.CFARule().isVALID()) { + reporter_->NoCFARule(entry_->offset, entry_->kind, CursorOffset()); + return false; + } + return DoRule(reg, Rule::mkValOffsetRule(Handler::kCFARegister, offset)); +} + +bool CallFrameInfo::State::DoRestore(unsigned reg) { + // DW_CFA_restore and DW_CFA_restore_extended don't make sense in a CIE. + if (entry_->kind == kCIE) { + reporter_->RestoreInCIE(entry_->offset, CursorOffset()); + return false; + } + Rule rule = cie_rules_.RegisterRule(reg); + if (!rule.isVALID()) { + // This isn't really the right thing to do, but since CFI generally + // only mentions callee-saves registers, and GCC's convention for + // callee-saves registers is that they are unchanged, it's a good + // approximation. + rule = Rule::mkSameValueRule(); + } + return DoRule(reg, rule); +} + +bool CallFrameInfo::ReadEntryPrologue(const char* cursor, Entry* entry) { + const char* buffer_end = buffer_ + buffer_length_; + + // Initialize enough of ENTRY for use in error reporting. + entry->offset = cursor - buffer_; + entry->start = cursor; + entry->kind = kUnknown; + entry->end = NULL; + + // Read the initial length. This sets reader_'s offset size. + size_t length_size; + uint64 length = reader_->ReadInitialLength(cursor, &length_size); + if (length_size > size_t(buffer_end - cursor)) return ReportIncomplete(entry); + cursor += length_size; + + // In a .eh_frame section, a length of zero marks the end of the series + // of entries. + if (length == 0 && eh_frame_) { + entry->kind = kTerminator; + entry->end = cursor; + return true; + } + + // Validate the length. + if (length > size_t(buffer_end - cursor)) return ReportIncomplete(entry); + + // The length is the number of bytes after the initial length field; + // we have that position handy at this point, so compute the end + // now. (If we're parsing 64-bit-offset DWARF on a 32-bit machine, + // and the length didn't fit in a size_t, we would have rejected it + // above.) + entry->end = cursor + length; + + // Parse the next field: either the offset of a CIE or a CIE id. + size_t offset_size = reader_->OffsetSize(); + if (offset_size > size_t(entry->end - cursor)) return ReportIncomplete(entry); + entry->id = reader_->ReadOffset(cursor); + + // Don't advance cursor past id field yet; in .eh_frame data we need + // the id's position to compute the section offset of an FDE's CIE. + + // Now we can decide what kind of entry this is. + if (eh_frame_) { + // In .eh_frame data, an ID of zero marks the entry as a CIE, and + // anything else is an offset from the id field of the FDE to the start + // of the CIE. + if (entry->id == 0) { + entry->kind = kCIE; + } else { + entry->kind = kFDE; + // Turn the offset from the id into an offset from the buffer's start. + entry->id = (cursor - buffer_) - entry->id; + } + } else { + // In DWARF CFI data, an ID of ~0 (of the appropriate width, given the + // offset size for the entry) marks the entry as a CIE, and anything + // else is the offset of the CIE from the beginning of the section. + if (offset_size == 4) + entry->kind = (entry->id == 0xffffffff) ? kCIE : kFDE; + else { + MOZ_ASSERT(offset_size == 8); + entry->kind = (entry->id == 0xffffffffffffffffULL) ? kCIE : kFDE; + } + } + + // Now advance cursor past the id. + cursor += offset_size; + + // The fields specific to this kind of entry start here. + entry->fields = cursor; + + entry->cie = NULL; + + return true; +} + +bool CallFrameInfo::ReadCIEFields(CIE* cie) { + const char* cursor = cie->fields; + size_t len; + + MOZ_ASSERT(cie->kind == kCIE); + + // Prepare for early exit. + cie->version = 0; + cie->augmentation.clear(); + cie->code_alignment_factor = 0; + cie->data_alignment_factor = 0; + cie->return_address_register = 0; + cie->has_z_augmentation = false; + cie->pointer_encoding = DW_EH_PE_absptr; + cie->instructions = 0; + + // Parse the version number. + if (cie->end - cursor < 1) return ReportIncomplete(cie); + cie->version = reader_->ReadOneByte(cursor); + cursor++; + + // If we don't recognize the version, we can't parse any more fields of the + // CIE. For DWARF CFI, we handle versions 1 through 4 (there was never a + // version 2 of CFI data). For .eh_frame, we handle versions 1 and 4 as well; + // the difference between those versions seems to be the same as for + // .debug_frame. + if (cie->version < 1 || cie->version > 4) { + reporter_->UnrecognizedVersion(cie->offset, cie->version); + return false; + } + + const char* augmentation_start = cursor; + const void* augmentation_end = + memchr(augmentation_start, '\0', cie->end - augmentation_start); + if (!augmentation_end) return ReportIncomplete(cie); + cursor = static_cast<const char*>(augmentation_end); + cie->augmentation = string(augmentation_start, cursor - augmentation_start); + // Skip the terminating '\0'. + cursor++; + + // Is this CFI augmented? + if (!cie->augmentation.empty()) { + // Is it an augmentation we recognize? + if (cie->augmentation[0] == DW_Z_augmentation_start) { + // Linux C++ ABI 'z' augmentation, used for exception handling data. + cie->has_z_augmentation = true; + } else { + // Not an augmentation we recognize. Augmentations can have arbitrary + // effects on the form of rest of the content, so we have to give up. + reporter_->UnrecognizedAugmentation(cie->offset, cie->augmentation); + return false; + } + } + + if (cie->version >= 4) { + // Check that the address_size and segment_size fields are plausible. + if (cie->end - cursor < 2) { + return ReportIncomplete(cie); + } + uint8_t address_size = reader_->ReadOneByte(cursor); + cursor++; + if (address_size != sizeof(void*)) { + // This is not per-se invalid CFI. But we can reasonably expect to + // be running on a target of the same word size as the CFI is for, + // so we reject this case. + reporter_->InvalidDwarf4Artefact(cie->offset, "Invalid address_size"); + return false; + } + uint8_t segment_size = reader_->ReadOneByte(cursor); + cursor++; + if (segment_size != 0) { + // This is also not per-se invalid CFI, but we don't currently handle + // the case of non-zero |segment_size|. + reporter_->InvalidDwarf4Artefact(cie->offset, "Invalid segment_size"); + return false; + } + // We only continue parsing if |segment_size| is zero. If this routine + // is ever changed to allow non-zero |segment_size|, then + // ReadFDEFields() below will have to be changed to match, per comments + // there. + } + + // Parse the code alignment factor. + cie->code_alignment_factor = reader_->ReadUnsignedLEB128(cursor, &len); + if (size_t(cie->end - cursor) < len) return ReportIncomplete(cie); + cursor += len; + + // Parse the data alignment factor. + cie->data_alignment_factor = reader_->ReadSignedLEB128(cursor, &len); + if (size_t(cie->end - cursor) < len) return ReportIncomplete(cie); + cursor += len; + + // Parse the return address register. This is a ubyte in version 1, and + // a ULEB128 in version 3. + if (cie->version == 1) { + if (cursor >= cie->end) return ReportIncomplete(cie); + cie->return_address_register = uint8(*cursor++); + } else { + cie->return_address_register = reader_->ReadUnsignedLEB128(cursor, &len); + if (size_t(cie->end - cursor) < len) return ReportIncomplete(cie); + cursor += len; + } + + // If we have a 'z' augmentation string, find the augmentation data and + // use the augmentation string to parse it. + if (cie->has_z_augmentation) { + uint64_t data_size = reader_->ReadUnsignedLEB128(cursor, &len); + if (size_t(cie->end - cursor) < len + data_size) + return ReportIncomplete(cie); + cursor += len; + const char* data = cursor; + cursor += data_size; + const char* data_end = cursor; + + cie->has_z_lsda = false; + cie->has_z_personality = false; + cie->has_z_signal_frame = false; + + // Walk the augmentation string, and extract values from the + // augmentation data as the string directs. + for (size_t i = 1; i < cie->augmentation.size(); i++) { + switch (cie->augmentation[i]) { + case DW_Z_has_LSDA: + // The CIE's augmentation data holds the language-specific data + // area pointer's encoding, and the FDE's augmentation data holds + // the pointer itself. + cie->has_z_lsda = true; + // Fetch the LSDA encoding from the augmentation data. + if (data >= data_end) return ReportIncomplete(cie); + cie->lsda_encoding = DwarfPointerEncoding(*data++); + if (!reader_->ValidEncoding(cie->lsda_encoding)) { + reporter_->InvalidPointerEncoding(cie->offset, cie->lsda_encoding); + return false; + } + // Don't check if the encoding is usable here --- we haven't + // read the FDE's fields yet, so we're not prepared for + // DW_EH_PE_funcrel, although that's a fine encoding for the + // LSDA to use, since it appears in the FDE. + break; + + case DW_Z_has_personality_routine: + // The CIE's augmentation data holds the personality routine + // pointer's encoding, followed by the pointer itself. + cie->has_z_personality = true; + // Fetch the personality routine pointer's encoding from the + // augmentation data. + if (data >= data_end) return ReportIncomplete(cie); + cie->personality_encoding = DwarfPointerEncoding(*data++); + if (!reader_->ValidEncoding(cie->personality_encoding)) { + reporter_->InvalidPointerEncoding(cie->offset, + cie->personality_encoding); + return false; + } + if (!reader_->UsableEncoding(cie->personality_encoding)) { + reporter_->UnusablePointerEncoding(cie->offset, + cie->personality_encoding); + return false; + } + // Fetch the personality routine's pointer itself from the data. + cie->personality_address = reader_->ReadEncodedPointer( + data, cie->personality_encoding, &len); + if (len > size_t(data_end - data)) return ReportIncomplete(cie); + data += len; + break; + + case DW_Z_has_FDE_address_encoding: + // The CIE's augmentation data holds the pointer encoding to use + // for addresses in the FDE. + if (data >= data_end) return ReportIncomplete(cie); + cie->pointer_encoding = DwarfPointerEncoding(*data++); + if (!reader_->ValidEncoding(cie->pointer_encoding)) { + reporter_->InvalidPointerEncoding(cie->offset, + cie->pointer_encoding); + return false; + } + if (!reader_->UsableEncoding(cie->pointer_encoding)) { + reporter_->UnusablePointerEncoding(cie->offset, + cie->pointer_encoding); + return false; + } + break; + + case DW_Z_is_signal_trampoline: + // Frames using this CIE are signal delivery frames. + cie->has_z_signal_frame = true; + break; + + default: + // An augmentation we don't recognize. + reporter_->UnrecognizedAugmentation(cie->offset, cie->augmentation); + return false; + } + } + } + + // The CIE's instructions start here. + cie->instructions = cursor; + + return true; +} + +bool CallFrameInfo::ReadFDEFields(FDE* fde) { + const char* cursor = fde->fields; + size_t size; + + // At this point, for Dwarf 4 and above, we are assuming that the + // associated CIE has its |segment_size| field equal to zero. This is + // checked for in ReadCIEFields() above. If ReadCIEFields() is ever + // changed to allow non-zero |segment_size| CIEs then we will have to read + // the segment_selector value at this point. + + fde->address = + reader_->ReadEncodedPointer(cursor, fde->cie->pointer_encoding, &size); + if (size > size_t(fde->end - cursor)) return ReportIncomplete(fde); + cursor += size; + reader_->SetFunctionBase(fde->address); + + // For the length, we strip off the upper nybble of the encoding used for + // the starting address. + DwarfPointerEncoding length_encoding = + DwarfPointerEncoding(fde->cie->pointer_encoding & 0x0f); + fde->size = reader_->ReadEncodedPointer(cursor, length_encoding, &size); + if (size > size_t(fde->end - cursor)) return ReportIncomplete(fde); + cursor += size; + + // If the CIE has a 'z' augmentation string, then augmentation data + // appears here. + if (fde->cie->has_z_augmentation) { + uint64_t data_size = reader_->ReadUnsignedLEB128(cursor, &size); + if (size_t(fde->end - cursor) < size + data_size) + return ReportIncomplete(fde); + cursor += size; + + // In the abstract, we should walk the augmentation string, and extract + // items from the FDE's augmentation data as we encounter augmentation + // string characters that specify their presence: the ordering of items + // in the augmentation string determines the arrangement of values in + // the augmentation data. + // + // In practice, there's only ever one value in FDE augmentation data + // that we support --- the LSDA pointer --- and we have to bail if we + // see any unrecognized augmentation string characters. So if there is + // anything here at all, we know what it is, and where it starts. + if (fde->cie->has_z_lsda) { + // Check whether the LSDA's pointer encoding is usable now: only once + // we've parsed the FDE's starting address do we call reader_-> + // SetFunctionBase, so that the DW_EH_PE_funcrel encoding becomes + // usable. + if (!reader_->UsableEncoding(fde->cie->lsda_encoding)) { + reporter_->UnusablePointerEncoding(fde->cie->offset, + fde->cie->lsda_encoding); + return false; + } + + fde->lsda_address = + reader_->ReadEncodedPointer(cursor, fde->cie->lsda_encoding, &size); + if (size > data_size) return ReportIncomplete(fde); + // Ideally, we would also complain here if there were unconsumed + // augmentation data. + } + + cursor += data_size; + } + + // The FDE's instructions start after those. + fde->instructions = cursor; + + return true; +} + +bool CallFrameInfo::Start() { + const char* buffer_end = buffer_ + buffer_length_; + const char* cursor; + bool all_ok = true; + const char* entry_end; + bool ok; + + // Traverse all the entries in buffer_, skipping CIEs and offering + // FDEs to the handler. + for (cursor = buffer_; cursor < buffer_end; + cursor = entry_end, all_ok = all_ok && ok) { + FDE fde; + + // Make it easy to skip this entry with 'continue': assume that + // things are not okay until we've checked all the data, and + // prepare the address of the next entry. + ok = false; + + // Read the entry's prologue. + if (!ReadEntryPrologue(cursor, &fde)) { + if (!fde.end) { + // If we couldn't even figure out this entry's extent, then we + // must stop processing entries altogether. + all_ok = false; + break; + } + entry_end = fde.end; + continue; + } + + // The next iteration picks up after this entry. + entry_end = fde.end; + + // Did we see an .eh_frame terminating mark? + if (fde.kind == kTerminator) { + // If there appears to be more data left in the section after the + // terminating mark, warn the user. But this is just a warning; + // we leave all_ok true. + if (fde.end < buffer_end) reporter_->EarlyEHTerminator(fde.offset); + break; + } + + // In this loop, we skip CIEs. We only parse them fully when we + // parse an FDE that refers to them. This limits our memory + // consumption (beyond the buffer itself) to that needed to + // process the largest single entry. + if (fde.kind != kFDE) { + ok = true; + continue; + } + + // Validate the CIE pointer. + if (fde.id > buffer_length_) { + reporter_->CIEPointerOutOfRange(fde.offset, fde.id); + continue; + } + + CIE cie; + + // Parse this FDE's CIE header. + if (!ReadEntryPrologue(buffer_ + fde.id, &cie)) continue; + // This had better be an actual CIE. + if (cie.kind != kCIE) { + reporter_->BadCIEId(fde.offset, fde.id); + continue; + } + if (!ReadCIEFields(&cie)) continue; + + // We now have the values that govern both the CIE and the FDE. + cie.cie = &cie; + fde.cie = &cie; + + // Parse the FDE's header. + if (!ReadFDEFields(&fde)) continue; + + // Call Entry to ask the consumer if they're interested. + if (!handler_->Entry(fde.offset, fde.address, fde.size, cie.version, + cie.augmentation, cie.return_address_register)) { + // The handler isn't interested in this entry. That's not an error. + ok = true; + continue; + } + + if (cie.has_z_augmentation) { + // Report the personality routine address, if we have one. + if (cie.has_z_personality) { + if (!handler_->PersonalityRoutine( + cie.personality_address, + IsIndirectEncoding(cie.personality_encoding))) + continue; + } + + // Report the language-specific data area address, if we have one. + if (cie.has_z_lsda) { + if (!handler_->LanguageSpecificDataArea( + fde.lsda_address, IsIndirectEncoding(cie.lsda_encoding))) + continue; + } + + // If this is a signal-handling frame, report that. + if (cie.has_z_signal_frame) { + if (!handler_->SignalHandler()) continue; + } + } + + // Interpret the CIE's instructions, and then the FDE's instructions. + State state(reader_, handler_, reporter_, fde.address); + ok = state.InterpretCIE(cie) && state.InterpretFDE(fde); + + // Tell the ByteReader that the function start address from the + // FDE header is no longer valid. + reader_->ClearFunctionBase(); + + // Report the end of the entry. + handler_->End(); + } + + return all_ok; +} + +const char* CallFrameInfo::KindName(EntryKind kind) { + if (kind == CallFrameInfo::kUnknown) + return "entry"; + else if (kind == CallFrameInfo::kCIE) + return "common information entry"; + else if (kind == CallFrameInfo::kFDE) + return "frame description entry"; + else { + MOZ_ASSERT(kind == CallFrameInfo::kTerminator); + return ".eh_frame sequence terminator"; + } +} + +bool CallFrameInfo::ReportIncomplete(Entry* entry) { + reporter_->Incomplete(entry->offset, entry->kind); + return false; +} + +void CallFrameInfo::Reporter::Incomplete(uint64 offset, + CallFrameInfo::EntryKind kind) { + char buf[300]; + SprintfLiteral(buf, "%s: CFI %s at offset 0x%llx in '%s': entry ends early\n", + filename_.c_str(), CallFrameInfo::KindName(kind), offset, + section_.c_str()); + log_(buf); +} + +void CallFrameInfo::Reporter::EarlyEHTerminator(uint64 offset) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI at offset 0x%llx in '%s': saw end-of-data marker" + " before end of section contents\n", + filename_.c_str(), offset, section_.c_str()); + log_(buf); +} + +void CallFrameInfo::Reporter::CIEPointerOutOfRange(uint64 offset, + uint64 cie_offset) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI frame description entry at offset 0x%llx in '%s':" + " CIE pointer is out of range: 0x%llx\n", + filename_.c_str(), offset, section_.c_str(), cie_offset); + log_(buf); +} + +void CallFrameInfo::Reporter::BadCIEId(uint64 offset, uint64 cie_offset) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI frame description entry at offset 0x%llx in '%s':" + " CIE pointer does not point to a CIE: 0x%llx\n", + filename_.c_str(), offset, section_.c_str(), cie_offset); + log_(buf); +} + +void CallFrameInfo::Reporter::UnrecognizedVersion(uint64 offset, int version) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI frame description entry at offset 0x%llx in '%s':" + " CIE specifies unrecognized version: %d\n", + filename_.c_str(), offset, section_.c_str(), version); + log_(buf); +} + +void CallFrameInfo::Reporter::UnrecognizedAugmentation(uint64 offset, + const string& aug) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI frame description entry at offset 0x%llx in '%s':" + " CIE specifies unrecognized augmentation: '%s'\n", + filename_.c_str(), offset, section_.c_str(), aug.c_str()); + log_(buf); +} + +void CallFrameInfo::Reporter::InvalidDwarf4Artefact(uint64 offset, + const char* what) { + char* what_safe = strndup(what, 100); + char buf[300]; + SprintfLiteral(buf, + "%s: CFI frame description entry at offset 0x%llx in '%s':" + " CIE specifies invalid Dwarf4 artefact: %s\n", + filename_.c_str(), offset, section_.c_str(), what_safe); + log_(buf); + free(what_safe); +} + +void CallFrameInfo::Reporter::InvalidPointerEncoding(uint64 offset, + uint8 encoding) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI common information entry at offset 0x%llx in '%s':" + " 'z' augmentation specifies invalid pointer encoding: " + "0x%02x\n", + filename_.c_str(), offset, section_.c_str(), encoding); + log_(buf); +} + +void CallFrameInfo::Reporter::UnusablePointerEncoding(uint64 offset, + uint8 encoding) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI common information entry at offset 0x%llx in '%s':" + " 'z' augmentation specifies a pointer encoding for which" + " we have no base address: 0x%02x\n", + filename_.c_str(), offset, section_.c_str(), encoding); + log_(buf); +} + +void CallFrameInfo::Reporter::RestoreInCIE(uint64 offset, uint64 insn_offset) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI common information entry at offset 0x%llx in '%s':" + " the DW_CFA_restore instruction at offset 0x%llx" + " cannot be used in a common information entry\n", + filename_.c_str(), offset, section_.c_str(), insn_offset); + log_(buf); +} + +void CallFrameInfo::Reporter::BadInstruction(uint64 offset, + CallFrameInfo::EntryKind kind, + uint64 insn_offset) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI %s at offset 0x%llx in section '%s':" + " the instruction at offset 0x%llx is unrecognized\n", + filename_.c_str(), CallFrameInfo::KindName(kind), offset, + section_.c_str(), insn_offset); + log_(buf); +} + +void CallFrameInfo::Reporter::NoCFARule(uint64 offset, + CallFrameInfo::EntryKind kind, + uint64 insn_offset) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI %s at offset 0x%llx in section '%s':" + " the instruction at offset 0x%llx assumes that a CFA rule " + "has been set, but none has been set\n", + filename_.c_str(), CallFrameInfo::KindName(kind), offset, + section_.c_str(), insn_offset); + log_(buf); +} + +void CallFrameInfo::Reporter::EmptyStateStack(uint64 offset, + CallFrameInfo::EntryKind kind, + uint64 insn_offset) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI %s at offset 0x%llx in section '%s':" + " the DW_CFA_restore_state instruction at offset 0x%llx" + " should pop a saved state from the stack, but the stack " + "is empty\n", + filename_.c_str(), CallFrameInfo::KindName(kind), offset, + section_.c_str(), insn_offset); + log_(buf); +} + +void CallFrameInfo::Reporter::ClearingCFARule(uint64 offset, + CallFrameInfo::EntryKind kind, + uint64 insn_offset) { + char buf[300]; + SprintfLiteral(buf, + "%s: CFI %s at offset 0x%llx in section '%s':" + " the DW_CFA_restore_state instruction at offset 0x%llx" + " would clear the CFA rule in effect\n", + filename_.c_str(), CallFrameInfo::KindName(kind), offset, + section_.c_str(), insn_offset); + log_(buf); +} + +unsigned int DwarfCFIToModule::RegisterNames::I386() { + /* + 8 "$eax", "$ecx", "$edx", "$ebx", "$esp", "$ebp", "$esi", "$edi", + 3 "$eip", "$eflags", "$unused1", + 8 "$st0", "$st1", "$st2", "$st3", "$st4", "$st5", "$st6", "$st7", + 2 "$unused2", "$unused3", + 8 "$xmm0", "$xmm1", "$xmm2", "$xmm3", "$xmm4", "$xmm5", "$xmm6", "$xmm7", + 8 "$mm0", "$mm1", "$mm2", "$mm3", "$mm4", "$mm5", "$mm6", "$mm7", + 3 "$fcw", "$fsw", "$mxcsr", + 8 "$es", "$cs", "$ss", "$ds", "$fs", "$gs", "$unused4", "$unused5", + 2 "$tr", "$ldtr" + */ + return 8 + 3 + 8 + 2 + 8 + 8 + 3 + 8 + 2; +} + +unsigned int DwarfCFIToModule::RegisterNames::X86_64() { + /* + 8 "$rax", "$rdx", "$rcx", "$rbx", "$rsi", "$rdi", "$rbp", "$rsp", + 8 "$r8", "$r9", "$r10", "$r11", "$r12", "$r13", "$r14", "$r15", + 1 "$rip", + 8 "$xmm0","$xmm1","$xmm2", "$xmm3", "$xmm4", "$xmm5", "$xmm6", "$xmm7", + 8 "$xmm8","$xmm9","$xmm10","$xmm11","$xmm12","$xmm13","$xmm14","$xmm15", + 8 "$st0", "$st1", "$st2", "$st3", "$st4", "$st5", "$st6", "$st7", + 8 "$mm0", "$mm1", "$mm2", "$mm3", "$mm4", "$mm5", "$mm6", "$mm7", + 1 "$rflags", + 8 "$es", "$cs", "$ss", "$ds", "$fs", "$gs", "$unused1", "$unused2", + 4 "$fs.base", "$gs.base", "$unused3", "$unused4", + 2 "$tr", "$ldtr", + 3 "$mxcsr", "$fcw", "$fsw" + */ + return 8 + 8 + 1 + 8 + 8 + 8 + 8 + 1 + 8 + 4 + 2 + 3; +} + +// Per ARM IHI 0040A, section 3.1 +unsigned int DwarfCFIToModule::RegisterNames::ARM() { + /* + 8 "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", + 8 "r8", "r9", "r10", "r11", "r12", "sp", "lr", "pc", + 8 "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", + 8 "fps", "cpsr", "", "", "", "", "", "", + 8 "", "", "", "", "", "", "", "", + 8 "", "", "", "", "", "", "", "", + 8 "", "", "", "", "", "", "", "", + 8 "", "", "", "", "", "", "", "", + 8 "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", + 8 "s8", "s9", "s10", "s11", "s12", "s13", "s14", "s15", + 8 "s16", "s17", "s18", "s19", "s20", "s21", "s22", "s23", + 8 "s24", "s25", "s26", "s27", "s28", "s29", "s30", "s31", + 8 "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7" + */ + return 13 * 8; +} + +// Per ARM IHI 0057A, section 3.1 +unsigned int DwarfCFIToModule::RegisterNames::ARM64() { + /* + 8 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", + 8 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", + 8 "x16" "x17", "x18", "x19", "x20", "x21", "x22", "x23", + 8 "x24", "x25", "x26", "x27", "x28", "x29", "x30","sp", + 8 "", "", "", "", "", "", "", "", + 8 "", "", "", "", "", "", "", "", + 8 "", "", "", "", "", "", "", "", + 8 "", "", "", "", "", "", "", "", + 8 "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + 8 "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + 8 "v16", "v17", "v18", "v19", "v20", "v21", "v22, "v23", + 8 "v24", "x25", "x26, "x27", "v28", "v29", "v30", "v31", + */ + return 12 * 8; +} + +unsigned int DwarfCFIToModule::RegisterNames::MIPS() { + /* + 8 "$zero", "$at", "$v0", "$v1", "$a0", "$a1", "$a2", "$a3", + 8 "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", + 8 "$s0", "$s1", "$s2", "$s3", "$s4", "$s5", "$s6", "$s7", + 8 "$t8", "$t9", "$k0", "$k1", "$gp", "$sp", "$fp", "$ra", + 9 "$lo", "$hi", "$pc", "$f0", "$f1", "$f2", "$f3", "$f4", "$f5", + 8 "$f6", "$f7", "$f8", "$f9", "$f10", "$f11", "$f12", "$f13", + 7 "$f14", "$f15", "$f16", "$f17", "$f18", "$f19", "$f20", + 7 "$f21", "$f22", "$f23", "$f24", "$f25", "$f26", "$f27", + 6 "$f28", "$f29", "$f30", "$f31", "$fcsr", "$fir" + */ + return 8 + 8 + 8 + 8 + 9 + 8 + 7 + 7 + 6; +} + +// See prototype for comments. +int32_t parseDwarfExpr(Summariser* summ, const ByteReader* reader, + ImageSlice expr, bool debug, bool pushCfaAtStart, + bool derefAtEnd) { + const char* cursor = expr.start_; + const char* end1 = cursor + expr.length_; + + char buf[100]; + if (debug) { + SprintfLiteral(buf, "LUL.DW << DwarfExpr, len is %d\n", + (int)(end1 - cursor)); + summ->Log(buf); + } + + // Add a marker for the start of this expression. In it, indicate + // whether or not the CFA should be pushed onto the stack prior to + // evaluation. + int32_t start_ix = + summ->AddPfxInstr(PfxInstr(PX_Start, pushCfaAtStart ? 1 : 0)); + MOZ_ASSERT(start_ix >= 0); + + while (cursor < end1) { + uint8 opc = reader->ReadOneByte(cursor); + cursor++; + + const char* nm = nullptr; + PfxExprOp pxop = PX_End; + + switch (opc) { + case DW_OP_lit0 ... DW_OP_lit31: { + int32_t simm32 = (int32_t)(opc - DW_OP_lit0); + if (debug) { + SprintfLiteral(buf, "LUL.DW DW_OP_lit%d\n", (int)simm32); + summ->Log(buf); + } + (void)summ->AddPfxInstr(PfxInstr(PX_SImm32, simm32)); + break; + } + + case DW_OP_breg0 ... DW_OP_breg31: { + size_t len; + int64_t n = reader->ReadSignedLEB128(cursor, &len); + cursor += len; + DW_REG_NUMBER reg = (DW_REG_NUMBER)(opc - DW_OP_breg0); + if (debug) { + SprintfLiteral(buf, "LUL.DW DW_OP_breg%d %lld\n", (int)reg, + (long long int)n); + summ->Log(buf); + } + // PfxInstr only allows a 32 bit signed offset. So we + // must fail if the immediate is out of range. + if (n < INT32_MIN || INT32_MAX < n) goto fail; + (void)summ->AddPfxInstr(PfxInstr(PX_DwReg, reg)); + (void)summ->AddPfxInstr(PfxInstr(PX_SImm32, (int32_t)n)); + (void)summ->AddPfxInstr(PfxInstr(PX_Add)); + break; + } + + case DW_OP_const4s: { + uint64_t u64 = reader->ReadFourBytes(cursor); + cursor += 4; + // u64 is guaranteed by |ReadFourBytes| to be in the + // range 0 .. FFFFFFFF inclusive. But to be safe: + uint32_t u32 = (uint32_t)(u64 & 0xFFFFFFFF); + int32_t s32 = (int32_t)u32; + if (debug) { + SprintfLiteral(buf, "LUL.DW DW_OP_const4s %d\n", (int)s32); + summ->Log(buf); + } + (void)summ->AddPfxInstr(PfxInstr(PX_SImm32, s32)); + break; + } + + case DW_OP_deref: + nm = "deref"; + pxop = PX_Deref; + goto no_operands; + case DW_OP_and: + nm = "and"; + pxop = PX_And; + goto no_operands; + case DW_OP_plus: + nm = "plus"; + pxop = PX_Add; + goto no_operands; + case DW_OP_minus: + nm = "minus"; + pxop = PX_Sub; + goto no_operands; + case DW_OP_shl: + nm = "shl"; + pxop = PX_Shl; + goto no_operands; + case DW_OP_ge: + nm = "ge"; + pxop = PX_CmpGES; + goto no_operands; + no_operands: + MOZ_ASSERT(nm && pxop != PX_End); + if (debug) { + SprintfLiteral(buf, "LUL.DW DW_OP_%s\n", nm); + summ->Log(buf); + } + (void)summ->AddPfxInstr(PfxInstr(pxop)); + break; + + default: + if (debug) { + SprintfLiteral(buf, "LUL.DW unknown opc %d\n", (int)opc); + summ->Log(buf); + } + goto fail; + + } // switch (opc) + + } // while (cursor < end1) + + MOZ_ASSERT(cursor >= end1); + + if (cursor > end1) { + // We overran the Dwarf expression. Give up. + goto fail; + } + + // For DW_CFA_expression, what the expression denotes is the address + // of where the previous value is located. The caller of this routine + // may therefore request one last dereference before the end marker is + // inserted. + if (derefAtEnd) { + (void)summ->AddPfxInstr(PfxInstr(PX_Deref)); + } + + // Insert an end marker, and declare success. + (void)summ->AddPfxInstr(PfxInstr(PX_End)); + if (debug) { + SprintfLiteral(buf, + "LUL.DW conversion of dwarf expression succeeded, " + "ix = %d\n", + (int)start_ix); + summ->Log(buf); + summ->Log("LUL.DW >>\n"); + } + return start_ix; + +fail: + if (debug) { + summ->Log("LUL.DW conversion of dwarf expression failed\n"); + summ->Log("LUL.DW >>\n"); + } + return -1; +} + +bool DwarfCFIToModule::Entry(size_t offset, uint64 address, uint64 length, + uint8 version, const string& augmentation, + unsigned return_address) { + if (DEBUG_DWARF) { + char buf[100]; + SprintfLiteral(buf, "LUL.DW DwarfCFIToModule::Entry 0x%llx,+%lld\n", + address, length); + summ_->Log(buf); + } + + summ_->Entry(address, length); + + // If dwarf2reader::CallFrameInfo can handle this version and + // augmentation, then we should be okay with that, so there's no + // need to check them here. + + // Get ready to collect entries. + return_address_ = return_address; + + // Breakpad STACK CFI records must provide a .ra rule, but DWARF CFI + // may not establish any rule for .ra if the return address column + // is an ordinary register, and that register holds the return + // address on entry to the function. So establish an initial .ra + // rule citing the return address register. + if (return_address_ < num_dw_regs_) { + summ_->Rule(address, return_address_, NODEREF, return_address, 0); + } + + return true; +} + +const UniqueString* DwarfCFIToModule::RegisterName(int i) { + if (i < 0) { + MOZ_ASSERT(i == kCFARegister); + return usu_->ToUniqueString(".cfa"); + } + unsigned reg = i; + if (reg == return_address_) return usu_->ToUniqueString(".ra"); + + char buf[30]; + SprintfLiteral(buf, "dwarf_reg_%u", reg); + return usu_->ToUniqueString(buf); +} + +bool DwarfCFIToModule::UndefinedRule(uint64 address, int reg) { + reporter_->UndefinedNotSupported(entry_offset_, RegisterName(reg)); + // Treat this as a non-fatal error. + return true; +} + +bool DwarfCFIToModule::SameValueRule(uint64 address, int reg) { + if (DEBUG_DWARF) { + char buf[100]; + SprintfLiteral(buf, "LUL.DW 0x%llx: old r%d = Same\n", address, reg); + summ_->Log(buf); + } + // reg + 0 + summ_->Rule(address, reg, NODEREF, reg, 0); + return true; +} + +bool DwarfCFIToModule::OffsetRule(uint64 address, int reg, int base_register, + long offset) { + if (DEBUG_DWARF) { + char buf[100]; + SprintfLiteral(buf, "LUL.DW 0x%llx: old r%d = *(r%d + %ld)\n", address, + reg, base_register, offset); + summ_->Log(buf); + } + // *(base_register + offset) + summ_->Rule(address, reg, DEREF, base_register, offset); + return true; +} + +bool DwarfCFIToModule::ValOffsetRule(uint64 address, int reg, int base_register, + long offset) { + if (DEBUG_DWARF) { + char buf[100]; + SprintfLiteral(buf, "LUL.DW 0x%llx: old r%d = r%d + %ld\n", address, reg, + base_register, offset); + summ_->Log(buf); + } + // base_register + offset + summ_->Rule(address, reg, NODEREF, base_register, offset); + return true; +} + +bool DwarfCFIToModule::RegisterRule(uint64 address, int reg, + int base_register) { + if (DEBUG_DWARF) { + char buf[100]; + SprintfLiteral(buf, "LUL.DW 0x%llx: old r%d = r%d\n", address, reg, + base_register); + summ_->Log(buf); + } + // base_register + 0 + summ_->Rule(address, reg, NODEREF, base_register, 0); + return true; +} + +bool DwarfCFIToModule::ExpressionRule(uint64 address, int reg, + const ImageSlice& expression) { + bool debug = !!DEBUG_DWARF; + int32_t start_ix = + parseDwarfExpr(summ_, reader_, expression, debug, true /*pushCfaAtStart*/, + true /*derefAtEnd*/); + if (start_ix >= 0) { + summ_->Rule(address, reg, PFXEXPR, 0, start_ix); + } else { + // Parsing of the Dwarf expression failed. Treat this as a + // non-fatal error, hence return |true| even on this path. + reporter_->ExpressionCouldNotBeSummarised(entry_offset_, RegisterName(reg)); + } + return true; +} + +bool DwarfCFIToModule::ValExpressionRule(uint64 address, int reg, + const ImageSlice& expression) { + bool debug = !!DEBUG_DWARF; + int32_t start_ix = + parseDwarfExpr(summ_, reader_, expression, debug, true /*pushCfaAtStart*/, + false /*!derefAtEnd*/); + if (start_ix >= 0) { + summ_->Rule(address, reg, PFXEXPR, 0, start_ix); + } else { + // Parsing of the Dwarf expression failed. Treat this as a + // non-fatal error, hence return |true| even on this path. + reporter_->ExpressionCouldNotBeSummarised(entry_offset_, RegisterName(reg)); + } + return true; +} + +bool DwarfCFIToModule::End() { + // module_->AddStackFrameEntry(entry_); + if (DEBUG_DWARF) { + summ_->Log("LUL.DW DwarfCFIToModule::End()\n"); + } + summ_->End(); + return true; +} + +void DwarfCFIToModule::Reporter::UndefinedNotSupported( + size_t offset, const UniqueString* reg) { + char buf[300]; + SprintfLiteral(buf, "DwarfCFIToModule::Reporter::UndefinedNotSupported()\n"); + log_(buf); + // BPLOG(INFO) << file_ << ", section '" << section_ + // << "': the call frame entry at offset 0x" + // << std::setbase(16) << offset << std::setbase(10) + // << " sets the rule for register '" << FromUniqueString(reg) + // << "' to 'undefined', but the Breakpad symbol file format cannot " + // << " express this"; +} + +// FIXME: move this somewhere sensible +static bool is_power_of_2(uint64_t n) { + int i, nSetBits = 0; + for (i = 0; i < 8 * (int)sizeof(n); i++) { + if ((n & ((uint64_t)1) << i) != 0) nSetBits++; + } + return nSetBits <= 1; +} + +void DwarfCFIToModule::Reporter::ExpressionCouldNotBeSummarised( + size_t offset, const UniqueString* reg) { + static uint64_t n_complaints = 0; // This isn't threadsafe + n_complaints++; + if (!is_power_of_2(n_complaints)) return; + char buf[300]; + SprintfLiteral(buf, + "DwarfCFIToModule::Reporter::" + "ExpressionCouldNotBeSummarised(shown %llu times)\n", + (unsigned long long int)n_complaints); + log_(buf); +} + +} // namespace lul diff --git a/tools/profiler/lul/LulDwarfExt.h b/tools/profiler/lul/LulDwarfExt.h new file mode 100644 index 0000000000..4ee6fe17a8 --- /dev/null +++ b/tools/profiler/lul/LulDwarfExt.h @@ -0,0 +1,1312 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ + +// Copyright 2006, 2010 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Original author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/dwarf/types.h +// src/common/dwarf/dwarf2enums.h +// src/common/dwarf/bytereader.h +// src/common/dwarf_cfi_to_module.h +// src/common/dwarf/dwarf2reader.h + +#ifndef LulDwarfExt_h +#define LulDwarfExt_h + +#include "LulDwarfSummariser.h" + +#include "mozilla/Assertions.h" + +#include <stdint.h> +#include <string> + +typedef signed char int8; +typedef short int16; +typedef int int32; +typedef long long int64; + +typedef unsigned char uint8; +typedef unsigned short uint16; +typedef unsigned int uint32; +typedef unsigned long long uint64; + +#ifdef __PTRDIFF_TYPE__ +typedef __PTRDIFF_TYPE__ intptr; +typedef unsigned __PTRDIFF_TYPE__ uintptr; +#else +# error "Can't find pointer-sized integral types." +#endif + +namespace lul { + +class UniqueString; + +// This represents a read-only slice of the "image" (the temporarily mmaped-in +// .so). It is used for representing byte ranges containing Dwarf expressions. +// Note that equality (operator==) is on slice contents, not slice locations. +struct ImageSlice { + const char* start_; + size_t length_; + ImageSlice() : start_(0), length_(0) {} + ImageSlice(const char* start, size_t length) + : start_(start), length_(length) {} + // Make one from a C string (for testing only). Note, the terminating zero + // is not included in the length. + explicit ImageSlice(const char* cstring) + : start_(cstring), length_(strlen(cstring)) {} + explicit ImageSlice(const std::string& str) + : start_(str.c_str()), length_(str.length()) {} + ImageSlice(const ImageSlice& other) + : start_(other.start_), length_(other.length_) {} + ImageSlice(ImageSlice& other) + : start_(other.start_), length_(other.length_) {} + bool operator==(const ImageSlice& other) const { + if (length_ != other.length_) { + return false; + } + // This relies on the fact that that memcmp returns zero whenever length_ + // is zero. + return memcmp(start_, other.start_, length_) == 0; + } +}; + +// Exception handling frame description pointer formats, as described +// by the Linux Standard Base Core Specification 4.0, section 11.5, +// DWARF Extensions. +enum DwarfPointerEncoding { + DW_EH_PE_absptr = 0x00, + DW_EH_PE_omit = 0xff, + DW_EH_PE_uleb128 = 0x01, + DW_EH_PE_udata2 = 0x02, + DW_EH_PE_udata4 = 0x03, + DW_EH_PE_udata8 = 0x04, + DW_EH_PE_sleb128 = 0x09, + DW_EH_PE_sdata2 = 0x0A, + DW_EH_PE_sdata4 = 0x0B, + DW_EH_PE_sdata8 = 0x0C, + DW_EH_PE_pcrel = 0x10, + DW_EH_PE_textrel = 0x20, + DW_EH_PE_datarel = 0x30, + DW_EH_PE_funcrel = 0x40, + DW_EH_PE_aligned = 0x50, + + // The GNU toolchain sources define this enum value as well, + // simply to help classify the lower nybble values into signed and + // unsigned groups. + DW_EH_PE_signed = 0x08, + + // This is not documented in LSB 4.0, but it is used in both the + // Linux and OS X toolchains. It can be added to any other + // encoding (except DW_EH_PE_aligned), and indicates that the + // encoded value represents the address at which the true address + // is stored, not the true address itself. + DW_EH_PE_indirect = 0x80 +}; + +// We can't use the obvious name of LITTLE_ENDIAN and BIG_ENDIAN +// because it conflicts with a macro +enum Endianness { ENDIANNESS_BIG, ENDIANNESS_LITTLE }; + +// A ByteReader knows how to read single- and multi-byte values of +// various endiannesses, sizes, and encodings, as used in DWARF +// debugging information and Linux C++ exception handling data. +class ByteReader { + public: + // Construct a ByteReader capable of reading one-, two-, four-, and + // eight-byte values according to ENDIANNESS, absolute machine-sized + // addresses, DWARF-style "initial length" values, signed and + // unsigned LEB128 numbers, and Linux C++ exception handling data's + // encoded pointers. + explicit ByteReader(enum Endianness endianness); + virtual ~ByteReader(); + + // Read a single byte from BUFFER and return it as an unsigned 8 bit + // number. + uint8 ReadOneByte(const char* buffer) const; + + // Read two bytes from BUFFER and return them as an unsigned 16 bit + // number, using this ByteReader's endianness. + uint16 ReadTwoBytes(const char* buffer) const; + + // Read four bytes from BUFFER and return them as an unsigned 32 bit + // number, using this ByteReader's endianness. This function returns + // a uint64 so that it is compatible with ReadAddress and + // ReadOffset. The number it returns will never be outside the range + // of an unsigned 32 bit integer. + uint64 ReadFourBytes(const char* buffer) const; + + // Read eight bytes from BUFFER and return them as an unsigned 64 + // bit number, using this ByteReader's endianness. + uint64 ReadEightBytes(const char* buffer) const; + + // Read an unsigned LEB128 (Little Endian Base 128) number from + // BUFFER and return it as an unsigned 64 bit integer. Set LEN to + // the number of bytes read. + // + // The unsigned LEB128 representation of an integer N is a variable + // number of bytes: + // + // - If N is between 0 and 0x7f, then its unsigned LEB128 + // representation is a single byte whose value is N. + // + // - Otherwise, its unsigned LEB128 representation is (N & 0x7f) | + // 0x80, followed by the unsigned LEB128 representation of N / + // 128, rounded towards negative infinity. + // + // In other words, we break VALUE into groups of seven bits, put + // them in little-endian order, and then write them as eight-bit + // bytes with the high bit on all but the last. + uint64 ReadUnsignedLEB128(const char* buffer, size_t* len) const; + + // Read a signed LEB128 number from BUFFER and return it as an + // signed 64 bit integer. Set LEN to the number of bytes read. + // + // The signed LEB128 representation of an integer N is a variable + // number of bytes: + // + // - If N is between -0x40 and 0x3f, then its signed LEB128 + // representation is a single byte whose value is N in two's + // complement. + // + // - Otherwise, its signed LEB128 representation is (N & 0x7f) | + // 0x80, followed by the signed LEB128 representation of N / 128, + // rounded towards negative infinity. + // + // In other words, we break VALUE into groups of seven bits, put + // them in little-endian order, and then write them as eight-bit + // bytes with the high bit on all but the last. + int64 ReadSignedLEB128(const char* buffer, size_t* len) const; + + // Indicate that addresses on this architecture are SIZE bytes long. SIZE + // must be either 4 or 8. (DWARF allows addresses to be any number of + // bytes in length from 1 to 255, but we only support 32- and 64-bit + // addresses at the moment.) You must call this before using the + // ReadAddress member function. + // + // For data in a .debug_info section, or something that .debug_info + // refers to like line number or macro data, the compilation unit + // header's address_size field indicates the address size to use. Call + // frame information doesn't indicate its address size (a shortcoming of + // the spec); you must supply the appropriate size based on the + // architecture of the target machine. + void SetAddressSize(uint8 size); + + // Return the current address size, in bytes. This is either 4, + // indicating 32-bit addresses, or 8, indicating 64-bit addresses. + uint8 AddressSize() const { return address_size_; } + + // Read an address from BUFFER and return it as an unsigned 64 bit + // integer, respecting this ByteReader's endianness and address size. You + // must call SetAddressSize before calling this function. + uint64 ReadAddress(const char* buffer) const; + + // DWARF actually defines two slightly different formats: 32-bit DWARF + // and 64-bit DWARF. This is *not* related to the size of registers or + // addresses on the target machine; it refers only to the size of section + // offsets and data lengths appearing in the DWARF data. One only needs + // 64-bit DWARF when the debugging data itself is larger than 4GiB. + // 32-bit DWARF can handle x86_64 or PPC64 code just fine, unless the + // debugging data itself is very large. + // + // DWARF information identifies itself as 32-bit or 64-bit DWARF: each + // compilation unit and call frame information entry begins with an + // "initial length" field, which, in addition to giving the length of the + // data, also indicates the size of section offsets and lengths appearing + // in that data. The ReadInitialLength member function, below, reads an + // initial length and sets the ByteReader's offset size as a side effect. + // Thus, in the normal process of reading DWARF data, the appropriate + // offset size is set automatically. So, you should only need to call + // SetOffsetSize if you are using the same ByteReader to jump from the + // midst of one block of DWARF data into another. + + // Read a DWARF "initial length" field from START, and return it as + // an unsigned 64 bit integer, respecting this ByteReader's + // endianness. Set *LEN to the length of the initial length in + // bytes, either four or twelve. As a side effect, set this + // ByteReader's offset size to either 4 (if we see a 32-bit DWARF + // initial length) or 8 (if we see a 64-bit DWARF initial length). + // + // A DWARF initial length is either: + // + // - a byte count stored as an unsigned 32-bit value less than + // 0xffffff00, indicating that the data whose length is being + // measured uses the 32-bit DWARF format, or + // + // - The 32-bit value 0xffffffff, followed by a 64-bit byte count, + // indicating that the data whose length is being measured uses + // the 64-bit DWARF format. + uint64 ReadInitialLength(const char* start, size_t* len); + + // Read an offset from BUFFER and return it as an unsigned 64 bit + // integer, respecting the ByteReader's endianness. In 32-bit DWARF, the + // offset is 4 bytes long; in 64-bit DWARF, the offset is eight bytes + // long. You must call ReadInitialLength or SetOffsetSize before calling + // this function; see the comments above for details. + uint64 ReadOffset(const char* buffer) const; + + // Return the current offset size, in bytes. + // A return value of 4 indicates that we are reading 32-bit DWARF. + // A return value of 8 indicates that we are reading 64-bit DWARF. + uint8 OffsetSize() const { return offset_size_; } + + // Indicate that section offsets and lengths are SIZE bytes long. SIZE + // must be either 4 (meaning 32-bit DWARF) or 8 (meaning 64-bit DWARF). + // Usually, you should not call this function yourself; instead, let a + // call to ReadInitialLength establish the data's offset size + // automatically. + void SetOffsetSize(uint8 size); + + // The Linux C++ ABI uses a variant of DWARF call frame information + // for exception handling. This data is included in the program's + // address space as the ".eh_frame" section, and intepreted at + // runtime to walk the stack, find exception handlers, and run + // cleanup code. The format is mostly the same as DWARF CFI, with + // some adjustments made to provide the additional + // exception-handling data, and to make the data easier to work with + // in memory --- for example, to allow it to be placed in read-only + // memory even when describing position-independent code. + // + // In particular, exception handling data can select a number of + // different encodings for pointers that appear in the data, as + // described by the DwarfPointerEncoding enum. There are actually + // four axes(!) to the encoding: + // + // - The pointer size: pointers can be 2, 4, or 8 bytes long, or use + // the DWARF LEB128 encoding. + // + // - The pointer's signedness: pointers can be signed or unsigned. + // + // - The pointer's base address: the data stored in the exception + // handling data can be the actual address (that is, an absolute + // pointer), or relative to one of a number of different base + // addreses --- including that of the encoded pointer itself, for + // a form of "pc-relative" addressing. + // + // - The pointer may be indirect: it may be the address where the + // true pointer is stored. (This is used to refer to things via + // global offset table entries, program linkage table entries, or + // other tricks used in position-independent code.) + // + // There are also two options that fall outside that matrix + // altogether: the pointer may be omitted, or it may have padding to + // align it on an appropriate address boundary. (That last option + // may seem like it should be just another axis, but it is not.) + + // Indicate that the exception handling data is loaded starting at + // SECTION_BASE, and that the start of its buffer in our own memory + // is BUFFER_BASE. This allows us to find the address that a given + // byte in our buffer would have when loaded into the program the + // data describes. We need this to resolve DW_EH_PE_pcrel pointers. + void SetCFIDataBase(uint64 section_base, const char* buffer_base); + + // Indicate that the base address of the program's ".text" section + // is TEXT_BASE. We need this to resolve DW_EH_PE_textrel pointers. + void SetTextBase(uint64 text_base); + + // Indicate that the base address for DW_EH_PE_datarel pointers is + // DATA_BASE. The proper value depends on the ABI; it is usually the + // address of the global offset table, held in a designated register in + // position-independent code. You will need to look at the startup code + // for the target system to be sure. I tried; my eyes bled. + void SetDataBase(uint64 data_base); + + // Indicate that the base address for the FDE we are processing is + // FUNCTION_BASE. This is the start address of DW_EH_PE_funcrel + // pointers. (This encoding does not seem to be used by the GNU + // toolchain.) + void SetFunctionBase(uint64 function_base); + + // Indicate that we are no longer processing any FDE, so any use of + // a DW_EH_PE_funcrel encoding is an error. + void ClearFunctionBase(); + + // Return true if ENCODING is a valid pointer encoding. + bool ValidEncoding(DwarfPointerEncoding encoding) const; + + // Return true if we have all the information we need to read a + // pointer that uses ENCODING. This checks that the appropriate + // SetFooBase function for ENCODING has been called. + bool UsableEncoding(DwarfPointerEncoding encoding) const; + + // Read an encoded pointer from BUFFER using ENCODING; return the + // absolute address it represents, and set *LEN to the pointer's + // length in bytes, including any padding for aligned pointers. + // + // This function calls 'abort' if ENCODING is invalid or refers to a + // base address this reader hasn't been given, so you should check + // with ValidEncoding and UsableEncoding first if you would rather + // die in a more helpful way. + uint64 ReadEncodedPointer(const char* buffer, DwarfPointerEncoding encoding, + size_t* len) const; + + private: + // Function pointer type for our address and offset readers. + typedef uint64 (ByteReader::*AddressReader)(const char*) const; + + // Read an offset from BUFFER and return it as an unsigned 64 bit + // integer. DWARF2/3 define offsets as either 4 or 8 bytes, + // generally depending on the amount of DWARF2/3 info present. + // This function pointer gets set by SetOffsetSize. + AddressReader offset_reader_; + + // Read an address from BUFFER and return it as an unsigned 64 bit + // integer. DWARF2/3 allow addresses to be any size from 0-255 + // bytes currently. Internally we support 4 and 8 byte addresses, + // and will CHECK on anything else. + // This function pointer gets set by SetAddressSize. + AddressReader address_reader_; + + Endianness endian_; + uint8 address_size_; + uint8 offset_size_; + + // Base addresses for Linux C++ exception handling data's encoded pointers. + bool have_section_base_, have_text_base_, have_data_base_; + bool have_function_base_; + uint64 section_base_; + uint64 text_base_, data_base_, function_base_; + const char* buffer_base_; +}; + +inline uint8 ByteReader::ReadOneByte(const char* buffer) const { + return buffer[0]; +} + +inline uint16 ByteReader::ReadTwoBytes(const char* signed_buffer) const { + const unsigned char* buffer = + reinterpret_cast<const unsigned char*>(signed_buffer); + const uint16 buffer0 = buffer[0]; + const uint16 buffer1 = buffer[1]; + if (endian_ == ENDIANNESS_LITTLE) { + return buffer0 | buffer1 << 8; + } else { + return buffer1 | buffer0 << 8; + } +} + +inline uint64 ByteReader::ReadFourBytes(const char* signed_buffer) const { + const unsigned char* buffer = + reinterpret_cast<const unsigned char*>(signed_buffer); + const uint32 buffer0 = buffer[0]; + const uint32 buffer1 = buffer[1]; + const uint32 buffer2 = buffer[2]; + const uint32 buffer3 = buffer[3]; + if (endian_ == ENDIANNESS_LITTLE) { + return buffer0 | buffer1 << 8 | buffer2 << 16 | buffer3 << 24; + } else { + return buffer3 | buffer2 << 8 | buffer1 << 16 | buffer0 << 24; + } +} + +inline uint64 ByteReader::ReadEightBytes(const char* signed_buffer) const { + const unsigned char* buffer = + reinterpret_cast<const unsigned char*>(signed_buffer); + const uint64 buffer0 = buffer[0]; + const uint64 buffer1 = buffer[1]; + const uint64 buffer2 = buffer[2]; + const uint64 buffer3 = buffer[3]; + const uint64 buffer4 = buffer[4]; + const uint64 buffer5 = buffer[5]; + const uint64 buffer6 = buffer[6]; + const uint64 buffer7 = buffer[7]; + if (endian_ == ENDIANNESS_LITTLE) { + return buffer0 | buffer1 << 8 | buffer2 << 16 | buffer3 << 24 | + buffer4 << 32 | buffer5 << 40 | buffer6 << 48 | buffer7 << 56; + } else { + return buffer7 | buffer6 << 8 | buffer5 << 16 | buffer4 << 24 | + buffer3 << 32 | buffer2 << 40 | buffer1 << 48 | buffer0 << 56; + } +} + +// Read an unsigned LEB128 number. Each byte contains 7 bits of +// information, plus one bit saying whether the number continues or +// not. + +inline uint64 ByteReader::ReadUnsignedLEB128(const char* buffer, + size_t* len) const { + uint64 result = 0; + size_t num_read = 0; + unsigned int shift = 0; + unsigned char byte; + + do { + byte = *buffer++; + num_read++; + + result |= (static_cast<uint64>(byte & 0x7f)) << shift; + + shift += 7; + + } while (byte & 0x80); + + *len = num_read; + + return result; +} + +// Read a signed LEB128 number. These are like regular LEB128 +// numbers, except the last byte may have a sign bit set. + +inline int64 ByteReader::ReadSignedLEB128(const char* buffer, + size_t* len) const { + int64 result = 0; + unsigned int shift = 0; + size_t num_read = 0; + unsigned char byte; + + do { + byte = *buffer++; + num_read++; + result |= (static_cast<uint64>(byte & 0x7f) << shift); + shift += 7; + } while (byte & 0x80); + + if ((shift < 8 * sizeof(result)) && (byte & 0x40)) + result |= -((static_cast<int64>(1)) << shift); + *len = num_read; + return result; +} + +inline uint64 ByteReader::ReadOffset(const char* buffer) const { + MOZ_ASSERT(this->offset_reader_); + return (this->*offset_reader_)(buffer); +} + +inline uint64 ByteReader::ReadAddress(const char* buffer) const { + MOZ_ASSERT(this->address_reader_); + return (this->*address_reader_)(buffer); +} + +inline void ByteReader::SetCFIDataBase(uint64 section_base, + const char* buffer_base) { + section_base_ = section_base; + buffer_base_ = buffer_base; + have_section_base_ = true; +} + +inline void ByteReader::SetTextBase(uint64 text_base) { + text_base_ = text_base; + have_text_base_ = true; +} + +inline void ByteReader::SetDataBase(uint64 data_base) { + data_base_ = data_base; + have_data_base_ = true; +} + +inline void ByteReader::SetFunctionBase(uint64 function_base) { + function_base_ = function_base; + have_function_base_ = true; +} + +inline void ByteReader::ClearFunctionBase() { have_function_base_ = false; } + +// (derived from) +// dwarf_cfi_to_module.h: Define the DwarfCFIToModule class, which +// accepts parsed DWARF call frame info and adds it to a Summariser object. + +// This class is a reader for DWARF's Call Frame Information. CFI +// describes how to unwind stack frames --- even for functions that do +// not follow fixed conventions for saving registers, whose frame size +// varies as they execute, etc. +// +// CFI describes, at each machine instruction, how to compute the +// stack frame's base address, how to find the return address, and +// where to find the saved values of the caller's registers (if the +// callee has stashed them somewhere to free up the registers for its +// own use). +// +// For example, suppose we have a function whose machine code looks +// like this (imagine an assembly language that looks like C, for a +// machine with 32-bit registers, and a stack that grows towards lower +// addresses): +// +// func: ; entry point; return address at sp +// func+0: sp = sp - 16 ; allocate space for stack frame +// func+1: sp[12] = r0 ; save r0 at sp+12 +// ... ; other code, not frame-related +// func+10: sp -= 4; *sp = x ; push some x on the stack +// ... ; other code, not frame-related +// func+20: r0 = sp[16] ; restore saved r0 +// func+21: sp += 20 ; pop whole stack frame +// func+22: pc = *sp; sp += 4 ; pop return address and jump to it +// +// DWARF CFI is (a very compressed representation of) a table with a +// row for each machine instruction address and a column for each +// register showing how to restore it, if possible. +// +// A special column named "CFA", for "Canonical Frame Address", tells how +// to compute the base address of the frame; registers' entries may +// refer to the CFA in describing where the registers are saved. +// +// Another special column, named "RA", represents the return address. +// +// For example, here is a complete (uncompressed) table describing the +// function above: +// +// insn cfa r0 r1 ... ra +// ======================================= +// func+0: sp cfa[0] +// func+1: sp+16 cfa[0] +// func+2: sp+16 cfa[-4] cfa[0] +// func+11: sp+20 cfa[-4] cfa[0] +// func+21: sp+20 cfa[0] +// func+22: sp cfa[0] +// +// Some things to note here: +// +// - Each row describes the state of affairs *before* executing the +// instruction at the given address. Thus, the row for func+0 +// describes the state before we allocate the stack frame. In the +// next row, the formula for computing the CFA has changed, +// reflecting that allocation. +// +// - The other entries are written in terms of the CFA; this allows +// them to remain unchanged as the stack pointer gets bumped around. +// For example, the rule for recovering the return address (the "ra" +// column) remains unchanged throughout the function, even as the +// stack pointer takes on three different offsets from the return +// address. +// +// - Although we haven't shown it, most calling conventions designate +// "callee-saves" and "caller-saves" registers. The callee must +// preserve the values of callee-saves registers; if it uses them, +// it must save their original values somewhere, and restore them +// before it returns. In contrast, the callee is free to trash +// caller-saves registers; if the callee uses these, it will +// probably not bother to save them anywhere, and the CFI will +// probably mark their values as "unrecoverable". +// +// (However, since the caller cannot assume the callee was going to +// save them, caller-saves registers are probably dead in the caller +// anyway, so compilers usually don't generate CFA for caller-saves +// registers.) +// +// - Exactly where the CFA points is a matter of convention that +// depends on the architecture and ABI in use. In the example, the +// CFA is the value the stack pointer had upon entry to the +// function, pointing at the saved return address. But on the x86, +// the call frame information generated by GCC follows the +// convention that the CFA is the address *after* the saved return +// address. +// +// But by definition, the CFA remains constant throughout the +// lifetime of the frame. This makes it a useful value for other +// columns to refer to. It is also gives debuggers a useful handle +// for identifying a frame. +// +// If you look at the table above, you'll notice that a given entry is +// often the same as the one immediately above it: most instructions +// change only one or two aspects of the stack frame, if they affect +// it at all. The DWARF format takes advantage of this fact, and +// reduces the size of the data by mentioning only the addresses and +// columns at which changes take place. So for the above, DWARF CFI +// data would only actually mention the following: +// +// insn cfa r0 r1 ... ra +// ======================================= +// func+0: sp cfa[0] +// func+1: sp+16 +// func+2: cfa[-4] +// func+11: sp+20 +// func+21: r0 +// func+22: sp +// +// In fact, this is the way the parser reports CFI to the consumer: as +// a series of statements of the form, "At address X, column Y changed +// to Z," and related conventions for describing the initial state. +// +// Naturally, it would be impractical to have to scan the entire +// program's CFI, noting changes as we go, just to recover the +// unwinding rules in effect at one particular instruction. To avoid +// this, CFI data is grouped into "entries", each of which covers a +// specified range of addresses and begins with a complete statement +// of the rules for all recoverable registers at that starting +// address. Each entry typically covers a single function. +// +// Thus, to compute the contents of a given row of the table --- that +// is, rules for recovering the CFA, RA, and registers at a given +// instruction --- the consumer should find the entry that covers that +// instruction's address, start with the initial state supplied at the +// beginning of the entry, and work forward until it has processed all +// the changes up to and including those for the present instruction. +// +// There are seven kinds of rules that can appear in an entry of the +// table: +// +// - "undefined": The given register is not preserved by the callee; +// its value cannot be recovered. +// +// - "same value": This register has the same value it did in the callee. +// +// - offset(N): The register is saved at offset N from the CFA. +// +// - val_offset(N): The value the register had in the caller is the +// CFA plus offset N. (This is usually only useful for describing +// the stack pointer.) +// +// - register(R): The register's value was saved in another register R. +// +// - expression(E): Evaluating the DWARF expression E using the +// current frame's registers' values yields the address at which the +// register was saved. +// +// - val_expression(E): Evaluating the DWARF expression E using the +// current frame's registers' values yields the value the register +// had in the caller. + +class CallFrameInfo { + public: + // The different kinds of entries one finds in CFI. Used internally, + // and for error reporting. + enum EntryKind { kUnknown, kCIE, kFDE, kTerminator }; + + // The handler class to which the parser hands the parsed call frame + // information. Defined below. + class Handler; + + // A reporter class, which CallFrameInfo uses to report errors + // encountered while parsing call frame information. Defined below. + class Reporter; + + // Create a DWARF CFI parser. BUFFER points to the contents of the + // .debug_frame section to parse; BUFFER_LENGTH is its length in bytes. + // REPORTER is an error reporter the parser should use to report + // problems. READER is a ByteReader instance that has the endianness and + // address size set properly. Report the data we find to HANDLER. + // + // This class can also parse Linux C++ exception handling data, as found + // in '.eh_frame' sections. This data is a variant of DWARF CFI that is + // placed in loadable segments so that it is present in the program's + // address space, and is interpreted by the C++ runtime to search the + // call stack for a handler interested in the exception being thrown, + // actually pop the frames, and find cleanup code to run. + // + // There are two differences between the call frame information described + // in the DWARF standard and the exception handling data Linux places in + // the .eh_frame section: + // + // - Exception handling data uses uses a different format for call frame + // information entry headers. The distinguished CIE id, the way FDEs + // refer to their CIEs, and the way the end of the series of entries is + // determined are all slightly different. + // + // If the constructor's EH_FRAME argument is true, then the + // CallFrameInfo parses the entry headers as Linux C++ exception + // handling data. If EH_FRAME is false or omitted, the CallFrameInfo + // parses standard DWARF call frame information. + // + // - Linux C++ exception handling data uses CIE augmentation strings + // beginning with 'z' to specify the presence of additional data after + // the CIE and FDE headers and special encodings used for addresses in + // frame description entries. + // + // CallFrameInfo can handle 'z' augmentations in either DWARF CFI or + // exception handling data if you have supplied READER with the base + // addresses needed to interpret the pointer encodings that 'z' + // augmentations can specify. See the ByteReader interface for details + // about the base addresses. See the CallFrameInfo::Handler interface + // for details about the additional information one might find in + // 'z'-augmented data. + // + // Thus: + // + // - If you are parsing standard DWARF CFI, as found in a .debug_frame + // section, you should pass false for the EH_FRAME argument, or omit + // it, and you need not worry about providing READER with the + // additional base addresses. + // + // - If you want to parse Linux C++ exception handling data from a + // .eh_frame section, you should pass EH_FRAME as true, and call + // READER's Set*Base member functions before calling our Start method. + // + // - If you want to parse DWARF CFI that uses the 'z' augmentations + // (although I don't think any toolchain ever emits such data), you + // could pass false for EH_FRAME, but call READER's Set*Base members. + // + // The extensions the Linux C++ ABI makes to DWARF for exception + // handling are described here, rather poorly: + // http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/dwarfext.html + // http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html + // + // The mechanics of C++ exception handling, personality routines, + // and language-specific data areas are described here, rather nicely: + // http://www.codesourcery.com/public/cxx-abi/abi-eh.html + + CallFrameInfo(const char* buffer, size_t buffer_length, ByteReader* reader, + Handler* handler, Reporter* reporter, bool eh_frame = false) + : buffer_(buffer), + buffer_length_(buffer_length), + reader_(reader), + handler_(handler), + reporter_(reporter), + eh_frame_(eh_frame) {} + + ~CallFrameInfo() {} + + // Parse the entries in BUFFER, reporting what we find to HANDLER. + // Return true if we reach the end of the section successfully, or + // false if we encounter an error. + bool Start(); + + // Return the textual name of KIND. For error reporting. + static const char* KindName(EntryKind kind); + + private: + struct CIE; + + // A CFI entry, either an FDE or a CIE. + struct Entry { + // The starting offset of the entry in the section, for error + // reporting. + size_t offset; + + // The start of this entry in the buffer. + const char* start; + + // Which kind of entry this is. + // + // We want to be able to use this for error reporting even while we're + // in the midst of parsing. Error reporting code may assume that kind, + // offset, and start fields are valid, although kind may be kUnknown. + EntryKind kind; + + // The end of this entry's common prologue (initial length and id), and + // the start of this entry's kind-specific fields. + const char* fields; + + // The start of this entry's instructions. + const char* instructions; + + // The address past the entry's last byte in the buffer. (Note that + // since offset points to the entry's initial length field, and the + // length field is the number of bytes after that field, this is not + // simply buffer_ + offset + length.) + const char* end; + + // For both DWARF CFI and .eh_frame sections, this is the CIE id in a + // CIE, and the offset of the associated CIE in an FDE. + uint64 id; + + // The CIE that applies to this entry, if we've parsed it. If this is a + // CIE, then this field points to this structure. + CIE* cie; + }; + + // A common information entry (CIE). + struct CIE : public Entry { + uint8 version; // CFI data version number + std::string augmentation; // vendor format extension markers + uint64 code_alignment_factor; // scale for code address adjustments + int data_alignment_factor; // scale for stack pointer adjustments + unsigned return_address_register; // which register holds the return addr + + // True if this CIE includes Linux C++ ABI 'z' augmentation data. + bool has_z_augmentation; + + // Parsed 'z' augmentation data. These are meaningful only if + // has_z_augmentation is true. + bool has_z_lsda; // The 'z' augmentation included 'L'. + bool has_z_personality; // The 'z' augmentation included 'P'. + bool has_z_signal_frame; // The 'z' augmentation included 'S'. + + // If has_z_lsda is true, this is the encoding to be used for language- + // specific data area pointers in FDEs. + DwarfPointerEncoding lsda_encoding; + + // If has_z_personality is true, this is the encoding used for the + // personality routine pointer in the augmentation data. + DwarfPointerEncoding personality_encoding; + + // If has_z_personality is true, this is the address of the personality + // routine --- or, if personality_encoding & DW_EH_PE_indirect, the + // address where the personality routine's address is stored. + uint64 personality_address; + + // This is the encoding used for addresses in the FDE header and + // in DW_CFA_set_loc instructions. This is always valid, whether + // or not we saw a 'z' augmentation string; its default value is + // DW_EH_PE_absptr, which is what normal DWARF CFI uses. + DwarfPointerEncoding pointer_encoding; + }; + + // A frame description entry (FDE). + struct FDE : public Entry { + uint64 address; // start address of described code + uint64 size; // size of described code, in bytes + + // If cie->has_z_lsda is true, then this is the language-specific data + // area's address --- or its address's address, if cie->lsda_encoding + // has the DW_EH_PE_indirect bit set. + uint64 lsda_address; + }; + + // Internal use. + class Rule; + class RuleMapLowLevel; + class RuleMap; + class State; + + // Parse the initial length and id of a CFI entry, either a CIE, an FDE, + // or a .eh_frame end-of-data mark. CURSOR points to the beginning of the + // data to parse. On success, populate ENTRY as appropriate, and return + // true. On failure, report the problem, and return false. Even if we + // return false, set ENTRY->end to the first byte after the entry if we + // were able to figure that out, or NULL if we weren't. + bool ReadEntryPrologue(const char* cursor, Entry* entry); + + // Parse the fields of a CIE after the entry prologue, including any 'z' + // augmentation data. Assume that the 'Entry' fields of CIE are + // populated; use CIE->fields and CIE->end as the start and limit for + // parsing. On success, populate the rest of *CIE, and return true; on + // failure, report the problem and return false. + bool ReadCIEFields(CIE* cie); + + // Parse the fields of an FDE after the entry prologue, including any 'z' + // augmentation data. Assume that the 'Entry' fields of *FDE are + // initialized; use FDE->fields and FDE->end as the start and limit for + // parsing. Assume that FDE->cie is fully initialized. On success, + // populate the rest of *FDE, and return true; on failure, report the + // problem and return false. + bool ReadFDEFields(FDE* fde); + + // Report that ENTRY is incomplete, and return false. This is just a + // trivial wrapper for invoking reporter_->Incomplete; it provides a + // little brevity. + bool ReportIncomplete(Entry* entry); + + // Return true if ENCODING has the DW_EH_PE_indirect bit set. + static bool IsIndirectEncoding(DwarfPointerEncoding encoding) { + return encoding & DW_EH_PE_indirect; + } + + // The contents of the DWARF .debug_info section we're parsing. + const char* buffer_; + size_t buffer_length_; + + // For reading multi-byte values with the appropriate endianness. + ByteReader* reader_; + + // The handler to which we should report the data we find. + Handler* handler_; + + // For reporting problems in the info we're parsing. + Reporter* reporter_; + + // True if we are processing .eh_frame-format data. + bool eh_frame_; +}; + +// The handler class for CallFrameInfo. The a CFI parser calls the +// member functions of a handler object to report the data it finds. +class CallFrameInfo::Handler { + public: + // The pseudo-register number for the canonical frame address. + enum { kCFARegister = DW_REG_CFA }; + + Handler() {} + virtual ~Handler() {} + + // The parser has found CFI for the machine code at ADDRESS, + // extending for LENGTH bytes. OFFSET is the offset of the frame + // description entry in the section, for use in error messages. + // VERSION is the version number of the CFI format. AUGMENTATION is + // a string describing any producer-specific extensions present in + // the data. RETURN_ADDRESS is the number of the register that holds + // the address to which the function should return. + // + // Entry should return true to process this CFI, or false to skip to + // the next entry. + // + // The parser invokes Entry for each Frame Description Entry (FDE) + // it finds. The parser doesn't report Common Information Entries + // to the handler explicitly; instead, if the handler elects to + // process a given FDE, the parser reiterates the appropriate CIE's + // contents at the beginning of the FDE's rules. + virtual bool Entry(size_t offset, uint64 address, uint64 length, + uint8 version, const std::string& augmentation, + unsigned return_address) = 0; + + // When the Entry function returns true, the parser calls these + // handler functions repeatedly to describe the rules for recovering + // registers at each instruction in the given range of machine code. + // Immediately after a call to Entry, the handler should assume that + // the rule for each callee-saves register is "unchanged" --- that + // is, that the register still has the value it had in the caller. + // + // If a *Rule function returns true, we continue processing this entry's + // instructions. If a *Rule function returns false, we stop evaluating + // instructions, and skip to the next entry. Either way, we call End + // before going on to the next entry. + // + // In all of these functions, if the REG parameter is kCFARegister, then + // the rule describes how to find the canonical frame address. + // kCFARegister may be passed as a BASE_REGISTER argument, meaning that + // the canonical frame address should be used as the base address for the + // computation. All other REG values will be positive. + + // At ADDRESS, register REG's value is not recoverable. + virtual bool UndefinedRule(uint64 address, int reg) = 0; + + // At ADDRESS, register REG's value is the same as that it had in + // the caller. + virtual bool SameValueRule(uint64 address, int reg) = 0; + + // At ADDRESS, register REG has been saved at offset OFFSET from + // BASE_REGISTER. + virtual bool OffsetRule(uint64 address, int reg, int base_register, + long offset) = 0; + + // At ADDRESS, the caller's value of register REG is the current + // value of BASE_REGISTER plus OFFSET. (This rule doesn't provide an + // address at which the register's value is saved.) + virtual bool ValOffsetRule(uint64 address, int reg, int base_register, + long offset) = 0; + + // At ADDRESS, register REG has been saved in BASE_REGISTER. This differs + // from ValOffsetRule(ADDRESS, REG, BASE_REGISTER, 0), in that + // BASE_REGISTER is the "home" for REG's saved value: if you want to + // assign to a variable whose home is REG in the calling frame, you + // should put the value in BASE_REGISTER. + virtual bool RegisterRule(uint64 address, int reg, int base_register) = 0; + + // At ADDRESS, the DWARF expression EXPRESSION yields the address at + // which REG was saved. + virtual bool ExpressionRule(uint64 address, int reg, + const ImageSlice& expression) = 0; + + // At ADDRESS, the DWARF expression EXPRESSION yields the caller's + // value for REG. (This rule doesn't provide an address at which the + // register's value is saved.) + virtual bool ValExpressionRule(uint64 address, int reg, + const ImageSlice& expression) = 0; + + // Indicate that the rules for the address range reported by the + // last call to Entry are complete. End should return true if + // everything is okay, or false if an error has occurred and parsing + // should stop. + virtual bool End() = 0; + + // Handler functions for Linux C++ exception handling data. These are + // only called if the data includes 'z' augmentation strings. + + // The Linux C++ ABI uses an extension of the DWARF CFI format to + // walk the stack to propagate exceptions from the throw to the + // appropriate catch, and do the appropriate cleanups along the way. + // CFI entries used for exception handling have two additional data + // associated with them: + // + // - The "language-specific data area" describes which exception + // types the function has 'catch' clauses for, and indicates how + // to go about re-entering the function at the appropriate catch + // clause. If the exception is not caught, it describes the + // destructors that must run before the frame is popped. + // + // - The "personality routine" is responsible for interpreting the + // language-specific data area's contents, and deciding whether + // the exception should continue to propagate down the stack, + // perhaps after doing some cleanup for this frame, or whether the + // exception will be caught here. + // + // In principle, the language-specific data area is opaque to + // everybody but the personality routine. In practice, these values + // may be useful or interesting to readers with extra context, and + // we have to at least skip them anyway, so we might as well report + // them to the handler. + + // This entry's exception handling personality routine's address is + // ADDRESS. If INDIRECT is true, then ADDRESS is the address at + // which the routine's address is stored. The default definition for + // this handler function simply returns true, allowing parsing of + // the entry to continue. + virtual bool PersonalityRoutine(uint64 address, bool indirect) { + return true; + } + + // This entry's language-specific data area (LSDA) is located at + // ADDRESS. If INDIRECT is true, then ADDRESS is the address at + // which the area's address is stored. The default definition for + // this handler function simply returns true, allowing parsing of + // the entry to continue. + virtual bool LanguageSpecificDataArea(uint64 address, bool indirect) { + return true; + } + + // This entry describes a signal trampoline --- this frame is the + // caller of a signal handler. The default definition for this + // handler function simply returns true, allowing parsing of the + // entry to continue. + // + // The best description of the rationale for and meaning of signal + // trampoline CFI entries seems to be in the GCC bug database: + // http://gcc.gnu.org/bugzilla/show_bug.cgi?id=26208 + virtual bool SignalHandler() { return true; } +}; + +// The CallFrameInfo class makes calls on an instance of this class to +// report errors or warn about problems in the data it is parsing. +// These messages are sent to the message sink |aLog| provided to the +// constructor. +class CallFrameInfo::Reporter { + public: + // Create an error reporter which attributes troubles to the section + // named SECTION in FILENAME. + // + // Normally SECTION would be .debug_frame, but the Mac puts CFI data + // in a Mach-O section named __debug_frame. If we support + // Linux-style exception handling data, we could be reading an + // .eh_frame section. + Reporter(void (*aLog)(const char*), const std::string& filename, + const std::string& section = ".debug_frame") + : log_(aLog), filename_(filename), section_(section) {} + virtual ~Reporter() {} + + // The CFI entry at OFFSET ends too early to be well-formed. KIND + // indicates what kind of entry it is; KIND can be kUnknown if we + // haven't parsed enough of the entry to tell yet. + virtual void Incomplete(uint64 offset, CallFrameInfo::EntryKind kind); + + // The .eh_frame data has a four-byte zero at OFFSET where the next + // entry's length would be; this is a terminator. However, the buffer + // length as given to the CallFrameInfo constructor says there should be + // more data. + virtual void EarlyEHTerminator(uint64 offset); + + // The FDE at OFFSET refers to the CIE at CIE_OFFSET, but the + // section is not that large. + virtual void CIEPointerOutOfRange(uint64 offset, uint64 cie_offset); + + // The FDE at OFFSET refers to the CIE at CIE_OFFSET, but the entry + // there is not a CIE. + virtual void BadCIEId(uint64 offset, uint64 cie_offset); + + // The FDE at OFFSET refers to a CIE with version number VERSION, + // which we don't recognize. We cannot parse DWARF CFI if it uses + // a version number we don't recognize. + virtual void UnrecognizedVersion(uint64 offset, int version); + + // The FDE at OFFSET refers to a CIE with augmentation AUGMENTATION, + // which we don't recognize. We cannot parse DWARF CFI if it uses + // augmentations we don't recognize. + virtual void UnrecognizedAugmentation(uint64 offset, + const std::string& augmentation); + + // The FDE at OFFSET contains an invalid or otherwise unusable Dwarf4 + // specific field (currently, only "address_size" or "segment_size"). + // Parsing DWARF CFI with unexpected values here seems dubious at best, + // so we stop. WHAT gives a little more information about what is wrong. + virtual void InvalidDwarf4Artefact(uint64 offset, const char* what); + + // The pointer encoding ENCODING, specified by the CIE at OFFSET, is not + // a valid encoding. + virtual void InvalidPointerEncoding(uint64 offset, uint8 encoding); + + // The pointer encoding ENCODING, specified by the CIE at OFFSET, depends + // on a base address which has not been supplied. + virtual void UnusablePointerEncoding(uint64 offset, uint8 encoding); + + // The CIE at OFFSET contains a DW_CFA_restore instruction at + // INSN_OFFSET, which may not appear in a CIE. + virtual void RestoreInCIE(uint64 offset, uint64 insn_offset); + + // The entry at OFFSET, of kind KIND, has an unrecognized + // instruction at INSN_OFFSET. + virtual void BadInstruction(uint64 offset, CallFrameInfo::EntryKind kind, + uint64 insn_offset); + + // The instruction at INSN_OFFSET in the entry at OFFSET, of kind + // KIND, establishes a rule that cites the CFA, but we have not + // established a CFA rule yet. + virtual void NoCFARule(uint64 offset, CallFrameInfo::EntryKind kind, + uint64 insn_offset); + + // The instruction at INSN_OFFSET in the entry at OFFSET, of kind + // KIND, is a DW_CFA_restore_state instruction, but the stack of + // saved states is empty. + virtual void EmptyStateStack(uint64 offset, CallFrameInfo::EntryKind kind, + uint64 insn_offset); + + // The DW_CFA_remember_state instruction at INSN_OFFSET in the entry + // at OFFSET, of kind KIND, would restore a state that has no CFA + // rule, whereas the current state does have a CFA rule. This is + // bogus input, which the CallFrameInfo::Handler interface doesn't + // (and shouldn't) have any way to report. + virtual void ClearingCFARule(uint64 offset, CallFrameInfo::EntryKind kind, + uint64 insn_offset); + + private: + // A logging sink function, as supplied by LUL's user. + void (*log_)(const char*); + + protected: + // The name of the file whose CFI we're reading. + std::string filename_; + + // The name of the CFI section in that file. + std::string section_; +}; + +using lul::CallFrameInfo; +using lul::Summariser; + +// A class that accepts parsed call frame information from the DWARF +// CFI parser and populates a google_breakpad::Module object with the +// contents. +class DwarfCFIToModule : public CallFrameInfo::Handler { + public: + // DwarfCFIToModule uses an instance of this class to report errors + // detected while converting DWARF CFI to Breakpad STACK CFI records. + class Reporter { + public: + // Create a reporter that writes messages to the message sink + // |aLog|. FILE is the name of the file we're processing, and + // SECTION is the name of the section within that file that we're + // looking at (.debug_frame, .eh_frame, etc.). + Reporter(void (*aLog)(const char*), const std::string& file, + const std::string& section) + : log_(aLog), file_(file), section_(section) {} + virtual ~Reporter() {} + + // The DWARF CFI entry at OFFSET says that REG is undefined, but the + // Breakpad symbol file format cannot express this. + virtual void UndefinedNotSupported(size_t offset, const UniqueString* reg); + + // The DWARF CFI entry at OFFSET says that REG uses a DWARF + // expression to find its value, but parseDwarfExpr could not + // convert it to a sequence of PfxInstrs. + virtual void ExpressionCouldNotBeSummarised(size_t offset, + const UniqueString* reg); + + private: + // A logging sink function, as supplied by LUL's user. + void (*log_)(const char*); + + protected: + std::string file_, section_; + }; + + // Register name tables. If TABLE is a vector returned by one of these + // functions, then TABLE[R] is the name of the register numbered R in + // DWARF call frame information. + class RegisterNames { + public: + // Intel's "x86" or IA-32. + static unsigned int I386(); + + // AMD x86_64, AMD64, Intel EM64T, or Intel 64 + static unsigned int X86_64(); + + // ARM. + static unsigned int ARM(); + + // AARCH64. + static unsigned int ARM64(); + + // MIPS. + static unsigned int MIPS(); + }; + + // Create a handler for the dwarf2reader::CallFrameInfo parser that + // records the stack unwinding information it receives in SUMM. + // + // Use REGISTER_NAMES[I] as the name of register number I; *this + // keeps a reference to the vector, so the vector should remain + // alive for as long as the DwarfCFIToModule does. + // + // Use REPORTER for reporting problems encountered in the conversion + // process. + DwarfCFIToModule(const unsigned int num_dw_regs, Reporter* reporter, + ByteReader* reader, + /*MOD*/ UniqueStringUniverse* usu, + /*OUT*/ Summariser* summ) + : summ_(summ), + usu_(usu), + num_dw_regs_(num_dw_regs), + reporter_(reporter), + reader_(reader), + return_address_(-1) {} + virtual ~DwarfCFIToModule() {} + + virtual bool Entry(size_t offset, uint64 address, uint64 length, + uint8 version, const std::string& augmentation, + unsigned return_address) override; + virtual bool UndefinedRule(uint64 address, int reg) override; + virtual bool SameValueRule(uint64 address, int reg) override; + virtual bool OffsetRule(uint64 address, int reg, int base_register, + long offset) override; + virtual bool ValOffsetRule(uint64 address, int reg, int base_register, + long offset) override; + virtual bool RegisterRule(uint64 address, int reg, + int base_register) override; + virtual bool ExpressionRule(uint64 address, int reg, + const ImageSlice& expression) override; + virtual bool ValExpressionRule(uint64 address, int reg, + const ImageSlice& expression) override; + virtual bool End() override; + + private: + // Return the name to use for register I. + const UniqueString* RegisterName(int i); + + // The Summariser to which we should give entries + Summariser* summ_; + + // Universe for creating UniqueStrings in, should that be necessary. + UniqueStringUniverse* usu_; + + // The number of Dwarf-defined register names for this architecture. + const unsigned int num_dw_regs_; + + // The reporter to use to report problems. + Reporter* reporter_; + + // The ByteReader to use for parsing Dwarf expressions. + ByteReader* reader_; + + // The section offset of the current frame description entry, for + // use in error messages. + size_t entry_offset_; + + // The return address column for that entry. + unsigned return_address_; +}; + +// Convert the Dwarf expression in |expr| into PfxInstrs stored in the +// SecMap referred to by |summ|, and return the index of the starting +// PfxInstr added, which must be >= 0. In case of failure return -1. +int32_t parseDwarfExpr(Summariser* summ, const ByteReader* reader, + ImageSlice expr, bool debug, bool pushCfaAtStart, + bool derefAtEnd); + +} // namespace lul + +#endif // LulDwarfExt_h diff --git a/tools/profiler/lul/LulDwarfInt.h b/tools/profiler/lul/LulDwarfInt.h new file mode 100644 index 0000000000..b72c6e08e3 --- /dev/null +++ b/tools/profiler/lul/LulDwarfInt.h @@ -0,0 +1,193 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ + +// Copyright (c) 2008, 2010 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// CFI reader author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// This file is derived from the following file in +// toolkit/crashreporter/google-breakpad: +// src/common/dwarf/dwarf2enums.h + +#ifndef LulDwarfInt_h +#define LulDwarfInt_h + +#include "LulCommonExt.h" +#include "LulDwarfExt.h" + +namespace lul { + +// These enums do not follow the google3 style only because they are +// known universally (specs, other implementations) by the names in +// exactly this capitalization. +// Tag names and codes. + +// Call Frame Info instructions. +enum DwarfCFI { + DW_CFA_advance_loc = 0x40, + DW_CFA_offset = 0x80, + DW_CFA_restore = 0xc0, + DW_CFA_nop = 0x00, + DW_CFA_set_loc = 0x01, + DW_CFA_advance_loc1 = 0x02, + DW_CFA_advance_loc2 = 0x03, + DW_CFA_advance_loc4 = 0x04, + DW_CFA_offset_extended = 0x05, + DW_CFA_restore_extended = 0x06, + DW_CFA_undefined = 0x07, + DW_CFA_same_value = 0x08, + DW_CFA_register = 0x09, + DW_CFA_remember_state = 0x0a, + DW_CFA_restore_state = 0x0b, + DW_CFA_def_cfa = 0x0c, + DW_CFA_def_cfa_register = 0x0d, + DW_CFA_def_cfa_offset = 0x0e, + DW_CFA_def_cfa_expression = 0x0f, + DW_CFA_expression = 0x10, + DW_CFA_offset_extended_sf = 0x11, + DW_CFA_def_cfa_sf = 0x12, + DW_CFA_def_cfa_offset_sf = 0x13, + DW_CFA_val_offset = 0x14, + DW_CFA_val_offset_sf = 0x15, + DW_CFA_val_expression = 0x16, + + // Opcodes in this range are reserved for user extensions. + DW_CFA_lo_user = 0x1c, + DW_CFA_hi_user = 0x3f, + + // SGI/MIPS specific. + DW_CFA_MIPS_advance_loc8 = 0x1d, + + // GNU extensions. + DW_CFA_GNU_window_save = 0x2d, + DW_CFA_GNU_args_size = 0x2e, + DW_CFA_GNU_negative_offset_extended = 0x2f +}; + +// Exception handling 'z' augmentation letters. +enum DwarfZAugmentationCodes { + // If the CFI augmentation string begins with 'z', then the CIE and FDE + // have an augmentation data area just before the instructions, whose + // contents are determined by the subsequent augmentation letters. + DW_Z_augmentation_start = 'z', + + // If this letter is present in a 'z' augmentation string, the CIE + // augmentation data includes a pointer encoding, and the FDE + // augmentation data includes a language-specific data area pointer, + // represented using that encoding. + DW_Z_has_LSDA = 'L', + + // If this letter is present in a 'z' augmentation string, the CIE + // augmentation data includes a pointer encoding, followed by a pointer + // to a personality routine, represented using that encoding. + DW_Z_has_personality_routine = 'P', + + // If this letter is present in a 'z' augmentation string, the CIE + // augmentation data includes a pointer encoding describing how the FDE's + // initial location, address range, and DW_CFA_set_loc operands are + // encoded. + DW_Z_has_FDE_address_encoding = 'R', + + // If this letter is present in a 'z' augmentation string, then code + // addresses covered by FDEs that cite this CIE are signal delivery + // trampolines. Return addresses of frames in trampolines should not be + // adjusted as described in section 6.4.4 of the DWARF 3 spec. + DW_Z_is_signal_trampoline = 'S' +}; + +// Expression opcodes +enum DwarfExpressionOpcodes { + DW_OP_addr = 0x03, + DW_OP_deref = 0x06, + DW_OP_const1s = 0x09, + DW_OP_const2u = 0x0a, + DW_OP_const2s = 0x0b, + DW_OP_const4u = 0x0c, + DW_OP_const4s = 0x0d, + DW_OP_const8u = 0x0e, + DW_OP_const8s = 0x0f, + DW_OP_constu = 0x10, + DW_OP_consts = 0x11, + DW_OP_dup = 0x12, + DW_OP_drop = 0x13, + DW_OP_over = 0x14, + DW_OP_pick = 0x15, + DW_OP_swap = 0x16, + DW_OP_rot = 0x17, + DW_OP_xderef = 0x18, + DW_OP_abs = 0x19, + DW_OP_and = 0x1a, + DW_OP_div = 0x1b, + DW_OP_minus = 0x1c, + DW_OP_mod = 0x1d, + DW_OP_mul = 0x1e, + DW_OP_neg = 0x1f, + DW_OP_not = 0x20, + DW_OP_or = 0x21, + DW_OP_plus = 0x22, + DW_OP_plus_uconst = 0x23, + DW_OP_shl = 0x24, + DW_OP_shr = 0x25, + DW_OP_shra = 0x26, + DW_OP_xor = 0x27, + DW_OP_skip = 0x2f, + DW_OP_bra = 0x28, + DW_OP_eq = 0x29, + DW_OP_ge = 0x2a, + DW_OP_gt = 0x2b, + DW_OP_le = 0x2c, + DW_OP_lt = 0x2d, + DW_OP_ne = 0x2e, + DW_OP_lit0 = 0x30, + DW_OP_lit31 = 0x4f, + DW_OP_reg0 = 0x50, + DW_OP_reg31 = 0x6f, + DW_OP_breg0 = 0x70, + DW_OP_breg31 = 0x8f, + DW_OP_regx = 0x90, + DW_OP_fbreg = 0x91, + DW_OP_bregx = 0x92, + DW_OP_piece = 0x93, + DW_OP_deref_size = 0x94, + DW_OP_xderef_size = 0x95, + DW_OP_nop = 0x96, + DW_OP_push_object_address = 0x97, + DW_OP_call2 = 0x98, + DW_OP_call4 = 0x99, + DW_OP_call_ref = 0x9a, + DW_OP_form_tls_address = 0x9b, + DW_OP_call_frame_cfa = 0x9c, + DW_OP_bit_piece = 0x9d, + DW_OP_lo_user = 0xe0, + DW_OP_hi_user = 0xff +}; + +} // namespace lul + +#endif // LulDwarfInt_h diff --git a/tools/profiler/lul/LulDwarfSummariser.cpp b/tools/profiler/lul/LulDwarfSummariser.cpp new file mode 100644 index 0000000000..e9172c3e18 --- /dev/null +++ b/tools/profiler/lul/LulDwarfSummariser.cpp @@ -0,0 +1,549 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "LulDwarfSummariser.h" + +#include "LulDwarfExt.h" + +#include "mozilla/Assertions.h" +#include "mozilla/Sprintf.h" + +// Set this to 1 for verbose logging +#define DEBUG_SUMMARISER 0 + +namespace lul { + +// Do |s64|'s lowest 32 bits sign extend back to |s64| itself? +static inline bool fitsIn32Bits(int64 s64) { + return s64 == ((s64 & 0xffffffff) ^ 0x80000000) - 0x80000000; +} + +// Check a LExpr prefix expression, starting at pfxInstrs[start] up to +// the next PX_End instruction, to ensure that: +// * It only mentions registers that are tracked on this target +// * The start point is sane +// If the expression is ok, return NULL. Else return a pointer +// a const char* holding a bit of text describing the problem. +static const char* checkPfxExpr(const vector<PfxInstr>* pfxInstrs, + int64_t start) { + size_t nInstrs = pfxInstrs->size(); + if (start < 0 || start >= (ssize_t)nInstrs) { + return "bogus start point"; + } + size_t i; + for (i = start; i < nInstrs; i++) { + PfxInstr pxi = (*pfxInstrs)[i]; + if (pxi.mOpcode == PX_End) break; + if (pxi.mOpcode == PX_DwReg && + !registerIsTracked((DW_REG_NUMBER)pxi.mOperand)) { + return "uses untracked reg"; + } + } + return nullptr; // success +} + +Summariser::Summariser(SecMap* aSecMap, uintptr_t aTextBias, + void (*aLog)(const char*)) + : mSecMap(aSecMap), mTextBias(aTextBias), mLog(aLog) { + mCurrAddr = 0; + mMax1Addr = 0; // Gives an empty range. + + // Initialise the running RuleSet to "haven't got a clue" status. + new (&mCurrRules) RuleSet(); +} + +void Summariser::Entry(uintptr_t aAddress, uintptr_t aLength) { + aAddress += mTextBias; + if (DEBUG_SUMMARISER) { + char buf[100]; + SprintfLiteral(buf, "LUL Entry(%llx, %llu)\n", + (unsigned long long int)aAddress, + (unsigned long long int)aLength); + mLog(buf); + } + // This throws away any previous summary, that is, assumes + // that the previous summary, if any, has been properly finished + // by a call to End(). + mCurrAddr = aAddress; + mMax1Addr = aAddress + aLength; + new (&mCurrRules) RuleSet(); +} + +void Summariser::Rule(uintptr_t aAddress, int aNewReg, LExprHow how, + int16_t oldReg, int64_t offset) { + aAddress += mTextBias; + if (DEBUG_SUMMARISER) { + char buf[100]; + if (how == NODEREF || how == DEREF) { + bool deref = how == DEREF; + SprintfLiteral(buf, "LUL 0x%llx old-r%d = %sr%d + %lld%s\n", + (unsigned long long int)aAddress, aNewReg, + deref ? "*(" : "", (int)oldReg, (long long int)offset, + deref ? ")" : ""); + } else if (how == PFXEXPR) { + SprintfLiteral(buf, "LUL 0x%llx old-r%d = pfx-expr-at %lld\n", + (unsigned long long int)aAddress, aNewReg, + (long long int)offset); + } else { + SprintfLiteral(buf, "LUL 0x%llx old-r%d = (invalid LExpr!)\n", + (unsigned long long int)aAddress, aNewReg); + } + mLog(buf); + } + + if (mCurrAddr < aAddress) { + // Flush the existing summary first. + mSecMap->AddRuleSet(&mCurrRules, mCurrAddr, aAddress - mCurrAddr); + if (DEBUG_SUMMARISER) { + mLog("LUL "); + mCurrRules.Print(mCurrAddr, aAddress - mCurrAddr, mLog); + mLog("\n"); + } + mCurrAddr = aAddress; + } + + // If for some reason summarisation fails, either or both of these + // become non-null and point at constant text describing the + // problem. Using two rather than just one avoids complications of + // having to concatenate two strings to produce a complete error message. + const char* reason1 = nullptr; + const char* reason2 = nullptr; + + // |offset| needs to be a 32 bit value that sign extends to 64 bits + // on a 64 bit target. We will need to incorporate |offset| into + // any LExpr made here. So we may as well check it right now. + if (!fitsIn32Bits(offset)) { + reason1 = "offset not in signed 32-bit range"; + goto cant_summarise; + } + + // FIXME: factor out common parts of the arch-dependent summarisers. + +#if defined(GP_ARCH_arm) + + // ----------------- arm ----------------- // + + // Now, can we add the rule to our summary? This depends on whether + // the registers and the overall expression are representable. This + // is the heart of the summarisation process. + switch (aNewReg) { + case DW_REG_CFA: + // This is a rule that defines the CFA. The only forms we + // choose to represent are: r7/11/12/13 + offset. The offset + // must fit into 32 bits since 'uintptr_t' is 32 bit on ARM, + // hence there is no need to check it for overflow. + if (how != NODEREF) { + reason1 = "rule for DW_REG_CFA: invalid |how|"; + goto cant_summarise; + } + switch (oldReg) { + case DW_REG_ARM_R7: + case DW_REG_ARM_R11: + case DW_REG_ARM_R12: + case DW_REG_ARM_R13: + break; + default: + reason1 = "rule for DW_REG_CFA: invalid |oldReg|"; + goto cant_summarise; + } + mCurrRules.mCfaExpr = LExpr(how, oldReg, offset); + break; + + case DW_REG_ARM_R7: + case DW_REG_ARM_R11: + case DW_REG_ARM_R12: + case DW_REG_ARM_R13: + case DW_REG_ARM_R14: + case DW_REG_ARM_R15: { + // This is a new rule for R7, R11, R12, R13 (SP), R14 (LR) or + // R15 (the return address). + switch (how) { + case NODEREF: + case DEREF: + // Check the old register is one we're tracking. + if (!registerIsTracked((DW_REG_NUMBER)oldReg) && + oldReg != DW_REG_CFA) { + reason1 = "rule for R7/11/12/13/14/15: uses untracked reg"; + goto cant_summarise; + } + break; + case PFXEXPR: { + // Check that the prefix expression only mentions tracked registers. + const vector<PfxInstr>* pfxInstrs = mSecMap->GetPfxInstrs(); + reason2 = checkPfxExpr(pfxInstrs, offset); + if (reason2) { + reason1 = "rule for R7/11/12/13/14/15: "; + goto cant_summarise; + } + break; + } + default: + goto cant_summarise; + } + LExpr expr = LExpr(how, oldReg, offset); + switch (aNewReg) { + case DW_REG_ARM_R7: + mCurrRules.mR7expr = expr; + break; + case DW_REG_ARM_R11: + mCurrRules.mR11expr = expr; + break; + case DW_REG_ARM_R12: + mCurrRules.mR12expr = expr; + break; + case DW_REG_ARM_R13: + mCurrRules.mR13expr = expr; + break; + case DW_REG_ARM_R14: + mCurrRules.mR14expr = expr; + break; + case DW_REG_ARM_R15: + mCurrRules.mR15expr = expr; + break; + default: + MOZ_ASSERT(0); + } + break; + } + + default: + // Leave |reason1| and |reason2| unset here. This program point + // is reached so often that it causes a flood of "Can't + // summarise" messages. In any case, we don't really care about + // the fact that this summary would produce a new value for a + // register that we're not tracking. We do on the other hand + // care if the summary's expression *uses* a register that we're + // not tracking. But in that case one of the above failures + // should tell us which. + goto cant_summarise; + } + + // Mark callee-saved registers (r4 .. r11) as unchanged, if there is + // no other information about them. FIXME: do this just once, at + // the point where the ruleset is committed. + if (mCurrRules.mR7expr.mHow == UNKNOWN) { + mCurrRules.mR7expr = LExpr(NODEREF, DW_REG_ARM_R7, 0); + } + if (mCurrRules.mR11expr.mHow == UNKNOWN) { + mCurrRules.mR11expr = LExpr(NODEREF, DW_REG_ARM_R11, 0); + } + if (mCurrRules.mR12expr.mHow == UNKNOWN) { + mCurrRules.mR12expr = LExpr(NODEREF, DW_REG_ARM_R12, 0); + } + + // The old r13 (SP) value before the call is always the same as the + // CFA. + mCurrRules.mR13expr = LExpr(NODEREF, DW_REG_CFA, 0); + + // If there's no information about R15 (the return address), say + // it's a copy of R14 (the link register). + if (mCurrRules.mR15expr.mHow == UNKNOWN) { + mCurrRules.mR15expr = LExpr(NODEREF, DW_REG_ARM_R14, 0); + } + +#elif defined(GP_ARCH_arm64) + + // ----------------- arm64 ----------------- // + + switch (aNewReg) { + case DW_REG_CFA: + if (how != NODEREF) { + reason1 = "rule for DW_REG_CFA: invalid |how|"; + goto cant_summarise; + } + switch (oldReg) { + case DW_REG_AARCH64_X29: + case DW_REG_AARCH64_SP: + break; + default: + reason1 = "rule for DW_REG_CFA: invalid |oldReg|"; + goto cant_summarise; + } + mCurrRules.mCfaExpr = LExpr(how, oldReg, offset); + break; + + case DW_REG_AARCH64_X29: + case DW_REG_AARCH64_X30: + case DW_REG_AARCH64_SP: { + switch (how) { + case NODEREF: + case DEREF: + // Check the old register is one we're tracking. + if (!registerIsTracked((DW_REG_NUMBER)oldReg) && + oldReg != DW_REG_CFA) { + reason1 = "rule for X29/X30/SP: uses untracked reg"; + goto cant_summarise; + } + break; + case PFXEXPR: { + // Check that the prefix expression only mentions tracked registers. + const vector<PfxInstr>* pfxInstrs = mSecMap->GetPfxInstrs(); + reason2 = checkPfxExpr(pfxInstrs, offset); + if (reason2) { + reason1 = "rule for X29/X30/SP: "; + goto cant_summarise; + } + break; + } + default: + goto cant_summarise; + } + LExpr expr = LExpr(how, oldReg, offset); + switch (aNewReg) { + case DW_REG_AARCH64_X29: + mCurrRules.mX29expr = expr; + break; + case DW_REG_AARCH64_X30: + mCurrRules.mX30expr = expr; + break; + case DW_REG_AARCH64_SP: + mCurrRules.mSPexpr = expr; + break; + default: + MOZ_ASSERT(0); + } + break; + } + default: + // Leave |reason1| and |reason2| unset here, for the reasons explained + // in the analogous point + goto cant_summarise; + } + + if (mCurrRules.mX29expr.mHow == UNKNOWN) { + mCurrRules.mX29expr = LExpr(NODEREF, DW_REG_AARCH64_X29, 0); + } + if (mCurrRules.mX30expr.mHow == UNKNOWN) { + mCurrRules.mX30expr = LExpr(NODEREF, DW_REG_AARCH64_X30, 0); + } + // On aarch64, it seems the old SP value before the call is always the + // same as the CFA. Therefore, in the absence of any other way to + // recover the SP, specify that the CFA should be copied. + if (mCurrRules.mSPexpr.mHow == UNKNOWN) { + mCurrRules.mSPexpr = LExpr(NODEREF, DW_REG_CFA, 0); + } +#elif defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + + // ---------------- x64/x86 ---------------- // + + // Now, can we add the rule to our summary? This depends on whether + // the registers and the overall expression are representable. This + // is the heart of the summarisation process. + switch (aNewReg) { + case DW_REG_CFA: { + // This is a rule that defines the CFA. The only forms we choose to + // represent are: = SP+offset, = FP+offset, or =prefix-expr. + switch (how) { + case NODEREF: + if (oldReg != DW_REG_INTEL_XSP && oldReg != DW_REG_INTEL_XBP) { + reason1 = "rule for DW_REG_CFA: invalid |oldReg|"; + goto cant_summarise; + } + break; + case DEREF: + reason1 = "rule for DW_REG_CFA: invalid |how|"; + goto cant_summarise; + case PFXEXPR: { + // Check that the prefix expression only mentions tracked registers. + const vector<PfxInstr>* pfxInstrs = mSecMap->GetPfxInstrs(); + reason2 = checkPfxExpr(pfxInstrs, offset); + if (reason2) { + reason1 = "rule for CFA: "; + goto cant_summarise; + } + break; + } + default: + goto cant_summarise; + } + mCurrRules.mCfaExpr = LExpr(how, oldReg, offset); + break; + } + + case DW_REG_INTEL_XSP: + case DW_REG_INTEL_XBP: + case DW_REG_INTEL_XIP: { + // This is a new rule for XSP, XBP or XIP (the return address). + switch (how) { + case NODEREF: + case DEREF: + // Check the old register is one we're tracking. + if (!registerIsTracked((DW_REG_NUMBER)oldReg) && + oldReg != DW_REG_CFA) { + reason1 = "rule for XSP/XBP/XIP: uses untracked reg"; + goto cant_summarise; + } + break; + case PFXEXPR: { + // Check that the prefix expression only mentions tracked registers. + const vector<PfxInstr>* pfxInstrs = mSecMap->GetPfxInstrs(); + reason2 = checkPfxExpr(pfxInstrs, offset); + if (reason2) { + reason1 = "rule for XSP/XBP/XIP: "; + goto cant_summarise; + } + break; + } + default: + goto cant_summarise; + } + LExpr expr = LExpr(how, oldReg, offset); + switch (aNewReg) { + case DW_REG_INTEL_XBP: + mCurrRules.mXbpExpr = expr; + break; + case DW_REG_INTEL_XSP: + mCurrRules.mXspExpr = expr; + break; + case DW_REG_INTEL_XIP: + mCurrRules.mXipExpr = expr; + break; + default: + MOZ_CRASH("impossible value for aNewReg"); + } + break; + } + + default: + // Leave |reason1| and |reason2| unset here, for the reasons + // explained in the analogous point in the ARM case just above. + goto cant_summarise; + } + + // On Intel, it seems the old SP value before the call is always the + // same as the CFA. Therefore, in the absence of any other way to + // recover the SP, specify that the CFA should be copied. + if (mCurrRules.mXspExpr.mHow == UNKNOWN) { + mCurrRules.mXspExpr = LExpr(NODEREF, DW_REG_CFA, 0); + } + + // Also, gcc says "Undef" for BP when it is unchanged. + if (mCurrRules.mXbpExpr.mHow == UNKNOWN) { + mCurrRules.mXbpExpr = LExpr(NODEREF, DW_REG_INTEL_XBP, 0); + } + +#elif defined(GP_ARCH_mips64) + // ---------------- mips ---------------- // + // + // Now, can we add the rule to our summary? This depends on whether + // the registers and the overall expression are representable. This + // is the heart of the summarisation process. + switch (aNewReg) { + case DW_REG_CFA: + // This is a rule that defines the CFA. The only forms we can + // represent are: = SP+offset or = FP+offset. + if (how != NODEREF) { + reason1 = "rule for DW_REG_CFA: invalid |how|"; + goto cant_summarise; + } + if (oldReg != DW_REG_MIPS_SP && oldReg != DW_REG_MIPS_FP) { + reason1 = "rule for DW_REG_CFA: invalid |oldReg|"; + goto cant_summarise; + } + mCurrRules.mCfaExpr = LExpr(how, oldReg, offset); + break; + + case DW_REG_MIPS_SP: + case DW_REG_MIPS_FP: + case DW_REG_MIPS_PC: { + // This is a new rule for SP, FP or PC (the return address). + switch (how) { + case NODEREF: + case DEREF: + // Check the old register is one we're tracking. + if (!registerIsTracked((DW_REG_NUMBER)oldReg) && + oldReg != DW_REG_CFA) { + reason1 = "rule for SP/FP/PC: uses untracked reg"; + goto cant_summarise; + } + break; + case PFXEXPR: { + // Check that the prefix expression only mentions tracked registers. + const vector<PfxInstr>* pfxInstrs = mSecMap->GetPfxInstrs(); + reason2 = checkPfxExpr(pfxInstrs, offset); + if (reason2) { + reason1 = "rule for SP/FP/PC: "; + goto cant_summarise; + } + break; + } + default: + goto cant_summarise; + } + LExpr expr = LExpr(how, oldReg, offset); + switch (aNewReg) { + case DW_REG_MIPS_FP: + mCurrRules.mFPexpr = expr; + break; + case DW_REG_MIPS_SP: + mCurrRules.mSPexpr = expr; + break; + case DW_REG_MIPS_PC: + mCurrRules.mPCexpr = expr; + break; + default: + MOZ_CRASH("impossible value for aNewReg"); + } + break; + } + default: + // Leave |reason1| and |reason2| unset here, for the reasons + // explained in the analogous point in the ARM case just above. + goto cant_summarise; + } + + // On MIPS, it seems the old SP value before the call is always the + // same as the CFA. Therefore, in the absence of any other way to + // recover the SP, specify that the CFA should be copied. + if (mCurrRules.mSPexpr.mHow == UNKNOWN) { + mCurrRules.mSPexpr = LExpr(NODEREF, DW_REG_CFA, 0); + } + + // Also, gcc says "Undef" for FP when it is unchanged. + if (mCurrRules.mFPexpr.mHow == UNKNOWN) { + mCurrRules.mFPexpr = LExpr(NODEREF, DW_REG_MIPS_FP, 0); + } + +#else + +# error "Unsupported arch" +#endif + + return; + +cant_summarise: + if (reason1 || reason2) { + char buf[200]; + SprintfLiteral(buf, + "LUL can't summarise: " + "SVMA=0x%llx: %s%s, expr=LExpr(%s,%u,%lld)\n", + (unsigned long long int)(aAddress - mTextBias), + reason1 ? reason1 : "", reason2 ? reason2 : "", + NameOf_LExprHow(how), (unsigned int)oldReg, + (long long int)offset); + mLog(buf); + } +} + +uint32_t Summariser::AddPfxInstr(PfxInstr pfxi) { + return mSecMap->AddPfxInstr(pfxi); +} + +void Summariser::End() { + if (DEBUG_SUMMARISER) { + mLog("LUL End\n"); + } + if (mCurrAddr < mMax1Addr) { + mSecMap->AddRuleSet(&mCurrRules, mCurrAddr, mMax1Addr - mCurrAddr); + if (DEBUG_SUMMARISER) { + mLog("LUL "); + mCurrRules.Print(mCurrAddr, mMax1Addr - mCurrAddr, mLog); + mLog("\n"); + } + } +} + +} // namespace lul diff --git a/tools/profiler/lul/LulDwarfSummariser.h b/tools/profiler/lul/LulDwarfSummariser.h new file mode 100644 index 0000000000..30f1ba23c1 --- /dev/null +++ b/tools/profiler/lul/LulDwarfSummariser.h @@ -0,0 +1,64 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef LulDwarfSummariser_h +#define LulDwarfSummariser_h + +#include "LulMainInt.h" + +namespace lul { + +class Summariser { + public: + Summariser(SecMap* aSecMap, uintptr_t aTextBias, void (*aLog)(const char*)); + + virtual void Entry(uintptr_t aAddress, uintptr_t aLength); + virtual void End(); + + // Tell the summariser that the value for |aNewReg| at |aAddress| is + // recovered using the LExpr that can be constructed using the + // components |how|, |oldReg| and |offset|. The summariser will + // inspect the components and may reject them for various reasons, + // but the hope is that it will find them acceptable and record this + // rule permanently. + virtual void Rule(uintptr_t aAddress, int aNewReg, LExprHow how, + int16_t oldReg, int64_t offset); + + virtual uint32_t AddPfxInstr(PfxInstr pfxi); + + // Send output to the logging sink, for debugging. + virtual void Log(const char* str) { mLog(str); } + + private: + // The SecMap in which we park the finished summaries (RuleSets) and + // also any PfxInstrs derived from Dwarf expressions. + SecMap* mSecMap; + + // Running state for the current summary (RuleSet) under construction. + RuleSet mCurrRules; + + // The start of the address range to which the RuleSet under + // construction applies. + uintptr_t mCurrAddr; + + // The highest address, plus one, for which the RuleSet under + // construction could possibly apply. If there are no further + // incoming events then mCurrRules will eventually be emitted + // as-is, for the range mCurrAddr.. mMax1Addr - 1, if that is + // nonempty. + uintptr_t mMax1Addr; + + // The bias value (to add to the SVMAs, to get AVMAs) to be used + // when adding entries into mSecMap. + uintptr_t mTextBias; + + // A logging sink, for debugging. + void (*mLog)(const char* aFmt); +}; + +} // namespace lul + +#endif // LulDwarfSummariser_h diff --git a/tools/profiler/lul/LulElf.cpp b/tools/profiler/lul/LulElf.cpp new file mode 100644 index 0000000000..28980a1349 --- /dev/null +++ b/tools/profiler/lul/LulElf.cpp @@ -0,0 +1,887 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ + +// Copyright (c) 2006, 2011, 2012 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Restructured in 2009 by: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// (derived from) +// dump_symbols.cc: implement google_breakpad::WriteSymbolFile: +// Find all the debugging info in a file and dump it as a Breakpad symbol file. +// +// dump_symbols.h: Read debugging information from an ELF file, and write +// it out as a Breakpad symbol file. + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/linux/dump_symbols.cc +// src/common/linux/elfutils.cc +// src/common/linux/file_id.cc + +#include <errno.h> +#include <fcntl.h> +#include <libgen.h> +#include <stdio.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <unistd.h> +#include <arpa/inet.h> + +#include <set> +#include <string> +#include <vector> + +#include "mozilla/Assertions.h" +#include "mozilla/Sprintf.h" + +#include "PlatformMacros.h" +#include "LulCommonExt.h" +#include "LulDwarfExt.h" +#include "LulElfInt.h" +#include "LulMainInt.h" + +#if defined(GP_PLAT_arm_android) && !defined(SHT_ARM_EXIDX) +// bionic and older glibsc don't define it +# define SHT_ARM_EXIDX (SHT_LOPROC + 1) +#endif + +#if (defined(GP_PLAT_amd64_linux) || defined(GP_PLAT_amd64_android)) && \ + !defined(SHT_X86_64_UNWIND) +// This is sometimes necessary on x86_64-android and x86_64-linux. +# define SHT_X86_64_UNWIND 0x70000001 +#endif + +// Old Linux header doesn't define EM_AARCH64 +#ifndef EM_AARCH64 +# define EM_AARCH64 183 +#endif + +// This namespace contains helper functions. +namespace { + +using lul::DwarfCFIToModule; +using lul::FindElfSectionByName; +using lul::GetOffset; +using lul::IsValidElf; +using lul::Module; +using lul::scoped_ptr; +using lul::Summariser; +using lul::UniqueStringUniverse; +using std::set; +using std::string; +using std::vector; + +// +// FDWrapper +// +// Wrapper class to make sure opened file is closed. +// +class FDWrapper { + public: + explicit FDWrapper(int fd) : fd_(fd) {} + ~FDWrapper() { + if (fd_ != -1) close(fd_); + } + int get() { return fd_; } + int release() { + int fd = fd_; + fd_ = -1; + return fd; + } + + private: + int fd_; +}; + +// +// MmapWrapper +// +// Wrapper class to make sure mapped regions are unmapped. +// +class MmapWrapper { + public: + MmapWrapper() : is_set_(false), base_(NULL), size_(0) {} + ~MmapWrapper() { + if (is_set_ && base_ != NULL) { + MOZ_ASSERT(size_ > 0); + munmap(base_, size_); + } + } + void set(void* mapped_address, size_t mapped_size) { + is_set_ = true; + base_ = mapped_address; + size_ = mapped_size; + } + void release() { + MOZ_ASSERT(is_set_); + is_set_ = false; + base_ = NULL; + size_ = 0; + } + + private: + bool is_set_; + void* base_; + size_t size_; +}; + +// Set NUM_DW_REGNAMES to be the number of Dwarf register names +// appropriate to the machine architecture given in HEADER. Return +// true on success, or false if HEADER's machine architecture is not +// supported. +template <typename ElfClass> +bool DwarfCFIRegisterNames(const typename ElfClass::Ehdr* elf_header, + unsigned int* num_dw_regnames) { + switch (elf_header->e_machine) { + case EM_386: + *num_dw_regnames = DwarfCFIToModule::RegisterNames::I386(); + return true; + case EM_ARM: + *num_dw_regnames = DwarfCFIToModule::RegisterNames::ARM(); + return true; + case EM_X86_64: + *num_dw_regnames = DwarfCFIToModule::RegisterNames::X86_64(); + return true; + case EM_MIPS: + *num_dw_regnames = DwarfCFIToModule::RegisterNames::MIPS(); + return true; + case EM_AARCH64: + *num_dw_regnames = DwarfCFIToModule::RegisterNames::ARM64(); + return true; + default: + MOZ_ASSERT(0); + return false; + } +} + +template <typename ElfClass> +bool LoadDwarfCFI(const string& dwarf_filename, + const typename ElfClass::Ehdr* elf_header, + const char* section_name, + const typename ElfClass::Shdr* section, const bool eh_frame, + const typename ElfClass::Shdr* got_section, + const typename ElfClass::Shdr* text_section, + const bool big_endian, SecMap* smap, uintptr_t text_bias, + UniqueStringUniverse* usu, void (*log)(const char*)) { + // Find the appropriate set of register names for this file's + // architecture. + unsigned int num_dw_regs = 0; + if (!DwarfCFIRegisterNames<ElfClass>(elf_header, &num_dw_regs)) { + fprintf(stderr, + "%s: unrecognized ELF machine architecture '%d';" + " cannot convert DWARF call frame information\n", + dwarf_filename.c_str(), elf_header->e_machine); + return false; + } + + const lul::Endianness endianness = + big_endian ? lul::ENDIANNESS_BIG : lul::ENDIANNESS_LITTLE; + + // Find the call frame information and its size. + const char* cfi = GetOffset<ElfClass, char>(elf_header, section->sh_offset); + size_t cfi_size = section->sh_size; + + // Plug together the parser, handler, and their entourages. + + // Here's a summariser, which will receive the output of the + // parser, create summaries, and add them to |smap|. + Summariser summ(smap, text_bias, log); + + lul::ByteReader reader(endianness); + reader.SetAddressSize(ElfClass::kAddrSize); + + DwarfCFIToModule::Reporter module_reporter(log, dwarf_filename, section_name); + DwarfCFIToModule handler(num_dw_regs, &module_reporter, &reader, usu, &summ); + + // Provide the base addresses for .eh_frame encoded pointers, if + // possible. + reader.SetCFIDataBase(section->sh_addr, cfi); + if (got_section) reader.SetDataBase(got_section->sh_addr); + if (text_section) reader.SetTextBase(text_section->sh_addr); + + lul::CallFrameInfo::Reporter dwarf_reporter(log, dwarf_filename, + section_name); + lul::CallFrameInfo parser(cfi, cfi_size, &reader, &handler, &dwarf_reporter, + eh_frame); + parser.Start(); + + return true; +} + +bool LoadELF(const string& obj_file, MmapWrapper* map_wrapper, + void** elf_header) { + int obj_fd = open(obj_file.c_str(), O_RDONLY); + if (obj_fd < 0) { + fprintf(stderr, "Failed to open ELF file '%s': %s\n", obj_file.c_str(), + strerror(errno)); + return false; + } + FDWrapper obj_fd_wrapper(obj_fd); + struct stat st; + if (fstat(obj_fd, &st) != 0 && st.st_size <= 0) { + fprintf(stderr, "Unable to fstat ELF file '%s': %s\n", obj_file.c_str(), + strerror(errno)); + return false; + } + // Mapping it read-only is good enough. In any case, mapping it + // read-write confuses Valgrind's debuginfo acquire/discard + // heuristics, making it hard to profile the profiler. + void* obj_base = mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, obj_fd, 0); + if (obj_base == MAP_FAILED) { + fprintf(stderr, "Failed to mmap ELF file '%s': %s\n", obj_file.c_str(), + strerror(errno)); + return false; + } + map_wrapper->set(obj_base, st.st_size); + *elf_header = obj_base; + if (!IsValidElf(*elf_header)) { + fprintf(stderr, "Not a valid ELF file: %s\n", obj_file.c_str()); + return false; + } + return true; +} + +// Get the endianness of ELF_HEADER. If it's invalid, return false. +template <typename ElfClass> +bool ElfEndianness(const typename ElfClass::Ehdr* elf_header, + bool* big_endian) { + if (elf_header->e_ident[EI_DATA] == ELFDATA2LSB) { + *big_endian = false; + return true; + } + if (elf_header->e_ident[EI_DATA] == ELFDATA2MSB) { + *big_endian = true; + return true; + } + + fprintf(stderr, "bad data encoding in ELF header: %d\n", + elf_header->e_ident[EI_DATA]); + return false; +} + +// +// LoadSymbolsInfo +// +// Holds the state between the two calls to LoadSymbols() in case it's necessary +// to follow the .gnu_debuglink section and load debug information from a +// different file. +// +template <typename ElfClass> +class LoadSymbolsInfo { + public: + typedef typename ElfClass::Addr Addr; + + explicit LoadSymbolsInfo(const vector<string>& dbg_dirs) + : debug_dirs_(dbg_dirs), has_loading_addr_(false) {} + + // Keeps track of which sections have been loaded so sections don't + // accidentally get loaded twice from two different files. + void LoadedSection(const string& section) { + if (loaded_sections_.count(section) == 0) { + loaded_sections_.insert(section); + } else { + fprintf(stderr, "Section %s has already been loaded.\n", section.c_str()); + } + } + + string debuglink_file() const { return debuglink_file_; } + + private: + const vector<string>& debug_dirs_; // Directories in which to + // search for the debug ELF file. + + string debuglink_file_; // Full path to the debug ELF file. + + bool has_loading_addr_; // Indicate if LOADING_ADDR_ is valid. + + set<string> loaded_sections_; // Tracks the Loaded ELF sections + // between calls to LoadSymbols(). +}; + +// Find the preferred loading address of the binary. +template <typename ElfClass> +typename ElfClass::Addr GetLoadingAddress( + const typename ElfClass::Phdr* program_headers, int nheader) { + typedef typename ElfClass::Phdr Phdr; + + // For non-PIC executables (e_type == ET_EXEC), the load address is + // the start address of the first PT_LOAD segment. (ELF requires + // the segments to be sorted by load address.) For PIC executables + // and dynamic libraries (e_type == ET_DYN), this address will + // normally be zero. + for (int i = 0; i < nheader; ++i) { + const Phdr& header = program_headers[i]; + if (header.p_type == PT_LOAD) return header.p_vaddr; + } + return 0; +} + +template <typename ElfClass> +bool LoadSymbols(const string& obj_file, const bool big_endian, + const typename ElfClass::Ehdr* elf_header, + const bool read_gnu_debug_link, + LoadSymbolsInfo<ElfClass>* info, SecMap* smap, void* rx_avma, + size_t rx_size, UniqueStringUniverse* usu, + void (*log)(const char*)) { + typedef typename ElfClass::Phdr Phdr; + typedef typename ElfClass::Shdr Shdr; + + char buf[500]; + SprintfLiteral(buf, "LoadSymbols: BEGIN %s\n", obj_file.c_str()); + buf[sizeof(buf) - 1] = 0; + log(buf); + + // This is how the text bias is calculated. + // BEGIN CALCULATE BIAS + uintptr_t loading_addr = GetLoadingAddress<ElfClass>( + GetOffset<ElfClass, Phdr>(elf_header, elf_header->e_phoff), + elf_header->e_phnum); + uintptr_t text_bias = ((uintptr_t)rx_avma) - loading_addr; + SprintfLiteral(buf, "LoadSymbols: rx_avma=%llx, text_bias=%llx", + (unsigned long long int)(uintptr_t)rx_avma, + (unsigned long long int)text_bias); + buf[sizeof(buf) - 1] = 0; + log(buf); + // END CALCULATE BIAS + + const Shdr* sections = + GetOffset<ElfClass, Shdr>(elf_header, elf_header->e_shoff); + const Shdr* section_names = sections + elf_header->e_shstrndx; + const char* names = + GetOffset<ElfClass, char>(elf_header, section_names->sh_offset); + const char* names_end = names + section_names->sh_size; + bool found_usable_info = false; + + // Dwarf Call Frame Information (CFI) is actually independent from + // the other DWARF debugging information, and can be used alone. + const Shdr* dwarf_cfi_section = + FindElfSectionByName<ElfClass>(".debug_frame", SHT_PROGBITS, sections, + names, names_end, elf_header->e_shnum); + if (dwarf_cfi_section) { + // Ignore the return value of this function; even without call frame + // information, the other debugging information could be perfectly + // useful. + info->LoadedSection(".debug_frame"); + bool result = LoadDwarfCFI<ElfClass>(obj_file, elf_header, ".debug_frame", + dwarf_cfi_section, false, 0, 0, + big_endian, smap, text_bias, usu, log); + found_usable_info = found_usable_info || result; + if (result) log("LoadSymbols: read CFI from .debug_frame"); + } + + // Linux C++ exception handling information can also provide + // unwinding data. + const Shdr* eh_frame_section = + FindElfSectionByName<ElfClass>(".eh_frame", SHT_PROGBITS, sections, names, + names_end, elf_header->e_shnum); +#if defined(GP_PLAT_amd64_linux) || defined(GP_PLAT_amd64_android) + if (!eh_frame_section) { + // Possibly depending on which linker created libxul.so, on x86_64-linux + // and -android, .eh_frame may instead have the SHT_X86_64_UNWIND type. + eh_frame_section = + FindElfSectionByName<ElfClass>(".eh_frame", SHT_X86_64_UNWIND, sections, + names, names_end, elf_header->e_shnum); + } +#endif + if (eh_frame_section) { + // Pointers in .eh_frame data may be relative to the base addresses of + // certain sections. Provide those sections if present. + const Shdr* got_section = FindElfSectionByName<ElfClass>( + ".got", SHT_PROGBITS, sections, names, names_end, elf_header->e_shnum); + const Shdr* text_section = FindElfSectionByName<ElfClass>( + ".text", SHT_PROGBITS, sections, names, names_end, elf_header->e_shnum); + info->LoadedSection(".eh_frame"); + // As above, ignore the return value of this function. + bool result = LoadDwarfCFI<ElfClass>( + obj_file, elf_header, ".eh_frame", eh_frame_section, true, got_section, + text_section, big_endian, smap, text_bias, usu, log); + found_usable_info = found_usable_info || result; + if (result) log("LoadSymbols: read CFI from .eh_frame"); + } + + SprintfLiteral(buf, "LoadSymbols: END %s\n", obj_file.c_str()); + buf[sizeof(buf) - 1] = 0; + log(buf); + + return found_usable_info; +} + +// Return the breakpad symbol file identifier for the architecture of +// ELF_HEADER. +template <typename ElfClass> +const char* ElfArchitecture(const typename ElfClass::Ehdr* elf_header) { + typedef typename ElfClass::Half Half; + Half arch = elf_header->e_machine; + switch (arch) { + case EM_386: + return "x86"; + case EM_ARM: + return "arm"; + case EM_AARCH64: + return "arm64"; + case EM_MIPS: + return "mips"; + case EM_PPC64: + return "ppc64"; + case EM_PPC: + return "ppc"; + case EM_S390: + return "s390"; + case EM_SPARC: + return "sparc"; + case EM_SPARCV9: + return "sparcv9"; + case EM_X86_64: + return "x86_64"; + default: + return NULL; + } +} + +// Format the Elf file identifier in IDENTIFIER as a UUID with the +// dashes removed. +string FormatIdentifier(unsigned char identifier[16]) { + char identifier_str[40]; + lul::FileID::ConvertIdentifierToString(identifier, identifier_str, + sizeof(identifier_str)); + string id_no_dash; + for (int i = 0; identifier_str[i] != '\0'; ++i) + if (identifier_str[i] != '-') id_no_dash += identifier_str[i]; + // Add an extra "0" by the end. PDB files on Windows have an 'age' + // number appended to the end of the file identifier; this isn't + // really used or necessary on other platforms, but be consistent. + id_no_dash += '0'; + return id_no_dash; +} + +// Return the non-directory portion of FILENAME: the portion after the +// last slash, or the whole filename if there are no slashes. +string BaseFileName(const string& filename) { + // Lots of copies! basename's behavior is less than ideal. + char* c_filename = strdup(filename.c_str()); + string base = basename(c_filename); + free(c_filename); + return base; +} + +template <typename ElfClass> +bool ReadSymbolDataElfClass(const typename ElfClass::Ehdr* elf_header, + const string& obj_filename, + const vector<string>& debug_dirs, SecMap* smap, + void* rx_avma, size_t rx_size, + UniqueStringUniverse* usu, + void (*log)(const char*)) { + typedef typename ElfClass::Ehdr Ehdr; + + unsigned char identifier[16]; + if (!lul ::FileID::ElfFileIdentifierFromMappedFile(elf_header, identifier)) { + fprintf(stderr, "%s: unable to generate file identifier\n", + obj_filename.c_str()); + return false; + } + + const char* architecture = ElfArchitecture<ElfClass>(elf_header); + if (!architecture) { + fprintf(stderr, "%s: unrecognized ELF machine architecture: %d\n", + obj_filename.c_str(), elf_header->e_machine); + return false; + } + + // Figure out what endianness this file is. + bool big_endian; + if (!ElfEndianness<ElfClass>(elf_header, &big_endian)) return false; + + string name = BaseFileName(obj_filename); + string os = "Linux"; + string id = FormatIdentifier(identifier); + + LoadSymbolsInfo<ElfClass> info(debug_dirs); + if (!LoadSymbols<ElfClass>(obj_filename, big_endian, elf_header, + !debug_dirs.empty(), &info, smap, rx_avma, rx_size, + usu, log)) { + const string debuglink_file = info.debuglink_file(); + if (debuglink_file.empty()) return false; + + // Load debuglink ELF file. + fprintf(stderr, "Found debugging info in %s\n", debuglink_file.c_str()); + MmapWrapper debug_map_wrapper; + Ehdr* debug_elf_header = NULL; + if (!LoadELF(debuglink_file, &debug_map_wrapper, + reinterpret_cast<void**>(&debug_elf_header))) + return false; + // Sanity checks to make sure everything matches up. + const char* debug_architecture = + ElfArchitecture<ElfClass>(debug_elf_header); + if (!debug_architecture) { + fprintf(stderr, "%s: unrecognized ELF machine architecture: %d\n", + debuglink_file.c_str(), debug_elf_header->e_machine); + return false; + } + if (strcmp(architecture, debug_architecture)) { + fprintf(stderr, + "%s with ELF machine architecture %s does not match " + "%s with ELF architecture %s\n", + debuglink_file.c_str(), debug_architecture, obj_filename.c_str(), + architecture); + return false; + } + + bool debug_big_endian; + if (!ElfEndianness<ElfClass>(debug_elf_header, &debug_big_endian)) + return false; + if (debug_big_endian != big_endian) { + fprintf(stderr, "%s and %s does not match in endianness\n", + obj_filename.c_str(), debuglink_file.c_str()); + return false; + } + + if (!LoadSymbols<ElfClass>(debuglink_file, debug_big_endian, + debug_elf_header, false, &info, smap, rx_avma, + rx_size, usu, log)) { + return false; + } + } + + return true; +} + +} // namespace + +namespace lul { + +bool ReadSymbolDataInternal(const uint8_t* obj_file, const string& obj_filename, + const vector<string>& debug_dirs, SecMap* smap, + void* rx_avma, size_t rx_size, + UniqueStringUniverse* usu, + void (*log)(const char*)) { + if (!IsValidElf(obj_file)) { + fprintf(stderr, "Not a valid ELF file: %s\n", obj_filename.c_str()); + return false; + } + + int elfclass = ElfClass(obj_file); + if (elfclass == ELFCLASS32) { + return ReadSymbolDataElfClass<ElfClass32>( + reinterpret_cast<const Elf32_Ehdr*>(obj_file), obj_filename, debug_dirs, + smap, rx_avma, rx_size, usu, log); + } + if (elfclass == ELFCLASS64) { + return ReadSymbolDataElfClass<ElfClass64>( + reinterpret_cast<const Elf64_Ehdr*>(obj_file), obj_filename, debug_dirs, + smap, rx_avma, rx_size, usu, log); + } + + return false; +} + +bool ReadSymbolData(const string& obj_file, const vector<string>& debug_dirs, + SecMap* smap, void* rx_avma, size_t rx_size, + UniqueStringUniverse* usu, void (*log)(const char*)) { + MmapWrapper map_wrapper; + void* elf_header = NULL; + if (!LoadELF(obj_file, &map_wrapper, &elf_header)) return false; + + return ReadSymbolDataInternal(reinterpret_cast<uint8_t*>(elf_header), + obj_file, debug_dirs, smap, rx_avma, rx_size, + usu, log); +} + +namespace { + +template <typename ElfClass> +void FindElfClassSection(const char* elf_base, const char* section_name, + typename ElfClass::Word section_type, + const void** section_start, int* section_size) { + typedef typename ElfClass::Ehdr Ehdr; + typedef typename ElfClass::Shdr Shdr; + + MOZ_ASSERT(elf_base); + MOZ_ASSERT(section_start); + MOZ_ASSERT(section_size); + + MOZ_ASSERT(strncmp(elf_base, ELFMAG, SELFMAG) == 0); + + const Ehdr* elf_header = reinterpret_cast<const Ehdr*>(elf_base); + MOZ_ASSERT(elf_header->e_ident[EI_CLASS] == ElfClass::kClass); + + const Shdr* sections = + GetOffset<ElfClass, Shdr>(elf_header, elf_header->e_shoff); + const Shdr* section_names = sections + elf_header->e_shstrndx; + const char* names = + GetOffset<ElfClass, char>(elf_header, section_names->sh_offset); + const char* names_end = names + section_names->sh_size; + + const Shdr* section = + FindElfSectionByName<ElfClass>(section_name, section_type, sections, + names, names_end, elf_header->e_shnum); + + if (section != NULL && section->sh_size > 0) { + *section_start = elf_base + section->sh_offset; + *section_size = section->sh_size; + } +} + +template <typename ElfClass> +void FindElfClassSegment(const char* elf_base, + typename ElfClass::Word segment_type, + const void** segment_start, int* segment_size) { + typedef typename ElfClass::Ehdr Ehdr; + typedef typename ElfClass::Phdr Phdr; + + MOZ_ASSERT(elf_base); + MOZ_ASSERT(segment_start); + MOZ_ASSERT(segment_size); + + MOZ_ASSERT(strncmp(elf_base, ELFMAG, SELFMAG) == 0); + + const Ehdr* elf_header = reinterpret_cast<const Ehdr*>(elf_base); + MOZ_ASSERT(elf_header->e_ident[EI_CLASS] == ElfClass::kClass); + + const Phdr* phdrs = + GetOffset<ElfClass, Phdr>(elf_header, elf_header->e_phoff); + + for (int i = 0; i < elf_header->e_phnum; ++i) { + if (phdrs[i].p_type == segment_type) { + *segment_start = elf_base + phdrs[i].p_offset; + *segment_size = phdrs[i].p_filesz; + return; + } + } +} + +} // namespace + +bool IsValidElf(const void* elf_base) { + return strncmp(reinterpret_cast<const char*>(elf_base), ELFMAG, SELFMAG) == 0; +} + +int ElfClass(const void* elf_base) { + const ElfW(Ehdr)* elf_header = reinterpret_cast<const ElfW(Ehdr)*>(elf_base); + + return elf_header->e_ident[EI_CLASS]; +} + +bool FindElfSection(const void* elf_mapped_base, const char* section_name, + uint32_t section_type, const void** section_start, + int* section_size, int* elfclass) { + MOZ_ASSERT(elf_mapped_base); + MOZ_ASSERT(section_start); + MOZ_ASSERT(section_size); + + *section_start = NULL; + *section_size = 0; + + if (!IsValidElf(elf_mapped_base)) return false; + + int cls = ElfClass(elf_mapped_base); + if (elfclass) { + *elfclass = cls; + } + + const char* elf_base = static_cast<const char*>(elf_mapped_base); + + if (cls == ELFCLASS32) { + FindElfClassSection<ElfClass32>(elf_base, section_name, section_type, + section_start, section_size); + return *section_start != NULL; + } else if (cls == ELFCLASS64) { + FindElfClassSection<ElfClass64>(elf_base, section_name, section_type, + section_start, section_size); + return *section_start != NULL; + } + + return false; +} + +bool FindElfSegment(const void* elf_mapped_base, uint32_t segment_type, + const void** segment_start, int* segment_size, + int* elfclass) { + MOZ_ASSERT(elf_mapped_base); + MOZ_ASSERT(segment_start); + MOZ_ASSERT(segment_size); + + *segment_start = NULL; + *segment_size = 0; + + if (!IsValidElf(elf_mapped_base)) return false; + + int cls = ElfClass(elf_mapped_base); + if (elfclass) { + *elfclass = cls; + } + + const char* elf_base = static_cast<const char*>(elf_mapped_base); + + if (cls == ELFCLASS32) { + FindElfClassSegment<ElfClass32>(elf_base, segment_type, segment_start, + segment_size); + return *segment_start != NULL; + } else if (cls == ELFCLASS64) { + FindElfClassSegment<ElfClass64>(elf_base, segment_type, segment_start, + segment_size); + return *segment_start != NULL; + } + + return false; +} + +// (derived from) +// file_id.cc: Return a unique identifier for a file +// +// See file_id.h for documentation +// + +// ELF note name and desc are 32-bits word padded. +#define NOTE_PADDING(a) ((a + 3) & ~3) + +// These functions are also used inside the crashed process, so be safe +// and use the syscall/libc wrappers instead of direct syscalls or libc. + +template <typename ElfClass> +static bool ElfClassBuildIDNoteIdentifier(const void* section, int length, + uint8_t identifier[kMDGUIDSize]) { + typedef typename ElfClass::Nhdr Nhdr; + + const void* section_end = reinterpret_cast<const char*>(section) + length; + const Nhdr* note_header = reinterpret_cast<const Nhdr*>(section); + while (reinterpret_cast<const void*>(note_header) < section_end) { + if (note_header->n_type == NT_GNU_BUILD_ID) break; + note_header = reinterpret_cast<const Nhdr*>( + reinterpret_cast<const char*>(note_header) + sizeof(Nhdr) + + NOTE_PADDING(note_header->n_namesz) + + NOTE_PADDING(note_header->n_descsz)); + } + if (reinterpret_cast<const void*>(note_header) >= section_end || + note_header->n_descsz == 0) { + return false; + } + + const char* build_id = reinterpret_cast<const char*>(note_header) + + sizeof(Nhdr) + NOTE_PADDING(note_header->n_namesz); + // Copy as many bits of the build ID as will fit + // into the GUID space. + memset(identifier, 0, kMDGUIDSize); + memcpy(identifier, build_id, + std::min(kMDGUIDSize, (size_t)note_header->n_descsz)); + + return true; +} + +// Attempt to locate a .note.gnu.build-id section in an ELF binary +// and copy as many bytes of it as will fit into |identifier|. +static bool FindElfBuildIDNote(const void* elf_mapped_base, + uint8_t identifier[kMDGUIDSize]) { + void* note_section; + int note_size, elfclass; + if ((!FindElfSegment(elf_mapped_base, PT_NOTE, (const void**)¬e_section, + ¬e_size, &elfclass) || + note_size == 0) && + (!FindElfSection(elf_mapped_base, ".note.gnu.build-id", SHT_NOTE, + (const void**)¬e_section, ¬e_size, &elfclass) || + note_size == 0)) { + return false; + } + + if (elfclass == ELFCLASS32) { + return ElfClassBuildIDNoteIdentifier<ElfClass32>(note_section, note_size, + identifier); + } else if (elfclass == ELFCLASS64) { + return ElfClassBuildIDNoteIdentifier<ElfClass64>(note_section, note_size, + identifier); + } + + return false; +} + +// Attempt to locate the .text section of an ELF binary and generate +// a simple hash by XORing the first page worth of bytes into |identifier|. +static bool HashElfTextSection(const void* elf_mapped_base, + uint8_t identifier[kMDGUIDSize]) { + void* text_section; + int text_size; + if (!FindElfSection(elf_mapped_base, ".text", SHT_PROGBITS, + (const void**)&text_section, &text_size, NULL) || + text_size == 0) { + return false; + } + + memset(identifier, 0, kMDGUIDSize); + const uint8_t* ptr = reinterpret_cast<const uint8_t*>(text_section); + const uint8_t* ptr_end = ptr + std::min(text_size, 4096); + while (ptr < ptr_end) { + for (unsigned i = 0; i < kMDGUIDSize; i++) identifier[i] ^= ptr[i]; + ptr += kMDGUIDSize; + } + return true; +} + +// static +bool FileID::ElfFileIdentifierFromMappedFile(const void* base, + uint8_t identifier[kMDGUIDSize]) { + // Look for a build id note first. + if (FindElfBuildIDNote(base, identifier)) return true; + + // Fall back on hashing the first page of the text section. + return HashElfTextSection(base, identifier); +} + +// static +void FileID::ConvertIdentifierToString(const uint8_t identifier[kMDGUIDSize], + char* buffer, int buffer_length) { + uint8_t identifier_swapped[kMDGUIDSize]; + + // Endian-ness swap to match dump processor expectation. + memcpy(identifier_swapped, identifier, kMDGUIDSize); + uint32_t* data1 = reinterpret_cast<uint32_t*>(identifier_swapped); + *data1 = htonl(*data1); + uint16_t* data2 = reinterpret_cast<uint16_t*>(identifier_swapped + 4); + *data2 = htons(*data2); + uint16_t* data3 = reinterpret_cast<uint16_t*>(identifier_swapped + 6); + *data3 = htons(*data3); + + int buffer_idx = 0; + for (unsigned int idx = 0; + (buffer_idx < buffer_length) && (idx < kMDGUIDSize); ++idx) { + int hi = (identifier_swapped[idx] >> 4) & 0x0F; + int lo = (identifier_swapped[idx]) & 0x0F; + + if (idx == 4 || idx == 6 || idx == 8 || idx == 10) + buffer[buffer_idx++] = '-'; + + buffer[buffer_idx++] = (hi >= 10) ? 'A' + hi - 10 : '0' + hi; + buffer[buffer_idx++] = (lo >= 10) ? 'A' + lo - 10 : '0' + lo; + } + + // NULL terminate + buffer[(buffer_idx < buffer_length) ? buffer_idx : buffer_idx - 1] = 0; +} + +} // namespace lul diff --git a/tools/profiler/lul/LulElfExt.h b/tools/profiler/lul/LulElfExt.h new file mode 100644 index 0000000000..73d9ff7f15 --- /dev/null +++ b/tools/profiler/lul/LulElfExt.h @@ -0,0 +1,69 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ + +// Copyright (c) 2006, 2011, 2012 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/linux/dump_symbols.h + +#ifndef LulElfExt_h +#define LulElfExt_h + +// These two functions are the external interface to the +// ELF/Dwarf/EXIDX reader. + +#include "LulMainInt.h" + +using lul::SecMap; + +namespace lul { + +class UniqueStringUniverse; + +// Find all the unwind information in OBJ_FILE, an ELF executable +// or shared library, and add it to SMAP. +bool ReadSymbolData(const std::string& obj_file, + const std::vector<std::string>& debug_dirs, SecMap* smap, + void* rx_avma, size_t rx_size, UniqueStringUniverse* usu, + void (*log)(const char*)); + +// The same as ReadSymbolData, except that OBJ_FILE is assumed to +// point to a mapped-in image of OBJ_FILENAME. +bool ReadSymbolDataInternal(const uint8_t* obj_file, + const std::string& obj_filename, + const std::vector<std::string>& debug_dirs, + SecMap* smap, void* rx_avma, size_t rx_size, + UniqueStringUniverse* usu, + void (*log)(const char*)); + +} // namespace lul + +#endif // LulElfExt_h diff --git a/tools/profiler/lul/LulElfInt.h b/tools/profiler/lul/LulElfInt.h new file mode 100644 index 0000000000..31ffba8ff0 --- /dev/null +++ b/tools/profiler/lul/LulElfInt.h @@ -0,0 +1,218 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ + +// Copyright (c) 2006, 2012, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/android/include/elf.h +// src/common/linux/elfutils.h +// src/common/linux/file_id.h +// src/common/linux/elfutils-inl.h + +#ifndef LulElfInt_h +#define LulElfInt_h + +// This header defines functions etc internal to the ELF reader. It +// should not be included outside of LulElf.cpp. + +#include <elf.h> +#include <stdlib.h> + +#include "mozilla/Assertions.h" + +#include "PlatformMacros.h" + +// (derived from) +// elfutils.h: Utilities for dealing with ELF files. +// +#include <link.h> + +#if defined(GP_OS_android) + +// From toolkit/crashreporter/google-breakpad/src/common/android/include/elf.h +// The Android headers don't always define this constant. +# ifndef EM_X86_64 +# define EM_X86_64 62 +# endif + +# ifndef EM_PPC64 +# define EM_PPC64 21 +# endif + +# ifndef EM_S390 +# define EM_S390 22 +# endif + +# ifndef NT_GNU_BUILD_ID +# define NT_GNU_BUILD_ID 3 +# endif + +# ifndef ElfW +# define ElfW(type) _ElfW(Elf, ELFSIZE, type) +# define _ElfW(e, w, t) _ElfW_1(e, w, _##t) +# define _ElfW_1(e, w, t) e##w##t +# endif + +#endif + +#if defined(GP_OS_freebsd) + +# ifndef ElfW +# define ElfW(type) Elf_##type +# endif + +#endif + +namespace lul { + +// Traits classes so consumers can write templatized code to deal +// with specific ELF bits. +struct ElfClass32 { + typedef Elf32_Addr Addr; + typedef Elf32_Ehdr Ehdr; + typedef Elf32_Nhdr Nhdr; + typedef Elf32_Phdr Phdr; + typedef Elf32_Shdr Shdr; + typedef Elf32_Half Half; + typedef Elf32_Off Off; + typedef Elf32_Word Word; + static const int kClass = ELFCLASS32; + static const size_t kAddrSize = sizeof(Elf32_Addr); +}; + +struct ElfClass64 { + typedef Elf64_Addr Addr; + typedef Elf64_Ehdr Ehdr; + typedef Elf64_Nhdr Nhdr; + typedef Elf64_Phdr Phdr; + typedef Elf64_Shdr Shdr; + typedef Elf64_Half Half; + typedef Elf64_Off Off; + typedef Elf64_Word Word; + static const int kClass = ELFCLASS64; + static const size_t kAddrSize = sizeof(Elf64_Addr); +}; + +bool IsValidElf(const void* elf_header); +int ElfClass(const void* elf_base); + +// Attempt to find a section named |section_name| of type |section_type| +// in the ELF binary data at |elf_mapped_base|. On success, returns true +// and sets |*section_start| to point to the start of the section data, +// and |*section_size| to the size of the section's data. If |elfclass| +// is not NULL, set |*elfclass| to the ELF file class. +bool FindElfSection(const void* elf_mapped_base, const char* section_name, + uint32_t section_type, const void** section_start, + int* section_size, int* elfclass); + +// Internal helper method, exposed for convenience for callers +// that already have more info. +template <typename ElfClass> +const typename ElfClass::Shdr* FindElfSectionByName( + const char* name, typename ElfClass::Word section_type, + const typename ElfClass::Shdr* sections, const char* section_names, + const char* names_end, int nsection); + +// Attempt to find the first segment of type |segment_type| in the ELF +// binary data at |elf_mapped_base|. On success, returns true and sets +// |*segment_start| to point to the start of the segment data, and +// and |*segment_size| to the size of the segment's data. If |elfclass| +// is not NULL, set |*elfclass| to the ELF file class. +bool FindElfSegment(const void* elf_mapped_base, uint32_t segment_type, + const void** segment_start, int* segment_size, + int* elfclass); + +// Convert an offset from an Elf header into a pointer to the mapped +// address in the current process. Takes an extra template parameter +// to specify the return type to avoid having to dynamic_cast the +// result. +template <typename ElfClass, typename T> +const T* GetOffset(const typename ElfClass::Ehdr* elf_header, + typename ElfClass::Off offset); + +// (derived from) +// file_id.h: Return a unique identifier for a file +// + +static const size_t kMDGUIDSize = sizeof(MDGUID); + +class FileID { + public: + // Load the identifier for the elf file mapped into memory at |base| into + // |identifier|. Return false if the identifier could not be created for the + // file. + static bool ElfFileIdentifierFromMappedFile(const void* base, + uint8_t identifier[kMDGUIDSize]); + + // Convert the |identifier| data to a NULL terminated string. The string will + // be formatted as a UUID (e.g., 22F065BB-FC9C-49F7-80FE-26A7CEBD7BCE). + // The |buffer| should be at least 37 bytes long to receive all of the data + // and termination. Shorter buffers will contain truncated data. + static void ConvertIdentifierToString(const uint8_t identifier[kMDGUIDSize], + char* buffer, int buffer_length); +}; + +template <typename ElfClass, typename T> +const T* GetOffset(const typename ElfClass::Ehdr* elf_header, + typename ElfClass::Off offset) { + return reinterpret_cast<const T*>(reinterpret_cast<uintptr_t>(elf_header) + + offset); +} + +template <typename ElfClass> +const typename ElfClass::Shdr* FindElfSectionByName( + const char* name, typename ElfClass::Word section_type, + const typename ElfClass::Shdr* sections, const char* section_names, + const char* names_end, int nsection) { + MOZ_ASSERT(name != NULL); + MOZ_ASSERT(sections != NULL); + MOZ_ASSERT(nsection > 0); + + int name_len = strlen(name); + if (name_len == 0) return NULL; + + for (int i = 0; i < nsection; ++i) { + const char* section_name = section_names + sections[i].sh_name; + if (sections[i].sh_type == section_type && + names_end - section_name >= name_len + 1 && + strcmp(name, section_name) == 0) { + return sections + i; + } + } + return NULL; +} + +} // namespace lul + +// And finally, the external interface, offered to LulMain.cpp +#include "LulElfExt.h" + +#endif // LulElfInt_h diff --git a/tools/profiler/lul/LulMain.cpp b/tools/profiler/lul/LulMain.cpp new file mode 100644 index 0000000000..7cf5508234 --- /dev/null +++ b/tools/profiler/lul/LulMain.cpp @@ -0,0 +1,2079 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "LulMain.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> // write(), only for testing LUL + +#include <algorithm> // std::sort +#include <string> +#include <utility> + +#include "GeckoProfiler.h" // for profiler_current_thread_id() +#include "LulCommonExt.h" +#include "LulElfExt.h" +#include "LulMainInt.h" +#include "mozilla/ArrayUtils.h" +#include "mozilla/Assertions.h" +#include "mozilla/CheckedInt.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/MemoryChecking.h" +#include "mozilla/Sprintf.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Unused.h" + +// Set this to 1 for verbose logging +#define DEBUG_MAIN 0 + +namespace lul { + +using mozilla::CheckedInt; +using mozilla::DebugOnly; +using mozilla::MallocSizeOf; +using mozilla::Unused; +using std::pair; +using std::string; +using std::vector; + +// WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING +// +// Some functions in this file are marked RUNS IN NO-MALLOC CONTEXT. +// Any such function -- and, hence, the transitive closure of those +// reachable from it -- must not do any dynamic memory allocation. +// Doing so risks deadlock. There is exactly one root function for +// the transitive closure: Lul::Unwind. +// +// WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING + +//////////////////////////////////////////////////////////////// +// RuleSet // +//////////////////////////////////////////////////////////////// + +static const char* NameOf_DW_REG(int16_t aReg) { + switch (aReg) { + case DW_REG_CFA: + return "cfa"; +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + case DW_REG_INTEL_XBP: + return "xbp"; + case DW_REG_INTEL_XSP: + return "xsp"; + case DW_REG_INTEL_XIP: + return "xip"; +#elif defined(GP_ARCH_arm) + case DW_REG_ARM_R7: + return "r7"; + case DW_REG_ARM_R11: + return "r11"; + case DW_REG_ARM_R12: + return "r12"; + case DW_REG_ARM_R13: + return "r13"; + case DW_REG_ARM_R14: + return "r14"; + case DW_REG_ARM_R15: + return "r15"; +#elif defined(GP_ARCH_arm64) + case DW_REG_AARCH64_X29: + return "x29"; + case DW_REG_AARCH64_X30: + return "x30"; + case DW_REG_AARCH64_SP: + return "sp"; +#elif defined(GP_ARCH_mips64) + case DW_REG_MIPS_SP: + return "sp"; + case DW_REG_MIPS_FP: + return "fp"; + case DW_REG_MIPS_PC: + return "pc"; +#else +# error "Unsupported arch" +#endif + default: + return "???"; + } +} + +string LExpr::ShowRule(const char* aNewReg) const { + char buf[64]; + string res = string(aNewReg) + "="; + switch (mHow) { + case UNKNOWN: + res += "Unknown"; + break; + case NODEREF: + SprintfLiteral(buf, "%s+%d", NameOf_DW_REG(mReg), (int)mOffset); + res += buf; + break; + case DEREF: + SprintfLiteral(buf, "*(%s+%d)", NameOf_DW_REG(mReg), (int)mOffset); + res += buf; + break; + case PFXEXPR: + SprintfLiteral(buf, "PfxExpr-at-%d", (int)mOffset); + res += buf; + break; + default: + res += "???"; + break; + } + return res; +} + +void RuleSet::Print(uintptr_t avma, uintptr_t len, + void (*aLog)(const char*)) const { + char buf[96]; + SprintfLiteral(buf, "[%llx .. %llx]: let ", (unsigned long long int)avma, + (unsigned long long int)(avma + len - 1)); + string res = string(buf); + res += mCfaExpr.ShowRule("cfa"); + res += " in"; + // For each reg we care about, print the recovery expression. +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + res += mXipExpr.ShowRule(" RA"); + res += mXspExpr.ShowRule(" SP"); + res += mXbpExpr.ShowRule(" BP"); +#elif defined(GP_ARCH_arm) + res += mR15expr.ShowRule(" R15"); + res += mR7expr.ShowRule(" R7"); + res += mR11expr.ShowRule(" R11"); + res += mR12expr.ShowRule(" R12"); + res += mR13expr.ShowRule(" R13"); + res += mR14expr.ShowRule(" R14"); +#elif defined(GP_ARCH_arm64) + res += mX29expr.ShowRule(" X29"); + res += mX30expr.ShowRule(" X30"); + res += mSPexpr.ShowRule(" SP"); +#elif defined(GP_ARCH_mips64) + res += mPCexpr.ShowRule(" PC"); + res += mSPexpr.ShowRule(" SP"); + res += mFPexpr.ShowRule(" FP"); +#else +# error "Unsupported arch" +#endif + aLog(res.c_str()); +} + +LExpr* RuleSet::ExprForRegno(DW_REG_NUMBER aRegno) { + switch (aRegno) { + case DW_REG_CFA: + return &mCfaExpr; +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + case DW_REG_INTEL_XIP: + return &mXipExpr; + case DW_REG_INTEL_XSP: + return &mXspExpr; + case DW_REG_INTEL_XBP: + return &mXbpExpr; +#elif defined(GP_ARCH_arm) + case DW_REG_ARM_R15: + return &mR15expr; + case DW_REG_ARM_R14: + return &mR14expr; + case DW_REG_ARM_R13: + return &mR13expr; + case DW_REG_ARM_R12: + return &mR12expr; + case DW_REG_ARM_R11: + return &mR11expr; + case DW_REG_ARM_R7: + return &mR7expr; +#elif defined(GP_ARCH_arm64) + case DW_REG_AARCH64_X29: + return &mX29expr; + case DW_REG_AARCH64_X30: + return &mX30expr; + case DW_REG_AARCH64_SP: + return &mSPexpr; +#elif defined(GP_ARCH_mips64) + case DW_REG_MIPS_SP: + return &mSPexpr; + case DW_REG_MIPS_FP: + return &mFPexpr; + case DW_REG_MIPS_PC: + return &mPCexpr; +#else +# error "Unknown arch" +#endif + default: + return nullptr; + } +} + +RuleSet::RuleSet() { + // All fields are of type LExpr and so are initialised by LExpr::LExpr(). +} + +//////////////////////////////////////////////////////////////// +// SecMap // +//////////////////////////////////////////////////////////////// + +// See header file LulMainInt.h for comments about invariants. + +SecMap::SecMap(uintptr_t mapStartAVMA, uint32_t mapLen, + void (*aLog)(const char*)) + : mUsable(false), + mUniqifier(new mozilla::HashMap<RuleSet, uint32_t, RuleSet, + InfallibleAllocPolicy>), + mLog(aLog) { + if (mapLen == 0) { + // Degenerate case. + mMapMinAVMA = 1; + mMapMaxAVMA = 0; + } else { + mMapMinAVMA = mapStartAVMA; + mMapMaxAVMA = mapStartAVMA + (uintptr_t)mapLen - 1; + } +} + +SecMap::~SecMap() { + mExtents.clear(); + mDictionary.clear(); + if (mUniqifier) { + mUniqifier->clear(); + mUniqifier = nullptr; + } +} + +// RUNS IN NO-MALLOC CONTEXT +RuleSet* SecMap::FindRuleSet(uintptr_t ia) { + // Binary search mExtents to find one that brackets |ia|. + // lo and hi need to be signed, else the loop termination tests + // don't work properly. Note that this works correctly even when + // mExtents.size() == 0. + + // Can't do this until the array has been sorted and preened. + MOZ_ASSERT(mUsable); + + long int lo = 0; + long int hi = (long int)mExtents.size() - 1; + while (true) { + // current unsearched space is from lo to hi, inclusive. + if (lo > hi) { + // not found + return nullptr; + } + long int mid = lo + ((hi - lo) / 2); + Extent* mid_extent = &mExtents[mid]; + uintptr_t mid_offset = mid_extent->offset(); + uintptr_t mid_len = mid_extent->len(); + uintptr_t mid_minAddr = mMapMinAVMA + mid_offset; + uintptr_t mid_maxAddr = mid_minAddr + mid_len - 1; + if (ia < mid_minAddr) { + hi = mid - 1; + continue; + } + if (ia > mid_maxAddr) { + lo = mid + 1; + continue; + } + MOZ_ASSERT(mid_minAddr <= ia && ia <= mid_maxAddr); + uint32_t mid_extent_dictIx = mid_extent->dictIx(); + MOZ_RELEASE_ASSERT(mid_extent_dictIx < mExtents.size()); + return &mDictionary[mid_extent_dictIx]; + } + // NOTREACHED +} + +// Add a RuleSet to the collection. The rule is copied in. Calling +// this makes the map non-searchable. +void SecMap::AddRuleSet(const RuleSet* rs, uintptr_t avma, uintptr_t len) { + mUsable = false; + + // Zero length RuleSet? Meaningless, but ignore it anyway. + if (len == 0) { + return; + } + + // Ignore attempts to add RuleSets whose address range doesn't fall within + // the declared address range for the SecMap. Maybe we should print some + // kind of error message rather than silently ignoring them. + if (!(avma >= mMapMinAVMA && avma + len - 1 <= mMapMaxAVMA)) { + return; + } + + // Because `mMapStartAVMA` .. `mMapEndAVMA` can specify at most a 2^32-1 byte + // chunk of address space, the following must now hold. + MOZ_RELEASE_ASSERT(len <= (uintptr_t)0xFFFFFFFF); + + // See if `mUniqifier` already has `rs`. If so set `dictIx` to the assigned + // dictionary index; if not, add `rs` to `mUniqifier` and assign a new + // dictionary index. This is the core of the RuleSet-de-duplication process. + uint32_t dictIx = 0; + mozilla::HashMap<RuleSet, uint32_t, RuleSet, InfallibleAllocPolicy>::AddPtr + p = mUniqifier->lookupForAdd(*rs); + if (!p) { + dictIx = mUniqifier->count(); + // If this ever fails, Extents::dictIx will need to be changed to be a + // type wider than the current uint16_t. + MOZ_RELEASE_ASSERT(dictIx < (1 << 16)); + // This returns `false` on OOM. We ignore the return value since we asked + // for it to use the InfallibleAllocPolicy. + DebugOnly<bool> addedOK = mUniqifier->add(p, *rs, dictIx); + MOZ_ASSERT(addedOK); + } else { + dictIx = p->value(); + } + + uint32_t offset = (uint32_t)(avma - mMapMinAVMA); + while (len > 0) { + // Because Extents::len is a uint16_t, we have to add multiple `mExtents` + // entries to cover the case where `len` is equal to or greater than 2^16. + // This happens only exceedingly rarely. In order to get more test + // coverage on what would otherwise be a very low probability (less than + // 0.0002%) corner case, we do this in steps of 4095. On libxul.so as of + // Sept 2020, this increases the number of `mExtents` entries by about + // 0.05%, hence has no meaningful effect on space use, but increases the + // use of this corner case, and hence its test coverage, by a factor of 250. + uint32_t this_step_len = (len > 4095) ? 4095 : len; + mExtents.emplace_back(offset, this_step_len, dictIx); + offset += this_step_len; + len -= this_step_len; + } +} + +// Add a PfxInstr to the vector of such instrs, and return the index +// in the vector. Calling this makes the map non-searchable. +uint32_t SecMap::AddPfxInstr(PfxInstr pfxi) { + mUsable = false; + mPfxInstrs.push_back(pfxi); + return mPfxInstrs.size() - 1; +} + +// Prepare the map for searching, by sorting it, de-overlapping entries and +// removing any resulting zero-length entries. At the start of this routine, +// all Extents should fall within [mMapMinAVMA, mMapMaxAVMA] and not have zero +// length, as a result of the checks in AddRuleSet(). +void SecMap::PrepareRuleSets() { + // At this point, the de-duped RuleSets are in `mUniqifier`, and + // `mDictionary` is empty. This method will, amongst other things, copy + // them into `mDictionary` in order of their assigned dictionary-index + // values, as established by `SecMap::AddRuleSet`, and free `mUniqifier`; + // after this method, it has no further use. + MOZ_RELEASE_ASSERT(mUniqifier); + MOZ_RELEASE_ASSERT(mDictionary.empty()); + + if (mExtents.empty()) { + mUniqifier->clear(); + mUniqifier = nullptr; + return; + } + + if (mMapMinAVMA == 1 && mMapMaxAVMA == 0) { + // The map is empty. This should never happen. + mExtents.clear(); + mUniqifier->clear(); + mUniqifier = nullptr; + return; + } + MOZ_RELEASE_ASSERT(mMapMinAVMA <= mMapMaxAVMA); + + // We must have at least one Extent, and as a consequence there must be at + // least one entry in the uniqifier. + MOZ_RELEASE_ASSERT(!mExtents.empty() && !mUniqifier->empty()); + +#ifdef DEBUG + // Check invariants on incoming Extents. + for (size_t i = 0; i < mExtents.size(); ++i) { + Extent* ext = &mExtents[i]; + uint32_t len = ext->len(); + MOZ_ASSERT(len > 0); + MOZ_ASSERT(len <= 4095 /* per '4095' in AddRuleSet() */); + uint32_t offset = ext->offset(); + uintptr_t avma = mMapMinAVMA + (uintptr_t)offset; + // Upper bounds test. There's no lower bounds test because `offset` is a + // positive displacement from `mMapMinAVMA`, so a small underrun will + // manifest as `len` being close to 2^32. + MOZ_ASSERT(avma + (uintptr_t)len - 1 <= mMapMaxAVMA); + } +#endif + + // Sort by start addresses. + std::sort(mExtents.begin(), mExtents.end(), + [](const Extent& ext1, const Extent& ext2) { + return ext1.offset() < ext2.offset(); + }); + + // Iteratively truncate any overlaps and remove any zero length + // entries that might result, or that may have been present + // initially. Unless the input is seriously screwy, this is + // expected to iterate only once. + while (true) { + size_t i; + size_t n = mExtents.size(); + size_t nZeroLen = 0; + + if (n == 0) { + break; + } + + for (i = 1; i < n; ++i) { + Extent* prev = &mExtents[i - 1]; + Extent* here = &mExtents[i]; + MOZ_ASSERT(prev->offset() <= here->offset()); + if (prev->offset() + prev->len() > here->offset()) { + prev->setLen(here->offset() - prev->offset()); + } + if (prev->len() == 0) { + nZeroLen++; + } + } + + if (mExtents[n - 1].len() == 0) { + nZeroLen++; + } + + // At this point, the entries are in-order and non-overlapping. + // If none of them are zero-length, we are done. + if (nZeroLen == 0) { + break; + } + + // Slide back the entries to remove the zero length ones. + size_t j = 0; // The write-point. + for (i = 0; i < n; ++i) { + if (mExtents[i].len() == 0) { + continue; + } + if (j != i) { + mExtents[j] = mExtents[i]; + } + ++j; + } + MOZ_ASSERT(i == n); + MOZ_ASSERT(nZeroLen <= n); + MOZ_ASSERT(j == n - nZeroLen); + while (nZeroLen > 0) { + mExtents.pop_back(); + nZeroLen--; + } + + MOZ_ASSERT(mExtents.size() == j); + } + + size_t nExtents = mExtents.size(); + +#ifdef DEBUG + // Do a final check on the extents: their address ranges must be + // ascending, non overlapping, non zero sized. + if (nExtents > 0) { + MOZ_ASSERT(mExtents[0].len() > 0); + for (size_t i = 1; i < nExtents; ++i) { + const Extent* prev = &mExtents[i - 1]; + const Extent* here = &mExtents[i]; + MOZ_ASSERT(prev->offset() < here->offset()); + MOZ_ASSERT(here->len() > 0); + MOZ_ASSERT(prev->offset() + prev->len() <= here->offset()); + } + } +#endif + + // Create the final dictionary by enumerating the uniqifier. + size_t nUniques = mUniqifier->count(); + + RuleSet dummy; + mozilla::PodZero(&dummy); + + mDictionary.reserve(nUniques); + for (size_t i = 0; i < nUniques; i++) { + mDictionary.push_back(dummy); + } + + for (auto iter = mUniqifier->iter(); !iter.done(); iter.next()) { + MOZ_RELEASE_ASSERT(iter.get().value() < nUniques); + mDictionary[iter.get().value()] = iter.get().key(); + } + + mUniqifier = nullptr; + + char buf[150]; + SprintfLiteral( + buf, + "PrepareRuleSets: %lu extents, %lu rulesets, " + "avma min/max 0x%llx, 0x%llx\n", + (unsigned long int)nExtents, (unsigned long int)mDictionary.size(), + (unsigned long long int)mMapMinAVMA, (unsigned long long int)mMapMaxAVMA); + buf[sizeof(buf) - 1] = 0; + mLog(buf); + + // Is now usable for binary search. + mUsable = true; + +#if 0 + mLog("\nRulesets after preening\n"); + for (size_t i = 0; i < nExtents; ++i) { + const Extent* extent = &mExtents[i]; + uintptr_t avma = mMapMinAVMA + (uintptr_t)extent->offset(); + mDictionary[extent->dictIx()].Print(avma, extent->len(), mLog); + mLog("\n"); + } + mLog("\n"); +#endif +} + +bool SecMap::IsEmpty() { return mExtents.empty(); } + +size_t SecMap::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { + size_t n = aMallocSizeOf(this); + + // It's conceivable that these calls would be unsafe with some + // implementations of std::vector, but it seems to be working for now... + n += aMallocSizeOf(mPfxInstrs.data()); + + if (mUniqifier) { + n += mUniqifier->shallowSizeOfIncludingThis(aMallocSizeOf); + } + n += aMallocSizeOf(mDictionary.data()); + n += aMallocSizeOf(mExtents.data()); + + return n; +} + +//////////////////////////////////////////////////////////////// +// SegArray // +//////////////////////////////////////////////////////////////// + +// A SegArray holds a set of address ranges that together exactly +// cover an address range, with no overlaps or holes. Each range has +// an associated value, which in this case has been specialised to be +// a simple boolean. The representation is kept to minimal canonical +// form in which adjacent ranges with the same associated value are +// merged together. Each range is represented by a |struct Seg|. +// +// SegArrays are used to keep track of which parts of the address +// space are known to contain instructions. +class SegArray { + public: + void add(uintptr_t lo, uintptr_t hi, bool val) { + if (lo > hi) { + return; + } + split_at(lo); + if (hi < UINTPTR_MAX) { + split_at(hi + 1); + } + std::vector<Seg>::size_type iLo, iHi, i; + iLo = find(lo); + iHi = find(hi); + for (i = iLo; i <= iHi; ++i) { + mSegs[i].val = val; + } + preen(); + } + + // RUNS IN NO-MALLOC CONTEXT + bool getBoundingCodeSegment(/*OUT*/ uintptr_t* rx_min, + /*OUT*/ uintptr_t* rx_max, uintptr_t addr) { + std::vector<Seg>::size_type i = find(addr); + if (!mSegs[i].val) { + return false; + } + *rx_min = mSegs[i].lo; + *rx_max = mSegs[i].hi; + return true; + } + + SegArray() { + Seg s(0, UINTPTR_MAX, false); + mSegs.push_back(s); + } + + private: + struct Seg { + Seg(uintptr_t lo, uintptr_t hi, bool val) : lo(lo), hi(hi), val(val) {} + uintptr_t lo; + uintptr_t hi; + bool val; + }; + + void preen() { + for (std::vector<Seg>::iterator iter = mSegs.begin(); + iter < mSegs.end() - 1; ++iter) { + if (iter[0].val != iter[1].val) { + continue; + } + iter[0].hi = iter[1].hi; + mSegs.erase(iter + 1); + // Back up one, so as not to miss an opportunity to merge + // with the entry after this one. + --iter; + } + } + + // RUNS IN NO-MALLOC CONTEXT + std::vector<Seg>::size_type find(uintptr_t a) { + long int lo = 0; + long int hi = (long int)mSegs.size(); + while (true) { + // The unsearched space is lo .. hi inclusive. + if (lo > hi) { + // Not found. This can't happen. + return (std::vector<Seg>::size_type)(-1); + } + long int mid = lo + ((hi - lo) / 2); + uintptr_t mid_lo = mSegs[mid].lo; + uintptr_t mid_hi = mSegs[mid].hi; + if (a < mid_lo) { + hi = mid - 1; + continue; + } + if (a > mid_hi) { + lo = mid + 1; + continue; + } + return (std::vector<Seg>::size_type)mid; + } + } + + void split_at(uintptr_t a) { + std::vector<Seg>::size_type i = find(a); + if (mSegs[i].lo == a) { + return; + } + mSegs.insert(mSegs.begin() + i + 1, mSegs[i]); + mSegs[i].hi = a - 1; + mSegs[i + 1].lo = a; + } + + void show() { + printf("<< %d entries:\n", (int)mSegs.size()); + for (std::vector<Seg>::iterator iter = mSegs.begin(); iter < mSegs.end(); + ++iter) { + printf(" %016llx %016llx %s\n", (unsigned long long int)(*iter).lo, + (unsigned long long int)(*iter).hi, + (*iter).val ? "true" : "false"); + } + printf(">>\n"); + } + + std::vector<Seg> mSegs; +}; + +//////////////////////////////////////////////////////////////// +// PriMap // +//////////////////////////////////////////////////////////////// + +class PriMap { + public: + explicit PriMap(void (*aLog)(const char*)) : mLog(aLog) {} + + // RUNS IN NO-MALLOC CONTEXT + pair<const RuleSet*, const vector<PfxInstr>*> Lookup(uintptr_t ia) { + SecMap* sm = FindSecMap(ia); + return pair<const RuleSet*, const vector<PfxInstr>*>( + sm ? sm->FindRuleSet(ia) : nullptr, sm ? sm->GetPfxInstrs() : nullptr); + } + + // Add a secondary map. No overlaps allowed w.r.t. existing + // secondary maps. + void AddSecMap(mozilla::UniquePtr<SecMap>&& aSecMap) { + // We can't add an empty SecMap to the PriMap. But that's OK + // since we'd never be able to find anything in it anyway. + if (aSecMap->IsEmpty()) { + return; + } + + // Iterate through the SecMaps and find the right place for this + // one. At the same time, ensure that the in-order + // non-overlapping invariant is preserved (and, generally, holds). + // FIXME: this gives a cost that is O(N^2) in the total number of + // shared objects in the system. ToDo: better. + MOZ_ASSERT(aSecMap->mMapMinAVMA <= aSecMap->mMapMaxAVMA); + + size_t num_secMaps = mSecMaps.size(); + uintptr_t i; + for (i = 0; i < num_secMaps; ++i) { + mozilla::UniquePtr<SecMap>& sm_i = mSecMaps[i]; + MOZ_ASSERT(sm_i->mMapMinAVMA <= sm_i->mMapMaxAVMA); + if (aSecMap->mMapMinAVMA < sm_i->mMapMaxAVMA) { + // |aSecMap| needs to be inserted immediately before mSecMaps[i]. + break; + } + } + MOZ_ASSERT(i <= num_secMaps); + if (i == num_secMaps) { + // It goes at the end. + mSecMaps.push_back(std::move(aSecMap)); + } else { + std::vector<mozilla::UniquePtr<SecMap>>::iterator iter = + mSecMaps.begin() + i; + mSecMaps.insert(iter, std::move(aSecMap)); + } + char buf[100]; + SprintfLiteral(buf, "AddSecMap: now have %d SecMaps\n", + (int)mSecMaps.size()); + buf[sizeof(buf) - 1] = 0; + mLog(buf); + } + + // Remove and delete any SecMaps in the mapping, that intersect + // with the specified address range. + void RemoveSecMapsInRange(uintptr_t avma_min, uintptr_t avma_max) { + MOZ_ASSERT(avma_min <= avma_max); + size_t num_secMaps = mSecMaps.size(); + if (num_secMaps > 0) { + intptr_t i; + // Iterate from end to start over the vector, so as to ensure + // that the special case where |avma_min| and |avma_max| denote + // the entire address space, can be completed in time proportional + // to the number of elements in the map. + for (i = (intptr_t)num_secMaps - 1; i >= 0; i--) { + mozilla::UniquePtr<SecMap>& sm_i = mSecMaps[i]; + if (sm_i->mMapMaxAVMA < avma_min || avma_max < sm_i->mMapMinAVMA) { + // There's no overlap. Move on. + continue; + } + // We need to remove mSecMaps[i] and slide all those above it + // downwards to cover the hole. + mSecMaps.erase(mSecMaps.begin() + i); + } + } + } + + // Return the number of currently contained SecMaps. + size_t CountSecMaps() { return mSecMaps.size(); } + + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { + size_t n = aMallocSizeOf(this); + + // It's conceivable that this call would be unsafe with some + // implementations of std::vector, but it seems to be working for now... + n += aMallocSizeOf(mSecMaps.data()); + + for (size_t i = 0; i < mSecMaps.size(); i++) { + n += mSecMaps[i]->SizeOfIncludingThis(aMallocSizeOf); + } + + return n; + } + + private: + // RUNS IN NO-MALLOC CONTEXT + SecMap* FindSecMap(uintptr_t ia) { + // Binary search mSecMaps to find one that brackets |ia|. + // lo and hi need to be signed, else the loop termination tests + // don't work properly. + long int lo = 0; + long int hi = (long int)mSecMaps.size() - 1; + while (true) { + // current unsearched space is from lo to hi, inclusive. + if (lo > hi) { + // not found + return nullptr; + } + long int mid = lo + ((hi - lo) / 2); + mozilla::UniquePtr<SecMap>& mid_secMap = mSecMaps[mid]; + uintptr_t mid_minAddr = mid_secMap->mMapMinAVMA; + uintptr_t mid_maxAddr = mid_secMap->mMapMaxAVMA; + if (ia < mid_minAddr) { + hi = mid - 1; + continue; + } + if (ia > mid_maxAddr) { + lo = mid + 1; + continue; + } + MOZ_ASSERT(mid_minAddr <= ia && ia <= mid_maxAddr); + return mid_secMap.get(); + } + // NOTREACHED + } + + private: + // sorted array of per-object ranges, non overlapping, non empty + std::vector<mozilla::UniquePtr<SecMap>> mSecMaps; + + // a logging sink, for debugging. + void (*mLog)(const char*); +}; + +//////////////////////////////////////////////////////////////// +// LUL // +//////////////////////////////////////////////////////////////// + +#define LUL_LOG(_str) \ + do { \ + char buf[200]; \ + SprintfLiteral(buf, "LUL: pid %" PRIu64 " tid %" PRIu64 " lul-obj %p: %s", \ + uint64_t(profiler_current_process_id().ToNumber()), \ + uint64_t(profiler_current_thread_id().ToNumber()), this, \ + (_str)); \ + buf[sizeof(buf) - 1] = 0; \ + mLog(buf); \ + } while (0) + +LUL::LUL(void (*aLog)(const char*)) + : mLog(aLog), + mAdminMode(true), + mAdminThreadId(profiler_current_thread_id()), + mPriMap(new PriMap(aLog)), + mSegArray(new SegArray()), + mUSU(new UniqueStringUniverse()) { + LUL_LOG("LUL::LUL: Created object"); +} + +LUL::~LUL() { + LUL_LOG("LUL::~LUL: Destroyed object"); + delete mPriMap; + delete mSegArray; + mLog = nullptr; + delete mUSU; +} + +void LUL::MaybeShowStats() { + // This is racey in the sense that it can't guarantee that + // n_new == n_new_Context + n_new_CFI + n_new_Scanned + // if it should happen that mStats is updated by some other thread + // in between computation of n_new and n_new_{Context,CFI,FP}. + // But it's just stats printing, so we don't really care. + uint32_t n_new = mStats - mStatsPrevious; + if (n_new >= 5000) { + uint32_t n_new_Context = mStats.mContext - mStatsPrevious.mContext; + uint32_t n_new_CFI = mStats.mCFI - mStatsPrevious.mCFI; + uint32_t n_new_FP = mStats.mFP - mStatsPrevious.mFP; + mStatsPrevious = mStats; + char buf[200]; + SprintfLiteral(buf, + "LUL frame stats: TOTAL %5u" + " CTX %4u CFI %4u FP %4u", + n_new, n_new_Context, n_new_CFI, n_new_FP); + buf[sizeof(buf) - 1] = 0; + mLog(buf); + } +} + +size_t LUL::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { + size_t n = aMallocSizeOf(this); + n += mPriMap->SizeOfIncludingThis(aMallocSizeOf); + + // Measurement of the following members may be added later if DMD finds it + // is worthwhile: + // - mSegArray + // - mUSU + + return n; +} + +void LUL::EnableUnwinding() { + LUL_LOG("LUL::EnableUnwinding"); + // Don't assert for Admin mode here. That is, tolerate a call here + // if we are already in Unwinding mode. + MOZ_RELEASE_ASSERT(profiler_current_thread_id() == mAdminThreadId); + + mAdminMode = false; +} + +void LUL::NotifyAfterMap(uintptr_t aRXavma, size_t aSize, const char* aFileName, + const void* aMappedImage) { + MOZ_RELEASE_ASSERT(mAdminMode); + MOZ_RELEASE_ASSERT(profiler_current_thread_id() == mAdminThreadId); + + mLog(":\n"); + char buf[200]; + SprintfLiteral(buf, "NotifyMap %llx %llu %s\n", + (unsigned long long int)aRXavma, (unsigned long long int)aSize, + aFileName); + buf[sizeof(buf) - 1] = 0; + mLog(buf); + + // We can't have a SecMap covering more than 2^32-1 bytes of address space. + // See the definition of SecMap for why. Rather than crash the system, just + // limit the SecMap's size accordingly. This case is never actually + // expected to happen. + if (((unsigned long long int)aSize) > 0xFFFFFFFFULL) { + aSize = (uintptr_t)0xFFFFFFFF; + } + MOZ_RELEASE_ASSERT(aSize <= 0xFFFFFFFF); + + // Ignore obviously-stupid notifications. + if (aSize > 0) { + // Here's a new mapping, for this object. + mozilla::UniquePtr<SecMap> smap = + mozilla::MakeUnique<SecMap>(aRXavma, (uint32_t)aSize, mLog); + + // Read CFI or EXIDX unwind data into |smap|. + if (!aMappedImage) { + (void)lul::ReadSymbolData(string(aFileName), std::vector<string>(), + smap.get(), (void*)aRXavma, aSize, mUSU, mLog); + } else { + (void)lul::ReadSymbolDataInternal( + (const uint8_t*)aMappedImage, string(aFileName), + std::vector<string>(), smap.get(), (void*)aRXavma, aSize, mUSU, mLog); + } + + mLog("NotifyMap .. preparing entries\n"); + + smap->PrepareRuleSets(); + + SprintfLiteral(buf, "NotifyMap got %lld entries\n", + (long long int)smap->Size()); + buf[sizeof(buf) - 1] = 0; + mLog(buf); + + // Add it to the primary map (the top level set of mapped objects). + mPriMap->AddSecMap(std::move(smap)); + + // Tell the segment array about the mapping, so that the stack + // scan and __kernel_syscall mechanisms know where valid code is. + mSegArray->add(aRXavma, aRXavma + aSize - 1, true); + } +} + +void LUL::NotifyExecutableArea(uintptr_t aRXavma, size_t aSize) { + MOZ_RELEASE_ASSERT(mAdminMode); + MOZ_RELEASE_ASSERT(profiler_current_thread_id() == mAdminThreadId); + + mLog(":\n"); + char buf[200]; + SprintfLiteral(buf, "NotifyExecutableArea %llx %llu\n", + (unsigned long long int)aRXavma, + (unsigned long long int)aSize); + buf[sizeof(buf) - 1] = 0; + mLog(buf); + + // Ignore obviously-stupid notifications. + if (aSize > 0) { + // Tell the segment array about the mapping, so that the stack + // scan and __kernel_syscall mechanisms know where valid code is. + mSegArray->add(aRXavma, aRXavma + aSize - 1, true); + } +} + +void LUL::NotifyBeforeUnmap(uintptr_t aRXavmaMin, uintptr_t aRXavmaMax) { + MOZ_RELEASE_ASSERT(mAdminMode); + MOZ_RELEASE_ASSERT(profiler_current_thread_id() == mAdminThreadId); + + mLog(":\n"); + char buf[100]; + SprintfLiteral(buf, "NotifyUnmap %016llx-%016llx\n", + (unsigned long long int)aRXavmaMin, + (unsigned long long int)aRXavmaMax); + buf[sizeof(buf) - 1] = 0; + mLog(buf); + + MOZ_ASSERT(aRXavmaMin <= aRXavmaMax); + + // Remove from the primary map, any secondary maps that intersect + // with the address range. Also delete the secondary maps. + mPriMap->RemoveSecMapsInRange(aRXavmaMin, aRXavmaMax); + + // Tell the segment array that the address range no longer + // contains valid code. + mSegArray->add(aRXavmaMin, aRXavmaMax, false); + + SprintfLiteral(buf, "NotifyUnmap: now have %d SecMaps\n", + (int)mPriMap->CountSecMaps()); + buf[sizeof(buf) - 1] = 0; + mLog(buf); +} + +size_t LUL::CountMappings() { + MOZ_RELEASE_ASSERT(mAdminMode); + MOZ_RELEASE_ASSERT(profiler_current_thread_id() == mAdminThreadId); + + return mPriMap->CountSecMaps(); +} + +// RUNS IN NO-MALLOC CONTEXT +static TaggedUWord DerefTUW(TaggedUWord aAddr, const StackImage* aStackImg) { + if (!aAddr.Valid()) { + return TaggedUWord(); + } + + // Lower limit check. |aAddr.Value()| is the lowest requested address + // and |aStackImg->mStartAvma| is the lowest address we actually have, + // so the comparison is straightforward. + if (aAddr.Value() < aStackImg->mStartAvma) { + return TaggedUWord(); + } + + // Upper limit check. We must compute the highest requested address + // and the highest address we actually have, but being careful to + // avoid overflow. In particular if |aAddr| is 0xFFF...FFF or the + // 3/7 values below that, then we will get overflow. See bug #1245477. + typedef CheckedInt<uintptr_t> CheckedUWord; + CheckedUWord highest_requested_plus_one = + CheckedUWord(aAddr.Value()) + CheckedUWord(sizeof(uintptr_t)); + CheckedUWord highest_available_plus_one = + CheckedUWord(aStackImg->mStartAvma) + CheckedUWord(aStackImg->mLen); + if (!highest_requested_plus_one.isValid() // overflow? + || !highest_available_plus_one.isValid() // overflow? + || (highest_requested_plus_one.value() > + highest_available_plus_one.value())) { // in range? + return TaggedUWord(); + } + + return TaggedUWord( + *(uintptr_t*)(&aStackImg + ->mContents[aAddr.Value() - aStackImg->mStartAvma])); +} + +// RUNS IN NO-MALLOC CONTEXT +static TaggedUWord EvaluateReg(int16_t aReg, const UnwindRegs* aOldRegs, + TaggedUWord aCFA) { + switch (aReg) { + case DW_REG_CFA: + return aCFA; +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + case DW_REG_INTEL_XBP: + return aOldRegs->xbp; + case DW_REG_INTEL_XSP: + return aOldRegs->xsp; + case DW_REG_INTEL_XIP: + return aOldRegs->xip; +#elif defined(GP_ARCH_arm) + case DW_REG_ARM_R7: + return aOldRegs->r7; + case DW_REG_ARM_R11: + return aOldRegs->r11; + case DW_REG_ARM_R12: + return aOldRegs->r12; + case DW_REG_ARM_R13: + return aOldRegs->r13; + case DW_REG_ARM_R14: + return aOldRegs->r14; + case DW_REG_ARM_R15: + return aOldRegs->r15; +#elif defined(GP_ARCH_arm64) + case DW_REG_AARCH64_X29: + return aOldRegs->x29; + case DW_REG_AARCH64_X30: + return aOldRegs->x30; + case DW_REG_AARCH64_SP: + return aOldRegs->sp; +#elif defined(GP_ARCH_mips64) + case DW_REG_MIPS_SP: + return aOldRegs->sp; + case DW_REG_MIPS_FP: + return aOldRegs->fp; + case DW_REG_MIPS_PC: + return aOldRegs->pc; +#else +# error "Unsupported arch" +#endif + default: + MOZ_ASSERT(0); + return TaggedUWord(); + } +} + +// RUNS IN NO-MALLOC CONTEXT +// See prototype for comment. +TaggedUWord EvaluatePfxExpr(int32_t start, const UnwindRegs* aOldRegs, + TaggedUWord aCFA, const StackImage* aStackImg, + const vector<PfxInstr>& aPfxInstrs) { + // A small evaluation stack, and a stack pointer, which points to + // the highest numbered in-use element. + const int N_STACK = 10; + TaggedUWord stack[N_STACK]; + int stackPointer = -1; + for (int i = 0; i < N_STACK; i++) stack[i] = TaggedUWord(); + +#define PUSH(_tuw) \ + do { \ + if (stackPointer >= N_STACK - 1) goto fail; /* overflow */ \ + stack[++stackPointer] = (_tuw); \ + } while (0) + +#define POP(_lval) \ + do { \ + if (stackPointer < 0) goto fail; /* underflow */ \ + _lval = stack[stackPointer--]; \ + } while (0) + + // Cursor in the instruction sequence. + size_t curr = start + 1; + + // Check the start point is sane. + size_t nInstrs = aPfxInstrs.size(); + if (start < 0 || (size_t)start >= nInstrs) goto fail; + + { + // The instruction sequence must start with PX_Start. If not, + // something is seriously wrong. + PfxInstr first = aPfxInstrs[start]; + if (first.mOpcode != PX_Start) goto fail; + + // Push the CFA on the stack to start with (or not), as required by + // the original DW_OP_*expression* CFI. + if (first.mOperand != 0) PUSH(aCFA); + } + + while (true) { + if (curr >= nInstrs) goto fail; // ran off the end of the sequence + + PfxInstr pfxi = aPfxInstrs[curr++]; + if (pfxi.mOpcode == PX_End) break; // we're done + + switch (pfxi.mOpcode) { + case PX_Start: + // This should appear only at the start of the sequence. + goto fail; + case PX_End: + // We just took care of that, so we shouldn't see it again. + MOZ_ASSERT(0); + goto fail; + case PX_SImm32: + PUSH(TaggedUWord((intptr_t)pfxi.mOperand)); + break; + case PX_DwReg: { + DW_REG_NUMBER reg = (DW_REG_NUMBER)pfxi.mOperand; + MOZ_ASSERT(reg != DW_REG_CFA); + PUSH(EvaluateReg(reg, aOldRegs, aCFA)); + break; + } + case PX_Deref: { + TaggedUWord addr; + POP(addr); + PUSH(DerefTUW(addr, aStackImg)); + break; + } + case PX_Add: { + TaggedUWord x, y; + POP(x); + POP(y); + PUSH(y + x); + break; + } + case PX_Sub: { + TaggedUWord x, y; + POP(x); + POP(y); + PUSH(y - x); + break; + } + case PX_And: { + TaggedUWord x, y; + POP(x); + POP(y); + PUSH(y & x); + break; + } + case PX_Or: { + TaggedUWord x, y; + POP(x); + POP(y); + PUSH(y | x); + break; + } + case PX_CmpGES: { + TaggedUWord x, y; + POP(x); + POP(y); + PUSH(y.CmpGEs(x)); + break; + } + case PX_Shl: { + TaggedUWord x, y; + POP(x); + POP(y); + PUSH(y << x); + break; + } + default: + MOZ_ASSERT(0); + goto fail; + } + } // while (true) + + // Evaluation finished. The top value on the stack is the result. + if (stackPointer >= 0) { + return stack[stackPointer]; + } + // Else fall through + +fail: + return TaggedUWord(); + +#undef PUSH +#undef POP +} + +// RUNS IN NO-MALLOC CONTEXT +TaggedUWord LExpr::EvaluateExpr(const UnwindRegs* aOldRegs, TaggedUWord aCFA, + const StackImage* aStackImg, + const vector<PfxInstr>* aPfxInstrs) const { + switch (mHow) { + case UNKNOWN: + return TaggedUWord(); + case NODEREF: { + TaggedUWord tuw = EvaluateReg(mReg, aOldRegs, aCFA); + tuw = tuw + TaggedUWord((intptr_t)mOffset); + return tuw; + } + case DEREF: { + TaggedUWord tuw = EvaluateReg(mReg, aOldRegs, aCFA); + tuw = tuw + TaggedUWord((intptr_t)mOffset); + return DerefTUW(tuw, aStackImg); + } + case PFXEXPR: { + MOZ_ASSERT(aPfxInstrs); + if (!aPfxInstrs) { + return TaggedUWord(); + } + return EvaluatePfxExpr(mOffset, aOldRegs, aCFA, aStackImg, *aPfxInstrs); + } + default: + MOZ_ASSERT(0); + return TaggedUWord(); + } +} + +// RUNS IN NO-MALLOC CONTEXT +static void UseRuleSet(/*MOD*/ UnwindRegs* aRegs, const StackImage* aStackImg, + const RuleSet* aRS, const vector<PfxInstr>* aPfxInstrs) { + // Take a copy of regs, since we'll need to refer to the old values + // whilst computing the new ones. + UnwindRegs old_regs = *aRegs; + + // Mark all the current register values as invalid, so that the + // caller can see, on our return, which ones have been computed + // anew. If we don't even manage to compute a new PC value, then + // the caller will have to abandon the unwind. + // FIXME: Create and use instead: aRegs->SetAllInvalid(); +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + aRegs->xbp = TaggedUWord(); + aRegs->xsp = TaggedUWord(); + aRegs->xip = TaggedUWord(); +#elif defined(GP_ARCH_arm) + aRegs->r7 = TaggedUWord(); + aRegs->r11 = TaggedUWord(); + aRegs->r12 = TaggedUWord(); + aRegs->r13 = TaggedUWord(); + aRegs->r14 = TaggedUWord(); + aRegs->r15 = TaggedUWord(); +#elif defined(GP_ARCH_arm64) + aRegs->x29 = TaggedUWord(); + aRegs->x30 = TaggedUWord(); + aRegs->sp = TaggedUWord(); + aRegs->pc = TaggedUWord(); +#elif defined(GP_ARCH_mips64) + aRegs->sp = TaggedUWord(); + aRegs->fp = TaggedUWord(); + aRegs->pc = TaggedUWord(); +#else +# error "Unsupported arch" +#endif + + // This is generally useful. + const TaggedUWord inval = TaggedUWord(); + + // First, compute the CFA. + TaggedUWord cfa = aRS->mCfaExpr.EvaluateExpr(&old_regs, inval /*old cfa*/, + aStackImg, aPfxInstrs); + + // If we didn't manage to compute the CFA, well .. that's ungood, + // but keep going anyway. It'll be OK provided none of the register + // value rules mention the CFA. In any case, compute the new values + // for each register that we're tracking. + +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + aRegs->xbp = + aRS->mXbpExpr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->xsp = + aRS->mXspExpr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->xip = + aRS->mXipExpr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); +#elif defined(GP_ARCH_arm) + aRegs->r7 = aRS->mR7expr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->r11 = + aRS->mR11expr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->r12 = + aRS->mR12expr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->r13 = + aRS->mR13expr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->r14 = + aRS->mR14expr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->r15 = + aRS->mR15expr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); +#elif defined(GP_ARCH_arm64) + aRegs->x29 = + aRS->mX29expr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->x30 = + aRS->mX30expr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->sp = aRS->mSPexpr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); +#elif defined(GP_ARCH_mips64) + aRegs->sp = aRS->mSPexpr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->fp = aRS->mFPexpr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); + aRegs->pc = aRS->mPCexpr.EvaluateExpr(&old_regs, cfa, aStackImg, aPfxInstrs); +#else +# error "Unsupported arch" +#endif + + // We're done. Any regs for which we didn't manage to compute a + // new value will now be marked as invalid. +} + +// RUNS IN NO-MALLOC CONTEXT +void LUL::Unwind(/*OUT*/ uintptr_t* aFramePCs, + /*OUT*/ uintptr_t* aFrameSPs, + /*OUT*/ size_t* aFramesUsed, + /*OUT*/ size_t* aFramePointerFramesAcquired, + size_t aFramesAvail, UnwindRegs* aStartRegs, + StackImage* aStackImg) { + MOZ_RELEASE_ASSERT(!mAdminMode); + + ///////////////////////////////////////////////////////// + // BEGIN UNWIND + + *aFramesUsed = 0; + + UnwindRegs regs = *aStartRegs; + TaggedUWord last_valid_sp = TaggedUWord(); + + while (true) { + if (DEBUG_MAIN) { + char buf[300]; + mLog("\n"); +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + SprintfLiteral( + buf, "LoopTop: rip %d/%llx rsp %d/%llx rbp %d/%llx\n", + (int)regs.xip.Valid(), (unsigned long long int)regs.xip.Value(), + (int)regs.xsp.Valid(), (unsigned long long int)regs.xsp.Value(), + (int)regs.xbp.Valid(), (unsigned long long int)regs.xbp.Value()); + buf[sizeof(buf) - 1] = 0; + mLog(buf); +#elif defined(GP_ARCH_arm) + SprintfLiteral( + buf, + "LoopTop: r15 %d/%llx r7 %d/%llx r11 %d/%llx" + " r12 %d/%llx r13 %d/%llx r14 %d/%llx\n", + (int)regs.r15.Valid(), (unsigned long long int)regs.r15.Value(), + (int)regs.r7.Valid(), (unsigned long long int)regs.r7.Value(), + (int)regs.r11.Valid(), (unsigned long long int)regs.r11.Value(), + (int)regs.r12.Valid(), (unsigned long long int)regs.r12.Value(), + (int)regs.r13.Valid(), (unsigned long long int)regs.r13.Value(), + (int)regs.r14.Valid(), (unsigned long long int)regs.r14.Value()); + buf[sizeof(buf) - 1] = 0; + mLog(buf); +#elif defined(GP_ARCH_arm64) + SprintfLiteral( + buf, + "LoopTop: pc %d/%llx x29 %d/%llx x30 %d/%llx" + " sp %d/%llx\n", + (int)regs.pc.Valid(), (unsigned long long int)regs.pc.Value(), + (int)regs.x29.Valid(), (unsigned long long int)regs.x29.Value(), + (int)regs.x30.Valid(), (unsigned long long int)regs.x30.Value(), + (int)regs.sp.Valid(), (unsigned long long int)regs.sp.Value()); + buf[sizeof(buf) - 1] = 0; + mLog(buf); +#elif defined(GP_ARCH_mips64) + SprintfLiteral( + buf, "LoopTop: pc %d/%llx sp %d/%llx fp %d/%llx\n", + (int)regs.pc.Valid(), (unsigned long long int)regs.pc.Value(), + (int)regs.sp.Valid(), (unsigned long long int)regs.sp.Value(), + (int)regs.fp.Valid(), (unsigned long long int)regs.fp.Value()); + buf[sizeof(buf) - 1] = 0; + mLog(buf); +#else +# error "Unsupported arch" +#endif + } + +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + TaggedUWord ia = regs.xip; + TaggedUWord sp = regs.xsp; +#elif defined(GP_ARCH_arm) + TaggedUWord ia = (*aFramesUsed == 0 ? regs.r15 : regs.r14); + TaggedUWord sp = regs.r13; +#elif defined(GP_ARCH_arm64) + TaggedUWord ia = (*aFramesUsed == 0 ? regs.pc : regs.x30); + TaggedUWord sp = regs.sp; +#elif defined(GP_ARCH_mips64) + TaggedUWord ia = regs.pc; + TaggedUWord sp = regs.sp; +#else +# error "Unsupported arch" +#endif + + if (*aFramesUsed >= aFramesAvail) { + break; + } + + // If we don't have a valid value for the PC, give up. + if (!ia.Valid()) { + break; + } + + // If this is the innermost frame, record the SP value, which + // presumably is valid. If this isn't the innermost frame, and we + // have a valid SP value, check that its SP value isn't less that + // the one we've seen so far, so as to catch potential SP value + // cycles. + if (*aFramesUsed == 0) { + last_valid_sp = sp; + } else { + MOZ_ASSERT(last_valid_sp.Valid()); + if (sp.Valid()) { + if (sp.Value() < last_valid_sp.Value()) { + // Hmm, SP going in the wrong direction. Let's stop. + break; + } + // Remember where we got to. + last_valid_sp = sp; + } + } + + aFramePCs[*aFramesUsed] = ia.Value(); + aFrameSPs[*aFramesUsed] = sp.Valid() ? sp.Value() : 0; + (*aFramesUsed)++; + + // Find the RuleSet for the current IA, if any. This will also + // query the backing (secondary) maps if it isn't found in the + // thread-local cache. + + // If this isn't the innermost frame, back up into the calling insn. + if (*aFramesUsed > 1) { + ia = ia + TaggedUWord((uintptr_t)(-1)); + } + + pair<const RuleSet*, const vector<PfxInstr>*> ruleset_and_pfxinstrs = + mPriMap->Lookup(ia.Value()); + const RuleSet* ruleset = ruleset_and_pfxinstrs.first; + const vector<PfxInstr>* pfxinstrs = ruleset_and_pfxinstrs.second; + + if (DEBUG_MAIN) { + char buf[100]; + SprintfLiteral(buf, "ruleset for 0x%llx = %p\n", + (unsigned long long int)ia.Value(), ruleset); + buf[sizeof(buf) - 1] = 0; + mLog(buf); + } + +#if defined(GP_PLAT_x86_android) || defined(GP_PLAT_x86_linux) + ///////////////////////////////////////////// + //// + // On 32 bit x86-linux, syscalls are often done via the VDSO + // function __kernel_vsyscall, which doesn't have a corresponding + // object that we can read debuginfo from. That effectively kills + // off all stack traces for threads blocked in syscalls. Hence + // special-case by looking at the code surrounding the program + // counter. + // + // 0xf7757420 <__kernel_vsyscall+0>: push %ecx + // 0xf7757421 <__kernel_vsyscall+1>: push %edx + // 0xf7757422 <__kernel_vsyscall+2>: push %ebp + // 0xf7757423 <__kernel_vsyscall+3>: mov %esp,%ebp + // 0xf7757425 <__kernel_vsyscall+5>: sysenter + // 0xf7757427 <__kernel_vsyscall+7>: nop + // 0xf7757428 <__kernel_vsyscall+8>: nop + // 0xf7757429 <__kernel_vsyscall+9>: nop + // 0xf775742a <__kernel_vsyscall+10>: nop + // 0xf775742b <__kernel_vsyscall+11>: nop + // 0xf775742c <__kernel_vsyscall+12>: nop + // 0xf775742d <__kernel_vsyscall+13>: nop + // 0xf775742e <__kernel_vsyscall+14>: int $0x80 + // 0xf7757430 <__kernel_vsyscall+16>: pop %ebp + // 0xf7757431 <__kernel_vsyscall+17>: pop %edx + // 0xf7757432 <__kernel_vsyscall+18>: pop %ecx + // 0xf7757433 <__kernel_vsyscall+19>: ret + // + // In cases where the sampled thread is blocked in a syscall, its + // program counter will point at "pop %ebp". Hence we look for + // the sequence "int $0x80; pop %ebp; pop %edx; pop %ecx; ret", and + // the corresponding register-recovery actions are: + // new_ebp = *(old_esp + 0) + // new eip = *(old_esp + 12) + // new_esp = old_esp + 16 + // + // It may also be the case that the program counter points two + // nops before the "int $0x80", viz, is __kernel_vsyscall+12, in + // the case where the syscall has been restarted but the thread + // hasn't been rescheduled. The code below doesn't handle that; + // it could easily be made to. + // + if (!ruleset && *aFramesUsed == 1 && ia.Valid() && sp.Valid()) { + uintptr_t insns_min, insns_max; + uintptr_t eip = ia.Value(); + bool b = mSegArray->getBoundingCodeSegment(&insns_min, &insns_max, eip); + if (b && eip - 2 >= insns_min && eip + 3 <= insns_max) { + uint8_t* eipC = (uint8_t*)eip; + if (eipC[-2] == 0xCD && eipC[-1] == 0x80 && eipC[0] == 0x5D && + eipC[1] == 0x5A && eipC[2] == 0x59 && eipC[3] == 0xC3) { + TaggedUWord sp_plus_0 = sp; + TaggedUWord sp_plus_12 = sp; + TaggedUWord sp_plus_16 = sp; + sp_plus_12 = sp_plus_12 + TaggedUWord(12); + sp_plus_16 = sp_plus_16 + TaggedUWord(16); + TaggedUWord new_ebp = DerefTUW(sp_plus_0, aStackImg); + TaggedUWord new_eip = DerefTUW(sp_plus_12, aStackImg); + TaggedUWord new_esp = sp_plus_16; + if (new_ebp.Valid() && new_eip.Valid() && new_esp.Valid()) { + regs.xbp = new_ebp; + regs.xip = new_eip; + regs.xsp = new_esp; + continue; + } + } + } + } + //// + ///////////////////////////////////////////// +#endif // defined(GP_PLAT_x86_android) || defined(GP_PLAT_x86_linux) + + // So, do we have a ruleset for this address? If so, use it now. + if (ruleset) { + if (DEBUG_MAIN) { + ruleset->Print(ia.Value(), 1 /*bogus, but doesn't matter*/, mLog); + mLog("\n"); + } + // Use the RuleSet to compute the registers for the previous + // frame. |regs| is modified in-place. + UseRuleSet(®s, aStackImg, ruleset, pfxinstrs); + continue; + } + +#if defined(GP_PLAT_amd64_linux) || defined(GP_PLAT_x86_linux) || \ + defined(GP_PLAT_amd64_android) || defined(GP_PLAT_x86_android) || \ + defined(GP_PLAT_amd64_freebsd) + // There's no RuleSet for the specified address. On amd64/x86_linux, see if + // it's possible to recover the caller's frame by using the frame pointer. + + // We seek to compute (new_IP, new_SP, new_BP) from (old_BP, stack image), + // and assume the following layout: + // + // <--- new_SP + // +----------+ + // | new_IP | (return address) + // +----------+ + // | new_BP | <--- old_BP + // +----------+ + // | .... | + // | .... | + // | .... | + // +----------+ <---- old_SP (arbitrary, but must be <= old_BP) + + const size_t wordSzB = sizeof(uintptr_t); + TaggedUWord old_xsp = regs.xsp; + + // points at new_BP ? + TaggedUWord old_xbp = regs.xbp; + // points at new_IP ? + TaggedUWord old_xbp_plus1 = regs.xbp + TaggedUWord(1 * wordSzB); + // is the new_SP ? + TaggedUWord old_xbp_plus2 = regs.xbp + TaggedUWord(2 * wordSzB); + + if (old_xbp.Valid() && old_xbp.IsAligned() && old_xsp.Valid() && + old_xsp.IsAligned() && old_xsp.Value() <= old_xbp.Value()) { + // We don't need to do any range, alignment or validity checks for + // addresses passed to DerefTUW, since that performs them itself, and + // returns an invalid value on failure. Any such value will poison + // subsequent uses, and we do a final check for validity before putting + // the computed values into |regs|. + TaggedUWord new_xbp = DerefTUW(old_xbp, aStackImg); + if (new_xbp.Valid() && new_xbp.IsAligned() && + old_xbp.Value() < new_xbp.Value()) { + TaggedUWord new_xip = DerefTUW(old_xbp_plus1, aStackImg); + TaggedUWord new_xsp = old_xbp_plus2; + if (new_xbp.Valid() && new_xip.Valid() && new_xsp.Valid()) { + regs.xbp = new_xbp; + regs.xip = new_xip; + regs.xsp = new_xsp; + (*aFramePointerFramesAcquired)++; + continue; + } + } + } +#elif defined(GP_ARCH_arm64) + // Here is an example of generated code for prologue and epilogue.. + // + // stp x29, x30, [sp, #-16]! + // mov x29, sp + // ... + // ldp x29, x30, [sp], #16 + // ret + // + // Next is another example of generated code. + // + // stp x20, x19, [sp, #-32]! + // stp x29, x30, [sp, #16] + // add x29, sp, #0x10 + // ... + // ldp x29, x30, [sp, #16] + // ldp x20, x19, [sp], #32 + // ret + // + // Previous x29 and x30 register are stored in the address of x29 register. + // But since sp register value depends on local variables, we cannot compute + // previous sp register from current sp/fp/lr register and there is no + // regular rule for sp register in prologue. But since return address is lr + // register, if x29 is valid, we will get return address without sp + // register. + // + // So we assume the following layout that if no rule set. x29 is frame + // pointer, so we will be able to compute x29 and x30 . + // + // +----------+ <--- new_sp (cannot compute) + // | .... | + // +----------+ + // | new_lr | (return address) + // +----------+ + // | new_fp | <--- old_fp + // +----------+ + // | .... | + // | .... | + // +----------+ <---- old_sp (arbitrary, but unused) + + TaggedUWord old_fp = regs.x29; + if (old_fp.Valid() && old_fp.IsAligned() && last_valid_sp.Valid() && + last_valid_sp.Value() <= old_fp.Value()) { + TaggedUWord new_fp = DerefTUW(old_fp, aStackImg); + if (new_fp.Valid() && new_fp.IsAligned() && + old_fp.Value() < new_fp.Value()) { + TaggedUWord old_fp_plus1 = old_fp + TaggedUWord(8); + TaggedUWord new_lr = DerefTUW(old_fp_plus1, aStackImg); + if (new_lr.Valid()) { + regs.x29 = new_fp; + regs.x30 = new_lr; + // When using frame pointer to walk stack, we cannot compute sp + // register since we cannot compute sp register from fp/lr/sp + // register, and there is no regular rule to compute previous sp + // register. So mark as invalid. + regs.sp = TaggedUWord(); + (*aFramePointerFramesAcquired)++; + continue; + } + } + } +#endif // defined(GP_PLAT_amd64_linux) || defined(GP_PLAT_x86_linux) || + // defined(GP_PLAT_amd64_android) || defined(GP_PLAT_x86_android) || + // defined(GP_PLAT_amd64_freebsd) + + // We failed to recover a frame either using CFI or FP chasing, and we + // have no other ways to recover the frame. So we have to give up. + break; + + } // top level unwind loop + + // END UNWIND + ///////////////////////////////////////////////////////// +} + +//////////////////////////////////////////////////////////////// +// LUL Unit Testing // +//////////////////////////////////////////////////////////////// + +static const int LUL_UNIT_TEST_STACK_SIZE = 32768; + +#if defined(GP_ARCH_mips64) +static __attribute__((noinline)) unsigned long __getpc(void) { + unsigned long rtaddr; + __asm__ volatile("move %0, $31" : "=r"(rtaddr)); + return rtaddr; +} +#endif + +// This function is innermost in the test call sequence. It uses LUL +// to unwind, and compares the result with the sequence specified in +// the director string. These need to agree in order for the test to +// pass. In order not to screw up the results, this function needs +// to have a not-very big stack frame, since we're only presenting +// the innermost LUL_UNIT_TEST_STACK_SIZE bytes of stack to LUL, and +// that chunk unavoidably includes the frame for this function. +// +// This function must not be inlined into its callers. Doing so will +// cause the expected-vs-actual backtrace consistency checking to +// fail. Prints summary results to |aLUL|'s logging sink and also +// returns a boolean indicating whether or not the test failed. +static __attribute__((noinline)) bool GetAndCheckStackTrace( + LUL* aLUL, const char* dstring) { + // Get hold of the current unwind-start registers. + UnwindRegs startRegs; + memset(&startRegs, 0, sizeof(startRegs)); +#if defined(GP_ARCH_amd64) + volatile uintptr_t block[3]; + MOZ_ASSERT(sizeof(block) == 24); + __asm__ __volatile__( + "leaq 0(%%rip), %%r15" + "\n\t" + "movq %%r15, 0(%0)" + "\n\t" + "movq %%rsp, 8(%0)" + "\n\t" + "movq %%rbp, 16(%0)" + "\n" + : + : "r"(&block[0]) + : "memory", "r15"); + startRegs.xip = TaggedUWord(block[0]); + startRegs.xsp = TaggedUWord(block[1]); + startRegs.xbp = TaggedUWord(block[2]); + const uintptr_t REDZONE_SIZE = 128; + uintptr_t start = block[1] - REDZONE_SIZE; +#elif defined(GP_PLAT_x86_linux) || defined(GP_PLAT_x86_android) + volatile uintptr_t block[3]; + MOZ_ASSERT(sizeof(block) == 12); + __asm__ __volatile__( + ".byte 0xE8,0x00,0x00,0x00,0x00" /*call next insn*/ + "\n\t" + "popl %%edi" + "\n\t" + "movl %%edi, 0(%0)" + "\n\t" + "movl %%esp, 4(%0)" + "\n\t" + "movl %%ebp, 8(%0)" + "\n" + : + : "r"(&block[0]) + : "memory", "edi"); + startRegs.xip = TaggedUWord(block[0]); + startRegs.xsp = TaggedUWord(block[1]); + startRegs.xbp = TaggedUWord(block[2]); + const uintptr_t REDZONE_SIZE = 0; + uintptr_t start = block[1] - REDZONE_SIZE; +#elif defined(GP_PLAT_arm_linux) || defined(GP_PLAT_arm_android) + volatile uintptr_t block[6]; + MOZ_ASSERT(sizeof(block) == 24); + __asm__ __volatile__( + "mov r0, r15" + "\n\t" + "str r0, [%0, #0]" + "\n\t" + "str r14, [%0, #4]" + "\n\t" + "str r13, [%0, #8]" + "\n\t" + "str r12, [%0, #12]" + "\n\t" + "str r11, [%0, #16]" + "\n\t" + "str r7, [%0, #20]" + "\n" + : + : "r"(&block[0]) + : "memory", "r0"); + startRegs.r15 = TaggedUWord(block[0]); + startRegs.r14 = TaggedUWord(block[1]); + startRegs.r13 = TaggedUWord(block[2]); + startRegs.r12 = TaggedUWord(block[3]); + startRegs.r11 = TaggedUWord(block[4]); + startRegs.r7 = TaggedUWord(block[5]); + const uintptr_t REDZONE_SIZE = 0; + uintptr_t start = block[1] - REDZONE_SIZE; +#elif defined(GP_ARCH_arm64) + volatile uintptr_t block[4]; + MOZ_ASSERT(sizeof(block) == 32); + __asm__ __volatile__( + "adr x0, . \n\t" + "str x0, [%0, #0] \n\t" + "str x29, [%0, #8] \n\t" + "str x30, [%0, #16] \n\t" + "mov x0, sp \n\t" + "str x0, [%0, #24] \n\t" + : + : "r"(&block[0]) + : "memory", "x0"); + startRegs.pc = TaggedUWord(block[0]); + startRegs.x29 = TaggedUWord(block[1]); + startRegs.x30 = TaggedUWord(block[2]); + startRegs.sp = TaggedUWord(block[3]); + const uintptr_t REDZONE_SIZE = 0; + uintptr_t start = block[1] - REDZONE_SIZE; +#elif defined(GP_ARCH_mips64) + volatile uintptr_t block[3]; + MOZ_ASSERT(sizeof(block) == 24); + __asm__ __volatile__( + "sd $29, 8(%0) \n" + "sd $30, 16(%0) \n" + : + : "r"(block) + : "memory"); + block[0] = __getpc(); + startRegs.pc = TaggedUWord(block[0]); + startRegs.sp = TaggedUWord(block[1]); + startRegs.fp = TaggedUWord(block[2]); + const uintptr_t REDZONE_SIZE = 0; + uintptr_t start = block[1] - REDZONE_SIZE; +#else +# error "Unsupported platform" +#endif + + // Get hold of the innermost LUL_UNIT_TEST_STACK_SIZE bytes of the + // stack. + uintptr_t end = start + LUL_UNIT_TEST_STACK_SIZE; + uintptr_t ws = sizeof(void*); + start &= ~(ws - 1); + end &= ~(ws - 1); + uintptr_t nToCopy = end - start; + if (nToCopy > lul::N_STACK_BYTES) { + nToCopy = lul::N_STACK_BYTES; + } + MOZ_ASSERT(nToCopy <= lul::N_STACK_BYTES); + StackImage* stackImg = new StackImage(); + stackImg->mLen = nToCopy; + stackImg->mStartAvma = start; + if (nToCopy > 0) { + MOZ_MAKE_MEM_DEFINED((void*)start, nToCopy); + memcpy(&stackImg->mContents[0], (void*)start, nToCopy); + } + + // Unwind it. + const int MAX_TEST_FRAMES = 64; + uintptr_t framePCs[MAX_TEST_FRAMES]; + uintptr_t frameSPs[MAX_TEST_FRAMES]; + size_t framesAvail = mozilla::ArrayLength(framePCs); + size_t framesUsed = 0; + size_t framePointerFramesAcquired = 0; + aLUL->Unwind(&framePCs[0], &frameSPs[0], &framesUsed, + &framePointerFramesAcquired, framesAvail, &startRegs, stackImg); + + delete stackImg; + + // if (0) { + // // Show what we have. + // fprintf(stderr, "Got %d frames:\n", (int)framesUsed); + // for (size_t i = 0; i < framesUsed; i++) { + // fprintf(stderr, " [%2d] SP %p PC %p\n", + // (int)i, (void*)frameSPs[i], (void*)framePCs[i]); + // } + // fprintf(stderr, "\n"); + //} + + // Check to see if there's a consistent binding between digits in + // the director string ('1' .. '8') and the PC values acquired by + // the unwind. If there isn't, the unwinding has failed somehow. + uintptr_t binding[8]; // binding for '1' .. binding for '8' + memset((void*)binding, 0, sizeof(binding)); + + // The general plan is to work backwards along the director string + // and forwards along the framePCs array. Doing so corresponds to + // working outwards from the innermost frame of the recursive test set. + const char* cursor = dstring; + + // Find the end. This leaves |cursor| two bytes past the first + // character we want to look at -- see comment below. + while (*cursor) cursor++; + + // Counts the number of consistent frames. + size_t nConsistent = 0; + + // Iterate back to the start of the director string. The starting + // points are a bit complex. We can't use framePCs[0] because that + // contains the PC in this frame (above). We can't use framePCs[1] + // because that will contain the PC at return point in the recursive + // test group (TestFn[1-8]) for their call "out" to this function, + // GetAndCheckStackTrace. Although LUL will compute a correct + // return address, that will not be the same return address as for a + // recursive call out of the the function to another function in the + // group. Hence we can only start consistency checking at + // framePCs[2]. + // + // To be consistent, then, we must ignore the last element in the + // director string as that corresponds to framePCs[1]. Hence the + // start points are: framePCs[2] and the director string 2 bytes + // before the terminating zero. + // + // Also as a result of this, the number of consistent frames counted + // will always be one less than the length of the director string + // (not including its terminating zero). + size_t frameIx; + for (cursor = cursor - 2, frameIx = 2; + cursor >= dstring && frameIx < framesUsed; cursor--, frameIx++) { + char c = *cursor; + uintptr_t pc = framePCs[frameIx]; + // If this doesn't hold, the director string is ill-formed. + MOZ_ASSERT(c >= '1' && c <= '8'); + int n = ((int)c) - ((int)'1'); + if (binding[n] == 0) { + // There's no binding for |c| yet, so install |pc| and carry on. + binding[n] = pc; + nConsistent++; + continue; + } + // There's a pre-existing binding for |c|. Check it's consistent. + if (binding[n] != pc) { + // Not consistent. Give up now. + break; + } + // Consistent. Keep going. + nConsistent++; + } + + // So, did we succeed? + bool passed = nConsistent + 1 == strlen(dstring); + + // Show the results. + char buf[200]; + SprintfLiteral(buf, "LULUnitTest: dstring = %s\n", dstring); + buf[sizeof(buf) - 1] = 0; + aLUL->mLog(buf); + SprintfLiteral(buf, "LULUnitTest: %d consistent, %d in dstring: %s\n", + (int)nConsistent, (int)strlen(dstring), + passed ? "PASS" : "FAIL"); + buf[sizeof(buf) - 1] = 0; + aLUL->mLog(buf); + + return !passed; +} + +// Macro magic to create a set of 8 mutually recursive functions with +// varying frame sizes. These will recurse amongst themselves as +// specified by |strP|, the directory string, and call +// GetAndCheckStackTrace when the string becomes empty, passing it the +// original value of the string. This checks the result, printing +// results on |aLUL|'s logging sink, and also returns a boolean +// indicating whether or not the results are acceptable (correct). + +#define DECL_TEST_FN(NAME) \ + bool NAME(LUL* aLUL, const char* strPorig, const char* strP); + +#define GEN_TEST_FN(NAME, FRAMESIZE) \ + bool NAME(LUL* aLUL, const char* strPorig, const char* strP) { \ + /* Create a frame of size (at least) FRAMESIZE, so that the */ \ + /* 8 functions created by this macro offer some variation in frame */ \ + /* sizes. This isn't as simple as it might seem, since a clever */ \ + /* optimizing compiler (eg, clang-5) detects that the array is unused */ \ + /* and removes it. We try to defeat this by passing it to a function */ \ + /* in a different compilation unit, and hoping that clang does not */ \ + /* notice that the call is a no-op. */ \ + char space[FRAMESIZE]; \ + Unused << write(1, space, 0); /* write zero bytes of |space| to stdout */ \ + \ + if (*strP == '\0') { \ + /* We've come to the end of the director string. */ \ + /* Take a stack snapshot. */ \ + /* We purposefully use a negation to avoid tail-call optimization */ \ + return !GetAndCheckStackTrace(aLUL, strPorig); \ + } else { \ + /* Recurse onwards. This is a bit subtle. The obvious */ \ + /* thing to do here is call onwards directly, from within the */ \ + /* arms of the case statement. That gives a problem in that */ \ + /* there will be multiple return points inside each function when */ \ + /* unwinding, so it will be difficult to check for consistency */ \ + /* against the director string. Instead, we make an indirect */ \ + /* call, so as to guarantee that there is only one call site */ \ + /* within each function. This does assume that the compiler */ \ + /* won't transform it back to the simple direct-call form. */ \ + /* To discourage it from doing so, the call is bracketed with */ \ + /* __asm__ __volatile__ sections so as to make it not-movable. */ \ + bool (*nextFn)(LUL*, const char*, const char*) = NULL; \ + switch (*strP) { \ + case '1': \ + nextFn = TestFn1; \ + break; \ + case '2': \ + nextFn = TestFn2; \ + break; \ + case '3': \ + nextFn = TestFn3; \ + break; \ + case '4': \ + nextFn = TestFn4; \ + break; \ + case '5': \ + nextFn = TestFn5; \ + break; \ + case '6': \ + nextFn = TestFn6; \ + break; \ + case '7': \ + nextFn = TestFn7; \ + break; \ + case '8': \ + nextFn = TestFn8; \ + break; \ + default: \ + nextFn = TestFn8; \ + break; \ + } \ + /* "use" |space| immediately after the recursive call, */ \ + /* so as to dissuade clang from deallocating the space while */ \ + /* the call is active, or otherwise messing with the stack frame. */ \ + __asm__ __volatile__("" ::: "cc", "memory"); \ + bool passed = nextFn(aLUL, strPorig, strP + 1); \ + Unused << write(1, space, 0); \ + __asm__ __volatile__("" ::: "cc", "memory"); \ + return passed; \ + } \ + } + +// The test functions are mutually recursive, so it is necessary to +// declare them before defining them. +DECL_TEST_FN(TestFn1) +DECL_TEST_FN(TestFn2) +DECL_TEST_FN(TestFn3) +DECL_TEST_FN(TestFn4) +DECL_TEST_FN(TestFn5) +DECL_TEST_FN(TestFn6) +DECL_TEST_FN(TestFn7) +DECL_TEST_FN(TestFn8) + +GEN_TEST_FN(TestFn1, 123) +GEN_TEST_FN(TestFn2, 456) +GEN_TEST_FN(TestFn3, 789) +GEN_TEST_FN(TestFn4, 23) +GEN_TEST_FN(TestFn5, 47) +GEN_TEST_FN(TestFn6, 117) +GEN_TEST_FN(TestFn7, 1) +GEN_TEST_FN(TestFn8, 99) + +// This starts the test sequence going. Call here to generate a +// sequence of calls as directed by the string |dstring|. The call +// sequence will, from its innermost frame, finish by calling +// GetAndCheckStackTrace() and passing it |dstring|. +// GetAndCheckStackTrace() will unwind the stack, check consistency +// of those results against |dstring|, and print a pass/fail message +// to aLUL's logging sink. It also updates the counters in *aNTests +// and aNTestsPassed. +__attribute__((noinline)) void TestUnw(/*OUT*/ int* aNTests, + /*OUT*/ int* aNTestsPassed, LUL* aLUL, + const char* dstring) { + // Ensure that the stack has at least this much space on it. This + // makes it safe to saw off the top LUL_UNIT_TEST_STACK_SIZE bytes + // and hand it to LUL. Safe in the sense that no segfault can + // happen because the stack is at least this big. This is all + // somewhat dubious in the sense that a sufficiently clever compiler + // (clang, for one) can figure out that space[] is unused and delete + // it from the frame. Hence the somewhat elaborate hoop jumping to + // fill it up before the call and to at least appear to use the + // value afterwards. + int i; + volatile char space[LUL_UNIT_TEST_STACK_SIZE]; + for (i = 0; i < LUL_UNIT_TEST_STACK_SIZE; i++) { + space[i] = (char)(i & 0x7F); + } + + // Really run the test. + bool passed = TestFn1(aLUL, dstring, dstring); + + // Appear to use space[], by visiting the value to compute some kind + // of checksum, and then (apparently) using the checksum. + int sum = 0; + for (i = 0; i < LUL_UNIT_TEST_STACK_SIZE; i++) { + // If this doesn't fool LLVM, I don't know what will. + sum += space[i] - 3 * i; + } + __asm__ __volatile__("" : : "r"(sum)); + + // Update the counters. + (*aNTests)++; + if (passed) { + (*aNTestsPassed)++; + } +} + +void RunLulUnitTests(/*OUT*/ int* aNTests, /*OUT*/ int* aNTestsPassed, + LUL* aLUL) { + aLUL->mLog(":\n"); + aLUL->mLog("LULUnitTest: BEGIN\n"); + *aNTests = *aNTestsPassed = 0; + TestUnw(aNTests, aNTestsPassed, aLUL, "11111111"); + TestUnw(aNTests, aNTestsPassed, aLUL, "11222211"); + TestUnw(aNTests, aNTestsPassed, aLUL, "111222333"); + TestUnw(aNTests, aNTestsPassed, aLUL, "1212121231212331212121212121212"); + TestUnw(aNTests, aNTestsPassed, aLUL, "31415827271828325332173258"); + TestUnw(aNTests, aNTestsPassed, aLUL, + "123456781122334455667788777777777777777777777"); + aLUL->mLog("LULUnitTest: END\n"); + aLUL->mLog(":\n"); +} + +} // namespace lul diff --git a/tools/profiler/lul/LulMain.h b/tools/profiler/lul/LulMain.h new file mode 100644 index 0000000000..d386bd5c4f --- /dev/null +++ b/tools/profiler/lul/LulMain.h @@ -0,0 +1,378 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef LulMain_h +#define LulMain_h + +#include "PlatformMacros.h" +#include "mozilla/Atomics.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/ProfilerUtils.h" + +// LUL: A Lightweight Unwind Library. +// This file provides the end-user (external) interface for LUL. + +// Some comments about naming in the implementation. These are safe +// to ignore if you are merely using LUL, but are important if you +// hack on its internals. +// +// Debuginfo readers in general have tended to use the word "address" +// to mean several different things. This sometimes makes them +// difficult to understand and maintain. LUL tries hard to avoid +// using the word "address" and instead uses the following more +// precise terms: +// +// * SVMA ("Stated Virtual Memory Address"): this is an address of a +// symbol (etc) as it is stated in the symbol table, or other +// metadata, of an object. Such values are typically small and +// start from zero or thereabouts, unless the object has been +// prelinked. +// +// * AVMA ("Actual Virtual Memory Address"): this is the address of a +// symbol (etc) in a running process, that is, once the associated +// object has been mapped into a process. Such values are typically +// much larger than SVMAs, since objects can get mapped arbitrarily +// far along the address space. +// +// * "Bias": the difference between AVMA and SVMA for a given symbol +// (specifically, AVMA - SVMA). The bias is always an integral +// number of pages. Once we know the bias for a given object's +// text section (for example), we can compute the AVMAs of all of +// its text symbols by adding the bias to their SVMAs. +// +// * "Image address": typically, to read debuginfo from an object we +// will temporarily mmap in the file so as to read symbol tables +// etc. Addresses in this temporary mapping are called "Image +// addresses". Note that the temporary mapping is entirely +// unrelated to the mappings of the file that the dynamic linker +// must perform merely in order to get the program to run. Hence +// image addresses are unrelated to either SVMAs or AVMAs. + +namespace lul { + +// A machine word plus validity tag. +class TaggedUWord { + public: + // RUNS IN NO-MALLOC CONTEXT + // Construct a valid one. + explicit TaggedUWord(uintptr_t w) : mValue(w), mValid(true) {} + + // RUNS IN NO-MALLOC CONTEXT + // Construct an invalid one. + TaggedUWord() : mValue(0), mValid(false) {} + + // RUNS IN NO-MALLOC CONTEXT + TaggedUWord operator+(TaggedUWord rhs) const { + return (Valid() && rhs.Valid()) ? TaggedUWord(Value() + rhs.Value()) + : TaggedUWord(); + } + + // RUNS IN NO-MALLOC CONTEXT + TaggedUWord operator-(TaggedUWord rhs) const { + return (Valid() && rhs.Valid()) ? TaggedUWord(Value() - rhs.Value()) + : TaggedUWord(); + } + + // RUNS IN NO-MALLOC CONTEXT + TaggedUWord operator&(TaggedUWord rhs) const { + return (Valid() && rhs.Valid()) ? TaggedUWord(Value() & rhs.Value()) + : TaggedUWord(); + } + + // RUNS IN NO-MALLOC CONTEXT + TaggedUWord operator|(TaggedUWord rhs) const { + return (Valid() && rhs.Valid()) ? TaggedUWord(Value() | rhs.Value()) + : TaggedUWord(); + } + + // RUNS IN NO-MALLOC CONTEXT + TaggedUWord CmpGEs(TaggedUWord rhs) const { + if (Valid() && rhs.Valid()) { + intptr_t s1 = (intptr_t)Value(); + intptr_t s2 = (intptr_t)rhs.Value(); + return TaggedUWord(s1 >= s2 ? 1 : 0); + } + return TaggedUWord(); + } + + // RUNS IN NO-MALLOC CONTEXT + TaggedUWord operator<<(TaggedUWord rhs) const { + if (Valid() && rhs.Valid()) { + uintptr_t shift = rhs.Value(); + if (shift < 8 * sizeof(uintptr_t)) return TaggedUWord(Value() << shift); + } + return TaggedUWord(); + } + + // RUNS IN NO-MALLOC CONTEXT + // Is equal? Note: non-validity on either side gives non-equality. + bool operator==(TaggedUWord other) const { + return (mValid && other.Valid()) ? (mValue == other.Value()) : false; + } + + // RUNS IN NO-MALLOC CONTEXT + // Is it word-aligned? + bool IsAligned() const { + return mValid && (mValue & (sizeof(uintptr_t) - 1)) == 0; + } + + // RUNS IN NO-MALLOC CONTEXT + uintptr_t Value() const { return mValue; } + + // RUNS IN NO-MALLOC CONTEXT + bool Valid() const { return mValid; } + + private: + uintptr_t mValue; + bool mValid; +}; + +// The registers, with validity tags, that will be unwound. + +struct UnwindRegs { +#if defined(GP_ARCH_arm) + TaggedUWord r7; + TaggedUWord r11; + TaggedUWord r12; + TaggedUWord r13; + TaggedUWord r14; + TaggedUWord r15; +#elif defined(GP_ARCH_arm64) + TaggedUWord x29; + TaggedUWord x30; + TaggedUWord sp; + TaggedUWord pc; +#elif defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + TaggedUWord xbp; + TaggedUWord xsp; + TaggedUWord xip; +#elif defined(GP_ARCH_mips64) + TaggedUWord sp; + TaggedUWord fp; + TaggedUWord pc; +#else +# error "Unknown plat" +#endif +}; + +// The maximum number of bytes in a stack snapshot. This value can be increased +// if necessary, but testing showed that 160k is enough to obtain good +// backtraces on x86_64 Linux. Most backtraces fit comfortably into 4-8k of +// stack space, but we do have some very deep stacks occasionally. Please see +// the comments in DoNativeBacktrace as to why it's OK to have this value be so +// large. +static const size_t N_STACK_BYTES = 160 * 1024; + +// The stack chunk image that will be unwound. +struct StackImage { + // [start_avma, +len) specify the address range in the buffer. + // Obviously we require 0 <= len <= N_STACK_BYTES. + uintptr_t mStartAvma; + size_t mLen; + uint8_t mContents[N_STACK_BYTES]; +}; + +// Statistics collection for the unwinder. +template <typename T> +class LULStats { + public: + LULStats() : mContext(0), mCFI(0), mFP(0) {} + + template <typename S> + explicit LULStats(const LULStats<S>& aOther) + : mContext(aOther.mContext), mCFI(aOther.mCFI), mFP(aOther.mFP) {} + + template <typename S> + LULStats<T>& operator=(const LULStats<S>& aOther) { + mContext = aOther.mContext; + mCFI = aOther.mCFI; + mFP = aOther.mFP; + return *this; + } + + template <typename S> + uint32_t operator-(const LULStats<S>& aOther) { + return (mContext - aOther.mContext) + (mCFI - aOther.mCFI) + + (mFP - aOther.mFP); + } + + T mContext; // Number of context frames + T mCFI; // Number of CFI/EXIDX frames + T mFP; // Number of frame-pointer recovered frames +}; + +// The core unwinder library class. Just one of these is needed, and +// it can be shared by multiple unwinder threads. +// +// The library operates in one of two modes. +// +// * Admin mode. The library is this state after creation. In Admin +// mode, no unwinding may be performed. It is however allowable to +// perform administrative tasks -- primarily, loading of unwind info +// -- in this mode. In particular, it is safe for the library to +// perform dynamic memory allocation in this mode. Safe in the +// sense that there is no risk of deadlock against unwinding threads +// that might -- because of where they have been sampled -- hold the +// system's malloc lock. +// +// * Unwind mode. In this mode, calls to ::Unwind may be made, but +// nothing else. ::Unwind guarantees not to make any dynamic memory +// requests, so as to guarantee that the calling thread won't +// deadlock in the case where it already holds the system's malloc lock. +// +// The library is created in Admin mode. After debuginfo is loaded, +// the caller must switch it into Unwind mode by calling +// ::EnableUnwinding. There is no way to switch it back to Admin mode +// after that. To safely switch back to Admin mode would require the +// caller (or other external agent) to guarantee that there are no +// pending ::Unwind calls. + +class PriMap; +class SegArray; +class UniqueStringUniverse; + +class LUL { + public: + // Create; supply a logging sink. Sets the object in Admin mode. + explicit LUL(void (*aLog)(const char*)); + + // Destroy. Caller is responsible for ensuring that no other + // threads are in Unwind calls. All resources are freed and all + // registered unwinder threads are deregistered. Can be called + // either in Admin or Unwind mode. + ~LUL(); + + // Notify the library that unwinding is now allowed and so + // admin-mode calls are no longer allowed. The object is initially + // created in admin mode. The only possible transition is + // admin->unwinding, therefore. + void EnableUnwinding(); + + // Notify of a new r-x mapping, and load the associated unwind info. + // The filename is strdup'd and used for debug printing. If + // aMappedImage is NULL, this function will mmap/munmap the file + // itself, so as to be able to read the unwind info. If + // aMappedImage is non-NULL then it is assumed to point to a + // called-supplied and caller-managed mapped image of the file. + // May only be called in Admin mode. + void NotifyAfterMap(uintptr_t aRXavma, size_t aSize, const char* aFileName, + const void* aMappedImage); + + // In rare cases we know an executable area exists but don't know + // what the associated file is. This call notifies LUL of such + // areas. This is important for correct functioning of stack + // scanning and of the x86-{linux,android} special-case + // __kernel_syscall function handling. + // This must be called only after the code area in + // question really has been mapped. + // May only be called in Admin mode. + void NotifyExecutableArea(uintptr_t aRXavma, size_t aSize); + + // Notify that a mapped area has been unmapped; discard any + // associated unwind info. Acquires mRWlock for writing. Note that + // to avoid segfaulting the stack-scan unwinder, which inspects code + // areas, this must be called before the code area in question is + // really unmapped. Note that, unlike NotifyAfterMap(), this + // function takes the start and end addresses of the range to be + // unmapped, rather than a start and a length parameter. This is so + // as to make it possible to notify an unmap for the entire address + // space using a single call. + // May only be called in Admin mode. + void NotifyBeforeUnmap(uintptr_t aAvmaMin, uintptr_t aAvmaMax); + + // Apply NotifyBeforeUnmap to the entire address space. This causes + // LUL to discard all unwind and executable-area information for the + // entire address space. + // May only be called in Admin mode. + void NotifyBeforeUnmapAll() { NotifyBeforeUnmap(0, UINTPTR_MAX); } + + // Returns the number of mappings currently registered. + // May only be called in Admin mode. + size_t CountMappings(); + + // Unwind |aStackImg| starting with the context in |aStartRegs|. + // Write the number of frames recovered in *aFramesUsed. Put + // the PC values in aFramePCs[0 .. *aFramesUsed-1] and + // the SP values in aFrameSPs[0 .. *aFramesUsed-1]. + // |aFramesAvail| is the size of the two output arrays and hence the + // largest possible value of *aFramesUsed. PC values are always + // valid, and the unwind will stop when the PC becomes invalid, but + // the SP values might be invalid, in which case the value zero will + // be written in the relevant frameSPs[] slot. + // + // This function assumes that the SP values increase as it unwinds + // away from the innermost frame -- that is, that the stack grows + // down. It monitors SP values as it unwinds to check they + // decrease, so as to avoid looping on corrupted stacks. + // + // May only be called in Unwind mode. Multiple threads may unwind + // at once. LUL user is responsible for ensuring that no thread makes + // any Admin calls whilst in Unwind mode. + // MOZ_CRASHes if the calling thread is not registered for unwinding. + // + // The calling thread must previously have been registered via a call to + // RegisterSampledThread. + void Unwind(/*OUT*/ uintptr_t* aFramePCs, + /*OUT*/ uintptr_t* aFrameSPs, + /*OUT*/ size_t* aFramesUsed, + /*OUT*/ size_t* aFramePointerFramesAcquired, size_t aFramesAvail, + UnwindRegs* aStartRegs, StackImage* aStackImg); + + // The logging sink. Call to send debug strings to the caller- + // specified destination. Can only be called by the Admin thread. + void (*mLog)(const char*); + + // Statistics relating to unwinding. These have to be atomic since + // unwinding can occur on different threads simultaneously. + LULStats<mozilla::Atomic<uint32_t>> mStats; + + // Possibly show the statistics. This may not be called from any + // registered sampling thread, since it involves I/O. + void MaybeShowStats(); + + size_t SizeOfIncludingThis(mozilla::MallocSizeOf) const; + + private: + // The statistics counters at the point where they were last printed. + LULStats<uint32_t> mStatsPrevious; + + // Are we in admin mode? Initially |true| but changes to |false| + // once unwinding begins. + bool mAdminMode; + + // The thread ID associated with admin mode. This is the only thread + // that is allowed do perform non-Unwind calls on this object. Conversely, + // no registered Unwinding thread may be the admin thread. This is so + // as to clearly partition the one thread that may do dynamic memory + // allocation from the threads that are being sampled, since the latter + // absolutely may not do dynamic memory allocation. + ProfilerThreadId mAdminThreadId; + + // The top level mapping from code address ranges to postprocessed + // unwind info. Basically a sorted array of (addr, len, info) + // records. This field is updated by NotifyAfterMap and NotifyBeforeUnmap. + PriMap* mPriMap; + + // An auxiliary structure that records which address ranges are + // mapped r-x, for the benefit of the stack scanner. + SegArray* mSegArray; + + // A UniqueStringUniverse that holds all the strdup'd strings created + // whilst reading unwind information. This is included so as to make + // it possible to free them in ~LUL. + UniqueStringUniverse* mUSU; +}; + +// Run unit tests on an initialised, loaded-up LUL instance, and print +// summary results on |aLUL|'s logging sink. Also return the number +// of tests run in *aNTests and the number that passed in +// *aNTestsPassed. +void RunLulUnitTests(/*OUT*/ int* aNTests, /*OUT*/ int* aNTestsPassed, + LUL* aLUL); + +} // namespace lul + +#endif // LulMain_h diff --git a/tools/profiler/lul/LulMainInt.h b/tools/profiler/lul/LulMainInt.h new file mode 100644 index 0000000000..001a4aecfb --- /dev/null +++ b/tools/profiler/lul/LulMainInt.h @@ -0,0 +1,631 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef LulMainInt_h +#define LulMainInt_h + +#include "PlatformMacros.h" +#include "LulMain.h" // for TaggedUWord + +#include <string> +#include <vector> + +#include "mozilla/Assertions.h" +#include "mozilla/HashFunctions.h" +#include "mozilla/HashTable.h" +#include "mozilla/Sprintf.h" + +// This file provides an internal interface inside LUL. If you are an +// end-user of LUL, do not include it in your code. The end-user +// interface is in LulMain.h. + +namespace lul { + +using std::vector; + +//////////////////////////////////////////////////////////////// +// DW_REG_ constants // +//////////////////////////////////////////////////////////////// + +// These are the Dwarf CFI register numbers, as (presumably) defined +// in the ELF ABI supplements for each architecture. + +enum DW_REG_NUMBER { + // No real register has this number. It's convenient to be able to + // treat the CFA (Canonical Frame Address) as "just another + // register", though. + DW_REG_CFA = -1, +#if defined(GP_ARCH_arm) + // ARM registers + DW_REG_ARM_R7 = 7, + DW_REG_ARM_R11 = 11, + DW_REG_ARM_R12 = 12, + DW_REG_ARM_R13 = 13, + DW_REG_ARM_R14 = 14, + DW_REG_ARM_R15 = 15, +#elif defined(GP_ARCH_arm64) + // aarch64 registers + DW_REG_AARCH64_X29 = 29, + DW_REG_AARCH64_X30 = 30, + DW_REG_AARCH64_SP = 31, +#elif defined(GP_ARCH_amd64) + // Because the X86 (32 bit) and AMD64 (64 bit) summarisers are + // combined, a merged set of register constants is needed. + DW_REG_INTEL_XBP = 6, + DW_REG_INTEL_XSP = 7, + DW_REG_INTEL_XIP = 16, +#elif defined(GP_ARCH_x86) + DW_REG_INTEL_XBP = 5, + DW_REG_INTEL_XSP = 4, + DW_REG_INTEL_XIP = 8, +#elif defined(GP_ARCH_mips64) + DW_REG_MIPS_SP = 29, + DW_REG_MIPS_FP = 30, + DW_REG_MIPS_PC = 34, +#else +# error "Unknown arch" +#endif +}; + +//////////////////////////////////////////////////////////////// +// PfxExpr // +//////////////////////////////////////////////////////////////// + +enum PfxExprOp { + // meaning of mOperand effect on stack + PX_Start, // bool start-with-CFA? start, with CFA on stack, or not + PX_End, // none stop; result is at top of stack + PX_SImm32, // int32 push signed int32 + PX_DwReg, // DW_REG_NUMBER push value of the specified reg + PX_Deref, // none pop X ; push *X + PX_Add, // none pop X ; pop Y ; push Y + X + PX_Sub, // none pop X ; pop Y ; push Y - X + PX_And, // none pop X ; pop Y ; push Y & X + PX_Or, // none pop X ; pop Y ; push Y | X + PX_CmpGES, // none pop X ; pop Y ; push (Y >=s X) ? 1 : 0 + PX_Shl // none pop X ; pop Y ; push Y << X +}; + +struct PfxInstr { + PfxInstr(PfxExprOp opcode, int32_t operand) + : mOpcode(opcode), mOperand(operand) {} + explicit PfxInstr(PfxExprOp opcode) : mOpcode(opcode), mOperand(0) {} + bool operator==(const PfxInstr& other) const { + return mOpcode == other.mOpcode && mOperand == other.mOperand; + } + PfxExprOp mOpcode; + int32_t mOperand; +}; + +static_assert(sizeof(PfxInstr) <= 8, "PfxInstr size changed unexpectedly"); + +// Evaluate the prefix expression whose PfxInstrs start at aPfxInstrs[start]. +// In the case of any mishap (stack over/underflow, running off the end of +// the instruction vector, obviously malformed sequences), +// return an invalid TaggedUWord. +// RUNS IN NO-MALLOC CONTEXT +TaggedUWord EvaluatePfxExpr(int32_t start, const UnwindRegs* aOldRegs, + TaggedUWord aCFA, const StackImage* aStackImg, + const vector<PfxInstr>& aPfxInstrs); + +//////////////////////////////////////////////////////////////// +// LExpr // +//////////////////////////////////////////////////////////////// + +// An expression -- very primitive. Denotes either "register + +// offset", a dereferenced version of the same, or a reference to a +// prefix expression stored elsewhere. So as to allow convenient +// handling of Dwarf-derived unwind info, the register may also denote +// the CFA. A large number of these need to be stored, so we ensure +// it fits into 8 bytes. See comment below on RuleSet to see how +// expressions fit into the bigger picture. + +enum LExprHow { + UNKNOWN = 0, // This LExpr denotes no value. + NODEREF, // Value is (mReg + mOffset). + DEREF, // Value is *(mReg + mOffset). + PFXEXPR // Value is EvaluatePfxExpr(secMap->mPfxInstrs[mOffset]) +}; + +inline static const char* NameOf_LExprHow(LExprHow how) { + switch (how) { + case UNKNOWN: + return "UNKNOWN"; + case NODEREF: + return "NODEREF"; + case DEREF: + return "DEREF"; + case PFXEXPR: + return "PFXEXPR"; + default: + return "LExpr-??"; + } +} + +struct LExpr { + // Denotes an expression with no value. + LExpr() : mHow(UNKNOWN), mReg(0), mOffset(0) {} + + // Denotes any expressible expression. + LExpr(LExprHow how, int16_t reg, int32_t offset) + : mHow(how), mReg(reg), mOffset(offset) { + switch (how) { + case UNKNOWN: + MOZ_ASSERT(reg == 0 && offset == 0); + break; + case NODEREF: + break; + case DEREF: + break; + case PFXEXPR: + MOZ_ASSERT(reg == 0 && offset >= 0); + break; + default: + MOZ_RELEASE_ASSERT(0, "LExpr::LExpr: invalid how"); + } + } + + // Hash it, carefully looking only at defined parts. + mozilla::HashNumber hash() const { + mozilla::HashNumber h = mHow; + switch (mHow) { + case UNKNOWN: + break; + case NODEREF: + case DEREF: + h = mozilla::AddToHash(h, mReg); + h = mozilla::AddToHash(h, mOffset); + break; + case PFXEXPR: + h = mozilla::AddToHash(h, mOffset); + break; + default: + MOZ_RELEASE_ASSERT(0, "LExpr::hash: invalid how"); + } + return h; + } + + // And structural equality. + bool equals(const LExpr& other) const { + if (mHow != other.mHow) { + return false; + } + switch (mHow) { + case UNKNOWN: + return true; + case NODEREF: + case DEREF: + return mReg == other.mReg && mOffset == other.mOffset; + case PFXEXPR: + return mOffset == other.mOffset; + default: + MOZ_RELEASE_ASSERT(0, "LExpr::equals: invalid how"); + } + } + + // Change the offset for an expression that references memory. + LExpr add_delta(long delta) { + MOZ_ASSERT(mHow == NODEREF); + // If this is a non-debug build and the above assertion would have + // failed, at least return LExpr() so that the machinery that uses + // the resulting expression fails in a repeatable way. + return (mHow == NODEREF) ? LExpr(mHow, mReg, mOffset + delta) + : LExpr(); // Gone bad + } + + // Dereference an expression that denotes a memory address. + LExpr deref() { + MOZ_ASSERT(mHow == NODEREF); + // Same rationale as for add_delta(). + return (mHow == NODEREF) ? LExpr(DEREF, mReg, mOffset) + : LExpr(); // Gone bad + } + + // Print a rule for recovery of |aNewReg| whose recovered value + // is this LExpr. + std::string ShowRule(const char* aNewReg) const; + + // Evaluate this expression, producing a TaggedUWord. |aOldRegs| + // holds register values that may be referred to by the expression. + // |aCFA| holds the CFA value, if any, that applies. |aStackImg| + // contains a chuck of stack that will be consulted if the expression + // references memory. |aPfxInstrs| holds the vector of PfxInstrs + // that will be consulted if this is a PFXEXPR. + // RUNS IN NO-MALLOC CONTEXT + TaggedUWord EvaluateExpr(const UnwindRegs* aOldRegs, TaggedUWord aCFA, + const StackImage* aStackImg, + const vector<PfxInstr>* aPfxInstrs) const; + + // Representation of expressions. If |mReg| is DW_REG_CFA (-1) then + // it denotes the CFA. All other allowed values for |mReg| are + // nonnegative and are DW_REG_ values. + LExprHow mHow : 8; + int16_t mReg; // A DW_REG_ value + int32_t mOffset; // 32-bit signed offset should be more than enough. +}; + +static_assert(sizeof(LExpr) <= 8, "LExpr size changed unexpectedly"); + +//////////////////////////////////////////////////////////////// +// RuleSet // +//////////////////////////////////////////////////////////////// + +// This is platform-dependent. It describes how to recover the CFA and then +// how to recover the registers for the previous frame. Such "recipes" are +// specific to particular ranges of machine code, but the associated range +// is not stored in RuleSet, because in general each RuleSet may be used +// for many such range fragments ("extents"). See the comments below for +// Extent and SecMap. +// +// The set of LExprs contained in a given RuleSet describe a DAG which +// says how to compute the caller's registers ("new registers") from +// the callee's registers ("old registers"). The DAG can contain a +// single internal node, which is the value of the CFA for the callee. +// It would be possible to construct a DAG that omits the CFA, but +// including it makes the summarisers simpler, and the Dwarf CFI spec +// has the CFA as a central concept. +// +// For this to make sense, |mCfaExpr| can't have +// |mReg| == DW_REG_CFA since we have no previous value for the CFA. +// All of the other |Expr| fields can -- and usually do -- specify +// |mReg| == DW_REG_CFA. +// +// With that in place, the unwind algorithm proceeds as follows. +// +// (0) Initially: we have values for the old registers, and a memory +// image. +// +// (1) Compute the CFA by evaluating |mCfaExpr|. Add the computed +// value to the set of "old registers". +// +// (2) Compute values for the registers by evaluating all of the other +// |Expr| fields in the RuleSet. These can depend on both the old +// register values and the just-computed CFA. +// +// If we are unwinding without computing a CFA, perhaps because the +// RuleSets are derived from EXIDX instead of Dwarf, then +// |mCfaExpr.mHow| will be LExpr::UNKNOWN, so the computed value will +// be invalid -- that is, TaggedUWord() -- and so any attempt to use +// that will result in the same value. But that's OK because the +// RuleSet would make no sense if depended on the CFA but specified no +// way to compute it. +// +// A RuleSet is not allowed to cover zero address range. Having zero +// length would break binary searching in SecMaps and PriMaps. + +class RuleSet { + public: + RuleSet(); + void Print(uintptr_t avma, uintptr_t len, void (*aLog)(const char*)) const; + + // Find the LExpr* for a given DW_REG_ value in this class. + LExpr* ExprForRegno(DW_REG_NUMBER aRegno); + + // How to compute the CFA. + LExpr mCfaExpr; + // How to compute caller register values. These may reference the + // value defined by |mCfaExpr|. +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + LExpr mXipExpr; // return address + LExpr mXspExpr; + LExpr mXbpExpr; +#elif defined(GP_ARCH_arm) + LExpr mR15expr; // return address + LExpr mR14expr; + LExpr mR13expr; + LExpr mR12expr; + LExpr mR11expr; + LExpr mR7expr; +#elif defined(GP_ARCH_arm64) + LExpr mX29expr; // frame pointer register + LExpr mX30expr; // link register + LExpr mSPexpr; +#elif defined(GP_ARCH_mips64) + LExpr mPCexpr; + LExpr mFPexpr; + LExpr mSPexpr; +#else +# error "Unknown arch" +#endif + + // Machinery in support of hashing. + typedef RuleSet Lookup; + + static mozilla::HashNumber hash(RuleSet rs) { + mozilla::HashNumber h = rs.mCfaExpr.hash(); +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + h = mozilla::AddToHash(h, rs.mXipExpr.hash()); + h = mozilla::AddToHash(h, rs.mXspExpr.hash()); + h = mozilla::AddToHash(h, rs.mXbpExpr.hash()); +#elif defined(GP_ARCH_arm) + h = mozilla::AddToHash(h, rs.mR15expr.hash()); + h = mozilla::AddToHash(h, rs.mR14expr.hash()); + h = mozilla::AddToHash(h, rs.mR13expr.hash()); + h = mozilla::AddToHash(h, rs.mR12expr.hash()); + h = mozilla::AddToHash(h, rs.mR11expr.hash()); + h = mozilla::AddToHash(h, rs.mR7expr.hash()); +#elif defined(GP_ARCH_arm64) + h = mozilla::AddToHash(h, rs.mX29expr.hash()); + h = mozilla::AddToHash(h, rs.mX30expr.hash()); + h = mozilla::AddToHash(h, rs.mSPexpr.hash()); +#elif defined(GP_ARCH_mips64) + h = mozilla::AddToHash(h, rs.mPCexpr.hash()); + h = mozilla::AddToHash(h, rs.mFPexpr.hash()); + h = mozilla::AddToHash(h, rs.mSPexpr.hash()); +#else +# error "Unknown arch" +#endif + return h; + } + + static bool match(const RuleSet& rs1, const RuleSet& rs2) { + return rs1.mCfaExpr.equals(rs2.mCfaExpr) && +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + rs1.mXipExpr.equals(rs2.mXipExpr) && + rs1.mXspExpr.equals(rs2.mXspExpr) && + rs1.mXbpExpr.equals(rs2.mXbpExpr); +#elif defined(GP_ARCH_arm) + rs1.mR15expr.equals(rs2.mR15expr) && + rs1.mR14expr.equals(rs2.mR14expr) && + rs1.mR13expr.equals(rs2.mR13expr) && + rs1.mR12expr.equals(rs2.mR12expr) && + rs1.mR11expr.equals(rs2.mR11expr) && rs1.mR7expr.equals(rs2.mR7expr); +#elif defined(GP_ARCH_arm64) + rs1.mX29expr.equals(rs2.mX29expr) && + rs1.mX30expr.equals(rs2.mX30expr) && rs1.mSPexpr.equals(rs2.mSPexpr); +#elif defined(GP_ARCH_mips64) + rs1.mPCexpr.equals(rs2.mPCexpr) && rs1.mFPexpr.equals(rs2.mFPexpr) && + rs1.mSPexpr.equals(rs2.mSPexpr); +#else +# error "Unknown arch" +#endif + } +}; + +// Returns |true| for Dwarf register numbers which are members +// of the set of registers that LUL unwinds on this target. +static inline bool registerIsTracked(DW_REG_NUMBER reg) { + switch (reg) { +#if defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) + case DW_REG_INTEL_XBP: + case DW_REG_INTEL_XSP: + case DW_REG_INTEL_XIP: + return true; +#elif defined(GP_ARCH_arm) + case DW_REG_ARM_R7: + case DW_REG_ARM_R11: + case DW_REG_ARM_R12: + case DW_REG_ARM_R13: + case DW_REG_ARM_R14: + case DW_REG_ARM_R15: + return true; +#elif defined(GP_ARCH_arm64) + case DW_REG_AARCH64_X29: + case DW_REG_AARCH64_X30: + case DW_REG_AARCH64_SP: + return true; +#elif defined(GP_ARCH_mips64) + case DW_REG_MIPS_FP: + case DW_REG_MIPS_SP: + case DW_REG_MIPS_PC: + return true; +#else +# error "Unknown arch" +#endif + default: + return false; + } +} + +//////////////////////////////////////////////////////////////// +// Extent // +//////////////////////////////////////////////////////////////// + +struct Extent { + // Three fields, which together take 8 bytes. + uint32_t mOffset; + uint16_t mLen; + uint16_t mDictIx; + + // What this means is: suppose we are looking for the unwind rules for some + // code address (AVMA) `avma`. If we can find some SecMap `secmap` such + // that `avma` falls in the range + // + // `[secmap.mMapMinAVMA, secmap.mMapMaxAVMA]` + // + // then the RuleSet to use is `secmap.mDictionary[dictIx]` iff we can find + // an `extent` in `secmap.mExtents` such that `avma` falls into the range + // + // `[secmap.mMapMinAVMA + extent.offset(), + // secmap.mMapMinAVMA + extent.offset() + extent.len())`. + // + // Packing Extent into the minimum space is important, since there will be + // huge numbers of Extents -- around 3 million for libxul.so as of Sept + // 2020. Here, we aim for an 8-byte size, with the field sizes chosen + // carefully, as follows: + // + // `offset` denotes a byte offset inside the text section for some shared + // object. libxul.so is by far the largest. As of Sept 2020 it has a text + // size of up to around 120MB, that is, close to 2^27 bytes. Hence a 32-bit + // `offset` field gives a safety margin of around a factor of 32 + // (== 2 ^(32 - 27)). + // + // `dictIx` indicates a unique `RuleSet` for some code address range. + // Experimentation on x86_64-linux indicates that only around 300 different + // `RuleSet`s exist, for libxul.so. A 16-bit bit field allows up to 65536 + // to be recorded, hence leaving us a generous safety margin. + // + // `len` indicates the length of the associated address range. + // + // Note the representation becomes unusable if either `offset` overflows 32 + // bits or `dictIx` overflows 16 bits. On the other hand, it does not + // matter (although is undesirable) if `len` overflows 16 bits, because in + // that case we can add multiple size-65535 entries to `secmap.mExtents` to + // cover the entire range. Hence the field sizes are biased so as to give a + // good safety margin for `offset` and `dictIx` at the cost of stealing bits + // from `len`. Almost all `len` values we will ever see in practice are + // 65535 or less, so stealing those bits does not matter much. + // + // If further compression is required, it would be feasible to implement + // Extent using 29 bits for the offset, 8 bits for the length and 11 bits + // for the dictionary index, giving a total of 6 bytes, provided that the + // data is packed into 3 uint16_t's. That would be a bit slower, though, + // due to the bit packing, and it would be more fragile, in the sense that + // it would fail for any object with more than 512MB of text segment, or + // with more than 2048 different `RuleSet`s. For the current (Sept 2020) + // libxul.so situation, though, it would work fine. + + Extent(uint32_t offset, uint32_t len, uint32_t dictIx) { + MOZ_RELEASE_ASSERT(len < (1 << 16)); + MOZ_RELEASE_ASSERT(dictIx < (1 << 16)); + mOffset = offset; + mLen = len; + mDictIx = dictIx; + } + inline uint32_t offset() const { return mOffset; } + inline uint32_t len() const { return mLen; } + inline uint32_t dictIx() const { return mDictIx; } + void setLen(uint32_t len) { + MOZ_RELEASE_ASSERT(len < (1 << 16)); + mLen = len; + } + void Print(void (*aLog)(const char*)) const { + char buf[64]; + SprintfLiteral(buf, "Extent(offs=0x%x, len=%u, dictIx=%u)", this->offset(), + this->len(), this->dictIx()); + aLog(buf); + } +}; + +static_assert(sizeof(Extent) == 8); + +//////////////////////////////////////////////////////////////// +// SecMap // +//////////////////////////////////////////////////////////////// + +// A SecMap may have zero address range, temporarily, whilst RuleSets +// are being added to it. But adding a zero-range SecMap to a PriMap +// will make it impossible to maintain the total order of the PriMap +// entries, and so that can't be allowed to happen. + +class SecMap { + public: + // In the constructor, `mapStartAVMA` and `mapLen` define the actual + // (in-process) virtual addresses covered by the SecMap. All RuleSets + // subsequently added to it by calling `AddRuleSet` must fall into this + // address range, and attempts to add ones outside the range will be + // ignored. This restriction exists because the type Extent (see below) + // indicates an address range for a RuleSet, but for reasons of compactness, + // it does not contain the start address of the range. Instead, it contains + // a 32-bit offset from the base address of the SecMap. This is also the + // reason why the map's size is a `uint32_t` and not a `uintptr_t`. + // + // The effect is to limit this mechanism to shared objects / executables + // whose text section size does not exceed 4GB (2^32 bytes). Given that, as + // of Sept 2020, libxul.so's text section size is around 120MB, this does + // not seem like much of a limitation. + // + // From the supplied `mapStartAVMA` and `mapLen`, fields `mMapMinAVMA` and + // `mMapMaxAVMA` are calculated. It is intended that no two SecMaps owned + // by the same PriMap contain overlapping address ranges, and the PriMap + // logic enforces that. + // + // Some invariants: + // + // mExtents is nonempty + // <=> mMapMinAVMA <= mMapMaxAVMA + // && mMapMinAVMA <= apply_delta(mExtents[0].offset()) + // && apply_delta(mExtents[#rulesets-1].offset() + // + mExtents[#rulesets-1].len() - 1) <= mMapMaxAVMA + // where + // apply_delta(off) = off + mMapMinAVMA + // + // This requires that no RuleSet has zero length. + // + // mExtents is empty + // <=> mMapMinAVMA > mMapMaxAVMA + // + // This doesn't constrain mMapMinAVMA and mMapMaxAVMA uniquely, so let's use + // mMapMinAVMA == 1 and mMapMaxAVMA == 0 to denote this case. + + SecMap(uintptr_t mapStartAVMA, uint32_t mapLen, void (*aLog)(const char*)); + ~SecMap(); + + // Binary search mRuleSets to find one that brackets |ia|, or nullptr + // if none is found. It's not allowable to do this until PrepareRuleSets + // has been called first. + RuleSet* FindRuleSet(uintptr_t ia); + + // Add a RuleSet to the collection. The rule is copied in. Calling + // this makes the map non-searchable. + void AddRuleSet(const RuleSet* rs, uintptr_t avma, uintptr_t len); + + // Add a PfxInstr to the vector of such instrs, and return the index + // in the vector. Calling this makes the map non-searchable. + uint32_t AddPfxInstr(PfxInstr pfxi); + + // Returns the entire vector of PfxInstrs. + const vector<PfxInstr>* GetPfxInstrs() { return &mPfxInstrs; } + + // Prepare the map for searching, by sorting it, de-overlapping entries and + // removing any resulting zero-length entries. At the start of this + // routine, all Extents should fall within [mMapMinAVMA, mMapMaxAVMA] and + // not have zero length, as a result of the checks in AddRuleSet(). + void PrepareRuleSets(); + + bool IsEmpty(); + + size_t Size() { return mExtents.size() + mDictionary.size(); } + + size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) const; + + // The extent of this SecMap as a whole. The extents of all contained + // RuleSets must fall inside this. See comment above for details. + uintptr_t mMapMinAVMA; + uintptr_t mMapMaxAVMA; + + private: + // False whilst adding entries; true once it is safe to call FindRuleSet. + // Transition (false->true) is caused by calling PrepareRuleSets(). + bool mUsable; + + // This is used to find and remove duplicate RuleSets while we are adding + // them to the SecMap. Almost all RuleSets are duplicates, so de-duping + // them is a huge space win. This is non-null while `mUsable` is false, and + // becomes null (is discarded) after the call to PrepareRuleSets, which + // copies all the entries into `mDictionary`. + mozilla::UniquePtr< + mozilla::HashMap<RuleSet, uint32_t, RuleSet, InfallibleAllocPolicy>> + mUniqifier; + + // This will contain final contents of `mUniqifier`, but ordered + // (implicitly) by the `uint32_t` value fields, for fast access. + vector<RuleSet> mDictionary; + + // A vector of Extents, sorted by offset value, nonoverlapping (post + // PrepareRuleSets()). + vector<Extent> mExtents; + + // A vector of PfxInstrs, which are referred to by the RuleSets. + // These are provided as a representation of Dwarf expressions + // (DW_CFA_val_expression, DW_CFA_expression, DW_CFA_def_cfa_expression), + // are relatively expensive to evaluate, and and are therefore + // expected to be used only occasionally. + // + // The vector holds a bunch of separate PfxInstr programs, each one + // starting with a PX_Start and terminated by a PX_End, all + // concatenated together. When a RuleSet can't recover a value + // using a self-contained LExpr, it uses a PFXEXPR whose mOffset is + // the index in this vector of start of the necessary PfxInstr program. + vector<PfxInstr> mPfxInstrs; + + // A logging sink, for debugging. + void (*mLog)(const char*); +}; + +} // namespace lul + +#endif // ndef LulMainInt_h diff --git a/tools/profiler/lul/platform-linux-lul.cpp b/tools/profiler/lul/platform-linux-lul.cpp new file mode 100644 index 0000000000..4027905c60 --- /dev/null +++ b/tools/profiler/lul/platform-linux-lul.cpp @@ -0,0 +1,75 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <stdio.h> +#include <signal.h> +#include <string.h> +#include <stdlib.h> +#include <time.h> + +#include "mozilla/ProfilerState.h" +#include "platform.h" +#include "PlatformMacros.h" +#include "LulMain.h" +#include "shared-libraries.h" +#include "AutoObjectMapper.h" + +// Contains miscellaneous helpers that are used to connect the Gecko Profiler +// and LUL. + +// Find out, in a platform-dependent way, where the code modules got +// mapped in the process' virtual address space, and get |aLUL| to +// load unwind info for them. +void read_procmaps(lul::LUL* aLUL) { + MOZ_ASSERT(aLUL->CountMappings() == 0); + +#if defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) + SharedLibraryInfo info = SharedLibraryInfo::GetInfoForSelf(); + + for (size_t i = 0; i < info.GetSize(); i++) { + const SharedLibrary& lib = info.GetEntry(i); + + std::string nativePath = lib.GetNativeDebugPath(); + + // We can use the standard POSIX-based mapper. + AutoObjectMapperPOSIX mapper(aLUL->mLog); + + // Ask |mapper| to map the object. Then hand its mapped address + // to NotifyAfterMap(). + void* image = nullptr; + size_t size = 0; + bool ok = mapper.Map(&image, &size, nativePath); + if (ok && image && size > 0) { + aLUL->NotifyAfterMap(lib.GetStart(), lib.GetEnd() - lib.GetStart(), + nativePath.c_str(), image); + } else if (!ok && lib.GetDebugName().IsEmpty()) { + // The object has no name and (as a consequence) the mapper failed to map + // it. This happens on Linux, where GetInfoForSelf() produces such a + // mapping for the VDSO. This is a problem on x86-{linux,android} because + // lack of knowledge about the mapped area inhibits LUL's special + // __kernel_syscall handling. Hence notify |aLUL| at least of the + // mapping, even though it can't read any unwind information for the area. + aLUL->NotifyExecutableArea(lib.GetStart(), lib.GetEnd() - lib.GetStart()); + } + + // |mapper| goes out of scope at this point and so its destructor + // unmaps the object. + } + +#else +# error "Unknown platform" +#endif +} + +// LUL needs a callback for its logging sink. +void logging_sink_for_LUL(const char* str) { + // These are only printed when Verbose logging is enabled (e.g. with + // MOZ_LOG="prof:5"). This is because LUL's logging is much more verbose than + // the rest of the profiler's logging, which occurs at the Info (3) and Debug + // (4) levels. + MOZ_LOG(gProfilerLog, mozilla::LogLevel::Verbose, + ("[%" PRIu64 "] %s", + uint64_t(profiler_current_process_id().ToNumber()), str)); +} diff --git a/tools/profiler/lul/platform-linux-lul.h b/tools/profiler/lul/platform-linux-lul.h new file mode 100644 index 0000000000..7c94299961 --- /dev/null +++ b/tools/profiler/lul/platform-linux-lul.h @@ -0,0 +1,19 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZ_PLATFORM_LINUX_LUL_H +#define MOZ_PLATFORM_LINUX_LUL_H + +#include "platform.h" + +// Find out, in a platform-dependent way, where the code modules got +// mapped in the process' virtual address space, and get |aLUL| to +// load unwind info for them. +void read_procmaps(lul::LUL* aLUL); + +// LUL needs a callback for its logging sink. +void logging_sink_for_LUL(const char* str); + +#endif /* ndef MOZ_PLATFORM_LINUX_LUL_H */ diff --git a/tools/profiler/moz.build b/tools/profiler/moz.build new file mode 100644 index 0000000000..ddb2ce5fff --- /dev/null +++ b/tools/profiler/moz.build @@ -0,0 +1,237 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +if CONFIG["MOZ_GECKO_PROFILER"]: + DEFINES["MOZ_REPLACE_MALLOC_PREFIX"] = "profiler" + XPIDL_MODULE = "profiler" + XPIDL_SOURCES += [ + "gecko/nsIProfiler.idl", + ] + EXPORTS += [ + "public/GeckoProfilerReporter.h", + "public/ProfilerChild.h", + "public/ProfilerCodeAddressService.h", + ] + UNIFIED_SOURCES += [ + "core/PageInformation.cpp", + "core/platform.cpp", + "core/ProfileBuffer.cpp", + "core/ProfileBufferEntry.cpp", + "core/ProfiledThreadData.cpp", + "core/ProfilerBacktrace.cpp", + "core/ProfilerCodeAddressService.cpp", + "core/ProfilerMarkers.cpp", + "gecko/ChildProfilerController.cpp", + "gecko/nsProfilerStartParams.cpp", + "gecko/ProfilerChild.cpp", + "gecko/ProfilerIOInterposeObserver.cpp", + ] + if CONFIG["MOZ_REPLACE_MALLOC"] and CONFIG["MOZ_PROFILER_MEMORY"]: + SOURCES += [ + "core/memory_hooks.cpp", # Non-unified because of order of #includes + ] + + XPCOM_MANIFESTS += [ + "gecko/components.conf", + ] + + if CONFIG["OS_TARGET"] == "Darwin": + # This file cannot be built in unified mode because it includes + # "nsLocalFile.h", which pulls in a system header which uses a type + # called TextRange, which conflicts with mozilla::TextRange due to + # a "using namespace mozilla;" declaration from a different file. + SOURCES += [ + "gecko/nsProfiler.cpp", + ] + else: + UNIFIED_SOURCES += [ + "gecko/nsProfiler.cpp", + ] + + if CONFIG["OS_TARGET"] in ("Android", "Linux"): + UNIFIED_SOURCES += [ + "core/ProfilerCPUFreq-linux-android.cpp", + ] + if CONFIG["OS_TARGET"] in ("Android", "Linux", "FreeBSD"): + if CONFIG["TARGET_CPU"] in ("arm", "aarch64", "x86", "x86_64", "mips64"): + UNIFIED_SOURCES += [ + "lul/AutoObjectMapper.cpp", + "lul/LulCommon.cpp", + "lul/LulDwarf.cpp", + "lul/LulDwarfSummariser.cpp", + "lul/LulElf.cpp", + "lul/LulMain.cpp", + "lul/platform-linux-lul.cpp", + ] + # These files cannot be built in unified mode because of name clashes with mozglue headers on Android. + SOURCES += [ + "core/shared-libraries-linux.cc", + ] + if not CONFIG["MOZ_CRASHREPORTER"]: + SOURCES += [ + "/toolkit/crashreporter/google-breakpad/src/common/linux/elfutils.cc", + "/toolkit/crashreporter/google-breakpad/src/common/linux/file_id.cc", + "/toolkit/crashreporter/google-breakpad/src/common/linux/linux_libc_support.cc", + "/toolkit/crashreporter/google-breakpad/src/common/linux/memory_mapped_file.cc", + ] + if not CONFIG["HAVE_GETCONTEXT"]: + SOURCES += [ + "/toolkit/crashreporter/google-breakpad/src/common/linux/breakpad_getcontext.S" + ] + if CONFIG["TARGET_CPU"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + UNIFIED_SOURCES += [ + "core/PowerCounters-linux.cpp", + ] + if CONFIG["TARGET_CPU"] == "arm" and CONFIG["OS_TARGET"] != "FreeBSD": + SOURCES += [ + "core/EHABIStackWalk.cpp", + ] + elif CONFIG["OS_TARGET"] == "Darwin": + UNIFIED_SOURCES += [ + "core/shared-libraries-macos.cc", + ] + if CONFIG["TARGET_CPU"] == "aarch64": + UNIFIED_SOURCES += [ + "core/PowerCounters-mac-arm64.cpp", + ] + if CONFIG["TARGET_CPU"] == "x86_64": + UNIFIED_SOURCES += [ + "core/PowerCounters-mac-amd64.cpp", + ] + elif CONFIG["OS_TARGET"] == "WINNT": + if CONFIG["CC_TYPE"] == "clang-cl": + UNIFIED_SOURCES += [ + "core/PowerCounters-win.cpp", + ] + SOURCES += { + "core/ETWTools.cpp", + } + SOURCES += [ + "core/ProfilerCPUFreq-win.cpp", + "core/shared-libraries-win32.cc", + ] + + LOCAL_INCLUDES += [ + "/caps", + "/docshell/base", + "/ipc/chromium/src", + "/mozglue/linker", + "/netwerk/base", + "/netwerk/protocol/http", + "/toolkit/components/jsoncpp/include", + "/toolkit/crashreporter/google-breakpad/src", + "/tools/profiler/core/", + "/tools/profiler/gecko/", + "/xpcom/base", + ] + + if CONFIG["OS_TARGET"] == "Android": + DEFINES["ANDROID_NDK_MAJOR_VERSION"] = CONFIG["ANDROID_NDK_MAJOR_VERSION"] + DEFINES["ANDROID_NDK_MINOR_VERSION"] = CONFIG["ANDROID_NDK_MINOR_VERSION"] + LOCAL_INCLUDES += [ + # We need access to Breakpad's getcontext(3) which is suitable for Android + "/toolkit/crashreporter/google-breakpad/src/common/android/include", + ] + + if CONFIG["MOZ_VTUNE"]: + DEFINES["MOZ_VTUNE_INSTRUMENTATION"] = True + UNIFIED_SOURCES += [ + "core/VTuneProfiler.cpp", + ] + + XPCSHELL_TESTS_MANIFESTS += ["tests/xpcshell/xpcshell.toml"] + MOCHITEST_CHROME_MANIFESTS += ["tests/chrome/chrome.toml"] + BROWSER_CHROME_MANIFESTS += ["tests/browser/browser.toml"] + +UNIFIED_SOURCES += [ + "core/MicroGeckoProfiler.cpp", + "core/ProfileAdditionalInformation.cpp", + "core/ProfilerBindings.cpp", + "core/ProfilerThreadRegistration.cpp", + "core/ProfilerThreadRegistrationData.cpp", + "core/ProfilerThreadRegistry.cpp", + "core/ProfilerUtils.cpp", + "gecko/ProfilerParent.cpp", +] + +IPDL_SOURCES += [ + "gecko/PProfiler.ipdl", + "gecko/ProfilerTypes.ipdlh", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +EXPORTS += [ + "public/ChildProfilerController.h", + "public/ETWTools.h", + "public/GeckoProfiler.h", + "public/MicroGeckoProfiler.h", + "public/ProfileAdditionalInformation.h", + "public/ProfilerBindings.h", + "public/ProfilerControl.h", + "public/ProfilerParent.h", + "public/ProfilerRustBindings.h", + "public/shared-libraries.h", +] + +EXPORTS.mozilla += [ + "public/ProfileBufferEntrySerializationGeckoExtensions.h", + "public/ProfileJSONWriter.h", + "public/ProfilerBandwidthCounter.h", + "public/ProfilerCounts.h", + "public/ProfilerLabels.h", + "public/ProfilerMarkers.h", + "public/ProfilerMarkersDetail.h", + "public/ProfilerMarkersPrerequisites.h", + "public/ProfilerMarkerTypes.h", + "public/ProfilerRunnable.h", + "public/ProfilerState.h", + "public/ProfilerThreadPlatformData.h", + "public/ProfilerThreadRegistration.h", + "public/ProfilerThreadRegistrationData.h", + "public/ProfilerThreadRegistrationInfo.h", + "public/ProfilerThreadRegistry.h", + "public/ProfilerThreadSleep.h", + "public/ProfilerThreadState.h", + "public/ProfilerUtils.h", +] + +GeneratedFile( + "rust-api/src/gecko_bindings/profiling_categories.rs", + script="../../mozglue/baseprofiler/build/generate_profiling_categories.py", + entry_point="generate_rust_enums", + inputs=["../../mozglue/baseprofiler/build/profiling_categories.yaml"], +) + +CONFIGURE_SUBST_FILES += [ + "rust-api/extra-bindgen-flags", +] + + +if CONFIG["COMPILE_ENVIRONMENT"]: + CbindgenHeader("profiler_ffi_generated.h", inputs=["rust-api"]) + + EXPORTS.mozilla += [ + "!profiler_ffi_generated.h", + ] + +USE_LIBS += [ + "jsoncpp", +] + +FINAL_LIBRARY = "xul" + +if CONFIG["ENABLE_TESTS"]: + DIRS += ["tests/gtest"] + +if CONFIG["CC_TYPE"] in ("clang", "gcc"): + CXXFLAGS += [ + "-Wno-error=stack-protector", + "-Wno-ignored-qualifiers", # due to use of breakpad headers + ] + +with Files("**"): + BUG_COMPONENT = ("Core", "Gecko Profiler") diff --git a/tools/profiler/public/ChildProfilerController.h b/tools/profiler/public/ChildProfilerController.h new file mode 100644 index 0000000000..8febc25b65 --- /dev/null +++ b/tools/profiler/public/ChildProfilerController.h @@ -0,0 +1,71 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ChildProfilerController_h +#define ChildProfilerController_h + +#include "base/process.h" +#include "mozilla/Attributes.h" +#include "mozilla/ipc/ProtocolUtils.h" +#include "mozilla/DataMutex.h" +#include "mozilla/RefPtr.h" +#include "nsISupportsImpl.h" +#include "nsStringFwd.h" +#include "ProfileAdditionalInformation.h" + +namespace mozilla { + +class ProfilerChild; +class PProfilerChild; +class PProfilerParent; + +// ChildProfilerController manages the setup and teardown of ProfilerChild. +// It's used on the main thread. +// It manages a background thread that ProfilerChild runs on. +class ChildProfilerController final { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ChildProfilerController) + +#ifdef MOZ_GECKO_PROFILER + static already_AddRefed<ChildProfilerController> Create( + mozilla::ipc::Endpoint<PProfilerChild>&& aEndpoint); + + [[nodiscard]] ProfileAndAdditionalInformation + GrabShutdownProfileAndShutdown(); + void Shutdown(); + + private: + ChildProfilerController(); + ~ChildProfilerController(); + void Init(mozilla::ipc::Endpoint<PProfilerChild>&& aEndpoint); + void ShutdownAndMaybeGrabShutdownProfileFirst( + ProfileAndAdditionalInformation* aOutShutdownProfileInformation); + + // Called on mThread: + void SetupProfilerChild(mozilla::ipc::Endpoint<PProfilerChild>&& aEndpoint); + void ShutdownProfilerChild( + ProfileAndAdditionalInformation* aOutShutdownProfileInformation); + + RefPtr<ProfilerChild> mProfilerChild; // only accessed on mThread + DataMutex<RefPtr<nsIThread>> mThread; +#else + static already_AddRefed<ChildProfilerController> Create( + mozilla::ipc::Endpoint<PProfilerChild>&& aEndpoint) { + return nullptr; + } + [[nodiscard]] ProfileAndAdditionalInformation + GrabShutdownProfileAndShutdown() { + return ProfileAndAdditionalInformation(std::move(EmptyCString())); + } + void Shutdown() {} + + private: + ~ChildProfilerController() {} +#endif // MOZ_GECKO_PROFILER +}; + +} // namespace mozilla + +#endif // ChildProfilerController_h diff --git a/tools/profiler/public/ETWTools.h b/tools/profiler/public/ETWTools.h new file mode 100644 index 0000000000..a1d986a6fd --- /dev/null +++ b/tools/profiler/public/ETWTools.h @@ -0,0 +1,391 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ETWTools_h +#define ETWTools_h + +#include "mozilla/BaseProfilerMarkers.h" +#include "mozilla/TimeStamp.h" +#include "nsString.h" + +#if defined(XP_WIN) && !defined(RUST_BINDGEN) && !defined(__MINGW32__) +# include "mozilla/ProfilerState.h" + +# include <windows.h> +# include <TraceLoggingProvider.h> +# include <vector> + +namespace ETW { + +extern std::atomic<ULONGLONG> gETWCollectionMask; + +// Forward-declare the g_hMyComponentProvider variable that you will use for +// tracing in this component +TRACELOGGING_DECLARE_PROVIDER(kFirefoxTraceLoggingProvider); + +void Init(); +void Shutdown(); + +// This describes the base fields for all markers (information extracted from +// MarkerOptions. +struct BaseMarkerDescription { + using MS = mozilla::MarkerSchema; + static constexpr MS::PayloadField PayloadFields[] = { + {"StartTime", MS::InputType::TimeStamp, "Start Time"}, + {"EndTime", MS::InputType::TimeStamp, "End Time"}, + {"InnerWindowId", MS::InputType::Uint64, "Inner Window ID"}, + {"CategoryPair", MS::InputType::Uint32, "Category Pair"}}; +}; + +// This is the MarkerType object for markers with no statically declared type, +// their name is written dynamically. +struct SimpleMarkerType { + using MS = mozilla::MarkerSchema; + static constexpr const char* Name = "SimpleMarker"; + static constexpr MS::PayloadField PayloadFields[] = { + {"MarkerName", MS::InputType::CString, "Simple Marker Name"}}; +}; + +// This gets the space required in the Tlg static struct to pack the fields. +template <typename T> +constexpr std::size_t GetPackingSpace() { + size_t length = 0; + for (size_t i = 0; i < std::size(T::PayloadFields); i++) { + length += std::string_view{T::PayloadFields[i].Key}.size() + 1; + length += sizeof(uint8_t); + } + return length; +} + +// Convert our InputType to Tlgs input type. +constexpr uint8_t GetTlgInputType(mozilla::MarkerSchema::InputType aInput) { + using InputType = mozilla::MarkerSchema::InputType; + switch (aInput) { + case InputType::Boolean: + return TlgInUINT8; + case InputType::Uint32: + return TlgInUINT32; + case InputType::Uint64: + case InputType::TimeStamp: + case InputType::TimeDuration: + return TlgInUINT64; + case InputType::CString: + return TlgInANSISTRING; + case InputType::String: + return TlgInUNICODESTRING; + default: + return 0; + } +} + +// This class represents the format ETW TraceLogging uses to describe its +// metadata. Since we have an abstraction layer we need to statically +// declare this ourselves and we cannot rely on TraceLogging's official +// macros. This does mean if TraceLogging ships big changes (on an SDK update) +// we may need to adapt. +__pragma(pack(push, 1)) _tlgEvtTagDecl(0); +template <typename T> +struct StaticMetaData { + _tlgEventMetadata_t metaData; + _tlgEvtTagType _tlgEvtTag; + char name[std::string_view{T::Name}.size() + 1]; + char fieldStorage[GetPackingSpace<BaseMarkerDescription>() + + GetPackingSpace<T>()]; + + // constexpr constructor + constexpr StaticMetaData() + : metaData{_TlgBlobEvent4, + 11, // WINEVENT_CHANNEL_TRACELOGGING, + 5, // Verbose + 0, + 0, + sizeof(StaticMetaData) - _tlg_EVENT_METADATA_PREAMBLE - 1}, + _tlgEvtTag(_tlgEvtTagInit) { + for (uint32_t i = 0; i < std::string_view{T::Name}.size() + 1; i++) { + name[i] = T::Name[i]; + } + + size_t pos = 0; + for (uint32_t i = 0; i < std::size(BaseMarkerDescription::PayloadFields); + i++) { + for (size_t c = 0; + c < std::string_view{BaseMarkerDescription::PayloadFields[i].Key} + .size() + + 1; + c++) { + fieldStorage[pos++] = BaseMarkerDescription::PayloadFields[i].Key[c]; + } + fieldStorage[pos++] = + GetTlgInputType(BaseMarkerDescription::PayloadFields[i].InputTy); + } + for (uint32_t i = 0; i < std::size(T::PayloadFields); i++) { + for (size_t c = 0; + c < std::string_view{T::PayloadFields[i].Key}.size() + 1; c++) { + fieldStorage[pos++] = T::PayloadFields[i].Key[c]; + } + fieldStorage[pos++] = GetTlgInputType(T::PayloadFields[i].InputTy); + } + } +}; +__pragma(pack(pop)); + +// This defines the amount of storage available on the stack to store POD +// values. +const size_t kStackStorage = 512; + +struct PayloadBuffer { + EVENT_DATA_DESCRIPTOR* mDescriptors = nullptr; + size_t mOffset = 0; + std::array<char, kStackStorage> mStorage; +}; + +// This processes POD objects and stores them in a temporary buffer. +// Theoretically we could probably avoid these assignments when passed a POD +// variable we know is going to be alive but that would require some more +// template magic. +template <typename T> +static void CreateDataDescForPayload(PayloadBuffer& aBuffer, + EVENT_DATA_DESCRIPTOR& aDescriptor, + const T& aPayload) { + static_assert(std::is_pod<T>::value, + "Writing a non-POD payload requires template specialization."); + + // Ensure we never overflow our stack buffer. + MOZ_RELEASE_ASSERT((aBuffer.mOffset + sizeof(T)) < kStackStorage); + + T* storedValue = + reinterpret_cast<T*>(aBuffer.mStorage.data() + aBuffer.mOffset); + *storedValue = aPayload; + aBuffer.mOffset += sizeof(T); + + EventDataDescCreate(&aDescriptor, storedValue, sizeof(T)); +} + +template <> +inline void CreateDataDescForPayload<mozilla::ProfilerString8View>( + PayloadBuffer& aBuffer, EVENT_DATA_DESCRIPTOR& aDescriptor, + const mozilla::ProfilerString8View& aPayload) { + EventDataDescCreate(&aDescriptor, aPayload.StringView().data(), + aPayload.StringView().size() + 1); +} + +template <> +inline void CreateDataDescForPayload<mozilla::TimeStamp>( + PayloadBuffer& aBuffer, EVENT_DATA_DESCRIPTOR& aDescriptor, + const mozilla::TimeStamp& aPayload) { + if (aPayload.RawQueryPerformanceCounterValue().isNothing()) { + // This should never happen? + EventDataDescCreate(&aDescriptor, nullptr, 0); + return; + } + + CreateDataDescForPayload(aBuffer, aDescriptor, + aPayload.RawQueryPerformanceCounterValue().value()); +} + +template <> +inline void CreateDataDescForPayload<mozilla::TimeDuration>( + PayloadBuffer& aBuffer, EVENT_DATA_DESCRIPTOR& aDescriptor, + const mozilla::TimeDuration& aPayload) { + CreateDataDescForPayload(aBuffer, aDescriptor, aPayload.ToMilliseconds()); +} + +// For reasons that are beyond me if this isn't marked inline it generates an +// unused function warning despite being a template specialization. +template <typename T> +inline void CreateDataDescForPayload(PayloadBuffer& aBuffer, + EVENT_DATA_DESCRIPTOR& aDescriptor, + const nsTString<T>& aPayload) { + EventDataDescCreate(&aDescriptor, aPayload.BeginReading(), + (aPayload.Length() + 1) * sizeof(T)); +} +template <typename T> +inline void CreateDataDescForPayload(PayloadBuffer& aBuffer, + EVENT_DATA_DESCRIPTOR& aDescriptor, + const nsTSubstring<T>& aPayload) { + EventDataDescCreate(&aDescriptor, aPayload.BeginReading(), + (aPayload.Length() + 1) * sizeof(T)); +} + +// Template specialization that writes out empty data descriptors for an empty +// Maybe<T> +template <typename T> +void CreateDataDescForPayload(PayloadBuffer& aBuffer, + EVENT_DATA_DESCRIPTOR& aDescriptor, + const mozilla::Maybe<T>& aPayload) { + if (aPayload.isNothing()) { + EventDataDescCreate(&aDescriptor, nullptr, 0); + } else { + CreateDataDescForPayload(aBuffer, aDescriptor, *aPayload); + } +} + +template <typename T, typename = void> +struct MarkerSupportsETW : std::false_type {}; +template <typename T> +struct MarkerSupportsETW<T, std::void_t<decltype(T::PayloadFields)>> + : std::true_type {}; + +template <typename T, typename = void> +struct MarkerHasTranslator : std::false_type {}; +template <typename T> +struct MarkerHasTranslator< + T, std::void_t<decltype(T::TranslateMarkerInputToSchema)>> + : std::true_type {}; + +struct BaseEventStorage { + uint64_t mStartTime; + uint64_t mEndTime; + uint64_t mWindowID; + uint32_t mCategoryPair; +}; + +static inline void StoreBaseEventDataDesc( + BaseEventStorage& aStorage, EVENT_DATA_DESCRIPTOR* aDescriptors, + const mozilla::MarkerCategory& aCategory, + const mozilla::MarkerOptions& aOptions) { + if (!aOptions.IsTimingUnspecified()) { + aStorage.mStartTime = + aOptions.Timing().StartTime().RawQueryPerformanceCounterValue().value(); + aStorage.mEndTime = + aOptions.Timing().EndTime().RawQueryPerformanceCounterValue().value(); + } + if (!aOptions.InnerWindowId().IsUnspecified()) { + aStorage.mWindowID = aOptions.InnerWindowId().Id(); + } + aStorage.mCategoryPair = uint32_t(aCategory.CategoryPair()); + EventDataDescCreate(&aDescriptors[2], &aStorage.mStartTime, sizeof(uint64_t)); + EventDataDescCreate(&aDescriptors[3], &aStorage.mEndTime, sizeof(uint64_t)); + EventDataDescCreate(&aDescriptors[4], &aStorage.mWindowID, sizeof(uint64_t)); + EventDataDescCreate(&aDescriptors[5], &aStorage.mCategoryPair, + sizeof(uint32_t)); +} + +// This is used for markers with no explicit type or markers which have not +// been converted to the updated schema yet. +static inline void EmitETWMarker(const mozilla::ProfilerString8View& aName, + const mozilla::MarkerCategory& aCategory, + const mozilla::MarkerOptions& aOptions = {}) { + if (!(gETWCollectionMask & + uint64_t(mozilla::MarkerSchema::ETWMarkerGroup::Generic))) { + return; + } + + static const __declspec(allocate(_tlgSegMetadataEvents)) __declspec( + align(1)) constexpr StaticMetaData<SimpleMarkerType> + staticData; + + std::array<EVENT_DATA_DESCRIPTOR, 7> descriptors = {}; + + // This is storage space allocated on the stack for POD values. + BaseEventStorage dataStorage = {}; + + StoreBaseEventDataDesc(dataStorage, descriptors.data(), aCategory, + std::move(aOptions)); + + EventDataDescCreate(&descriptors[6], aName.StringView().data(), + aName.StringView().size() + 1); + _tlgWriteTransfer(kFirefoxTraceLoggingProvider, &staticData.metaData.Channel, + NULL, NULL, descriptors.size(), descriptors.data()); +} + +template <typename MarkerType, typename... PayloadArguments> +static inline void EmitETWMarker(const mozilla::ProfilerString8View& aName, + const mozilla::MarkerCategory& aCategory, + const mozilla::MarkerOptions& aOptions, + MarkerType aMarkerType, + const PayloadArguments&... aPayloadArguments) { + // If our MarkerType has not been made to support ETW, emit only the base + // event. Avoid attempting to compile the rest of the function. + if constexpr (!MarkerSupportsETW<MarkerType>::value) { + return EmitETWMarker(aName, aCategory, aOptions); + } else { + if (!(gETWCollectionMask & uint64_t(MarkerType::Group))) { + return; + } + + static const __declspec(allocate(_tlgSegMetadataEvents)) __declspec( + align(1)) constexpr StaticMetaData<MarkerType> + staticData; + + // Allocate the exact amount of descriptors required by this event. + std::array<EVENT_DATA_DESCRIPTOR, + 2 + std::size(MarkerType::PayloadFields) + + std::size(BaseMarkerDescription::PayloadFields)> + descriptors = {}; + + // Memory allocated on the stack for storing intermediate values. + BaseEventStorage dataStorage = {}; + PayloadBuffer buffer; + + StoreBaseEventDataDesc(dataStorage, descriptors.data(), aCategory, + aOptions); + + if constexpr (MarkerHasTranslator<MarkerType>::value) { + // When this function is implemented the arguments are passed back to the + // MarkerType object which is expected to call OutputMarkerSchema with + // the correct argument format. + buffer.mDescriptors = descriptors.data() + 2 + + std::size(BaseMarkerDescription::PayloadFields); + MarkerType::TranslateMarkerInputToSchema(&buffer, aPayloadArguments...); + } else { + const size_t argCount = sizeof...(PayloadArguments); + static_assert( + argCount == std::size(MarkerType::PayloadFields), + "Number and type of fields must be equal to number and type of " + "payload arguments. If this is not the case a " + "TranslateMarkerInputToSchema function must be defined."); + size_t i = 2 + std::size(BaseMarkerDescription::PayloadFields); + (CreateDataDescForPayload(buffer, descriptors[i++], aPayloadArguments), + ...); + } + + _tlgWriteTransfer(kFirefoxTraceLoggingProvider, + &staticData.metaData.Channel, NULL, NULL, + descriptors.size(), descriptors.data()); + } +} + +// This function allows markers to specify a translator function for when +// their arguments to profiler_add_marker do not exactly match the schema or +// when they need to make other adjustments to the data. +template <typename MarkerType, typename... PayloadArguments> +void OutputMarkerSchema(void* aContext, MarkerType aMarkerType, + const PayloadArguments&... aPayloadArguments) { + const size_t argCount = sizeof...(PayloadArguments); + static_assert(argCount == std::size(MarkerType::PayloadFields), + "Number and type of fields must be equal to number and type of " + "payload arguments."); + + PayloadBuffer* buffer = static_cast<PayloadBuffer*>(aContext); + size_t i = 0; + (CreateDataDescForPayload(*buffer, buffer->mDescriptors[i++], + aPayloadArguments), + ...); +} +} // namespace ETW + +#else +namespace ETW { +static inline void Init() {} +static inline void Shutdown() {} +static inline void EmitETWMarker(const mozilla::ProfilerString8View& aName, + const mozilla::MarkerCategory& aCategory, + const mozilla::MarkerOptions& aOptions = {}) {} +template <typename MarkerType, typename... PayloadArguments> +static inline void EmitETWMarker(const mozilla::ProfilerString8View& aName, + const mozilla::MarkerCategory& aCategory, + const mozilla::MarkerOptions& aOptions, + MarkerType aMarkerType, + const PayloadArguments&... aPayloadArguments) { +} +template <typename MarkerType, typename... PayloadArguments> +void OutputMarkerSchema(void* aContext, MarkerType aMarkerType, + const PayloadArguments&... aPayloadArguments) {} +} // namespace ETW +#endif + +#endif // ETWTools_h diff --git a/tools/profiler/public/GeckoProfiler.h b/tools/profiler/public/GeckoProfiler.h new file mode 100644 index 0000000000..f7c045297e --- /dev/null +++ b/tools/profiler/public/GeckoProfiler.h @@ -0,0 +1,435 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// The Gecko Profiler is an always-on profiler that takes fast and low overhead +// samples of the program execution using only userspace functionality for +// portability. The goal of this module is to provide performance data in a +// generic cross-platform way without requiring custom tools or kernel support. +// +// Samples are collected to form a timeline with optional timeline event +// (markers) used for filtering. The samples include both native stacks and +// platform-independent "label stack" frames. + +#ifndef GeckoProfiler_h +#define GeckoProfiler_h + +// Everything in here is also safe to include unconditionally, and only defines +// empty macros if MOZ_GECKO_PROFILER is unset. +// If your file only uses particular APIs (e.g., only markers), please consider +// including only the needed headers instead of this one, to reduce compilation +// dependencies. +#include "BaseProfiler.h" +#include "ProfileAdditionalInformation.h" +#include "mozilla/ProfilerCounts.h" +#include "mozilla/ProfilerLabels.h" +#include "mozilla/ProfilerMarkers.h" +#include "mozilla/ProfilerState.h" +#include "mozilla/ProfilerThreadSleep.h" +#include "mozilla/ProfilerThreadState.h" +#include "mozilla/ProgressLogger.h" +#include "mozilla/Result.h" +#include "mozilla/ResultVariant.h" + +#ifndef MOZ_GECKO_PROFILER + +# include "mozilla/UniquePtr.h" + +// This file can be #included unconditionally. However, everything within this +// file must be guarded by a #ifdef MOZ_GECKO_PROFILER, *except* for the +// following macros and functions, which encapsulate the most common operations +// and thus avoid the need for many #ifdefs. + +# define PROFILER_REGISTER_THREAD(name) +# define PROFILER_UNREGISTER_THREAD() +# define AUTO_PROFILER_REGISTER_THREAD(name) + +# define PROFILER_JS_INTERRUPT_CALLBACK() + +# define PROFILER_SET_JS_CONTEXT(cx) +# define PROFILER_CLEAR_JS_CONTEXT() + +// Function stubs for when MOZ_GECKO_PROFILER is not defined. + +// This won't be used, it's just there to allow the empty definition of +// `profiler_get_backtrace`. +struct ProfilerBacktrace {}; +using UniqueProfilerBacktrace = mozilla::UniquePtr<ProfilerBacktrace>; + +// Get/Capture-backtrace functions can return nullptr or false, the result +// should be fed to another empty macro or stub anyway. + +static inline UniqueProfilerBacktrace profiler_get_backtrace() { + return nullptr; +} + +// This won't be used, it's just there to allow the empty definitions of +// `profiler_capture_backtrace_into` and `profiler_capture_backtrace`. +struct ProfileChunkedBuffer {}; + +static inline bool profiler_capture_backtrace_into( + mozilla::ProfileChunkedBuffer& aChunkedBuffer, + mozilla::StackCaptureOptions aCaptureOptions) { + return false; +} +static inline mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> +profiler_capture_backtrace() { + return nullptr; +} + +static inline void profiler_set_process_name( + const nsACString& aProcessName, const nsACString* aETLDplus1 = nullptr) {} + +static inline void profiler_received_exit_profile( + const nsACString& aExitProfile) {} + +static inline void profiler_register_page(uint64_t aTabID, + uint64_t aInnerWindowID, + const nsCString& aUrl, + uint64_t aEmbedderInnerWindowID, + bool aIsPrivateBrowsing) {} +static inline void profiler_unregister_page(uint64_t aRegisteredInnerWindowID) { +} + +static inline void GetProfilerEnvVarsForChildProcess( + std::function<void(const char* key, const char* value)>&& aSetEnv) {} + +static inline void profiler_record_wakeup_count( + const nsACString& aProcessType) {} + +#else // !MOZ_GECKO_PROFILER + +# include "js/ProfilingStack.h" +# include "mozilla/Assertions.h" +# include "mozilla/Atomics.h" +# include "mozilla/Attributes.h" +# include "mozilla/BaseProfilerRAIIMacro.h" +# include "mozilla/Maybe.h" +# include "mozilla/PowerOfTwo.h" +# include "mozilla/ThreadLocal.h" +# include "mozilla/TimeStamp.h" +# include "mozilla/UniquePtr.h" +# include "nscore.h" +# include "nsINamed.h" +# include "nsString.h" +# include "nsThreadUtils.h" + +# include <functional> +# include <stdint.h> + +class ProfilerBacktrace; +class ProfilerCodeAddressService; +struct JSContext; + +namespace mozilla { +class ProfileBufferControlledChunkManager; +class ProfileChunkedBuffer; +namespace baseprofiler { +class SpliceableJSONWriter; +} // namespace baseprofiler +} // namespace mozilla +class nsIURI; + +enum class ProfilerError { + IsInactive, + JsonGenerationFailed, +}; + +template <typename T> +using ProfilerResult = mozilla::Result<T, ProfilerError>; + +//--------------------------------------------------------------------------- +// Give information to the profiler +//--------------------------------------------------------------------------- + +// Register/unregister threads with the profiler. Both functions operate the +// same whether the profiler is active or inactive. +# define PROFILER_REGISTER_THREAD(name) \ + do { \ + char stackTop; \ + profiler_register_thread(name, &stackTop); \ + } while (0) +# define PROFILER_UNREGISTER_THREAD() profiler_unregister_thread() +ProfilingStack* profiler_register_thread(const char* name, void* guessStackTop); +void profiler_unregister_thread(); + +// Registers a DOM Window (the JS global `window`) with the profiler. Each +// Window _roughly_ corresponds to a single document loaded within a +// browsing context. Both the Window Id and Browser Id are recorded to allow +// correlating different Windows loaded within the same tab or frame element. +// +// We register pages for each navigations but we do not register +// history.pushState or history.replaceState since they correspond to the same +// Inner Window ID. When a browsing context is first loaded, the first url +// loaded in it will be about:blank. Because of that, this call keeps the first +// non-about:blank registration of window and discards the previous one. +// +// "aTabID" is the BrowserId of that document belongs to. +// That's used to determine the tab of that page. +// "aInnerWindowID" is the ID of the `window` global object of that +// document. +// "aUrl" is the URL of the page. +// "aEmbedderInnerWindowID" is the inner window id of embedder. It's used to +// determine sub documents of a page. +// "aIsPrivateBrowsing" is true if this browsing context happens in a +// private browsing context. +void profiler_register_page(uint64_t aTabID, uint64_t aInnerWindowID, + const nsCString& aUrl, + uint64_t aEmbedderInnerWindowID, + bool aIsPrivateBrowsing); +// Unregister page with the profiler. +// +// Take a Inner Window ID and unregister the page entry that has the same ID. +void profiler_unregister_page(uint64_t aRegisteredInnerWindowID); + +// Remove all registered and unregistered pages in the profiler. +void profiler_clear_all_pages(); + +class BaseProfilerCount; +void profiler_add_sampled_counter(BaseProfilerCount* aCounter); +void profiler_remove_sampled_counter(BaseProfilerCount* aCounter); + +// Register and unregister a thread within a scope. +# define AUTO_PROFILER_REGISTER_THREAD(name) \ + mozilla::AutoProfilerRegisterThread PROFILER_RAII(name) + +enum class SamplingState { + JustStopped, // Sampling loop has just stopped without sampling, between the + // callback registration and now. + SamplingPaused, // Profiler is active but sampling loop has gone through a + // pause. + NoStackSamplingCompleted, // A full sampling loop has completed in + // no-stack-sampling mode. + SamplingCompleted // A full sampling loop has completed. +}; + +using PostSamplingCallback = std::function<void(SamplingState)>; + +// Install a callback to be invoked at the end of the next sampling loop. +// - `false` if profiler is not active, `aCallback` will stay untouched. +// - `true` if `aCallback` was successfully moved-from into internal storage, +// and *will* be invoked at the end of the next sampling cycle. Note that this +// will happen on the Sampler thread, and will block further sampling, so +// please be mindful not to block for a long time (e.g., just dispatch a +// runnable to another thread.) Calling profiler functions from the callback +// is allowed. +[[nodiscard]] bool profiler_callback_after_sampling( + PostSamplingCallback&& aCallback); + +// Called by the JSRuntime's operation callback. This is used to start profiling +// on auxiliary threads. Operates the same whether the profiler is active or +// not. +# define PROFILER_JS_INTERRUPT_CALLBACK() profiler_js_interrupt_callback() +void profiler_js_interrupt_callback(); + +// Set and clear the current thread's JSContext. +# define PROFILER_SET_JS_CONTEXT(cx) profiler_set_js_context(cx) +# define PROFILER_CLEAR_JS_CONTEXT() profiler_clear_js_context() +void profiler_set_js_context(JSContext* aCx); +void profiler_clear_js_context(); + +//--------------------------------------------------------------------------- +// Get information from the profiler +//--------------------------------------------------------------------------- + +// Get the chunk manager used in the current profiling session, or null. +mozilla::ProfileBufferControlledChunkManager* +profiler_get_controlled_chunk_manager(); + +// The number of milliseconds since the process started. Operates the same +// whether the profiler is active or inactive. +double profiler_time(); + +// An object of this class is passed to profiler_suspend_and_sample_thread(). +// For each stack frame, one of the Collect methods will be called. +class ProfilerStackCollector { + public: + // Some collectors need to worry about possibly overwriting previous + // generations of data. If that's not an issue, this can return Nothing, + // which is the default behaviour. + virtual mozilla::Maybe<uint64_t> SamplePositionInBuffer() { + return mozilla::Nothing(); + } + virtual mozilla::Maybe<uint64_t> BufferRangeStart() { + return mozilla::Nothing(); + } + + // This method will be called once if the thread being suspended is the main + // thread. Default behaviour is to do nothing. + virtual void SetIsMainThread() {} + + // WARNING: The target thread is suspended when the Collect methods are + // called. Do not try to allocate or acquire any locks, or you could + // deadlock. The target thread will have resumed by the time this function + // returns. + + virtual void CollectNativeLeafAddr(void* aAddr) = 0; + + virtual void CollectJitReturnAddr(void* aAddr) = 0; + + virtual void CollectWasmFrame(const char* aLabel) = 0; + + virtual void CollectProfilingStackFrame( + const js::ProfilingStackFrame& aFrame) = 0; +}; + +// This method suspends the thread identified by aThreadId, samples its +// profiling stack, JS stack, and (optionally) native stack, passing the +// collected frames into aCollector. aFeatures dictates which compiler features +// are used. |Leaf| is the only relevant one. +// Use `ProfilerThreadId{}` (unspecified) to sample the current thread. +void profiler_suspend_and_sample_thread(ProfilerThreadId aThreadId, + uint32_t aFeatures, + ProfilerStackCollector& aCollector, + bool aSampleNative = true); + +struct ProfilerBacktraceDestructor { + void operator()(ProfilerBacktrace*); +}; + +using UniqueProfilerBacktrace = + mozilla::UniquePtr<ProfilerBacktrace, ProfilerBacktraceDestructor>; + +// Immediately capture the current thread's call stack, store it in the provided +// buffer (usually to avoid allocations if you can construct the buffer on the +// stack). Returns false if unsuccessful, or if the profiler is inactive. +bool profiler_capture_backtrace_into( + mozilla::ProfileChunkedBuffer& aChunkedBuffer, + mozilla::StackCaptureOptions aCaptureOptions); + +// Immediately capture the current thread's call stack, and return it in a +// ProfileChunkedBuffer (usually for later use in MarkerStack::TakeBacktrace()). +// May be null if unsuccessful, or if the profiler is inactive. +mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> profiler_capture_backtrace(); + +// Immediately capture the current thread's call stack, and return it in a +// ProfilerBacktrace (usually for later use in marker function that take a +// ProfilerBacktrace). May be null if unsuccessful, or if the profiler is +// inactive. +UniqueProfilerBacktrace profiler_get_backtrace(); + +struct ProfilerStats { + unsigned n = 0; + double sum = 0; + double min = std::numeric_limits<double>::max(); + double max = 0; + void Count(double v) { + ++n; + sum += v; + if (v < min) { + min = v; + } + if (v > max) { + max = v; + } + } +}; + +struct ProfilerBufferInfo { + // Index of the oldest entry. + uint64_t mRangeStart; + // Index of the newest entry. + uint64_t mRangeEnd; + // Buffer capacity in number of 8-byte entries. + uint32_t mEntryCount; + // Sampling stats: Interval between successive samplings. + ProfilerStats mIntervalsUs; + // Sampling stats: Total sampling duration. (Split detail below.) + ProfilerStats mOverheadsUs; + // Sampling stats: Time to acquire the lock before sampling. + ProfilerStats mLockingsUs; + // Sampling stats: Time to discard expired data. + ProfilerStats mCleaningsUs; + // Sampling stats: Time to collect counter data. + ProfilerStats mCountersUs; + // Sampling stats: Time to sample thread stacks. + ProfilerStats mThreadsUs; +}; + +// Get information about the current buffer status. +// Returns Nothing() if the profiler is inactive. +// +// This information may be useful to a user-interface displaying the current +// status of the profiler, allowing the user to get a sense for how fast the +// buffer is being written to, and how much data is visible. +mozilla::Maybe<ProfilerBufferInfo> profiler_get_buffer_info(); + +// Record through glean how many times profiler_thread_wake has been +// called. +void profiler_record_wakeup_count(const nsACString& aProcessType); + +//--------------------------------------------------------------------------- +// Output profiles +//--------------------------------------------------------------------------- + +// Set a user-friendly process name, used in JSON stream. Allows an optional +// detailed name which may include private info (eTLD+1 in fission) +void profiler_set_process_name(const nsACString& aProcessName, + const nsACString* aETLDplus1 = nullptr); + +// Record an exit profile from a child process. +void profiler_received_exit_profile(const nsACString& aExitProfile); + +// Get the profile encoded as a JSON string. A no-op (returning nullptr) if the +// profiler is inactive. +// If aIsShuttingDown is true, the current time is included as the process +// shutdown time in the JSON's "meta" object. +mozilla::UniquePtr<char[]> profiler_get_profile(double aSinceTime = 0, + bool aIsShuttingDown = false); + +// Write the profile for this process (excluding subprocesses) into aWriter. +// Returns a failed result if the profiler is inactive. +ProfilerResult<mozilla::ProfileGenerationAdditionalInformation> +profiler_stream_json_for_this_process( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter, double aSinceTime = 0, + bool aIsShuttingDown = false, + ProfilerCodeAddressService* aService = nullptr, + mozilla::ProgressLogger aProgressLogger = {}); + +// Get the profile and write it into a file. A no-op if the profile is +// inactive. +// +// This function is 'extern "C"' so that it is easily callable from a debugger +// in a build without debugging information (a workaround for +// http://llvm.org/bugs/show_bug.cgi?id=22211). +extern "C" { +void profiler_save_profile_to_file(const char* aFilename); +} + +//--------------------------------------------------------------------------- +// RAII classes +//--------------------------------------------------------------------------- + +namespace mozilla { + +// Convenience class to register and unregister a thread with the profiler. +// Needs to be the first object on the stack of the thread. +class MOZ_RAII AutoProfilerRegisterThread final { + public: + explicit AutoProfilerRegisterThread(const char* aName) { + profiler_register_thread(aName, this); + } + + ~AutoProfilerRegisterThread() { profiler_unregister_thread(); } + + private: + AutoProfilerRegisterThread(const AutoProfilerRegisterThread&) = delete; + AutoProfilerRegisterThread& operator=(const AutoProfilerRegisterThread&) = + delete; +}; + +// Get the MOZ_PROFILER_STARTUP* environment variables that should be +// supplied to a child process that is about to be launched, in order +// to make that child process start with the same profiler settings as +// in the current process. The given function is invoked once for +// each variable to be set. +void GetProfilerEnvVarsForChildProcess( + std::function<void(const char* key, const char* value)>&& aSetEnv); + +} // namespace mozilla + +#endif // !MOZ_GECKO_PROFILER + +#endif // GeckoProfiler_h diff --git a/tools/profiler/public/GeckoProfilerReporter.h b/tools/profiler/public/GeckoProfilerReporter.h new file mode 100644 index 0000000000..f5bf41f223 --- /dev/null +++ b/tools/profiler/public/GeckoProfilerReporter.h @@ -0,0 +1,26 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef GeckoProfilerReporter_h +#define GeckoProfilerReporter_h + +#include "nsIMemoryReporter.h" + +class GeckoProfilerReporter final : public nsIMemoryReporter { + public: + NS_DECL_ISUPPORTS + + GeckoProfilerReporter() {} + + NS_IMETHOD + CollectReports(nsIHandleReportCallback* aHandleReport, nsISupports* aData, + bool aAnonymize) override; + + private: + ~GeckoProfilerReporter() {} +}; + +#endif diff --git a/tools/profiler/public/GeckoTraceEvent.h b/tools/profiler/public/GeckoTraceEvent.h new file mode 100644 index 0000000000..75affaf9c8 --- /dev/null +++ b/tools/profiler/public/GeckoTraceEvent.h @@ -0,0 +1,1060 @@ +// Copyright (c) 2012 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file under third_party_mods/chromium or at: +// http://src.chromium.org/svn/trunk/src/LICENSE + +#ifndef GECKO_TRACE_EVENT_H_ +#define GECKO_TRACE_EVENT_H_ + +#include "MicroGeckoProfiler.h" + +// Extracted from Chromium's src/base/debug/trace_event.h, modified to talk to +// the Gecko profiler. + +#if defined(RTC_DISABLE_TRACE_EVENTS) +# define RTC_TRACE_EVENTS_ENABLED 0 +#else +# define RTC_TRACE_EVENTS_ENABLED 1 +#endif + +// Type values for identifying types in the TraceValue union. +#define TRACE_VALUE_TYPE_BOOL (static_cast<unsigned char>(1)) +#define TRACE_VALUE_TYPE_UINT (static_cast<unsigned char>(2)) +#define TRACE_VALUE_TYPE_INT (static_cast<unsigned char>(3)) +#define TRACE_VALUE_TYPE_DOUBLE (static_cast<unsigned char>(4)) +#define TRACE_VALUE_TYPE_POINTER (static_cast<unsigned char>(5)) +#define TRACE_VALUE_TYPE_STRING (static_cast<unsigned char>(6)) +#define TRACE_VALUE_TYPE_COPY_STRING (static_cast<unsigned char>(7)) + +#if RTC_TRACE_EVENTS_ENABLED + +// This header is designed to give you trace_event macros without specifying +// how the events actually get collected and stored. If you need to expose trace +// event to some other universe, you can copy-and-paste this file, +// implement the TRACE_EVENT_API macros, and do any other necessary fixup for +// the target platform. The end result is that multiple libraries can funnel +// events through to a shared trace event collector. + +// Trace events are for tracking application performance and resource usage. +// Macros are provided to track: +// Begin and end of function calls +// Counters +// +// Events are issued against categories. Whereas RTC_LOG's +// categories are statically defined, TRACE categories are created +// implicitly with a string. For example: +// TRACE_EVENT_INSTANT0("MY_SUBSYSTEM", "SomeImportantEvent") +// +// Events can be INSTANT, or can be pairs of BEGIN and END in the same scope: +// TRACE_EVENT_BEGIN0("MY_SUBSYSTEM", "SomethingCostly") +// doSomethingCostly() +// TRACE_EVENT_END0("MY_SUBSYSTEM", "SomethingCostly") +// Note: our tools can't always determine the correct BEGIN/END pairs unless +// these are used in the same scope. Use ASYNC_BEGIN/ASYNC_END macros if you +// need them to be in separate scopes. +// +// A common use case is to trace entire function scopes. This +// issues a trace BEGIN and END automatically: +// void doSomethingCostly() { +// TRACE_EVENT0("MY_SUBSYSTEM", "doSomethingCostly"); +// ... +// } +// +// Additional parameters can be associated with an event: +// void doSomethingCostly2(int howMuch) { +// TRACE_EVENT1("MY_SUBSYSTEM", "doSomethingCostly", +// "howMuch", howMuch); +// ... +// } +// +// The trace system will automatically add to this information the +// current process id, thread id, and a timestamp in microseconds. +// +// To trace an asynchronous procedure such as an IPC send/receive, use +// ASYNC_BEGIN and ASYNC_END: +// [single threaded sender code] +// static int send_count = 0; +// ++send_count; +// TRACE_EVENT_ASYNC_BEGIN0("ipc", "message", send_count); +// Send(new MyMessage(send_count)); +// [receive code] +// void OnMyMessage(send_count) { +// TRACE_EVENT_ASYNC_END0("ipc", "message", send_count); +// } +// The third parameter is a unique ID to match ASYNC_BEGIN/ASYNC_END pairs. +// ASYNC_BEGIN and ASYNC_END can occur on any thread of any traced process. +// Pointers can be used for the ID parameter, and they will be mangled +// internally so that the same pointer on two different processes will not +// match. For example: +// class MyTracedClass { +// public: +// MyTracedClass() { +// TRACE_EVENT_ASYNC_BEGIN0("category", "MyTracedClass", this); +// } +// ~MyTracedClass() { +// TRACE_EVENT_ASYNC_END0("category", "MyTracedClass", this); +// } +// } +// +// Trace event also supports counters, which is a way to track a quantity +// as it varies over time. Counters are created with the following macro: +// TRACE_COUNTER1("MY_SUBSYSTEM", "myCounter", g_myCounterValue); +// +// Counters are process-specific. The macro itself can be issued from any +// thread, however. +// +// Sometimes, you want to track two counters at once. You can do this with two +// counter macros: +// TRACE_COUNTER1("MY_SUBSYSTEM", "myCounter0", g_myCounterValue[0]); +// TRACE_COUNTER1("MY_SUBSYSTEM", "myCounter1", g_myCounterValue[1]); +// Or you can do it with a combined macro: +// TRACE_COUNTER2("MY_SUBSYSTEM", "myCounter", +// "bytesPinned", g_myCounterValue[0], +// "bytesAllocated", g_myCounterValue[1]); +// This indicates to the tracing UI that these counters should be displayed +// in a single graph, as a summed area chart. +// +// Since counters are in a global namespace, you may want to disembiguate with a +// unique ID, by using the TRACE_COUNTER_ID* variations. +// +// By default, trace collection is compiled in, but turned off at runtime. +// Collecting trace data is the responsibility of the embedding +// application. In Chrome's case, navigating to about:tracing will turn on +// tracing and display data collected across all active processes. +// +// +// Memory scoping note: +// Tracing copies the pointers, not the string content, of the strings passed +// in for category, name, and arg_names. Thus, the following code will +// cause problems: +// char* str = strdup("impprtantName"); +// TRACE_EVENT_INSTANT0("SUBSYSTEM", str); // BAD! +// free(str); // Trace system now has dangling pointer +// +// To avoid this issue with the `name` and `arg_name` parameters, use the +// TRACE_EVENT_COPY_XXX overloads of the macros at additional runtime overhead. +// Notes: The category must always be in a long-lived char* (i.e. static const). +// The `arg_values`, when used, are always deep copied with the _COPY +// macros. +// +// When are string argument values copied: +// const char* arg_values are only referenced by default: +// TRACE_EVENT1("category", "name", +// "arg1", "literal string is only referenced"); +// Use TRACE_STR_COPY to force copying of a const char*: +// TRACE_EVENT1("category", "name", +// "arg1", TRACE_STR_COPY("string will be copied")); +// std::string arg_values are always copied: +// TRACE_EVENT1("category", "name", +// "arg1", std::string("string will be copied")); +// +// +// Thread Safety: +// Thread safety is provided by methods defined in event_tracer.h. See the file +// for details. + +// By default, const char* argument values are assumed to have long-lived scope +// and will not be copied. Use this macro to force a const char* to be copied. +# define TRACE_STR_COPY(str) \ + webrtc::trace_event_internal::TraceStringWithCopy(str) + +// This will mark the trace event as disabled by default. The user will need +// to explicitly enable the event. +# define TRACE_DISABLED_BY_DEFAULT(name) "disabled-by-default-" name + +// By default, uint64 ID argument values are not mangled with the Process ID in +// TRACE_EVENT_ASYNC macros. Use this macro to force Process ID mangling. +# define TRACE_ID_MANGLE(id) \ + webrtc::trace_event_internal::TraceID::ForceMangle(id) + +// Records a pair of begin and end events called "name" for the current +// scope, with 0, 1 or 2 associated arguments. If the category is not +// enabled, then this does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +# define TRACE_EVENT0(category, name) \ + INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name) +# define TRACE_EVENT1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, arg1_name, arg1_val) +# define TRACE_EVENT2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) + +// Records a single event called "name" immediately, with 0, 1 or 2 +// associated arguments. If the category is not enabled, then this +// does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +# define TRACE_EVENT_INSTANT0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_NONE) +# define TRACE_EVENT_INSTANT1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) +# define TRACE_EVENT_INSTANT2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ + arg2_name, arg2_val) +# define TRACE_EVENT_COPY_INSTANT0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_COPY) +# define TRACE_EVENT_COPY_INSTANT1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val) +# define TRACE_EVENT_COPY_INSTANT2(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_INSTANT, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val, \ + arg2_name, arg2_val) + +// Records a single BEGIN event called "name" immediately, with 0, 1 or 2 +// associated arguments. If the category is not enabled, then this +// does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +# define TRACE_EVENT_BEGIN0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_NONE) +# define TRACE_EVENT_BEGIN1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) +# define TRACE_EVENT_BEGIN2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ + arg2_name, arg2_val) +# define TRACE_EVENT_COPY_BEGIN0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_COPY) +# define TRACE_EVENT_COPY_BEGIN1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val) +# define TRACE_EVENT_COPY_BEGIN2(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_BEGIN, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val, \ + arg2_name, arg2_val) + +// Records a single END event for "name" immediately. If the category +// is not enabled, then this does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +# define TRACE_EVENT_END0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_NONE) +# define TRACE_EVENT_END1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val) +# define TRACE_EVENT_END2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_NONE, arg1_name, arg1_val, \ + arg2_name, arg2_val) +# define TRACE_EVENT_COPY_END0(category, name) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_COPY) +# define TRACE_EVENT_COPY_END1(category, name, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val) +# define TRACE_EVENT_COPY_END2(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_END, category, name, \ + TRACE_EVENT_FLAG_COPY, arg1_name, arg1_val, \ + arg2_name, arg2_val) + +// Records the value of a counter called "name" immediately. Value +// must be representable as a 32 bit integer. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +# define TRACE_COUNTER1(category, name, value) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_NONE, "value", \ + static_cast<int>(value)) +# define TRACE_COPY_COUNTER1(category, name, value) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_COPY, "value", \ + static_cast<int>(value)) + +// Records the values of a multi-parted counter called "name" immediately. +// The UI will treat value1 and value2 as parts of a whole, displaying their +// values as a stacked-bar chart. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +# define TRACE_COUNTER2(category, name, value1_name, value1_val, value2_name, \ + value2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_NONE, value1_name, \ + static_cast<int>(value1_val), value2_name, \ + static_cast<int>(value2_val)) +# define TRACE_COPY_COUNTER2(category, name, value1_name, value1_val, \ + value2_name, value2_val) \ + INTERNAL_TRACE_EVENT_ADD(TRACE_EVENT_PHASE_COUNTER, category, name, \ + TRACE_EVENT_FLAG_COPY, value1_name, \ + static_cast<int>(value1_val), value2_name, \ + static_cast<int>(value2_val)) + +// Records the value of a counter called "name" immediately. Value +// must be representable as a 32 bit integer. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +// - `id` is used to disambiguate counters with the same name. It must either +// be a pointer or an integer value up to 64 bits. If it's a pointer, the bits +// will be xored with a hash of the process ID so that the same pointer on +// two different processes will not collide. +# define TRACE_COUNTER_ID1(category, name, id, value) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "value", \ + static_cast<int>(value)) +# define TRACE_COPY_COUNTER_ID1(category, name, id, value) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_COUNTER, category, \ + name, id, TRACE_EVENT_FLAG_COPY, "value", \ + static_cast<int>(value)) + +// Records the values of a multi-parted counter called "name" immediately. +// The UI will treat value1 and value2 as parts of a whole, displaying their +// values as a stacked-bar chart. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +// - `id` is used to disambiguate counters with the same name. It must either +// be a pointer or an integer value up to 64 bits. If it's a pointer, the bits +// will be xored with a hash of the process ID so that the same pointer on +// two different processes will not collide. +# define TRACE_COUNTER_ID2(category, name, id, value1_name, value1_val, \ + value2_name, value2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID( \ + TRACE_EVENT_PHASE_COUNTER, category, name, id, TRACE_EVENT_FLAG_NONE, \ + value1_name, static_cast<int>(value1_val), value2_name, \ + static_cast<int>(value2_val)) +# define TRACE_COPY_COUNTER_ID2(category, name, id, value1_name, value1_val, \ + value2_name, value2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID( \ + TRACE_EVENT_PHASE_COUNTER, category, name, id, TRACE_EVENT_FLAG_COPY, \ + value1_name, static_cast<int>(value1_val), value2_name, \ + static_cast<int>(value2_val)) + +// Records a single ASYNC_BEGIN event called "name" immediately, with 0, 1 or 2 +// associated arguments. If the category is not enabled, then this +// does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +// - `id` is used to match the ASYNC_BEGIN event with the ASYNC_END event. ASYNC +// events are considered to match if their category, name and id values all +// match. `id` must either be a pointer or an integer value up to 64 bits. If +// it's a pointer, the bits will be xored with a hash of the process ID so +// that the same pointer on two different processes will not collide. +// An asynchronous operation can consist of multiple phases. The first phase is +// defined by the ASYNC_BEGIN calls. Additional phases can be defined using the +// ASYNC_STEP macros. When the operation completes, call ASYNC_END. +// An ASYNC trace typically occur on a single thread (if not, they will only be +// drawn on the thread defined in the ASYNC_BEGIN event), but all events in that +// operation must use the same `name` and `id`. Each event can have its own +// args. +# define TRACE_EVENT_ASYNC_BEGIN0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE) +# define TRACE_EVENT_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, \ + arg1_name, arg1_val) +# define TRACE_EVENT_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, \ + arg1_name, arg1_val, arg2_name, arg2_val) +# define TRACE_EVENT_COPY_ASYNC_BEGIN0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY) +# define TRACE_EVENT_COPY_ASYNC_BEGIN1(category, name, id, arg1_name, \ + arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY, \ + arg1_name, arg1_val) +# define TRACE_EVENT_COPY_ASYNC_BEGIN2(category, name, id, arg1_name, \ + arg1_val, arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY, \ + arg1_name, arg1_val, arg2_name, arg2_val) + +// Records a single ASYNC_STEP event for `step` immediately. If the category +// is not enabled, then this does nothing. The `name` and `id` must match the +// ASYNC_BEGIN event above. The `step` param identifies this step within the +// async event. This should be called at the beginning of the next phase of an +// asynchronous operation. +# define TRACE_EVENT_ASYNC_STEP0(category, name, id, step) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step) +# define TRACE_EVENT_ASYNC_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step, arg1_name, arg1_val) +# define TRACE_EVENT_COPY_ASYNC_STEP0(category, name, id, step) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_COPY, "step", \ + step) +# define TRACE_EVENT_COPY_ASYNC_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_STEP, category, \ + name, id, TRACE_EVENT_FLAG_COPY, "step", \ + step, arg1_name, arg1_val) + +// Records a single ASYNC_END event for "name" immediately. If the category +// is not enabled, then this does nothing. +# define TRACE_EVENT_ASYNC_END0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE) +# define TRACE_EVENT_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE, \ + arg1_name, arg1_val) +# define TRACE_EVENT_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE, \ + arg1_name, arg1_val, arg2_name, arg2_val) +# define TRACE_EVENT_COPY_ASYNC_END0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_COPY) +# define TRACE_EVENT_COPY_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_COPY, \ + arg1_name, arg1_val) +# define TRACE_EVENT_COPY_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_ASYNC_END, category, \ + name, id, TRACE_EVENT_FLAG_COPY, \ + arg1_name, arg1_val, arg2_name, arg2_val) + +// Records a single FLOW_BEGIN event called "name" immediately, with 0, 1 or 2 +// associated arguments. If the category is not enabled, then this +// does nothing. +// - category and name strings must have application lifetime (statics or +// literals). They may not include " chars. +// - `id` is used to match the FLOW_BEGIN event with the FLOW_END event. FLOW +// events are considered to match if their category, name and id values all +// match. `id` must either be a pointer or an integer value up to 64 bits. If +// it's a pointer, the bits will be xored with a hash of the process ID so +// that the same pointer on two different processes will not collide. +// FLOW events are different from ASYNC events in how they are drawn by the +// tracing UI. A FLOW defines asynchronous data flow, such as posting a task +// (FLOW_BEGIN) and later executing that task (FLOW_END). Expect FLOWs to be +// drawn as lines or arrows from FLOW_BEGIN scopes to FLOW_END scopes. Similar +// to ASYNC, a FLOW can consist of multiple phases. The first phase is defined +// by the FLOW_BEGIN calls. Additional phases can be defined using the FLOW_STEP +// macros. When the operation completes, call FLOW_END. An async operation can +// span threads and processes, but all events in that operation must use the +// same `name` and `id`. Each event can have its own args. +# define TRACE_EVENT_FLOW_BEGIN0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE) +# define TRACE_EVENT_FLOW_BEGIN1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, \ + arg1_name, arg1_val) +# define TRACE_EVENT_FLOW_BEGIN2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_NONE, \ + arg1_name, arg1_val, arg2_name, arg2_val) +# define TRACE_EVENT_COPY_FLOW_BEGIN0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY) +# define TRACE_EVENT_COPY_FLOW_BEGIN1(category, name, id, arg1_name, \ + arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY, \ + arg1_name, arg1_val) +# define TRACE_EVENT_COPY_FLOW_BEGIN2(category, name, id, arg1_name, \ + arg1_val, arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_BEGIN, category, \ + name, id, TRACE_EVENT_FLAG_COPY, \ + arg1_name, arg1_val, arg2_name, arg2_val) + +// Records a single FLOW_STEP event for `step` immediately. If the category +// is not enabled, then this does nothing. The `name` and `id` must match the +// FLOW_BEGIN event above. The `step` param identifies this step within the +// async event. This should be called at the beginning of the next phase of an +// asynchronous operation. +# define TRACE_EVENT_FLOW_STEP0(category, name, id, step) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step) +# define TRACE_EVENT_FLOW_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, category, \ + name, id, TRACE_EVENT_FLAG_NONE, "step", \ + step, arg1_name, arg1_val) +# define TRACE_EVENT_COPY_FLOW_STEP0(category, name, id, step) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, category, \ + name, id, TRACE_EVENT_FLAG_COPY, "step", \ + step) +# define TRACE_EVENT_COPY_FLOW_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_STEP, category, \ + name, id, TRACE_EVENT_FLAG_COPY, "step", \ + step, arg1_name, arg1_val) + +// Records a single FLOW_END event for "name" immediately. If the category +// is not enabled, then this does nothing. +# define TRACE_EVENT_FLOW_END0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE) +# define TRACE_EVENT_FLOW_END1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE, \ + arg1_name, arg1_val) +# define TRACE_EVENT_FLOW_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, \ + name, id, TRACE_EVENT_FLAG_NONE, \ + arg1_name, arg1_val, arg2_name, arg2_val) +# define TRACE_EVENT_COPY_FLOW_END0(category, name, id) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, \ + name, id, TRACE_EVENT_FLAG_COPY) +# define TRACE_EVENT_COPY_FLOW_END1(category, name, id, arg1_name, arg1_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, \ + name, id, TRACE_EVENT_FLAG_COPY, \ + arg1_name, arg1_val) +# define TRACE_EVENT_COPY_FLOW_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + INTERNAL_TRACE_EVENT_ADD_WITH_ID(TRACE_EVENT_PHASE_FLOW_END, category, \ + name, id, TRACE_EVENT_FLAG_COPY, \ + arg1_name, arg1_val, arg2_name, arg2_val) + +//////////////////////////////////////////////////////////////////////////////// +// Implementation specific tracing API definitions. + +// Get a pointer to the enabled state of the given trace category. Only +// long-lived literal strings should be given as the category name. The returned +// pointer can be held permanently in a local static for example. If the +// unsigned char is non-zero, tracing is enabled. If tracing is enabled, +// TRACE_EVENT_API_ADD_TRACE_EVENT can be called. It's OK if tracing is disabled +// between the load of the tracing state and the call to +// TRACE_EVENT_API_ADD_TRACE_EVENT, because this flag only provides an early out +// for best performance when tracing is disabled. +// const unsigned char* +// TRACE_EVENT_API_GET_CATEGORY_ENABLED(const char* category_name) +# define TRACE_EVENT_API_GET_CATEGORY_ENABLED \ + webrtc::EventTracer::GetCategoryEnabled + +// Add a trace event to the platform tracing system. +// void TRACE_EVENT_API_ADD_TRACE_EVENT( +// char phase, +// const unsigned char* category_enabled, +// const char* name, +// unsigned long long id, +// int num_args, +// const char** arg_names, +// const unsigned char* arg_types, +// const unsigned long long* arg_values, +// unsigned char flags) +# define TRACE_EVENT_API_ADD_TRACE_EVENT MOZ_INTERNAL_UPROFILER_SIMPLE_EVENT + +//////////////////////////////////////////////////////////////////////////////// + +// Implementation detail: trace event macros create temporary variables +// to keep instrumentation overhead low. These macros give each temporary +// variable a unique name based on the line number to prevent name collissions. +# define INTERNAL_TRACE_EVENT_UID3(a, b) trace_event_unique_##a##b +# define INTERNAL_TRACE_EVENT_UID2(a, b) INTERNAL_TRACE_EVENT_UID3(a, b) +# define INTERNAL_TRACE_EVENT_UID(name_prefix) \ + INTERNAL_TRACE_EVENT_UID2(name_prefix, __LINE__) + +# if WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS +# define INTERNAL_TRACE_EVENT_INFO_TYPE const unsigned char* +# else +# define INTERNAL_TRACE_EVENT_INFO_TYPE static const unsigned char* +# endif // WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS + +// Implementation detail: internal macro to create static category. +# define INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category) \ + INTERNAL_TRACE_EVENT_INFO_TYPE INTERNAL_TRACE_EVENT_UID(catstatic) = \ + reinterpret_cast<const unsigned char*>(category); + +// Implementation detail: internal macro to create static category and add +// event if the category is enabled. +# define INTERNAL_TRACE_EVENT_ADD(phase, category, name, flags, ...) \ + do { \ + INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ + if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ + webrtc::trace_event_internal::AddTraceEvent( \ + phase, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ + webrtc::trace_event_internal::kNoEventId, flags, ##__VA_ARGS__); \ + } \ + } while (0) + +// Implementation detail: internal macro to create static category and add begin +// event if the category is enabled. Also adds the end event when the scope +// ends. +# define INTERNAL_TRACE_EVENT_ADD_SCOPED(category, name, ...) \ + INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ + webrtc::trace_event_internal::TraceEndOnScopeClose \ + INTERNAL_TRACE_EVENT_UID(profileScope); \ + if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ + webrtc::trace_event_internal::AddTraceEvent( \ + TRACE_EVENT_PHASE_BEGIN, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ + webrtc::trace_event_internal::kNoEventId, TRACE_EVENT_FLAG_NONE, \ + ##__VA_ARGS__); \ + INTERNAL_TRACE_EVENT_UID(profileScope) \ + .Initialize(INTERNAL_TRACE_EVENT_UID(catstatic), name); \ + } + +// Implementation detail: internal macro to create static category and add +// event if the category is enabled. +# define INTERNAL_TRACE_EVENT_ADD_WITH_ID(phase, category, name, id, flags, \ + ...) \ + do { \ + INTERNAL_TRACE_EVENT_GET_CATEGORY_INFO(category); \ + if (*INTERNAL_TRACE_EVENT_UID(catstatic)) { \ + unsigned char trace_event_flags = flags | TRACE_EVENT_FLAG_HAS_ID; \ + webrtc::trace_event_internal::TraceID trace_event_trace_id( \ + id, &trace_event_flags); \ + webrtc::trace_event_internal::AddTraceEvent( \ + phase, INTERNAL_TRACE_EVENT_UID(catstatic), name, \ + trace_event_trace_id.data(), trace_event_flags, ##__VA_ARGS__); \ + } \ + } while (0) + +# ifdef MOZ_GECKO_PROFILER +# define MOZ_INTERNAL_UPROFILER_SIMPLE_EVENT(phase, category_enabled, name, \ + id, num_args, arg_names, \ + arg_types, arg_values, flags) \ + uprofiler_simple_event_marker(name, phase, num_args, arg_names, \ + arg_types, arg_values); +# else +# define MOZ_INTERNAL_UPROFILER_SIMPLE_EVENT(phase, category_enabled, name, \ + id, num_args, arg_names, \ + arg_types, arg_values, flags) +# endif + +// Notes regarding the following definitions: +// New values can be added and propagated to third party libraries, but existing +// definitions must never be changed, because third party libraries may use old +// definitions. + +// Phase indicates the nature of an event entry. E.g. part of a begin/end pair. +# define TRACE_EVENT_PHASE_BEGIN ('B') +# define TRACE_EVENT_PHASE_END ('E') +# define TRACE_EVENT_PHASE_INSTANT ('I') +# define TRACE_EVENT_PHASE_ASYNC_BEGIN ('S') +# define TRACE_EVENT_PHASE_ASYNC_STEP ('T') +# define TRACE_EVENT_PHASE_ASYNC_END ('F') +# define TRACE_EVENT_PHASE_FLOW_BEGIN ('s') +# define TRACE_EVENT_PHASE_FLOW_STEP ('t') +# define TRACE_EVENT_PHASE_FLOW_END ('f') +# define TRACE_EVENT_PHASE_METADATA ('M') +# define TRACE_EVENT_PHASE_COUNTER ('C') + +// Flags for changing the behavior of TRACE_EVENT_API_ADD_TRACE_EVENT. +# define TRACE_EVENT_FLAG_NONE (static_cast<unsigned char>(0)) +# define TRACE_EVENT_FLAG_COPY (static_cast<unsigned char>(1 << 0)) +# define TRACE_EVENT_FLAG_HAS_ID (static_cast<unsigned char>(1 << 1)) +# define TRACE_EVENT_FLAG_MANGLE_ID (static_cast<unsigned char>(1 << 2)) + +namespace webrtc { +namespace trace_event_internal { + +// Specify these values when the corresponding argument of AddTraceEvent is not +// used. +const int kZeroNumArgs = 0; +const unsigned long long kNoEventId = 0; + +// TraceID encapsulates an ID that can either be an integer or pointer. Pointers +// are mangled with the Process ID so that they are unlikely to collide when the +// same pointer is used on different processes. +class TraceID { + public: + class ForceMangle { + public: + explicit ForceMangle(unsigned long long id) : data_(id) {} + explicit ForceMangle(unsigned long id) : data_(id) {} + explicit ForceMangle(unsigned int id) : data_(id) {} + explicit ForceMangle(unsigned short id) : data_(id) {} + explicit ForceMangle(unsigned char id) : data_(id) {} + explicit ForceMangle(long long id) + : data_(static_cast<unsigned long long>(id)) {} + explicit ForceMangle(long id) + : data_(static_cast<unsigned long long>(id)) {} + explicit ForceMangle(int id) : data_(static_cast<unsigned long long>(id)) {} + explicit ForceMangle(short id) + : data_(static_cast<unsigned long long>(id)) {} + explicit ForceMangle(signed char id) + : data_(static_cast<unsigned long long>(id)) {} + + unsigned long long data() const { return data_; } + + private: + unsigned long long data_; + }; + + explicit TraceID(const void* id, unsigned char* flags) + : data_( + static_cast<unsigned long long>(reinterpret_cast<uintptr_t>(id))) { + *flags |= TRACE_EVENT_FLAG_MANGLE_ID; + } + explicit TraceID(ForceMangle id, unsigned char* flags) : data_(id.data()) { + *flags |= TRACE_EVENT_FLAG_MANGLE_ID; + } + explicit TraceID(unsigned long long id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned long id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned int id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned short id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(unsigned char id, unsigned char* flags) : data_(id) { + (void)flags; + } + explicit TraceID(long long id, unsigned char* flags) + : data_(static_cast<unsigned long long>(id)) { + (void)flags; + } + explicit TraceID(long id, unsigned char* flags) + : data_(static_cast<unsigned long long>(id)) { + (void)flags; + } + explicit TraceID(int id, unsigned char* flags) + : data_(static_cast<unsigned long long>(id)) { + (void)flags; + } + explicit TraceID(short id, unsigned char* flags) + : data_(static_cast<unsigned long long>(id)) { + (void)flags; + } + explicit TraceID(signed char id, unsigned char* flags) + : data_(static_cast<unsigned long long>(id)) { + (void)flags; + } + + unsigned long long data() const { return data_; } + + private: + unsigned long long data_; +}; + +// Simple union to store various types as unsigned long long. +union TraceValueUnion { + bool as_bool; + unsigned long long as_uint; + long long as_int; + double as_double; + const void* as_pointer; + const char* as_string; +}; + +// Simple container for const char* that should be copied instead of retained. +class TraceStringWithCopy { + public: + explicit TraceStringWithCopy(const char* str) : str_(str) {} + operator const char*() const { return str_; } + + private: + const char* str_; +}; + +// Define SetTraceValue for each allowed type. It stores the type and +// value in the return arguments. This allows this API to avoid declaring any +// structures so that it is portable to third_party libraries. +# define INTERNAL_DECLARE_SET_TRACE_VALUE(actual_type, union_member, \ + value_type_id) \ + static inline void SetTraceValue(actual_type arg, unsigned char* type, \ + unsigned long long* value) { \ + TraceValueUnion type_value; \ + type_value.union_member = arg; \ + *type = value_type_id; \ + *value = type_value.as_uint; \ + } +// Simpler form for int types that can be safely casted. +# define INTERNAL_DECLARE_SET_TRACE_VALUE_INT(actual_type, value_type_id) \ + static inline void SetTraceValue(actual_type arg, unsigned char* type, \ + unsigned long long* value) { \ + *type = value_type_id; \ + *value = static_cast<unsigned long long>(arg); \ + } + +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned long long, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned long, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned int, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned short, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(unsigned char, TRACE_VALUE_TYPE_UINT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(long long, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(long, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(int, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(short, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE_INT(signed char, TRACE_VALUE_TYPE_INT) +INTERNAL_DECLARE_SET_TRACE_VALUE(bool, as_bool, TRACE_VALUE_TYPE_BOOL) +INTERNAL_DECLARE_SET_TRACE_VALUE(double, as_double, TRACE_VALUE_TYPE_DOUBLE) +INTERNAL_DECLARE_SET_TRACE_VALUE(const void*, as_pointer, + TRACE_VALUE_TYPE_POINTER) +INTERNAL_DECLARE_SET_TRACE_VALUE(const char*, as_string, + TRACE_VALUE_TYPE_STRING) +INTERNAL_DECLARE_SET_TRACE_VALUE(const TraceStringWithCopy&, as_string, + TRACE_VALUE_TYPE_COPY_STRING) + +# undef INTERNAL_DECLARE_SET_TRACE_VALUE +# undef INTERNAL_DECLARE_SET_TRACE_VALUE_INT + +// std::string version of SetTraceValue so that trace arguments can be strings. +static inline void SetTraceValue(const std::string& arg, unsigned char* type, + unsigned long long* value) { + TraceValueUnion type_value; + type_value.as_string = arg.c_str(); + *type = TRACE_VALUE_TYPE_COPY_STRING; + *value = type_value.as_uint; +} + +// These AddTraceEvent template functions are defined here instead of in the +// macro, because the arg_values could be temporary objects, such as +// std::string. In order to store pointers to the internal c_str and pass +// through to the tracing API, the arg_values must live throughout +// these procedures. + +static inline void AddTraceEvent(char phase, + const unsigned char* category_enabled, + const char* name, unsigned long long id, + unsigned char flags) { + TRACE_EVENT_API_ADD_TRACE_EVENT(phase, category_enabled, name, id, + kZeroNumArgs, nullptr, nullptr, nullptr, + flags); +} + +template <class ARG1_TYPE> +static inline void AddTraceEvent(char phase, + const unsigned char* category_enabled, + const char* name, unsigned long long id, + unsigned char flags, const char* arg1_name, + const ARG1_TYPE& arg1_val) { + const int num_args = 1; + unsigned char arg_types[1]; + unsigned long long arg_values[1]; + SetTraceValue(arg1_val, &arg_types[0], &arg_values[0]); + TRACE_EVENT_API_ADD_TRACE_EVENT(phase, category_enabled, name, id, num_args, + &arg1_name, arg_types, arg_values, flags); +} + +template <class ARG1_TYPE, class ARG2_TYPE> +static inline void AddTraceEvent(char phase, + const unsigned char* category_enabled, + const char* name, unsigned long long id, + unsigned char flags, const char* arg1_name, + const ARG1_TYPE& arg1_val, + const char* arg2_name, + const ARG2_TYPE& arg2_val) { + const int num_args = 2; + const char* arg_names[2] = {arg1_name, arg2_name}; + unsigned char arg_types[2]; + unsigned long long arg_values[2]; + SetTraceValue(arg1_val, &arg_types[0], &arg_values[0]); + SetTraceValue(arg2_val, &arg_types[1], &arg_values[1]); + TRACE_EVENT_API_ADD_TRACE_EVENT(phase, category_enabled, name, id, num_args, + arg_names, arg_types, arg_values, flags); +} + +// Used by TRACE_EVENTx macro. Do not use directly. +class TraceEndOnScopeClose { + public: + // Note: members of data_ intentionally left uninitialized. See Initialize. + TraceEndOnScopeClose() : p_data_(nullptr) {} + ~TraceEndOnScopeClose() { + if (p_data_) AddEventIfEnabled(); + } + + void Initialize(const unsigned char* category_enabled, const char* name) { + data_.category_enabled = category_enabled; + data_.name = name; + p_data_ = &data_; + } + + private: + // Add the end event if the category is still enabled. + void AddEventIfEnabled() { + // Only called when p_data_ is non-null. + if (*p_data_->category_enabled) { + TRACE_EVENT_API_ADD_TRACE_EVENT(TRACE_EVENT_PHASE_END, + p_data_->category_enabled, p_data_->name, + kNoEventId, kZeroNumArgs, nullptr, + nullptr, nullptr, TRACE_EVENT_FLAG_NONE); + } + } + + // This Data struct workaround is to avoid initializing all the members + // in Data during construction of this object, since this object is always + // constructed, even when tracing is disabled. If the members of Data were + // members of this class instead, compiler warnings occur about potential + // uninitialized accesses. + struct Data { + const unsigned char* category_enabled; + const char* name; + }; + Data* p_data_; + Data data_; +}; + +} // namespace trace_event_internal +} // namespace webrtc +#else + +//////////////////////////////////////////////////////////////////////////////// +// This section defines no-op alternatives to the tracing macros when +// RTC_DISABLE_TRACE_EVENTS is defined. + +# define RTC_NOOP() \ + do { \ + } while (0) + +# define TRACE_STR_COPY(str) RTC_NOOP() + +# define TRACE_DISABLED_BY_DEFAULT(name) "disabled-by-default-" name + +# define TRACE_ID_MANGLE(id) 0 + +# define TRACE_EVENT0(category, name) RTC_NOOP() +# define TRACE_EVENT1(category, name, arg1_name, arg1_val) RTC_NOOP() +# define TRACE_EVENT2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() + +# define TRACE_EVENT_INSTANT0(category, name) RTC_NOOP() +# define TRACE_EVENT_INSTANT1(category, name, arg1_name, arg1_val) RTC_NOOP() + +# define TRACE_EVENT_INSTANT2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() + +# define TRACE_EVENT_COPY_INSTANT0(category, name) RTC_NOOP() +# define TRACE_EVENT_COPY_INSTANT1(category, name, arg1_name, arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_INSTANT2(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() + +# define TRACE_EVENT_BEGIN0(category, name) RTC_NOOP() +# define TRACE_EVENT_BEGIN1(category, name, arg1_name, arg1_val) RTC_NOOP() +# define TRACE_EVENT_BEGIN2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_BEGIN0(category, name) RTC_NOOP() +# define TRACE_EVENT_COPY_BEGIN1(category, name, arg1_name, arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_BEGIN2(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() + +# define TRACE_EVENT_END0(category, name) RTC_NOOP() +# define TRACE_EVENT_END1(category, name, arg1_name, arg1_val) RTC_NOOP() +# define TRACE_EVENT_END2(category, name, arg1_name, arg1_val, arg2_name, \ + arg2_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_END0(category, name) RTC_NOOP() +# define TRACE_EVENT_COPY_END1(category, name, arg1_name, arg1_val) RTC_NOOP() +# define TRACE_EVENT_COPY_END2(category, name, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() + +# define TRACE_COUNTER1(category, name, value) RTC_NOOP() +# define TRACE_COPY_COUNTER1(category, name, value) RTC_NOOP() + +# define TRACE_COUNTER2(category, name, value1_name, value1_val, value2_name, \ + value2_val) \ + RTC_NOOP() +# define TRACE_COPY_COUNTER2(category, name, value1_name, value1_val, \ + value2_name, value2_val) \ + RTC_NOOP() + +# define TRACE_COUNTER_ID1(category, name, id, value) RTC_NOOP() +# define TRACE_COPY_COUNTER_ID1(category, name, id, value) RTC_NOOP() + +# define TRACE_COUNTER_ID2(category, name, id, value1_name, value1_val, \ + value2_name, value2_val) \ + RTC_NOOP() +# define TRACE_COPY_COUNTER_ID2(category, name, id, value1_name, value1_val, \ + value2_name, value2_val) \ + RTC_NOOP() + +# define TRACE_EVENT_ASYNC_BEGIN0(category, name, id) RTC_NOOP() +# define TRACE_EVENT_ASYNC_BEGIN1(category, name, id, arg1_name, arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_ASYNC_BEGIN2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_ASYNC_BEGIN0(category, name, id) RTC_NOOP() +# define TRACE_EVENT_COPY_ASYNC_BEGIN1(category, name, id, arg1_name, \ + arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_ASYNC_BEGIN2(category, name, id, arg1_name, \ + arg1_val, arg2_name, arg2_val) \ + RTC_NOOP() + +# define TRACE_EVENT_ASYNC_STEP0(category, name, id, step) RTC_NOOP() +# define TRACE_EVENT_ASYNC_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_ASYNC_STEP0(category, name, id, step) RTC_NOOP() +# define TRACE_EVENT_COPY_ASYNC_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + RTC_NOOP() + +# define TRACE_EVENT_ASYNC_END0(category, name, id) RTC_NOOP() +# define TRACE_EVENT_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_ASYNC_END0(category, name, id) RTC_NOOP() +# define TRACE_EVENT_COPY_ASYNC_END1(category, name, id, arg1_name, arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_ASYNC_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() + +# define TRACE_EVENT_FLOW_BEGIN0(category, name, id) RTC_NOOP() +# define TRACE_EVENT_FLOW_BEGIN1(category, name, id, arg1_name, arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_FLOW_BEGIN2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_FLOW_BEGIN0(category, name, id) RTC_NOOP() +# define TRACE_EVENT_COPY_FLOW_BEGIN1(category, name, id, arg1_name, \ + arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_FLOW_BEGIN2(category, name, id, arg1_name, \ + arg1_val, arg2_name, arg2_val) \ + RTC_NOOP() + +# define TRACE_EVENT_FLOW_STEP0(category, name, id, step) RTC_NOOP() +# define TRACE_EVENT_FLOW_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_FLOW_STEP0(category, name, id, step) RTC_NOOP() +# define TRACE_EVENT_COPY_FLOW_STEP1(category, name, id, step, arg1_name, \ + arg1_val) \ + RTC_NOOP() + +# define TRACE_EVENT_FLOW_END0(category, name, id) RTC_NOOP() +# define TRACE_EVENT_FLOW_END1(category, name, id, arg1_name, arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_FLOW_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_FLOW_END0(category, name, id) RTC_NOOP() +# define TRACE_EVENT_COPY_FLOW_END1(category, name, id, arg1_name, arg1_val) \ + RTC_NOOP() +# define TRACE_EVENT_COPY_FLOW_END2(category, name, id, arg1_name, arg1_val, \ + arg2_name, arg2_val) \ + RTC_NOOP() + +# define TRACE_EVENT_API_GET_CATEGORY_ENABLED "" + +# define TRACE_EVENT_API_ADD_TRACE_EVENT RTC_NOOP() + +#endif // RTC_TRACE_EVENTS_ENABLED + +#endif // GECKO_TRACE_EVENT_H_ diff --git a/tools/profiler/public/MicroGeckoProfiler.h b/tools/profiler/public/MicroGeckoProfiler.h new file mode 100644 index 0000000000..7b735e1eec --- /dev/null +++ b/tools/profiler/public/MicroGeckoProfiler.h @@ -0,0 +1,130 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This contains things related to the Gecko profiler, for use in third_party +// code. It is very minimal and is designed to be used by patching over +// upstream code. +// Only use the C ABI and guard C++ code with #ifdefs, don't pull anything from +// Gecko, it must be possible to include the header file into any C++ codebase. + +#ifndef MICRO_GECKO_PROFILER +#define MICRO_GECKO_PROFILER + +#ifdef __cplusplus +extern "C" { +#endif + +#include <mozilla/Types.h> +#include <stdio.h> + +#ifdef _WIN32 +# include <libloaderapi.h> +#else +# include <dlfcn.h> +#endif + +extern MOZ_EXPORT void uprofiler_register_thread(const char* aName, + void* aGuessStackTop); + +extern MOZ_EXPORT void uprofiler_unregister_thread(); + +extern MOZ_EXPORT void uprofiler_simple_event_marker( + const char* name, char phase, int num_args, const char** arg_names, + const unsigned char* arg_types, const unsigned long long* arg_values); +#ifdef __cplusplus +} + +struct AutoRegisterProfiler { + AutoRegisterProfiler(const char* name, char* stacktop) { + if (getenv("MOZ_UPROFILER_LOG_THREAD_CREATION")) { + printf("### UProfiler: new thread: '%s'\n", name); + } + uprofiler_register_thread(name, stacktop); + } + ~AutoRegisterProfiler() { uprofiler_unregister_thread(); } +}; +#endif // __cplusplus + +void uprofiler_simple_event_marker(const char* name, char phase, int num_args, + const char** arg_names, + const unsigned char* arg_types, + const unsigned long long* arg_values); + +struct UprofilerFuncPtrs { + void (*register_thread)(const char* aName, void* aGuessStackTop); + void (*unregister_thread)(); + void (*simple_event_marker)(const char* name, char phase, int num_args, + const char** arg_names, + const unsigned char* arg_types, + const unsigned long long* arg_values); +}; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" + +static void register_thread_noop(const char* aName, void* aGuessStackTop) { + /* no-op */ +} +static void unregister_thread_noop() { /* no-op */ +} +static void simple_event_marker_noop(const char* name, char phase, int num_args, + const char** arg_names, + const unsigned char* arg_types, + const unsigned long long* arg_values) { + /* no-op */ +} + +#pragma GCC diagnostic pop + +#if defined(_WIN32) +# define UPROFILER_OPENLIB() GetModuleHandle(NULL) +#else +# define UPROFILER_OPENLIB() dlopen(NULL, RTLD_NOW) +#endif + +#if defined(_WIN32) +# define UPROFILER_GET_SYM(handle, sym) GetProcAddress(handle, sym) +#else +# define UPROFILER_GET_SYM(handle, sym) dlsym(handle, sym) +#endif + +#if defined(_WIN32) +# define UPROFILER_PRINT_ERROR(func) fprintf(stderr, "%s error\n", #func); +#else +# define UPROFILER_PRINT_ERROR(func) \ + fprintf(stderr, "%s error: %s\n", #func, dlerror()); +#endif + +// Assumes that a variable of type UprofilerFuncPtrs, named uprofiler +// is accessible in the scope +#define UPROFILER_GET_FUNCTIONS() \ + void* handle = UPROFILER_OPENLIB(); \ + if (!handle) { \ + UPROFILER_PRINT_ERROR(UPROFILER_OPENLIB); \ + uprofiler.register_thread = register_thread_noop; \ + uprofiler.unregister_thread = unregister_thread_noop; \ + uprofiler.simple_event_marker = simple_event_marker_noop; \ + } \ + uprofiler.register_thread = \ + UPROFILER_GET_SYM(handle, "uprofiler_register_thread"); \ + if (!uprofiler.register_thread) { \ + UPROFILER_PRINT_ERROR(uprofiler_unregister_thread); \ + uprofiler.register_thread = register_thread_noop; \ + } \ + uprofiler.unregister_thread = \ + UPROFILER_GET_SYM(handle, "uprofiler_unregister_thread"); \ + if (!uprofiler.unregister_thread) { \ + UPROFILER_PRINT_ERROR(uprofiler_unregister_thread); \ + uprofiler.unregister_thread = unregister_thread_noop; \ + } \ + uprofiler.simple_event_marker = \ + UPROFILER_GET_SYM(handle, "uprofiler_simple_event_marker"); \ + if (!uprofiler.simple_event_marker) { \ + UPROFILER_PRINT_ERROR(uprofiler_simple_event_marker); \ + uprofiler.simple_event_marker = simple_event_marker_noop; \ + } + +#endif // MICRO_GECKO_PROFILER diff --git a/tools/profiler/public/ProfileAdditionalInformation.h b/tools/profiler/public/ProfileAdditionalInformation.h new file mode 100644 index 0000000000..7c9e3db2f6 --- /dev/null +++ b/tools/profiler/public/ProfileAdditionalInformation.h @@ -0,0 +1,84 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// The Gecko Profiler is an always-on profiler that takes fast and low overhead +// samples of the program execution using only userspace functionality for +// portability. The goal of this module is to provide performance data in a +// generic cross-platform way without requiring custom tools or kernel support. +// +// Samples are collected to form a timeline with optional timeline event +// (markers) used for filtering. The samples include both native stacks and +// platform-independent "label stack" frames. + +#ifndef ProfileAdditionalInformation_h +#define ProfileAdditionalInformation_h + +#include "shared-libraries.h" +#include "js/Value.h" +#include "nsString.h" + +namespace IPC { +class MessageReader; +class MessageWriter; +template <typename T> +struct ParamTraits; +} // namespace IPC + +namespace mozilla { +// This structure contains additional information gathered while generating the +// profile json and iterating the buffer. +struct ProfileGenerationAdditionalInformation { + ProfileGenerationAdditionalInformation() = default; + explicit ProfileGenerationAdditionalInformation( + const SharedLibraryInfo&& aSharedLibraries) + : mSharedLibraries(aSharedLibraries) {} + + size_t SizeOf() const { return mSharedLibraries.SizeOf(); } + + void Append(ProfileGenerationAdditionalInformation&& aOther) { + mSharedLibraries.AddAllSharedLibraries(aOther.mSharedLibraries); + } + + void FinishGathering() { mSharedLibraries.DeduplicateEntries(); } + + void ToJSValue(JSContext* aCx, JS::MutableHandle<JS::Value> aRetVal) const; + + SharedLibraryInfo mSharedLibraries; +}; + +struct ProfileAndAdditionalInformation { + ProfileAndAdditionalInformation() = default; + explicit ProfileAndAdditionalInformation(const nsCString&& aProfile) + : mProfile(aProfile) {} + + ProfileAndAdditionalInformation( + const nsCString&& aProfile, + const ProfileGenerationAdditionalInformation&& aAdditionalInformation) + : mProfile(aProfile), + mAdditionalInformation(Some(aAdditionalInformation)) {} + + size_t SizeOf() const { + size_t size = mProfile.Length(); + if (mAdditionalInformation.isSome()) { + size += mAdditionalInformation->SizeOf(); + } + return size; + } + + nsCString mProfile; + Maybe<ProfileGenerationAdditionalInformation> mAdditionalInformation; +}; +} // namespace mozilla + +namespace IPC { +template <> +struct ParamTraits<mozilla::ProfileGenerationAdditionalInformation> { + typedef mozilla::ProfileGenerationAdditionalInformation paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam); + static bool Read(MessageReader* aReader, paramType* aResult); +}; +} // namespace IPC + +#endif // ProfileAdditionalInformation_h diff --git a/tools/profiler/public/ProfileBufferEntrySerializationGeckoExtensions.h b/tools/profiler/public/ProfileBufferEntrySerializationGeckoExtensions.h new file mode 100644 index 0000000000..1578bd2ddc --- /dev/null +++ b/tools/profiler/public/ProfileBufferEntrySerializationGeckoExtensions.h @@ -0,0 +1,160 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfileBufferEntrySerializationGeckoExtensions_h +#define ProfileBufferEntrySerializationGeckoExtensions_h + +#include "mozilla/ProfileBufferEntrySerialization.h" + +#include "js/AllocPolicy.h" +#include "js/Utility.h" +#include "nsString.h" + +namespace mozilla { + +// ---------------------------------------------------------------------------- +// ns[C]String + +// nsString or nsCString contents are serialized as the number of bytes (encoded +// as ULEB128) and all the characters in the string. The terminal '\0' is +// omitted. +// Make sure you write and read with the same character type! +// +// Usage: `nsCString s = ...; aEW.WriteObject(s);` +template <typename CHAR> +struct ProfileBufferEntryWriter::Serializer<nsTString<CHAR>> { + static Length Bytes(const nsTString<CHAR>& aS) { + const auto length = aS.Length(); + return ProfileBufferEntryWriter::ULEB128Size(length) + + static_cast<Length>(length * sizeof(CHAR)); + } + + static void Write(ProfileBufferEntryWriter& aEW, const nsTString<CHAR>& aS) { + const auto length = aS.Length(); + aEW.WriteULEB128(length); + // Copy the bytes from the string's buffer. + aEW.WriteBytes(aS.Data(), length * sizeof(CHAR)); + } +}; + +template <typename CHAR> +struct ProfileBufferEntryReader::Deserializer<nsTString<CHAR>> { + static void ReadInto(ProfileBufferEntryReader& aER, nsTString<CHAR>& aS) { + aS = Read(aER); + } + + static nsTString<CHAR> Read(ProfileBufferEntryReader& aER) { + const Length length = aER.ReadULEB128<Length>(); + nsTString<CHAR> s; + // BulkWrite is the most efficient way to copy bytes into the target string. + auto writerOrErr = s.BulkWrite(length, 0, true); + MOZ_RELEASE_ASSERT(!writerOrErr.isErr()); + + auto writer = writerOrErr.unwrap(); + + aER.ReadBytes(writer.Elements(), length * sizeof(CHAR)); + writer.Finish(length, true); + return s; + } +}; + +// ---------------------------------------------------------------------------- +// nsAuto[C]String + +// nsAuto[C]String contents are serialized as the number of bytes (encoded as +// ULEB128) and all the characters in the string. The terminal '\0' is omitted. +// Make sure you write and read with the same character type! +// +// Usage: `nsAutoCString s = ...; aEW.WriteObject(s);` +template <typename CHAR, size_t N> +struct ProfileBufferEntryWriter::Serializer<nsTAutoStringN<CHAR, N>> { + static Length Bytes(const nsTAutoStringN<CHAR, N>& aS) { + const auto length = aS.Length(); + return ProfileBufferEntryWriter::ULEB128Size(length) + + static_cast<Length>(length * sizeof(CHAR)); + } + + static void Write(ProfileBufferEntryWriter& aEW, + const nsTAutoStringN<CHAR, N>& aS) { + const auto length = aS.Length(); + aEW.WriteULEB128(length); + // Copy the bytes from the string's buffer. + aEW.WriteBytes(aS.BeginReading(), length * sizeof(CHAR)); + } +}; + +template <typename CHAR, size_t N> +struct ProfileBufferEntryReader::Deserializer<nsTAutoStringN<CHAR, N>> { + static void ReadInto(ProfileBufferEntryReader& aER, + nsTAutoStringN<CHAR, N>& aS) { + aS = Read(aER); + } + + static nsTAutoStringN<CHAR, N> Read(ProfileBufferEntryReader& aER) { + const auto length = aER.ReadULEB128<Length>(); + nsTAutoStringN<CHAR, N> s; + // BulkWrite is the most efficient way to copy bytes into the target string. + auto writerOrErr = s.BulkWrite(length, 0, true); + MOZ_RELEASE_ASSERT(!writerOrErr.isErr()); + + auto writer = writerOrErr.unwrap(); + aER.ReadBytes(writer.Elements(), length * sizeof(CHAR)); + writer.Finish(length, true); + return s; + } +}; + +// ---------------------------------------------------------------------------- +// JS::UniqueChars + +// JS::UniqueChars contents are serialized as the number of bytes (encoded as +// ULEB128) and all the characters in the string. The terminal '\0' is omitted. +// Note: A nullptr pointer will be serialized like an empty string, so when +// deserializing it will result in an allocated buffer only containing a +// single null terminator. +// +// Usage: `JS::UniqueChars s = ...; aEW.WriteObject(s);` +template <> +struct ProfileBufferEntryWriter::Serializer<JS::UniqueChars> { + static Length Bytes(const JS::UniqueChars& aS) { + if (!aS) { + return ProfileBufferEntryWriter::ULEB128Size<Length>(0); + } + const auto len = static_cast<Length>(strlen(aS.get())); + return ProfileBufferEntryWriter::ULEB128Size(len) + len; + } + + static void Write(ProfileBufferEntryWriter& aEW, const JS::UniqueChars& aS) { + if (!aS) { + aEW.WriteULEB128<Length>(0); + return; + } + const auto len = static_cast<Length>(strlen(aS.get())); + aEW.WriteULEB128(len); + aEW.WriteBytes(aS.get(), len); + } +}; + +template <> +struct ProfileBufferEntryReader::Deserializer<JS::UniqueChars> { + static void ReadInto(ProfileBufferEntryReader& aER, JS::UniqueChars& aS) { + aS = Read(aER); + } + + static JS::UniqueChars Read(ProfileBufferEntryReader& aER) { + const auto len = aER.ReadULEB128<Length>(); + // Use the same allocation policy as JS_smprintf. + char* buffer = + static_cast<char*>(js::SystemAllocPolicy{}.pod_malloc<char>(len + 1)); + aER.ReadBytes(buffer, len); + buffer[len] = '\0'; + return JS::UniqueChars(buffer); + } +}; + +} // namespace mozilla + +#endif // ProfileBufferEntrySerializationGeckoExtensions_h diff --git a/tools/profiler/public/ProfileJSONWriter.h b/tools/profiler/public/ProfileJSONWriter.h new file mode 100644 index 0000000000..8d23d7a890 --- /dev/null +++ b/tools/profiler/public/ProfileJSONWriter.h @@ -0,0 +1,19 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef PROFILEJSONWRITER_H +#define PROFILEJSONWRITER_H + +#include "mozilla/BaseProfileJSONWriter.h" + +using ChunkedJSONWriteFunc = mozilla::baseprofiler::ChunkedJSONWriteFunc; +using JSONSchemaWriter = mozilla::baseprofiler::JSONSchemaWriter; +using OStreamJSONWriteFunc = mozilla::baseprofiler::OStreamJSONWriteFunc; +using SpliceableChunkedJSONWriter = + mozilla::baseprofiler::SpliceableChunkedJSONWriter; +using SpliceableJSONWriter = mozilla::baseprofiler::SpliceableJSONWriter; +using UniqueJSONStrings = mozilla::baseprofiler::UniqueJSONStrings; + +#endif // PROFILEJSONWRITER_H diff --git a/tools/profiler/public/ProfilerBandwidthCounter.h b/tools/profiler/public/ProfilerBandwidthCounter.h new file mode 100644 index 0000000000..c83fd02f32 --- /dev/null +++ b/tools/profiler/public/ProfilerBandwidthCounter.h @@ -0,0 +1,106 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerBandwidthCounter_h +#define ProfilerBandwidthCounter_h + +#ifndef MOZ_GECKO_PROFILER + +namespace mozilla { + +inline void profiler_count_bandwidth_read_bytes(int64_t aCount) {} +inline void profiler_count_bandwidth_written_bytes(int64_t aCount) {} + +} // namespace mozilla + +#else + +# include "mozilla/ProfilerMarkers.h" +# include "mozilla/ProfilerCounts.h" + +class ProfilerBandwidthCounter final : public BaseProfilerCount { + public: + ProfilerBandwidthCounter() + : BaseProfilerCount("bandwidth", &mCounter, &mNumber, "Bandwidth", + "Amount of data transfered") { + Register(); + } + + void Register() { + profiler_add_sampled_counter(this); + mRegistered = true; + } + + bool IsRegistered() { return mRegistered; } + void MarkUnregistered() { mRegistered = false; } + + void Add(int64_t aNumber) { + if (!mRegistered) { + Register(); + } + mCounter += aNumber; + mNumber++; + } + + ProfilerAtomicSigned mCounter; + ProfilerAtomicUnsigned mNumber; + bool mRegistered; +}; + +namespace geckoprofiler::markers { + +using namespace mozilla; + +struct NetworkIOMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("NetIO"); + } + static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter, + int64_t aRead, int64_t aWritten) { + if (aRead) { + aWriter.IntProperty("read", aRead); + } + if (aWritten) { + aWriter.IntProperty("written", aWritten); + } + } + + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable}; + + schema.AddKeyLabelFormat("read", "Read", MS::Format::Bytes); + schema.AddKeyLabelFormat("written", "Written", MS::Format::Bytes); + + return schema; + } +}; + +} // namespace geckoprofiler::markers + +void profiler_count_bandwidth_bytes(int64_t aCount); + +namespace mozilla { + +inline void profiler_count_bandwidth_read_bytes(int64_t aCount) { + if (MOZ_UNLIKELY(profiler_feature_active(ProfilerFeature::Bandwidth))) { + profiler_count_bandwidth_bytes(aCount); + } + // This marker will appear on the Socket Thread. + PROFILER_MARKER("Read", NETWORK, {}, NetworkIOMarker, aCount, 0); +} + +inline void profiler_count_bandwidth_written_bytes(int64_t aCount) { + if (MOZ_UNLIKELY(profiler_feature_active(ProfilerFeature::Bandwidth))) { + profiler_count_bandwidth_bytes(aCount); + } + // This marker will appear on the Socket Thread. + PROFILER_MARKER("Write", NETWORK, {}, NetworkIOMarker, 0, aCount); +} + +} // namespace mozilla + +#endif // !MOZ_GECKO_PROFILER + +#endif // ProfilerBandwidthCounter_h diff --git a/tools/profiler/public/ProfilerBindings.h b/tools/profiler/public/ProfilerBindings.h new file mode 100644 index 0000000000..a5c0daf069 --- /dev/null +++ b/tools/profiler/public/ProfilerBindings.h @@ -0,0 +1,165 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* FFI functions for Profiler Rust API to call into profiler */ + +#ifndef ProfilerBindings_h +#define ProfilerBindings_h + +#include "mozilla/BaseProfilerMarkersPrerequisites.h" + +#include <cstddef> +#include <stdint.h> + +namespace mozilla { +class AutoProfilerLabel; +class MarkerSchema; +class MarkerTiming; +class TimeStamp; +enum class StackCaptureOptions; + +namespace baseprofiler { +enum class ProfilingCategoryPair : uint32_t; +class SpliceableJSONWriter; +} // namespace baseprofiler + +} // namespace mozilla + +namespace JS { +enum class ProfilingCategoryPair : uint32_t; +} // namespace JS + +// Everything in here is safe to include unconditionally, implementations must +// take !MOZ_GECKO_PROFILER into account. +extern "C" { + +void gecko_profiler_register_thread(const char* aName); +void gecko_profiler_unregister_thread(); + +void gecko_profiler_construct_label(mozilla::AutoProfilerLabel* aAutoLabel, + JS::ProfilingCategoryPair aCategoryPair); +void gecko_profiler_destruct_label(mozilla::AutoProfilerLabel* aAutoLabel); + +// Construct, clone and destruct the timestamp for profiler time. +void gecko_profiler_construct_timestamp_now(mozilla::TimeStamp* aTimeStamp); +void gecko_profiler_clone_timestamp(const mozilla::TimeStamp* aSrcTimeStamp, + mozilla::TimeStamp* aDestTimeStamp); +void gecko_profiler_destruct_timestamp(mozilla::TimeStamp* aTimeStamp); + +// Addition and subtraction for timestamp. +void gecko_profiler_add_timestamp(const mozilla::TimeStamp* aTimeStamp, + mozilla::TimeStamp* aDestTimeStamp, + double aMicroseconds); +void gecko_profiler_subtract_timestamp(const mozilla::TimeStamp* aTimeStamp, + mozilla::TimeStamp* aDestTimeStamp, + double aMicroseconds); + +// Various MarkerTiming constructors and a destructor. +void gecko_profiler_construct_marker_timing_instant_at( + mozilla::MarkerTiming* aMarkerTiming, const mozilla::TimeStamp* aTime); +void gecko_profiler_construct_marker_timing_instant_now( + mozilla::MarkerTiming* aMarkerTiming); +void gecko_profiler_construct_marker_timing_interval( + mozilla::MarkerTiming* aMarkerTiming, const mozilla::TimeStamp* aStartTime, + const mozilla::TimeStamp* aEndTime); +void gecko_profiler_construct_marker_timing_interval_until_now_from( + mozilla::MarkerTiming* aMarkerTiming, const mozilla::TimeStamp* aStartTime); +void gecko_profiler_construct_marker_timing_interval_start( + mozilla::MarkerTiming* aMarkerTiming, const mozilla::TimeStamp* aTime); +void gecko_profiler_construct_marker_timing_interval_end( + mozilla::MarkerTiming* aMarkerTiming, const mozilla::TimeStamp* aTime); +void gecko_profiler_destruct_marker_timing( + mozilla::MarkerTiming* aMarkerTiming); + +// MarkerSchema constructors and destructor. +void gecko_profiler_construct_marker_schema( + mozilla::MarkerSchema* aMarkerSchema, + const mozilla::MarkerSchema::Location* aLocations, size_t aLength); +void gecko_profiler_construct_marker_schema_with_special_front_end_location( + mozilla::MarkerSchema* aMarkerSchema); +void gecko_profiler_destruct_marker_schema( + mozilla::MarkerSchema* aMarkerSchema); + +// MarkerSchema methods for adding labels. +void gecko_profiler_marker_schema_set_chart_label( + mozilla::MarkerSchema* aSchema, const char* aLabel, size_t aLabelLength); +void gecko_profiler_marker_schema_set_tooltip_label( + mozilla::MarkerSchema* aSchema, const char* aLabel, size_t aLabelLength); +void gecko_profiler_marker_schema_set_table_label( + mozilla::MarkerSchema* aSchema, const char* aLabel, size_t aLabelLength); +void gecko_profiler_marker_schema_set_all_labels(mozilla::MarkerSchema* aSchema, + const char* aLabel, + size_t aLabelLength); + +// MarkerSchema methods for adding key/key-label values. +void gecko_profiler_marker_schema_add_key_format( + mozilla::MarkerSchema* aSchema, const char* aKey, size_t aKeyLength, + mozilla::MarkerSchema::Format aFormat); +void gecko_profiler_marker_schema_add_key_label_format( + mozilla::MarkerSchema* aSchema, const char* aKey, size_t aKeyLength, + const char* aLabel, size_t aLabelLength, + mozilla::MarkerSchema::Format aFormat); +void gecko_profiler_marker_schema_add_key_format_searchable( + mozilla::MarkerSchema* aSchema, const char* aKey, size_t aKeyLength, + mozilla::MarkerSchema::Format aFormat, + mozilla::MarkerSchema::Searchable aSearchable); +void gecko_profiler_marker_schema_add_key_label_format_searchable( + mozilla::MarkerSchema* aSchema, const char* aKey, size_t aKeyLength, + const char* aLabel, size_t aLabelLength, + mozilla::MarkerSchema::Format aFormat, + mozilla::MarkerSchema::Searchable aSearchable); +void gecko_profiler_marker_schema_add_static_label_value( + mozilla::MarkerSchema* aSchema, const char* aLabel, size_t aLabelLength, + const char* aValue, size_t aValueLength); + +// Stream MarkerSchema to SpliceableJSONWriter. +void gecko_profiler_marker_schema_stream( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, mozilla::MarkerSchema* aMarkerSchema, + void* aStreamedNamesSet); + +// Various SpliceableJSONWriter methods to add properties. +void gecko_profiler_json_writer_int_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, int64_t aValue); +void gecko_profiler_json_writer_float_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, double aValue); +void gecko_profiler_json_writer_bool_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, bool aValue); +void gecko_profiler_json_writer_string_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, const char* aValue, size_t aValueLength); +void gecko_profiler_json_writer_unique_string_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength, const char* aValue, size_t aValueLength); +void gecko_profiler_json_writer_null_property( + mozilla::baseprofiler::SpliceableJSONWriter* aWriter, const char* aName, + size_t aNameLength); + +// Marker APIs. +void gecko_profiler_add_marker_untyped( + const char* aName, size_t aNameLength, + mozilla::baseprofiler::ProfilingCategoryPair aCategoryPair, + mozilla::MarkerTiming* aMarkerTiming, + mozilla::StackCaptureOptions aStackCaptureOptions); +void gecko_profiler_add_marker_text( + const char* aName, size_t aNameLength, + mozilla::baseprofiler::ProfilingCategoryPair aCategoryPair, + mozilla::MarkerTiming* aMarkerTiming, + mozilla::StackCaptureOptions aStackCaptureOptions, const char* aText, + size_t aTextLength); +void gecko_profiler_add_marker( + const char* aName, size_t aNameLength, + mozilla::baseprofiler::ProfilingCategoryPair aCategoryPair, + mozilla::MarkerTiming* aMarkerTiming, + mozilla::StackCaptureOptions aStackCaptureOptions, uint8_t aMarkerTag, + const uint8_t* aPayload, size_t aPayloadSize); + +} // extern "C" + +#endif // ProfilerBindings_h diff --git a/tools/profiler/public/ProfilerChild.h b/tools/profiler/public/ProfilerChild.h new file mode 100644 index 0000000000..a781784aae --- /dev/null +++ b/tools/profiler/public/ProfilerChild.h @@ -0,0 +1,106 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerChild_h +#define ProfilerChild_h + +#include "mozilla/BaseProfilerDetail.h" +#include "mozilla/DataMutex.h" +#include "mozilla/PProfilerChild.h" +#include "mozilla/ProfileBufferControlledChunkManager.h" +#include "mozilla/ProgressLogger.h" +#include "mozilla/RefPtr.h" +#include "ProfileAdditionalInformation.h" + +class nsIThread; +struct PRThread; + +namespace mozilla { + +// The ProfilerChild actor is created in all processes except for the main +// process. The corresponding ProfilerParent actor is created in the main +// process, and it will notify us about profiler state changes and request +// profiles from us. +class ProfilerChild final : public PProfilerChild, + public mozilla::ipc::IShmemAllocator { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ProfilerChild, final) + + ProfilerChild(); + + // Collects and returns a profile. + // This method can be used to grab a profile just before PProfiler is torn + // down. The collected profile should then be sent through a different + // message channel that is guaranteed to stay open long enough. + ProfileAndAdditionalInformation GrabShutdownProfile(); + + void Destroy(); + + // This should be called regularly from outside of the profiler lock. + static void ProcessPendingUpdate(); + + static bool IsLockedOnCurrentThread(); + + private: + virtual ~ProfilerChild(); + + mozilla::ipc::IPCResult RecvStart(const ProfilerInitParams& params, + StartResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvEnsureStarted( + const ProfilerInitParams& params, + EnsureStartedResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvStop(StopResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvPause(PauseResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvResume(ResumeResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvPauseSampling( + PauseSamplingResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvResumeSampling( + ResumeSamplingResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvWaitOnePeriodicSampling( + WaitOnePeriodicSamplingResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvAwaitNextChunkManagerUpdate( + AwaitNextChunkManagerUpdateResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvDestroyReleasedChunksAtOrBefore( + const TimeStamp& aTimeStamp) override; + mozilla::ipc::IPCResult RecvGatherProfile( + GatherProfileResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvGetGatherProfileProgress( + GetGatherProfileProgressResolver&& aResolve) override; + mozilla::ipc::IPCResult RecvClearAllPages() override; + + void ActorDestroy(ActorDestroyReason aActorDestroyReason) override; + + FORWARD_SHMEM_ALLOCATOR_TO(PProfilerChild) + + void SetupChunkManager(); + void ResetChunkManager(); + void ResolveChunkUpdate( + PProfilerChild::AwaitNextChunkManagerUpdateResolver& aResolve); + void ProcessChunkManagerUpdate( + ProfileBufferControlledChunkManager::Update&& aUpdate); + + static void GatherProfileThreadFunction(void* already_AddRefedParameters); + + nsCOMPtr<nsIThread> mThread; + bool mDestroyed; + + ProfileBufferControlledChunkManager* mChunkManager = nullptr; + AwaitNextChunkManagerUpdateResolver mAwaitNextChunkManagerUpdateResolver; + ProfileBufferControlledChunkManager::Update mChunkManagerUpdate; + + struct ProfilerChildAndUpdate { + RefPtr<ProfilerChild> mProfilerChild; + ProfileBufferControlledChunkManager::Update mUpdate; + }; + static DataMutexBase<ProfilerChildAndUpdate, + baseprofiler::detail::BaseProfilerMutex> + sPendingChunkManagerUpdate; + + RefPtr<ProgressLogger::SharedProgress> mGatherProfileProgress; +}; + +} // namespace mozilla + +#endif // ProfilerChild_h diff --git a/tools/profiler/public/ProfilerCodeAddressService.h b/tools/profiler/public/ProfilerCodeAddressService.h new file mode 100644 index 0000000000..9d75c363b3 --- /dev/null +++ b/tools/profiler/public/ProfilerCodeAddressService.h @@ -0,0 +1,52 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerCodeAddressService_h +#define ProfilerCodeAddressService_h + +#include "CodeAddressService.h" +#include "nsTArray.h" + +namespace mozilla { + +// This SymbolTable struct, and the CompactSymbolTable struct in the +// profiler rust module, have the exact same memory layout. +// nsTArray and ThinVec are FFI-compatible, because the thin-vec crate is +// being compiled with the "gecko-ffi" feature enabled. +struct SymbolTable { + SymbolTable() = default; + SymbolTable(SymbolTable&& aOther) = default; + + nsTArray<uint32_t> mAddrs; + nsTArray<uint32_t> mIndex; + nsTArray<uint8_t> mBuffer; +}; + +} // namespace mozilla + +/** + * Cache and look up function symbol names. + * + * We don't template this on AllocPolicy since we need to use nsTArray in + * SymbolTable above, which doesn't work with AllocPolicy. (We can't switch + * to Vector, as we would lose FFI compatibility with ThinVec.) + */ +class ProfilerCodeAddressService : public mozilla::CodeAddressService<> { + public: + // Like GetLocation, but only returns the symbol name. + bool GetFunction(const void* aPc, nsACString& aResult); + + private: +#if defined(XP_LINUX) || defined(XP_FREEBSD) + // Map of library names (owned by mLibraryStrings) to SymbolTables filled + // in by profiler_get_symbol_table. + mozilla::HashMap<const char*, mozilla::SymbolTable, + mozilla::DefaultHasher<const char*>, AllocPolicy> + mSymbolTables; +#endif +}; + +#endif // ProfilerCodeAddressService_h diff --git a/tools/profiler/public/ProfilerControl.h b/tools/profiler/public/ProfilerControl.h new file mode 100644 index 0000000000..466d15eb69 --- /dev/null +++ b/tools/profiler/public/ProfilerControl.h @@ -0,0 +1,190 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// APIs that control the lifetime of the profiler: Initialization, start, pause, +// resume, stop, and shutdown. + +#ifndef ProfilerControl_h +#define ProfilerControl_h + +#include "mozilla/BaseProfilerRAIIMacro.h" + +// Everything in here is also safe to include unconditionally, and only defines +// empty macros if MOZ_GECKO_PROFILER is unset. +// If your file only uses particular APIs (e.g., only markers), please consider +// including only the needed headers instead of this one, to reduce compilation +// dependencies. + +enum class IsFastShutdown { + No, + Yes, +}; + +#ifndef MOZ_GECKO_PROFILER + +// This file can be #included unconditionally. However, everything within this +// file must be guarded by a #ifdef MOZ_GECKO_PROFILER, *except* for the +// following macros and functions, which encapsulate the most common operations +// and thus avoid the need for many #ifdefs. + +# define AUTO_PROFILER_INIT ::profiler_init_main_thread_id() +# define AUTO_PROFILER_INIT2 + +// Function stubs for when MOZ_GECKO_PROFILER is not defined. + +static inline void profiler_init(void* stackTop) {} + +static inline void profiler_shutdown( + IsFastShutdown aIsFastShutdown = IsFastShutdown::No) {} + +#else // !MOZ_GECKO_PROFILER + +# include "BaseProfiler.h" +# include "mozilla/Attributes.h" +# include "mozilla/Maybe.h" +# include "mozilla/MozPromise.h" +# include "mozilla/PowerOfTwo.h" +# include "mozilla/Vector.h" + +//--------------------------------------------------------------------------- +// Start and stop the profiler +//--------------------------------------------------------------------------- + +static constexpr mozilla::PowerOfTwo32 PROFILER_DEFAULT_ENTRIES = + mozilla::baseprofiler::BASE_PROFILER_DEFAULT_ENTRIES; + +static constexpr mozilla::PowerOfTwo32 PROFILER_DEFAULT_STARTUP_ENTRIES = + mozilla::baseprofiler::BASE_PROFILER_DEFAULT_STARTUP_ENTRIES; + +# define PROFILER_DEFAULT_INTERVAL BASE_PROFILER_DEFAULT_INTERVAL +# define PROFILER_MAX_INTERVAL BASE_PROFILER_MAX_INTERVAL + +# define PROFILER_DEFAULT_ACTIVE_TAB_ID 0 + +// Initialize the profiler. If MOZ_PROFILER_STARTUP is set the profiler will +// also be started. This call must happen before any other profiler calls +// (except profiler_start(), which will call profiler_init() if it hasn't +// already run). +void profiler_init(void* stackTop); +void profiler_init_threadmanager(); + +// Call this as early as possible +# define AUTO_PROFILER_INIT mozilla::AutoProfilerInit PROFILER_RAII +// Call this after the nsThreadManager is Init()ed +# define AUTO_PROFILER_INIT2 mozilla::AutoProfilerInit2 PROFILER_RAII + +// Clean up the profiler module, stopping it if required. This function may +// also save a shutdown profile if requested. No profiler calls should happen +// after this point and all profiling stack labels should have been popped. +void profiler_shutdown(IsFastShutdown aIsFastShutdown = IsFastShutdown::No); + +// Start the profiler -- initializing it first if necessary -- with the +// selected options. Stops and restarts the profiler if it is already active. +// After starting the profiler is "active". The samples will be recorded in a +// circular buffer. +// "aCapacity" is the maximum number of 8-bytes entries in the profiler's +// circular buffer. +// "aInterval" the sampling interval, measured in millseconds. +// "aFeatures" is the feature set. Features unsupported by this +// platform/configuration are ignored. +// "aFilters" is the list of thread filters. Threads that do not match any +// of the filters are not profiled. A filter matches a thread if +// (a) the thread name contains the filter as a case-insensitive +// substring, or +// (b) the filter is of the form "pid:<n>" where n is the process +// id of the process that the thread is running in. +// "aActiveTabID" BrowserId of the active browser screen's active tab. +// It's being used to determine the profiled tab. It's "0" if +// we failed to get the ID. +// "aDuration" is the duration of entries in the profiler's circular buffer. +// Returns as soon as this process' profiler has started, the returned promise +// gets resolved when profilers in sub-processes (if any) have started. +RefPtr<mozilla::GenericPromise> profiler_start( + mozilla::PowerOfTwo32 aCapacity, double aInterval, uint32_t aFeatures, + const char** aFilters, uint32_t aFilterCount, uint64_t aActiveTabID, + const mozilla::Maybe<double>& aDuration = mozilla::Nothing()); + +// Stop the profiler and discard the profile without saving it. A no-op if the +// profiler is inactive. After stopping the profiler is "inactive". +// Returns as soon as this process' profiler has stopped, the returned promise +// gets resolved when profilers in sub-processes (if any) have stopped. +RefPtr<mozilla::GenericPromise> profiler_stop(); + +// If the profiler is inactive, start it. If it's already active, restart it if +// the requested settings differ from the current settings. Both the check and +// the state change are performed while the profiler state is locked. +// The only difference to profiler_start is that the current buffer contents are +// not discarded if the profiler is already running with the requested settings. +void profiler_ensure_started( + mozilla::PowerOfTwo32 aCapacity, double aInterval, uint32_t aFeatures, + const char** aFilters, uint32_t aFilterCount, uint64_t aActiveTabID, + const mozilla::Maybe<double>& aDuration = mozilla::Nothing()); + +//--------------------------------------------------------------------------- +// Control the profiler +//--------------------------------------------------------------------------- + +// Pause and resume the profiler. No-ops if the profiler is inactive. While +// paused the profile will not take any samples and will not record any data +// into its buffers. The profiler remains fully initialized in this state. +// Timeline markers will still be stored. This feature will keep JavaScript +// profiling enabled, thus allowing toggling the profiler without invalidating +// the JIT. +// Returns as soon as this process' profiler has paused/resumed, the returned +// promise gets resolved when profilers in sub-processes (if any) have +// paused/resumed. +RefPtr<mozilla::GenericPromise> profiler_pause(); +RefPtr<mozilla::GenericPromise> profiler_resume(); + +// Only pause and resume the periodic sampling loop, including stack sampling, +// counters, and profiling overheads. +// Returns as soon as this process' profiler has paused/resumed sampling, the +// returned promise gets resolved when profilers in sub-processes (if any) have +// paused/resumed sampling. +RefPtr<mozilla::GenericPromise> profiler_pause_sampling(); +RefPtr<mozilla::GenericPromise> profiler_resume_sampling(); + +//--------------------------------------------------------------------------- +// Get information from the profiler +//--------------------------------------------------------------------------- + +// Get the params used to start the profiler. Returns 0 and an empty vector +// (via outparams) if the profile is inactive. It's possible that the features +// returned may be slightly different to those requested due to required +// adjustments. +void profiler_get_start_params( + int* aEntrySize, mozilla::Maybe<double>* aDuration, double* aInterval, + uint32_t* aFeatures, + mozilla::Vector<const char*, 0, mozilla::MallocAllocPolicy>* aFilters, + uint64_t* aActiveTabID); + +//--------------------------------------------------------------------------- +// RAII classes +//--------------------------------------------------------------------------- + +namespace mozilla { + +class MOZ_RAII AutoProfilerInit { + public: + explicit AutoProfilerInit() { profiler_init(this); } + + ~AutoProfilerInit() { profiler_shutdown(); } + + private: +}; + +class MOZ_RAII AutoProfilerInit2 { + public: + explicit AutoProfilerInit2() { profiler_init_threadmanager(); } + + private: +}; + +} // namespace mozilla + +#endif // !MOZ_GECKO_PROFILER + +#endif // ProfilerControl_h diff --git a/tools/profiler/public/ProfilerCounts.h b/tools/profiler/public/ProfilerCounts.h new file mode 100644 index 0000000000..cebca81e2c --- /dev/null +++ b/tools/profiler/public/ProfilerCounts.h @@ -0,0 +1,297 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerCounts_h +#define ProfilerCounts_h + +#ifndef MOZ_GECKO_PROFILER + +# define PROFILER_DEFINE_COUNT_TOTAL(label, category, description) +# define PROFILER_DEFINE_COUNT(label, category, description) +# define PROFILER_DEFINE_STATIC_COUNT_TOTAL(label, category, description) +# define AUTO_PROFILER_TOTAL(label, count) +# define AUTO_PROFILER_COUNT(label) +# define AUTO_PROFILER_STATIC_COUNT(label, count) + +#else + +# include "mozilla/Assertions.h" +# include "mozilla/Atomics.h" + +class BaseProfilerCount; +void profiler_add_sampled_counter(BaseProfilerCount* aCounter); +void profiler_remove_sampled_counter(BaseProfilerCount* aCounter); + +typedef mozilla::Atomic<int64_t, mozilla::MemoryOrdering::Relaxed> + ProfilerAtomicSigned; +typedef mozilla::Atomic<uint64_t, mozilla::MemoryOrdering::Relaxed> + ProfilerAtomicUnsigned; + +// Counter support +// There are two types of counters: +// 1) a simple counter which can be added to or subtracted from. This could +// track the number of objects of a type, the number of calls to something +// (reflow, JIT, etc). +// 2) a combined counter which has the above, plus a number-of-calls counter +// that is incremented by 1 for each call to modify the count. This provides +// an optional source for a 'heatmap' of access. This can be used (for +// example) to track the amount of memory allocated, and provide a heatmap of +// memory operations (allocs/frees). +// +// Counters are sampled by the profiler once per sample-period. At this time, +// all counters are global to the process. In the future, there might be more +// versions with per-thread or other discriminators. +// +// Typical usage: +// There are two ways to use counters: With heap-created counter objects, +// or using macros. Note: the macros use statics, and will be slightly +// faster/smaller, and you need to care about creating them before using +// them. They're similar to the use-pattern for the other AUTO_PROFILER* +// macros, but they do need the PROFILER_DEFINE* to be use to instantiate +// the statics. +// +// PROFILER_DEFINE_COUNT(mything, "JIT", "Some JIT byte count") +// ... +// void foo() { ... AUTO_PROFILER_COUNT(mything, number_of_bytes_used); ... } +// +// or (to also get a heatmap) +// +// PROFILER_DEFINE_COUNT_TOTAL(mything, "JIT", "Some JIT byte count") +// ... +// void foo() { +// ... +// AUTO_PROFILER_COUNT_TOTAL(mything, number_of_bytes_generated); +// ... +// } +// +// To use without statics/macros: +// +// UniquePtr<ProfilerCounter> myCounter; +// ... +// myCounter = +// MakeUnique<ProfilerCounter>("mything", "JIT", "Some JIT byte count")); +// ... +// void foo() { ... myCounter->Add(number_of_bytes_generated0; ... } + +class BaseProfilerCount { + public: + BaseProfilerCount(const char* aLabel, ProfilerAtomicSigned* aCounter, + ProfilerAtomicUnsigned* aNumber, const char* aCategory, + const char* aDescription) + : mLabel(aLabel), + mCategory(aCategory), + mDescription(aDescription), + mCounter(aCounter), + mNumber(aNumber) { +# define COUNTER_CANARY 0xDEADBEEF +# ifdef DEBUG + mCanary = COUNTER_CANARY; + mPrevNumber = 0; +# endif + // Can't call profiler_* here since this may be non-xul-library + } + + virtual ~BaseProfilerCount() { +# ifdef DEBUG + mCanary = 0; +# endif + } + + struct CountSample { + int64_t count; + uint64_t number; + // This field indicates if the sample has already been consummed by a call + // to the Sample() method. This allows the profiler to discard duplicate + // samples if the counter sampling rate is lower than the profiler sampling + // rate. This can happen for example with some power meters that sample up + // to every 10ms. + // It should always be true when calling Sample() for the first time. + bool isSampleNew; + }; + virtual CountSample Sample() { + MOZ_ASSERT(mCanary == COUNTER_CANARY); + + CountSample result; + result.count = *mCounter; + result.number = mNumber ? *mNumber : 0; +# ifdef DEBUG + MOZ_ASSERT(result.number >= mPrevNumber); + mPrevNumber = result.number; +# endif + result.isSampleNew = true; + return result; + } + + void Clear() { + *mCounter = 0; + // We don't reset *mNumber or mPrevNumber. We encode numbers as + // positive deltas, and currently we only care about the deltas (for + // e.g. heatmaps). If we ever need to clear mNumber as well, we can an + // alternative method (Reset()) to do so. + } + + // We don't define ++ and Add() here, since the static defines directly + // increment the atomic counters, and the subclasses implement ++ and + // Add() directly. + + // These typically are static strings (for example if you use the macros + // below) + const char* mLabel; + const char* mCategory; + const char* mDescription; + // We're ok with these being un-ordered in race conditions. These are + // pointers because we want to be able to use statics and increment them + // directly. Otherwise we could just have them inline, and not need the + // constructor args. + // These can be static globals (using the macros below), though they + // don't have to be - their lifetime must be longer than the use of them + // by the profiler (see profiler_add/remove_sampled_counter()). If you're + // using a lot of these, they probably should be allocated at runtime (see + // class ProfilerCountOnly below). + ProfilerAtomicSigned* mCounter; + ProfilerAtomicUnsigned* mNumber; // may be null + +# ifdef DEBUG + uint32_t mCanary; + uint64_t mPrevNumber; // value of number from the last Sample() +# endif +}; + +// Designed to be allocated dynamically, and simply incremented with obj++ +// or obj->Add(n) +class ProfilerCounter final : public BaseProfilerCount { + public: + ProfilerCounter(const char* aLabel, const char* aCategory, + const char* aDescription) + : BaseProfilerCount(aLabel, &mCounter, nullptr, aCategory, aDescription) { + // Assume we're in libxul + profiler_add_sampled_counter(this); + } + + virtual ~ProfilerCounter() { profiler_remove_sampled_counter(this); } + + BaseProfilerCount& operator++() { + Add(1); + return *this; + } + + void Add(int64_t aNumber) { mCounter += aNumber; } + + ProfilerAtomicSigned mCounter; +}; + +// Also keeps a heatmap (number of calls to ++/Add()) +class ProfilerCounterTotal final : public BaseProfilerCount { + public: + ProfilerCounterTotal(const char* aLabel, const char* aCategory, + const char* aDescription) + : BaseProfilerCount(aLabel, &mCounter, &mNumber, aCategory, + aDescription) { + // Assume we're in libxul + profiler_add_sampled_counter(this); + } + + virtual ~ProfilerCounterTotal() { profiler_remove_sampled_counter(this); } + + BaseProfilerCount& operator++() { + Add(1); + return *this; + } + + void Add(int64_t aNumber) { + mCounter += aNumber; + mNumber++; + } + + ProfilerAtomicSigned mCounter; + ProfilerAtomicUnsigned mNumber; +}; + +// Defines a counter that is sampled on each profiler tick, with a running +// count (signed), and number-of-instances. Note that because these are two +// independent Atomics, there is a possiblity that count will not include +// the last call, but number of uses will. I think this is not worth +// worrying about +# define PROFILER_DEFINE_COUNT_TOTAL(label, category, description) \ + ProfilerAtomicSigned profiler_count_##label(0); \ + ProfilerAtomicUnsigned profiler_number_##label(0); \ + const char profiler_category_##label[] = category; \ + const char profiler_description_##label[] = description; \ + mozilla::UniquePtr<BaseProfilerCount> AutoCount_##label; + +// This counts, but doesn't keep track of the number of calls to +// AUTO_PROFILER_COUNT() +# define PROFILER_DEFINE_COUNT(label, category, description) \ + ProfilerAtomicSigned profiler_count_##label(0); \ + const char profiler_category_##label[] = category; \ + const char profiler_description_##label[] = description; \ + mozilla::UniquePtr<BaseProfilerCount> AutoCount_##label; + +// This will create a static initializer if used, but avoids a possible +// allocation. +# define PROFILER_DEFINE_STATIC_COUNT_TOTAL(label, category, description) \ + ProfilerAtomicSigned profiler_count_##label(0); \ + ProfilerAtomicUnsigned profiler_number_##label(0); \ + BaseProfilerCount AutoCount_##label(#label, &profiler_count_##label, \ + &profiler_number_##label, category, \ + description); + +// If we didn't care about static initializers, we could avoid the need for +// a ptr to the BaseProfilerCount object. + +// XXX It would be better to do this without the if() and without the +// theoretical race to set the UniquePtr (i.e. possible leak). +# define AUTO_PROFILER_COUNT_TOTAL(label, count) \ + do { \ + profiler_number_##label++; /* do this first*/ \ + profiler_count_##label += count; \ + if (!AutoCount_##label) { \ + /* Ignore that we could call this twice in theory, and that we leak \ + * them \ + */ \ + AutoCount_##label.reset(new BaseProfilerCount( \ + #label, &profiler_count_##label, &profiler_number_##label, \ + profiler_category_##label, profiler_description_##label)); \ + profiler_add_sampled_counter(AutoCount_##label.get()); \ + } \ + } while (0) + +# define AUTO_PROFILER_COUNT(label, count) \ + do { \ + profiler_count_##label += count; /* do this first*/ \ + if (!AutoCount_##label) { \ + /* Ignore that we could call this twice in theory, and that we leak \ + * them \ + */ \ + AutoCount_##label.reset(new BaseProfilerCount( \ + #label, nullptr, &profiler_number_##label, \ + profiler_category_##label, profiler_description_##label)); \ + profiler_add_sampled_counter(AutoCount_##label.get()); \ + } \ + } while (0) + +# define AUTO_PROFILER_STATIC_COUNT(label, count) \ + do { \ + profiler_number_##label++; /* do this first*/ \ + profiler_count_##label += count; \ + } while (0) + +// if we need to force the allocation +# define AUTO_PROFILER_FORCE_ALLOCATION(label) \ + do { \ + if (!AutoCount_##label) { \ + /* Ignore that we could call this twice in theory, and that we leak \ + * them \ + */ \ + AutoCount_##label.reset(new BaseProfilerCount( \ + #label, &profiler_count_##label, &profiler_number_##label, \ + profiler_category_##label, profiler_description_##label)); \ + } \ + } while (0) + +#endif // !MOZ_GECKO_PROFILER + +#endif // ProfilerCounts_h diff --git a/tools/profiler/public/ProfilerLabels.h b/tools/profiler/public/ProfilerLabels.h new file mode 100644 index 0000000000..a1585d8dd8 --- /dev/null +++ b/tools/profiler/public/ProfilerLabels.h @@ -0,0 +1,326 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This header contains all definitions related to profiler labels. +// It is safe to include unconditionally, and only defines empty macros if +// MOZ_GECKO_PROFILER is not set. + +#ifndef ProfilerLabels_h +#define ProfilerLabels_h + +#include "mozilla/ProfilerState.h" +#include "mozilla/ProfilerThreadState.h" + +#include "js/ProfilingCategory.h" +#include "js/ProfilingStack.h" +#include "js/RootingAPI.h" +#include "mozilla/Assertions.h" +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/BaseProfilerRAIIMacro.h" +#include "mozilla/Maybe.h" +#include "mozilla/ProfilerThreadRegistration.h" +#include "mozilla/ThreadLocal.h" +#include "nsString.h" + +#include <stdint.h> + +struct JSContext; + +// Insert an RAII object in this scope to enter a label stack frame. Any +// samples collected in this scope will contain this label in their stack. +// The label argument must be a static C string. It is usually of the +// form "ClassName::FunctionName". (Ideally we'd use the compiler to provide +// that for us, but __func__ gives us the function name without the class +// name.) If the label applies to only part of a function, you can qualify it +// like this: "ClassName::FunctionName:PartName". +// +// Use AUTO_PROFILER_LABEL_DYNAMIC_* if you want to add additional / dynamic +// information to the label stack frame, and AUTO_PROFILER_LABEL_HOT if you're +// instrumenting functions for which overhead on the order of nanoseconds is +// noticeable. +#define AUTO_PROFILER_LABEL(label, categoryPair) \ + mozilla::AutoProfilerLabel PROFILER_RAII( \ + label, nullptr, JS::ProfilingCategoryPair::categoryPair) + +// Like AUTO_PROFILER_LABEL, but for super-hot code where overhead must be +// kept to the absolute minimum. This variant doesn't push the label if the +// profiler isn't running. +// Don't use this for long-running functions: If the profiler is started in +// the middle of the function, this label won't be on the stack until the +// function is entered the next time. As a result, category information for +// samples at the start of the profile can be misleading. +// For short-running functions, that's often an acceptable trade-off. +#define AUTO_PROFILER_LABEL_HOT(label, categoryPair) \ + mozilla::AutoProfilerLabelHot PROFILER_RAII( \ + label, nullptr, JS::ProfilingCategoryPair::categoryPair) + +// Similar to AUTO_PROFILER_LABEL, but that adds the RELEVANT_FOR_JS flag. +#define AUTO_PROFILER_LABEL_RELEVANT_FOR_JS(label, categoryPair) \ + mozilla::AutoProfilerLabel PROFILER_RAII( \ + label, nullptr, JS::ProfilingCategoryPair::categoryPair, \ + uint32_t(js::ProfilingStackFrame::Flags::RELEVANT_FOR_JS)) + +// Similar to AUTO_PROFILER_LABEL, but with only one argument: the category +// pair. The label string is taken from the category pair. This is convenient +// for labels like AUTO_PROFILER_LABEL_CATEGORY_PAIR(GRAPHICS_LayerBuilding) +// which would otherwise just repeat the string. +#define AUTO_PROFILER_LABEL_CATEGORY_PAIR(categoryPair) \ + mozilla::AutoProfilerLabel PROFILER_RAII( \ + "", nullptr, JS::ProfilingCategoryPair::categoryPair, \ + uint32_t( \ + js::ProfilingStackFrame::Flags::LABEL_DETERMINED_BY_CATEGORY_PAIR)) + +// Similar to AUTO_PROFILER_LABEL_CATEGORY_PAIR but adding the RELEVANT_FOR_JS +// flag. +#define AUTO_PROFILER_LABEL_CATEGORY_PAIR_RELEVANT_FOR_JS(categoryPair) \ + mozilla::AutoProfilerLabel PROFILER_RAII( \ + "", nullptr, JS::ProfilingCategoryPair::categoryPair, \ + uint32_t( \ + js::ProfilingStackFrame::Flags::LABEL_DETERMINED_BY_CATEGORY_PAIR) | \ + uint32_t(js::ProfilingStackFrame::Flags::RELEVANT_FOR_JS)) + +// Similar to AUTO_PROFILER_LABEL, but with an additional string. The inserted +// RAII object stores the cStr pointer in a field; it does not copy the string. +// +// WARNING: This means that the string you pass to this macro needs to live at +// least until the end of the current scope. Be careful using this macro with +// ns[C]String; the other AUTO_PROFILER_LABEL_DYNAMIC_* macros below are +// preferred because they avoid this problem. +// +// If the profiler samples the current thread and walks the label stack while +// this RAII object is on the stack, it will copy the supplied string into the +// profile buffer. So there's one string copy operation, and it happens at +// sample time. +// +// Compare this to the plain AUTO_PROFILER_LABEL macro, which only accepts +// literal strings: When the label stack frames generated by +// AUTO_PROFILER_LABEL are sampled, no string copy needs to be made because the +// profile buffer can just store the raw pointers to the literal strings. +// Consequently, AUTO_PROFILER_LABEL frames take up considerably less space in +// the profile buffer than AUTO_PROFILER_LABEL_DYNAMIC_* frames. +#define AUTO_PROFILER_LABEL_DYNAMIC_CSTR(label, categoryPair, cStr) \ + mozilla::AutoProfilerLabel PROFILER_RAII( \ + label, cStr, JS::ProfilingCategoryPair::categoryPair) + +// Like AUTO_PROFILER_LABEL_DYNAMIC_CSTR, but with the NONSENSITIVE flag to +// note that it does not contain sensitive information (so we can include it +// in, for example, the BackgroundHangMonitor) +#define AUTO_PROFILER_LABEL_DYNAMIC_CSTR_NONSENSITIVE(label, categoryPair, \ + cStr) \ + mozilla::AutoProfilerLabel PROFILER_RAII( \ + label, cStr, JS::ProfilingCategoryPair::categoryPair, \ + uint32_t(js::ProfilingStackFrame::Flags::NONSENSITIVE)) + +// Similar to AUTO_PROFILER_LABEL_DYNAMIC_CSTR, but takes an nsACString. +// +// Note: The use of the Maybe<>s ensures the scopes for the dynamic string and +// the AutoProfilerLabel are appropriate, while also not incurring the runtime +// cost of the string assignment unless the profiler is active. Therefore, +// unlike AUTO_PROFILER_LABEL and AUTO_PROFILER_LABEL_DYNAMIC_CSTR, this macro +// doesn't push/pop a label when the profiler is inactive. +#define AUTO_PROFILER_LABEL_DYNAMIC_NSCSTRING(label, categoryPair, nsCStr) \ + mozilla::Maybe<nsAutoCString> autoCStr; \ + mozilla::Maybe<mozilla::AutoProfilerLabel> raiiObjectNsCString; \ + if (profiler_is_active()) { \ + autoCStr.emplace(nsCStr); \ + raiiObjectNsCString.emplace(label, autoCStr->get(), \ + JS::ProfilingCategoryPair::categoryPair); \ + } + +#define AUTO_PROFILER_LABEL_DYNAMIC_NSCSTRING_RELEVANT_FOR_JS( \ + label, categoryPair, nsCStr) \ + mozilla::Maybe<nsAutoCString> autoCStr; \ + mozilla::Maybe<mozilla::AutoProfilerLabel> raiiObjectNsCString; \ + if (profiler_is_active()) { \ + autoCStr.emplace(nsCStr); \ + raiiObjectNsCString.emplace( \ + label, autoCStr->get(), JS::ProfilingCategoryPair::categoryPair, \ + uint32_t(js::ProfilingStackFrame::Flags::RELEVANT_FOR_JS)); \ + } + +// Match the conditions for MOZ_ENABLE_BACKGROUND_HANG_MONITOR +#if defined(NIGHTLY_BUILD) && !defined(MOZ_DEBUG) && !defined(MOZ_TSAN) && \ + !defined(MOZ_ASAN) +# define SHOULD_CREATE_ALL_NONSENSITIVE_LABEL_FRAMES true +#else +# define SHOULD_CREATE_ALL_NONSENSITIVE_LABEL_FRAMES profiler_is_active() +#endif + +// See note above AUTO_PROFILER_LABEL_DYNAMIC_CSTR_NONSENSITIVE +#define AUTO_PROFILER_LABEL_DYNAMIC_NSCSTRING_NONSENSITIVE( \ + label, categoryPair, nsCStr) \ + mozilla::Maybe<nsAutoCString> autoCStr; \ + mozilla::Maybe<mozilla::AutoProfilerLabel> raiiObjectNsCString; \ + if (SHOULD_CREATE_ALL_NONSENSITIVE_LABEL_FRAMES) { \ + autoCStr.emplace(nsCStr); \ + raiiObjectNsCString.emplace( \ + label, autoCStr->get(), JS::ProfilingCategoryPair::categoryPair, \ + uint32_t(js::ProfilingStackFrame::Flags::NONSENSITIVE)); \ + } + +// Similar to AUTO_PROFILER_LABEL_DYNAMIC_CSTR, but takes an nsString that is +// is lossily converted to an ASCII string. +// +// Note: The use of the Maybe<>s ensures the scopes for the converted dynamic +// string and the AutoProfilerLabel are appropriate, while also not incurring +// the runtime cost of the string conversion unless the profiler is active. +// Therefore, unlike AUTO_PROFILER_LABEL and AUTO_PROFILER_LABEL_DYNAMIC_CSTR, +// this macro doesn't push/pop a label when the profiler is inactive. +#define AUTO_PROFILER_LABEL_DYNAMIC_LOSSY_NSSTRING(label, categoryPair, nsStr) \ + mozilla::Maybe<NS_LossyConvertUTF16toASCII> asciiStr; \ + mozilla::Maybe<mozilla::AutoProfilerLabel> raiiObjectLossyNsString; \ + if (profiler_is_active()) { \ + asciiStr.emplace(nsStr); \ + raiiObjectLossyNsString.emplace(label, asciiStr->get(), \ + JS::ProfilingCategoryPair::categoryPair); \ + } + +// Similar to AUTO_PROFILER_LABEL, but accepting a JSContext* parameter, and a +// no-op if the profiler is disabled. +// Used to annotate functions for which overhead in the range of nanoseconds is +// noticeable. It avoids overhead from the TLS lookup because it can get the +// ProfilingStack from the JS context, and avoids almost all overhead in the +// case where the profiler is disabled. +#define AUTO_PROFILER_LABEL_FAST(label, categoryPair, ctx) \ + mozilla::AutoProfilerLabelHot PROFILER_RAII( \ + ctx, label, nullptr, JS::ProfilingCategoryPair::categoryPair) + +// Similar to AUTO_PROFILER_LABEL_FAST, but also takes an extra string and an +// additional set of flags. The flags parameter should carry values from the +// js::ProfilingStackFrame::Flags enum. +#define AUTO_PROFILER_LABEL_DYNAMIC_FAST(label, dynamicString, categoryPair, \ + ctx, flags) \ + mozilla::AutoProfilerLabelHot PROFILER_RAII( \ + ctx, label, dynamicString, JS::ProfilingCategoryPair::categoryPair, \ + flags) + +namespace mozilla { + +#ifndef MOZ_GECKO_PROFILER + +class MOZ_RAII AutoProfilerLabel { + public: + // This is the AUTO_PROFILER_LABEL and AUTO_PROFILER_LABEL_DYNAMIC variant. + AutoProfilerLabel(const char* aLabel, const char* aDynamicString, + JS::ProfilingCategoryPair aCategoryPair, + uint32_t aFlags = 0) {} + + ~AutoProfilerLabel() {} +}; + +class MOZ_RAII AutoProfilerLabelHot { + public: + // This is the AUTO_PROFILER_LABEL_HOT variant. + AutoProfilerLabelHot(const char* aLabel, const char* aDynamicString, + JS::ProfilingCategoryPair aCategoryPair, + uint32_t aFlags = 0) {} + + // This is the AUTO_PROFILER_LABEL_FAST variant. + AutoProfilerLabelHot(JSContext* aJSContext, const char* aLabel, + const char* aDynamicString, + JS::ProfilingCategoryPair aCategoryPair, + uint32_t aFlags) {} + + ~AutoProfilerLabelHot() {} +}; + +#else // !MOZ_GECKO_PROFILER + +// This class creates a non-owning ProfilingStack reference. Objects of this +// class are stack-allocated, and so exist within a thread, and are thus bounded +// by the lifetime of the thread, which ensures that the references held can't +// be used after the ProfilingStack is destroyed. +class MOZ_RAII AutoProfilerLabel { + public: + // This is the AUTO_PROFILER_LABEL and AUTO_PROFILER_LABEL_DYNAMIC variant. + AutoProfilerLabel(const char* aLabel, const char* aDynamicString, + JS::ProfilingCategoryPair aCategoryPair, + uint32_t aFlags = 0) { + // Get the ProfilingStack from TLS. + mProfilingStack = profiler::ThreadRegistration::WithOnThreadRefOr( + [](profiler::ThreadRegistration::OnThreadRef aThread) { + return &aThread.UnlockedConstReaderAndAtomicRWRef() + .ProfilingStackRef(); + }, + nullptr); + if (mProfilingStack) { + mProfilingStack->pushLabelFrame(aLabel, aDynamicString, this, + aCategoryPair, aFlags); + } + } + + ~AutoProfilerLabel() { + // This function runs both on and off the main thread. + + if (mProfilingStack) { + mProfilingStack->pop(); + } + } + + private: + // We save a ProfilingStack pointer in the ctor so we don't have to redo the + // TLS lookup in the dtor. + ProfilingStack* mProfilingStack; +}; + +class MOZ_RAII AutoProfilerLabelHot { + public: + // This is the AUTO_PROFILER_LABEL_HOT variant. It does nothing if + // the profiler is inactive. + AutoProfilerLabelHot(const char* aLabel, const char* aDynamicString, + JS::ProfilingCategoryPair aCategoryPair, + uint32_t aFlags = 0) { + if (MOZ_LIKELY(!profiler_is_active())) { + mProfilingStack = nullptr; + return; + } + + // Get the ProfilingStack from TLS. + mProfilingStack = profiler::ThreadRegistration::WithOnThreadRefOr( + [](profiler::ThreadRegistration::OnThreadRef aThread) { + return &aThread.UnlockedConstReaderAndAtomicRWRef() + .ProfilingStackRef(); + }, + nullptr); + if (mProfilingStack) { + mProfilingStack->pushLabelFrame(aLabel, aDynamicString, this, + aCategoryPair, aFlags); + } + } + + // This is the AUTO_PROFILER_LABEL_FAST variant. It retrieves the + // ProfilingStack from the JSContext and does nothing if the profiler is + // inactive. + AutoProfilerLabelHot(JSContext* aJSContext, const char* aLabel, + const char* aDynamicString, + JS::ProfilingCategoryPair aCategoryPair, + uint32_t aFlags) { + mProfilingStack = js::GetContextProfilingStackIfEnabled(aJSContext); + if (MOZ_UNLIKELY(mProfilingStack)) { + mProfilingStack->pushLabelFrame(aLabel, aDynamicString, this, + aCategoryPair, aFlags); + } + } + + ~AutoProfilerLabelHot() { + // This function runs both on and off the main thread. + if (MOZ_UNLIKELY(mProfilingStack)) { + mProfilingStack->pop(); + } + } + + private: + // We save a ProfilingStack pointer in the ctor so we don't have to redo the + // TLS lookup in the dtor. + ProfilingStack* mProfilingStack; +}; + +#endif // !MOZ_GECKO_PROFILER + +} // namespace mozilla + +#endif // ProfilerLabels_h diff --git a/tools/profiler/public/ProfilerMarkerTypes.h b/tools/profiler/public/ProfilerMarkerTypes.h new file mode 100644 index 0000000000..0868c70e30 --- /dev/null +++ b/tools/profiler/public/ProfilerMarkerTypes.h @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerMarkerTypes_h +#define ProfilerMarkerTypes_h + +// This header contains common marker type definitions that rely on xpcom. +// +// It #include's "mozilla/BaseProfilerMarkerTypess.h" and "ProfilerMarkers.h", +// see these files for more marker types, how to define other marker types, and +// how to add markers to the profiler buffers. + +// !!! /!\ WORK IN PROGRESS /!\ !!! +// This file contains draft marker definitions, but most are not used yet. +// Further work is needed to complete these definitions, and use them to convert +// existing PROFILER_ADD_MARKER calls. See meta bug 1661394. + +#include "mozilla/BaseProfilerMarkerTypes.h" +#include "mozilla/ProfilerMarkers.h" +#include "js/ProfilingFrameIterator.h" +#include "js/Utility.h" +#include "mozilla/Preferences.h" +#include "mozilla/ServoTraversalStatistics.h" + +namespace geckoprofiler::markers { + +// Import some common markers from mozilla::baseprofiler::markers. +using MediaSampleMarker = mozilla::baseprofiler::markers::MediaSampleMarker; +using VideoFallingBehindMarker = + mozilla::baseprofiler::markers::VideoFallingBehindMarker; +using ContentBuildMarker = mozilla::baseprofiler::markers::ContentBuildMarker; +using MediaEngineMarker = mozilla::baseprofiler::markers::MediaEngineMarker; +using MediaEngineTextMarker = + mozilla::baseprofiler::markers::MediaEngineTextMarker; + +} // namespace geckoprofiler::markers + +#endif // ProfilerMarkerTypes_h diff --git a/tools/profiler/public/ProfilerMarkers.h b/tools/profiler/public/ProfilerMarkers.h new file mode 100644 index 0000000000..b1ba8c762f --- /dev/null +++ b/tools/profiler/public/ProfilerMarkers.h @@ -0,0 +1,509 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Markers are useful to delimit something important happening such as the first +// paint. Unlike labels, which are only recorded in the profile buffer if a +// sample is collected while the label is on the label stack, markers will +// always be recorded in the profile buffer. +// +// This header contains definitions necessary to add markers to the Gecko +// Profiler buffer. +// +// It #include's "mozilla/BaseProfilerMarkers.h", see that header for base +// definitions necessary to create marker types. +// +// If common marker types are needed, #include "ProfilerMarkerTypes.h" instead. +// +// But if you want to create your own marker type locally, you can #include this +// header only; look at ProfilerMarkerTypes.h for examples of how to define +// types. +// +// To then record markers: +// - Use `baseprofiler::AddMarker(...)` from mozglue or other libraries that are +// outside of xul, especially if they may happen outside of xpcom's lifetime +// (typically startup, shutdown, or tests). +// - Otherwise #include "ProfilerMarkers.h" instead, and use +// `profiler_add_marker(...)`. +// See these functions for more details. + +#ifndef ProfilerMarkers_h +#define ProfilerMarkers_h + +#include "mozilla/BaseProfilerMarkers.h" +#include "mozilla/ProfilerMarkersDetail.h" +#include "mozilla/ProfilerLabels.h" +#include "nsJSUtils.h" // for nsJSUtils::GetCurrentlyRunningCodeInnerWindowID +#include "nsString.h" +#include "ETWTools.h" + +class nsIDocShell; + +namespace geckoprofiler::markers::detail { +// Please do not use anything from the detail namespace outside the profiler. + +#ifdef MOZ_GECKO_PROFILER +mozilla::Maybe<uint64_t> profiler_get_inner_window_id_from_docshell( + nsIDocShell* aDocshell); +#else +inline mozilla::Maybe<uint64_t> profiler_get_inner_window_id_from_docshell( + nsIDocShell* aDocshell) { + return mozilla::Nothing(); +} +#endif // MOZ_GECKO_PROFILER + +} // namespace geckoprofiler::markers::detail + +// This is a helper function to get the Inner Window ID from DocShell but it's +// not a recommended method to get it and it's not encouraged to use this +// function. If there is a computed inner window ID, `window`, or `Document` +// available in the call site, please use them. Use this function as a last +// resort. +inline mozilla::MarkerInnerWindowId MarkerInnerWindowIdFromDocShell( + nsIDocShell* aDocshell) { + mozilla::Maybe<uint64_t> id = geckoprofiler::markers::detail:: + profiler_get_inner_window_id_from_docshell(aDocshell); + if (!id) { + return mozilla::MarkerInnerWindowId::NoId(); + } + return mozilla::MarkerInnerWindowId(*id); +} + +// This is a helper function to get the Inner Window ID from a JS Context but +// it's not a recommended method to get it and it's not encouraged to use this +// function. If there is a computed inner window ID, `window`, or `Document` +// available in the call site, please use them. Use this function as a last +// resort. +inline mozilla::MarkerInnerWindowId MarkerInnerWindowIdFromJSContext( + JSContext* aContext) { + return mozilla::MarkerInnerWindowId( + nsJSUtils::GetCurrentlyRunningCodeInnerWindowID(aContext)); +} + +// Bring category names from Base Profiler into the geckoprofiler::category +// namespace, for consistency with other Gecko Profiler identifiers. +namespace geckoprofiler::category { +using namespace ::mozilla::baseprofiler::category; +} + +#ifdef MOZ_GECKO_PROFILER +// Forward-declaration. TODO: Move to more common header, see bug 1681416. +bool profiler_capture_backtrace_into( + mozilla::ProfileChunkedBuffer& aChunkedBuffer, + mozilla::StackCaptureOptions aCaptureOptions); + +// Add a marker to a given buffer. `AddMarker()` and related macros should be +// used in most cases, see below for more information about them and the +// paramters; This function may be useful when markers need to be recorded in a +// local buffer outside of the main profiler buffer. +template <typename MarkerType, typename... PayloadArguments> +mozilla::ProfileBufferBlockIndex AddMarkerToBuffer( + mozilla::ProfileChunkedBuffer& aBuffer, + const mozilla::ProfilerString8View& aName, + const mozilla::MarkerCategory& aCategory, mozilla::MarkerOptions&& aOptions, + MarkerType aMarkerType, const PayloadArguments&... aPayloadArguments) { + AUTO_PROFILER_LABEL("AddMarkerToBuffer", PROFILER); + mozilla::Unused << aMarkerType; // Only the empty object type is useful. + return mozilla::base_profiler_markers_detail::AddMarkerToBuffer<MarkerType>( + aBuffer, aName, aCategory, std::move(aOptions), + profiler_active_without_feature(ProfilerFeature::NoMarkerStacks) + ? ::profiler_capture_backtrace_into + : nullptr, + aPayloadArguments...); +} + +// Add a marker (without payload) to a given buffer. +inline mozilla::ProfileBufferBlockIndex AddMarkerToBuffer( + mozilla::ProfileChunkedBuffer& aBuffer, + const mozilla::ProfilerString8View& aName, + const mozilla::MarkerCategory& aCategory, + mozilla::MarkerOptions&& aOptions = {}) { + return AddMarkerToBuffer(aBuffer, aName, aCategory, std::move(aOptions), + mozilla::baseprofiler::markers::NoPayload{}); +} +#endif + +// Internally we need to check specifically if gecko is collecting markers. +[[nodiscard]] inline bool profiler_thread_is_being_gecko_profiled_for_markers( + const ProfilerThreadId& aThreadId) { + return profiler_thread_is_being_profiled(aThreadId, + ThreadProfilingFeatures::Markers); +} + +// ETW collects on all threads. So when it is collecting these should always +// return true. +[[nodiscard]] inline bool profiler_thread_is_being_profiled_for_markers() { + return profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers) || + profiler_is_etw_collecting_markers(); +} + +[[nodiscard]] inline bool profiler_thread_is_being_profiled_for_markers( + const ProfilerThreadId& aThreadId) { + return profiler_thread_is_being_profiled(aThreadId, + ThreadProfilingFeatures::Markers) || + profiler_is_etw_collecting_markers(); +} + +// Add a marker to the Gecko Profiler buffer. +// - aName: Main name of this marker. +// - aCategory: Category for this marker. +// - aOptions: Optional settings (such as timing, inner window id, +// backtrace...), see `MarkerOptions` for details. +// - aMarkerType: Empty object that specifies the type of marker. +// - aPayloadArguments: Arguments expected by this marker type's +// ` StreamJSONMarkerData` function. +template <typename MarkerType, typename... PayloadArguments> +mozilla::ProfileBufferBlockIndex profiler_add_marker_impl( + const mozilla::ProfilerString8View& aName, + const mozilla::MarkerCategory& aCategory, mozilla::MarkerOptions&& aOptions, + MarkerType aMarkerType, const PayloadArguments&... aPayloadArguments) { +#ifndef MOZ_GECKO_PROFILER + return {}; +#else +# ifndef RUST_BINDGEN + // Bindgen can't take Windows.h and as such can't parse this. + ETW::EmitETWMarker(aName, aCategory, aOptions, aMarkerType, + aPayloadArguments...); +# endif + if (!profiler_thread_is_being_gecko_profiled_for_markers( + aOptions.ThreadId().ThreadId())) { + return {}; + } + AUTO_PROFILER_LABEL("profiler_add_marker", PROFILER); + return ::AddMarkerToBuffer(profiler_get_core_buffer(), aName, aCategory, + std::move(aOptions), aMarkerType, + aPayloadArguments...); +#endif +} + +// Add a marker (without payload) to the Gecko Profiler buffer. +inline mozilla::ProfileBufferBlockIndex profiler_add_marker_impl( + const mozilla::ProfilerString8View& aName, + const mozilla::MarkerCategory& aCategory, + mozilla::MarkerOptions&& aOptions = {}) { + return profiler_add_marker_impl(aName, aCategory, std::move(aOptions), + mozilla::baseprofiler::markers::NoPayload{}); +} + +// `profiler_add_marker` is a macro rather than a function so that arguments to +// it aren't unconditionally evaluated when not profiled. Some of the arguments +// might be non-trivial, see bug 1843534. +// +// The check used around `::profiler_add_marker_impl()` is a bit subtle. +// Naively, you might want to do +// `profiler_thread_is_being_profiled_for_markers()`, but markers can be +// targeted to different threads. +// So we do a cheaper `profiler_is_collecting_markers()` check instead to +// avoid any marker overhead when not profiling. This also allows us to do a +// single check that also checks if ETW is enabled. +#define profiler_add_marker(...) \ + do { \ + if (profiler_is_collecting_markers()) { \ + ::profiler_add_marker_impl(__VA_ARGS__); \ + } \ + } while (false) + +// Same as `profiler_add_marker()` (without payload). This macro is safe to use +// even if MOZ_GECKO_PROFILER is not #defined. +#define PROFILER_MARKER_UNTYPED(markerName, categoryName, ...) \ + do { \ + AUTO_PROFILER_STATS(PROFILER_MARKER_UNTYPED); \ + profiler_add_marker(markerName, ::geckoprofiler::category::categoryName, \ + ##__VA_ARGS__); \ + } while (false) + +// Same as `profiler_add_marker()` (with payload). This macro is safe to use +// even if MOZ_GECKO_PROFILER is not #defined. +#define PROFILER_MARKER(markerName, categoryName, options, MarkerType, ...) \ + do { \ + AUTO_PROFILER_STATS(PROFILER_MARKER_with_##MarkerType); \ + profiler_add_marker(markerName, ::geckoprofiler::category::categoryName, \ + options, ::geckoprofiler::markers::MarkerType{}, \ + ##__VA_ARGS__); \ + } while (false) + +namespace geckoprofiler::markers { +// Most common marker types. Others are in ProfilerMarkerTypes.h. +using TextMarker = ::mozilla::baseprofiler::markers::TextMarker; +using Tracing = mozilla::baseprofiler::markers::Tracing; +} // namespace geckoprofiler::markers + +// Add a text marker. This macro is safe to use even if MOZ_GECKO_PROFILER is +// not #defined. +#define PROFILER_MARKER_TEXT(markerName, categoryName, options, text) \ + do { \ + AUTO_PROFILER_STATS(PROFILER_MARKER_TEXT); \ + profiler_add_marker(markerName, ::geckoprofiler::category::categoryName, \ + options, ::geckoprofiler::markers::TextMarker{}, \ + text); \ + } while (false) + +// RAII object that adds a PROFILER_MARKER_TEXT when destroyed; the marker's +// timing will be the interval from construction (unless an instant or start +// time is already specified in the provided options) until destruction. +class MOZ_RAII AutoProfilerTextMarker { + public: + AutoProfilerTextMarker(const char* aMarkerName, + const mozilla::MarkerCategory& aCategory, + mozilla::MarkerOptions&& aOptions, + const nsACString& aText) + : mMarkerName(aMarkerName), + mCategory(aCategory), + mOptions(std::move(aOptions)), + mText(aText) { + MOZ_ASSERT(mOptions.Timing().EndTime().IsNull(), + "AutoProfilerTextMarker options shouldn't have an end time"); + if (profiler_is_active_and_unpaused() && + mOptions.Timing().StartTime().IsNull()) { + mOptions.Set(mozilla::MarkerTiming::InstantNow()); + } + } + + ~AutoProfilerTextMarker() { + if (profiler_is_active_and_unpaused()) { + AUTO_PROFILER_LABEL("TextMarker", PROFILER); + mOptions.TimingRef().SetIntervalEnd(); + AUTO_PROFILER_STATS(AUTO_PROFILER_MARKER_TEXT); + profiler_add_marker( + mozilla::ProfilerString8View::WrapNullTerminatedString(mMarkerName), + mCategory, std::move(mOptions), geckoprofiler::markers::TextMarker{}, + mText); + } + } + + protected: + const char* mMarkerName; + mozilla::MarkerCategory mCategory; + mozilla::MarkerOptions mOptions; + nsCString mText; +}; + +// Creates an AutoProfilerTextMarker RAII object. This macro is safe to use +// even if MOZ_GECKO_PROFILER is not #defined. +#define AUTO_PROFILER_MARKER_TEXT(markerName, categoryName, options, text) \ + AutoProfilerTextMarker PROFILER_RAII( \ + markerName, ::mozilla::baseprofiler::category::categoryName, options, \ + text) + +class MOZ_RAII AutoProfilerTracing { + public: + AutoProfilerTracing(const char* aCategoryString, const char* aMarkerName, + mozilla::MarkerCategory aCategoryPair, + const mozilla::Maybe<uint64_t>& aInnerWindowID) + : mCategoryString(aCategoryString), + mMarkerName(aMarkerName), + mCategoryPair(aCategoryPair), + mInnerWindowID(aInnerWindowID) { + profiler_add_marker( + mozilla::ProfilerString8View::WrapNullTerminatedString(mMarkerName), + mCategoryPair, + {mozilla::MarkerTiming::IntervalStart(), + mozilla::MarkerInnerWindowId(mInnerWindowID)}, + geckoprofiler::markers::Tracing{}, + mozilla::ProfilerString8View::WrapNullTerminatedString( + mCategoryString)); + } + + AutoProfilerTracing( + const char* aCategoryString, const char* aMarkerName, + mozilla::MarkerCategory aCategoryPair, + mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> aBacktrace, + const mozilla::Maybe<uint64_t>& aInnerWindowID) + : mCategoryString(aCategoryString), + mMarkerName(aMarkerName), + mCategoryPair(aCategoryPair), + mInnerWindowID(aInnerWindowID) { + profiler_add_marker( + mozilla::ProfilerString8View::WrapNullTerminatedString(mMarkerName), + mCategoryPair, + {mozilla::MarkerTiming::IntervalStart(), + mozilla::MarkerInnerWindowId(mInnerWindowID), + mozilla::MarkerStack::TakeBacktrace(std::move(aBacktrace))}, + geckoprofiler::markers::Tracing{}, + mozilla::ProfilerString8View::WrapNullTerminatedString( + mCategoryString)); + } + + ~AutoProfilerTracing() { + profiler_add_marker( + mozilla::ProfilerString8View::WrapNullTerminatedString(mMarkerName), + mCategoryPair, + {mozilla::MarkerTiming::IntervalEnd(), + mozilla::MarkerInnerWindowId(mInnerWindowID)}, + geckoprofiler::markers::Tracing{}, + mozilla::ProfilerString8View::WrapNullTerminatedString( + mCategoryString)); + } + + protected: + const char* mCategoryString; + const char* mMarkerName; + const mozilla::MarkerCategory mCategoryPair; + const mozilla::Maybe<uint64_t> mInnerWindowID; +}; + +// Adds a START/END pair of tracing markers. +#define AUTO_PROFILER_TRACING_MARKER(categoryString, markerName, categoryPair) \ + AutoProfilerTracing PROFILER_RAII(categoryString, markerName, \ + geckoprofiler::category::categoryPair, \ + mozilla::Nothing()) +#define AUTO_PROFILER_TRACING_MARKER_INNERWINDOWID( \ + categoryString, markerName, categoryPair, innerWindowId) \ + AutoProfilerTracing PROFILER_RAII(categoryString, markerName, \ + geckoprofiler::category::categoryPair, \ + mozilla::Some(innerWindowId)) +#define AUTO_PROFILER_TRACING_MARKER_DOCSHELL(categoryString, markerName, \ + categoryPair, docShell) \ + AutoProfilerTracing PROFILER_RAII( \ + categoryString, markerName, geckoprofiler::category::categoryPair, \ + geckoprofiler::markers::detail:: \ + profiler_get_inner_window_id_from_docshell(docShell)) + +#ifdef MOZ_GECKO_PROFILER +extern template mozilla::ProfileBufferBlockIndex AddMarkerToBuffer( + mozilla::ProfileChunkedBuffer&, const mozilla::ProfilerString8View&, + const mozilla::MarkerCategory&, mozilla::MarkerOptions&&, + mozilla::baseprofiler::markers::NoPayload); + +extern template mozilla::ProfileBufferBlockIndex AddMarkerToBuffer( + mozilla::ProfileChunkedBuffer&, const mozilla::ProfilerString8View&, + const mozilla::MarkerCategory&, mozilla::MarkerOptions&&, + mozilla::baseprofiler::markers::TextMarker, const std::string&); + +extern template mozilla::ProfileBufferBlockIndex profiler_add_marker_impl( + const mozilla::ProfilerString8View&, const mozilla::MarkerCategory&, + mozilla::MarkerOptions&&, mozilla::baseprofiler::markers::TextMarker, + const std::string&); + +extern template mozilla::ProfileBufferBlockIndex profiler_add_marker_impl( + const mozilla::ProfilerString8View&, const mozilla::MarkerCategory&, + mozilla::MarkerOptions&&, mozilla::baseprofiler::markers::TextMarker, + const nsCString&); + +extern template mozilla::ProfileBufferBlockIndex profiler_add_marker_impl( + const mozilla::ProfilerString8View&, const mozilla::MarkerCategory&, + mozilla::MarkerOptions&&, mozilla::baseprofiler::markers::Tracing, + const mozilla::ProfilerString8View&); +#endif // MOZ_GECKO_PROFILER + +namespace mozilla { + +namespace detail { +// GCC doesn't allow this to live inside the class. +template <typename PayloadType> +static void StreamPayload(baseprofiler::SpliceableJSONWriter& aWriter, + const Span<const char> aKey, + const PayloadType& aPayload) { + aWriter.StringProperty(aKey, aPayload); +} + +template <typename PayloadType> +inline void StreamPayload(baseprofiler::SpliceableJSONWriter& aWriter, + const Span<const char> aKey, + const Maybe<PayloadType>& aPayload) { + if (aPayload.isSome()) { + StreamPayload(aWriter, aKey, *aPayload); + } else { + aWriter.NullProperty(aKey); + } +} + +template <> +inline void StreamPayload<bool>(baseprofiler::SpliceableJSONWriter& aWriter, + const Span<const char> aKey, + const bool& aPayload) { + aWriter.BoolProperty(aKey, aPayload); +} + +template <> +inline void StreamPayload<ProfilerString16View>( + baseprofiler::SpliceableJSONWriter& aWriter, const Span<const char> aKey, + const ProfilerString16View& aPayload) { + aWriter.StringProperty(aKey, NS_ConvertUTF16toUTF8(aPayload)); +} + +template <> +inline void StreamPayload<ProfilerString8View>( + baseprofiler::SpliceableJSONWriter& aWriter, const Span<const char> aKey, + const ProfilerString8View& aPayload) { + aWriter.StringProperty(aKey, aPayload); +} +} // namespace detail + +// This helper class is used by MarkerTypes that want to support the general +// MarkerType object schema. When using this the markers will also transmit +// their payload to the ETW tracer as well as requiring less inline code. +// This is a curiously recurring template, the template argument is the child +// class itself. +template <typename T> +struct BaseMarkerType { + static constexpr const char* AllLabels = nullptr; + static constexpr const char* ChartLabel = nullptr; + static constexpr const char* TableLabel = nullptr; + static constexpr const char* TooltipLabel = nullptr; + + static constexpr MarkerSchema::ETWMarkerGroup Group = + MarkerSchema::ETWMarkerGroup::Generic; + + static MarkerSchema MarkerTypeDisplay() { + using MS = MarkerSchema; + MS schema{T::Locations, std::size(T::Locations)}; + if (T::AllLabels) { + schema.SetAllLabels(T::AllLabels); + } + if (T::ChartLabel) { + schema.SetChartLabel(T::ChartLabel); + } + if (T::TableLabel) { + schema.SetTableLabel(T::TableLabel); + } + if (T::TooltipLabel) { + schema.SetTooltipLabel(T::TooltipLabel); + } + for (const MS::PayloadField field : T::PayloadFields) { + if (field.Label) { + if (uint32_t(field.Flags) & uint32_t(MS::PayloadFlags::Searchable)) { + schema.AddKeyLabelFormatSearchable(field.Key, field.Label, field.Fmt, + MS::Searchable::Searchable); + } else { + schema.AddKeyLabelFormat(field.Key, field.Label, field.Fmt); + } + } else { + if (uint32_t(field.Flags) & uint32_t(MS::PayloadFlags::Searchable)) { + schema.AddKeyFormatSearchable(field.Key, field.Fmt, + MS::Searchable::Searchable); + } else { + schema.AddKeyFormat(field.Key, field.Fmt); + } + } + } + if (T::Description) { + schema.AddStaticLabelValue("Description", T::Description); + } + return schema; + } + + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan(T::Name); + } + + // This is called by the child class since the child class version of this + // function is used to infer the argument types by the profile buffer and + // allows the child to do any special data conversion it needs to do. + // Optionally the child can opt not to use this at all and write the data + // out itself. + template <typename... PayloadArguments> + static void StreamJSONMarkerDataImpl( + baseprofiler::SpliceableJSONWriter& aWriter, + const PayloadArguments&... aPayloadArguments) { + size_t i = 0; + (detail::StreamPayload(aWriter, MakeStringSpan(T::PayloadFields[i++].Key), + aPayloadArguments), + ...); + } +}; + +} // namespace mozilla +#endif // ProfilerMarkers_h diff --git a/tools/profiler/public/ProfilerMarkersDetail.h b/tools/profiler/public/ProfilerMarkersDetail.h new file mode 100644 index 0000000000..2308a14bb2 --- /dev/null +++ b/tools/profiler/public/ProfilerMarkersDetail.h @@ -0,0 +1,31 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerMarkersDetail_h +#define ProfilerMarkersDetail_h + +#ifndef ProfilerMarkers_h +# error "This header should only be #included by ProfilerMarkers.h" +#endif + +#include "mozilla/ProfilerMarkersPrerequisites.h" + +#ifdef MOZ_GECKO_PROFILER + +// ~~ HERE BE DRAGONS ~~ +// +// Everything below is internal implementation detail, you shouldn't need to +// look at it unless working on the profiler code. + +// Header that specializes the (de)serializers for xpcom types. +# include "mozilla/ProfileBufferEntrySerializationGeckoExtensions.h" + +// Implemented in platform.cpp +mozilla::ProfileChunkedBuffer& profiler_get_core_buffer(); + +#endif // MOZ_GECKO_PROFILER + +#endif // ProfilerMarkersDetail_h diff --git a/tools/profiler/public/ProfilerMarkersPrerequisites.h b/tools/profiler/public/ProfilerMarkersPrerequisites.h new file mode 100644 index 0000000000..0f10f7efe2 --- /dev/null +++ b/tools/profiler/public/ProfilerMarkersPrerequisites.h @@ -0,0 +1,31 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This header contains basic definitions required to create marker types, and +// to add markers to the profiler buffers. +// +// In most cases, #include "mozilla/ProfilerMarkers.h" instead, or +// #include "mozilla/ProfilerMarkerTypes.h" for common marker types. + +#ifndef ProfilerMarkersPrerequisites_h +#define ProfilerMarkersPrerequisites_h + +#include "mozilla/BaseProfilerMarkersPrerequisites.h" +#include "mozilla/ProfilerThreadState.h" + +#ifdef MOZ_GECKO_PROFILER + +namespace geckoprofiler::markers { + +// Default marker payload types, with no extra information, not even a marker +// type and payload. This is intended for label-only markers. +using NoPayload = ::mozilla::baseprofiler::markers::NoPayload; + +} // namespace geckoprofiler::markers + +#endif // MOZ_GECKO_PROFILER + +#endif // ProfilerMarkersPrerequisites_h diff --git a/tools/profiler/public/ProfilerParent.h b/tools/profiler/public/ProfilerParent.h new file mode 100644 index 0000000000..8bd5c71721 --- /dev/null +++ b/tools/profiler/public/ProfilerParent.h @@ -0,0 +1,119 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerParent_h +#define ProfilerParent_h + +#include "mozilla/PProfilerParent.h" +#include "mozilla/RefPtr.h" + +class nsIProfilerStartParams; + +namespace mozilla { + +class ProfileBufferGlobalController; +class ProfilerParentTracker; + +// This is the main process side of the PProfiler protocol. +// ProfilerParent instances only exist on the main thread of the main process. +// The other side (ProfilerChild) lives on a background thread in the other +// process. +// The creation of PProfiler actors is initiated from the main process, after +// the other process has been launched. +// ProfilerParent instances are destroyed once the message channel closes, +// which can be triggered by either process, depending on which one shuts down +// first. +// All ProfilerParent instances are registered with a manager class called +// ProfilerParentTracker, which has the list of living ProfilerParent instances +// and handles shutdown. +class ProfilerParent final : public PProfilerParent { + public: + NS_INLINE_DECL_REFCOUNTING(ProfilerParent, final) + + static mozilla::ipc::Endpoint<PProfilerChild> CreateForProcess( + base::ProcessId aOtherPid); + +#ifdef MOZ_GECKO_PROFILER + using SingleProcessProfilePromise = + MozPromise<IPCProfileAndAdditionalInformation, ResponseRejectReason, + true>; + + struct SingleProcessProfilePromiseAndChildPid { + RefPtr<SingleProcessProfilePromise> profilePromise; + base::ProcessId childPid; + }; + + using SingleProcessProgressPromise = + MozPromise<GatherProfileProgress, ResponseRejectReason, true>; + + // The following static methods can be called on any thread, but they are + // no-ops on anything other than the main thread. + // If called on the main thread, the call will be broadcast to all + // registered processes (all processes for which we have a ProfilerParent + // object). + // At the moment, the main process always calls these methods on the main + // thread, and that's the only process in which we need to forward these + // calls to other processes. The other processes will call these methods on + // the ProfilerChild background thread, but those processes don't need to + // forward these calls any further. + + // Returns the profiles to expect, as promises and child pids. + static nsTArray<SingleProcessProfilePromiseAndChildPid> GatherProfiles(); + + // Send a request to get the GatherProfiles() progress update from one child + // process, returns a promise to be resolved with that progress. + // The promise RefPtr may be null if the child process is unknown. + // Progress may be invalid, if the request arrived after the child process + // had already responded to the main GatherProfile() IPC, or something went + // very wrong in that process. + static RefPtr<SingleProcessProgressPromise> RequestGatherProfileProgress( + base::ProcessId aChildPid); + + // This will start the profiler in all child processes. The returned promise + // will be resolved when all child have completed their operation + // (successfully or not.) + [[nodiscard]] static RefPtr<GenericPromise> ProfilerStarted( + nsIProfilerStartParams* aParams); + static void ProfilerWillStopIfStarted(); + [[nodiscard]] static RefPtr<GenericPromise> ProfilerStopped(); + [[nodiscard]] static RefPtr<GenericPromise> ProfilerPaused(); + [[nodiscard]] static RefPtr<GenericPromise> ProfilerResumed(); + [[nodiscard]] static RefPtr<GenericPromise> ProfilerPausedSampling(); + [[nodiscard]] static RefPtr<GenericPromise> ProfilerResumedSampling(); + static void ClearAllPages(); + + [[nodiscard]] static RefPtr<GenericPromise> WaitOnePeriodicSampling(); + + // Create a "Final" update that the Child can return to its Parent. + static ProfileBufferChunkManagerUpdate MakeFinalUpdate(); + + // True if the ProfilerParent holds a lock on this thread. + static bool IsLockedOnCurrentThread(); + + private: + friend class ProfileBufferGlobalController; + friend class ProfilerParentTracker; + + explicit ProfilerParent(base::ProcessId aChildPid); + + void Init(); + void ActorDestroy(ActorDestroyReason aActorDestroyReason) override; + + void RequestChunkManagerUpdate(); + + base::ProcessId mChildPid; + nsTArray<MozPromiseHolder<SingleProcessProfilePromise>> + mPendingRequestedProfiles; + bool mDestroyed; +#endif // MOZ_GECKO_PROFILER + + private: + virtual ~ProfilerParent(); +}; + +} // namespace mozilla + +#endif // ProfilerParent_h diff --git a/tools/profiler/public/ProfilerRunnable.h b/tools/profiler/public/ProfilerRunnable.h new file mode 100644 index 0000000000..b3b4e64043 --- /dev/null +++ b/tools/profiler/public/ProfilerRunnable.h @@ -0,0 +1,68 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerRunnable_h +#define ProfilerRunnable_h + +#include "GeckoProfiler.h" +#include "nsIThreadPool.h" + +#if !defined(MOZ_GECKO_PROFILER) || !defined(MOZ_COLLECTING_RUNNABLE_TELEMETRY) +# define AUTO_PROFILE_FOLLOWING_RUNNABLE(runnable) +#else +# define AUTO_PROFILE_FOLLOWING_RUNNABLE(runnable) \ + mozilla::Maybe<mozilla::AutoProfileRunnable> raiiRunnableMarker; \ + if (profiler_thread_is_being_profiled_for_markers()) { \ + raiiRunnableMarker.emplace(runnable); \ + } + +namespace mozilla { + +class MOZ_RAII AutoProfileRunnable { + public: + explicit AutoProfileRunnable(Runnable* aRunnable) + : mStartTime(TimeStamp::Now()) { + aRunnable->GetName(mName); + } + explicit AutoProfileRunnable(nsIRunnable* aRunnable) + : mStartTime(TimeStamp::Now()) { + nsCOMPtr<nsIThreadPool> threadPool = do_QueryInterface(aRunnable); + if (threadPool) { + // nsThreadPool::Run has its own call to AUTO_PROFILE_FOLLOWING_RUNNABLE, + // avoid nesting runnable markers. + return; + } + + nsCOMPtr<nsINamed> named = do_QueryInterface(aRunnable); + if (named) { + named->GetName(mName); + } + } + explicit AutoProfileRunnable(nsACString& aName) + : mStartTime(TimeStamp::Now()), mName(aName) {} + + ~AutoProfileRunnable() { + if (mName.IsEmpty()) { + return; + } + + AUTO_PROFILER_LABEL("AutoProfileRunnable", PROFILER); + AUTO_PROFILER_STATS(AUTO_PROFILE_RUNNABLE); + profiler_add_marker("Runnable", ::mozilla::baseprofiler::category::OTHER, + MarkerTiming::IntervalUntilNowFrom(mStartTime), + geckoprofiler::markers::TextMarker{}, mName); + } + + protected: + TimeStamp mStartTime; + nsAutoCString mName; +}; + +} // namespace mozilla + +#endif + +#endif // ProfilerRunnable_h diff --git a/tools/profiler/public/ProfilerRustBindings.h b/tools/profiler/public/ProfilerRustBindings.h new file mode 100644 index 0000000000..bf290838a1 --- /dev/null +++ b/tools/profiler/public/ProfilerRustBindings.h @@ -0,0 +1,12 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +#ifndef ProfilerRustBindings_h +#define ProfilerRustBindings_h + +#include "mozilla/profiler_ffi_generated.h" + +// Add any non-generated support code here + +#endif // ProfilerRustBindings_h diff --git a/tools/profiler/public/ProfilerState.h b/tools/profiler/public/ProfilerState.h new file mode 100644 index 0000000000..40e1517c91 --- /dev/null +++ b/tools/profiler/public/ProfilerState.h @@ -0,0 +1,436 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This header contains most functions that give information about the Profiler: +// Whether it is active or not, paused, and the selected features. +// It is safe to include unconditionally, but uses of structs and functions must +// be guarded by `#ifdef MOZ_GECKO_PROFILER`. + +#ifndef ProfilerState_h +#define ProfilerState_h + +#include <mozilla/DefineEnum.h> +#include <mozilla/EnumSet.h> +#include "mozilla/ProfilerUtils.h" + +#include <functional> + +//--------------------------------------------------------------------------- +// Profiler features +//--------------------------------------------------------------------------- + +#if defined(__APPLE__) && defined(__aarch64__) +# define POWER_HELP "Sample per process power use" +#elif defined(__APPLE__) && defined(__x86_64__) +# define POWER_HELP \ + "Record the power used by the entire system with each sample." +#elif defined(__linux__) && defined(__x86_64__) +# define POWER_HELP \ + "Record the power used by the entire system with each sample. " \ + "Only available with Intel CPUs and requires setting " \ + "the sysctl kernel.perf_event_paranoid to 0." + +#elif defined(_MSC_VER) +# define POWER_HELP \ + "Record the value of every energy meter available on the system with " \ + "each sample. Only available on Windows 11 with Intel CPUs." +#else +# define POWER_HELP "Not supported on this platform." +#endif + +// Higher-order macro containing all the feature info in one place. Define +// |MACRO| appropriately to extract the relevant parts. Note that the number +// values are used internally only and so can be changed without consequence. +// Any changes to this list should also be applied to the feature list in +// toolkit/components/extensions/schemas/geckoProfiler.json. +// *** Synchronize with lists in BaseProfilerState.h and geckoProfiler.json *** +#define PROFILER_FOR_EACH_FEATURE(MACRO) \ + MACRO(0, "java", Java, "Profile Java code, Android only") \ + \ + MACRO(1, "js", JS, \ + "Get the JS engine to expose the JS stack to the profiler") \ + \ + MACRO(2, "mainthreadio", MainThreadIO, "Add main thread file I/O") \ + \ + MACRO(3, "fileio", FileIO, \ + "Add file I/O from all profiled threads, implies mainthreadio") \ + \ + MACRO(4, "fileioall", FileIOAll, \ + "Add file I/O from all threads, implies fileio") \ + \ + MACRO(5, "nomarkerstacks", NoMarkerStacks, \ + "Markers do not capture stacks, to reduce overhead") \ + \ + MACRO(6, "screenshots", Screenshots, \ + "Take a snapshot of the window on every composition") \ + \ + MACRO(7, "seqstyle", SequentialStyle, \ + "Disable parallel traversal in styling") \ + \ + MACRO(8, "stackwalk", StackWalk, \ + "Walk the C++ stack, not available on all platforms") \ + \ + MACRO(9, "jsallocations", JSAllocations, \ + "Have the JavaScript engine track allocations") \ + \ + MACRO(10, "nostacksampling", NoStackSampling, \ + "Disable all stack sampling: Cancels \"js\", \"stackwalk\" and " \ + "labels") \ + \ + MACRO(11, "nativeallocations", NativeAllocations, \ + "Collect the stacks from a smaller subset of all native " \ + "allocations, biasing towards collecting larger allocations") \ + \ + MACRO(12, "ipcmessages", IPCMessages, \ + "Have the IPC layer track cross-process messages") \ + \ + MACRO(13, "audiocallbacktracing", AudioCallbackTracing, \ + "Audio callback tracing") \ + \ + MACRO(14, "cpu", CPUUtilization, "CPU utilization") \ + \ + MACRO(15, "notimerresolutionchange", NoTimerResolutionChange, \ + "Do not adjust the timer resolution for sampling, so that other " \ + "Firefox timers do not get affected") \ + \ + MACRO(16, "cpuallthreads", CPUAllThreads, \ + "Sample the CPU utilization of all registered threads") \ + \ + MACRO(17, "samplingallthreads", SamplingAllThreads, \ + "Sample the stacks of all registered threads") \ + \ + MACRO(18, "markersallthreads", MarkersAllThreads, \ + "Record markers from all registered threads") \ + \ + MACRO(19, "unregisteredthreads", UnregisteredThreads, \ + "Discover and profile unregistered threads -- beware: expensive!") \ + \ + MACRO(20, "processcpu", ProcessCPU, \ + "Sample the CPU utilization of each process") \ + \ + MACRO(21, "power", Power, POWER_HELP) \ + \ + MACRO(22, "cpufreq", CPUFrequency, \ + "Record the clock frequency of " \ + "every CPU core for every profiler sample.") \ + \ + MACRO(23, "bandwidth", Bandwidth, \ + "Record the network bandwidth used for every profiler sample.") +// *** Synchronize with lists in BaseProfilerState.h and geckoProfiler.json *** + +struct ProfilerFeature { +#define DECLARE(n_, str_, Name_, desc_) \ + static constexpr uint32_t Name_ = (1u << n_); \ + [[nodiscard]] static constexpr bool Has##Name_(uint32_t aFeatures) { \ + return aFeatures & Name_; \ + } \ + static constexpr void Set##Name_(uint32_t& aFeatures) { \ + aFeatures |= Name_; \ + } \ + static constexpr void Clear##Name_(uint32_t& aFeatures) { \ + aFeatures &= ~Name_; \ + } + + // Define a bitfield constant, a getter, and two setters for each feature. + PROFILER_FOR_EACH_FEATURE(DECLARE) + +#undef DECLARE +}; + +// clang-format off +MOZ_DEFINE_ENUM_CLASS(ProfilingState,( + // A callback will be invoked ... + AlreadyActive, // if the profiler is active when the callback is added. + RemovingCallback, // when the callback is removed. + Started, // after the profiler has started. + Pausing, // before the profiler is paused. + Resumed, // after the profiler has resumed. + GeneratingProfile, // before a profile is created. + Stopping, // before the profiler stops (unless restarting afterward). + ShuttingDown // before the profiler is shut down. +)); +// clang-format on + +[[nodiscard]] inline static const char* ProfilingStateToString( + ProfilingState aProfilingState) { + switch (aProfilingState) { + case ProfilingState::AlreadyActive: + return "Profiler already active"; + case ProfilingState::RemovingCallback: + return "Callback being removed"; + case ProfilingState::Started: + return "Profiler started"; + case ProfilingState::Pausing: + return "Profiler pausing"; + case ProfilingState::Resumed: + return "Profiler resumed"; + case ProfilingState::GeneratingProfile: + return "Generating profile"; + case ProfilingState::Stopping: + return "Profiler stopping"; + case ProfilingState::ShuttingDown: + return "Profiler shutting down"; + default: + MOZ_ASSERT_UNREACHABLE("Unexpected ProfilingState enum value"); + return "?"; + } +} + +using ProfilingStateSet = mozilla::EnumSet<ProfilingState>; + +[[nodiscard]] constexpr ProfilingStateSet AllProfilingStates() { + ProfilingStateSet set; + using Value = std::underlying_type_t<ProfilingState>; + for (Value stateValue = 0; + stateValue <= static_cast<Value>(kHighestProfilingState); ++stateValue) { + set += static_cast<ProfilingState>(stateValue); + } + return set; +} + +// Type of callbacks to be invoked at certain state changes. +// It must NOT call profiler_add/remove_state_change_callback(). +using ProfilingStateChangeCallback = std::function<void(ProfilingState)>; + +#ifndef MOZ_GECKO_PROFILER + +[[nodiscard]] inline bool profiler_is_active() { return false; } +[[nodiscard]] inline bool profiler_is_active_and_unpaused() { return false; } +[[nodiscard]] inline bool profiler_is_collecting_markers() { return false; } +[[nodiscard]] inline bool profiler_is_etw_collecting_markers() { return false; } +[[nodiscard]] inline bool profiler_feature_active(uint32_t aFeature) { + return false; +} +[[nodiscard]] inline bool profiler_is_locked_on_current_thread() { + return false; +} +inline void profiler_add_state_change_callback( + ProfilingStateSet aProfilingStateSet, + ProfilingStateChangeCallback&& aCallback, uintptr_t aUniqueIdentifier = 0) { +} +inline void profiler_remove_state_change_callback(uintptr_t aUniqueIdentifier) { +} + +#else // !MOZ_GECKO_PROFILER + +# include "mozilla/Atomics.h" +# include "mozilla/Maybe.h" + +# include <stdint.h> + +namespace mozilla::profiler::detail { + +// RacyFeatures is only defined in this header file so that its methods can +// be inlined into profiler_is_active(). Please do not use anything from the +// detail namespace outside the profiler. + +// Within the profiler's code, the preferred way to check profiler activeness +// and features is via ActivePS(). However, that requires locking gPSMutex. +// There are some hot operations where absolute precision isn't required, so we +// duplicate the activeness/feature state in a lock-free manner in this class. +class RacyFeatures { + public: + static void SetActive(uint32_t aFeatures) { + sActiveAndFeatures = Active | aFeatures; + } + + static void SetETWCollectionActive() { + sActiveAndFeatures |= ETWCollectionEnabled; + } + + static void SetETWCollectionInactive() { + sActiveAndFeatures &= ~ETWCollectionEnabled; + } + + static void SetInactive() { sActiveAndFeatures = 0; } + + static void SetPaused() { sActiveAndFeatures |= Paused; } + + static void SetUnpaused() { sActiveAndFeatures &= ~Paused; } + + static void SetSamplingPaused() { sActiveAndFeatures |= SamplingPaused; } + + static void SetSamplingUnpaused() { sActiveAndFeatures &= ~SamplingPaused; } + + [[nodiscard]] static Maybe<uint32_t> FeaturesIfActive() { + if (uint32_t af = sActiveAndFeatures; af & Active) { + // Active, remove the Active&Paused bits to get all features. + return Some(af & ~(Active | Paused | SamplingPaused)); + } + return Nothing(); + } + + [[nodiscard]] static Maybe<uint32_t> FeaturesIfActiveAndUnpaused() { + if (uint32_t af = sActiveAndFeatures; (af & (Active | Paused)) == Active) { + // Active but not fully paused, remove the Active and sampling-paused bits + // to get all features. + return Some(af & ~(Active | SamplingPaused)); + } + return Nothing(); + } + + // This implementation must be kept in sync with `gecko_profiler::is_active` + // in the Profiler Rust API. + [[nodiscard]] static bool IsActive() { + return uint32_t(sActiveAndFeatures) & Active; + } + + [[nodiscard]] static bool IsActiveWithFeature(uint32_t aFeature) { + uint32_t af = sActiveAndFeatures; // copy it first + return (af & Active) && (af & aFeature); + } + + [[nodiscard]] static bool IsActiveWithoutFeature(uint32_t aFeature) { + uint32_t af = sActiveAndFeatures; // copy it first + return (af & Active) && !(af & aFeature); + } + + // True if profiler is active, and not fully paused. + // Note that periodic sampling *could* be paused! + // This implementation must be kept in sync with + // `gecko_profiler::can_accept_markers` in the Profiler Rust API. + [[nodiscard]] static bool IsActiveAndUnpaused() { + uint32_t af = sActiveAndFeatures; // copy it first + return (af & Active) && !(af & Paused); + } + + // True if profiler is active, and sampling is not paused (though generic + // `SetPaused()` or specific `SetSamplingPaused()`). + [[nodiscard]] static bool IsActiveAndSamplingUnpaused() { + uint32_t af = sActiveAndFeatures; // copy it first + return (af & Active) && !(af & (Paused | SamplingPaused)); + } + + [[nodiscard]] static bool IsCollectingMarkers() { + uint32_t af = sActiveAndFeatures; // copy it first + return ((af & Active) && !(af & Paused)) || (af & ETWCollectionEnabled); + } + + [[nodiscard]] static bool IsETWCollecting() { + uint32_t af = sActiveAndFeatures; // copy it first + return (af & ETWCollectionEnabled); + } + + private: + static constexpr uint32_t Active = 1u << 31; + static constexpr uint32_t Paused = 1u << 30; + static constexpr uint32_t SamplingPaused = 1u << 29; + static constexpr uint32_t ETWCollectionEnabled = 1u << 28; + +// Ensure Active/Paused don't overlap with any of the feature bits. +# define NO_OVERLAP(n_, str_, Name_, desc_) \ + static_assert(ProfilerFeature::Name_ != SamplingPaused, \ + "bad feature value"); + + PROFILER_FOR_EACH_FEATURE(NO_OVERLAP); + +# undef NO_OVERLAP + + // We combine the active bit with the feature bits so they can be read or + // written in a single atomic operation. + static Atomic<uint32_t, MemoryOrdering::Relaxed> sActiveAndFeatures; +}; + +} // namespace mozilla::profiler::detail + +//--------------------------------------------------------------------------- +// Get information from the profiler +//--------------------------------------------------------------------------- + +// Is the profiler active? Note: the return value of this function can become +// immediately out-of-date. E.g. the profile might be active but then +// profiler_stop() is called immediately afterward. One common and reasonable +// pattern of usage is the following: +// +// if (profiler_is_active()) { +// ExpensiveData expensiveData = CreateExpensiveData(); +// PROFILER_OPERATION(expensiveData); +// } +// +// where PROFILER_OPERATION is a no-op if the profiler is inactive. In this +// case the profiler_is_active() check is just an optimization -- it prevents +// us calling CreateExpensiveData() unnecessarily in most cases, but the +// expensive data will end up being created but not used if another thread +// stops the profiler between the CreateExpensiveData() and PROFILER_OPERATION +// calls. +[[nodiscard]] inline bool profiler_is_active() { + return mozilla::profiler::detail::RacyFeatures::IsActive(); +} + +// Same as profiler_is_active(), but also checks if the profiler is not paused. +[[nodiscard]] inline bool profiler_is_active_and_unpaused() { + return mozilla::profiler::detail::RacyFeatures::IsActiveAndUnpaused(); +} + +// Same as profiler_is_active_and_unpaused(), but also checks if the ETW is +// collecting markers. +[[nodiscard]] inline bool profiler_is_collecting_markers() { + return mozilla::profiler::detail::RacyFeatures::IsCollectingMarkers(); +} + +// Reports if our ETW tracelogger is running. +[[nodiscard]] inline bool profiler_is_etw_collecting_markers() { + return mozilla::profiler::detail::RacyFeatures::IsETWCollecting(); +} + +// Is the profiler active and paused? Returns false if the profiler is inactive. +[[nodiscard]] bool profiler_is_paused(); + +// Is the profiler active and sampling is paused? Returns false if the profiler +// is inactive. +[[nodiscard]] bool profiler_is_sampling_paused(); + +// Get all the features supported by the profiler that are accepted by +// profiler_start(). The result is the same whether the profiler is active or +// not. +[[nodiscard]] uint32_t profiler_get_available_features(); + +// Returns the full feature set if the profiler is active. +// Note: the return value can become immediately out-of-date, much like the +// return value of profiler_is_active(). +[[nodiscard]] inline mozilla::Maybe<uint32_t> profiler_features_if_active() { + return mozilla::profiler::detail::RacyFeatures::FeaturesIfActive(); +} + +// Returns the full feature set if the profiler is active and unpaused. +// Note: the return value can become immediately out-of-date, much like the +// return value of profiler_is_active(). +[[nodiscard]] inline mozilla::Maybe<uint32_t> +profiler_features_if_active_and_unpaused() { + return mozilla::profiler::detail::RacyFeatures::FeaturesIfActiveAndUnpaused(); +} + +// Check if a profiler feature (specified via the ProfilerFeature type) is +// active. Returns false if the profiler is inactive. Note: the return value +// can become immediately out-of-date, much like the return value of +// profiler_is_active(). +[[nodiscard]] bool profiler_feature_active(uint32_t aFeature); + +// Check if the profiler is active without a feature (specified via the +// ProfilerFeature type). Note: the return value can become immediately +// out-of-date, much like the return value of profiler_is_active(). +[[nodiscard]] bool profiler_active_without_feature(uint32_t aFeature); + +// Returns true if any of the profiler mutexes are currently locked *on the +// current thread*. This may be used by re-entrant code that may call profiler +// functions while the same of a different profiler mutex is locked, which could +// deadlock. +[[nodiscard]] bool profiler_is_locked_on_current_thread(); + +// Install a callback to be invoked at any of the given profiling state changes. +// An optional non-zero identifier may be given, to allow later removal of the +// callback, the caller is responsible for making sure it's really unique (e.g., +// by using a pointer to an object it owns.) +void profiler_add_state_change_callback( + ProfilingStateSet aProfilingStateSet, + ProfilingStateChangeCallback&& aCallback, uintptr_t aUniqueIdentifier = 0); + +// Remove the callback with the given non-zero identifier. +void profiler_remove_state_change_callback(uintptr_t aUniqueIdentifier); + +#endif // MOZ_GECKO_PROFILER + +#endif // ProfilerState_h diff --git a/tools/profiler/public/ProfilerThreadPlatformData.h b/tools/profiler/public/ProfilerThreadPlatformData.h new file mode 100644 index 0000000000..c243a8ee02 --- /dev/null +++ b/tools/profiler/public/ProfilerThreadPlatformData.h @@ -0,0 +1,80 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerThreadPlatformData_h +#define ProfilerThreadPlatformData_h + +#include "mozilla/ProfilerUtils.h" + +#if defined(__APPLE__) +# include <mach/mach_types.h> +#elif defined(__linux__) || defined(__ANDROID__) || defined(__FreeBSD__) +# include "mozilla/Maybe.h" +# include <time.h> +#endif + +namespace mozilla::profiler { + +class PlatformData { +#if (defined(_MSC_VER) || defined(__MINGW32__)) && defined(MOZ_GECKO_PROFILER) + public: + explicit PlatformData(ProfilerThreadId aThreadId); + ~PlatformData(); + + // Faking win32's HANDLE, because #including "windows.h" here causes trouble + // (e.g., it #defines `Yield` as nothing!) + // This type is static_check'ed against HANDLE in platform-win32.cpp. + using WindowsHandle = void*; + WindowsHandle ProfiledThread() const { return mProfiledThread; } + + private: + WindowsHandle mProfiledThread; +#elif defined(__APPLE__) && defined(MOZ_GECKO_PROFILER) + public: + explicit PlatformData(ProfilerThreadId aThreadId); + ~PlatformData(); + thread_act_t ProfiledThread() const { return mProfiledThread; } + + private: + // Note: for mProfiledThread Mach primitives are used instead of pthread's + // because the latter doesn't provide thread manipulation primitives + // required. For details, consult "Mac OS X Internals" book, Section 7.3. + thread_act_t mProfiledThread; +#elif (defined(__linux__) || defined(__ANDROID__) || defined(__FreeBSD__)) && \ + defined(MOZ_GECKO_PROFILER) + public: + explicit PlatformData(ProfilerThreadId aThreadId); + ~PlatformData(); + // Clock Id for this profiled thread. `Nothing` if `pthread_getcpuclockid` + // failed (e.g., if the system doesn't support per-thread clocks). + Maybe<clockid_t> GetClockId() const { return mClockId; } + + private: + Maybe<clockid_t> mClockId; +#else + public: + explicit PlatformData(ProfilerThreadId aThreadId) {} +#endif +}; + +/** + * Return the number of nanoseconds of CPU time used since thread start. + * + * @return true on success. + */ +#if defined(MOZ_GECKO_PROFILER) +bool GetCpuTimeSinceThreadStartInNs(uint64_t* aResult, + const PlatformData& aPlatformData); +#else +static inline bool GetCpuTimeSinceThreadStartInNs( + uint64_t* aResult, const PlatformData& aPlatformData) { + return false; +} +#endif + +} // namespace mozilla::profiler + +#endif // ProfilerThreadPlatformData_h diff --git a/tools/profiler/public/ProfilerThreadRegistration.h b/tools/profiler/public/ProfilerThreadRegistration.h new file mode 100644 index 0000000000..6d1c755bba --- /dev/null +++ b/tools/profiler/public/ProfilerThreadRegistration.h @@ -0,0 +1,369 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerThreadRegistration_h +#define ProfilerThreadRegistration_h + +#include "mozilla/BaseProfilerDetail.h" +#include "mozilla/ProfilerThreadRegistrationData.h" +#include "mozilla/ThreadLocal.h" + +namespace mozilla::profiler { + +class ThreadRegistry; + +// To use as RAII object, or through RegisterThread/UnregisterThread. +// Automatically registers itself with TLS and Profiler. +// It can be safely nested, but nested instances are just ignored. +// See Get.../With... functions for how to access the data. +class ThreadRegistration { + private: + using DataMutex = baseprofiler::detail::BaseProfilerMutex; + using DataLock = baseprofiler::detail::BaseProfilerAutoLock; + + public: + // Constructor to use as RAII auto-registration object. + // It stores itself in the TLS (its effective owner), and gives its pointer to + // the Profiler. + ThreadRegistration(const char* aName, const void* aStackTop); + + // Destruction reverses construction: Remove pointer from the Profiler (except + // for the main thread, because it should be done by the profiler itself) and + // from the TLS. + ~ThreadRegistration(); + + // Manual construction&destruction, if RAII is not possible or too expensive + // in stack space. + // RegisterThread() *must* be paired with exactly one UnregisterThread() on + // the same thread. (Extra UnregisterThread() calls are handled safely, but + // they may cause profiling of this thread to stop earlier than expected.) + static ProfilingStack* RegisterThread(const char* aName, + const void* aStackTop); + static void UnregisterThread(); + + [[nodiscard]] static bool IsRegistered() { return GetFromTLS(); } + + // Prevent copies&moves. + ThreadRegistration(const ThreadRegistration&) = delete; + ThreadRegistration& operator=(const ThreadRegistration&) = delete; + + // Aliases to data accessors (removing the ThreadRegistration prefix). + + using UnlockedConstReader = ThreadRegistrationUnlockedConstReader; + using UnlockedConstReaderAndAtomicRW = + ThreadRegistrationUnlockedConstReaderAndAtomicRW; + using UnlockedRWForLockedProfiler = + ThreadRegistrationUnlockedRWForLockedProfiler; + using UnlockedReaderAndAtomicRWOnThread = + ThreadRegistrationUnlockedReaderAndAtomicRWOnThread; + using LockedRWFromAnyThread = ThreadRegistrationLockedRWFromAnyThread; + using LockedRWOnThread = ThreadRegistrationLockedRWOnThread; + + // On-thread access from the TLS, providing the following data accessors: + // UnlockedConstReader, UnlockedConstReaderAndAtomicRW, + // UnlockedRWForLockedProfiler, UnlockedReaderAndAtomicRWOnThread, and + // LockedRWOnThread. + // (See ThreadRegistry class for OFF-thread access.) + + // Reference-like class pointing at the ThreadRegistration for the current + // thread. + class OnThreadRef { + public: + // const UnlockedConstReader + + [[nodiscard]] const UnlockedConstReader& UnlockedConstReaderCRef() const { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedConstReader(F&& aF) const { + return std::forward<F>(aF)(UnlockedConstReaderCRef()); + } + + // const UnlockedConstReaderAndAtomicRW + + [[nodiscard]] const UnlockedConstReaderAndAtomicRW& + UnlockedConstReaderAndAtomicRWCRef() const { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedConstReaderAndAtomicRW(F&& aF) const { + return std::forward<F>(aF)(UnlockedConstReaderAndAtomicRWCRef()); + } + + // UnlockedConstReaderAndAtomicRW + + [[nodiscard]] UnlockedConstReaderAndAtomicRW& + UnlockedConstReaderAndAtomicRWRef() { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedConstReaderAndAtomicRW(F&& aF) { + return std::forward<F>(aF)(UnlockedConstReaderAndAtomicRWRef()); + } + + // const UnlockedRWForLockedProfiler + + [[nodiscard]] const UnlockedRWForLockedProfiler& + UnlockedRWForLockedProfilerCRef() const { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedRWForLockedProfiler(F&& aF) const { + return std::forward<F>(aF)(UnlockedRWForLockedProfilerCRef()); + } + + // UnlockedRWForLockedProfiler + + [[nodiscard]] UnlockedRWForLockedProfiler& + UnlockedRWForLockedProfilerRef() { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedRWForLockedProfiler(F&& aF) { + return std::forward<F>(aF)(UnlockedRWForLockedProfilerRef()); + } + + // const UnlockedReaderAndAtomicRWOnThread + + [[nodiscard]] const UnlockedReaderAndAtomicRWOnThread& + UnlockedReaderAndAtomicRWOnThreadCRef() const { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedReaderAndAtomicRWOnThread(F&& aF) const { + return std::forward<F>(aF)(UnlockedReaderAndAtomicRWOnThreadCRef()); + } + + // UnlockedReaderAndAtomicRWOnThread + + [[nodiscard]] UnlockedReaderAndAtomicRWOnThread& + UnlockedReaderAndAtomicRWOnThreadRef() { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedReaderAndAtomicRWOnThread(F&& aF) { + return std::forward<F>(aF)(UnlockedReaderAndAtomicRWOnThreadRef()); + } + + // const LockedRWOnThread through ConstRWOnThreadWithLock + + // Locking order: Profiler, ThreadRegistry, ThreadRegistration. + class ConstRWOnThreadWithLock { + public: + [[nodiscard]] const LockedRWOnThread& DataCRef() const { + return mLockedRWOnThread; + } + [[nodiscard]] const LockedRWOnThread* operator->() const { + return &mLockedRWOnThread; + } + + private: + friend class OnThreadRef; + ConstRWOnThreadWithLock(const LockedRWOnThread& aLockedRWOnThread, + DataMutex& aDataMutex) + : mLockedRWOnThread(aLockedRWOnThread), mDataLock(aDataMutex) {} + + const LockedRWOnThread& mLockedRWOnThread; + DataLock mDataLock; + }; + + [[nodiscard]] ConstRWOnThreadWithLock ConstLockedRWOnThread() const { + return ConstRWOnThreadWithLock{mThreadRegistration->mData, + mThreadRegistration->mDataMutex}; + } + + template <typename F> + auto WithConstLockedRWOnThread(F&& aF) const { + ConstRWOnThreadWithLock lockedData = ConstLockedRWOnThread(); + return std::forward<F>(aF)(lockedData.DataCRef()); + } + + // LockedRWOnThread through RWOnThreadWithLock + + // Locking order: Profiler, ThreadRegistry, ThreadRegistration. + class RWOnThreadWithLock { + public: + [[nodiscard]] const LockedRWOnThread& DataCRef() const { + return mLockedRWOnThread; + } + [[nodiscard]] LockedRWOnThread& DataRef() { return mLockedRWOnThread; } + [[nodiscard]] const LockedRWOnThread* operator->() const { + return &mLockedRWOnThread; + } + [[nodiscard]] LockedRWOnThread* operator->() { + return &mLockedRWOnThread; + } + + private: + friend class OnThreadRef; + RWOnThreadWithLock(LockedRWOnThread& aLockedRWOnThread, + DataMutex& aDataMutex) + : mLockedRWOnThread(aLockedRWOnThread), mDataLock(aDataMutex) {} + + LockedRWOnThread& mLockedRWOnThread; + DataLock mDataLock; + }; + + [[nodiscard]] RWOnThreadWithLock GetLockedRWOnThread() { + return RWOnThreadWithLock{mThreadRegistration->mData, + mThreadRegistration->mDataMutex}; + } + + template <typename F> + auto WithLockedRWOnThread(F&& aF) { + RWOnThreadWithLock lockedData = GetLockedRWOnThread(); + return std::forward<F>(aF)(lockedData.DataRef()); + } + + // This is needed to allow OnThreadPtr::operator-> to return a temporary + // OnThreadRef object, for which `->` must work; Here it provides a pointer + // to itself, so that the next follow-up `->` will work as member accessor. + OnThreadRef* operator->() && { return this; } + + private: + // Only ThreadRegistration should construct an OnThreadRef. + friend class ThreadRegistration; + explicit OnThreadRef(ThreadRegistration& aThreadRegistration) + : mThreadRegistration(&aThreadRegistration) {} + + // Allow ThreadRegistry to read mThreadRegistration. + friend class ThreadRegistry; + + // Guaranted to be non-null by construction from a reference. + ThreadRegistration* mThreadRegistration; + }; + + // Pointer-like class pointing at the ThreadRegistration for the current + // thread, if one was registered. + class OnThreadPtr { + public: + [[nodiscard]] explicit operator bool() const { return mThreadRegistration; } + + // Note that this resolves to a temporary OnThreadRef object, which has all + // the allowed data accessors. + [[nodiscard]] OnThreadRef operator*() const { + MOZ_ASSERT(mThreadRegistration); + return OnThreadRef(*mThreadRegistration); + } + + // Note that this resolves to a temporary OnThreadRef object, which also + // overloads operator-> and has all the allowed data accessors. + [[nodiscard]] OnThreadRef operator->() const { + MOZ_ASSERT(mThreadRegistration); + return OnThreadRef(*mThreadRegistration); + } + + private: + friend class ThreadRegistration; + explicit OnThreadPtr(ThreadRegistration* aThreadRegistration) + : mThreadRegistration(aThreadRegistration) {} + + ThreadRegistration* mThreadRegistration; + }; + + [[nodiscard]] static OnThreadPtr GetOnThreadPtr() { + return OnThreadPtr{GetFromTLS()}; + } + + // Call `F(OnThreadRef)`. + template <typename F> + static void WithOnThreadRef(F&& aF) { + const auto* tls = GetTLS(); + if (tls) { + ThreadRegistration* tr = tls->get(); + if (tr) { + std::forward<F>(aF)(OnThreadRef{*tr}); + } + } + } + + // Call `F(OnThreadRef)`. + template <typename F, typename FallbackReturn> + [[nodiscard]] static auto WithOnThreadRefOr(F&& aF, + FallbackReturn&& aFallbackReturn) + -> decltype(std::forward<F>(aF)(std::declval<OnThreadRef>())) { + const auto* tls = GetTLS(); + if (tls) { + ThreadRegistration* tr = tls->get(); + if (tr) { + return std::forward<F>(aF)(OnThreadRef{*tr}); + } + } + return std::forward<FallbackReturn>(aFallbackReturn); + } + + [[nodiscard]] static bool IsDataMutexLockedOnCurrentThread() { + if (const ThreadRegistration* tr = GetFromTLS(); tr) { + return tr->mDataMutex.IsLockedOnCurrentThread(); + } + return false; + } + + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { + DataLock lock(mDataMutex); + return mData.SizeOfExcludingThis(aMallocSizeOf); + } + + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { + // aMallocSizeOf can only be used on head-allocated objects. Stack + // allocations and static objects are not counted. + return (mIsOnHeap ? aMallocSizeOf(this) : 0) + + SizeOfExcludingThis(aMallocSizeOf); + } + + private: + friend class ThreadRegistry; + + // This is what is embedded inside ThreadRegistration. + // References to sub-classes will be provided, to limit access as appropriate. + class EmbeddedData final : public LockedRWOnThread { + private: + // Only ThreadRegistration can construct (its embedded) `mData`. + friend class ThreadRegistration; + EmbeddedData(const char* aName, const void* aStackTop) + : LockedRWOnThread(aName, aStackTop) {} + }; + EmbeddedData mData; + + // Used when writing on self thread, and for any access from any thread. + // Locking order: Profiler, ThreadRegistry, ThreadRegistration. + mutable DataMutex mDataMutex; + + // In case of nested (non-RAII) registrations. Only accessed on thread. + int mOtherRegistrations = 0; + + // Set to true if allocated by `RegisterThread()`. Otherwise we assume that it + // is on the stack. + bool mIsOnHeap = false; + + // Only accessed by ThreadRegistry on this thread. + bool mIsRegistryLockedSharedOnThisThread = false; + + static MOZ_THREAD_LOCAL(ThreadRegistration*) tlsThreadRegistration; + + [[nodiscard]] static decltype(tlsThreadRegistration)* GetTLS() { + if (tlsThreadRegistration.init()) + return &tlsThreadRegistration; + else + return nullptr; + } + + [[nodiscard]] static ThreadRegistration* GetFromTLS() { + const auto tls = GetTLS(); + return tls ? tls->get() : nullptr; + } +}; + +} // namespace mozilla::profiler + +#endif // ProfilerThreadRegistration_h diff --git a/tools/profiler/public/ProfilerThreadRegistrationData.h b/tools/profiler/public/ProfilerThreadRegistrationData.h new file mode 100644 index 0000000000..7c14290e4c --- /dev/null +++ b/tools/profiler/public/ProfilerThreadRegistrationData.h @@ -0,0 +1,537 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This header contains classes that hold data related to thread profiling: +// Data members are stored `protected` in `ThreadRegistrationData`. +// Non-virtual sub-classes of ProfilerThreadRegistrationData provide layers of +// public accessors to subsets of the data. Each level builds on the previous +// one and adds further access to more data, but always with the appropriate +// guards where necessary. +// These classes have protected constructors, so only some trusted classes +// `ThreadRegistration` and `ThreadRegistry` will be able to construct them, and +// then give limited access depending on who asks (the owning thread or another +// one), and how much data they actually need. +// +// The hierarchy is, from base to most derived: +// - ThreadRegistrationData +// - ThreadRegistrationUnlockedConstReader +// - ThreadRegistrationUnlockedConstReaderAndAtomicRW +// - ThreadRegistrationUnlockedRWForLockedProfiler +// - ThreadRegistrationUnlockedReaderAndAtomicRWOnThread +// - ThreadRegistrationLockedRWFromAnyThread +// - ThreadRegistrationLockedRWOnThread +// - ThreadRegistration::EmbeddedData (actual data member in ThreadRegistration) +// +// Tech detail: These classes need to be a single hierarchy so that +// `ThreadRegistration` can contain the most-derived class, and from there can +// publish references to base classes without relying on Undefined Behavior. +// (It's not allowed to have some object and give a reference to a sub-class, +// unless that object was *really* constructed as that sub-class at least, even +// if that sub-class only adds member functions!) +// And where appropriate, these references will come along with the required +// lock. + +#ifndef ProfilerThreadRegistrationData_h +#define ProfilerThreadRegistrationData_h + +#include "js/ProfilingFrameIterator.h" +#include "js/ProfilingStack.h" +#include "mozilla/Atomics.h" +#include "mozilla/BaseProfilerDetail.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/ProfilerThreadPlatformData.h" +#include "mozilla/ProfilerThreadRegistrationInfo.h" +#include "nsCOMPtr.h" +#include "nsIThread.h" + +class ProfiledThreadData; +class PSAutoLock; +struct JSContext; + +// Enum listing which profiling features are active for a single thread. +enum class ThreadProfilingFeatures : uint32_t { + // The thread is not being profiled at all (either the profiler is not + // running, or this thread is not examined during profiling.) + NotProfiled = 0u, + + // Single features, binary exclusive. May be `Combine()`d. + CPUUtilization = 1u << 0, + Sampling = 1u << 1, + Markers = 1u << 2, + + // All possible features. Usually used as a mask to see if any feature is + // active at a given time. + Any = CPUUtilization | Sampling | Markers +}; + +// Binary OR of one of more ThreadProfilingFeatures, to mix all arguments. +template <typename... Ts> +[[nodiscard]] constexpr ThreadProfilingFeatures Combine( + ThreadProfilingFeatures a1, Ts... as) { + static_assert((true && ... && + std::is_same_v<std::remove_cv_t<std::remove_reference_t<Ts>>, + ThreadProfilingFeatures>)); + return static_cast<ThreadProfilingFeatures>( + (static_cast<std::underlying_type_t<ThreadProfilingFeatures>>(a1) | ... | + static_cast<std::underlying_type_t<ThreadProfilingFeatures>>(as))); +} + +// Binary AND of one of more ThreadProfilingFeatures, to find features common to +// all arguments. +template <typename... Ts> +[[nodiscard]] constexpr ThreadProfilingFeatures Intersect( + ThreadProfilingFeatures a1, Ts... as) { + static_assert((true && ... && + std::is_same_v<std::remove_cv_t<std::remove_reference_t<Ts>>, + ThreadProfilingFeatures>)); + return static_cast<ThreadProfilingFeatures>( + (static_cast<std::underlying_type_t<ThreadProfilingFeatures>>(a1) & ... & + static_cast<std::underlying_type_t<ThreadProfilingFeatures>>(as))); +} + +// Are there features in common between the two given sets? +// Mostly useful to test if any of a set of features is present in another set. +template <typename... Ts> +[[nodiscard]] constexpr bool DoFeaturesIntersect(ThreadProfilingFeatures a1, + ThreadProfilingFeatures a2) { + return Intersect(a1, a2) != ThreadProfilingFeatures::NotProfiled; +} + +namespace mozilla::profiler { + +// All data members related to thread profiling are stored here. +// See derived classes below, which give limited unlocked/locked read/write +// access in different situations, and will be available through +// ThreadRegistration and ThreadRegistry. +class ThreadRegistrationData { + public: + // No public accessors here. See derived classes for accessors, and + // Get.../With... functions for who can use these accessors. + + size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { + // Not including data that is not fully owned here. + return 0; + } + + size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { + return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); + } + + static constexpr size_t MAX_JS_FRAMES = 1024; + using JsFrame = JS::ProfilingFrameIterator::Frame; + using JsFrameBuffer = JsFrame[MAX_JS_FRAMES]; + + // `protected` to allow derived classes to read all data members. + protected: + ThreadRegistrationData(const char* aName, const void* aStackTop); + +#ifdef DEBUG + // Destructor only used to check invariants. + ~ThreadRegistrationData() { + MOZ_ASSERT((mProfilingFeatures != ThreadProfilingFeatures::NotProfiled) == + !!mProfiledThreadData); + MOZ_ASSERT(!mProfiledThreadData, + "mProfiledThreadData pointer should have been reset before " + "~ThreadRegistrationData"); + } +#endif // DEBUG + + // Permanent thread information. + // Set at construction, read from anywhere, moved-from at destruction. + ThreadRegistrationInfo mInfo; + + // Contains profiler labels and JS frames. + // Deep-written on thread only, deep-read from thread and suspended thread. + ProfilingStack mProfilingStack; + + // In practice, only read from thread and suspended thread. + PlatformData mPlatformData; + + // Only read from thread and suspended thread. + const void* const mStackTop; + + // Written from thread, read from thread and suspended thread. + nsCOMPtr<nsIThread> mThread; + + // If this is a JS thread, this is its JSContext, which is required for any + // JS sampling. + // Written from thread, read from thread and suspended thread. + JSContext* mJSContext = nullptr; + + // If mJSContext is not null AND the thread is being profiled, this points at + // the start of a JsFrameBuffer to be used for on-thread synchronous sampling. + JsFrame* mJsFrameBuffer = nullptr; + + // The profiler needs to start and stop JS sampling of JS threads at various + // times. However, the JS engine can only do the required actions on the + // JS thread itself ("on-thread"), not from another thread ("off-thread"). + // Therefore, we have the following two-step process. + // + // - The profiler requests (on-thread or off-thread) that the JS sampling be + // started/stopped, by changing mJSSampling to the appropriate REQUESTED + // state. + // + // - The relevant JS thread polls (on-thread) for changes to mJSSampling. + // When it sees a REQUESTED state, it performs the appropriate actions to + // actually start/stop JS sampling, and changes mJSSampling out of the + // REQUESTED state. + // + // The state machine is as follows. + // + // INACTIVE --> ACTIVE_REQUESTED + // ^ ^ | + // | _/ | + // | _/ | + // | / | + // | v v + // INACTIVE_REQUESTED <-- ACTIVE + // + // The polling is done in the following two ways. + // + // - Via the interrupt callback mechanism; the JS thread must call + // profiler_js_interrupt_callback() from its own interrupt callback. + // This is how sampling must be started/stopped for threads where the + // request was made off-thread. + // + // - When {Start,Stop}JSSampling() is called on-thread, we can immediately + // follow it with a PollJSSampling() call to avoid the delay between the + // two steps. Likewise, setJSContext() calls PollJSSampling(). + // + // One non-obvious thing about all this: these JS sampling requests are made + // on all threads, even non-JS threads. mContext needs to also be set (via + // setJSContext(), which can only happen for JS threads) for any JS sampling + // to actually happen. + // + enum { + INACTIVE = 0, + ACTIVE_REQUESTED = 1, + ACTIVE = 2, + INACTIVE_REQUESTED = 3, + } mJSSampling = INACTIVE; + + uint32_t mJSFlags = 0; + + // Flags to conveniently track various JS instrumentations. + enum class JSInstrumentationFlags { + StackSampling = 0x1, + Allocations = 0x2, + }; + + [[nodiscard]] bool JSAllocationsEnabled() const { + return mJSFlags & uint32_t(JSInstrumentationFlags::Allocations); + } + + // The following members may be modified from another thread. + // They need to be atomic, because LockData() does not prevent reads from + // the owning thread. + + // mSleep tracks whether the thread is sleeping, and if so, whether it has + // been previously observed. This is used for an optimization: in some + // cases, when a thread is asleep, we duplicate the previous sample, which + // is cheaper than taking a new sample. + // + // mSleep is atomic because it is accessed from multiple threads. + // + // - It is written only by this thread, via setSleeping() and setAwake(). + // + // - It is read by SamplerThread::Run(). + // + // There are two cases where racing between threads can cause an issue. + // + // - If CanDuplicateLastSampleDueToSleep() returns false but that result is + // invalidated before being acted upon, we will take a full sample + // unnecessarily. This is additional work but won't cause any correctness + // issues. (In actual fact, this case is impossible. In order to go from + // CanDuplicateLastSampleDueToSleep() returning false to it returning true + // requires an intermediate call to it in order for mSleep to go from + // SLEEPING_NOT_OBSERVED to SLEEPING_OBSERVED.) + // + // - If CanDuplicateLastSampleDueToSleep() returns true but that result is + // invalidated before being acted upon -- i.e. the thread wakes up before + // DuplicateLastSample() is called -- we will duplicate the previous + // sample. This is inaccurate, but only slightly... we will effectively + // treat the thread as having slept a tiny bit longer than it really did. + // + // This latter inaccuracy could be avoided by moving the + // CanDuplicateLastSampleDueToSleep() check within the thread-freezing code, + // e.g. the section where Tick() is called. But that would reduce the + // effectiveness of the optimization because more code would have to be run + // before we can tell that duplication is allowed. + // + static const int AWAKE = 0; + static const int SLEEPING_NOT_OBSERVED = 1; + static const int SLEEPING_OBSERVED = 2; + // Read&written from thread and suspended thread. + Atomic<int> mSleep{AWAKE}; + Atomic<uint64_t> mThreadCpuTimeInNsAtLastSleep{0}; + +#ifdef NIGHTLY_BUILD + // The first wake is the thread creation. + Atomic<uint64_t, MemoryOrdering::Relaxed> mWakeCount{1}; + mutable baseprofiler::detail::BaseProfilerMutex mRecordWakeCountMutex; + mutable uint64_t mAlreadyRecordedWakeCount = 0; + mutable uint64_t mAlreadyRecordedCpuTimeInMs = 0; +#endif + + // Is this thread currently being profiled, and with which features? + // Written from profiler, read from any thread. + // Invariant: `!!mProfilingFeatures == !!mProfiledThreadData` (set together.) + Atomic<ThreadProfilingFeatures, MemoryOrdering::Relaxed> mProfilingFeatures{ + ThreadProfilingFeatures::NotProfiled}; + + // If the profiler is active and this thread is selected for profiling, this + // points at the relevant ProfiledThreadData. + // Fully controlled by the profiler. + // Invariant: `!!mProfilingFeatures == !!mProfiledThreadData` (set together). + ProfiledThreadData* mProfiledThreadData = nullptr; +}; + +// Accessing const data from any thread. +class ThreadRegistrationUnlockedConstReader : public ThreadRegistrationData { + public: + [[nodiscard]] const ThreadRegistrationInfo& Info() const { return mInfo; } + + [[nodiscard]] const PlatformData& PlatformDataCRef() const { + return mPlatformData; + } + + [[nodiscard]] const void* StackTop() const { return mStackTop; } + + protected: + ThreadRegistrationUnlockedConstReader(const char* aName, + const void* aStackTop) + : ThreadRegistrationData(aName, aStackTop) {} +}; + +// Accessing atomic data from any thread. +class ThreadRegistrationUnlockedConstReaderAndAtomicRW + : public ThreadRegistrationUnlockedConstReader { + public: + [[nodiscard]] const ProfilingStack& ProfilingStackCRef() const { + return mProfilingStack; + } + [[nodiscard]] ProfilingStack& ProfilingStackRef() { return mProfilingStack; } + + // Similar to `profiler_is_active()`, this atomic flag may become out-of-date. + // It should only be used as an indication to know whether this thread is + // probably being profiled (with some specific features), to avoid doing + // expensive operations otherwise. Edge cases: + // - This thread could get `NotProfiled`, but the profiler has just started, + // so some very early data may be missing. No real impact on profiling. + // - This thread could see profiled features, but the profiled has just + // stopped, so some some work will be done and then discarded when finally + // attempting to write to the buffer. No impact on profiling. + // - This thread could see profiled features, but the profiler will quickly + // stop and restart, so this thread will write information relevant to the + // previous profiling session. Very rare, and little impact on profiling. + [[nodiscard]] ThreadProfilingFeatures ProfilingFeatures() const { + return mProfilingFeatures; + } + + // Call this whenever the current thread sleeps. Calling it twice in a row + // without an intervening setAwake() call is an error. + void SetSleeping() { + MOZ_ASSERT(mSleep == AWAKE); + mSleep = SLEEPING_NOT_OBSERVED; + } + + // Call this whenever the current thread wakes. Calling it twice in a row + // without an intervening setSleeping() call is an error. + void SetAwake() { + MOZ_ASSERT(mSleep != AWAKE); + mSleep = AWAKE; +#ifdef NIGHTLY_BUILD + ++mWakeCount; +#endif + } + + // Returns the CPU time used by the thread since the previous call to this + // method or since the thread was started if this is the first call. + uint64_t GetNewCpuTimeInNs() { + uint64_t newCpuTimeNs; + if (!GetCpuTimeSinceThreadStartInNs(&newCpuTimeNs, PlatformDataCRef())) { + newCpuTimeNs = 0; + } + uint64_t before = mThreadCpuTimeInNsAtLastSleep; + uint64_t result = + MOZ_LIKELY(newCpuTimeNs > before) ? newCpuTimeNs - before : 0; + mThreadCpuTimeInNsAtLastSleep = newCpuTimeNs; + return result; + } + +#ifdef NIGHTLY_BUILD + void RecordWakeCount() const; +#endif + + // This is called on every profiler restart. Put things that should happen + // at that time here. + void ReinitializeOnResume() { + // This is needed to cause an initial sample to be taken from sleeping + // threads that had been observed prior to the profiler stopping and + // restarting. Otherwise sleeping threads would not have any samples to + // copy forward while sleeping. + (void)mSleep.compareExchange(SLEEPING_OBSERVED, SLEEPING_NOT_OBSERVED); + } + + // This returns true for the second and subsequent calls in each sleep + // cycle, so that the sampler can skip its full sampling and reuse the first + // asleep sample instead. + [[nodiscard]] bool CanDuplicateLastSampleDueToSleep() { + if (mSleep == AWAKE) { + return false; + } + if (mSleep.compareExchange(SLEEPING_NOT_OBSERVED, SLEEPING_OBSERVED)) { + return false; + } + return true; + } + + [[nodiscard]] bool IsSleeping() const { return mSleep != AWAKE; } + + protected: + ThreadRegistrationUnlockedConstReaderAndAtomicRW(const char* aName, + const void* aStackTop) + : ThreadRegistrationUnlockedConstReader(aName, aStackTop) {} +}; + +// Like above, with special PSAutoLock-guarded accessors. +class ThreadRegistrationUnlockedRWForLockedProfiler + : public ThreadRegistrationUnlockedConstReaderAndAtomicRW { + public: + // IMPORTANT! IMPORTANT! IMPORTANT! IMPORTANT! IMPORTANT! IMPORTANT! + // Only add functions that take a `const PSAutoLock&` proof-of-lock. + // (Because there is no other lock.) + + [[nodiscard]] const ProfiledThreadData* GetProfiledThreadData( + const PSAutoLock&) const { + return mProfiledThreadData; + } + + [[nodiscard]] ProfiledThreadData* GetProfiledThreadData(const PSAutoLock&) { + return mProfiledThreadData; + } + + protected: + ThreadRegistrationUnlockedRWForLockedProfiler(const char* aName, + const void* aStackTop) + : ThreadRegistrationUnlockedConstReaderAndAtomicRW(aName, aStackTop) {} +}; + +// Reading data, unlocked from the thread, or locked otherwise. +// This data MUST only be written from the thread with lock (i.e., in +// LockedRWOnThread through RWOnThreadWithLock.) +class ThreadRegistrationUnlockedReaderAndAtomicRWOnThread + : public ThreadRegistrationUnlockedRWForLockedProfiler { + public: + // IMPORTANT! IMPORTANT! IMPORTANT! IMPORTANT! IMPORTANT! IMPORTANT! + // Non-atomic members read here MUST be written from LockedRWOnThread (to + // guarantee that they are only modified on this thread.) + + [[nodiscard]] JSContext* GetJSContext() const { return mJSContext; } + + protected: + ThreadRegistrationUnlockedReaderAndAtomicRWOnThread(const char* aName, + const void* aStackTop) + : ThreadRegistrationUnlockedRWForLockedProfiler(aName, aStackTop) {} +}; + +// Accessing locked data from the thread, or from any thread through the locked +// profiler: + +// Like above, and profiler can also read&write mutex-protected members. +class ThreadRegistrationLockedRWFromAnyThread + : public ThreadRegistrationUnlockedReaderAndAtomicRWOnThread { + public: + void SetProfilingFeaturesAndData(ThreadProfilingFeatures aProfilingFeatures, + ProfiledThreadData* aProfiledThreadData, + const PSAutoLock&); + void ClearProfilingFeaturesAndData(const PSAutoLock&); + + // Not null when JSContext is not null AND this thread is being profiled. + // Points at the start of JsFrameBuffer. + [[nodiscard]] JsFrame* GetJsFrameBuffer() const { return mJsFrameBuffer; } + + [[nodiscard]] const nsCOMPtr<nsIEventTarget> GetEventTarget() const { + return mThread; + } + + void ResetMainThread(nsIThread* aThread) { mThread = aThread; } + + // aDelay is the time the event that is currently running on the thread was + // queued before starting to run (if a PrioritizedEventQueue + // (i.e. MainThread), this will be 0 for any event at a lower priority + // than Input). + // aRunning is the time the event has been running. If no event is running + // these will both be TimeDuration() (i.e. 0). Both are out params, and are + // always set. Their initial value is discarded. + void GetRunningEventDelay(const TimeStamp& aNow, TimeDuration& aDelay, + TimeDuration& aRunning) { + if (mThread) { // can be null right at the start of a process + TimeStamp start; + mThread->GetRunningEventDelay(&aDelay, &start); + if (!start.IsNull()) { + // Note: the timestamp used here will be from when we started to + // suspend and sample the thread; which is also the timestamp + // associated with the sample. + aRunning = aNow - start; + return; + } + } + aDelay = TimeDuration(); + aRunning = TimeDuration(); + } + + // Request that this thread start JS sampling. JS sampling won't actually + // start until a subsequent PollJSSampling() call occurs *and* mContext has + // been set. + void StartJSSampling(uint32_t aJSFlags) { + // This function runs on-thread or off-thread. + + MOZ_RELEASE_ASSERT(mJSSampling == INACTIVE || + mJSSampling == INACTIVE_REQUESTED); + mJSSampling = ACTIVE_REQUESTED; + mJSFlags = aJSFlags; + } + + // Request that this thread stop JS sampling. JS sampling won't actually + // stop until a subsequent PollJSSampling() call occurs. + void StopJSSampling() { + // This function runs on-thread or off-thread. + + MOZ_RELEASE_ASSERT(mJSSampling == ACTIVE || + mJSSampling == ACTIVE_REQUESTED); + mJSSampling = INACTIVE_REQUESTED; + } + + protected: + ThreadRegistrationLockedRWFromAnyThread(const char* aName, + const void* aStackTop) + : ThreadRegistrationUnlockedReaderAndAtomicRWOnThread(aName, aStackTop) {} +}; + +// Accessing data, locked, from the thread. +// If any non-atomic data is readable from UnlockedReaderAndAtomicRWOnThread, +// it must be written from here, and not in base classes: Since this data is +// only written on the thread, it can be read from the same thread without +// lock; but writing must be locked so that other threads can safely read it, +// typically from LockedRWFromAnyThread. +class ThreadRegistrationLockedRWOnThread + : public ThreadRegistrationLockedRWFromAnyThread { + public: + void SetJSContext(JSContext* aJSContext); + void ClearJSContext(); + + // Poll to see if JS sampling should be started/stopped. + void PollJSSampling(); + + public: + ThreadRegistrationLockedRWOnThread(const char* aName, const void* aStackTop) + : ThreadRegistrationLockedRWFromAnyThread(aName, aStackTop) {} +}; + +} // namespace mozilla::profiler + +#endif // ProfilerThreadRegistrationData_h diff --git a/tools/profiler/public/ProfilerThreadRegistrationInfo.h b/tools/profiler/public/ProfilerThreadRegistrationInfo.h new file mode 100644 index 0000000000..e116c3059e --- /dev/null +++ b/tools/profiler/public/ProfilerThreadRegistrationInfo.h @@ -0,0 +1,64 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerThreadRegistrationInfo_h +#define ProfilerThreadRegistrationInfo_h + +#include "mozilla/BaseAndGeckoProfilerDetail.h" +#include "mozilla/ProfilerUtils.h" +#include "mozilla/TimeStamp.h" + +#include <string> + +namespace mozilla::profiler { + +// This class contains immutable information about a thread which needs to be +// stored across restarts of the profiler and which can be useful even after the +// thread has stopped running. +class ThreadRegistrationInfo { + public: + // Construct on the thread. + explicit ThreadRegistrationInfo(const char* aName) : mName(aName) {} + + // Construct for a foreign thread (e.g., Java). + ThreadRegistrationInfo(const char* aName, ProfilerThreadId aThreadId, + bool aIsMainThread, const TimeStamp& aRegisterTime) + : mName(aName), + mRegisterTime(aRegisterTime), + mThreadId(aThreadId), + mIsMainThread(aIsMainThread) {} + + // Only allow move construction, for extraction when the thread ends. + ThreadRegistrationInfo(ThreadRegistrationInfo&&) = default; + + // Other copies/moves disallowed. + ThreadRegistrationInfo(const ThreadRegistrationInfo&) = delete; + ThreadRegistrationInfo& operator=(const ThreadRegistrationInfo&) = delete; + ThreadRegistrationInfo& operator=(ThreadRegistrationInfo&&) = delete; + + [[nodiscard]] const char* Name() const { return mName.c_str(); } + [[nodiscard]] const TimeStamp& RegisterTime() const { return mRegisterTime; } + [[nodiscard]] ProfilerThreadId ThreadId() const { return mThreadId; } + [[nodiscard]] bool IsMainThread() const { return mIsMainThread; } + + private: + static TimeStamp ExistingRegisterTimeOrNow() { + TimeStamp registerTime = baseprofiler::detail::GetThreadRegistrationTime(); + if (!registerTime) { + registerTime = TimeStamp::Now(); + } + return registerTime; + } + + const std::string mName; + const TimeStamp mRegisterTime = ExistingRegisterTimeOrNow(); + const ProfilerThreadId mThreadId = profiler_current_thread_id(); + const bool mIsMainThread = profiler_is_main_thread(); +}; + +} // namespace mozilla::profiler + +#endif // ProfilerThreadRegistrationInfo_h diff --git a/tools/profiler/public/ProfilerThreadRegistry.h b/tools/profiler/public/ProfilerThreadRegistry.h new file mode 100644 index 0000000000..4d0fd3ef68 --- /dev/null +++ b/tools/profiler/public/ProfilerThreadRegistry.h @@ -0,0 +1,321 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerThreadRegistry_h +#define ProfilerThreadRegistry_h + +#include "mozilla/BaseProfilerDetail.h" +#include "mozilla/ProfilerThreadRegistration.h" +#include "mozilla/Vector.h" + +namespace mozilla::profiler { + +class ThreadRegistry { + private: + using RegistryMutex = baseprofiler::detail::BaseProfilerSharedMutex; + using RegistryLockExclusive = + baseprofiler::detail::BaseProfilerAutoLockExclusive; + using RegistryLockShared = baseprofiler::detail::BaseProfilerAutoLockShared; + + public: + // Aliases to data accessors (removing the ThreadRegistration prefix). + + using UnlockedConstReader = ThreadRegistrationUnlockedConstReader; + using UnlockedConstReaderAndAtomicRW = + ThreadRegistrationUnlockedConstReaderAndAtomicRW; + using UnlockedRWForLockedProfiler = + ThreadRegistrationUnlockedRWForLockedProfiler; + using UnlockedReaderAndAtomicRWOnThread = + ThreadRegistrationUnlockedReaderAndAtomicRWOnThread; + using LockedRWFromAnyThread = ThreadRegistrationLockedRWFromAnyThread; + using LockedRWOnThread = ThreadRegistrationLockedRWOnThread; + + // Off-thread access through the registry, providing the following data + // accessors: UnlockedConstReader, UnlockedConstReaderAndAtomicRW, + // UnlockedRWForLockedProfiler, and LockedRWFromAnyThread. + // (See ThreadRegistration class for ON-thread access.) + + // Reference-like class pointing at a ThreadRegistration. + // It should only exist while sRegistryMutex is locked. + class OffThreadRef { + public: + // const UnlockedConstReader + + [[nodiscard]] const UnlockedConstReader& UnlockedConstReaderCRef() const { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedConstReader(F&& aF) const { + return std::forward<F>(aF)(UnlockedConstReaderCRef()); + } + + // const UnlockedConstReaderAndAtomicRW + + [[nodiscard]] const UnlockedConstReaderAndAtomicRW& + UnlockedConstReaderAndAtomicRWCRef() const { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedConstReaderAndAtomicRW(F&& aF) const { + return std::forward<F>(aF)(UnlockedConstReaderAndAtomicRWCRef()); + } + + // UnlockedConstReaderAndAtomicRW + + [[nodiscard]] UnlockedConstReaderAndAtomicRW& + UnlockedConstReaderAndAtomicRWRef() { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedConstReaderAndAtomicRW(F&& aF) { + return std::forward<F>(aF)(UnlockedConstReaderAndAtomicRWRef()); + } + + // const UnlockedRWForLockedProfiler + + [[nodiscard]] const UnlockedRWForLockedProfiler& + UnlockedRWForLockedProfilerCRef() const { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedRWForLockedProfiler(F&& aF) const { + return std::forward<F>(aF)(UnlockedRWForLockedProfilerCRef()); + } + + // UnlockedRWForLockedProfiler + + [[nodiscard]] UnlockedRWForLockedProfiler& + UnlockedRWForLockedProfilerRef() { + return mThreadRegistration->mData; + } + + template <typename F> + auto WithUnlockedRWForLockedProfiler(F&& aF) { + return std::forward<F>(aF)(UnlockedRWForLockedProfilerRef()); + } + + // const LockedRWFromAnyThread through ConstRWFromAnyThreadWithLock + + class ConstRWFromAnyThreadWithLock { + public: + [[nodiscard]] const LockedRWFromAnyThread& DataCRef() const { + return mLockedRWFromAnyThread; + } + [[nodiscard]] const LockedRWFromAnyThread* operator->() const { + return &mLockedRWFromAnyThread; + } + + ConstRWFromAnyThreadWithLock( + const LockedRWFromAnyThread& aLockedRWFromAnyThread, + ThreadRegistration::DataMutex& aDataMutex) + : mLockedRWFromAnyThread(aLockedRWFromAnyThread), + mDataLock(aDataMutex) {} + + private: + const LockedRWFromAnyThread& mLockedRWFromAnyThread; + ThreadRegistration::DataLock mDataLock; + }; + + [[nodiscard]] ConstRWFromAnyThreadWithLock ConstLockedRWFromAnyThread() + const { + return ConstRWFromAnyThreadWithLock{mThreadRegistration->mData, + mThreadRegistration->mDataMutex}; + } + + template <typename F> + auto WithConstLockedRWFromAnyThread(F&& aF) const { + ConstRWFromAnyThreadWithLock lockedData = ConstLockedRWFromAnyThread(); + return std::forward<F>(aF)(lockedData.DataCRef()); + } + + // LockedRWFromAnyThread through RWFromAnyThreadWithLock + + class RWFromAnyThreadWithLock { + public: + [[nodiscard]] const LockedRWFromAnyThread& DataCRef() const { + return mLockedRWFromAnyThread; + } + [[nodiscard]] LockedRWFromAnyThread& DataRef() { + return mLockedRWFromAnyThread; + } + [[nodiscard]] const LockedRWFromAnyThread* operator->() const { + return &mLockedRWFromAnyThread; + } + [[nodiscard]] LockedRWFromAnyThread* operator->() { + return &mLockedRWFromAnyThread; + } + + // In some situations, it may be useful to do some on-thread operations if + // we are indeed on this thread now. The lock is still held here; caller + // should not use this pointer longer than this RWFromAnyThreadWithLock. + [[nodiscard]] LockedRWOnThread* GetLockedRWOnThread() { + if (mLockedRWFromAnyThread.Info().ThreadId() == + profiler_current_thread_id()) { + // mLockedRWFromAnyThread references a subclass of the + // ThreadRegistration's mData, so it's safe to downcast it to another + // hierarchy level of the object. + return &static_cast<LockedRWOnThread&>(mLockedRWFromAnyThread); + } + return nullptr; + } + + private: + friend class OffThreadRef; + RWFromAnyThreadWithLock(LockedRWFromAnyThread& aLockedRWFromAnyThread, + ThreadRegistration::DataMutex& aDataMutex) + : mLockedRWFromAnyThread(aLockedRWFromAnyThread), + mDataLock(aDataMutex) {} + + LockedRWFromAnyThread& mLockedRWFromAnyThread; + ThreadRegistration::DataLock mDataLock; + }; + + [[nodiscard]] RWFromAnyThreadWithLock GetLockedRWFromAnyThread() { + return RWFromAnyThreadWithLock{mThreadRegistration->mData, + mThreadRegistration->mDataMutex}; + } + + template <typename F> + auto WithLockedRWFromAnyThread(F&& aF) { + RWFromAnyThreadWithLock lockedData = GetLockedRWFromAnyThread(); + return std::forward<F>(aF)(lockedData.DataRef()); + } + + private: + // Only ThreadRegistry should construct an OnThreadRef. + friend class ThreadRegistry; + explicit OffThreadRef(ThreadRegistration& aThreadRegistration) + : mThreadRegistration(&aThreadRegistration) {} + + // If we have an ON-thread ref, it's safe to convert to an OFF-thread ref. + explicit OffThreadRef(ThreadRegistration::OnThreadRef aOnThreadRef) + : mThreadRegistration(aOnThreadRef.mThreadRegistration) {} + + [[nodiscard]] bool IsPointingAt( + ThreadRegistration& aThreadRegistration) const { + return mThreadRegistration == &aThreadRegistration; + } + + // Guaranted to be non-null by construction. + ThreadRegistration* mThreadRegistration; + }; + + // Lock the registry non-exclusively and allow iteration. E.g.: + // `for (OffThreadRef thread : LockedRegistry{}) { ... }` + // Do *not* export copies/references, as they could become dangling. + // Locking order: Profiler, ThreadRegistry, ThreadRegistration. + class LockedRegistry { + public: + LockedRegistry() + : mRegistryLock([]() -> RegistryMutex& { + MOZ_ASSERT(!IsRegistryMutexLockedOnCurrentThread(), + "Recursive locking detected"); + // In DEBUG builds, *before* we attempt to lock sRegistryMutex, we + // want to check that the ThreadRegistration mutex is *not* locked + // on this thread, to avoid inversion deadlocks. + MOZ_ASSERT(!ThreadRegistration::IsDataMutexLockedOnCurrentThread()); + return sRegistryMutex; + }()) { + ThreadRegistration::WithOnThreadRef( + [](ThreadRegistration::OnThreadRef aOnThreadRef) { + aOnThreadRef.mThreadRegistration + ->mIsRegistryLockedSharedOnThisThread = true; + }); + } + + ~LockedRegistry() { + ThreadRegistration::WithOnThreadRef( + [](ThreadRegistration::OnThreadRef aOnThreadRef) { + aOnThreadRef.mThreadRegistration + ->mIsRegistryLockedSharedOnThisThread = false; + }); + } + + [[nodiscard]] const OffThreadRef* begin() const { + return sRegistryContainer.begin(); + } + [[nodiscard]] OffThreadRef* begin() { return sRegistryContainer.begin(); } + [[nodiscard]] const OffThreadRef* end() const { + return sRegistryContainer.end(); + } + [[nodiscard]] OffThreadRef* end() { return sRegistryContainer.end(); } + + private: + RegistryLockShared mRegistryLock; + }; + + // Call `F(OffThreadRef)` for the given aThreadId. + template <typename F> + static void WithOffThreadRef(ProfilerThreadId aThreadId, F&& aF) { + for (OffThreadRef thread : LockedRegistry{}) { + if (thread.UnlockedConstReaderCRef().Info().ThreadId() == aThreadId) { + std::forward<F>(aF)(thread); + break; + } + } + } + + template <typename F, typename FallbackReturn> + [[nodiscard]] static auto WithOffThreadRefOr(ProfilerThreadId aThreadId, + F&& aF, + FallbackReturn&& aFallbackReturn) + -> decltype(std::forward<F>(aF)(std::declval<OffThreadRef>())) { + for (OffThreadRef thread : LockedRegistry{}) { + if (thread.UnlockedConstReaderCRef().Info().ThreadId() == aThreadId) { + return std::forward<F>(aF)(thread); + } + } + return std::forward<FallbackReturn>(aFallbackReturn); + } + + static size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) { + LockedRegistry lockedRegistry; + // "Ex" because we don't count static objects, but we count whatever they + // allocated on the heap. + size_t bytes = sRegistryContainer.sizeOfExcludingThis(aMallocSizeOf); + for (const OffThreadRef& offThreadRef : lockedRegistry) { + bytes += + offThreadRef.mThreadRegistration->SizeOfExcludingThis(aMallocSizeOf); + } + return bytes; + } + + static size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) { + return SizeOfExcludingThis(aMallocSizeOf); + } + + [[nodiscard]] static bool IsRegistryMutexLockedOnCurrentThread() { + return sRegistryMutex.IsLockedExclusiveOnCurrentThread() || + ThreadRegistration::WithOnThreadRefOr( + [](ThreadRegistration::OnThreadRef aOnThreadRef) { + return aOnThreadRef.mThreadRegistration + ->mIsRegistryLockedSharedOnThisThread; + }, + false); + } + + private: + using RegistryContainer = Vector<OffThreadRef>; + + static RegistryContainer sRegistryContainer; + + // Mutex protecting the registry. + // Locking order: Profiler, ThreadRegistry, ThreadRegistration. + static RegistryMutex sRegistryMutex; + + // Only allow ThreadRegistration to (un)register itself. + friend class ThreadRegistration; + static void Register(ThreadRegistration::OnThreadRef aOnThreadRef); + static void Unregister(ThreadRegistration::OnThreadRef aOnThreadRef); +}; + +} // namespace mozilla::profiler + +#endif // ProfilerThreadRegistry_h diff --git a/tools/profiler/public/ProfilerThreadSleep.h b/tools/profiler/public/ProfilerThreadSleep.h new file mode 100644 index 0000000000..730176d39f --- /dev/null +++ b/tools/profiler/public/ProfilerThreadSleep.h @@ -0,0 +1,58 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// APIs that inform the profiler when a thread is effectively asleep so that we +// can avoid sampling it more than once. + +#ifndef ProfilerThreadSleep_h +#define ProfilerThreadSleep_h + +#ifndef MOZ_GECKO_PROFILER + +// This file can be #included unconditionally. However, everything within this +// file must be guarded by a #ifdef MOZ_GECKO_PROFILER, *except* for the +// following macros and functions, which encapsulate the most common operations +// and thus avoid the need for many #ifdefs. + +# define AUTO_PROFILER_THREAD_SLEEP + +static inline void profiler_thread_sleep() {} + +static inline void profiler_thread_wake() {} + +#else // !MOZ_GECKO_PROFILER + +# include "mozilla/Attributes.h" +# include "mozilla/BaseProfilerRAIIMacro.h" + +// These functions tell the profiler that a thread went to sleep so that we can +// avoid sampling it more than once while it's sleeping. Calling +// profiler_thread_sleep() twice without an intervening profiler_thread_wake() +// is an error. All three functions operate the same whether the profiler is +// active or inactive. +void profiler_thread_sleep(); +void profiler_thread_wake(); + +// Mark a thread as asleep within a scope. +// (See also AUTO_PROFILER_THREAD_WAKE in ProfilerThreadState.h) +# define AUTO_PROFILER_THREAD_SLEEP \ + mozilla::AutoProfilerThreadSleep PROFILER_RAII + +namespace mozilla { + +// (See also AutoProfilerThreadWake in ProfilerThreadState.h) +class MOZ_RAII AutoProfilerThreadSleep { + public: + explicit AutoProfilerThreadSleep() { profiler_thread_sleep(); } + + ~AutoProfilerThreadSleep() { profiler_thread_wake(); } +}; + +} // namespace mozilla + +#endif // !MOZ_GECKO_PROFILER + +#endif // ProfilerThreadSleep_h diff --git a/tools/profiler/public/ProfilerThreadState.h b/tools/profiler/public/ProfilerThreadState.h new file mode 100644 index 0000000000..6ac48e41dd --- /dev/null +++ b/tools/profiler/public/ProfilerThreadState.h @@ -0,0 +1,128 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This header contains functions that give information about the Profiler state +// with regards to the current thread. + +#ifndef ProfilerThreadState_h +#define ProfilerThreadState_h + +#include "mozilla/ProfilerState.h" +#include "mozilla/ProfilerThreadRegistration.h" +#include "mozilla/ProfilerThreadRegistry.h" +#include "mozilla/ProfilerThreadSleep.h" + +// During profiling, if the current thread is registered, return true +// (regardless of whether it is actively being profiled). +// (Same caveats and recommended usage as profiler_is_active().) +[[nodiscard]] inline bool profiler_is_active_and_thread_is_registered() { + return profiler_is_active() && + mozilla::profiler::ThreadRegistration::IsRegistered(); +} + +// Is the profiler active and unpaused, and is the current thread being +// profiled for any of the given features? (Same caveats and recommended usage +// as profiler_is_active().) +[[nodiscard]] inline bool profiler_thread_is_being_profiled( + ThreadProfilingFeatures aThreadProfilingFeatures) { + return profiler_is_active_and_unpaused() && + mozilla::profiler::ThreadRegistration::WithOnThreadRefOr( + [aThreadProfilingFeatures]( + mozilla::profiler::ThreadRegistration::OnThreadRef aTR) { + return DoFeaturesIntersect( + aTR.UnlockedConstReaderAndAtomicRWCRef().ProfilingFeatures(), + aThreadProfilingFeatures); + }, + false); +} + +// Is the profiler active and unpaused, and is the given thread being profiled? +// (Same caveats and recommended usage as profiler_is_active().) +// Safe to use with the current thread id, or unspecified ProfilerThreadId (same +// as current thread id). +[[nodiscard]] inline bool profiler_thread_is_being_profiled( + const ProfilerThreadId& aThreadId, + ThreadProfilingFeatures aThreadProfilingFeatures) { + if (!profiler_is_active_and_unpaused()) { + return false; + } + + if (!aThreadId.IsSpecified() || aThreadId == profiler_current_thread_id()) { + // For the current thread id, use the ThreadRegistration directly, it is + // more efficient. + return mozilla::profiler::ThreadRegistration::WithOnThreadRefOr( + [aThreadProfilingFeatures]( + mozilla::profiler::ThreadRegistration::OnThreadRef aTR) { + return DoFeaturesIntersect( + aTR.UnlockedConstReaderAndAtomicRWCRef().ProfilingFeatures(), + aThreadProfilingFeatures); + }, + false); + } + + // For other threads, go through the ThreadRegistry. + return mozilla::profiler::ThreadRegistry::WithOffThreadRefOr( + aThreadId, + [aThreadProfilingFeatures]( + mozilla::profiler::ThreadRegistry::OffThreadRef aTR) { + return DoFeaturesIntersect( + aTR.UnlockedConstReaderAndAtomicRWCRef().ProfilingFeatures(), + aThreadProfilingFeatures); + }, + false); +} + +// Is the current thread registered and sleeping? +[[nodiscard]] inline bool profiler_thread_is_sleeping() { + return profiler_is_active() && + mozilla::profiler::ThreadRegistration::WithOnThreadRefOr( + [](mozilla::profiler::ThreadRegistration::OnThreadRef aTR) { + return aTR.UnlockedConstReaderAndAtomicRWCRef().IsSleeping(); + }, + false); +} + +#ifndef MOZ_GECKO_PROFILER + +# define AUTO_PROFILER_THREAD_WAKE + +#else // !MOZ_GECKO_PROFILER + +// Mark a thread as awake within a scope. +// (See also AUTO_PROFILER_THREAD_SLEEP in mozilla/ProfilerThreadSleep.h) +# define AUTO_PROFILER_THREAD_WAKE \ + mozilla::AutoProfilerThreadWake PROFILER_RAII + +namespace mozilla { + +// Temporarily wake up the profiling of a thread while servicing events such as +// Asynchronous Procedure Calls (APCs). +// (See also AutoProfilerThreadSleep in ProfilerThreadSleep.h) +class MOZ_RAII AutoProfilerThreadWake { + public: + explicit AutoProfilerThreadWake() + : mIssuedWake(profiler_thread_is_sleeping()) { + if (mIssuedWake) { + profiler_thread_wake(); + } + } + + ~AutoProfilerThreadWake() { + if (mIssuedWake) { + MOZ_ASSERT(!profiler_thread_is_sleeping()); + profiler_thread_sleep(); + } + } + + private: + bool mIssuedWake; +}; + +} // namespace mozilla + +#endif // !MOZ_GECKO_PROFILER + +#endif // ProfilerThreadState_h diff --git a/tools/profiler/public/ProfilerUtils.h b/tools/profiler/public/ProfilerUtils.h new file mode 100644 index 0000000000..3969761e18 --- /dev/null +++ b/tools/profiler/public/ProfilerUtils.h @@ -0,0 +1,32 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ProfilerUtils_h +#define ProfilerUtils_h + +// This header contains most process- and thread-related functions. +// It is safe to include unconditionally. + +#include "mozilla/BaseProfilerUtils.h" + +using ProfilerProcessId = mozilla::baseprofiler::BaseProfilerProcessId; +using ProfilerThreadId = mozilla::baseprofiler::BaseProfilerThreadId; + +// Get the current process's ID. +[[nodiscard]] ProfilerProcessId profiler_current_process_id(); + +// Get the current thread's ID. +[[nodiscard]] ProfilerThreadId profiler_current_thread_id(); + +// Must be called at least once from the main thread, before any other main- +// thread id function. +void profiler_init_main_thread_id(); + +[[nodiscard]] ProfilerThreadId profiler_main_thread_id(); + +[[nodiscard]] bool profiler_is_main_thread(); + +#endif // ProfilerUtils_h diff --git a/tools/profiler/public/shared-libraries.h b/tools/profiler/public/shared-libraries.h new file mode 100644 index 0000000000..9b36d0fc3f --- /dev/null +++ b/tools/profiler/public/shared-libraries.h @@ -0,0 +1,220 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef SHARED_LIBRARIES_H_ +#define SHARED_LIBRARIES_H_ + +#include "nsNativeCharsetUtils.h" +#include "nsString.h" +#include <nsID.h> + +#include <algorithm> +#include <stdint.h> +#include <stdlib.h> +#include <string> +#include <vector> + +namespace IPC { +class MessageReader; +class MessageWriter; +template <typename T> +struct ParamTraits; +} // namespace IPC + +class SharedLibrary { + public: + SharedLibrary(uintptr_t aStart, uintptr_t aEnd, uintptr_t aOffset, + const nsCString& aBreakpadId, const nsCString& aCodeId, + const nsString& aModuleName, const nsString& aModulePath, + const nsString& aDebugName, const nsString& aDebugPath, + const nsCString& aVersion, const char* aArch) + : mStart(aStart), + mEnd(aEnd), + mOffset(aOffset), + mBreakpadId(aBreakpadId), + mCodeId(aCodeId), + mModuleName(aModuleName), + mModulePath(aModulePath), + mDebugName(aDebugName), + mDebugPath(aDebugPath), + mVersion(aVersion), + mArch(aArch) {} + + bool operator==(const SharedLibrary& other) const { + return (mStart == other.mStart) && (mEnd == other.mEnd) && + (mOffset == other.mOffset) && (mModuleName == other.mModuleName) && + (mModulePath == other.mModulePath) && + (mDebugName == other.mDebugName) && + (mDebugPath == other.mDebugPath) && + (mBreakpadId == other.mBreakpadId) && (mCodeId == other.mCodeId) && + (mVersion == other.mVersion) && (mArch == other.mArch); + } + + uintptr_t GetStart() const { return mStart; } + uintptr_t GetEnd() const { return mEnd; } + uintptr_t GetOffset() const { return mOffset; } + const nsCString& GetBreakpadId() const { return mBreakpadId; } + const nsCString& GetCodeId() const { return mCodeId; } + const nsString& GetModuleName() const { return mModuleName; } + const nsString& GetModulePath() const { return mModulePath; } + const std::string GetNativeDebugPath() const { + nsAutoCString debugPathStr; + + NS_CopyUnicodeToNative(mDebugPath, debugPathStr); + + return debugPathStr.get(); + } + const nsString& GetDebugName() const { return mDebugName; } + const nsString& GetDebugPath() const { return mDebugPath; } + const nsCString& GetVersion() const { return mVersion; } + const std::string& GetArch() const { return mArch; } + size_t SizeOf() const { + return sizeof *this + mBreakpadId.Length() + mCodeId.Length() + + mModuleName.Length() * 2 + mModulePath.Length() * 2 + + mDebugName.Length() * 2 + mDebugPath.Length() * 2 + + mVersion.Length() + mArch.size(); + } + + SharedLibrary() : mStart{0}, mEnd{0}, mOffset{0} {} + + private: + uintptr_t mStart; + uintptr_t mEnd; + uintptr_t mOffset; + nsCString mBreakpadId; + // A string carrying an identifier for a binary. + // + // All platforms have different formats: + // - Windows: The code ID for a Windows PE file. + // It's the PE timestamp and PE image size. + // - macOS: The code ID for a macOS / iOS binary (mach-O). + // It's the mach-O UUID without dashes and without the trailing 0 for the + // breakpad ID. + // - Linux/Android: The code ID for a Linux ELF file. + // It's the complete build ID, as hex string. + nsCString mCodeId; + nsString mModuleName; + nsString mModulePath; + nsString mDebugName; + nsString mDebugPath; + nsCString mVersion; + std::string mArch; + + friend struct IPC::ParamTraits<SharedLibrary>; +}; + +static bool CompareAddresses(const SharedLibrary& first, + const SharedLibrary& second) { + return first.GetStart() < second.GetStart(); +} + +class SharedLibraryInfo { + public: +#ifdef MOZ_GECKO_PROFILER + static SharedLibraryInfo GetInfoForSelf(); +# ifdef XP_WIN + static SharedLibraryInfo GetInfoFromPath(const wchar_t* aPath); +# endif + + static void Initialize(); +#else + static SharedLibraryInfo GetInfoForSelf() { return SharedLibraryInfo(); } +# ifdef XP_WIN + static SharedLibraryInfo GetInfoFromPath(const wchar_t* aPath) { + return SharedLibraryInfo(); + } +# endif + + static void Initialize() {} +#endif + + void AddSharedLibrary(SharedLibrary entry) { mEntries.push_back(entry); } + + void AddAllSharedLibraries(const SharedLibraryInfo& sharedLibraryInfo) { + mEntries.insert(mEntries.end(), sharedLibraryInfo.mEntries.begin(), + sharedLibraryInfo.mEntries.end()); + } + + const SharedLibrary& GetEntry(size_t i) const { return mEntries[i]; } + + SharedLibrary& GetMutableEntry(size_t i) { return mEntries[i]; } + + // Removes items in the range [first, last) + // i.e. element at the "last" index is not removed + void RemoveEntries(size_t first, size_t last) { + mEntries.erase(mEntries.begin() + first, mEntries.begin() + last); + } + + bool Contains(const SharedLibrary& searchItem) const { + return (mEntries.end() != + std::find(mEntries.begin(), mEntries.end(), searchItem)); + } + + size_t GetSize() const { return mEntries.size(); } + + void SortByAddress() { + std::sort(mEntries.begin(), mEntries.end(), CompareAddresses); + } + + // Remove duplicate entries from the vector. + // + // We purposefully don't use the operator== implementation of SharedLibrary + // because it compares all the fields including mStart, mEnd and mOffset which + // are not the same across different processes. + void DeduplicateEntries() { + static auto cmpSort = [](const SharedLibrary& a, const SharedLibrary& b) { + return std::tie(a.GetModuleName(), a.GetBreakpadId()) < + std::tie(b.GetModuleName(), b.GetBreakpadId()); + }; + static auto cmpEqual = [](const SharedLibrary& a, const SharedLibrary& b) { + return std::tie(a.GetModuleName(), a.GetBreakpadId()) == + std::tie(b.GetModuleName(), b.GetBreakpadId()); + }; + // std::unique requires the vector to be sorted first. It can only remove + // consecutive duplicate elements. + std::sort(mEntries.begin(), mEntries.end(), cmpSort); + // Remove the duplicates since it's sorted now. + mEntries.erase(std::unique(mEntries.begin(), mEntries.end(), cmpEqual), + mEntries.end()); + } + + void Clear() { mEntries.clear(); } + + size_t SizeOf() const { + size_t size = 0; + + for (const auto& item : mEntries) { + size += item.SizeOf(); + } + + return size; + } + + private: + std::vector<SharedLibrary> mEntries; + + friend struct IPC::ParamTraits<SharedLibraryInfo>; +}; + +namespace IPC { +template <> +struct ParamTraits<SharedLibrary> { + typedef SharedLibrary paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam); + static bool Read(MessageReader* aReader, paramType* aResult); +}; + +template <> +struct ParamTraits<SharedLibraryInfo> { + typedef SharedLibraryInfo paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam); + static bool Read(MessageReader* aReader, paramType* aResult); +}; +} // namespace IPC + +#endif diff --git a/tools/profiler/rust-api/Cargo.toml b/tools/profiler/rust-api/Cargo.toml new file mode 100644 index 0000000000..7efc739c78 --- /dev/null +++ b/tools/profiler/rust-api/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "gecko-profiler" +version = "0.1.0" +authors = ["The Mozilla Project Developers"] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +profiler-macros = { path = "./macros" } +lazy_static = "1" +serde = { version = "1.0", features = ["derive"] } +bincode = "1" +mozbuild = "0.1" + +[build-dependencies] +lazy_static = "1" +bindgen = {version = "0.69", default-features = false} +mozbuild = "0.1" + +[features] +# This feature is being set by Gecko. If it's not set, all public functions and +# structs will be no-op. +enabled = [] diff --git a/tools/profiler/rust-api/README.md b/tools/profiler/rust-api/README.md new file mode 100644 index 0000000000..60926a85c7 --- /dev/null +++ b/tools/profiler/rust-api/README.md @@ -0,0 +1,5 @@ +# Gecko Profiler API for Rust + +This crate is the collection of all the API endpoints for Gecko Profiler. Please use this crate instead of using raw FFI calls. + +See the module documentations for more information about the specific API endpoints. diff --git a/tools/profiler/rust-api/build.rs b/tools/profiler/rust-api/build.rs new file mode 100644 index 0000000000..a27e52d03c --- /dev/null +++ b/tools/profiler/rust-api/build.rs @@ -0,0 +1,118 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +//! Build script for the Gecko Profiler bindings. +//! +//! This file is executed by cargo when this crate is built. It generates the +//! `$OUT_DIR/bindings.rs` file which is then included by `src/gecko_bindings/mod.rs`. + +#[macro_use] +extern crate lazy_static; + +use bindgen::{Builder, CodegenConfig}; +use std::env; +use std::fs; +use std::path::PathBuf; + +lazy_static! { + static ref OUTDIR_PATH: PathBuf = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("gecko"); +} + +const BINDINGS_FILE: &str = "bindings.rs"; + +lazy_static! { + static ref BINDGEN_FLAGS: Vec<String> = { + // Load build-specific config overrides. + let path = mozbuild::TOPOBJDIR.join("tools/profiler/rust-api/extra-bindgen-flags"); + println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); + fs::read_to_string(path).expect("Failed to read extra-bindgen-flags file") + .split_whitespace() + .map(std::borrow::ToOwned::to_owned) + .collect() + }; + static ref SEARCH_PATHS: Vec<PathBuf> = vec![ + mozbuild::TOPOBJDIR.join("dist/include"), + mozbuild::TOPOBJDIR.join("dist/include/nspr"), + ]; +} + +fn search_include(name: &str) -> Option<PathBuf> { + for path in SEARCH_PATHS.iter() { + let file = path.join(name); + if file.is_file() { + return Some(file); + } + } + None +} + +fn add_include(name: &str) -> String { + let file = match search_include(name) { + Some(file) => file, + None => panic!("Include not found: {}", name), + }; + let file_path = String::from(file.to_str().unwrap()); + println!("cargo:rerun-if-changed={}", file_path); + file_path +} + +fn generate_bindings() { + let mut builder = Builder::default() + .enable_cxx_namespaces() + .with_codegen_config(CodegenConfig::TYPES | CodegenConfig::VARS | CodegenConfig::FUNCTIONS) + .disable_untagged_union() + .size_t_is_usize(true); + + for dir in SEARCH_PATHS.iter() { + builder = builder.clang_arg("-I").clang_arg(dir.to_str().unwrap()); + } + + builder = builder + .clang_arg("-include") + .clang_arg(add_include("mozilla-config.h")); + + for item in &*BINDGEN_FLAGS { + builder = builder.clang_arg(item); + } + + let bindings = builder + .header(add_include("GeckoProfiler.h")) + .header(add_include("ProfilerBindings.h")) + .allowlist_function("gecko_profiler_.*") + .allowlist_var("mozilla::profiler::detail::RacyFeatures::sActiveAndFeatures") + .allowlist_type("mozilla::profiler::detail::RacyFeatures") + .rustified_enum("mozilla::StackCaptureOptions") + .rustified_enum("mozilla::MarkerSchema_Location") + .rustified_enum("mozilla::MarkerSchema_Format") + .rustified_enum("mozilla::MarkerSchema_Searchable") + // Converting std::string to an opaque type makes some platforms build + // successfully. Otherwise, it fails to build because MarkerSchema has + // some std::strings as its fields. + .opaque_type("std::string") + // std::vector needs to be converted to an opaque type because, if it's + // not an opaque type, bindgen can't find its size properly and + // MarkerSchema's total size reduces. That causes a heap buffer overflow. + .opaque_type("std::vector") + .raw_line("pub use self::root::*;") + // Tell cargo to invalidate the built crate whenever any of the + // included header files changed. + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + // Finish the builder and generate the bindings. + .generate() + // Unwrap the Result and panic on failure. + .expect("Unable to generate bindings"); + + let out_file = OUTDIR_PATH.join(BINDINGS_FILE); + bindings + .write_to_file(out_file) + .expect("Couldn't write bindings!"); +} + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:out_dir={}", env::var("OUT_DIR").unwrap()); + + fs::create_dir_all(&*OUTDIR_PATH).unwrap(); + generate_bindings(); +} diff --git a/tools/profiler/rust-api/cbindgen.toml b/tools/profiler/rust-api/cbindgen.toml new file mode 100644 index 0000000000..3f0df0f34f --- /dev/null +++ b/tools/profiler/rust-api/cbindgen.toml @@ -0,0 +1,15 @@ +header = """/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */""" +autogen_warning = """/* DO NOT MODIFY THIS MANUALLY! This file was generated using cbindgen. See RunCbindgen.py */ +#ifndef ProfilerRustBindings_h +#error "Don't include this file directly, instead include ProfilerRustBindings.h" +#endif +""" +include_version = true +braces = "SameLine" +line_length = 100 +tab_width = 2 +language = "C++" +# Put FFI calls in the `mozilla::profiler::ffi` namespace. +namespaces = ["mozilla", "profiler", "ffi"] diff --git a/tools/profiler/rust-api/extra-bindgen-flags.in b/tools/profiler/rust-api/extra-bindgen-flags.in new file mode 100644 index 0000000000..b0275a031b --- /dev/null +++ b/tools/profiler/rust-api/extra-bindgen-flags.in @@ -0,0 +1 @@ +@BINDGEN_SYSTEM_FLAGS@ @NSPR_CFLAGS@ diff --git a/tools/profiler/rust-api/macros/Cargo.toml b/tools/profiler/rust-api/macros/Cargo.toml new file mode 100644 index 0000000000..c089965401 --- /dev/null +++ b/tools/profiler/rust-api/macros/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "profiler-macros" +version = "0.1.0" +authors = ["The Mozilla Project Developers"] +edition = "2018" +license = "MPL-2.0" + +[lib] +proc-macro = true + +[dependencies] +syn = "2" +quote = "1.0" diff --git a/tools/profiler/rust-api/macros/src/lib.rs b/tools/profiler/rust-api/macros/src/lib.rs new file mode 100644 index 0000000000..aca65cced3 --- /dev/null +++ b/tools/profiler/rust-api/macros/src/lib.rs @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![deny(warnings)] + +//! A procedural macro as a syntactical sugar to `gecko_profiler_label!` macro. +//! You can use this macro on top of functions to automatically append the +//! label frame to the function. +//! +//! Example usage: +//! ```rust +//! #[gecko_profiler_fn_label(DOM)] +//! fn foo(bar: u32) -> u32 { +//! bar +//! } +//! +//! #[gecko_profiler_fn_label(Javascript, IonMonkey)] +//! pub fn bar(baz: i8) -> i8 { +//! baz +//! } +//! ``` +//! +//! See the documentation of `gecko_profiler_label!` macro to learn more about +//! its parameters. + +extern crate proc_macro; + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, ItemFn}; + +#[proc_macro_attribute] +pub fn gecko_profiler_fn_label(attrs: TokenStream, input: TokenStream) -> TokenStream { + let mut attr_args = Vec::new(); + let attr_parser = syn::meta::parser(|meta| { + attr_args.push(meta.path); + Ok(()) + }); + parse_macro_input!(attrs with attr_parser); + let input = parse_macro_input!(input as ItemFn); + + if attr_args.is_empty() || attr_args.len() > 2 { + panic!("Expected one or two arguments as ProfilingCategory or ProfilingCategoryPair but {} arguments provided!", attr_args.len()); + } + + let category_name = &attr_args[0]; + // Try to get the subcategory if possible. Otherwise, use `None`. + let subcategory_if_provided = match attr_args.get(1) { + Some(subcategory) => quote!(, #subcategory), + None => quote!(), + }; + + let ItemFn { + attrs, + vis, + sig, + block, + } = input; + let stmts = &block.stmts; + + let new_fn = quote! { + #(#attrs)* #vis #sig { + gecko_profiler_label!(#category_name#subcategory_if_provided); + #(#stmts)* + } + }; + + new_fn.into() +} diff --git a/tools/profiler/rust-api/src/gecko_bindings/glue.rs b/tools/profiler/rust-api/src/gecko_bindings/glue.rs new file mode 100644 index 0000000000..531f727a00 --- /dev/null +++ b/tools/profiler/rust-api/src/gecko_bindings/glue.rs @@ -0,0 +1,53 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::gecko_bindings::{bindings, structs::mozilla}; +use crate::json_writer::JSONWriter; +use crate::marker::deserializer_tags_state::{ + get_marker_type_functions_read_guard, MarkerTypeFunctions, +}; +use std::ops::DerefMut; +use std::os::raw::{c_char, c_void}; + +#[no_mangle] +pub unsafe extern "C" fn gecko_profiler_serialize_marker_for_tag( + deserializer_tag: u8, + payload: *const u8, + payload_size: usize, + json_writer: &mut mozilla::baseprofiler::SpliceableJSONWriter, +) { + let marker_type_functions = get_marker_type_functions_read_guard(); + let &MarkerTypeFunctions { + transmute_and_stream_fn, + marker_type_name_fn, + .. + } = marker_type_functions.get(deserializer_tag); + let mut json_writer = JSONWriter::new(&mut *json_writer); + + // Serialize the marker type name first. + json_writer.string_property("type", marker_type_name_fn()); + // Serialize the marker payload now. + transmute_and_stream_fn(payload, payload_size, &mut json_writer); +} + +#[no_mangle] +pub unsafe extern "C" fn gecko_profiler_stream_marker_schemas( + json_writer: &mut mozilla::baseprofiler::SpliceableJSONWriter, + streamed_names_set: *mut c_void, +) { + let marker_type_functions = get_marker_type_functions_read_guard(); + + for funcs in marker_type_functions.iter() { + let marker_name = (funcs.marker_type_name_fn)(); + let mut marker_schema = (funcs.marker_type_display_fn)(); + + bindings::gecko_profiler_marker_schema_stream( + json_writer, + marker_name.as_ptr() as *const c_char, + marker_name.len(), + marker_schema.pin.deref_mut().as_mut_ptr(), + streamed_names_set, + ) + } +} diff --git a/tools/profiler/rust-api/src/gecko_bindings/mod.rs b/tools/profiler/rust-api/src/gecko_bindings/mod.rs new file mode 100644 index 0000000000..f1ec667bb2 --- /dev/null +++ b/tools/profiler/rust-api/src/gecko_bindings/mod.rs @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +//! Gecko's C++ bindings for the profiler. + +#[allow( + dead_code, + non_camel_case_types, + non_snake_case, + non_upper_case_globals, + missing_docs +)] +pub mod structs { + include!(concat!(env!("OUT_DIR"), "/gecko/bindings.rs")); +} + +pub use self::structs as bindings; + +mod glue; +pub mod profiling_categories; diff --git a/tools/profiler/rust-api/src/gecko_bindings/profiling_categories.rs b/tools/profiler/rust-api/src/gecko_bindings/profiling_categories.rs new file mode 100644 index 0000000000..0f24aa9c35 --- /dev/null +++ b/tools/profiler/rust-api/src/gecko_bindings/profiling_categories.rs @@ -0,0 +1,32 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! This file contains the generated ProfilingCategory and ProfilingCategoryPair enums. +//! +//! The contents of this module are generated by +//! `mozglue/baseprofiler/generate_profiling_categories.py`, from +//! 'mozglue/baseprofiler/core/profiling_categories.yaml`. + +include!(mozbuild::objdir_path!( + "tools/profiler/rust-api/src/gecko_bindings/profiling_categories.rs" +)); + +/// Helper macro that returns the profiling category pair from either only +/// "category", or "category + sub category" pair. Refer to `profiling_categories.yaml` +/// or generated `profiling_categories.rs` to see all the marker categories. +/// This is useful to make the APIs similar to each other since +/// `gecko_profiler_label!` API also requires the same syntax. +/// +/// Example usages: +/// - `gecko_profiler_category!(DOM)` +/// - `gecko_profiler_category!(JavaScript, Parsing)` +#[macro_export] +macro_rules! gecko_profiler_category { + ($category:ident) => { + $crate::ProfilingCategoryPair::$category(None) + }; + ($category:ident, $subcategory:ident) => { + $crate::ProfilingCategoryPair::$category(Some($crate::$category::$subcategory)) + }; +} diff --git a/tools/profiler/rust-api/src/json_writer.rs b/tools/profiler/rust-api/src/json_writer.rs new file mode 100644 index 0000000000..66c12dda04 --- /dev/null +++ b/tools/profiler/rust-api/src/json_writer.rs @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Gecko JSON writer support for marker API. + +use crate::gecko_bindings::{bindings, structs::mozilla}; +use std::os::raw::c_char; + +/// Wrapper for the C++ SpliceableJSONWriter object. It exposes some methods to +/// add various properties to the JSON. +#[derive(Debug)] +pub struct JSONWriter<'a>(&'a mut mozilla::baseprofiler::SpliceableJSONWriter); + +impl<'a> JSONWriter<'a> { + /// Constructor for the JSONWriter object. It takes a C++ SpliceableJSONWriter + /// reference as its argument and stores it for later accesses. + pub(crate) fn new(json_writer: &'a mut mozilla::baseprofiler::SpliceableJSONWriter) -> Self { + JSONWriter(json_writer) + } + + /// Adds an int property to the JSON. + /// Prints: "<name>": <value> + pub fn int_property(&mut self, name: &str, value: i64) { + unsafe { + bindings::gecko_profiler_json_writer_int_property( + self.0, + name.as_ptr() as *const c_char, + name.len(), + value, + ); + } + } + + /// Adds a float property to the JSON. + /// Prints: "<name>": <value> + pub fn float_property(&mut self, name: &str, value: f64) { + unsafe { + bindings::gecko_profiler_json_writer_float_property( + self.0, + name.as_ptr() as *const c_char, + name.len(), + value, + ); + } + } + + /// Adds an bool property to the JSON. + /// Prints: "<name>": <value> + pub fn bool_property(&mut self, name: &str, value: bool) { + unsafe { + bindings::gecko_profiler_json_writer_bool_property( + self.0, + name.as_ptr() as *const c_char, + name.len(), + value, + ); + } + } + + /// Adds a string property to the JSON. + /// Prints: "<name>": "<value>" + pub fn string_property(&mut self, name: &str, value: &str) { + unsafe { + bindings::gecko_profiler_json_writer_string_property( + self.0, + name.as_ptr() as *const c_char, + name.len(), + value.as_ptr() as *const c_char, + value.len(), + ); + } + } + + /// Adds a unique string property to the JSON. + /// Prints: "<name>": <string_table_index> + pub fn unique_string_property(&mut self, name: &str, value: &str) { + unsafe { + bindings::gecko_profiler_json_writer_unique_string_property( + self.0, + name.as_ptr() as *const c_char, + name.len(), + value.as_ptr() as *const c_char, + value.len(), + ); + } + } + + /// Adds a null property to the JSON. + /// Prints: "<name>": null + pub fn null_property(&mut self, name: &str) { + unsafe { + bindings::gecko_profiler_json_writer_null_property( + self.0, + name.as_ptr() as *const c_char, + name.len(), + ); + } + } +} diff --git a/tools/profiler/rust-api/src/label.rs b/tools/profiler/rust-api/src/label.rs new file mode 100644 index 0000000000..10970c90ad --- /dev/null +++ b/tools/profiler/rust-api/src/label.rs @@ -0,0 +1,137 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Gecko profiler label support. +//! +//! Use the `profiler_label!` macro directly instead of using `AutoProfilerLabel`. +//! See the `profiler_label!` macro documentation on how to use it. + +#[cfg(feature = "enabled")] +use crate::gecko_bindings::{ + bindings, profiling_categories::ProfilingCategoryPair, structs::mozilla, +}; + +/// RAII object that constructs and destroys a C++ AutoProfilerLabel object +/// pointed to be the specified reference. +/// Use `profiler_label!` macro directly instead of this, if possible. +#[cfg(feature = "enabled")] +pub struct AutoProfilerLabel<'a>(&'a mut mozilla::AutoProfilerLabel); + +#[cfg(feature = "enabled")] +impl<'a> AutoProfilerLabel<'a> { + /// Creates a new AutoProfilerLabel with the specified label type. + /// + /// unsafe since the caller must ensure that `label` is allocated on the + /// stack. + #[inline] + pub unsafe fn new( + label: &mut std::mem::MaybeUninit<mozilla::AutoProfilerLabel>, + category_pair: ProfilingCategoryPair, + ) -> AutoProfilerLabel { + bindings::gecko_profiler_construct_label( + label.as_mut_ptr(), + category_pair.to_cpp_enum_value(), + ); + AutoProfilerLabel(&mut *label.as_mut_ptr()) + } +} + +#[cfg(feature = "enabled")] +impl<'a> Drop for AutoProfilerLabel<'a> { + #[inline] + fn drop(&mut self) { + unsafe { + bindings::gecko_profiler_destruct_label(self.0); + } + } +} + +/// Place a Gecko profiler label on the stack. +/// +/// The first `category` argument must be the name of a variant of `ProfilerLabelCategoryPair` +/// and the second optional `subcategory` argument must be one of the sub variants of +/// `ProfilerLabelCategoryPair`. All options can be seen either in the +/// profiling_categories.yaml file or generated profiling_categories.rs file. +/// +/// Example usage: +/// ```rust +/// gecko_profiler_label!(Layout); +/// gecko_profiler_label!(JavaScript, Parsing); +/// ``` +/// You can wrap this macro with a block to only label a specific part of a function. +#[cfg(feature = "enabled")] +#[macro_export] +macro_rules! gecko_profiler_label { + ($category:ident) => { + gecko_profiler_label!($crate::ProfilingCategoryPair::$category(None)) + }; + ($category:ident, $subcategory:ident) => { + gecko_profiler_label!($crate::ProfilingCategoryPair::$category(Some( + $crate::$category::$subcategory + ))) + }; + + ($category_path:expr) => { + let mut _profiler_label = ::std::mem::MaybeUninit::< + $crate::gecko_bindings::structs::mozilla::AutoProfilerLabel, + >::uninit(); + let _profiler_label = if $crate::is_active() { + unsafe { + Some($crate::AutoProfilerLabel::new( + &mut _profiler_label, + $category_path, + )) + } + } else { + None + }; + }; +} + +/// No-op when MOZ_GECKO_PROFILER is not defined. +#[cfg(not(feature = "enabled"))] +#[macro_export] +macro_rules! gecko_profiler_label { + ($category:ident) => {}; + ($category:ident, $subcategory:ident) => {}; +} + +#[cfg(test)] +mod tests { + use profiler_macros::gecko_profiler_fn_label; + + #[test] + fn test_gecko_profiler_label() { + gecko_profiler_label!(Layout); + gecko_profiler_label!(JavaScript, Parsing); + } + + #[gecko_profiler_fn_label(DOM)] + fn foo(bar: u32) -> u32 { + bar + } + + #[gecko_profiler_fn_label(Javascript, IonMonkey)] + pub(self) fn bar(baz: i8) -> i8 { + baz + } + + struct A; + + impl A { + #[gecko_profiler_fn_label(Idle)] + pub fn test(&self) -> i8 { + 1 + } + } + + #[test] + fn test_gecko_profiler_fn_label() { + let _: u32 = foo(100000); + let _: i8 = bar(127); + + let a = A; + let _ = a.test(100); + } +} diff --git a/tools/profiler/rust-api/src/lib.rs b/tools/profiler/rust-api/src/lib.rs new file mode 100644 index 0000000000..f92e2937e2 --- /dev/null +++ b/tools/profiler/rust-api/src/lib.rs @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +///! Profiler Rust API + +#[macro_use] +extern crate lazy_static; + +pub mod gecko_bindings; +mod json_writer; +mod label; +mod marker; +mod profiler_state; +mod thread; +mod time; + +pub use gecko_bindings::profiling_categories::*; +pub use json_writer::*; +#[cfg(feature = "enabled")] +pub use label::*; +pub use marker::options::*; +pub use marker::schema::MarkerSchema; +pub use marker::*; +pub use profiler_macros::gecko_profiler_fn_label; +pub use profiler_state::*; +pub use thread::*; +pub use time::*; + +pub use serde::{Deserialize, Serialize}; diff --git a/tools/profiler/rust-api/src/marker/deserializer_tags_state.rs b/tools/profiler/rust-api/src/marker/deserializer_tags_state.rs new file mode 100644 index 0000000000..890cc3f263 --- /dev/null +++ b/tools/profiler/rust-api/src/marker/deserializer_tags_state.rs @@ -0,0 +1,116 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::json_writer::JSONWriter; +use crate::marker::schema::MarkerSchema; +use crate::marker::{transmute_and_stream, ProfilerMarker}; +use std::collections::HashMap; +use std::sync::{RwLock, RwLockReadGuard}; + +lazy_static! { + static ref DESERIALIZER_TAGS_STATE: RwLock<DeserializerTagsState> = + RwLock::new(DeserializerTagsState::new()); +} + +/// A state that keeps track of each marker types and their deserializer tags. +/// They are added during the marker insertion and read during the marker serialization. +pub struct DeserializerTagsState { + /// C++ side accepts only u8 values, but we only know usize values as the + /// unique marker type values. So, we need to keep track of each + /// "marker tag -> deserializer tag" conversions to directly get the + /// deserializer tags of the already added marker types. + pub marker_tag_to_deserializer_tag: HashMap<usize, u8>, + /// Vector of marker type functions. + /// 1-based, i.e.: [0] -> tag 1. Elements are pushed to the end of the vector + /// whenever a new marker type is used in a Firefox session; the content is + /// kept between profiler runs in that session. On the C++ side, we have the + /// same algorithm (althought it's a sized array). See `sMarkerTypeFunctions1Based`. + pub marker_type_functions_1_based: Vec<MarkerTypeFunctions>, +} + +/// Functions that will be stored per marker type, so we can serialize the marker +/// schema and stream the marker payload for a specific type. +pub struct MarkerTypeFunctions { + /// A function that returns the name of the marker type. + pub marker_type_name_fn: fn() -> &'static str, + /// A function that returns a `MarkerSchema`, which contains all the + /// information needed to stream the display schema associated with a + /// marker type. + pub marker_type_display_fn: fn() -> MarkerSchema, + /// A function that can read a serialized payload from bytes and streams it + /// as JSON object properties. + pub transmute_and_stream_fn: + unsafe fn(payload: *const u8, payload_size: usize, json_writer: &mut JSONWriter), +} + +impl DeserializerTagsState { + fn new() -> Self { + DeserializerTagsState { + marker_tag_to_deserializer_tag: HashMap::new(), + marker_type_functions_1_based: vec![], + } + } +} + +/// Get or insert the deserializer tag for each marker type. The tag storage +/// is limited to 255 marker types. This is the same with the C++ side. It's +/// unlikely to reach to this limit, but if that's the case, C++ side needs +/// to change the uint8_t type for the deserializer tag as well. +pub fn get_or_insert_deserializer_tag<T>() -> u8 +where + T: ProfilerMarker, +{ + let unique_marker_tag = &T::marker_type_name as *const _ as usize; + let mut state = DESERIALIZER_TAGS_STATE.write().unwrap(); + + match state.marker_tag_to_deserializer_tag.get(&unique_marker_tag) { + None => { + // It's impossible to have length more than u8. + let deserializer_tag = state.marker_type_functions_1_based.len() as u8 + 1; + debug_assert!( + deserializer_tag < 250, + "Too many rust marker payload types! Please consider increasing the profiler \ + buffer tag size." + ); + + state + .marker_tag_to_deserializer_tag + .insert(unique_marker_tag, deserializer_tag); + state + .marker_type_functions_1_based + .push(MarkerTypeFunctions { + marker_type_name_fn: T::marker_type_name, + marker_type_display_fn: T::marker_type_display, + transmute_and_stream_fn: transmute_and_stream::<T>, + }); + deserializer_tag + } + Some(deserializer_tag) => *deserializer_tag, + } +} + +/// A guard that will be used by the marker FFI functions for getting marker type functions. +pub struct MarkerTypeFunctionsReadGuard { + guard: RwLockReadGuard<'static, DeserializerTagsState>, +} + +impl MarkerTypeFunctionsReadGuard { + pub fn iter<'a>(&'a self) -> impl Iterator<Item = &'a MarkerTypeFunctions> { + self.guard.marker_type_functions_1_based.iter() + } + + pub fn get<'a>(&'a self, deserializer_tag: u8) -> &'a MarkerTypeFunctions { + self.guard + .marker_type_functions_1_based + .get(deserializer_tag as usize - 1) + .expect("Failed to find the marker type functions for given deserializer tag") + } +} + +/// Locks the DESERIALIZER_TAGS_STATE and returns the marker type functions read guard. +pub fn get_marker_type_functions_read_guard() -> MarkerTypeFunctionsReadGuard { + MarkerTypeFunctionsReadGuard { + guard: DESERIALIZER_TAGS_STATE.read().unwrap(), + } +} diff --git a/tools/profiler/rust-api/src/marker/mod.rs b/tools/profiler/rust-api/src/marker/mod.rs new file mode 100644 index 0000000000..1981c0a322 --- /dev/null +++ b/tools/profiler/rust-api/src/marker/mod.rs @@ -0,0 +1,283 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! ## Gecko profiler marker support +//! +//! This marker API has a few different functions that you can use to mark a part of your code. +//! There are three main marker functions to use from Rust: [`add_untyped_marker`], +//! [`add_text_marker`] and [`add_marker`]. They are similar to what we have on +//! the C++ side. Please take a look at the marker documentation in the Firefox +//! source docs to learn more about them: +//! https://firefox-source-docs.mozilla.org/tools/profiler/markers-guide.html +//! +//! ### Simple marker without any additional data +//! +//! The simplest way to add a marker without any additional information is the +//! [`add_untyped_marker`] API. You can use it to mark a part of the code with +//! only a name. E.g.: +//! +//! ``` +//! gecko_profiler::add_untyped_marker( +//! // Name of the marker as a string. +//! "Marker Name", +//! // Category with an optional sub-category. +//! gecko_profiler_category!(Graphics, DisplayListBuilding), +//! // MarkerOptions that keeps options like marker timing and marker stack. +//! Default::default(), +//! ); +//! ``` +//! +//! Please see the [`gecko_profiler_category!`], [`MarkerOptions`],[`MarkerTiming`] +//! and [`MarkerStack`] to learn more about these. +//! +//! You can also give explicit [`MarkerOptions`] value like these: +//! +//! ``` +//! // With both timing and stack fields: +//! MarkerOptions { timing: MarkerTiming::instant_now(), stack: MarkerStack::Full } +//! // Or with some fields as default: +//! MarkerOptions { timing: MarkerTiming::instant_now(), ..Default::default() } +//! ``` +//! +//! ### Marker with only an additional text for more information: +//! +//! The next and slightly more advanced API is [`add_text_marker`]. +//! This is used to add a marker name + a string value for extra information. +//! E.g.: +//! +//! ``` +//! let info = "info about this marker"; +//! ... +//! gecko_profiler::add_text_marker( +//! // Name of the marker as a string. +//! "Marker Name", +//! // Category with an optional sub-category. +//! gecko_profiler_category!(DOM), +//! // MarkerOptions that keeps options like marker timing and marker stack. +//! MarkerOptions { +//! timing: MarkerTiming::instant_now(), +//! ..Default::default() +//! }, +//! // Additional information as a string. +//! info, +//! ); +//! ``` +//! +//! ### Marker with a more complex payload and different visualization in the profiler front-end. +//! +//! [`add_marker`] is the most advanced API that you can use to add different types +//! of values as data to your marker and customize the visualization of that marker +//! in the profiler front-end (profiler.firefox.com). +//! +//! To be able to add a a marker, first you need to create your marker payload +//! struct in your codebase and implement the [`ProfilerMarker`] trait like this: +//! +//! ``` +//! #[derive(Serialize, Deserialize, Debug)] +//! pub struct TestMarker { +//! a: u32, +//! b: String, +//! } +//! +//! // Please see the documentation of [`ProfilerMarker`]. +//! impl gecko_profiler::ProfilerMarker for TestMarker { +//! fn marker_type_name() -> &'static str { +//! "marker type from rust" +//! } +//! fn marker_type_display() -> gecko_profiler::MarkerSchema { +//! use gecko_profiler::marker::schema::*; +//! let mut schema = MarkerSchema::new(&[Location::MarkerChart]); +//! schema.set_chart_label("Name: {marker.name}"); +//! schema.set_tooltip_label("{marker.data.a}"); +//! schema.add_key_label_format("a", "A Value", Format::Integer); +//! schema.add_key_label_format("b", "B Value", Format::String); +//! schema +//! } +//! fn stream_json_marker_data(&self, json_writer: &mut gecko_profiler::JSONWriter) { +//! json_writer.int_property("a", self.a.into()); +//! json_writer.string_property("b", &self.b); +//! } +//! } +//! ``` +//! +//! Once you've created this payload and implemented the [`ProfilerMarker`], you +//! can now add this marker in the code that you would like to measure. E.g.: +//! +//! ``` +//! gecko_profiler::add_marker( +//! // Name of the marker as a string. +//! "Marker Name", +//! // Category with an optional sub-category. +//! gecko_profiler_category!(Graphics, DisplayListBuilding), +//! // MarkerOptions that keeps options like marker timing and marker stack. +//! Default::default(), +//! // Marker payload. +//! TestMarker {a: 12, b: "hello".to_owned()}, +//! ); +//! ``` + +pub(crate) mod deserializer_tags_state; +pub mod options; +pub mod schema; + +pub use options::*; +pub use schema::MarkerSchema; + +use crate::gecko_bindings::{bindings, profiling_categories::ProfilingCategoryPair}; +use crate::json_writer::JSONWriter; +use crate::marker::deserializer_tags_state::get_or_insert_deserializer_tag; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use std::os::raw::c_char; + +/// Marker API to add a new simple marker without any payload. +/// Please see the module documentation on how to add a marker with this API. +pub fn add_untyped_marker(name: &str, category: ProfilingCategoryPair, mut options: MarkerOptions) { + if !crate::profiler_state::can_accept_markers() { + // Nothing to do. + return; + } + + unsafe { + bindings::gecko_profiler_add_marker_untyped( + name.as_ptr() as *const c_char, + name.len(), + category.to_cpp_enum_value(), + options.timing.0.as_mut_ptr(), + options.stack, + ) + } +} + +/// Marker API to add a new marker with additional text for details. +/// Please see the module documentation on how to add a marker with this API. +pub fn add_text_marker( + name: &str, + category: ProfilingCategoryPair, + mut options: MarkerOptions, + text: &str, +) { + if !crate::profiler_state::can_accept_markers() { + // Nothing to do. + return; + } + + unsafe { + bindings::gecko_profiler_add_marker_text( + name.as_ptr() as *const c_char, + name.len(), + category.to_cpp_enum_value(), + options.timing.0.as_mut_ptr(), + options.stack, + text.as_ptr() as *const c_char, + text.len(), + ) + } +} + +/// Trait that every profiler marker payload struct needs to implement. +/// This will tell the profiler back-end how to serialize it as json and +/// the front-end how to display the marker. +/// Please also see the documentation here: +/// https://firefox-source-docs.mozilla.org/tools/profiler/markers-guide.html#how-to-define-new-marker-types +/// +/// - `marker_type_name`: Returns a static string as the marker type name. This +/// should be unique and it is used to keep track of the type of markers in the +/// profiler storage, and to identify them uniquely on the profiler front-end. +/// - `marker_type_display`: Where and how to display the marker and its data. +/// Returns a `MarkerSchema` object which will be forwarded to the profiler +/// front-end. +/// - `stream_json_marker_data`: Data specific to this marker type should be +/// serialized to JSON for the profiler front-end. All the common marker data +/// like marker name, category, timing will be serialized automatically. But +/// marker specific data should be serialized here. +pub trait ProfilerMarker: Serialize + DeserializeOwned { + /// A static method that returns the name of the marker type. + fn marker_type_name() -> &'static str; + /// A static method that returns a `MarkerSchema`, which contains all the + /// information needed to stream the display schema associated with a + /// marker type. + fn marker_type_display() -> MarkerSchema; + /// A method that streams the marker payload data as JSON object properties. + /// Please see the [JSONWriter] struct to see its methods. + fn stream_json_marker_data(&self, json_writer: &mut JSONWriter); +} + +/// A function that deserializes the marker payload and streams it to the JSON. +unsafe fn transmute_and_stream<T>( + payload: *const u8, + payload_size: usize, + json_writer: &mut JSONWriter, +) where + T: ProfilerMarker, +{ + let payload_slice = std::slice::from_raw_parts(payload, payload_size); + let payload: T = bincode::deserialize(&payload_slice).unwrap(); + payload.stream_json_marker_data(json_writer); +} + +/// Main marker API to add a new marker to profiler buffer. +/// Please see the module documentation on how to add a marker with this API. +pub fn add_marker<T>( + name: &str, + category: ProfilingCategoryPair, + mut options: MarkerOptions, + payload: T, +) where + T: ProfilerMarker, +{ + if !crate::profiler_state::can_accept_markers() { + // Nothing to do. + return; + } + + let encoded_payload: Vec<u8> = bincode::serialize(&payload).unwrap(); + let payload_size = encoded_payload.len(); + let maker_tag = get_or_insert_deserializer_tag::<T>(); + + unsafe { + bindings::gecko_profiler_add_marker( + name.as_ptr() as *const c_char, + name.len(), + category.to_cpp_enum_value(), + options.timing.0.as_mut_ptr(), + options.stack, + maker_tag, + encoded_payload.as_ptr(), + payload_size, + ) + } +} + +/// Tracing marker type for Rust code. +/// This must be kept in sync with the `mozilla::baseprofiler::markers::Tracing` +/// C++ counterpart. +#[derive(Serialize, Deserialize, Debug)] +pub struct Tracing(pub String); + +impl ProfilerMarker for Tracing { + fn marker_type_name() -> &'static str { + "tracing" + } + + fn stream_json_marker_data(&self, json_writer: &mut JSONWriter) { + if self.0.len() != 0 { + json_writer.string_property("category", &self.0); + } + } + + // Tracing marker is a bit special because we have the same schema in the + // C++ side. This function will only get called when no Tracing markers are + // generated from the C++ side. But, most of the time, this will not be called + // when there is another C++ Tracing marker. + fn marker_type_display() -> MarkerSchema { + use crate::marker::schema::*; + let mut schema = MarkerSchema::new(&[ + Location::MarkerChart, + Location::MarkerTable, + Location::TimelineOverview, + ]); + schema.add_key_label_format("category", "Type", Format::String); + schema + } +} diff --git a/tools/profiler/rust-api/src/marker/options.rs b/tools/profiler/rust-api/src/marker/options.rs new file mode 100644 index 0000000000..a5d4e11094 --- /dev/null +++ b/tools/profiler/rust-api/src/marker/options.rs @@ -0,0 +1,138 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Different options for the marker API. +//! See [`MarkerOptions`] and its fields. + +use crate::gecko_bindings::{bindings, structs::mozilla}; +use crate::ProfilerTime; +use std::mem::MaybeUninit; + +/// Marker option that contains marker timing information. +/// This class encapsulates the logic for correctly storing a marker based on its +/// constructor types. Use the static methods to create the MarkerTiming. This is +/// a transient object that is being used to enforce the constraints of the +/// combinations of the data. +/// +/// Implementation details: This is a RAII object that constructs and destroys a +/// C++ MarkerTiming object pointed to a specified reference. It allocates the +/// marker timing on stack and it's safe to move around because it's a +/// trivially-copyable object that only contains a few numbers. +#[derive(Debug)] +pub struct MarkerTiming(pub(crate) MaybeUninit<mozilla::MarkerTiming>); + +impl MarkerTiming { + /// Instant marker timing at a specific time. + pub fn instant_at(time: ProfilerTime) -> MarkerTiming { + let mut marker_timing = MaybeUninit::<mozilla::MarkerTiming>::uninit(); + unsafe { + bindings::gecko_profiler_construct_marker_timing_instant_at( + marker_timing.as_mut_ptr(), + &time.0, + ); + } + MarkerTiming(marker_timing) + } + + /// Instant marker timing at this time. + pub fn instant_now() -> MarkerTiming { + let mut marker_timing = MaybeUninit::<mozilla::MarkerTiming>::uninit(); + unsafe { + bindings::gecko_profiler_construct_marker_timing_instant_now( + marker_timing.as_mut_ptr(), + ); + } + MarkerTiming(marker_timing) + } + + /// Interval marker timing with start and end times. + pub fn interval(start_time: ProfilerTime, end_time: ProfilerTime) -> MarkerTiming { + let mut marker_timing = MaybeUninit::<mozilla::MarkerTiming>::uninit(); + unsafe { + bindings::gecko_profiler_construct_marker_timing_interval( + marker_timing.as_mut_ptr(), + &start_time.0, + &end_time.0, + ); + } + MarkerTiming(marker_timing) + } + + /// Interval marker with a start time and end time as "now". + pub fn interval_until_now_from(start_time: ProfilerTime) -> MarkerTiming { + let mut marker_timing = MaybeUninit::<mozilla::MarkerTiming>::uninit(); + unsafe { + bindings::gecko_profiler_construct_marker_timing_interval_until_now_from( + marker_timing.as_mut_ptr(), + &start_time.0, + ); + } + MarkerTiming(marker_timing) + } + + /// Interval start marker with only start time. This is a partial marker and + /// it requires another marker with `instant_end` to be complete. + pub fn interval_start(time: ProfilerTime) -> MarkerTiming { + let mut marker_timing = MaybeUninit::<mozilla::MarkerTiming>::uninit(); + unsafe { + bindings::gecko_profiler_construct_marker_timing_interval_start( + marker_timing.as_mut_ptr(), + &time.0, + ); + } + MarkerTiming(marker_timing) + } + + /// Interval end marker with only end time. This is a partial marker and + /// it requires another marker with `interval_start` to be complete. + pub fn interval_end(time: ProfilerTime) -> MarkerTiming { + let mut marker_timing = MaybeUninit::<mozilla::MarkerTiming>::uninit(); + unsafe { + bindings::gecko_profiler_construct_marker_timing_interval_end( + marker_timing.as_mut_ptr(), + &time.0, + ); + } + MarkerTiming(marker_timing) + } +} + +impl Default for MarkerTiming { + fn default() -> Self { + MarkerTiming::instant_now() + } +} + +impl Drop for MarkerTiming { + fn drop(&mut self) { + unsafe { + bindings::gecko_profiler_destruct_marker_timing(self.0.as_mut_ptr()); + } + } +} + +/// Marker option that contains marker stack information. +pub type MarkerStack = mozilla::StackCaptureOptions; + +impl Default for MarkerStack { + fn default() -> Self { + MarkerStack::NoStack + } +} + +/// This class combines each of the possible marker options above. +/// Use Default::default() for the options that you don't want to provide or the +/// options you want to leave as default. Example usage: +/// +/// ```rust +/// MarkerOptions { +/// timing: MarkerTiming::instant_now(), +/// ..Default::default() +/// } +/// ``` +#[derive(Debug, Default)] +pub struct MarkerOptions { + pub timing: MarkerTiming, + pub stack: MarkerStack, +} diff --git a/tools/profiler/rust-api/src/marker/schema.rs b/tools/profiler/rust-api/src/marker/schema.rs new file mode 100644 index 0000000000..9368582f11 --- /dev/null +++ b/tools/profiler/rust-api/src/marker/schema.rs @@ -0,0 +1,233 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! [`MarkerSchema`] and other enums that will be used by `MarkerSchema`. + +use crate::gecko_bindings::{bindings, structs::mozilla}; +use std::mem::MaybeUninit; +use std::ops::DerefMut; +use std::os::raw::c_char; +use std::pin::Pin; + +/// Marker locations to be displayed in the profiler front-end. +pub type Location = mozilla::MarkerSchema_Location; + +/// Formats of marker properties for profiler front-end. +pub type Format = mozilla::MarkerSchema_Format; + +/// Whether it's searchable or not in the profiler front-end. +pub type Searchable = mozilla::MarkerSchema_Searchable; + +/// This object collects all the information necessary to stream the JSON schema +/// that informs the front-end how to display a type of markers. +/// It will be created and populated in `marker_type_display()` functions in each +/// marker type definition, see add/set functions. +/// +/// It's a RAII object that constructs and destroys a C++ MarkerSchema object +/// pointed to a specified reference. +pub struct MarkerSchema { + pub(crate) pin: Pin<Box<MaybeUninit<mozilla::MarkerSchema>>>, +} + +impl MarkerSchema { + // Initialize a marker schema with the given `Location`s. + pub fn new(locations: &[Location]) -> Self { + let mut marker_schema = Box::pin(std::mem::MaybeUninit::<mozilla::MarkerSchema>::uninit()); + + unsafe { + bindings::gecko_profiler_construct_marker_schema( + marker_schema.deref_mut().as_mut_ptr(), + locations.as_ptr(), + locations.len(), + ); + } + MarkerSchema { pin: marker_schema } + } + + /// Marker schema for types that have special frontend handling. + /// Nothing else should be set in this case. + pub fn new_with_special_frontend_location() -> Self { + let mut marker_schema = Box::pin(std::mem::MaybeUninit::<mozilla::MarkerSchema>::uninit()); + unsafe { + bindings::gecko_profiler_construct_marker_schema_with_special_front_end_location( + marker_schema.deref_mut().as_mut_ptr(), + ); + } + MarkerSchema { pin: marker_schema } + } + + /// Optional label in the marker chart. + /// If not provided, the marker "name" will be used. The given string + /// can contain element keys in braces to include data elements streamed by + /// `stream_json_marker_data()`. E.g.: "This is {marker.data.text}" + pub fn set_chart_label(&mut self, label: &str) -> &mut Self { + unsafe { + bindings::gecko_profiler_marker_schema_set_chart_label( + self.pin.deref_mut().as_mut_ptr(), + label.as_ptr() as *const c_char, + label.len(), + ); + } + self + } + + /// Optional label in the marker chart tooltip. + /// If not provided, the marker "name" will be used. The given string + /// can contain element keys in braces to include data elements streamed by + /// `stream_json_marker_data()`. E.g.: "This is {marker.data.text}" + pub fn set_tooltip_label(&mut self, label: &str) -> &mut Self { + unsafe { + bindings::gecko_profiler_marker_schema_set_tooltip_label( + self.pin.deref_mut().as_mut_ptr(), + label.as_ptr() as *const c_char, + label.len(), + ); + } + self + } + + /// Optional label in the marker table. + /// If not provided, the marker "name" will be used. The given string + /// can contain element keys in braces to include data elements streamed by + /// `stream_json_marker_data()`. E.g.: "This is {marker.data.text}" + pub fn set_table_label(&mut self, label: &str) -> &mut Self { + unsafe { + bindings::gecko_profiler_marker_schema_set_table_label( + self.pin.deref_mut().as_mut_ptr(), + label.as_ptr() as *const c_char, + label.len(), + ); + } + self + } + + /// Set all marker chart / marker tooltip / marker table labels with the same text. + /// Same as the individual methods, the given string can contain element keys + /// in braces to include data elements streamed by `stream_json_marker_data()`. + /// E.g.: "This is {marker.data.text}" + pub fn set_all_labels(&mut self, label: &str) -> &mut Self { + unsafe { + bindings::gecko_profiler_marker_schema_set_all_labels( + self.pin.deref_mut().as_mut_ptr(), + label.as_ptr() as *const c_char, + label.len(), + ); + } + self + } + + // Each data element that is streamed by `stream_json_marker_data()` can be + // displayed as indicated by using one of the `add_...` function below. + // Each `add...` will add a line in the full marker description. Parameters: + // - `key`: Element property name as streamed by `stream_json_marker_data()`. + // - `label`: Optional label. Defaults to the key name. + // - `format`: How to format the data element value, see `Format` above. + // - `searchable`: Optional, indicates if the value is used in searches, + // defaults to false. + + /// Add a key / format row for the marker data element. + /// - `key`: Element property name as streamed by `stream_json_marker_data()`. + /// - `format`: How to format the data element value, see `Format` above. + pub fn add_key_format(&mut self, key: &str, format: Format) -> &mut Self { + unsafe { + bindings::gecko_profiler_marker_schema_add_key_format( + self.pin.deref_mut().as_mut_ptr(), + key.as_ptr() as *const c_char, + key.len(), + format, + ); + } + self + } + + /// Add a key / label / format row for the marker data element. + /// - `key`: Element property name as streamed by `stream_json_marker_data()`. + /// - `label`: Optional label. Defaults to the key name. + /// - `format`: How to format the data element value, see `Format` above. + pub fn add_key_label_format(&mut self, key: &str, label: &str, format: Format) -> &mut Self { + unsafe { + bindings::gecko_profiler_marker_schema_add_key_label_format( + self.pin.deref_mut().as_mut_ptr(), + key.as_ptr() as *const c_char, + key.len(), + label.as_ptr() as *const c_char, + label.len(), + format, + ); + } + self + } + + /// Add a key / format / searchable row for the marker data element. + /// - `key`: Element property name as streamed by `stream_json_marker_data()`. + /// - `format`: How to format the data element value, see `Format` above. + pub fn add_key_format_searchable( + &mut self, + key: &str, + format: Format, + searchable: Searchable, + ) -> &mut Self { + unsafe { + bindings::gecko_profiler_marker_schema_add_key_format_searchable( + self.pin.deref_mut().as_mut_ptr(), + key.as_ptr() as *const c_char, + key.len(), + format, + searchable, + ); + } + self + } + + /// Add a key / label / format / searchable row for the marker data element. + /// - `key`: Element property name as streamed by `stream_json_marker_data()`. + /// - `label`: Optional label. Defaults to the key name. + /// - `format`: How to format the data element value, see `Format` above. + /// - `searchable`: Optional, indicates if the value is used in searches, + /// defaults to false. + pub fn add_key_label_format_searchable( + &mut self, + key: &str, + label: &str, + format: Format, + searchable: Searchable, + ) -> &mut Self { + unsafe { + bindings::gecko_profiler_marker_schema_add_key_label_format_searchable( + self.pin.deref_mut().as_mut_ptr(), + key.as_ptr() as *const c_char, + key.len(), + label.as_ptr() as *const c_char, + label.len(), + format, + searchable, + ); + } + self + } + + /// Add a key / value static row. + /// - `key`: Element property name as streamed by `stream_json_marker_data()`. + /// - `value`: Static value to display. + pub fn add_static_label_value(&mut self, label: &str, value: &str) -> &mut Self { + unsafe { + bindings::gecko_profiler_marker_schema_add_static_label_value( + self.pin.deref_mut().as_mut_ptr(), + label.as_ptr() as *const c_char, + label.len(), + value.as_ptr() as *const c_char, + value.len(), + ); + } + self + } +} + +impl Drop for MarkerSchema { + fn drop(&mut self) { + unsafe { + bindings::gecko_profiler_destruct_marker_schema(self.pin.deref_mut().as_mut_ptr()); + } + } +} diff --git a/tools/profiler/rust-api/src/profiler_state.rs b/tools/profiler/rust-api/src/profiler_state.rs new file mode 100644 index 0000000000..c4858db22c --- /dev/null +++ b/tools/profiler/rust-api/src/profiler_state.rs @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Gecko profiler state. + +/// Whether the Gecko profiler is currently active. +/// A typical use of this API: +/// ```rust +/// if gecko_profiler::is_active() { +/// // do something. +/// } +/// ``` +/// +/// This implementation must be kept in sync with +/// `mozilla::profiler::detail::RacyFeatures::IsActive`. +#[cfg(feature = "enabled")] +#[inline] +pub fn is_active() -> bool { + use crate::gecko_bindings::structs::mozilla::profiler::detail; + + let active_and_features = get_active_and_features(); + (active_and_features & detail::RacyFeatures_Active) != 0 +} + +/// Always false when MOZ_GECKO_PROFILER is not defined. +#[cfg(not(feature = "enabled"))] +#[inline] +pub fn is_active() -> bool { + false +} + +/// Whether the Gecko Profiler can accept markers. +/// Similar to `is_active`, but with some extra checks that determine if the +/// profiler would currently store markers. So this should be used before +/// doing some potentially-expensive work that's used in a marker. E.g.: +/// +/// ```rust +/// if gecko_profiler::can_accept_markers() { +/// // Do something expensive and add the marker with that data. +/// } +/// ``` +/// +/// This implementation must be kept in sync with +/// `mozilla::profiler::detail::RacyFeatures::IsActiveAndUnpaused`. +#[cfg(feature = "enabled")] +#[inline] +pub fn can_accept_markers() -> bool { + use crate::gecko_bindings::structs::mozilla::profiler::detail; + + let active_and_features = get_active_and_features(); + (active_and_features & detail::RacyFeatures_Active) != 0 + && (active_and_features & detail::RacyFeatures_Paused) == 0 +} + +/// Always false when MOZ_GECKO_PROFILER is not defined. +#[cfg(not(feature = "enabled"))] +#[inline] +pub fn can_accept_markers() -> bool { + false +} + +/// Returns the value of atomic `RacyFeatures::sActiveAndFeatures` from the C++ side. +#[cfg(feature = "enabled")] +#[inline] +fn get_active_and_features() -> u32 { + use crate::gecko_bindings::structs::mozilla::profiler::detail; + use std::sync::atomic::{AtomicU32, Ordering}; + + // This is reaching for the C++ atomic value instead of calling an FFI + // function to return this value. Because, calling an FFI function is much + // more expensive compared to this method. That's why it's worth to go with + // this solution for performance. But it's crucial to keep the implementation + // of this and the callers in sync with the C++ counterparts. + let active_and_features: &AtomicU32 = unsafe { + let ptr: *const u32 = std::ptr::addr_of!(detail::RacyFeatures_sActiveAndFeatures); + // TODO: Switch this to use `AtomicU32::from_ptr` once our Rust MSRV is at least 1.75.0 + &*ptr.cast() + }; + active_and_features.load(Ordering::Relaxed) +} diff --git a/tools/profiler/rust-api/src/thread.rs b/tools/profiler/rust-api/src/thread.rs new file mode 100644 index 0000000000..353469a4bb --- /dev/null +++ b/tools/profiler/rust-api/src/thread.rs @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +///! Profiler API for thread registration and unregistration. +use crate::gecko_bindings::bindings; +use std::ffi::CString; + +/// Register a thread with the Gecko Profiler. +pub fn register_thread(thread_name: &str) { + let name = CString::new(thread_name).unwrap(); + unsafe { + // gecko_profiler_register_thread copies the passed name here. + bindings::gecko_profiler_register_thread(name.as_ptr()); + } +} + +/// Unregister a thread with the Gecko Profiler. +pub fn unregister_thread() { + unsafe { + bindings::gecko_profiler_unregister_thread(); + } +} diff --git a/tools/profiler/rust-api/src/time.rs b/tools/profiler/rust-api/src/time.rs new file mode 100644 index 0000000000..56315690c9 --- /dev/null +++ b/tools/profiler/rust-api/src/time.rs @@ -0,0 +1,71 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Gecko profiler time. + +use crate::gecko_bindings::{bindings, structs::mozilla}; +use std::mem::MaybeUninit; + +/// Profiler time for the marker API. +/// This should be used as the `MarkerTiming` parameter. +/// E.g.: +/// +/// ``` +/// let start = ProfilerTime::now(); +/// // ...some code... +/// gecko_profiler::add_untyped_marker( +/// "marker name", +/// category, +/// MarkerOptions { +/// timing: MarkerTiming::interval_until_now_from(start), +/// ..Default::default() +/// }, +/// ); +/// ``` +#[derive(Debug)] +pub struct ProfilerTime(pub(crate) mozilla::TimeStamp); + +impl ProfilerTime { + pub fn now() -> ProfilerTime { + let mut marker_timing = MaybeUninit::<mozilla::TimeStamp>::uninit(); + unsafe { + bindings::gecko_profiler_construct_timestamp_now(marker_timing.as_mut_ptr()); + ProfilerTime(marker_timing.assume_init()) + } + } + + pub fn add_microseconds(self, microseconds: f64) -> Self { + let mut dest = MaybeUninit::<mozilla::TimeStamp>::uninit(); + unsafe { + bindings::gecko_profiler_add_timestamp(&self.0, dest.as_mut_ptr(), microseconds); + ProfilerTime(dest.assume_init()) + } + } + + pub fn subtract_microseconds(self, microseconds: f64) -> Self { + let mut dest = MaybeUninit::<mozilla::TimeStamp>::uninit(); + unsafe { + bindings::gecko_profiler_subtract_timestamp(&self.0, dest.as_mut_ptr(), microseconds); + ProfilerTime(dest.assume_init()) + } + } +} + +impl Clone for ProfilerTime { + fn clone(&self) -> Self { + let mut dest = MaybeUninit::<mozilla::TimeStamp>::uninit(); + unsafe { + bindings::gecko_profiler_clone_timestamp(&self.0, dest.as_mut_ptr()); + ProfilerTime(dest.assume_init()) + } + } +} + +impl Drop for ProfilerTime { + fn drop(&mut self) { + unsafe { + bindings::gecko_profiler_destruct_timestamp(&mut self.0); + } + } +} diff --git a/tools/profiler/rust-helper/Cargo.toml b/tools/profiler/rust-helper/Cargo.toml new file mode 100644 index 0000000000..fa02796fb2 --- /dev/null +++ b/tools/profiler/rust-helper/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "profiler_helper" +version = "0.1.0" +authors = ["Markus Stange <mstange@themasta.com>"] +license = "MPL-2.0" + +[dependencies] +memmap2 = "0.9" +rustc-demangle = "0.1" +uuid = "1.0" + +[dependencies.object] +version = "0.32.0" +optional = true +default-features = false +features = ["std", "read_core", "elf"] + +[dependencies.thin-vec] +version = "0.2.1" +features = ["gecko-ffi"] + +[features] +parse_elf = ["object"] diff --git a/tools/profiler/rust-helper/src/compact_symbol_table.rs b/tools/profiler/rust-helper/src/compact_symbol_table.rs new file mode 100644 index 0000000000..12c4ca081b --- /dev/null +++ b/tools/profiler/rust-helper/src/compact_symbol_table.rs @@ -0,0 +1,40 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::collections::HashMap; +use thin_vec::ThinVec; + +#[repr(C)] +pub struct CompactSymbolTable { + pub addr: ThinVec<u32>, + pub index: ThinVec<u32>, + pub buffer: ThinVec<u8>, +} + +impl CompactSymbolTable { + pub fn new() -> Self { + Self { + addr: ThinVec::new(), + index: ThinVec::new(), + buffer: ThinVec::new(), + } + } + + pub fn from_map(map: HashMap<u32, &str>) -> Self { + let mut table = Self::new(); + let mut entries: Vec<_> = map.into_iter().collect(); + entries.sort_by_key(|&(addr, _)| addr); + for (addr, name) in entries { + table.addr.push(addr); + table.index.push(table.buffer.len() as u32); + table.add_name(name); + } + table.index.push(table.buffer.len() as u32); + table + } + + fn add_name(&mut self, name: &str) { + self.buffer.extend_from_slice(name.as_bytes()); + } +} diff --git a/tools/profiler/rust-helper/src/elf.rs b/tools/profiler/rust-helper/src/elf.rs new file mode 100644 index 0000000000..4930884f05 --- /dev/null +++ b/tools/profiler/rust-helper/src/elf.rs @@ -0,0 +1,101 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use compact_symbol_table::CompactSymbolTable; +use object::read::{NativeFile, Object}; +use object::{ObjectSection, ObjectSymbol, SectionKind, SymbolKind}; +use std::cmp; +use std::collections::HashMap; +use uuid::Uuid; + +const UUID_SIZE: usize = 16; +const PAGE_SIZE: usize = 4096; + +fn get_symbol_map<'a: 'b, 'b, T>(object_file: &'b T) -> HashMap<u32, &'a str> +where + T: Object<'a, 'b>, +{ + object_file + .dynamic_symbols() + .chain(object_file.symbols()) + .filter(|symbol| symbol.kind() == SymbolKind::Text) + .filter_map(|symbol| { + symbol + .name() + .map(|name| (symbol.address() as u32, name)) + .ok() + }) + .collect() +} + +pub fn get_compact_symbol_table( + buffer: &[u8], + breakpad_id: Option<&str>, +) -> Option<CompactSymbolTable> { + let elf_file = NativeFile::parse(buffer).ok()?; + let elf_id = get_elf_id(&elf_file)?; + if !breakpad_id.map_or(true, |id| id == format!("{:X}0", elf_id.as_simple())) { + return None; + } + return Some(CompactSymbolTable::from_map(get_symbol_map(&elf_file))); +} + +fn create_elf_id(identifier: &[u8], little_endian: bool) -> Uuid { + // Make sure that we have exactly UUID_SIZE bytes available + let mut data = [0 as u8; UUID_SIZE]; + let len = cmp::min(identifier.len(), UUID_SIZE); + data[0..len].copy_from_slice(&identifier[0..len]); + + if little_endian { + // The file ELF file targets a little endian architecture. Convert to + // network byte order (big endian) to match the Breakpad processor's + // expectations. For big endian object files, this is not needed. + data[0..4].reverse(); // uuid field 1 + data[4..6].reverse(); // uuid field 2 + data[6..8].reverse(); // uuid field 3 + } + + Uuid::from_bytes(data) +} + +/// Tries to obtain the object identifier of an ELF object. +/// +/// As opposed to Mach-O, ELF does not specify a unique ID for object files in +/// its header. Compilers and linkers usually add either `SHT_NOTE` sections or +/// `PT_NOTE` program header elements for this purpose. If one of these notes +/// is present, ElfFile's build_id() method will find it. +/// +/// If neither of the above are present, this function will hash the first page +/// of the `.text` section (program code). This matches what the Breakpad +/// processor does. +/// +/// If all of the above fails, this function will return `None`. +pub fn get_elf_id(elf_file: &NativeFile) -> Option<Uuid> { + if let Ok(Some(identifier)) = elf_file.build_id() { + return Some(create_elf_id(identifier, elf_file.is_little_endian())); + } + + // We were not able to locate the build ID, so fall back to hashing the + // first page of the ".text" (program code) section. This algorithm XORs + // 16-byte chunks directly into a UUID buffer. + if let Some(section_data) = find_text_section(elf_file) { + let mut hash = [0; UUID_SIZE]; + for i in 0..cmp::min(section_data.len(), PAGE_SIZE) { + hash[i % UUID_SIZE] ^= section_data[i]; + } + return Some(create_elf_id(&hash, elf_file.is_little_endian())); + } + + None +} + +/// Returns a reference to the data of the the .text section in an ELF binary. +fn find_text_section<'elf>(elf_file: &'elf NativeFile) -> Option<&'elf [u8]> { + if let Some(section) = elf_file.section_by_name(".text") { + if section.kind() == SectionKind::Text { + return section.data().ok(); + } + } + None +} diff --git a/tools/profiler/rust-helper/src/lib.rs b/tools/profiler/rust-helper/src/lib.rs new file mode 100644 index 0000000000..22f8e04a2e --- /dev/null +++ b/tools/profiler/rust-helper/src/lib.rs @@ -0,0 +1,107 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +extern crate memmap2; +extern crate rustc_demangle; +extern crate thin_vec; +extern crate uuid; + +#[cfg(feature = "parse_elf")] +extern crate object; + +mod compact_symbol_table; + +#[cfg(feature = "parse_elf")] +mod elf; + +#[cfg(feature = "parse_elf")] +use memmap2::MmapOptions; +#[cfg(feature = "parse_elf")] +use std::fs::File; + +use compact_symbol_table::CompactSymbolTable; +use rustc_demangle::try_demangle; +use std::ffi::CStr; +use std::mem; +use std::os::raw::c_char; +use std::ptr; + +#[cfg(feature = "parse_elf")] +pub fn get_compact_symbol_table_from_file( + debug_path: &str, + breakpad_id: Option<&str>, +) -> Option<CompactSymbolTable> { + let file = File::open(debug_path).ok()?; + let buffer = unsafe { MmapOptions::new().map(&file).ok()? }; + elf::get_compact_symbol_table(&buffer, breakpad_id) +} + +#[cfg(not(feature = "parse_elf"))] +pub fn get_compact_symbol_table_from_file( + _debug_path: &str, + _breakpad_id: Option<&str>, +) -> Option<CompactSymbolTable> { + None +} + +#[no_mangle] +pub unsafe extern "C" fn profiler_get_symbol_table( + debug_path: *const c_char, + breakpad_id: *const c_char, + symbol_table: &mut CompactSymbolTable, +) -> bool { + let debug_path = CStr::from_ptr(debug_path).to_string_lossy(); + let breakpad_id = if breakpad_id.is_null() { + None + } else { + match CStr::from_ptr(breakpad_id).to_str() { + Ok(s) => Some(s), + Err(_) => return false, + } + }; + + match get_compact_symbol_table_from_file(&debug_path, breakpad_id.map(|id| id.as_ref())) { + Some(mut st) => { + std::mem::swap(symbol_table, &mut st); + true + } + None => false, + } +} + +#[no_mangle] +pub unsafe extern "C" fn profiler_demangle_rust( + mangled: *const c_char, + buffer: *mut c_char, + buffer_len: usize, +) -> bool { + assert!(!mangled.is_null()); + assert!(!buffer.is_null()); + + if buffer_len == 0 { + return false; + } + + let buffer: *mut u8 = mem::transmute(buffer); + let mangled = match CStr::from_ptr(mangled).to_str() { + Ok(s) => s, + Err(_) => return false, + }; + + match try_demangle(mangled) { + Ok(demangled) => { + let mut demangled = format!("{:#}", demangled); + if !demangled.is_ascii() { + return false; + } + demangled.truncate(buffer_len - 1); + + let bytes = demangled.as_bytes(); + ptr::copy(bytes.as_ptr(), buffer, bytes.len()); + ptr::write(buffer.offset(bytes.len() as isize), 0); + true + } + Err(_) => false, + } +} diff --git a/tools/profiler/tests/browser/browser.toml b/tools/profiler/tests/browser/browser.toml new file mode 100644 index 0000000000..57d8ad0875 --- /dev/null +++ b/tools/profiler/tests/browser/browser.toml @@ -0,0 +1,104 @@ +[DEFAULT] +skip-if = ["tsan"] # Bug 1804081 - TSan times out on pretty much all of these tests +support-files = [ + "../shared-head.js", + "head.js", +] + +["browser_test_feature_ipcmessages.js"] +support-files = ["simple.html"] + +["browser_test_feature_jsallocations.js"] +support-files = ["do_work_500ms.html"] + +["browser_test_feature_nostacksampling.js"] +support-files = ["do_work_500ms.html"] + +["browser_test_marker_network_cancel.js"] +https_first_disabled = true +support-files = ["simple.html"] + +["browser_test_marker_network_private_browsing.js"] +support-files = ["simple.html"] + +["browser_test_marker_network_redirect.js"] +https_first_disabled = true +support-files = [ + "redirect.sjs", + "simple.html", + "page_with_resources.html", + "firefox-logo-nightly.svg", +] + +["browser_test_marker_network_serviceworker_cache_first.js"] +support-files = [ + "serviceworkers/serviceworker-utils.js", + "serviceworkers/serviceworker_register.html", + "serviceworkers/serviceworker_page.html", + "serviceworkers/firefox-logo-nightly.svg", + "serviceworkers/serviceworker_cache_first.js", +] + +["browser_test_marker_network_serviceworker_no_fetch_handler.js"] +support-files = [ + "serviceworkers/serviceworker-utils.js", + "serviceworkers/serviceworker_register.html", + "serviceworkers/serviceworker_page.html", + "serviceworkers/firefox-logo-nightly.svg", + "serviceworkers/serviceworker_no_fetch_handler.js", +] + +["browser_test_marker_network_serviceworker_no_respondWith_in_fetch_handler.js"] +support-files = [ + "serviceworkers/serviceworker-utils.js", + "serviceworkers/serviceworker_register.html", + "serviceworkers/serviceworker_page.html", + "serviceworkers/firefox-logo-nightly.svg", + "serviceworkers/serviceworker_no_respondWith_in_fetch_handler.js", +] + +["browser_test_marker_network_serviceworker_synthetized_response.js"] +support-files = [ + "serviceworkers/serviceworker-utils.js", + "serviceworkers/serviceworker_register.html", + "serviceworkers/serviceworker_simple.html", + "serviceworkers/firefox-logo-nightly.svg", + "serviceworkers/serviceworker_synthetized_response.js", +] + +["browser_test_marker_network_simple.js"] +https_first_disabled = true +support-files = ["simple.html"] + +["browser_test_marker_network_sts.js"] +support-files = ["simple.html"] + +["browser_test_markers_gc_cc.js"] + +["browser_test_markers_parent_process.js"] + +["browser_test_markers_preferencereads.js"] +support-files = ["single_frame.html"] + +["browser_test_profile_capture_by_pid.js"] +https_first_disabled = true +support-files = ["single_frame.html"] + +["browser_test_profile_fission.js"] +support-files = ["single_frame.html"] + +["browser_test_profile_multi_frame_page_info.js"] +https_first_disabled = true +support-files = [ + "multi_frame.html", + "single_frame.html", +] + +["browser_test_profile_single_frame_page_info.js"] +https_first_disabled = true +support-files = ["single_frame.html"] + +["browser_test_profile_slow_capture.js"] +https_first_disabled = true +support-files = ["single_frame.html"] +skip-if = ["!debug"] diff --git a/tools/profiler/tests/browser/browser_test_feature_ipcmessages.js b/tools/profiler/tests/browser/browser_test_feature_ipcmessages.js new file mode 100644 index 0000000000..f5fb2921a1 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_feature_ipcmessages.js @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +requestLongerTimeout(10); + +async function waitForLoad() { + return SpecialPowers.spawn(gBrowser.selectedBrowser, [], () => { + return new Promise(function (resolve) { + if (content.document.readyState !== "complete") { + content.document.addEventListener("readystatechange", () => { + if (content.document.readyState === "complete") { + resolve(); + } + }); + } else { + resolve(); + } + }); + }); +} + +/** + * Test the IPCMessages feature. + */ +add_task(async function test_profile_feature_ipcmessges() { + const url = BASE_URL + "simple.html"; + + info("Open a tab while profiling IPC messages."); + await startProfiler({ features: ["js", "ipcmessages"] }); + info("Started the profiler sucessfully! Now, let's open a tab."); + + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + info("We opened a tab!"); + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + info("Now let's wait until it's fully loaded."); + await waitForLoad(); + + info( + "Check that some IPC profile markers were generated when " + + "the feature is enabled." + ); + { + const { parentThread, contentThread } = + await waitSamplingAndStopProfilerAndGetThreads(contentPid); + + Assert.greater( + getPayloadsOfType(parentThread, "IPC").length, + 0, + "IPC profile markers were recorded for the parent process' main " + + "thread when the IPCMessages feature was turned on." + ); + + Assert.greater( + getPayloadsOfType(contentThread, "IPC").length, + 0, + "IPC profile markers were recorded for the content process' main " + + "thread when the IPCMessages feature was turned on." + ); + } + }); + + info("Now open a tab without profiling IPC messages."); + await startProfiler({ features: ["js"] }); + + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + await waitForLoad(); + + info( + "Check that no IPC profile markers were recorded when the " + + "feature is turned off." + ); + { + const { parentThread, contentThread } = + await waitSamplingAndStopProfilerAndGetThreads(contentPid); + Assert.equal( + getPayloadsOfType(parentThread, "IPC").length, + 0, + "No IPC profile markers were recorded for the parent process' main " + + "thread when the IPCMessages feature was turned off." + ); + + Assert.equal( + getPayloadsOfType(contentThread, "IPC").length, + 0, + "No IPC profile markers were recorded for the content process' main " + + "thread when the IPCMessages feature was turned off." + ); + } + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_feature_jsallocations.js b/tools/profiler/tests/browser/browser_test_feature_jsallocations.js new file mode 100644 index 0000000000..60d072bed9 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_feature_jsallocations.js @@ -0,0 +1,74 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +requestLongerTimeout(10); + +/** + * Test the JS Allocations feature. This is done as a browser test to ensure that + * we realistically try out how the JS allocations are running. This ensures that + * we are collecting allocations for the content process and the parent process. + */ +add_task(async function test_profile_feature_jsallocations() { + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + await startProfiler({ features: ["js", "jsallocations"] }); + + const url = BASE_URL + "do_work_500ms.html"; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + // Wait 500ms so that the tab finishes executing. + await wait(500); + + // Check that we can get some allocations when the feature is turned on. + { + const { parentThread, contentThread } = + await waitSamplingAndStopProfilerAndGetThreads(contentPid); + Assert.greater( + getPayloadsOfType(parentThread, "JS allocation").length, + 0, + "Allocations were recorded for the parent process' main thread when the " + + "JS Allocation feature was turned on." + ); + Assert.greater( + getPayloadsOfType(contentThread, "JS allocation").length, + 0, + "Allocations were recorded for the content process' main thread when the " + + "JS Allocation feature was turned on." + ); + } + + await startProfiler({ features: ["js"] }); + // Now reload the tab with a clean run. + gBrowser.reload(); + await wait(500); + + // Check that no allocations were recorded, and allocation tracking was correctly + // turned off. + { + const { parentThread, contentThread } = + await waitSamplingAndStopProfilerAndGetThreads(contentPid); + Assert.equal( + getPayloadsOfType(parentThread, "JS allocation").length, + 0, + "No allocations were recorded for the parent processes' main thread when " + + "JS allocation was not turned on." + ); + + Assert.equal( + getPayloadsOfType(contentThread, "JS allocation").length, + 0, + "No allocations were recorded for the content processes' main thread when " + + "JS allocation was not turned on." + ); + } + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_feature_nostacksampling.js b/tools/profiler/tests/browser/browser_test_feature_nostacksampling.js new file mode 100644 index 0000000000..323a87e191 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_feature_nostacksampling.js @@ -0,0 +1,72 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test the No Stack Sampling feature. + */ +add_task(async function test_profile_feature_nostacksampling() { + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + await startProfiler({ features: ["js", "nostacksampling"] }); + + const url = BASE_URL + "do_work_500ms.html"; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + // Wait 500ms so that the tab finishes executing. + await wait(500); + + // Check that we can get no stacks when the feature is turned on. + { + const { parentThread, contentThread } = + await stopProfilerNowAndGetThreads(contentPid); + Assert.equal( + parentThread.samples.data.length, + 0, + "Stack samples were recorded from the parent process' main thread" + + "when the No Stack Sampling feature was turned on." + ); + Assert.equal( + contentThread.samples.data.length, + 0, + "Stack samples were recorded from the content process' main thread" + + "when the No Stack Sampling feature was turned on." + ); + } + + // Flush out any straggling allocation markers that may have not been collected + // yet by starting and stopping the profiler once. + await startProfiler({ features: ["js"] }); + + // Now reload the tab with a clean run. + gBrowser.reload(); + await wait(500); + + // Check that stack samples were recorded. + { + const { parentThread, contentThread } = + await waitSamplingAndStopProfilerAndGetThreads(contentPid); + Assert.greater( + parentThread.samples.data.length, + 0, + "No Stack samples were recorded from the parent process' main thread" + + "when the No Stack Sampling feature was not turned on." + ); + + Assert.greater( + contentThread.samples.data.length, + 0, + "No Stack samples were recorded from the content process' main thread" + + "when the No Stack Sampling feature was not turned on." + ); + } + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_marker_network_cancel.js b/tools/profiler/tests/browser/browser_test_marker_network_cancel.js new file mode 100644 index 0000000000..df3de2b99a --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_marker_network_cancel.js @@ -0,0 +1,71 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we emit network markers with the cancel status. + */ +add_task(async function test_network_markers_early_cancel() { + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const url = BASE_URL + "simple.html?cacheBust=" + Math.random(); + const options = { + gBrowser, + url: "about:blank", + waitForLoad: false, + }; + + const tab = await BrowserTestUtils.openNewForegroundTab(options); + const loadPromise = BrowserTestUtils.waitForDocLoadAndStopIt(url, tab); + BrowserTestUtils.startLoadingURIString(tab.linkedBrowser, url); + const contentPid = await SpecialPowers.spawn( + tab.linkedBrowser, + [], + () => Services.appinfo.processID + ); + await loadPromise; + const { parentThread, contentThread } = await stopProfilerNowAndGetThreads( + contentPid + ); + BrowserTestUtils.removeTab(tab); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + + info("parent process: " + JSON.stringify(parentNetworkMarkers, null, 2)); + info("content process: " + JSON.stringify(contentNetworkMarkers, null, 2)); + + Assert.equal( + parentNetworkMarkers.length, + 2, + `We should get a pair of network markers in the parent thread.` + ); + + // We don't test the markers in the content process, because depending on some + // timing we can have 0 or 1 (and maybe even 2 (?)). + + const parentStopMarker = parentNetworkMarkers[1]; + + const expectedProperties = { + name: Expect.stringMatches(`Load \\d+:.*${escapeStringRegexp(url)}`), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_CANCEL", + URI: url, + requestMethod: "GET", + contentType: null, + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + cache: "Unresolved", + }), + }; + + Assert.objectContains(parentStopMarker, expectedProperties); +}); diff --git a/tools/profiler/tests/browser/browser_test_marker_network_private_browsing.js b/tools/profiler/tests/browser/browser_test_marker_network_private_browsing.js new file mode 100644 index 0000000000..85d312d217 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_marker_network_private_browsing.js @@ -0,0 +1,91 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we emit network markers accordingly + */ +add_task(async function test_network_markers() { + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const win = await BrowserTestUtils.openNewBrowserWindow({ + private: true, + fission: true, + }); + try { + const url = BASE_URL_HTTPS + "simple.html?cacheBust=" + Math.random(); + const contentBrowser = win.gBrowser.selectedBrowser; + BrowserTestUtils.startLoadingURIString(contentBrowser, url); + await BrowserTestUtils.browserLoaded(contentBrowser, false, url); + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + const { parentThread, contentThread } = await stopProfilerNowAndGetThreads( + contentPid + ); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + info(JSON.stringify(parentNetworkMarkers, null, 2)); + info(JSON.stringify(contentNetworkMarkers, null, 2)); + + Assert.equal( + parentNetworkMarkers.length, + 2, + `We should get a pair of network markers in the parent thread.` + ); + Assert.equal( + contentNetworkMarkers.length, + 2, + `We should get a pair of network markers in the content thread.` + ); + + const parentStopMarker = parentNetworkMarkers[1]; + const contentStopMarker = contentNetworkMarkers[1]; + + const expectedProperties = { + name: Expect.stringMatches(`Load \\d+:.*${escapeStringRegexp(url)}`), + data: Expect.objectContains({ + status: "STATUS_STOP", + URI: url, + requestMethod: "GET", + contentType: "text/html", + startTime: Expect.number(), + endTime: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + id: Expect.number(), + count: Expect.number(), + pri: Expect.number(), + isPrivateBrowsing: true, + }), + }; + + Assert.objectContains(parentStopMarker, expectedProperties); + // The cache information is missing from the content marker, it's only part + // of the parent marker. See Bug 1544821. + Assert.objectContains(parentStopMarker.data, { + // Because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + }); + Assert.objectContains(contentStopMarker, expectedProperties); + } finally { + await BrowserTestUtils.closeWindow(win); + } +}); diff --git a/tools/profiler/tests/browser/browser_test_marker_network_redirect.js b/tools/profiler/tests/browser/browser_test_marker_network_redirect.js new file mode 100644 index 0000000000..28478c2b3b --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_marker_network_redirect.js @@ -0,0 +1,341 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we emit network markers accordingly. + * In this file we'll test the redirect cases. + */ +add_task(async function test_network_markers_service_worker_setup() { + // Disabling cache makes the result more predictable especially in verify mode. + await SpecialPowers.pushPrefEnv({ + set: [ + ["browser.cache.disk.enable", false], + ["browser.cache.memory.enable", false], + ], + }); +}); + +add_task(async function test_network_markers_redirect_simple() { + // In this test, we request an HTML page that gets redirected. This is a + // top-level navigation. + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const targetFileNameWithCacheBust = "simple.html"; + const url = + BASE_URL + + "redirect.sjs?" + + encodeURIComponent(targetFileNameWithCacheBust); + const targetUrl = BASE_URL + targetFileNameWithCacheBust; + + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + const { parentThread, contentThread } = await stopProfilerNowAndGetThreads( + contentPid + ); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + info(JSON.stringify(parentNetworkMarkers, null, 2)); + info(JSON.stringify(contentNetworkMarkers, null, 2)); + + Assert.equal( + parentNetworkMarkers.length, + 4, + `We should get 2 pairs of network markers in the parent thread.` + ); + + /* It looks like that for a redirection for the top level navigation, the + * content thread sees the markers for the second request only. + * See Bug 1692879. */ + Assert.equal( + contentNetworkMarkers.length, + 2, + `We should get one pair of network markers in the content thread.` + ); + + const parentRedirectMarker = parentNetworkMarkers[1]; + const parentStopMarker = parentNetworkMarkers[3]; + // There's no content redirect marker for the reason outlined above. + const contentStopMarker = contentNetworkMarkers[1]; + + Assert.objectContains(parentRedirectMarker, { + name: Expect.stringMatches(`Load \\d+:.*${escapeStringRegexp(url)}`), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_REDIRECT", + URI: url, + RedirectURI: targetUrl, + requestMethod: "GET", + contentType: null, + startTime: Expect.number(), + endTime: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + id: Expect.number(), + redirectId: parentStopMarker.data.id, + pri: Expect.number(), + cache: Expect.stringMatches(/Missed|Unresolved/), + redirectType: "Permanent", + isHttpToHttpsRedirect: false, + }), + }); + + const expectedProperties = { + name: Expect.stringMatches( + `Load \\d+:.*${escapeStringRegexp(targetUrl)}` + ), + }; + const expectedDataProperties = { + type: "Network", + status: "STATUS_STOP", + URI: targetUrl, + requestMethod: "GET", + contentType: "text/html", + startTime: Expect.number(), + endTime: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + id: Expect.number(), + count: Expect.number(), + pri: Expect.number(), + }; + + Assert.objectContains(parentStopMarker, expectedProperties); + Assert.objectContains(contentStopMarker, expectedProperties); + + // The cache information is missing from the content marker, it's only part + // of the parent marker. See Bug 1544821. + Assert.objectContainsOnly(parentStopMarker.data, { + ...expectedDataProperties, + // Because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + }); + Assert.objectContainsOnly(contentStopMarker.data, expectedDataProperties); + }); +}); + +add_task(async function test_network_markers_redirect_resources() { + // In this test we request an HTML file that itself contains resources that + // are redirected. + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const url = BASE_URL + "page_with_resources.html?cacheBust=" + Math.random(); + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + const { parentThread, contentThread } = await stopProfilerNowAndGetThreads( + contentPid + ); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + info(JSON.stringify(parentNetworkMarkers, null, 2)); + info(JSON.stringify(contentNetworkMarkers, null, 2)); + + Assert.equal( + parentNetworkMarkers.length, + 8, + `We should get 4 pairs of network markers in the parent thread.` + // 1 - The main page + // 2 - The SVG + // 3 - The redirected request for the second SVG request. + // 4 - The SVG, again + ); + + /* In this second test, the top level navigation request isn't redirected. + * Contrary to Bug 1692879 we get all network markers for redirected + * resources. */ + Assert.equal( + contentNetworkMarkers.length, + 8, + `We should get 4 pairs of network markers in the content thread.` + ); + + // The same resource firefox-logo-nightly.svg is requested twice, but the + // second time it is redirected. + // We're not interested in the main page, as we test that in other files. + // In this page we're only interested in the marker for requested resources. + + const parentPairs = getPairsOfNetworkMarkers(parentNetworkMarkers); + const contentPairs = getPairsOfNetworkMarkers(contentNetworkMarkers); + + // First, make sure we properly matched all start with stop markers. This + // means that both arrays should contain only arrays of 2 elements. + parentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the parent process.` + ) + ); + contentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the content process.` + ) + ); + + const parentFirstStopMarker = parentPairs[1][1]; + const parentRedirectMarker = parentPairs[2][1]; + const parentSecondStopMarker = parentPairs[3][1]; + const contentFirstStopMarker = contentPairs[1][1]; + const contentRedirectMarker = contentPairs[2][1]; + const contentSecondStopMarker = contentPairs[3][1]; + + const expectedCommonDataProperties = { + type: "Network", + requestMethod: "GET", + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + innerWindowID: Expect.number(), + }; + + // These properties are present when a connection is fully opened. This is + // most often the case, unless we're in verify mode, because in that case + // we run the same tests several times in the same Firefox and they might be + // cached, or in chaos mode Firefox may make all requests sequentially on + // the same connection. + // In these cases, these properties won't always be present. + const expectedConnectionProperties = { + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + }; + + const expectedPropertiesForStopMarker = { + name: Expect.stringMatches(/Load \d+:.*\/firefox-logo-nightly\.svg/), + }; + + const expectedDataPropertiesForStopMarker = { + ...expectedCommonDataProperties, + ...expectedConnectionProperties, + status: "STATUS_STOP", + URI: Expect.stringContains("/firefox-logo-nightly.svg"), + contentType: "image/svg+xml", + count: Expect.number(), + }; + + const expectedPropertiesForRedirectMarker = { + name: Expect.stringMatches( + /Load \d+:.*\/redirect.sjs\?firefox-logo-nightly\.svg/ + ), + }; + + const expectedDataPropertiesForRedirectMarker = { + ...expectedCommonDataProperties, + ...expectedConnectionProperties, + status: "STATUS_REDIRECT", + URI: Expect.stringContains("/redirect.sjs?firefox-logo-nightly.svg"), + RedirectURI: Expect.stringContains("/firefox-logo-nightly.svg"), + contentType: null, + redirectType: "Permanent", + isHttpToHttpsRedirect: false, + }; + + Assert.objectContains( + parentFirstStopMarker, + expectedPropertiesForStopMarker + ); + Assert.objectContainsOnly(parentFirstStopMarker.data, { + ...expectedDataPropertiesForStopMarker, + // The cache information is missing from the content marker, it's only part + // of the parent marker. See Bug 1544821. + // Also, because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + }); + + Assert.objectContains( + contentFirstStopMarker, + expectedPropertiesForStopMarker + ); + Assert.objectContainsOnly( + contentFirstStopMarker.data, + expectedDataPropertiesForStopMarker + ); + + Assert.objectContains( + parentRedirectMarker, + expectedPropertiesForRedirectMarker + ); + Assert.objectContainsOnly(parentRedirectMarker.data, { + ...expectedDataPropertiesForRedirectMarker, + redirectId: parentSecondStopMarker.data.id, + // See above for the full explanation about the cache property. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + }); + + Assert.objectContains( + contentRedirectMarker, + expectedPropertiesForRedirectMarker + ); + Assert.objectContainsOnly(contentRedirectMarker.data, { + ...expectedDataPropertiesForRedirectMarker, + redirectId: contentSecondStopMarker.data.id, + }); + + Assert.objectContains( + parentSecondStopMarker, + expectedPropertiesForStopMarker + ); + Assert.objectContainsOnly(parentSecondStopMarker.data, { + ...expectedDataPropertiesForStopMarker, + // The "count" property is absent from the content marker. + count: Expect.number(), + // See above for the full explanation about the cache property. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + }); + + Assert.objectContains( + contentSecondStopMarker, + expectedPropertiesForStopMarker + ); + Assert.objectContainsOnly( + contentSecondStopMarker.data, + expectedDataPropertiesForStopMarker + ); + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_cache_first.js b/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_cache_first.js new file mode 100644 index 0000000000..c1ad49b262 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_cache_first.js @@ -0,0 +1,378 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we emit network markers accordingly. + * In this file we'll test a caching service worker. This service worker will + * fetch and store requests at install time, and serve them when the page + * requests them. + */ + +const serviceWorkerFileName = "serviceworker_cache_first.js"; +registerCleanupFunction(() => SpecialPowers.removeAllServiceWorkerData()); + +add_task(async function test_network_markers_service_worker_setup() { + // Disabling cache makes the result more predictable. Also this makes things + // simpler when dealing with service workers. + await SpecialPowers.pushPrefEnv({ + set: [ + ["browser.cache.disk.enable", false], + ["browser.cache.memory.enable", false], + ], + }); +}); + +add_task(async function test_network_markers_service_worker_register() { + // In this first step, we request an HTML page that will register a service + // worker. We'll wait until the service worker is fully installed before + // checking various things. + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const url = `${BASE_URL_HTTPS}serviceworkers/serviceworker_register.html`; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + await SpecialPowers.spawn( + contentBrowser, + [serviceWorkerFileName], + async function (serviceWorkerFileName) { + await content.wrappedJSObject.registerServiceWorkerAndWait( + serviceWorkerFileName + ); + } + ); + + const { parentThread, contentThread, profile } = + await stopProfilerNowAndGetThreads(contentPid); + + // The service worker work happens in a third "thread" or process, let's try + // to find it. + // Currently the fetches happen on the main thread for the content process, + // this may change in the future and we may have to adapt this function. + // Also please note this isn't necessarily the same content process as the + // ones for the tab. + const { serviceWorkerParentThread } = findServiceWorkerThreads(profile); + + // Here are a few sanity checks. + ok( + serviceWorkerParentThread, + "We should find a thread for the service worker." + ); + + Assert.notEqual( + serviceWorkerParentThread.pid, + parentThread.pid, + "We should have a different pid than the parent thread." + ); + Assert.notEqual( + serviceWorkerParentThread.tid, + parentThread.tid, + "We should have a different tid than the parent thread." + ); + + // Let's make sure we actually have a registered service workers. + const workers = await SpecialPowers.registeredServiceWorkers(); + Assert.equal( + workers.length, + 1, + "One service worker should be properly registered." + ); + + // By logging a few information about the threads we make debugging easier. + logInformationForThread("parentThread information", parentThread); + logInformationForThread("contentThread information", contentThread); + logInformationForThread( + "serviceWorkerParentThread information", + serviceWorkerParentThread + ); + + // Now let's check the marker payloads. + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread) + // When we load a page, Firefox will check the service worker freshness + // after a few seconds. So when the test lasts a long time (with some test + // environments) we might see spurious markers about that that we're not + // interesting in in this part of the test. They're only present in the + // parent process. + .filter(marker => !marker.data.URI.includes(serviceWorkerFileName)); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + const serviceWorkerNetworkMarkers = getInflatedNetworkMarkers( + serviceWorkerParentThread + ); + + // Some more logs for debugging purposes. + info( + "Parent network markers: " + JSON.stringify(parentNetworkMarkers, null, 2) + ); + info( + "Content network markers: " + + JSON.stringify(contentNetworkMarkers, null, 2) + ); + info( + "Serviceworker network markers: " + + JSON.stringify(serviceWorkerNetworkMarkers, null, 2) + ); + + const parentPairs = getPairsOfNetworkMarkers(parentNetworkMarkers); + const contentPairs = getPairsOfNetworkMarkers(contentNetworkMarkers); + const serviceWorkerPairs = getPairsOfNetworkMarkers( + serviceWorkerNetworkMarkers + ); + + // First, make sure we properly matched all start with stop markers. This + // means that both arrays should contain only arrays of 2 elements. + parentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the parent process.` + ) + ); + contentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the content process.` + ) + ); + serviceWorkerPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the service worker process.` + ) + ); + + // Let's look at all pairs and make sure we requested all expected files. + const parentStopMarkers = parentPairs.map(([_, stopMarker]) => stopMarker); + + // These are the files cached by the service worker. We should see markers + // for both the parent thread and the service worker thread. + const expectedFiles = [ + "serviceworker_page.html", + "firefox-logo-nightly.svg", + ].map(filename => `${BASE_URL_HTTPS}serviceworkers/${filename}`); + + for (const expectedFile of expectedFiles) { + info( + `Checking if "${expectedFile}" is present in the network markers in both processes.` + ); + const parentMarker = parentStopMarkers.find( + marker => marker.data.URI === expectedFile + ); + + const expectedProperties = { + name: Expect.stringMatches( + `Load \\d+:.*${escapeStringRegexp(expectedFile)}` + ), + data: Expect.objectContains({ + status: "STATUS_STOP", + URI: expectedFile, + requestMethod: "GET", + contentType: Expect.stringMatches(/^(text\/html|image\/svg\+xml)$/), + startTime: Expect.number(), + endTime: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + id: Expect.number(), + count: Expect.number(), + pri: Expect.number(), + }), + }; + + Assert.objectContains(parentMarker, expectedProperties); + } + }); +}); + +add_task(async function test_network_markers_service_worker_use() { + // In this test we request an HTML file that itself contains resources that + // are redirected. + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const url = `${BASE_URL_HTTPS}serviceworkers/serviceworker_page.html`; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + const { parentThread, contentThread } = await stopProfilerNowAndGetThreads( + contentPid + ); + + // By logging a few information about the threads we make debugging easier. + logInformationForThread("parentThread information", parentThread); + logInformationForThread("contentThread information", contentThread); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread) + // When we load a page, Firefox will check the service worker freshness + // after a few seconds. So when the test lasts a long time (with some test + // environments) we might see spurious markers about that that we're not + // interesting in in this part of the test. They're only present in the + // parent process. + .filter(marker => !marker.data.URI.includes(serviceWorkerFileName)); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + + // Here are some logs to ease debugging. + info( + "Parent network markers: " + JSON.stringify(parentNetworkMarkers, null, 2) + ); + info( + "Content network markers: " + + JSON.stringify(contentNetworkMarkers, null, 2) + ); + + const parentPairs = getPairsOfNetworkMarkers(parentNetworkMarkers); + const contentPairs = getPairsOfNetworkMarkers(contentNetworkMarkers); + + // These are the files cached by the service worker. We should see markers + // for the parent thread and the content thread. + const expectedFiles = [ + "serviceworker_page.html", + "firefox-logo-nightly.svg", + ].map(filename => `${BASE_URL_HTTPS}serviceworkers/${filename}`); + + // First, make sure we properly matched all start with stop markers. This + // means that both arrays should contain only arrays of 2 elements. + parentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the parent process.` + ) + ); + + contentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the content process.` + ) + ); + + // Let's look at all pairs and make sure we requested all expected files. + const parentEndMarkers = parentPairs.map(([_, endMarker]) => endMarker); + const contentStopMarkers = contentPairs.map( + ([_, stopMarker]) => stopMarker + ); + + Assert.equal( + parentEndMarkers.length, + expectedFiles.length * 2, // one redirect + one stop + "There should be twice as many end markers in the parent process as requested files." + ); + Assert.equal( + contentStopMarkers.length, + expectedFiles.length, + "There should be as many stop markers in the content process as requested files." + ); + + for (const [i, expectedFile] of expectedFiles.entries()) { + info( + `Checking if "${expectedFile}" if present in the network markers in both processes.` + ); + const [parentRedirectMarker, parentStopMarker] = parentEndMarkers.filter( + marker => marker.data.URI === expectedFile + ); + const contentMarker = contentStopMarkers.find( + marker => marker.data.URI === expectedFile + ); + + const commonDataProperties = { + type: "Network", + URI: expectedFile, + requestMethod: "GET", + contentType: Expect.stringMatches(/^(text\/html|image\/svg\+xml)$/), + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + }; + + const expectedProperties = { + name: Expect.stringMatches( + `Load \\d+:.*${escapeStringRegexp(expectedFile)}` + ), + }; + + Assert.objectContains(parentRedirectMarker, expectedProperties); + Assert.objectContains(parentStopMarker, expectedProperties); + Assert.objectContains(contentMarker, expectedProperties); + if (i === 0) { + // This is the top level navigation, the HTML file. + Assert.objectContainsOnly(parentRedirectMarker.data, { + ...commonDataProperties, + status: "STATUS_REDIRECT", + contentType: null, + cache: "Unresolved", + RedirectURI: expectedFile, + redirectType: "Internal", + redirectId: parentStopMarker.data.id, + isHttpToHttpsRedirect: false, + }); + + Assert.objectContainsOnly(parentStopMarker.data, { + ...commonDataProperties, + status: "STATUS_STOP", + }); + + Assert.objectContainsOnly(contentMarker.data, { + ...commonDataProperties, + status: "STATUS_STOP", + }); + } else { + Assert.objectContainsOnly(parentRedirectMarker.data, { + ...commonDataProperties, + status: "STATUS_REDIRECT", + contentType: null, + cache: "Unresolved", + innerWindowID: Expect.number(), + RedirectURI: expectedFile, + redirectType: "Internal", + redirectId: parentStopMarker.data.id, + isHttpToHttpsRedirect: false, + }); + + Assert.objectContainsOnly( + parentStopMarker.data, + // Note: in the future we may have more properties. We're using the + // "Only" flavor of the matcher so that we don't forget to update this + // test when this changes. + { + ...commonDataProperties, + innerWindowID: Expect.number(), + status: "STATUS_STOP", + } + ); + + Assert.objectContainsOnly(contentMarker.data, { + ...commonDataProperties, + innerWindowID: Expect.number(), + status: "STATUS_STOP", + }); + } + } + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_no_fetch_handler.js b/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_no_fetch_handler.js new file mode 100644 index 0000000000..ad2cc81661 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_no_fetch_handler.js @@ -0,0 +1,218 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we emit network markers accordingly. + * In this file we'll test the case of a service worker that has no fetch + * handlers. In this case, a fetch is done to the network. There may be + * shortcuts in our code in this case, that's why it's important to test it + * separately. + */ + +const serviceWorkerFileName = "serviceworker_no_fetch_handler.js"; +registerCleanupFunction(() => SpecialPowers.removeAllServiceWorkerData()); + +add_task(async function test_network_markers_service_worker_setup() { + // Disabling cache makes the result more predictable. Also this makes things + // simpler when dealing with service workers. + await SpecialPowers.pushPrefEnv({ + set: [ + ["browser.cache.disk.enable", false], + ["browser.cache.memory.enable", false], + ], + }); +}); + +add_task(async function test_network_markers_service_worker_register() { + // In this first step, we request an HTML page that will register a service + // worker. We'll wait until the service worker is fully installed before + // checking various things. + const url = `${BASE_URL_HTTPS}serviceworkers/serviceworker_register.html`; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + await SpecialPowers.spawn( + contentBrowser, + [serviceWorkerFileName], + async function (serviceWorkerFileName) { + await content.wrappedJSObject.registerServiceWorkerAndWait( + serviceWorkerFileName + ); + } + ); + + // Let's make sure we actually have a registered service workers. + const workers = await SpecialPowers.registeredServiceWorkers(); + Assert.equal( + workers.length, + 1, + "One service worker should be properly registered." + ); + }); +}); + +add_task(async function test_network_markers_service_worker_use() { + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const url = `${BASE_URL_HTTPS}serviceworkers/serviceworker_page.html`; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + const { parentThread, contentThread } = await stopProfilerNowAndGetThreads( + contentPid + ); + + // By logging a few information about the threads we make debugging easier. + logInformationForThread("parentThread information", parentThread); + logInformationForThread("contentThread information", contentThread); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread) + // When we load a page, Firefox will check the service worker freshness + // after a few seconds. So when the test lasts a long time (with some test + // environments) we might see spurious markers about that that we're not + // interesting in in this part of the test. They're only present in the + // parent process. + .filter(marker => !marker.data.URI.includes(serviceWorkerFileName)); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + + // Here are some logs to ease debugging. + info( + "Parent network markers:" + JSON.stringify(parentNetworkMarkers, null, 2) + ); + info( + "Content network markers:" + + JSON.stringify(contentNetworkMarkers, null, 2) + ); + + const parentPairs = getPairsOfNetworkMarkers(parentNetworkMarkers); + const contentPairs = getPairsOfNetworkMarkers(contentNetworkMarkers); + + // First, make sure we properly matched all start with stop markers. This + // means that both arrays should contain only arrays of 2 elements. + parentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the parent process.` + ) + ); + + contentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the content process.` + ) + ); + + // Let's look at all pairs and make sure we requested all expected files. + const parentStopMarkers = parentPairs.map(([_, stopMarker]) => stopMarker); + const contentStopMarkers = contentPairs.map( + ([_, stopMarker]) => stopMarker + ); + + // These are the files requested by the page. + // We should see markers for the parent thread and the content thread. + const expectedFiles = [ + // Please take care that the first element is the top level navigation, as + // this is special-cased below. + "serviceworker_page.html", + "firefox-logo-nightly.svg", + ].map(filename => `${BASE_URL_HTTPS}serviceworkers/${filename}`); + + Assert.equal( + parentStopMarkers.length, + expectedFiles.length, + "There should be as many stop markers in the parent process as requested files." + ); + Assert.equal( + contentStopMarkers.length, + expectedFiles.length, + "There should be as many stop markers in the content process as requested files." + ); + + for (const [i, expectedFile] of expectedFiles.entries()) { + info( + `Checking if "${expectedFile}" if present in the network markers in both processes.` + ); + const parentMarker = parentStopMarkers.find( + marker => marker.data.URI === expectedFile + ); + const contentMarker = contentStopMarkers.find( + marker => marker.data.URI === expectedFile + ); + + const commonProperties = { + name: Expect.stringMatches( + `Load \\d+:.*${escapeStringRegexp(expectedFile)}` + ), + }; + Assert.objectContains(parentMarker, commonProperties); + Assert.objectContains(contentMarker, commonProperties); + + // We get the full set of properties in this case, because we do an actual + // fetch to the network. + const commonDataProperties = { + type: "Network", + status: "STATUS_STOP", + URI: expectedFile, + requestMethod: "GET", + contentType: Expect.stringMatches(/^(text\/html|image\/svg\+xml)$/), + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + count: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + }; + + if (i === 0) { + // The first marker is special cased: this is the top level navigation + // serviceworker_page.html, + // and in this case we don't have all the same properties. Especially + // the innerWindowID information is missing. + Assert.objectContainsOnly(parentMarker.data, { + ...commonDataProperties, + // Note that the parent process has the "cache" information, but not the content + // process. See Bug 1544821. + // Also because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + }); + + Assert.objectContainsOnly(contentMarker.data, commonDataProperties); + } else { + // This is the other file firefox-logo-nightly.svg. + Assert.objectContainsOnly(parentMarker.data, { + ...commonDataProperties, + // Because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + innerWindowID: Expect.number(), + }); + + Assert.objectContainsOnly(contentMarker.data, { + ...commonDataProperties, + innerWindowID: Expect.number(), + }); + } + } + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_no_respondWith_in_fetch_handler.js b/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_no_respondWith_in_fetch_handler.js new file mode 100644 index 0000000000..973ae61a7f --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_no_respondWith_in_fetch_handler.js @@ -0,0 +1,294 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we emit network markers accordingly. + * In this file we'll test the case of a service worker that has a fetch + * handler, but no respondWith. In this case, some process called "reset + * interception" happens, and the fetch is still carried on by our code. Because + * this is a bit of an edge case, it's important to have a test for this case. + */ + +const serviceWorkerFileName = + "serviceworker_no_respondWith_in_fetch_handler.js"; +registerCleanupFunction(() => SpecialPowers.removeAllServiceWorkerData()); + +add_task(async function test_network_markers_service_worker_setup() { + // Disabling cache makes the result more predictable. Also this makes things + // simpler when dealing with service workers. + await SpecialPowers.pushPrefEnv({ + set: [ + ["browser.cache.disk.enable", false], + ["browser.cache.memory.enable", false], + ], + }); +}); + +add_task(async function test_network_markers_service_worker_register() { + // In this first step, we request an HTML page that will register a service + // worker. We'll wait until the service worker is fully installed before + // checking various things. + const url = `${BASE_URL_HTTPS}serviceworkers/serviceworker_register.html`; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + await SpecialPowers.spawn( + contentBrowser, + [serviceWorkerFileName], + async function (serviceWorkerFileName) { + await content.wrappedJSObject.registerServiceWorkerAndWait( + serviceWorkerFileName + ); + } + ); + + // Let's make sure we actually have a registered service workers. + const workers = await SpecialPowers.registeredServiceWorkers(); + Assert.equal( + workers.length, + 1, + "One service worker should be properly registered." + ); + }); +}); + +add_task(async function test_network_markers_service_worker_use() { + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const url = `${BASE_URL_HTTPS}serviceworkers/serviceworker_page.html`; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + const { parentThread, contentThread } = await stopProfilerNowAndGetThreads( + contentPid + ); + + // By logging a few information about the threads we make debugging easier. + logInformationForThread("parentThread information", parentThread); + logInformationForThread("contentThread information", contentThread); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread) + // When we load a page, Firefox will check the service worker freshness + // after a few seconds. So when the test lasts a long time (with some test + // environments) we might see spurious markers about that that we're not + // interesting in in this part of the test. They're only present in the + // parent process. + .filter(marker => !marker.data.URI.includes(serviceWorkerFileName)); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + + // Here are some logs to ease debugging. + info( + "Parent network markers:" + JSON.stringify(parentNetworkMarkers, null, 2) + ); + info( + "Content network markers:" + + JSON.stringify(contentNetworkMarkers, null, 2) + ); + + const parentPairs = getPairsOfNetworkMarkers(parentNetworkMarkers); + const contentPairs = getPairsOfNetworkMarkers(contentNetworkMarkers); + + // First, make sure we properly matched all start with stop markers. This + // means that both arrays should contain only arrays of 2 elements. + parentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the parent process.` + ) + ); + + contentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the content process.` + ) + ); + + // Let's look at all pairs and make sure we requested all expected files. + // In this test, we should have redirect markers as well as stop markers, + // because this case generates internal redirects. We may want to change + // that in the future, or handle this specially in the frontend. + // Let's create various arrays to help assert. + + const parentEndMarkers = parentPairs.map(([_, stopMarker]) => stopMarker); + const parentStopMarkers = parentEndMarkers.filter( + marker => marker.data.status === "STATUS_STOP" + ); + const parentRedirectMarkers = parentEndMarkers.filter( + marker => marker.data.status === "STATUS_REDIRECT" + ); + const contentEndMarkers = contentPairs.map(([_, stopMarker]) => stopMarker); + const contentStopMarkers = contentEndMarkers.filter( + marker => marker.data.status === "STATUS_STOP" + ); + const contentRedirectMarkers = contentEndMarkers.filter( + marker => marker.data.status === "STATUS_REDIRECT" + ); + + // These are the files requested by the page. + // We should see markers for the parent thread and the content thread. + const expectedFiles = [ + // Please take care that the first element is the top level navigation, as + // this is special-cased below. + "serviceworker_page.html", + "firefox-logo-nightly.svg", + ].map(filename => `${BASE_URL_HTTPS}serviceworkers/${filename}`); + + Assert.equal( + parentStopMarkers.length, + expectedFiles.length, + "There should be as many stop markers in the parent process as requested files." + ); + Assert.equal( + parentRedirectMarkers.length, + expectedFiles.length * 2, // http -> intercepted, intercepted -> http + "There should be twice as many redirect markers in the parent process as requested files." + ); + Assert.equal( + contentStopMarkers.length, + expectedFiles.length, + "There should be as many stop markers in the content process as requested files." + ); + // Note: there will no redirect markers in the content process for + // ServiceWorker fallbacks request to network. + // See Bug 1793940. + Assert.equal( + contentRedirectMarkers.length, + 0, + "There should be no redirect markers in the content process than requested files." + ); + + for (const [i, expectedFile] of expectedFiles.entries()) { + info( + `Checking if "${expectedFile}" if present in the network markers in both processes.` + ); + const [parentRedirectMarkerIntercept, parentRedirectMarkerReset] = + parentRedirectMarkers.filter( + marker => marker.data.URI === expectedFile + ); + const parentStopMarker = parentStopMarkers.find( + marker => marker.data.URI === expectedFile + ); + const contentStopMarker = contentStopMarkers.find( + marker => marker.data.URI === expectedFile + ); + + const commonProperties = { + name: Expect.stringMatches( + `Load \\d+:.*${escapeStringRegexp(expectedFile)}` + ), + }; + Assert.objectContains(parentRedirectMarkerIntercept, commonProperties); + Assert.objectContains(parentRedirectMarkerReset, commonProperties); + Assert.objectContains(parentStopMarker, commonProperties); + Assert.objectContains(contentStopMarker, commonProperties); + // Note: there's no check for the contentRedirectMarker, because there's + // no marker for a top level navigation redirect in the content process. + + // We get the full set of properties in this case, because we do an actual + // fetch to the network. + const commonDataProperties = { + type: "Network", + status: "STATUS_STOP", + URI: expectedFile, + requestMethod: "GET", + contentType: Expect.stringMatches(/^(text\/html|image\/svg\+xml)$/), + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + count: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + }; + + const commonRedirectProperties = { + type: "Network", + status: "STATUS_REDIRECT", + URI: expectedFile, + RedirectURI: expectedFile, + requestMethod: "GET", + contentType: null, + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + redirectType: "Internal", + isHttpToHttpsRedirect: false, + }; + + if (i === 0) { + // The first marker is special cased: this is the top level navigation + // serviceworker_page.html, + // and in this case we don't have all the same properties. Especially + // the innerWindowID information is missing. + Assert.objectContainsOnly(parentStopMarker.data, { + ...commonDataProperties, + // Note that the parent process has the "cache" information, but not the content + // process. See Bug 1544821. + // Also, because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + }); + Assert.objectContainsOnly(contentStopMarker.data, commonDataProperties); + + Assert.objectContainsOnly(parentRedirectMarkerIntercept.data, { + ...commonRedirectProperties, + redirectId: parentRedirectMarkerReset.data.id, + cache: "Unresolved", + }); + Assert.objectContainsOnly(parentRedirectMarkerReset.data, { + ...commonRedirectProperties, + redirectId: parentStopMarker.data.id, + }); + + // Note: there's no check for the contentRedirectMarker, because there's + // no marker for a top level navigation redirect in the content process. + } else { + // This is the other file firefox-logo-nightly.svg. + Assert.objectContainsOnly(parentStopMarker.data, { + ...commonDataProperties, + // Because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + innerWindowID: Expect.number(), + }); + Assert.objectContains(contentStopMarker, commonProperties); + Assert.objectContainsOnly(contentStopMarker.data, { + ...commonDataProperties, + innerWindowID: Expect.number(), + }); + + Assert.objectContainsOnly(parentRedirectMarkerIntercept.data, { + ...commonRedirectProperties, + innerWindowID: Expect.number(), + redirectId: parentRedirectMarkerReset.data.id, + cache: "Unresolved", + }); + Assert.objectContainsOnly(parentRedirectMarkerReset.data, { + ...commonRedirectProperties, + innerWindowID: Expect.number(), + redirectId: parentStopMarker.data.id, + }); + } + } + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_synthetized_response.js b/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_synthetized_response.js new file mode 100644 index 0000000000..060592840a --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_marker_network_serviceworker_synthetized_response.js @@ -0,0 +1,480 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we emit network markers accordingly. + * In this file we'll test a service worker that returns a synthetized response. + * This means the service worker will make up a response by itself. + */ + +const serviceWorkerFileName = "serviceworker_synthetized_response.js"; +registerCleanupFunction(() => SpecialPowers.removeAllServiceWorkerData()); + +add_task(async function test_network_markers_service_worker_setup() { + // Disabling cache makes the result more predictable. Also this makes things + // simpler when dealing with service workers. + await SpecialPowers.pushPrefEnv({ + set: [ + ["browser.cache.disk.enable", false], + ["browser.cache.memory.enable", false], + ], + }); +}); + +add_task(async function test_network_markers_service_worker_register() { + // In this first step, we request an HTML page that will register a service + // worker. We'll wait until the service worker is fully installed before + // checking various things. + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + const url = `${BASE_URL_HTTPS}serviceworkers/serviceworker_register.html`; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + await SpecialPowers.spawn( + contentBrowser, + [serviceWorkerFileName], + async function (serviceWorkerFileName) { + await content.wrappedJSObject.registerServiceWorkerAndWait( + serviceWorkerFileName + ); + } + ); + + // Let's make sure we actually have a registered service workers. + const workers = await SpecialPowers.registeredServiceWorkers(); + Assert.equal( + workers.length, + 1, + "One service worker should be properly registered." + ); + }); +}); + +add_task(async function test_network_markers_service_worker_use() { + // In this test, we'll first load a plain html file, then do some fetch + // requests in the context of the page. One request is served with a + // synthetized response, the other request is served with a real "fetch" done + // by the service worker. + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const url = `${BASE_URL_HTTPS}serviceworkers/serviceworker_simple.html`; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + await SpecialPowers.spawn(contentBrowser, [], async () => { + // This request is served directly by the service worker as a synthetized response. + await content + .fetch("firefox-generated.svg") + .then(res => res.arrayBuffer()); + + // This request is served by a fetch done inside the service worker. + await content + .fetch("firefox-logo-nightly.svg") + .then(res => res.arrayBuffer()); + }); + + const { parentThread, contentThread, profile } = + await stopProfilerNowAndGetThreads(contentPid); + + // The service worker work happens in a third "thread" or process, let's try + // to find it. + // Currently the fetches happen on the main thread for the content process, + // this may change in the future and we may have to adapt this function. + // Also please note this isn't necessarily the same content process as the + // ones for the tab. + const { serviceWorkerParentThread } = findServiceWorkerThreads(profile); + + ok( + serviceWorkerParentThread, + "We should find a thread for the service worker." + ); + + // By logging a few information about the threads we make debugging easier. + logInformationForThread("parentThread information", parentThread); + logInformationForThread("contentThread information", contentThread); + logInformationForThread( + "serviceWorkerParentThread information", + serviceWorkerParentThread + ); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread) + // When we load a page, Firefox will check the service worker freshness + // after a few seconds. So when the test lasts a long time (with some test + // environments) we might see spurious markers about that that we're not + // interesting in in this part of the test. They're only present in the + // parent process. + .filter(marker => !marker.data.URI.includes(serviceWorkerFileName)); + + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + const serviceWorkerNetworkMarkers = getInflatedNetworkMarkers( + serviceWorkerParentThread + ); + + // Some more logs for debugging purposes. + info( + "Parent network markers: " + JSON.stringify(parentNetworkMarkers, null, 2) + ); + info( + "Content network markers: " + + JSON.stringify(contentNetworkMarkers, null, 2) + ); + info( + "Serviceworker network markers: " + + JSON.stringify(serviceWorkerNetworkMarkers, null, 2) + ); + + const parentPairs = getPairsOfNetworkMarkers(parentNetworkMarkers); + const contentPairs = getPairsOfNetworkMarkers(contentNetworkMarkers); + const serviceWorkerPairs = getPairsOfNetworkMarkers( + serviceWorkerNetworkMarkers + ); + + // First, make sure we properly matched all start with stop markers. This + // means that both arrays should contain only arrays of 2 elements. + parentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the parent process.` + ) + ); + + contentPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the content process.` + ) + ); + serviceWorkerPairs.forEach(pair => + Assert.equal( + pair.length, + 2, + `For the URL ${pair[0].data.URI} we should get 2 markers in the service worker process.` + ) + ); + + // Let's look at all pairs and make sure we requested all expected files. + // In this test, we should have redirect markers as well as stop markers, + // because this case generates internal redirects. + // Let's create various arrays to help assert. + + let parentStopMarkers = parentPairs.map(([_, stopMarker]) => stopMarker); + const contentStopMarkers = contentPairs.map( + ([_, stopMarker]) => stopMarker + ); + // In this test we have very different results in the various threads, so + // we'll assert every case separately. + // A simple function to help constructing better assertions: + const fullUrl = filename => `${BASE_URL_HTTPS}serviceworkers/${filename}`; + + { + // In the parent process, we have 8 network markers: + // - twice the html file -- because it's not cached by the SW, we get the + // marker both for the initial request and for the request initied from the + // SW. + // - twice the firefox svg file -- similar situation + // - once the generated svg file -- this one isn't fetched by the SW but + // rather forged directly, so there's no "second fetch", and thus we have + // only one marker. + // - for each of these files, we have first an internal redirect from the + // main channel to the service worker. => 3 redirect markers more. + Assert.equal( + parentStopMarkers.length, + 8, // 3 html files, 3 firefox svg files, 2 generated svg file + "There should be 8 stop markers in the parent process." + ); + + // The "1" requests are the initial requests that are intercepted, coming + // from the web page, while the "2" requests are requests to the network, + // coming from the service worker. The 1 were requested before 2, 2 ends + // before 1. + // "Intercept" requests are the internal redirects from the main channel + // to the service worker. They happen before others. + const [ + htmlFetchIntercept, + htmlFetch1, + htmlFetch2, + generatedSvgIntercept, + generatedSvgFetch, + firefoxSvgIntercept, + firefoxSvgFetch1, + firefoxSvgFetch2, + ] = parentStopMarkers; + + /* ----- /HTML FILE ---- */ + Assert.objectContains(htmlFetchIntercept, { + name: Expect.stringMatches(/Load \d+:.*serviceworker_simple.html/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_REDIRECT", + URI: fullUrl("serviceworker_simple.html"), + requestMethod: "GET", + contentType: null, + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + redirectId: htmlFetch1.data.id, + redirectType: "Internal", + isHttpToHttpsRedirect: false, + RedirectURI: fullUrl("serviceworker_simple.html"), + cache: "Unresolved", + }), + }); + + Assert.objectContains(htmlFetch1, { + name: Expect.stringMatches(/Load \d+:.*serviceworker_simple.html/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_STOP", + URI: fullUrl("serviceworker_simple.html"), + requestMethod: "GET", + contentType: "text/html", + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + }), + }); + Assert.objectContains(htmlFetch2, { + name: Expect.stringMatches(/Load \d+:.*serviceworker_simple.html/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_STOP", + URI: fullUrl("serviceworker_simple.html"), + requestMethod: "GET", + contentType: "text/html", + // Because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + startTime: Expect.number(), + endTime: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + id: Expect.number(), + count: Expect.number(), + pri: Expect.number(), + }), + }); + /* ----- /HTML FILE ---- */ + + /* ----- GENERATED SVG FILE ---- */ + Assert.objectContains(generatedSvgIntercept, { + name: Expect.stringMatches(/Load \d+:.*firefox-generated.svg/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_REDIRECT", + URI: fullUrl("firefox-generated.svg"), + requestMethod: "GET", + contentType: null, + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + redirectId: generatedSvgFetch.data.id, + redirectType: "Internal", + isHttpToHttpsRedirect: false, + RedirectURI: fullUrl("firefox-generated.svg"), + cache: "Unresolved", + innerWindowID: Expect.number(), + }), + }); + Assert.objectContains(generatedSvgFetch, { + name: Expect.stringMatches(/Load \d+:.*firefox-generated.svg/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_STOP", + URI: fullUrl("firefox-generated.svg"), + requestMethod: "GET", + contentType: "image/svg+xml", + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + innerWindowID: Expect.number(), + }), + }); + /* ----- ∕GENERATED SVG FILE ---- */ + /* ----- REQUESTED SVG FILE ---- */ + Assert.objectContains(firefoxSvgIntercept, { + name: Expect.stringMatches(/Load \d+:.*firefox-logo-nightly.svg/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_REDIRECT", + URI: fullUrl("firefox-logo-nightly.svg"), + requestMethod: "GET", + contentType: null, + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + redirectId: firefoxSvgFetch1.data.id, + redirectType: "Internal", + isHttpToHttpsRedirect: false, + RedirectURI: fullUrl("firefox-logo-nightly.svg"), + cache: "Unresolved", + innerWindowID: Expect.number(), + }), + }); + Assert.objectContains(firefoxSvgFetch1, { + name: Expect.stringMatches(/Load \d+:.*firefox-logo-nightly.svg/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_STOP", + URI: fullUrl("firefox-logo-nightly.svg"), + requestMethod: "GET", + contentType: "image/svg+xml", + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + innerWindowID: Expect.number(), + }), + }); + Assert.objectContains(firefoxSvgFetch2, { + name: Expect.stringMatches(/Load \d+:.*firefox-logo-nightly.svg/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_STOP", + URI: fullUrl("firefox-logo-nightly.svg"), + requestMethod: "GET", + contentType: "image/svg+xml", + // Because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + startTime: Expect.number(), + endTime: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + id: Expect.number(), + count: Expect.number(), + pri: Expect.number(), + // Note: no innerWindowID here, is that a bug? + }), + }); + /* ----- ∕REQUESTED SVG FILE ---- */ + } + + // It's possible that the service worker thread IS the content thread, in + // that case we'll get all markers in the same thread. + // The "1" requests are the initial requests that are intercepted, coming + // from the web page, while the "2" requests are the requests coming from + // the service worker. + let htmlFetch1, generatedSvgFetch1, firefoxSvgFetch1; + + // First, let's handle the case where the threads are different: + if (serviceWorkerParentThread !== contentThread) { + // In the content process (that is the process for the web page), we have + // 3 network markers: + // - 1 for the HTML page + // - 1 for the generated svg file + // - 1 for the firefox svg file + // Indeed, the service worker interception is invisible from the context + // of the web page, so we just get 3 "normal" requests. However these + // requests will miss all timing information, because they're hidden by + // the service worker interception. We may want to fix this... + Assert.equal( + contentStopMarkers.length, + 3, // 1 for each file + "There should be 3 stop markers in the content process." + ); + + [htmlFetch1, generatedSvgFetch1, firefoxSvgFetch1] = contentStopMarkers; + } else { + // Else case: the service worker parent thread IS the content thread + // (note: this is always the case with fission). In that case all network + // markers tested in the above block are together in the same object. + Assert.equal( + contentStopMarkers.length, + 5, + "There should be 5 stop markers in the combined process (containing both the content page and the service worker)" + ); + + // Because of how the test is done, these markers are ordered by the + // position of the START markers. + [ + // For the htmlFetch request, note that 2 is before 1, because that's + // the top level navigation. Indeed for the top level navigation + // everything happens first in the main process, possibly before a + // content process even exists, and the content process is merely + // notified at the end. + htmlFetch1, + generatedSvgFetch1, + firefoxSvgFetch1, + ] = contentStopMarkers; + } + + // Let's test first the markers coming from the content page. + Assert.objectContains(htmlFetch1, { + name: Expect.stringMatches(/Load \d+:.*serviceworker_simple.html/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_STOP", + URI: fullUrl("serviceworker_simple.html"), + requestMethod: "GET", + contentType: "text/html", + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + }), + }); + Assert.objectContains(generatedSvgFetch1, { + name: Expect.stringMatches(/Load \d+:.*firefox-generated.svg/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_STOP", + URI: fullUrl("firefox-generated.svg"), + requestMethod: "GET", + contentType: "image/svg+xml", + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + innerWindowID: Expect.number(), + }), + }); + Assert.objectContains(firefoxSvgFetch1, { + name: Expect.stringMatches(/Load \d+:.*firefox-logo-nightly.svg/), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_STOP", + URI: fullUrl("firefox-logo-nightly.svg"), + requestMethod: "GET", + contentType: "image/svg+xml", + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + pri: Expect.number(), + innerWindowID: Expect.number(), + }), + }); + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_marker_network_simple.js b/tools/profiler/tests/browser/browser_test_marker_network_simple.js new file mode 100644 index 0000000000..15894305a7 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_marker_network_simple.js @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we emit network markers accordingly + */ +add_task(async function test_network_markers() { + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const url = BASE_URL + "simple.html?cacheBust=" + Math.random(); + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + const { parentThread, contentThread } = await stopProfilerNowAndGetThreads( + contentPid + ); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + info(JSON.stringify(parentNetworkMarkers, null, 2)); + info(JSON.stringify(contentNetworkMarkers, null, 2)); + + Assert.equal( + parentNetworkMarkers.length, + 2, + `We should get a pair of network markers in the parent thread.` + ); + Assert.equal( + contentNetworkMarkers.length, + 2, + `We should get a pair of network markers in the content thread.` + ); + + const parentStopMarker = parentNetworkMarkers[1]; + const contentStopMarker = contentNetworkMarkers[1]; + + const expectedProperties = { + name: Expect.stringMatches(`Load \\d+:.*${escapeStringRegexp(url)}`), + data: Expect.objectContains({ + status: "STATUS_STOP", + URI: url, + requestMethod: "GET", + contentType: "text/html", + startTime: Expect.number(), + endTime: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + id: Expect.number(), + count: Expect.number(), + pri: Expect.number(), + }), + }; + + Assert.objectContains(parentStopMarker, expectedProperties); + // The cache information is missing from the content marker, it's only part + // of the parent marker. See Bug 1544821. + Assert.objectContains(parentStopMarker.data, { + // Because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + }); + Assert.objectContains(contentStopMarker, expectedProperties); + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_marker_network_sts.js b/tools/profiler/tests/browser/browser_test_marker_network_sts.js new file mode 100644 index 0000000000..26f2a1c756 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_marker_network_sts.js @@ -0,0 +1,130 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we emit network markers accordingly. + * In this file we'll test that we behave properly with STS redirections. + */ + +add_task(async function test_network_markers_service_worker_setup() { + await SpecialPowers.pushPrefEnv({ + set: [ + // Disabling cache makes the result more predictable especially in verify mode. + ["browser.cache.disk.enable", false], + ["browser.cache.memory.enable", false], + // We want to test upgrading requests + ["dom.security.https_only_mode", true], + ], + }); +}); + +add_task(async function test_network_markers_redirect_to_https() { + // In this test, we request an HTML page with http that gets redirected to https. + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + startProfilerForMarkerTests(); + + const url = BASE_URL + "simple.html"; + const targetUrl = BASE_URL_HTTPS + "simple.html"; + + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + const { parentThread, contentThread } = await stopProfilerNowAndGetThreads( + contentPid + ); + + const parentNetworkMarkers = getInflatedNetworkMarkers(parentThread); + const contentNetworkMarkers = getInflatedNetworkMarkers(contentThread); + info(JSON.stringify(parentNetworkMarkers, null, 2)); + info(JSON.stringify(contentNetworkMarkers, null, 2)); + + Assert.equal( + parentNetworkMarkers.length, + 4, + `We should get 2 pairs of network markers in the parent thread.` + ); + + /* It looks like that for a redirection for the top level navigation, the + * content thread sees the markers for the second request only. + * See Bug 1692879. */ + Assert.equal( + contentNetworkMarkers.length, + 2, + `We should get one pair of network markers in the content thread.` + ); + + const parentRedirectMarker = parentNetworkMarkers[1]; + const parentStopMarker = parentNetworkMarkers[3]; + // There's no content redirect marker for the reason outlined above. + const contentStopMarker = contentNetworkMarkers[1]; + + Assert.objectContains(parentRedirectMarker, { + name: Expect.stringMatches(`Load \\d+:.*${escapeStringRegexp(url)}`), + data: Expect.objectContainsOnly({ + type: "Network", + status: "STATUS_REDIRECT", + URI: url, + RedirectURI: targetUrl, + requestMethod: "GET", + contentType: null, + startTime: Expect.number(), + endTime: Expect.number(), + id: Expect.number(), + redirectId: parentStopMarker.data.id, + pri: Expect.number(), + cache: "Unresolved", + redirectType: "Permanent", + isHttpToHttpsRedirect: true, + }), + }); + + const expectedProperties = { + name: Expect.stringMatches( + `Load \\d+:.*${escapeStringRegexp(targetUrl)}` + ), + }; + const expectedDataProperties = { + type: "Network", + status: "STATUS_STOP", + URI: targetUrl, + requestMethod: "GET", + contentType: "text/html", + startTime: Expect.number(), + endTime: Expect.number(), + domainLookupStart: Expect.number(), + domainLookupEnd: Expect.number(), + connectStart: Expect.number(), + tcpConnectEnd: Expect.number(), + connectEnd: Expect.number(), + requestStart: Expect.number(), + responseStart: Expect.number(), + responseEnd: Expect.number(), + id: Expect.number(), + count: Expect.number(), + pri: Expect.number(), + }; + + Assert.objectContains(parentStopMarker, expectedProperties); + Assert.objectContains(contentStopMarker, expectedProperties); + + // The cache information is missing from the content marker, it's only part + // of the parent marker. See Bug 1544821. + Assert.objectContainsOnly(parentStopMarker.data, { + ...expectedDataProperties, + // Because the request races with the cache, these 2 values are valid: + // "Missed" when the cache answered before we get a result from the network. + // "Unresolved" when we got a response from the network before the cache subsystem. + cache: Expect.stringMatches(/^(Missed|Unresolved)$/), + }); + Assert.objectContainsOnly(contentStopMarker.data, expectedDataProperties); + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_markers_gc_cc.js b/tools/profiler/tests/browser/browser_test_markers_gc_cc.js new file mode 100644 index 0000000000..a4a94d60cc --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_markers_gc_cc.js @@ -0,0 +1,49 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async function test_markers_gc_cc() { + info("Test GC&CC markers."); + + info("Create a throwaway profile."); + await startProfiler({}); + let tempProfileContainer = { profile: null }; + tempProfileContainer.profile = await waitSamplingAndStopAndGetProfile(); + + info("Restart the profiler."); + await startProfiler({}); + + info("Throw away the previous profile, which should be garbage-collected."); + Assert.equal( + typeof tempProfileContainer.profile, + "object", + "Previously-captured profile should be an object" + ); + delete tempProfileContainer.profile; + Assert.equal( + typeof tempProfileContainer.profile, + "undefined", + "Deleted profile should now be undefined" + ); + + info("Force GC&CC"); + SpecialPowers.gc(); + SpecialPowers.forceShrinkingGC(); + SpecialPowers.forceCC(); + SpecialPowers.gc(); + SpecialPowers.forceShrinkingGC(); + SpecialPowers.forceCC(); + + info("Stop the profiler and get the profile."); + const profile = await waitSamplingAndStopAndGetProfile(); + + const markers = getInflatedMarkerData(profile.threads[0]); + Assert.ok( + markers.some(({ data }) => data?.type === "GCSlice"), + "A GCSlice marker was recorded" + ); + Assert.ok( + markers.some(({ data }) => data?.type === "CCSlice"), + "A CCSlice marker was recorded" + ); +}); diff --git a/tools/profiler/tests/browser/browser_test_markers_parent_process.js b/tools/profiler/tests/browser/browser_test_markers_parent_process.js new file mode 100644 index 0000000000..28b82f8054 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_markers_parent_process.js @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async function test_markers_parent_process() { + info("Test markers that are generated by the browser's parent process."); + + info("Start the profiler in nostacksampling mode."); + await startProfiler({ features: ["nostacksampling"] }); + + info("Dispatch a DOMEvent"); + window.dispatchEvent(new Event("synthetic")); + + info("Stop the profiler and get the profile."); + const profile = await stopNowAndGetProfile(); + + const markers = getInflatedMarkerData(profile.threads[0]); + { + const domEventStart = markers.find( + ({ phase, data }) => + phase === INTERVAL_START && data?.eventType === "synthetic" + ); + const domEventEnd = markers.find( + ({ phase, data }) => + phase === INTERVAL_END && data?.eventType === "synthetic" + ); + ok(domEventStart, "A start DOMEvent was generated"); + ok(domEventEnd, "An end DOMEvent was generated"); + ok( + domEventEnd.data.latency > 0, + "DOMEvent had a a latency value generated." + ); + ok(domEventEnd.data.type === "DOMEvent"); + ok(domEventEnd.name === "DOMEvent"); + } + // Add more marker tests. +}); diff --git a/tools/profiler/tests/browser/browser_test_markers_preferencereads.js b/tools/profiler/tests/browser/browser_test_markers_preferencereads.js new file mode 100644 index 0000000000..0ae183f874 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_markers_preferencereads.js @@ -0,0 +1,73 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +requestLongerTimeout(10); + +const kContentPref = "font.size.variable.x-western"; + +function countPrefReadsInThread(pref, thread) { + let count = 0; + for (let payload of getPayloadsOfType(thread, "Preference")) { + if (payload.prefName === pref) { + count++; + } + } + return count; +} + +async function waitForPaintAfterLoad() { + return SpecialPowers.spawn(gBrowser.selectedBrowser, [], () => { + return new Promise(function (resolve) { + function listener() { + if (content.document.readyState == "complete") { + content.requestAnimationFrame(() => content.setTimeout(resolve, 0)); + } + } + if (content.document.readyState != "complete") { + content.document.addEventListener("readystatechange", listener); + } else { + listener(); + } + }); + }); +} + +/** + * Test the Preference Read markers. + */ +add_task(async function test_profile_preferencereads_markers() { + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + await startProfiler({ features: ["js"] }); + + const url = BASE_URL + "single_frame.html"; + await BrowserTestUtils.withNewTab(url, async contentBrowser => { + const contentPid = await SpecialPowers.spawn( + contentBrowser, + [], + () => Services.appinfo.processID + ); + + await waitForPaintAfterLoad(); + + // Ensure we read a pref in the content process. + await SpecialPowers.spawn(contentBrowser, [kContentPref], pref => { + Services.prefs.getIntPref(pref); + }); + + // Check that some Preference Read profile markers were generated. + { + const { contentThread } = await stopProfilerNowAndGetThreads(contentPid); + + Assert.greater( + countPrefReadsInThread(kContentPref, contentThread), + 0, + `Preference Read profile markers for ${kContentPref} were recorded.` + ); + } + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_profile_capture_by_pid.js b/tools/profiler/tests/browser/browser_test_profile_capture_by_pid.js new file mode 100644 index 0000000000..14d76dbcaf --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_profile_capture_by_pid.js @@ -0,0 +1,199 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function ProcessHasSamplerThread(process) { + return process.threads.some(t => t.name == "SamplerThread"); +} + +async function GetPidsWithSamplerThread() { + let parentProc = await ChromeUtils.requestProcInfo(); + + let pids = parentProc.children + .filter(ProcessHasSamplerThread) + .map(proc => proc.pid); + if (ProcessHasSamplerThread(parentProc)) { + pids.unshift(parentProc.pid); + } + return pids; +} + +// fnFilterWithContentId: Called with content child pid, returns filters to use. +// E.g.: 123 => ["GeckoMain", "pid:123"], or 123 => ["pid:456"]. +async function test_with_filter(fnFilterWithContentId) { + Assert.ok(!Services.profiler.IsActive()); + info("Clear the previous pages just in case we still some open tabs."); + await Services.profiler.ClearAllPages(); + + info("Open a tab with single_frame.html in it."); + const url = BASE_URL + "single_frame.html"; + return BrowserTestUtils.withNewTab(url, async function (contentBrowser) { + const contentPid = await SpecialPowers.spawn(contentBrowser, [], () => { + return Services.appinfo.processID; + }); + + Assert.deepEqual( + await GetPidsWithSamplerThread(), + [], + "There should be no SamplerThreads before starting the profiler" + ); + + info("Start the profiler to test filters including 'pid:<content>'."); + await startProfiler({ threads: fnFilterWithContentId(contentPid) }); + + let pidsWithSamplerThread = null; + await TestUtils.waitForCondition( + async function () { + let pidsStringBefore = JSON.stringify(pidsWithSamplerThread); + pidsWithSamplerThread = await GetPidsWithSamplerThread(); + return JSON.stringify(pidsWithSamplerThread) == pidsStringBefore; + }, + "Wait for sampler threads to stabilize after profiler start", + /* interval (ms) */ 250, + /* maxTries */ 10 + ); + + info("Capture the profile data."); + const profile = await waitSamplingAndStopAndGetProfile(); + + await TestUtils.waitForCondition(async function () { + return !(await GetPidsWithSamplerThread()).length; + }, "Wait for all sampler threads to stop after profiler stop"); + + return { contentPid, pidsWithSamplerThread, profile }; + }); +} + +add_task(async function browser_test_profile_capture_along_with_content_pid() { + const { contentPid, pidsWithSamplerThread, profile } = await test_with_filter( + contentPid => ["GeckoMain", "pid:" + contentPid] + ); + + Assert.greater( + pidsWithSamplerThread.length, + 2, + "There should be lots of SamplerThreads after starting the profiler" + ); + + let contentProcessIndex = profile.processes.findIndex( + p => p.threads[0].pid == contentPid + ); + Assert.notEqual( + contentProcessIndex, + -1, + "The content process should be present" + ); + + // Note: Some threads may not be registered, so we can't expect that many. But + // 10 is much more than the default 4. + Assert.greater( + profile.processes[contentProcessIndex].threads.length, + 10, + "The content process should have many threads" + ); + + Assert.equal( + profile.threads.length, + 1, + "The parent process should have only one thread" + ); + Assert.equal( + profile.threads[0].name, + "GeckoMain", + "The parent process should have the main thread" + ); +}); + +add_task(async function browser_test_profile_capture_along_with_other_pid() { + const parentPid = Services.appinfo.processID; + const { contentPid, pidsWithSamplerThread, profile } = await test_with_filter( + contentPid => ["GeckoMain", "pid:" + parentPid] + ); + + Assert.greater( + pidsWithSamplerThread.length, + 2, + "There should be lots of SamplerThreads after starting the profiler" + ); + + let contentProcessIndex = profile.processes.findIndex( + p => p.threads[0].pid == contentPid + ); + Assert.notEqual( + contentProcessIndex, + -1, + "The content process should be present" + ); + + Assert.equal( + profile.processes[contentProcessIndex].threads.length, + 1, + "The content process should have only one thread" + ); + + // Note: Some threads may not be registered, so we can't expect that many. But + // 10 is much more than the default 4. + Assert.greater( + profile.threads.length, + 10, + "The parent process should have many threads" + ); +}); + +add_task(async function browser_test_profile_capture_by_only_content_pid() { + const parentPid = Services.appinfo.processID; + const { contentPid, pidsWithSamplerThread, profile } = await test_with_filter( + contentPid => ["pid:" + contentPid] + ); + + // The sampler thread always runs in the parent process, see bug 1754100. + Assert.deepEqual( + pidsWithSamplerThread, + [parentPid, contentPid], + "There should only be SamplerThreads in the parent and the target child" + ); + + Assert.equal( + profile.processes.length, + 1, + "There should only be one child process" + ); + // Note: Some threads may not be registered, so we can't expect that many. But + // 10 is much more than the default 4. + Assert.greater( + profile.processes[0].threads.length, + 10, + "The child process should have many threads" + ); + Assert.equal( + profile.processes[0].threads[0].pid, + contentPid, + "The only child process should be our content" + ); +}); + +add_task(async function browser_test_profile_capture_by_only_parent_pid() { + const parentPid = Services.appinfo.processID; + const { pidsWithSamplerThread, profile } = await test_with_filter( + contentPid => ["pid:" + parentPid] + ); + + Assert.deepEqual( + pidsWithSamplerThread, + [parentPid], + "There should only be a SamplerThread in the parent" + ); + + // Note: Some threads may not be registered, so we can't expect that many. But + // 10 is much more than the default 4. + Assert.greater( + profile.threads.length, + 10, + "The parent process should have many threads" + ); + Assert.equal( + profile.processes.length, + 0, + "There should be no child processes" + ); +}); diff --git a/tools/profiler/tests/browser/browser_test_profile_fission.js b/tools/profiler/tests/browser/browser_test_profile_fission.js new file mode 100644 index 0000000000..ce4996590d --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_profile_fission.js @@ -0,0 +1,191 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +if (SpecialPowers.useRemoteSubframes) { + // Bug 1586105: these tests could time out in some extremely slow conditions, + // when fission is enabled. + // Requesting a longer timeout should make it pass. + requestLongerTimeout(2); +} + +add_task(async function test_profile_fission_no_private_browsing() { + // Requesting the complete log to be able to debug Bug 1586105. + SimpleTest.requestCompleteLog(); + Assert.ok(!Services.profiler.IsActive()); + info("Clear the previous pages just in case we still have some open tabs."); + await Services.profiler.ClearAllPages(); + + info( + "Start the profiler to test the page information with single frame page." + ); + await startProfiler(); + + info("Open a private window with single_frame.html in it."); + const win = await BrowserTestUtils.openNewBrowserWindow({ + fission: true, + }); + + try { + const url = BASE_URL_HTTPS + "single_frame.html"; + const contentBrowser = win.gBrowser.selectedBrowser; + BrowserTestUtils.startLoadingURIString(contentBrowser, url); + await BrowserTestUtils.browserLoaded(contentBrowser, false, url); + + const parentPid = Services.appinfo.processID; + const contentPid = await SpecialPowers.spawn(contentBrowser, [], () => { + return Services.appinfo.processID; + }); + + // Getting the active Browser ID to assert the page info tabID later. + const activeTabID = contentBrowser.browsingContext.browserId; + + info("Capture the profile data."); + const { profile, contentProcess, contentThread } = + await stopProfilerNowAndGetThreads(contentPid); + + Assert.equal( + contentThread.isPrivateBrowsing, + false, + "The content process has the private browsing flag set to false." + ); + + Assert.equal( + contentThread.userContextId, + 0, + "The content process has the information about the container used for this process" + ); + + info( + "Check if the captured page is the one with correct values we created." + ); + + let pageFound = false; + for (const page of contentProcess.pages) { + if (page.url == url) { + Assert.equal(page.url, url); + Assert.equal(typeof page.tabID, "number"); + Assert.equal(page.tabID, activeTabID); + Assert.equal(typeof page.innerWindowID, "number"); + // Top level document will have no embedder. + Assert.equal(page.embedderInnerWindowID, 0); + Assert.equal(typeof page.isPrivateBrowsing, "boolean"); + Assert.equal(page.isPrivateBrowsing, false); + pageFound = true; + break; + } + } + Assert.equal(pageFound, true); + + info("Check that the profiling logs exist with the expected properties."); + Assert.equal(typeof profile.profilingLog, "object"); + Assert.equal(typeof profile.profilingLog[parentPid], "object"); + const parentLog = profile.profilingLog[parentPid]; + Assert.equal(typeof parentLog.profilingLogBegin_TSms, "number"); + Assert.equal(typeof parentLog.profilingLogEnd_TSms, "number"); + Assert.equal(typeof parentLog.bufferGlobalController, "object"); + Assert.equal( + typeof parentLog.bufferGlobalController.controllerCreationTime_TSms, + "number" + ); + + Assert.equal(typeof profile.profileGatheringLog, "object"); + Assert.equal(typeof profile.profileGatheringLog[parentPid], "object"); + Assert.equal( + typeof profile.profileGatheringLog[parentPid] + .profileGatheringLogBegin_TSms, + "number" + ); + Assert.equal( + typeof profile.profileGatheringLog[parentPid].profileGatheringLogEnd_TSms, + "number" + ); + + Assert.equal(typeof contentProcess.profilingLog, "object"); + Assert.equal(typeof contentProcess.profilingLog[contentPid], "object"); + Assert.equal( + typeof contentProcess.profilingLog[contentPid].profilingLogBegin_TSms, + "number" + ); + Assert.equal( + typeof contentProcess.profilingLog[contentPid].profilingLogEnd_TSms, + "number" + ); + + Assert.equal(typeof contentProcess.profileGatheringLog, "undefined"); + } finally { + await BrowserTestUtils.closeWindow(win); + } +}); + +add_task(async function test_profile_fission_private_browsing() { + // Requesting the complete log to be able to debug Bug 1586105. + SimpleTest.requestCompleteLog(); + Assert.ok(!Services.profiler.IsActive()); + info("Clear the previous pages just in case we still have some open tabs."); + await Services.profiler.ClearAllPages(); + + info( + "Start the profiler to test the page information with single frame page." + ); + await startProfiler(); + + info("Open a private window with single_frame.html in it."); + const win = await BrowserTestUtils.openNewBrowserWindow({ + private: true, + fission: true, + }); + + try { + const url = BASE_URL_HTTPS + "single_frame.html"; + const contentBrowser = win.gBrowser.selectedBrowser; + BrowserTestUtils.startLoadingURIString(contentBrowser, url); + await BrowserTestUtils.browserLoaded(contentBrowser, false, url); + + const contentPid = await SpecialPowers.spawn(contentBrowser, [], () => { + return Services.appinfo.processID; + }); + + // Getting the active Browser ID to assert the page info tabID later. + const activeTabID = contentBrowser.browsingContext.browserId; + + info("Capture the profile data."); + const { contentProcess, contentThread } = + await stopProfilerNowAndGetThreads(contentPid); + + Assert.equal( + contentThread.isPrivateBrowsing, + true, + "The content process has the private browsing flag set to true." + ); + + Assert.equal( + contentThread.userContextId, + 0, + "The content process has the information about the container used for this process" + ); + + info( + "Check if the captured page is the one with correct values we created." + ); + + let pageFound = false; + for (const page of contentProcess.pages) { + if (page.url == url) { + Assert.equal(page.url, url); + Assert.equal(typeof page.tabID, "number"); + Assert.equal(page.tabID, activeTabID); + Assert.equal(typeof page.innerWindowID, "number"); + // Top level document will have no embedder. + Assert.equal(page.embedderInnerWindowID, 0); + Assert.equal(typeof page.isPrivateBrowsing, "boolean"); + Assert.equal(page.isPrivateBrowsing, true); + pageFound = true; + break; + } + } + Assert.equal(pageFound, true); + } finally { + await BrowserTestUtils.closeWindow(win); + } +}); diff --git a/tools/profiler/tests/browser/browser_test_profile_multi_frame_page_info.js b/tools/profiler/tests/browser/browser_test_profile_multi_frame_page_info.js new file mode 100644 index 0000000000..854587678d --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_profile_multi_frame_page_info.js @@ -0,0 +1,83 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +if (SpecialPowers.useRemoteSubframes) { + // Bug 1586105: these tests could time out in some extremely slow conditions, + // when fission is enabled. + // Requesting a longer timeout should make it pass. + requestLongerTimeout(2); +} + +add_task(async function test_profile_multi_frame_page_info() { + // Requesting the complete log to be able to debug Bug 1586105. + SimpleTest.requestCompleteLog(); + Assert.ok(!Services.profiler.IsActive()); + info("Clear the previous pages just in case we still have some open tabs."); + await Services.profiler.ClearAllPages(); + + info( + "Start the profiler to test the page information with multi frame page." + ); + await startProfiler(); + + info("Open a tab with multi_frame.html in it."); + // multi_frame.html embeds single_frame.html inside an iframe. + const url = BASE_URL + "multi_frame.html"; + await BrowserTestUtils.withNewTab(url, async function (contentBrowser) { + const contentPid = await SpecialPowers.spawn(contentBrowser, [], () => { + return Services.appinfo.processID; + }); + + // Getting the active Browser ID to assert the page info tabID later. + const win = Services.wm.getMostRecentWindow("navigator:browser"); + const activeTabID = win.gBrowser.selectedBrowser.browsingContext.browserId; + + info("Capture the profile data."); + const { contentProcess } = await stopProfilerNowAndGetThreads(contentPid); + + info( + "Check if the captured pages are the ones with correct values we created." + ); + + let parentPage; + let foundPage = 0; + for (const page of contentProcess.pages) { + // Parent page + if (page.url == url) { + Assert.equal(page.url, url); + Assert.equal(typeof page.tabID, "number"); + Assert.equal(page.tabID, activeTabID); + Assert.equal(typeof page.innerWindowID, "number"); + // Top level document will have no embedder. + Assert.equal(page.embedderInnerWindowID, 0); + Assert.equal(typeof page.isPrivateBrowsing, "boolean"); + Assert.equal(page.isPrivateBrowsing, false); + parentPage = page; + foundPage++; + break; + } + } + + Assert.notEqual(typeof parentPage, "undefined"); + + for (const page of contentProcess.pages) { + // Child page (iframe) + if (page.url == BASE_URL + "single_frame.html") { + Assert.equal(page.url, BASE_URL + "single_frame.html"); + Assert.equal(typeof page.tabID, "number"); + Assert.equal(page.tabID, activeTabID); + Assert.equal(typeof page.innerWindowID, "number"); + Assert.equal(typeof page.embedderInnerWindowID, "number"); + Assert.notEqual(typeof parentPage, "undefined"); + Assert.equal(page.embedderInnerWindowID, parentPage.innerWindowID); + Assert.equal(typeof page.isPrivateBrowsing, "boolean"); + Assert.equal(page.isPrivateBrowsing, false); + foundPage++; + break; + } + } + + Assert.equal(foundPage, 2); + }); +}); diff --git a/tools/profiler/tests/browser/browser_test_profile_single_frame_page_info.js b/tools/profiler/tests/browser/browser_test_profile_single_frame_page_info.js new file mode 100644 index 0000000000..0385413c16 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_profile_single_frame_page_info.js @@ -0,0 +1,132 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +if (SpecialPowers.useRemoteSubframes) { + // Bug 1586105: these tests could time out in some extremely slow conditions, + // when fission is enabled. + // Requesting a longer timeout should make it pass. + requestLongerTimeout(2); +} + +add_task(async function test_profile_single_frame_page_info() { + // Requesting the complete log to be able to debug Bug 1586105. + SimpleTest.requestCompleteLog(); + Assert.ok(!Services.profiler.IsActive()); + info("Clear the previous pages just in case we still have some open tabs."); + await Services.profiler.ClearAllPages(); + + info( + "Start the profiler to test the page information with single frame page." + ); + await startProfiler(); + + info("Open a tab with single_frame.html in it."); + const url = BASE_URL + "single_frame.html"; + await BrowserTestUtils.withNewTab(url, async function (contentBrowser) { + const contentPid = await SpecialPowers.spawn(contentBrowser, [], () => { + return Services.appinfo.processID; + }); + + // Getting the active Browser ID to assert the page info tabID later. + const win = Services.wm.getMostRecentWindow("navigator:browser"); + const activeTabID = win.gBrowser.selectedBrowser.browsingContext.browserId; + + info("Capture the profile data."); + const { contentProcess } = await stopProfilerNowAndGetThreads(contentPid); + + info( + "Check if the captured page is the one with correct values we created." + ); + + let pageFound = false; + for (const page of contentProcess.pages) { + if (page.url == url) { + Assert.equal(page.url, url); + Assert.equal(typeof page.tabID, "number"); + Assert.equal(page.tabID, activeTabID); + Assert.equal(typeof page.innerWindowID, "number"); + // Top level document will have no embedder. + Assert.equal(page.embedderInnerWindowID, 0); + Assert.equal(typeof page.isPrivateBrowsing, "boolean"); + Assert.equal(page.isPrivateBrowsing, false); + pageFound = true; + break; + } + } + Assert.equal(pageFound, true); + }); +}); + +add_task(async function test_profile_private_browsing() { + // Requesting the complete log to be able to debug Bug 1586105. + SimpleTest.requestCompleteLog(); + Assert.ok(!Services.profiler.IsActive()); + info("Clear the previous pages just in case we still have some open tabs."); + await Services.profiler.ClearAllPages(); + + info( + "Start the profiler to test the page information with single frame page." + ); + await startProfiler(); + + info("Open a private window with single_frame.html in it."); + const win = await BrowserTestUtils.openNewBrowserWindow({ + fission: false, + private: true, + }); + + try { + const url = BASE_URL_HTTPS + "single_frame.html"; + const contentBrowser = win.gBrowser.selectedBrowser; + BrowserTestUtils.startLoadingURIString(contentBrowser, url); + await BrowserTestUtils.browserLoaded(contentBrowser, false, url); + + const contentPid = await SpecialPowers.spawn(contentBrowser, [], () => { + return Services.appinfo.processID; + }); + + // Getting the active Browser ID to assert the page info tabID later. + const activeTabID = contentBrowser.browsingContext.browserId; + + info("Capture the profile data."); + const { contentProcess, contentThread } = + await stopProfilerNowAndGetThreads(contentPid); + + // This information is available with fission only. + Assert.equal( + contentThread.isPrivateBrowsing, + undefined, + "The content process has no private browsing flag." + ); + + Assert.equal( + contentThread.userContextId, + undefined, + "The content process has no information about the container used for this process." + ); + + info( + "Check if the captured page is the one with correct values we created." + ); + + let pageFound = false; + for (const page of contentProcess.pages) { + if (page.url == url) { + Assert.equal(page.url, url); + Assert.equal(typeof page.tabID, "number"); + Assert.equal(page.tabID, activeTabID); + Assert.equal(typeof page.innerWindowID, "number"); + // Top level document will have no embedder. + Assert.equal(page.embedderInnerWindowID, 0); + Assert.equal(typeof page.isPrivateBrowsing, "boolean"); + Assert.equal(page.isPrivateBrowsing, true); + pageFound = true; + break; + } + } + Assert.equal(pageFound, true); + } finally { + await BrowserTestUtils.closeWindow(win); + } +}); diff --git a/tools/profiler/tests/browser/browser_test_profile_slow_capture.js b/tools/profiler/tests/browser/browser_test_profile_slow_capture.js new file mode 100644 index 0000000000..4a675b84d1 --- /dev/null +++ b/tools/profiler/tests/browser/browser_test_profile_slow_capture.js @@ -0,0 +1,104 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async function browser_test_profile_slow_capture() { + Assert.ok(!Services.profiler.IsActive()); + info("Clear the previous pages just in case we still some open tabs."); + await Services.profiler.ClearAllPages(); + + info( + "Start the profiler to test the page information with single frame page." + ); + await startProfiler({ threads: ["GeckoMain", "test-debug-child-slow-json"] }); + + info("Open a tab with single_frame.html in it."); + const url = BASE_URL + "single_frame.html"; + await BrowserTestUtils.withNewTab(url, async function (contentBrowser) { + const contentPid = await SpecialPowers.spawn(contentBrowser, [], () => { + return Services.appinfo.processID; + }); + + // Getting the active Browser ID to assert the page info tabID later. + const win = Services.wm.getMostRecentWindow("navigator:browser"); + const activeTabID = win.gBrowser.selectedBrowser.browsingContext.browserId; + + info("Capture the profile data."); + const profile = await waitSamplingAndStopAndGetProfile(); + + let pageFound = false; + // We need to find the correct content process for that tab. + let contentProcess = profile.processes.find( + p => p.threads[0].pid == contentPid + ); + + if (!contentProcess) { + throw new Error( + `Could not find the content process with given pid: ${contentPid}` + ); + } + + info( + "Check if the captured page is the one with correct values we created." + ); + + for (const page of contentProcess.pages) { + if (page.url == url) { + Assert.equal(page.url, url); + Assert.equal(typeof page.tabID, "number"); + Assert.equal(page.tabID, activeTabID); + Assert.equal(typeof page.innerWindowID, "number"); + // Top level document will have no embedder. + Assert.equal(page.embedderInnerWindowID, 0); + pageFound = true; + break; + } + } + Assert.equal(pageFound, true); + + info("Flush slow processes with a quick profile."); + await startProfiler(); + for (let i = 0; i < 10; ++i) { + await Services.profiler.waitOnePeriodicSampling(); + } + await stopNowAndGetProfile(); + }); +}); + +add_task(async function browser_test_profile_very_slow_capture() { + Assert.ok(!Services.profiler.IsActive()); + info("Clear the previous pages just in case we still some open tabs."); + await Services.profiler.ClearAllPages(); + + info( + "Start the profiler to test the page information with single frame page." + ); + await startProfiler({ + threads: ["GeckoMain", "test-debug-child-very-slow-json"], + }); + + info("Open a tab with single_frame.html in it."); + const url = BASE_URL + "single_frame.html"; + await BrowserTestUtils.withNewTab(url, async function (contentBrowser) { + const contentPid = await SpecialPowers.spawn(contentBrowser, [], () => { + return Services.appinfo.processID; + }); + + info("Capture the profile data."); + const profile = await waitSamplingAndStopAndGetProfile(); + + info("Check that the content process is missing."); + + let contentProcessIndex = profile.processes.findIndex( + p => p.threads[0].pid == contentPid + ); + Assert.equal(contentProcessIndex, -1); + + info("Flush slow processes with a quick profile."); + await startProfiler(); + for (let i = 0; i < 10; ++i) { + await Services.profiler.waitOnePeriodicSampling(); + } + await stopNowAndGetProfile(); + }); +}); diff --git a/tools/profiler/tests/browser/do_work_500ms.html b/tools/profiler/tests/browser/do_work_500ms.html new file mode 100644 index 0000000000..9713a80671 --- /dev/null +++ b/tools/profiler/tests/browser/do_work_500ms.html @@ -0,0 +1,41 @@ +<!DOCTYPE html> +<html> +<head> + <meta charset="utf-8"> + <title>Do some work for 500ms</title> + <script> + const milliseconds = 500; + const millisecondsPerBatch = 10; + const end = Date.now() + milliseconds; + window.total = 0; + let i = 0; + + /** + * Do work for a set number of milliseconds, but only do the work in batches + * so the browser does not get unresponsive. + */ + function doWork() { + const batchEnd = Date.now() + millisecondsPerBatch; + // Do some work for a set amount of time. + while (Date.now() < end) { + // Do some kind of work that is non-deterministic to guard against optimizations. + window.total += Math.random(); + i++; + + // Check if a batch is done yet. + if (Date.now() > batchEnd) { + // Defer the rest of the work into a micro task. Keep on doing this until + // the total milliseconds have elapsed. + setTimeout(doWork, 0); + return; + } + } + } + + doWork(); + </script> +</head> +<body> + Do some work for 500ms. +</body> +</html> diff --git a/tools/profiler/tests/browser/firefox-logo-nightly.svg b/tools/profiler/tests/browser/firefox-logo-nightly.svg new file mode 100644 index 0000000000..f1af370d87 --- /dev/null +++ b/tools/profiler/tests/browser/firefox-logo-nightly.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 953.37 984"><defs><linearGradient id="linear-gradient" x1="-14706.28" y1="9250.14" x2="-14443.04" y2="9250.14" gradientTransform="matrix(0.76, 0.03, 0.05, -1.12, 11485.47, 11148)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#0083ff"/><stop offset="0.1" stop-color="#0092f8"/><stop offset="0.31" stop-color="#00abeb"/><stop offset="0.52" stop-color="#00bee1"/><stop offset="0.75" stop-color="#00c8dc"/><stop offset="1" stop-color="#00ccda"/></linearGradient><radialGradient id="radial-gradient" cx="-7588.66" cy="8866.53" r="791.23" gradientTransform="matrix(1.23, 0, 0, -1.22, 9958.21, 11048.11)" gradientUnits="userSpaceOnUse"><stop offset="0.02" stop-color="#005fe7"/><stop offset="0.18" stop-color="#0042b4"/><stop offset="0.32" stop-color="#002989"/><stop offset="0.4" stop-color="#002079"/><stop offset="0.47" stop-color="#131d78"/><stop offset="0.66" stop-color="#3b1676"/><stop offset="0.75" stop-color="#4a1475"/></radialGradient><linearGradient id="linear-gradient-2" x1="539.64" y1="254.8" x2="348.2" y2="881.03" gradientTransform="matrix(1, 0, 0, -1, 1, 984)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#000f43" stop-opacity="0.4"/><stop offset="0.48" stop-color="#001962" stop-opacity="0.17"/><stop offset="1" stop-color="#002079" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-3" x1="540.64" y1="254.8" x2="349.2" y2="881.03" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" xlink:href="#linear-gradient-2"/><linearGradient id="linear-gradient-4" x1="-8367.12" y1="7348.87" x2="-8482.36" y2="7357.76" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#812cc9"/><stop offset="1" stop-color="#005fe7"/></linearGradient><linearGradient id="linear-gradient-5" x1="-8449.89" y1="7496.97" x2="-8341.94" y2="7609.09" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0.05" stop-color="#005fe7"/><stop offset="0.18" stop-color="#065de6"/><stop offset="0.35" stop-color="#1856e1"/><stop offset="0.56" stop-color="#354adb"/><stop offset="0.78" stop-color="#5d3ad1"/><stop offset="0.95" stop-color="#812cc9"/></linearGradient><linearGradient id="linear-gradient-6" x1="-8653.41" y1="7245.3" x2="-8422.52" y2="7244.76" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#002079"/><stop offset="0.99" stop-color="#a238ff"/></linearGradient><radialGradient id="radial-gradient-2" cx="644.11" cy="599.83" fx="785.0454815336918" fy="470.6889181532662" r="793.95" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" gradientUnits="userSpaceOnUse"><stop offset="0.2" stop-color="#00fdff"/><stop offset="0.26" stop-color="#0af1ff"/><stop offset="0.37" stop-color="#23d2ff"/><stop offset="0.52" stop-color="#4da0ff"/><stop offset="0.69" stop-color="#855bff"/><stop offset="0.77" stop-color="#a238ff"/><stop offset="0.81" stop-color="#a738fd"/><stop offset="0.86" stop-color="#b539f9"/><stop offset="0.9" stop-color="#cd39f1"/><stop offset="0.96" stop-color="#ee3ae6"/><stop offset="0.98" stop-color="#ff3be0"/></radialGradient><linearGradient id="linear-gradient-7" x1="-7458.97" y1="9093.17" x2="-7531.06" y2="8282.84" gradientTransform="matrix(1.23, 0, 0, -1.22, 9958.21, 11048.11)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#00ec00"/><stop offset="0.1" stop-color="#00e244"/><stop offset="0.22" stop-color="#00d694"/><stop offset="0.31" stop-color="#00cfc7"/><stop offset="0.35" stop-color="#00ccda"/><stop offset="0.42" stop-color="#0bc2dd" stop-opacity="0.92"/><stop offset="0.57" stop-color="#29a7e4" stop-opacity="0.72"/><stop offset="0.77" stop-color="#597df0" stop-opacity="0.4"/><stop offset="1" stop-color="#9448ff" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-8" x1="-8926.61" y1="7680.53" x2="-8790.14" y2="7680.53" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#005fe7"/><stop offset="0.46" stop-color="#0071f3" stop-opacity="0.51"/><stop offset="0.83" stop-color="#007efc" stop-opacity="0.14"/><stop offset="1" stop-color="#0083ff" stop-opacity="0"/></linearGradient><radialGradient id="radial-gradient-3" cx="-8914.62" cy="7721.05" r="165.97" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0.63" stop-color="#ffe302" stop-opacity="0"/><stop offset="0.67" stop-color="#ffe302" stop-opacity="0.05"/><stop offset="0.75" stop-color="#ffe302" stop-opacity="0.19"/><stop offset="0.86" stop-color="#ffe302" stop-opacity="0.4"/><stop offset="0.99" stop-color="#ffe302" stop-opacity="0.7"/></radialGradient><linearGradient id="linear-gradient-9" x1="214.02" y1="2032.47" x2="96.19" y2="2284.31" gradientTransform="matrix(0.99, 0.1, 0.1, -0.99, -250.1, 2306.29)" gradientUnits="userSpaceOnUse"><stop offset="0.19" stop-color="#4a1475" stop-opacity="0.5"/><stop offset="0.62" stop-color="#2277ac" stop-opacity="0.23"/><stop offset="0.94" stop-color="#00ccda" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-10" x1="-38.44" y1="278.18" x2="55.67" y2="171.29" gradientTransform="matrix(0.99, 0.1, 0.1, -0.99, 229.04, 745.87)" gradientUnits="userSpaceOnUse"><stop offset="0.01" stop-color="#002079" stop-opacity="0.5"/><stop offset="1" stop-color="#0083ff" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-11" x1="142.45" y1="96.25" x2="142.5" y2="149.68" gradientTransform="matrix(0.99, 0.1, 0.1, -0.99, 229.04, 745.87)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#4a1475" stop-opacity="0.9"/><stop offset="0.18" stop-color="#6720a2" stop-opacity="0.6"/><stop offset="0.38" stop-color="#812acb" stop-opacity="0.34"/><stop offset="0.57" stop-color="#9332e8" stop-opacity="0.15"/><stop offset="0.76" stop-color="#9e36f9" stop-opacity="0.04"/><stop offset="0.93" stop-color="#a238ff" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-12" x1="620.52" y1="947.88" x2="926.18" y2="264.39" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#00ec00" stop-opacity="0"/><stop offset="0.28" stop-color="#00dc6d" stop-opacity="0.5"/><stop offset="0.5" stop-color="#00d1bb" stop-opacity="0.86"/><stop offset="0.6" stop-color="#00ccda"/><stop offset="0.68" stop-color="#04c9db"/><stop offset="0.75" stop-color="#0fc1df"/><stop offset="0.83" stop-color="#23b2e6"/><stop offset="0.9" stop-color="#3e9ef0"/><stop offset="0.98" stop-color="#6184fc"/><stop offset="0.99" stop-color="#6680fe"/></linearGradient><linearGradient id="linear-gradient-13" x1="680.88" y1="554.79" x2="536.1" y2="166.04" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#0083ff"/><stop offset="0.04" stop-color="#0083ff" stop-opacity="0.92"/><stop offset="0.14" stop-color="#0083ff" stop-opacity="0.71"/><stop offset="0.26" stop-color="#0083ff" stop-opacity="0.52"/><stop offset="0.37" stop-color="#0083ff" stop-opacity="0.36"/><stop offset="0.49" stop-color="#0083ff" stop-opacity="0.23"/><stop offset="0.61" stop-color="#0083ff" stop-opacity="0.13"/><stop offset="0.73" stop-color="#0083ff" stop-opacity="0.06"/><stop offset="0.86" stop-color="#0083ff" stop-opacity="0.01"/><stop offset="1" stop-color="#0083ff" stop-opacity="0"/></linearGradient></defs><title>firefox-logo-nightly</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><g id="Layer_2-2" data-name="Layer 2"><g id="Firefox"><path d="M770.28,91.56c-23.95,27.88-35.1,90.64-10.82,154.26s61.5,49.8,84.7,114.67c30.62,85.6,16.37,200.59,16.37,200.59s36.81,106.61,62.47-6.63C979.79,341.74,770.28,143.94,770.28,91.56Z" style="fill:url(#linear-gradient)"/><path id="_Path_" data-name=" Path " d="M476.92,972.83c245.24,0,443.9-199.74,443.9-446s-198.66-446-443.66-446S33.5,280.51,33.5,526.8C33,773.33,231.92,972.83,476.92,972.83Z" style="fill:url(#radial-gradient)"/><path d="M810.67,803.64a246.8,246.8,0,0,1-30.12,18.18,705.31,705.31,0,0,0,38.3-63c9.46-10.47,18.13-20.65,25.19-31.65,3.44-5.41,7.31-12.08,11.42-19.82,24.92-44.9,52.4-117.56,53.18-192.2v-5.66a257.25,257.25,0,0,0-5.71-55.75c.2,1.43.38,2.86.56,4.29-.22-1.1-.41-2.21-.64-3.31.37,2,.66,4,1,6,5.09,43.22,1.47,85.37-16.68,116.45-.29.45-.58.88-.87,1.32,9.41-47.23,12.56-99.39,2.09-151.6,0,0-4.19-25.38-35.38-102.44-18-44.35-49.83-80.72-78-107.21-24.69-30.55-47.11-51-59.47-64.06C689.72,126,678.9,105.61,674.45,92.31c-3.85-1.93-53.14-49.81-57.05-51.63-21.51,33.35-89.16,137.67-57,235.15,14.58,44.17,51.47,90,90.07,115.74,1.69,1.94,23,25,33.09,77.16,10.45,53.85,5,95.86-16.54,158C641.73,681.24,577,735.12,516.3,740.63c-129.67,11.78-177.15-65.11-177.15-65.11C385.49,694,436.72,690.17,467.87,671c31.4-19.43,50.39-33.83,65.81-28.15C548.86,648.43,561,632,550.1,615a78.5,78.5,0,0,0-79.4-34.57c-31.43,5.11-60.23,30-101.41,5.89a86.29,86.29,0,0,1-7.73-5.06c-2.71-1.79,8.83,2.72,6.13.69-8-4.35-22.2-13.84-25.88-17.22-.61-.56,6.22,2.18,5.61,1.62-38.51-31.71-33.7-53.13-32.49-66.57,1-10.75,8-24.52,19.75-30.11,5.69,3.11,9.24,5.48,9.24,5.48s-2.43-5-3.74-7.58c.46-.2.9-.15,1.36-.34,4.66,2.25,15,8.1,20.41,11.67,7.07,5,9.33,9.44,9.33,9.44s1.86-1,.48-5.37c-.5-1.78-2.65-7.45-9.65-13.17h.44A81.61,81.61,0,0,1,374.42,478c2-7.18,5.53-14.68,4.75-28.09-.48-9.43-.26-11.87-1.92-15.51-1.49-3.13.83-4.35,3.42-1.1a32.5,32.5,0,0,0-2.21-7.4v-.24c3.23-11.24,68.25-40.46,73-43.88A67.2,67.2,0,0,0,470.59,361c3.62-5.76,6.34-13.85,7-26.11.36-8.84-3.76-14.73-69.51-21.62-18-1.77-28.53-14.8-34.53-26.82-1.09-2.59-2.21-4.94-3.33-7.28a57.68,57.68,0,0,1-2.56-8.43c10.75-30.87,28.81-57,55.37-76.7,1.45-1.32-5.78.34-4.34-1,1.69-1.54,12.71-6,14.79-7,2.54-1.2-10.88-6.9-22.73-5.51-12.07,1.36-14.63,2.8-21.07,5.53,2.67-2.66,11.17-6.15,9.18-6.13-13,2-29.18,9.56-43,18.12a10.66,10.66,0,0,1,.83-4.35c-6.44,2.73-22.26,13.79-26.87,23.14a44.29,44.29,0,0,0,.27-5.4,84.17,84.17,0,0,0-13.19,13.82l-.24.22c-37.36-15-70.23-16-98.05-9.28-6.09-6.11-9.06-1.64-22.91-32.07-.94-1.83.72,1.81,0,0-2.28-5.9,1.39,7.87,0,0-23.28,18.37-53.92,39.19-68.63,53.89-.18.59,17.16-4.9,0,0-6,1.72-5.6,5.28-6.51,37.5-.22,2.44,0,5.18-.22,7.38-11.75,15-19.75,27.64-22.78,34.21-15.19,26.18-31.93,67-48.15,131.55A334.82,334.82,0,0,1,75.2,398.36C61.71,432.63,48.67,486.44,46.07,569.3A482.08,482.08,0,0,1,58.6,518.64,473,473,0,0,0,93.33,719.71c9.33,22.82,24.76,57.46,51,95.4C226.9,902,343.31,956,472.21,956,606.79,956,727.64,897.13,810.67,803.64Z" style="fill:url(#linear-gradient-2)"/><path d="M810.67,803.64a246.8,246.8,0,0,1-30.12,18.18,705.31,705.31,0,0,0,38.3-63c9.46-10.47,18.13-20.65,25.19-31.65,3.44-5.41,7.31-12.08,11.42-19.82,24.92-44.9,52.4-117.56,53.18-192.2v-5.66a257.25,257.25,0,0,0-5.71-55.75c.2,1.43.38,2.86.56,4.29-.22-1.1-.41-2.21-.64-3.31.37,2,.66,4,1,6,5.09,43.22,1.47,85.37-16.68,116.45-.29.45-.58.88-.87,1.32,9.41-47.23,12.56-99.39,2.09-151.6,0,0-4.19-25.38-35.38-102.44-18-44.35-49.83-80.72-78-107.21-24.69-30.55-47.11-51-59.47-64.06C689.72,126,678.9,105.61,674.45,92.31c-3.85-1.93-53.14-49.81-57.05-51.63-21.51,33.35-89.16,137.67-57,235.15,14.58,44.17,51.47,90,90.07,115.74,1.69,1.94,23,25,33.09,77.16,10.45,53.85,5,95.86-16.54,158C641.73,681.24,577,735.12,516.3,740.63c-129.67,11.78-177.15-65.11-177.15-65.11C385.49,694,436.72,690.17,467.87,671c31.4-19.43,50.39-33.83,65.81-28.15C548.86,648.43,561,632,550.1,615a78.5,78.5,0,0,0-79.4-34.57c-31.43,5.11-60.23,30-101.41,5.89a86.29,86.29,0,0,1-7.73-5.06c-2.71-1.79,8.83,2.72,6.13.69-8-4.35-22.2-13.84-25.88-17.22-.61-.56,6.22,2.18,5.61,1.62-38.51-31.71-33.7-53.13-32.49-66.57,1-10.75,8-24.52,19.75-30.11,5.69,3.11,9.24,5.48,9.24,5.48s-2.43-5-3.74-7.58c.46-.2.9-.15,1.36-.34,4.66,2.25,15,8.1,20.41,11.67,7.07,5,9.33,9.44,9.33,9.44s1.86-1,.48-5.37c-.5-1.78-2.65-7.45-9.65-13.17h.44A81.61,81.61,0,0,1,374.42,478c2-7.18,5.53-14.68,4.75-28.09-.48-9.43-.26-11.87-1.92-15.51-1.49-3.13.83-4.35,3.42-1.1a32.5,32.5,0,0,0-2.21-7.4v-.24c3.23-11.24,68.25-40.46,73-43.88A67.2,67.2,0,0,0,470.59,361c3.62-5.76,6.34-13.85,7-26.11.36-8.84-3.76-14.73-69.51-21.62-18-1.77-28.53-14.8-34.53-26.82-1.09-2.59-2.21-4.94-3.33-7.28a57.68,57.68,0,0,1-2.56-8.43c10.75-30.87,28.81-57,55.37-76.7,1.45-1.32-5.78.34-4.34-1,1.69-1.54,12.71-6,14.79-7,2.54-1.2-10.88-6.9-22.73-5.51-12.07,1.36-14.63,2.8-21.07,5.53,2.67-2.66,11.17-6.15,9.18-6.13-13,2-29.18,9.56-43,18.12a10.66,10.66,0,0,1,.83-4.35c-6.44,2.73-22.26,13.79-26.87,23.14a44.29,44.29,0,0,0,.27-5.4,84.17,84.17,0,0,0-13.19,13.82l-.24.22c-37.36-15-70.23-16-98.05-9.28-6.09-6.11-9.06-1.64-22.91-32.07-.94-1.83.72,1.81,0,0-2.28-5.9,1.39,7.87,0,0-23.28,18.37-53.92,39.19-68.63,53.89-.18.59,17.16-4.9,0,0-6,1.72-5.6,5.28-6.51,37.5-.22,2.44,0,5.18-.22,7.38-11.75,15-19.75,27.64-22.78,34.21-15.19,26.18-31.93,67-48.15,131.55A334.82,334.82,0,0,1,75.2,398.36C61.71,432.63,48.67,486.44,46.07,569.3A482.08,482.08,0,0,1,58.6,518.64,473,473,0,0,0,93.33,719.71c9.33,22.82,24.76,57.46,51,95.4C226.9,902,343.31,956,472.21,956,606.79,956,727.64,897.13,810.67,803.64Z" style="fill:url(#linear-gradient-3)"/><path d="M711.1,866.71c162.87-18.86,235-186.7,142.38-190C769.85,674,634,875.61,711.1,866.71Z" style="fill:url(#linear-gradient-4)"/><path d="M865.21,642.42C977.26,577.21,948,436.34,948,436.34s-43.25,50.24-72.62,130.32C846.4,646,797.84,681.81,865.21,642.42Z" style="fill:url(#linear-gradient-5)"/><path d="M509.47,950.06C665.7,999.91,800,876.84,717.21,835.74,642,798.68,435.32,926.49,509.47,950.06Z" style="fill:url(#linear-gradient-6)"/><path d="M638.58,21.42l.53-.57A1.7,1.7,0,0,0,638.58,21.42ZM876.85,702.23c3.8-5.36,8.94-22.53,13.48-30.21,27.58-44.52,27.78-80,27.78-80.84,16.66-83.22,15.15-117.2,4.9-180-8.25-50.6-44.32-123.09-75.57-158-32.2-36-9.51-24.25-40.69-50.52-27.33-30.29-53.82-60.29-68.25-72.36C634.22,43.09,636.57,24.58,638.58,21.42c-.34.37-.84.92-1.47,1.64C635.87,18.14,635,14,635,14s-57,57-69,152c-7.83,62,15.38,126.68,49,168a381.62,381.62,0,0,0,59,58h0c25.4,36.48,39.38,81.49,39.38,129.91,0,121.24-98.34,219.53-219.65,219.53a220.14,220.14,0,0,1-49.13-5.52c-57.24-10.92-90.3-39.8-106.78-59.41-9.45-11.23-13.46-19.42-13.46-19.42,51.28,18.37,108,14.53,142.47-4.52,34.75-19.26,55.77-33.55,72.84-27.92,16.82,5.61,30.21-10.67,18.2-27.54-11.77-16.85-42.4-41-87.88-34.29-34.79,5.07-66.66,29.76-112.24,5.84a97.34,97.34,0,0,1-8.55-5c-3-1.77,9.77,2.69,6.79.68-8.87-4.32-24.57-13.73-28.64-17.07-.68-.56,6.88,2.16,6.2,1.6-42.62-31.45-37.3-52.69-36-66,1.07-10.66,8.81-24.32,21.86-29.86,6.3,3.08,10.23,5.43,10.23,5.43s-2.69-4.92-4.14-7.51c.51-.19,1-.15,1.5-.34,5.16,2.23,16.58,8,22.59,11.57,7.83,4.95,10.32,9.36,10.32,9.36s2.06-1,.54-5.33c-.56-1.77-2.93-7.39-10.68-13.07h.48a91.65,91.65,0,0,1,13.13,8.17c2.19-7.12,6.12-14.56,5.25-27.86-.53-9.35-.28-11.78-2.12-15.39-1.65-3.1.92-4.31,3.78-1.09a29.73,29.73,0,0,0-2.44-7.34v-.24c3.57-11.14,75.53-40.12,80.77-43.51a70.24,70.24,0,0,0,21.17-20.63c4-5.72,7-13.73,7.75-25.89.25-5.48-1.44-9.82-20.5-14-11.44-2.49-29.14-4.91-56.43-7.47-19.9-1.76-31.58-14.68-38.21-26.6-1.21-2.57-2.45-4.9-3.68-7.22a53.41,53.41,0,0,1-2.83-8.36,158.47,158.47,0,0,1,61.28-76.06c1.6-1.31-6.4.33-4.8-1,1.87-1.52,14.06-5.93,16.37-6.92,2.81-1.19-12-6.84-25.16-5.47-13.36,1.35-16.19,2.78-23.32,5.49,3-2.64,12.37-6.1,10.16-6.08-14.4,2-32.3,9.48-47.6,18a9.72,9.72,0,0,1,.92-4.31c-7.13,2.71-24.64,13.67-29.73,23a39.79,39.79,0,0,0,.29-5.35,88.55,88.55,0,0,0-14.6,13.7l-.27.22C258.14,196,221.75,195,191,201.72c-6.74-6.06-17.57-15.23-32.89-45.4-1-1.82-1.6,3.75-2.4,2-6-13.81-9.55-36.44-9-52,0,0-12.32,5.61-22.51,29.06-1.89,4.21-3.11,6.54-4.32,8.87-.56.68,1.27-7.7,1-7.24-1.77,3-6.36,7.19-8.37,12.62-1.38,4-3.32,6.27-4.56,11.29l-.29.46c-.1-1.48.37-6.08,0-5.14A235.4,235.4,0,0,0,95.34,186c-5.49,18-11.88,42.61-12.89,74.57-.24,2.42,0,5.14-.25,7.32-13,14.83-21.86,27.39-25.2,33.91-16.81,26-35.33,66.44-53.29,130.46a319.35,319.35,0,0,1,28.54-50C17.32,416.25,2.89,469.62,0,551.8a436.92,436.92,0,0,1,13.87-50.24C11.29,556.36,17.68,624.3,52.32,701c20.57,45,67.92,136.6,183.62,208h0s39.36,29.3,107,51.26c5,1.81,10.06,3.6,15.23,5.33q-2.43-1-4.71-2A484.9,484.9,0,0,0,492.27,984c175.18.15,226.85-70.2,226.85-70.2l-.51.38q3.71-3.49,7.14-7.26c-27.64,26.08-90.75,27.84-114.3,26,40.22-11.81,66.69-21.81,118.17-41.52q9-3.36,18.48-7.64l2-.94c1.25-.58,2.49-1.13,3.75-1.74a349.3,349.3,0,0,0,70.26-44c51.7-41.3,63-81.56,68.83-108.1-.82,2.54-3.37,8.47-5.17,12.32-13.31,28.48-42.84,46-74.91,61a689.05,689.05,0,0,0,42.38-62.44C865.77,729.39,869,713.15,876.85,702.23Z" style="fill:url(#radial-gradient-2)"/><path d="M813.92,801c21.08-23.24,40-49.82,54.35-80,36.9-77.58,94-206.58,49-341.31C881.77,273.22,833,215,771.11,158.12,670.56,65.76,642.48,24.52,642.48,0c0,0-116.09,129.41-65.74,264.38s153.46,130,221.68,270.87c80.27,165.74-64.95,346.61-185,397.24,7.35-1.63,267-60.38,280.61-208.88C893.68,726.34,887.83,767.41,813.92,801Z" style="fill:url(#linear-gradient-7)"/><path d="M477.59,319.37c.39-8.77-4.16-14.66-76.68-21.46-29.84-2.76-41.26-30.33-44.75-41.94-10.61,27.56-15,56.49-12.64,91.48,1.61,22.92,17,47.52,24.37,62,0,0,1.64-2.13,2.39-2.91,13.86-14.43,71.94-36.42,77.39-39.54C453.69,363.16,476.58,346.44,477.59,319.37Z" style="fill:url(#linear-gradient-8)"/><path d="M477.59,319.37c.39-8.77-4.16-14.66-76.68-21.46-29.84-2.76-41.26-30.33-44.75-41.94-10.61,27.56-15,56.49-12.64,91.48,1.61,22.92,17,47.52,24.37,62,0,0,1.64-2.13,2.39-2.91,13.86-14.43,71.94-36.42,77.39-39.54C453.69,363.16,476.58,346.44,477.59,319.37Z" style="opacity:0.5;isolation:isolate;fill:url(#radial-gradient-3)"/><path d="M158.31,156.47c-1-1.82-1.6,3.75-2.4,2-6-13.81-9.58-36.2-8.72-52,0,0-12.32,5.61-22.51,29.06-1.89,4.21-3.11,6.54-4.32,8.86-.56.68,1.27-7.7,1-7.24-1.77,3-6.36,7.19-8.35,12.38-1.65,4.24-3.35,6.52-4.61,11.77-.39,1.43.39-6.32,0-5.38C84.72,201.68,80.19,271,82.69,268,133.17,214.14,191,201.36,191,201.36c-6.15-4.53-19.53-17.63-32.7-44.89Z" style="fill:url(#linear-gradient-9)"/><path d="M349.84,720.1c-69.72-29.77-149-71.75-146-167.14C207.92,427.35,321,452.18,321,452.18c-4.27,1-15.68,9.16-19.72,17.82-4.27,10.83-12.07,35.28,11.55,60.9,37.09,40.19-76.2,95.36,98.66,199.57,4.41,2.4-41-1.43-61.64-10.36Z" style="fill:url(#linear-gradient-10)"/><path d="M325.07,657.5c49.44,17.21,107,14.19,141.52-4.86,23.09-12.85,52.7-33.43,70.92-28.35-15.78-6.24-27.73-9.15-42.1-9.86-2.45,0-5.38,0-8-.32a136,136,0,0,0-15.76.86c-8.9.82-18.77,6.43-27.74,5.53-.48,0,8.7-3.77,8-3.61-4.75,1-9.92,1.21-15.37,1.88-3.47.39-6.45.82-9.89,1-103,8.73-190-55.81-190-55.81-7.41,25,33.17,74.3,88.52,93.57Z" style="opacity:0.5;isolation:isolate;fill:url(#linear-gradient-11)"/><path d="M813.74,801.65c104.16-102.27,156.86-226.58,134.58-366,0,0,8.9,71.5-24.85,144.63,16.21-71.39,18.1-160.11-25-252C841,205.64,746.45,141.11,710.35,114.19,655.66,73.4,633,31.87,632.57,23.3c-16.34,33.48-65.77,148.2-5.31,247,56.64,92.56,145.86,120,208.33,205C950.67,631.67,813.74,801.65,813.74,801.65Z" style="fill:url(#linear-gradient-12)"/><path d="M798.81,535.55C762.41,460.35,717,427.55,674,392c5,7,6.23,9.47,9,14,37.83,40.32,93.61,138.66,53.11,262.11C659.88,900.48,355,791.06,323,760.32,335.93,894.81,561,959.16,707.6,872,791,793,858.47,658.79,798.81,535.55Z" style="fill:url(#linear-gradient-13)"/></g></g></g></g></svg>
\ No newline at end of file diff --git a/tools/profiler/tests/browser/head.js b/tools/profiler/tests/browser/head.js new file mode 100644 index 0000000000..ef0e3128c0 --- /dev/null +++ b/tools/profiler/tests/browser/head.js @@ -0,0 +1,159 @@ +/* import-globals-from ../shared-head.js */ + +Services.scriptloader.loadSubScript( + "chrome://mochitests/content/browser/tools/profiler/tests/browser/shared-head.js", + this +); + +const BASE_URL = "http://example.com/browser/tools/profiler/tests/browser/"; +const BASE_URL_HTTPS = + "https://example.com/browser/tools/profiler/tests/browser/"; + +registerCleanupFunction(async () => { + if (Services.profiler.IsActive()) { + info( + "The profiler was found to still be running at the end of the test, which means that some error likely occured. Let's stop it to prevent issues with following tests!" + ); + await Services.profiler.StopProfiler(); + } +}); + +/** + * This is a helper function that will stop the profiler and returns the main + * threads for the parent process and the content process with PID contentPid. + * This happens immediately, without waiting for any sampling to happen or + * finish. Use waitSamplingAndStopProfilerAndGetThreads below instead to wait + * for samples before stopping. + * This returns also the full profile in case the caller wants more information. + * + * @param {number} contentPid + * @returns {Promise<{profile, parentThread, contentProcess, contentThread}>} + */ +async function stopProfilerNowAndGetThreads(contentPid) { + const profile = await stopNowAndGetProfile(); + + const parentThread = profile.threads[0]; + const contentProcess = profile.processes.find( + p => p.threads[0].pid == contentPid + ); + if (!contentProcess) { + throw new Error( + `Could not find the content process with given pid: ${contentPid}` + ); + } + + if (!parentThread) { + throw new Error("The parent thread was not found in the profile."); + } + + const contentThread = contentProcess.threads[0]; + if (!contentThread) { + throw new Error("The content thread was not found in the profile."); + } + + return { profile, parentThread, contentProcess, contentThread }; +} + +/** + * This is a helper function that will stop the profiler and returns the main + * threads for the parent process and the content process with PID contentPid. + * As opposed to stopProfilerNowAndGetThreads (with "Now") above, the profiler + * in that PID will not stop until there is at least one periodic sample taken. + * + * @param {number} contentPid + * @returns {Promise<{profile, parentThread, contentProcess, contentThread}>} + */ +async function waitSamplingAndStopProfilerAndGetThreads(contentPid) { + await Services.profiler.waitOnePeriodicSampling(); + + return stopProfilerNowAndGetThreads(contentPid); +} + +/** This tries to find the service worker thread by targeting a very specific + * UserTiming marker. Indeed we use performance.mark to add this marker from the + * service worker's events. + * Then from this thread we get its parent thread. Indeed the parent thread is + * where all network stuff happens, so this is useful for network marker tests. + * + * @param {Object} profile + * @returns {{ serviceWorkerThread: Object, serviceWorkerParentThread: Object }} the found threads + */ +function findServiceWorkerThreads(profile) { + const allThreads = [ + profile.threads, + ...profile.processes.map(process => process.threads), + ].flat(); + + const serviceWorkerThread = allThreads.find( + ({ processType, markers }) => + processType === "tab" && + markers.data.some(markerTuple => { + const data = markerTuple[markers.schema.data]; + return ( + data && + data.type === "UserTiming" && + data.name === "__serviceworker_event" + ); + }) + ); + + if (!serviceWorkerThread) { + info( + "We couldn't find a service worker thread. Here are all the threads in this profile:" + ); + allThreads.forEach(logInformationForThread.bind(null, "")); + return null; + } + + const serviceWorkerParentThread = allThreads.find( + ({ name, pid }) => pid === serviceWorkerThread.pid && name === "GeckoMain" + ); + + if (!serviceWorkerParentThread) { + info( + `We couldn't find a parent thread for the service worker thread (pid: ${serviceWorkerThread.pid}, tid: ${serviceWorkerThread.tid}).` + ); + info("Here are all the threads in this profile:"); + allThreads.forEach(logInformationForThread.bind(null, "")); + + // Let's write the profile on disk if MOZ_UPLOAD_DIR is present + const path = Services.env.get("MOZ_UPLOAD_DIR"); + if (path) { + const profileName = `profile_${Date.now()}.json`; + const profilePath = PathUtils.join(path, profileName); + info( + `We wrote down the profile on disk as an artifact, with name ${profileName}.` + ); + // This function returns a Promise, but we're not waiting on it because + // we're in a synchronous function. Hopefully writing will be finished + // when the process ends. + IOUtils.writeJSON(profilePath, profile).catch(err => + console.error("An error happened when writing the profile on disk", err) + ); + } + throw new Error( + "We couldn't find a parent thread for the service worker thread. Please read logs to find more information." + ); + } + + return { serviceWorkerThread, serviceWorkerParentThread }; +} + +/** + * This logs some basic information about the passed thread. + * + * @param {string} prefix + * @param {Object} thread + */ +function logInformationForThread(prefix, thread) { + if (!thread) { + info(prefix + ": thread is null or undefined."); + return; + } + + const { name, pid, tid, processName, processType } = thread; + info( + `${prefix}: ` + + `name(${name}) pid(${pid}) tid(${tid}) processName(${processName}) processType(${processType})` + ); +} diff --git a/tools/profiler/tests/browser/multi_frame.html b/tools/profiler/tests/browser/multi_frame.html new file mode 100644 index 0000000000..b2efcedd50 --- /dev/null +++ b/tools/profiler/tests/browser/multi_frame.html @@ -0,0 +1,11 @@ +<!DOCTYPE html> +<html> +<head> + <meta charset="utf-8"> + <title>Multi Frame</title> +</head> +<body> + Multi Frame + <iframe src="single_frame.html"></iframe> +</body> +</html> diff --git a/tools/profiler/tests/browser/page_with_resources.html b/tools/profiler/tests/browser/page_with_resources.html new file mode 100644 index 0000000000..9d2bb8f218 --- /dev/null +++ b/tools/profiler/tests/browser/page_with_resources.html @@ -0,0 +1,11 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"/> + </head> + <body> + Testing + <img src='firefox-logo-nightly.svg' width="24"/> + <img src='redirect.sjs?firefox-logo-nightly.svg' width="24"/> + </body> +</html> diff --git a/tools/profiler/tests/browser/redirect.sjs b/tools/profiler/tests/browser/redirect.sjs new file mode 100644 index 0000000000..2a325c3d0b --- /dev/null +++ b/tools/profiler/tests/browser/redirect.sjs @@ -0,0 +1,8 @@ +function handleRequest(request, response) { + response.setStatusLine(request.httpVersion, 301, "Moved Permanently"); + response.setHeader( + "Location", + decodeURIComponent(request.queryString), + false + ); +} diff --git a/tools/profiler/tests/browser/serviceworkers/firefox-logo-nightly.svg b/tools/profiler/tests/browser/serviceworkers/firefox-logo-nightly.svg new file mode 100644 index 0000000000..f1af370d87 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/firefox-logo-nightly.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 953.37 984"><defs><linearGradient id="linear-gradient" x1="-14706.28" y1="9250.14" x2="-14443.04" y2="9250.14" gradientTransform="matrix(0.76, 0.03, 0.05, -1.12, 11485.47, 11148)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#0083ff"/><stop offset="0.1" stop-color="#0092f8"/><stop offset="0.31" stop-color="#00abeb"/><stop offset="0.52" stop-color="#00bee1"/><stop offset="0.75" stop-color="#00c8dc"/><stop offset="1" stop-color="#00ccda"/></linearGradient><radialGradient id="radial-gradient" cx="-7588.66" cy="8866.53" r="791.23" gradientTransform="matrix(1.23, 0, 0, -1.22, 9958.21, 11048.11)" gradientUnits="userSpaceOnUse"><stop offset="0.02" stop-color="#005fe7"/><stop offset="0.18" stop-color="#0042b4"/><stop offset="0.32" stop-color="#002989"/><stop offset="0.4" stop-color="#002079"/><stop offset="0.47" stop-color="#131d78"/><stop offset="0.66" stop-color="#3b1676"/><stop offset="0.75" stop-color="#4a1475"/></radialGradient><linearGradient id="linear-gradient-2" x1="539.64" y1="254.8" x2="348.2" y2="881.03" gradientTransform="matrix(1, 0, 0, -1, 1, 984)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#000f43" stop-opacity="0.4"/><stop offset="0.48" stop-color="#001962" stop-opacity="0.17"/><stop offset="1" stop-color="#002079" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-3" x1="540.64" y1="254.8" x2="349.2" y2="881.03" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" xlink:href="#linear-gradient-2"/><linearGradient id="linear-gradient-4" x1="-8367.12" y1="7348.87" x2="-8482.36" y2="7357.76" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#812cc9"/><stop offset="1" stop-color="#005fe7"/></linearGradient><linearGradient id="linear-gradient-5" x1="-8449.89" y1="7496.97" x2="-8341.94" y2="7609.09" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0.05" stop-color="#005fe7"/><stop offset="0.18" stop-color="#065de6"/><stop offset="0.35" stop-color="#1856e1"/><stop offset="0.56" stop-color="#354adb"/><stop offset="0.78" stop-color="#5d3ad1"/><stop offset="0.95" stop-color="#812cc9"/></linearGradient><linearGradient id="linear-gradient-6" x1="-8653.41" y1="7245.3" x2="-8422.52" y2="7244.76" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#002079"/><stop offset="0.99" stop-color="#a238ff"/></linearGradient><radialGradient id="radial-gradient-2" cx="644.11" cy="599.83" fx="785.0454815336918" fy="470.6889181532662" r="793.95" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" gradientUnits="userSpaceOnUse"><stop offset="0.2" stop-color="#00fdff"/><stop offset="0.26" stop-color="#0af1ff"/><stop offset="0.37" stop-color="#23d2ff"/><stop offset="0.52" stop-color="#4da0ff"/><stop offset="0.69" stop-color="#855bff"/><stop offset="0.77" stop-color="#a238ff"/><stop offset="0.81" stop-color="#a738fd"/><stop offset="0.86" stop-color="#b539f9"/><stop offset="0.9" stop-color="#cd39f1"/><stop offset="0.96" stop-color="#ee3ae6"/><stop offset="0.98" stop-color="#ff3be0"/></radialGradient><linearGradient id="linear-gradient-7" x1="-7458.97" y1="9093.17" x2="-7531.06" y2="8282.84" gradientTransform="matrix(1.23, 0, 0, -1.22, 9958.21, 11048.11)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#00ec00"/><stop offset="0.1" stop-color="#00e244"/><stop offset="0.22" stop-color="#00d694"/><stop offset="0.31" stop-color="#00cfc7"/><stop offset="0.35" stop-color="#00ccda"/><stop offset="0.42" stop-color="#0bc2dd" stop-opacity="0.92"/><stop offset="0.57" stop-color="#29a7e4" stop-opacity="0.72"/><stop offset="0.77" stop-color="#597df0" stop-opacity="0.4"/><stop offset="1" stop-color="#9448ff" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-8" x1="-8926.61" y1="7680.53" x2="-8790.14" y2="7680.53" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#005fe7"/><stop offset="0.46" stop-color="#0071f3" stop-opacity="0.51"/><stop offset="0.83" stop-color="#007efc" stop-opacity="0.14"/><stop offset="1" stop-color="#0083ff" stop-opacity="0"/></linearGradient><radialGradient id="radial-gradient-3" cx="-8914.62" cy="7721.05" r="165.97" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0.63" stop-color="#ffe302" stop-opacity="0"/><stop offset="0.67" stop-color="#ffe302" stop-opacity="0.05"/><stop offset="0.75" stop-color="#ffe302" stop-opacity="0.19"/><stop offset="0.86" stop-color="#ffe302" stop-opacity="0.4"/><stop offset="0.99" stop-color="#ffe302" stop-opacity="0.7"/></radialGradient><linearGradient id="linear-gradient-9" x1="214.02" y1="2032.47" x2="96.19" y2="2284.31" gradientTransform="matrix(0.99, 0.1, 0.1, -0.99, -250.1, 2306.29)" gradientUnits="userSpaceOnUse"><stop offset="0.19" stop-color="#4a1475" stop-opacity="0.5"/><stop offset="0.62" stop-color="#2277ac" stop-opacity="0.23"/><stop offset="0.94" stop-color="#00ccda" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-10" x1="-38.44" y1="278.18" x2="55.67" y2="171.29" gradientTransform="matrix(0.99, 0.1, 0.1, -0.99, 229.04, 745.87)" gradientUnits="userSpaceOnUse"><stop offset="0.01" stop-color="#002079" stop-opacity="0.5"/><stop offset="1" stop-color="#0083ff" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-11" x1="142.45" y1="96.25" x2="142.5" y2="149.68" gradientTransform="matrix(0.99, 0.1, 0.1, -0.99, 229.04, 745.87)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#4a1475" stop-opacity="0.9"/><stop offset="0.18" stop-color="#6720a2" stop-opacity="0.6"/><stop offset="0.38" stop-color="#812acb" stop-opacity="0.34"/><stop offset="0.57" stop-color="#9332e8" stop-opacity="0.15"/><stop offset="0.76" stop-color="#9e36f9" stop-opacity="0.04"/><stop offset="0.93" stop-color="#a238ff" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-12" x1="620.52" y1="947.88" x2="926.18" y2="264.39" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#00ec00" stop-opacity="0"/><stop offset="0.28" stop-color="#00dc6d" stop-opacity="0.5"/><stop offset="0.5" stop-color="#00d1bb" stop-opacity="0.86"/><stop offset="0.6" stop-color="#00ccda"/><stop offset="0.68" stop-color="#04c9db"/><stop offset="0.75" stop-color="#0fc1df"/><stop offset="0.83" stop-color="#23b2e6"/><stop offset="0.9" stop-color="#3e9ef0"/><stop offset="0.98" stop-color="#6184fc"/><stop offset="0.99" stop-color="#6680fe"/></linearGradient><linearGradient id="linear-gradient-13" x1="680.88" y1="554.79" x2="536.1" y2="166.04" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#0083ff"/><stop offset="0.04" stop-color="#0083ff" stop-opacity="0.92"/><stop offset="0.14" stop-color="#0083ff" stop-opacity="0.71"/><stop offset="0.26" stop-color="#0083ff" stop-opacity="0.52"/><stop offset="0.37" stop-color="#0083ff" stop-opacity="0.36"/><stop offset="0.49" stop-color="#0083ff" stop-opacity="0.23"/><stop offset="0.61" stop-color="#0083ff" stop-opacity="0.13"/><stop offset="0.73" stop-color="#0083ff" stop-opacity="0.06"/><stop offset="0.86" stop-color="#0083ff" stop-opacity="0.01"/><stop offset="1" stop-color="#0083ff" stop-opacity="0"/></linearGradient></defs><title>firefox-logo-nightly</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><g id="Layer_2-2" data-name="Layer 2"><g id="Firefox"><path d="M770.28,91.56c-23.95,27.88-35.1,90.64-10.82,154.26s61.5,49.8,84.7,114.67c30.62,85.6,16.37,200.59,16.37,200.59s36.81,106.61,62.47-6.63C979.79,341.74,770.28,143.94,770.28,91.56Z" style="fill:url(#linear-gradient)"/><path id="_Path_" data-name=" Path " d="M476.92,972.83c245.24,0,443.9-199.74,443.9-446s-198.66-446-443.66-446S33.5,280.51,33.5,526.8C33,773.33,231.92,972.83,476.92,972.83Z" style="fill:url(#radial-gradient)"/><path d="M810.67,803.64a246.8,246.8,0,0,1-30.12,18.18,705.31,705.31,0,0,0,38.3-63c9.46-10.47,18.13-20.65,25.19-31.65,3.44-5.41,7.31-12.08,11.42-19.82,24.92-44.9,52.4-117.56,53.18-192.2v-5.66a257.25,257.25,0,0,0-5.71-55.75c.2,1.43.38,2.86.56,4.29-.22-1.1-.41-2.21-.64-3.31.37,2,.66,4,1,6,5.09,43.22,1.47,85.37-16.68,116.45-.29.45-.58.88-.87,1.32,9.41-47.23,12.56-99.39,2.09-151.6,0,0-4.19-25.38-35.38-102.44-18-44.35-49.83-80.72-78-107.21-24.69-30.55-47.11-51-59.47-64.06C689.72,126,678.9,105.61,674.45,92.31c-3.85-1.93-53.14-49.81-57.05-51.63-21.51,33.35-89.16,137.67-57,235.15,14.58,44.17,51.47,90,90.07,115.74,1.69,1.94,23,25,33.09,77.16,10.45,53.85,5,95.86-16.54,158C641.73,681.24,577,735.12,516.3,740.63c-129.67,11.78-177.15-65.11-177.15-65.11C385.49,694,436.72,690.17,467.87,671c31.4-19.43,50.39-33.83,65.81-28.15C548.86,648.43,561,632,550.1,615a78.5,78.5,0,0,0-79.4-34.57c-31.43,5.11-60.23,30-101.41,5.89a86.29,86.29,0,0,1-7.73-5.06c-2.71-1.79,8.83,2.72,6.13.69-8-4.35-22.2-13.84-25.88-17.22-.61-.56,6.22,2.18,5.61,1.62-38.51-31.71-33.7-53.13-32.49-66.57,1-10.75,8-24.52,19.75-30.11,5.69,3.11,9.24,5.48,9.24,5.48s-2.43-5-3.74-7.58c.46-.2.9-.15,1.36-.34,4.66,2.25,15,8.1,20.41,11.67,7.07,5,9.33,9.44,9.33,9.44s1.86-1,.48-5.37c-.5-1.78-2.65-7.45-9.65-13.17h.44A81.61,81.61,0,0,1,374.42,478c2-7.18,5.53-14.68,4.75-28.09-.48-9.43-.26-11.87-1.92-15.51-1.49-3.13.83-4.35,3.42-1.1a32.5,32.5,0,0,0-2.21-7.4v-.24c3.23-11.24,68.25-40.46,73-43.88A67.2,67.2,0,0,0,470.59,361c3.62-5.76,6.34-13.85,7-26.11.36-8.84-3.76-14.73-69.51-21.62-18-1.77-28.53-14.8-34.53-26.82-1.09-2.59-2.21-4.94-3.33-7.28a57.68,57.68,0,0,1-2.56-8.43c10.75-30.87,28.81-57,55.37-76.7,1.45-1.32-5.78.34-4.34-1,1.69-1.54,12.71-6,14.79-7,2.54-1.2-10.88-6.9-22.73-5.51-12.07,1.36-14.63,2.8-21.07,5.53,2.67-2.66,11.17-6.15,9.18-6.13-13,2-29.18,9.56-43,18.12a10.66,10.66,0,0,1,.83-4.35c-6.44,2.73-22.26,13.79-26.87,23.14a44.29,44.29,0,0,0,.27-5.4,84.17,84.17,0,0,0-13.19,13.82l-.24.22c-37.36-15-70.23-16-98.05-9.28-6.09-6.11-9.06-1.64-22.91-32.07-.94-1.83.72,1.81,0,0-2.28-5.9,1.39,7.87,0,0-23.28,18.37-53.92,39.19-68.63,53.89-.18.59,17.16-4.9,0,0-6,1.72-5.6,5.28-6.51,37.5-.22,2.44,0,5.18-.22,7.38-11.75,15-19.75,27.64-22.78,34.21-15.19,26.18-31.93,67-48.15,131.55A334.82,334.82,0,0,1,75.2,398.36C61.71,432.63,48.67,486.44,46.07,569.3A482.08,482.08,0,0,1,58.6,518.64,473,473,0,0,0,93.33,719.71c9.33,22.82,24.76,57.46,51,95.4C226.9,902,343.31,956,472.21,956,606.79,956,727.64,897.13,810.67,803.64Z" style="fill:url(#linear-gradient-2)"/><path d="M810.67,803.64a246.8,246.8,0,0,1-30.12,18.18,705.31,705.31,0,0,0,38.3-63c9.46-10.47,18.13-20.65,25.19-31.65,3.44-5.41,7.31-12.08,11.42-19.82,24.92-44.9,52.4-117.56,53.18-192.2v-5.66a257.25,257.25,0,0,0-5.71-55.75c.2,1.43.38,2.86.56,4.29-.22-1.1-.41-2.21-.64-3.31.37,2,.66,4,1,6,5.09,43.22,1.47,85.37-16.68,116.45-.29.45-.58.88-.87,1.32,9.41-47.23,12.56-99.39,2.09-151.6,0,0-4.19-25.38-35.38-102.44-18-44.35-49.83-80.72-78-107.21-24.69-30.55-47.11-51-59.47-64.06C689.72,126,678.9,105.61,674.45,92.31c-3.85-1.93-53.14-49.81-57.05-51.63-21.51,33.35-89.16,137.67-57,235.15,14.58,44.17,51.47,90,90.07,115.74,1.69,1.94,23,25,33.09,77.16,10.45,53.85,5,95.86-16.54,158C641.73,681.24,577,735.12,516.3,740.63c-129.67,11.78-177.15-65.11-177.15-65.11C385.49,694,436.72,690.17,467.87,671c31.4-19.43,50.39-33.83,65.81-28.15C548.86,648.43,561,632,550.1,615a78.5,78.5,0,0,0-79.4-34.57c-31.43,5.11-60.23,30-101.41,5.89a86.29,86.29,0,0,1-7.73-5.06c-2.71-1.79,8.83,2.72,6.13.69-8-4.35-22.2-13.84-25.88-17.22-.61-.56,6.22,2.18,5.61,1.62-38.51-31.71-33.7-53.13-32.49-66.57,1-10.75,8-24.52,19.75-30.11,5.69,3.11,9.24,5.48,9.24,5.48s-2.43-5-3.74-7.58c.46-.2.9-.15,1.36-.34,4.66,2.25,15,8.1,20.41,11.67,7.07,5,9.33,9.44,9.33,9.44s1.86-1,.48-5.37c-.5-1.78-2.65-7.45-9.65-13.17h.44A81.61,81.61,0,0,1,374.42,478c2-7.18,5.53-14.68,4.75-28.09-.48-9.43-.26-11.87-1.92-15.51-1.49-3.13.83-4.35,3.42-1.1a32.5,32.5,0,0,0-2.21-7.4v-.24c3.23-11.24,68.25-40.46,73-43.88A67.2,67.2,0,0,0,470.59,361c3.62-5.76,6.34-13.85,7-26.11.36-8.84-3.76-14.73-69.51-21.62-18-1.77-28.53-14.8-34.53-26.82-1.09-2.59-2.21-4.94-3.33-7.28a57.68,57.68,0,0,1-2.56-8.43c10.75-30.87,28.81-57,55.37-76.7,1.45-1.32-5.78.34-4.34-1,1.69-1.54,12.71-6,14.79-7,2.54-1.2-10.88-6.9-22.73-5.51-12.07,1.36-14.63,2.8-21.07,5.53,2.67-2.66,11.17-6.15,9.18-6.13-13,2-29.18,9.56-43,18.12a10.66,10.66,0,0,1,.83-4.35c-6.44,2.73-22.26,13.79-26.87,23.14a44.29,44.29,0,0,0,.27-5.4,84.17,84.17,0,0,0-13.19,13.82l-.24.22c-37.36-15-70.23-16-98.05-9.28-6.09-6.11-9.06-1.64-22.91-32.07-.94-1.83.72,1.81,0,0-2.28-5.9,1.39,7.87,0,0-23.28,18.37-53.92,39.19-68.63,53.89-.18.59,17.16-4.9,0,0-6,1.72-5.6,5.28-6.51,37.5-.22,2.44,0,5.18-.22,7.38-11.75,15-19.75,27.64-22.78,34.21-15.19,26.18-31.93,67-48.15,131.55A334.82,334.82,0,0,1,75.2,398.36C61.71,432.63,48.67,486.44,46.07,569.3A482.08,482.08,0,0,1,58.6,518.64,473,473,0,0,0,93.33,719.71c9.33,22.82,24.76,57.46,51,95.4C226.9,902,343.31,956,472.21,956,606.79,956,727.64,897.13,810.67,803.64Z" style="fill:url(#linear-gradient-3)"/><path d="M711.1,866.71c162.87-18.86,235-186.7,142.38-190C769.85,674,634,875.61,711.1,866.71Z" style="fill:url(#linear-gradient-4)"/><path d="M865.21,642.42C977.26,577.21,948,436.34,948,436.34s-43.25,50.24-72.62,130.32C846.4,646,797.84,681.81,865.21,642.42Z" style="fill:url(#linear-gradient-5)"/><path d="M509.47,950.06C665.7,999.91,800,876.84,717.21,835.74,642,798.68,435.32,926.49,509.47,950.06Z" style="fill:url(#linear-gradient-6)"/><path d="M638.58,21.42l.53-.57A1.7,1.7,0,0,0,638.58,21.42ZM876.85,702.23c3.8-5.36,8.94-22.53,13.48-30.21,27.58-44.52,27.78-80,27.78-80.84,16.66-83.22,15.15-117.2,4.9-180-8.25-50.6-44.32-123.09-75.57-158-32.2-36-9.51-24.25-40.69-50.52-27.33-30.29-53.82-60.29-68.25-72.36C634.22,43.09,636.57,24.58,638.58,21.42c-.34.37-.84.92-1.47,1.64C635.87,18.14,635,14,635,14s-57,57-69,152c-7.83,62,15.38,126.68,49,168a381.62,381.62,0,0,0,59,58h0c25.4,36.48,39.38,81.49,39.38,129.91,0,121.24-98.34,219.53-219.65,219.53a220.14,220.14,0,0,1-49.13-5.52c-57.24-10.92-90.3-39.8-106.78-59.41-9.45-11.23-13.46-19.42-13.46-19.42,51.28,18.37,108,14.53,142.47-4.52,34.75-19.26,55.77-33.55,72.84-27.92,16.82,5.61,30.21-10.67,18.2-27.54-11.77-16.85-42.4-41-87.88-34.29-34.79,5.07-66.66,29.76-112.24,5.84a97.34,97.34,0,0,1-8.55-5c-3-1.77,9.77,2.69,6.79.68-8.87-4.32-24.57-13.73-28.64-17.07-.68-.56,6.88,2.16,6.2,1.6-42.62-31.45-37.3-52.69-36-66,1.07-10.66,8.81-24.32,21.86-29.86,6.3,3.08,10.23,5.43,10.23,5.43s-2.69-4.92-4.14-7.51c.51-.19,1-.15,1.5-.34,5.16,2.23,16.58,8,22.59,11.57,7.83,4.95,10.32,9.36,10.32,9.36s2.06-1,.54-5.33c-.56-1.77-2.93-7.39-10.68-13.07h.48a91.65,91.65,0,0,1,13.13,8.17c2.19-7.12,6.12-14.56,5.25-27.86-.53-9.35-.28-11.78-2.12-15.39-1.65-3.1.92-4.31,3.78-1.09a29.73,29.73,0,0,0-2.44-7.34v-.24c3.57-11.14,75.53-40.12,80.77-43.51a70.24,70.24,0,0,0,21.17-20.63c4-5.72,7-13.73,7.75-25.89.25-5.48-1.44-9.82-20.5-14-11.44-2.49-29.14-4.91-56.43-7.47-19.9-1.76-31.58-14.68-38.21-26.6-1.21-2.57-2.45-4.9-3.68-7.22a53.41,53.41,0,0,1-2.83-8.36,158.47,158.47,0,0,1,61.28-76.06c1.6-1.31-6.4.33-4.8-1,1.87-1.52,14.06-5.93,16.37-6.92,2.81-1.19-12-6.84-25.16-5.47-13.36,1.35-16.19,2.78-23.32,5.49,3-2.64,12.37-6.1,10.16-6.08-14.4,2-32.3,9.48-47.6,18a9.72,9.72,0,0,1,.92-4.31c-7.13,2.71-24.64,13.67-29.73,23a39.79,39.79,0,0,0,.29-5.35,88.55,88.55,0,0,0-14.6,13.7l-.27.22C258.14,196,221.75,195,191,201.72c-6.74-6.06-17.57-15.23-32.89-45.4-1-1.82-1.6,3.75-2.4,2-6-13.81-9.55-36.44-9-52,0,0-12.32,5.61-22.51,29.06-1.89,4.21-3.11,6.54-4.32,8.87-.56.68,1.27-7.7,1-7.24-1.77,3-6.36,7.19-8.37,12.62-1.38,4-3.32,6.27-4.56,11.29l-.29.46c-.1-1.48.37-6.08,0-5.14A235.4,235.4,0,0,0,95.34,186c-5.49,18-11.88,42.61-12.89,74.57-.24,2.42,0,5.14-.25,7.32-13,14.83-21.86,27.39-25.2,33.91-16.81,26-35.33,66.44-53.29,130.46a319.35,319.35,0,0,1,28.54-50C17.32,416.25,2.89,469.62,0,551.8a436.92,436.92,0,0,1,13.87-50.24C11.29,556.36,17.68,624.3,52.32,701c20.57,45,67.92,136.6,183.62,208h0s39.36,29.3,107,51.26c5,1.81,10.06,3.6,15.23,5.33q-2.43-1-4.71-2A484.9,484.9,0,0,0,492.27,984c175.18.15,226.85-70.2,226.85-70.2l-.51.38q3.71-3.49,7.14-7.26c-27.64,26.08-90.75,27.84-114.3,26,40.22-11.81,66.69-21.81,118.17-41.52q9-3.36,18.48-7.64l2-.94c1.25-.58,2.49-1.13,3.75-1.74a349.3,349.3,0,0,0,70.26-44c51.7-41.3,63-81.56,68.83-108.1-.82,2.54-3.37,8.47-5.17,12.32-13.31,28.48-42.84,46-74.91,61a689.05,689.05,0,0,0,42.38-62.44C865.77,729.39,869,713.15,876.85,702.23Z" style="fill:url(#radial-gradient-2)"/><path d="M813.92,801c21.08-23.24,40-49.82,54.35-80,36.9-77.58,94-206.58,49-341.31C881.77,273.22,833,215,771.11,158.12,670.56,65.76,642.48,24.52,642.48,0c0,0-116.09,129.41-65.74,264.38s153.46,130,221.68,270.87c80.27,165.74-64.95,346.61-185,397.24,7.35-1.63,267-60.38,280.61-208.88C893.68,726.34,887.83,767.41,813.92,801Z" style="fill:url(#linear-gradient-7)"/><path d="M477.59,319.37c.39-8.77-4.16-14.66-76.68-21.46-29.84-2.76-41.26-30.33-44.75-41.94-10.61,27.56-15,56.49-12.64,91.48,1.61,22.92,17,47.52,24.37,62,0,0,1.64-2.13,2.39-2.91,13.86-14.43,71.94-36.42,77.39-39.54C453.69,363.16,476.58,346.44,477.59,319.37Z" style="fill:url(#linear-gradient-8)"/><path d="M477.59,319.37c.39-8.77-4.16-14.66-76.68-21.46-29.84-2.76-41.26-30.33-44.75-41.94-10.61,27.56-15,56.49-12.64,91.48,1.61,22.92,17,47.52,24.37,62,0,0,1.64-2.13,2.39-2.91,13.86-14.43,71.94-36.42,77.39-39.54C453.69,363.16,476.58,346.44,477.59,319.37Z" style="opacity:0.5;isolation:isolate;fill:url(#radial-gradient-3)"/><path d="M158.31,156.47c-1-1.82-1.6,3.75-2.4,2-6-13.81-9.58-36.2-8.72-52,0,0-12.32,5.61-22.51,29.06-1.89,4.21-3.11,6.54-4.32,8.86-.56.68,1.27-7.7,1-7.24-1.77,3-6.36,7.19-8.35,12.38-1.65,4.24-3.35,6.52-4.61,11.77-.39,1.43.39-6.32,0-5.38C84.72,201.68,80.19,271,82.69,268,133.17,214.14,191,201.36,191,201.36c-6.15-4.53-19.53-17.63-32.7-44.89Z" style="fill:url(#linear-gradient-9)"/><path d="M349.84,720.1c-69.72-29.77-149-71.75-146-167.14C207.92,427.35,321,452.18,321,452.18c-4.27,1-15.68,9.16-19.72,17.82-4.27,10.83-12.07,35.28,11.55,60.9,37.09,40.19-76.2,95.36,98.66,199.57,4.41,2.4-41-1.43-61.64-10.36Z" style="fill:url(#linear-gradient-10)"/><path d="M325.07,657.5c49.44,17.21,107,14.19,141.52-4.86,23.09-12.85,52.7-33.43,70.92-28.35-15.78-6.24-27.73-9.15-42.1-9.86-2.45,0-5.38,0-8-.32a136,136,0,0,0-15.76.86c-8.9.82-18.77,6.43-27.74,5.53-.48,0,8.7-3.77,8-3.61-4.75,1-9.92,1.21-15.37,1.88-3.47.39-6.45.82-9.89,1-103,8.73-190-55.81-190-55.81-7.41,25,33.17,74.3,88.52,93.57Z" style="opacity:0.5;isolation:isolate;fill:url(#linear-gradient-11)"/><path d="M813.74,801.65c104.16-102.27,156.86-226.58,134.58-366,0,0,8.9,71.5-24.85,144.63,16.21-71.39,18.1-160.11-25-252C841,205.64,746.45,141.11,710.35,114.19,655.66,73.4,633,31.87,632.57,23.3c-16.34,33.48-65.77,148.2-5.31,247,56.64,92.56,145.86,120,208.33,205C950.67,631.67,813.74,801.65,813.74,801.65Z" style="fill:url(#linear-gradient-12)"/><path d="M798.81,535.55C762.41,460.35,717,427.55,674,392c5,7,6.23,9.47,9,14,37.83,40.32,93.61,138.66,53.11,262.11C659.88,900.48,355,791.06,323,760.32,335.93,894.81,561,959.16,707.6,872,791,793,858.47,658.79,798.81,535.55Z" style="fill:url(#linear-gradient-13)"/></g></g></g></g></svg>
\ No newline at end of file diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker-utils.js b/tools/profiler/tests/browser/serviceworkers/serviceworker-utils.js new file mode 100644 index 0000000000..16a9f0c91f --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker-utils.js @@ -0,0 +1,39 @@ +// Most of this file has been stolen from dom/serviceworkers/test/utils.js. + +function waitForState(worker, state) { + return new Promise((resolve, reject) => { + function onStateChange() { + if (worker.state === state) { + worker.removeEventListener("statechange", onStateChange); + resolve(); + } + if (worker.state === "redundant") { + worker.removeEventListener("statechange", onStateChange); + reject(new Error("The service worker failed to install.")); + } + } + + // First add an event listener, so we won't miss any change that happens + // before we check the current state. + worker.addEventListener("statechange", onStateChange); + + // Now check if the worker is already in the desired state. + onStateChange(); + }); +} + +async function registerServiceWorkerAndWait(serviceWorkerFile) { + if (!serviceWorkerFile) { + throw new Error( + "No service worker filename has been specified. Please specify a valid filename." + ); + } + + console.log(`...registering the serviceworker "${serviceWorkerFile}"`); + const reg = await navigator.serviceWorker.register(`./${serviceWorkerFile}`, { + scope: "./", + }); + console.log("...waiting for activation"); + await waitForState(reg.installing, "activated"); + console.log("...activated!"); +} diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_cache_first.js b/tools/profiler/tests/browser/serviceworkers/serviceworker_cache_first.js new file mode 100644 index 0000000000..baa07fd6d8 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_cache_first.js @@ -0,0 +1,34 @@ +const files = ["serviceworker_page.html", "firefox-logo-nightly.svg"]; +const cacheName = "v1"; + +self.addEventListener("install", event => { + performance.mark("__serviceworker_event"); + console.log("[SW]:", "Install event"); + + event.waitUntil(cacheAssets()); +}); + +async function cacheAssets() { + const cache = await caches.open(cacheName); + await cache.addAll(files); +} + +self.addEventListener("fetch", event => { + performance.mark("__serviceworker_event"); + console.log("Handling fetch event for", event.request.url); + event.respondWith(handleFetch(event.request)); +}); + +async function handleFetch(request) { + const cachedResponse = await caches.match(request); + if (cachedResponse) { + console.log("Found response in cache:", cachedResponse); + + return cachedResponse; + } + console.log("No response found in cache. About to fetch from network..."); + + const networkResponse = await fetch(request); + console.log("Response from network is:", networkResponse); + return networkResponse; +} diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_no_fetch_handler.js b/tools/profiler/tests/browser/serviceworkers/serviceworker_no_fetch_handler.js new file mode 100644 index 0000000000..f656665ca0 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_no_fetch_handler.js @@ -0,0 +1,4 @@ +self.addEventListener("install", event => { + performance.mark("__serviceworker_event"); + console.log("[SW]:", "Install event"); +}); diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_no_respondWith_in_fetch_handler.js b/tools/profiler/tests/browser/serviceworkers/serviceworker_no_respondWith_in_fetch_handler.js new file mode 100644 index 0000000000..255c8269a1 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_no_respondWith_in_fetch_handler.js @@ -0,0 +1,9 @@ +self.addEventListener("install", event => { + performance.mark("__serviceworker_event"); + console.log("[SW]:", "Install event"); +}); + +self.addEventListener("fetch", event => { + performance.mark("__serviceworker_event"); + console.log("Handling fetch event for", event.request.url); +}); diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_page.html b/tools/profiler/tests/browser/serviceworkers/serviceworker_page.html new file mode 100644 index 0000000000..1c2100a9d6 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_page.html @@ -0,0 +1,10 @@ +<!doctype html> +<html> + <head> + <meta charset='utf-8'> + <meta name='viewport' content='initial-scale=1'> + </head> + <body> + <img src='firefox-logo-nightly.svg' width="24"> + </body> +</html> diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_register.html b/tools/profiler/tests/browser/serviceworkers/serviceworker_register.html new file mode 100644 index 0000000000..86719787f4 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_register.html @@ -0,0 +1,9 @@ +<!doctype html> +<html> + <head> + <meta charset='utf-8'> + <script src='serviceworker-utils.js'></script> + </head> + <body> + </body> +</html> diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_simple.html b/tools/profiler/tests/browser/serviceworkers/serviceworker_simple.html new file mode 100644 index 0000000000..f7c32d02c3 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_simple.html @@ -0,0 +1,9 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"/> + </head> + <body> + Testing + </body> +</html> diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_synthetized_response.js b/tools/profiler/tests/browser/serviceworkers/serviceworker_synthetized_response.js new file mode 100644 index 0000000000..891b679a5f --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_synthetized_response.js @@ -0,0 +1,27 @@ +self.addEventListener("install", event => { + performance.mark("__serviceworker_event"); + dump("[SW]:", "Install event\n"); +}); + +self.addEventListener("fetch", event => { + performance.mark("__serviceworker_event"); + dump(`Handling fetch event for ${event.request.url}\n`); + event.respondWith(handleFetch(event.request)); +}); + +async function handleFetch(request) { + if (request.url.endsWith("-generated.svg")) { + dump( + "An icon file that should be generated was requested, let's answer directly.\n" + ); + return new Response( + `<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 953.37 984"><defs><linearGradient id="linear-gradient" x1="-14706.28" y1="9250.14" x2="-14443.04" y2="9250.14" gradientTransform="matrix(0.76, 0.03, 0.05, -1.12, 11485.47, 11148)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#0083ff"/><stop offset="0.1" stop-color="#0092f8"/><stop offset="0.31" stop-color="#00abeb"/><stop offset="0.52" stop-color="#00bee1"/><stop offset="0.75" stop-color="#00c8dc"/><stop offset="1" stop-color="#00ccda"/></linearGradient><radialGradient id="radial-gradient" cx="-7588.66" cy="8866.53" r="791.23" gradientTransform="matrix(1.23, 0, 0, -1.22, 9958.21, 11048.11)" gradientUnits="userSpaceOnUse"><stop offset="0.02" stop-color="#005fe7"/><stop offset="0.18" stop-color="#0042b4"/><stop offset="0.32" stop-color="#002989"/><stop offset="0.4" stop-color="#002079"/><stop offset="0.47" stop-color="#131d78"/><stop offset="0.66" stop-color="#3b1676"/><stop offset="0.75" stop-color="#4a1475"/></radialGradient><linearGradient id="linear-gradient-2" x1="539.64" y1="254.8" x2="348.2" y2="881.03" gradientTransform="matrix(1, 0, 0, -1, 1, 984)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#000f43" stop-opacity="0.4"/><stop offset="0.48" stop-color="#001962" stop-opacity="0.17"/><stop offset="1" stop-color="#002079" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-3" x1="540.64" y1="254.8" x2="349.2" y2="881.03" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" xlink:href="#linear-gradient-2"/><linearGradient id="linear-gradient-4" x1="-8367.12" y1="7348.87" x2="-8482.36" y2="7357.76" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#812cc9"/><stop offset="1" stop-color="#005fe7"/></linearGradient><linearGradient id="linear-gradient-5" x1="-8449.89" y1="7496.97" x2="-8341.94" y2="7609.09" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0.05" stop-color="#005fe7"/><stop offset="0.18" stop-color="#065de6"/><stop offset="0.35" stop-color="#1856e1"/><stop offset="0.56" stop-color="#354adb"/><stop offset="0.78" stop-color="#5d3ad1"/><stop offset="0.95" stop-color="#812cc9"/></linearGradient><linearGradient id="linear-gradient-6" x1="-8653.41" y1="7245.3" x2="-8422.52" y2="7244.76" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#002079"/><stop offset="0.99" stop-color="#a238ff"/></linearGradient><radialGradient id="radial-gradient-2" cx="644.11" cy="599.83" fx="785.0454815336918" fy="470.6889181532662" r="793.95" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" gradientUnits="userSpaceOnUse"><stop offset="0.2" stop-color="#00fdff"/><stop offset="0.26" stop-color="#0af1ff"/><stop offset="0.37" stop-color="#23d2ff"/><stop offset="0.52" stop-color="#4da0ff"/><stop offset="0.69" stop-color="#855bff"/><stop offset="0.77" stop-color="#a238ff"/><stop offset="0.81" stop-color="#a738fd"/><stop offset="0.86" stop-color="#b539f9"/><stop offset="0.9" stop-color="#cd39f1"/><stop offset="0.96" stop-color="#ee3ae6"/><stop offset="0.98" stop-color="#ff3be0"/></radialGradient><linearGradient id="linear-gradient-7" x1="-7458.97" y1="9093.17" x2="-7531.06" y2="8282.84" gradientTransform="matrix(1.23, 0, 0, -1.22, 9958.21, 11048.11)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#00ec00"/><stop offset="0.1" stop-color="#00e244"/><stop offset="0.22" stop-color="#00d694"/><stop offset="0.31" stop-color="#00cfc7"/><stop offset="0.35" stop-color="#00ccda"/><stop offset="0.42" stop-color="#0bc2dd" stop-opacity="0.92"/><stop offset="0.57" stop-color="#29a7e4" stop-opacity="0.72"/><stop offset="0.77" stop-color="#597df0" stop-opacity="0.4"/><stop offset="1" stop-color="#9448ff" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-8" x1="-8926.61" y1="7680.53" x2="-8790.14" y2="7680.53" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#005fe7"/><stop offset="0.46" stop-color="#0071f3" stop-opacity="0.51"/><stop offset="0.83" stop-color="#007efc" stop-opacity="0.14"/><stop offset="1" stop-color="#0083ff" stop-opacity="0"/></linearGradient><radialGradient id="radial-gradient-3" cx="-8914.62" cy="7721.05" r="165.97" gradientTransform="matrix(1.22, 0.12, 0.12, -1.22, 10241.06, 10765.32)" gradientUnits="userSpaceOnUse"><stop offset="0.63" stop-color="#ffe302" stop-opacity="0"/><stop offset="0.67" stop-color="#ffe302" stop-opacity="0.05"/><stop offset="0.75" stop-color="#ffe302" stop-opacity="0.19"/><stop offset="0.86" stop-color="#ffe302" stop-opacity="0.4"/><stop offset="0.99" stop-color="#ffe302" stop-opacity="0.7"/></radialGradient><linearGradient id="linear-gradient-9" x1="214.02" y1="2032.47" x2="96.19" y2="2284.31" gradientTransform="matrix(0.99, 0.1, 0.1, -0.99, -250.1, 2306.29)" gradientUnits="userSpaceOnUse"><stop offset="0.19" stop-color="#4a1475" stop-opacity="0.5"/><stop offset="0.62" stop-color="#2277ac" stop-opacity="0.23"/><stop offset="0.94" stop-color="#00ccda" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-10" x1="-38.44" y1="278.18" x2="55.67" y2="171.29" gradientTransform="matrix(0.99, 0.1, 0.1, -0.99, 229.04, 745.87)" gradientUnits="userSpaceOnUse"><stop offset="0.01" stop-color="#002079" stop-opacity="0.5"/><stop offset="1" stop-color="#0083ff" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-11" x1="142.45" y1="96.25" x2="142.5" y2="149.68" gradientTransform="matrix(0.99, 0.1, 0.1, -0.99, 229.04, 745.87)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#4a1475" stop-opacity="0.9"/><stop offset="0.18" stop-color="#6720a2" stop-opacity="0.6"/><stop offset="0.38" stop-color="#812acb" stop-opacity="0.34"/><stop offset="0.57" stop-color="#9332e8" stop-opacity="0.15"/><stop offset="0.76" stop-color="#9e36f9" stop-opacity="0.04"/><stop offset="0.93" stop-color="#a238ff" stop-opacity="0"/></linearGradient><linearGradient id="linear-gradient-12" x1="620.52" y1="947.88" x2="926.18" y2="264.39" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#00ec00" stop-opacity="0"/><stop offset="0.28" stop-color="#00dc6d" stop-opacity="0.5"/><stop offset="0.5" stop-color="#00d1bb" stop-opacity="0.86"/><stop offset="0.6" stop-color="#00ccda"/><stop offset="0.68" stop-color="#04c9db"/><stop offset="0.75" stop-color="#0fc1df"/><stop offset="0.83" stop-color="#23b2e6"/><stop offset="0.9" stop-color="#3e9ef0"/><stop offset="0.98" stop-color="#6184fc"/><stop offset="0.99" stop-color="#6680fe"/></linearGradient><linearGradient id="linear-gradient-13" x1="680.88" y1="554.79" x2="536.1" y2="166.04" gradientTransform="matrix(1, 0, 0, -1, 0, 984)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#0083ff"/><stop offset="0.04" stop-color="#0083ff" stop-opacity="0.92"/><stop offset="0.14" stop-color="#0083ff" stop-opacity="0.71"/><stop offset="0.26" stop-color="#0083ff" stop-opacity="0.52"/><stop offset="0.37" stop-color="#0083ff" stop-opacity="0.36"/><stop offset="0.49" stop-color="#0083ff" stop-opacity="0.23"/><stop offset="0.61" stop-color="#0083ff" stop-opacity="0.13"/><stop offset="0.73" stop-color="#0083ff" stop-opacity="0.06"/><stop offset="0.86" stop-color="#0083ff" stop-opacity="0.01"/><stop offset="1" stop-color="#0083ff" stop-opacity="0"/></linearGradient></defs><title>firefox-logo-nightly</title><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><g id="Layer_2-2" data-name="Layer 2"><g id="Firefox"><path d="M770.28,91.56c-23.95,27.88-35.1,90.64-10.82,154.26s61.5,49.8,84.7,114.67c30.62,85.6,16.37,200.59,16.37,200.59s36.81,106.61,62.47-6.63C979.79,341.74,770.28,143.94,770.28,91.56Z" style="fill:url(#linear-gradient)"/><path id="_Path_" data-name=" Path " d="M476.92,972.83c245.24,0,443.9-199.74,443.9-446s-198.66-446-443.66-446S33.5,280.51,33.5,526.8C33,773.33,231.92,972.83,476.92,972.83Z" style="fill:url(#radial-gradient)"/><path d="M810.67,803.64a246.8,246.8,0,0,1-30.12,18.18,705.31,705.31,0,0,0,38.3-63c9.46-10.47,18.13-20.65,25.19-31.65,3.44-5.41,7.31-12.08,11.42-19.82,24.92-44.9,52.4-117.56,53.18-192.2v-5.66a257.25,257.25,0,0,0-5.71-55.75c.2,1.43.38,2.86.56,4.29-.22-1.1-.41-2.21-.64-3.31.37,2,.66,4,1,6,5.09,43.22,1.47,85.37-16.68,116.45-.29.45-.58.88-.87,1.32,9.41-47.23,12.56-99.39,2.09-151.6,0,0-4.19-25.38-35.38-102.44-18-44.35-49.83-80.72-78-107.21-24.69-30.55-47.11-51-59.47-64.06C689.72,126,678.9,105.61,674.45,92.31c-3.85-1.93-53.14-49.81-57.05-51.63-21.51,33.35-89.16,137.67-57,235.15,14.58,44.17,51.47,90,90.07,115.74,1.69,1.94,23,25,33.09,77.16,10.45,53.85,5,95.86-16.54,158C641.73,681.24,577,735.12,516.3,740.63c-129.67,11.78-177.15-65.11-177.15-65.11C385.49,694,436.72,690.17,467.87,671c31.4-19.43,50.39-33.83,65.81-28.15C548.86,648.43,561,632,550.1,615a78.5,78.5,0,0,0-79.4-34.57c-31.43,5.11-60.23,30-101.41,5.89a86.29,86.29,0,0,1-7.73-5.06c-2.71-1.79,8.83,2.72,6.13.69-8-4.35-22.2-13.84-25.88-17.22-.61-.56,6.22,2.18,5.61,1.62-38.51-31.71-33.7-53.13-32.49-66.57,1-10.75,8-24.52,19.75-30.11,5.69,3.11,9.24,5.48,9.24,5.48s-2.43-5-3.74-7.58c.46-.2.9-.15,1.36-.34,4.66,2.25,15,8.1,20.41,11.67,7.07,5,9.33,9.44,9.33,9.44s1.86-1,.48-5.37c-.5-1.78-2.65-7.45-9.65-13.17h.44A81.61,81.61,0,0,1,374.42,478c2-7.18,5.53-14.68,4.75-28.09-.48-9.43-.26-11.87-1.92-15.51-1.49-3.13.83-4.35,3.42-1.1a32.5,32.5,0,0,0-2.21-7.4v-.24c3.23-11.24,68.25-40.46,73-43.88A67.2,67.2,0,0,0,470.59,361c3.62-5.76,6.34-13.85,7-26.11.36-8.84-3.76-14.73-69.51-21.62-18-1.77-28.53-14.8-34.53-26.82-1.09-2.59-2.21-4.94-3.33-7.28a57.68,57.68,0,0,1-2.56-8.43c10.75-30.87,28.81-57,55.37-76.7,1.45-1.32-5.78.34-4.34-1,1.69-1.54,12.71-6,14.79-7,2.54-1.2-10.88-6.9-22.73-5.51-12.07,1.36-14.63,2.8-21.07,5.53,2.67-2.66,11.17-6.15,9.18-6.13-13,2-29.18,9.56-43,18.12a10.66,10.66,0,0,1,.83-4.35c-6.44,2.73-22.26,13.79-26.87,23.14a44.29,44.29,0,0,0,.27-5.4,84.17,84.17,0,0,0-13.19,13.82l-.24.22c-37.36-15-70.23-16-98.05-9.28-6.09-6.11-9.06-1.64-22.91-32.07-.94-1.83.72,1.81,0,0-2.28-5.9,1.39,7.87,0,0-23.28,18.37-53.92,39.19-68.63,53.89-.18.59,17.16-4.9,0,0-6,1.72-5.6,5.28-6.51,37.5-.22,2.44,0,5.18-.22,7.38-11.75,15-19.75,27.64-22.78,34.21-15.19,26.18-31.93,67-48.15,131.55A334.82,334.82,0,0,1,75.2,398.36C61.71,432.63,48.67,486.44,46.07,569.3A482.08,482.08,0,0,1,58.6,518.64,473,473,0,0,0,93.33,719.71c9.33,22.82,24.76,57.46,51,95.4C226.9,902,343.31,956,472.21,956,606.79,956,727.64,897.13,810.67,803.64Z" style="fill:url(#linear-gradient-2)"/><path d="M810.67,803.64a246.8,246.8,0,0,1-30.12,18.18,705.31,705.31,0,0,0,38.3-63c9.46-10.47,18.13-20.65,25.19-31.65,3.44-5.41,7.31-12.08,11.42-19.82,24.92-44.9,52.4-117.56,53.18-192.2v-5.66a257.25,257.25,0,0,0-5.71-55.75c.2,1.43.38,2.86.56,4.29-.22-1.1-.41-2.21-.64-3.31.37,2,.66,4,1,6,5.09,43.22,1.47,85.37-16.68,116.45-.29.45-.58.88-.87,1.32,9.41-47.23,12.56-99.39,2.09-151.6,0,0-4.19-25.38-35.38-102.44-18-44.35-49.83-80.72-78-107.21-24.69-30.55-47.11-51-59.47-64.06C689.72,126,678.9,105.61,674.45,92.31c-3.85-1.93-53.14-49.81-57.05-51.63-21.51,33.35-89.16,137.67-57,235.15,14.58,44.17,51.47,90,90.07,115.74,1.69,1.94,23,25,33.09,77.16,10.45,53.85,5,95.86-16.54,158C641.73,681.24,577,735.12,516.3,740.63c-129.67,11.78-177.15-65.11-177.15-65.11C385.49,694,436.72,690.17,467.87,671c31.4-19.43,50.39-33.83,65.81-28.15C548.86,648.43,561,632,550.1,615a78.5,78.5,0,0,0-79.4-34.57c-31.43,5.11-60.23,30-101.41,5.89a86.29,86.29,0,0,1-7.73-5.06c-2.71-1.79,8.83,2.72,6.13.69-8-4.35-22.2-13.84-25.88-17.22-.61-.56,6.22,2.18,5.61,1.62-38.51-31.71-33.7-53.13-32.49-66.57,1-10.75,8-24.52,19.75-30.11,5.69,3.11,9.24,5.48,9.24,5.48s-2.43-5-3.74-7.58c.46-.2.9-.15,1.36-.34,4.66,2.25,15,8.1,20.41,11.67,7.07,5,9.33,9.44,9.33,9.44s1.86-1,.48-5.37c-.5-1.78-2.65-7.45-9.65-13.17h.44A81.61,81.61,0,0,1,374.42,478c2-7.18,5.53-14.68,4.75-28.09-.48-9.43-.26-11.87-1.92-15.51-1.49-3.13.83-4.35,3.42-1.1a32.5,32.5,0,0,0-2.21-7.4v-.24c3.23-11.24,68.25-40.46,73-43.88A67.2,67.2,0,0,0,470.59,361c3.62-5.76,6.34-13.85,7-26.11.36-8.84-3.76-14.73-69.51-21.62-18-1.77-28.53-14.8-34.53-26.82-1.09-2.59-2.21-4.94-3.33-7.28a57.68,57.68,0,0,1-2.56-8.43c10.75-30.87,28.81-57,55.37-76.7,1.45-1.32-5.78.34-4.34-1,1.69-1.54,12.71-6,14.79-7,2.54-1.2-10.88-6.9-22.73-5.51-12.07,1.36-14.63,2.8-21.07,5.53,2.67-2.66,11.17-6.15,9.18-6.13-13,2-29.18,9.56-43,18.12a10.66,10.66,0,0,1,.83-4.35c-6.44,2.73-22.26,13.79-26.87,23.14a44.29,44.29,0,0,0,.27-5.4,84.17,84.17,0,0,0-13.19,13.82l-.24.22c-37.36-15-70.23-16-98.05-9.28-6.09-6.11-9.06-1.64-22.91-32.07-.94-1.83.72,1.81,0,0-2.28-5.9,1.39,7.87,0,0-23.28,18.37-53.92,39.19-68.63,53.89-.18.59,17.16-4.9,0,0-6,1.72-5.6,5.28-6.51,37.5-.22,2.44,0,5.18-.22,7.38-11.75,15-19.75,27.64-22.78,34.21-15.19,26.18-31.93,67-48.15,131.55A334.82,334.82,0,0,1,75.2,398.36C61.71,432.63,48.67,486.44,46.07,569.3A482.08,482.08,0,0,1,58.6,518.64,473,473,0,0,0,93.33,719.71c9.33,22.82,24.76,57.46,51,95.4C226.9,902,343.31,956,472.21,956,606.79,956,727.64,897.13,810.67,803.64Z" style="fill:url(#linear-gradient-3)"/><path d="M711.1,866.71c162.87-18.86,235-186.7,142.38-190C769.85,674,634,875.61,711.1,866.71Z" style="fill:url(#linear-gradient-4)"/><path d="M865.21,642.42C977.26,577.21,948,436.34,948,436.34s-43.25,50.24-72.62,130.32C846.4,646,797.84,681.81,865.21,642.42Z" style="fill:url(#linear-gradient-5)"/><path d="M509.47,950.06C665.7,999.91,800,876.84,717.21,835.74,642,798.68,435.32,926.49,509.47,950.06Z" style="fill:url(#linear-gradient-6)"/><path d="M638.58,21.42l.53-.57A1.7,1.7,0,0,0,638.58,21.42ZM876.85,702.23c3.8-5.36,8.94-22.53,13.48-30.21,27.58-44.52,27.78-80,27.78-80.84,16.66-83.22,15.15-117.2,4.9-180-8.25-50.6-44.32-123.09-75.57-158-32.2-36-9.51-24.25-40.69-50.52-27.33-30.29-53.82-60.29-68.25-72.36C634.22,43.09,636.57,24.58,638.58,21.42c-.34.37-.84.92-1.47,1.64C635.87,18.14,635,14,635,14s-57,57-69,152c-7.83,62,15.38,126.68,49,168a381.62,381.62,0,0,0,59,58h0c25.4,36.48,39.38,81.49,39.38,129.91,0,121.24-98.34,219.53-219.65,219.53a220.14,220.14,0,0,1-49.13-5.52c-57.24-10.92-90.3-39.8-106.78-59.41-9.45-11.23-13.46-19.42-13.46-19.42,51.28,18.37,108,14.53,142.47-4.52,34.75-19.26,55.77-33.55,72.84-27.92,16.82,5.61,30.21-10.67,18.2-27.54-11.77-16.85-42.4-41-87.88-34.29-34.79,5.07-66.66,29.76-112.24,5.84a97.34,97.34,0,0,1-8.55-5c-3-1.77,9.77,2.69,6.79.68-8.87-4.32-24.57-13.73-28.64-17.07-.68-.56,6.88,2.16,6.2,1.6-42.62-31.45-37.3-52.69-36-66,1.07-10.66,8.81-24.32,21.86-29.86,6.3,3.08,10.23,5.43,10.23,5.43s-2.69-4.92-4.14-7.51c.51-.19,1-.15,1.5-.34,5.16,2.23,16.58,8,22.59,11.57,7.83,4.95,10.32,9.36,10.32,9.36s2.06-1,.54-5.33c-.56-1.77-2.93-7.39-10.68-13.07h.48a91.65,91.65,0,0,1,13.13,8.17c2.19-7.12,6.12-14.56,5.25-27.86-.53-9.35-.28-11.78-2.12-15.39-1.65-3.1.92-4.31,3.78-1.09a29.73,29.73,0,0,0-2.44-7.34v-.24c3.57-11.14,75.53-40.12,80.77-43.51a70.24,70.24,0,0,0,21.17-20.63c4-5.72,7-13.73,7.75-25.89.25-5.48-1.44-9.82-20.5-14-11.44-2.49-29.14-4.91-56.43-7.47-19.9-1.76-31.58-14.68-38.21-26.6-1.21-2.57-2.45-4.9-3.68-7.22a53.41,53.41,0,0,1-2.83-8.36,158.47,158.47,0,0,1,61.28-76.06c1.6-1.31-6.4.33-4.8-1,1.87-1.52,14.06-5.93,16.37-6.92,2.81-1.19-12-6.84-25.16-5.47-13.36,1.35-16.19,2.78-23.32,5.49,3-2.64,12.37-6.1,10.16-6.08-14.4,2-32.3,9.48-47.6,18a9.72,9.72,0,0,1,.92-4.31c-7.13,2.71-24.64,13.67-29.73,23a39.79,39.79,0,0,0,.29-5.35,88.55,88.55,0,0,0-14.6,13.7l-.27.22C258.14,196,221.75,195,191,201.72c-6.74-6.06-17.57-15.23-32.89-45.4-1-1.82-1.6,3.75-2.4,2-6-13.81-9.55-36.44-9-52,0,0-12.32,5.61-22.51,29.06-1.89,4.21-3.11,6.54-4.32,8.87-.56.68,1.27-7.7,1-7.24-1.77,3-6.36,7.19-8.37,12.62-1.38,4-3.32,6.27-4.56,11.29l-.29.46c-.1-1.48.37-6.08,0-5.14A235.4,235.4,0,0,0,95.34,186c-5.49,18-11.88,42.61-12.89,74.57-.24,2.42,0,5.14-.25,7.32-13,14.83-21.86,27.39-25.2,33.91-16.81,26-35.33,66.44-53.29,130.46a319.35,319.35,0,0,1,28.54-50C17.32,416.25,2.89,469.62,0,551.8a436.92,436.92,0,0,1,13.87-50.24C11.29,556.36,17.68,624.3,52.32,701c20.57,45,67.92,136.6,183.62,208h0s39.36,29.3,107,51.26c5,1.81,10.06,3.6,15.23,5.33q-2.43-1-4.71-2A484.9,484.9,0,0,0,492.27,984c175.18.15,226.85-70.2,226.85-70.2l-.51.38q3.71-3.49,7.14-7.26c-27.64,26.08-90.75,27.84-114.3,26,40.22-11.81,66.69-21.81,118.17-41.52q9-3.36,18.48-7.64l2-.94c1.25-.58,2.49-1.13,3.75-1.74a349.3,349.3,0,0,0,70.26-44c51.7-41.3,63-81.56,68.83-108.1-.82,2.54-3.37,8.47-5.17,12.32-13.31,28.48-42.84,46-74.91,61a689.05,689.05,0,0,0,42.38-62.44C865.77,729.39,869,713.15,876.85,702.23Z" style="fill:url(#radial-gradient-2)"/><path d="M813.92,801c21.08-23.24,40-49.82,54.35-80,36.9-77.58,94-206.58,49-341.31C881.77,273.22,833,215,771.11,158.12,670.56,65.76,642.48,24.52,642.48,0c0,0-116.09,129.41-65.74,264.38s153.46,130,221.68,270.87c80.27,165.74-64.95,346.61-185,397.24,7.35-1.63,267-60.38,280.61-208.88C893.68,726.34,887.83,767.41,813.92,801Z" style="fill:url(#linear-gradient-7)"/><path d="M477.59,319.37c.39-8.77-4.16-14.66-76.68-21.46-29.84-2.76-41.26-30.33-44.75-41.94-10.61,27.56-15,56.49-12.64,91.48,1.61,22.92,17,47.52,24.37,62,0,0,1.64-2.13,2.39-2.91,13.86-14.43,71.94-36.42,77.39-39.54C453.69,363.16,476.58,346.44,477.59,319.37Z" style="fill:url(#linear-gradient-8)"/><path d="M477.59,319.37c.39-8.77-4.16-14.66-76.68-21.46-29.84-2.76-41.26-30.33-44.75-41.94-10.61,27.56-15,56.49-12.64,91.48,1.61,22.92,17,47.52,24.37,62,0,0,1.64-2.13,2.39-2.91,13.86-14.43,71.94-36.42,77.39-39.54C453.69,363.16,476.58,346.44,477.59,319.37Z" style="opacity:0.5;isolation:isolate;fill:url(#radial-gradient-3)"/><path d="M158.31,156.47c-1-1.82-1.6,3.75-2.4,2-6-13.81-9.58-36.2-8.72-52,0,0-12.32,5.61-22.51,29.06-1.89,4.21-3.11,6.54-4.32,8.86-.56.68,1.27-7.7,1-7.24-1.77,3-6.36,7.19-8.35,12.38-1.65,4.24-3.35,6.52-4.61,11.77-.39,1.43.39-6.32,0-5.38C84.72,201.68,80.19,271,82.69,268,133.17,214.14,191,201.36,191,201.36c-6.15-4.53-19.53-17.63-32.7-44.89Z" style="fill:url(#linear-gradient-9)"/><path d="M349.84,720.1c-69.72-29.77-149-71.75-146-167.14C207.92,427.35,321,452.18,321,452.18c-4.27,1-15.68,9.16-19.72,17.82-4.27,10.83-12.07,35.28,11.55,60.9,37.09,40.19-76.2,95.36,98.66,199.57,4.41,2.4-41-1.43-61.64-10.36Z" style="fill:url(#linear-gradient-10)"/><path d="M325.07,657.5c49.44,17.21,107,14.19,141.52-4.86,23.09-12.85,52.7-33.43,70.92-28.35-15.78-6.24-27.73-9.15-42.1-9.86-2.45,0-5.38,0-8-.32a136,136,0,0,0-15.76.86c-8.9.82-18.77,6.43-27.74,5.53-.48,0,8.7-3.77,8-3.61-4.75,1-9.92,1.21-15.37,1.88-3.47.39-6.45.82-9.89,1-103,8.73-190-55.81-190-55.81-7.41,25,33.17,74.3,88.52,93.57Z" style="opacity:0.5;isolation:isolate;fill:url(#linear-gradient-11)"/><path d="M813.74,801.65c104.16-102.27,156.86-226.58,134.58-366,0,0,8.9,71.5-24.85,144.63,16.21-71.39,18.1-160.11-25-252C841,205.64,746.45,141.11,710.35,114.19,655.66,73.4,633,31.87,632.57,23.3c-16.34,33.48-65.77,148.2-5.31,247,56.64,92.56,145.86,120,208.33,205C950.67,631.67,813.74,801.65,813.74,801.65Z" style="fill:url(#linear-gradient-12)"/><path d="M798.81,535.55C762.41,460.35,717,427.55,674,392c5,7,6.23,9.47,9,14,37.83,40.32,93.61,138.66,53.11,262.11C659.88,900.48,355,791.06,323,760.32,335.93,894.81,561,959.16,707.6,872,791,793,858.47,658.79,798.81,535.55Z" style="fill:url(#linear-gradient-13)"/></g></g></g></g></svg>`, + { headers: { "content-type": "image/svg+xml" } } + ); + } + + dump( + `A normal URL ${request.url} has been requested, let's fetch it from the network.\n` + ); + return fetch(request); +} diff --git a/tools/profiler/tests/browser/simple.html b/tools/profiler/tests/browser/simple.html new file mode 100644 index 0000000000..f7c32d02c3 --- /dev/null +++ b/tools/profiler/tests/browser/simple.html @@ -0,0 +1,9 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"/> + </head> + <body> + Testing + </body> +</html> diff --git a/tools/profiler/tests/browser/single_frame.html b/tools/profiler/tests/browser/single_frame.html new file mode 100644 index 0000000000..ebdfc41da2 --- /dev/null +++ b/tools/profiler/tests/browser/single_frame.html @@ -0,0 +1,10 @@ +<!DOCTYPE html> +<html> +<head> + <meta charset="utf-8"> + <title>Single Frame</title> +</head> +<body> + Single Frame +</body> +</html> diff --git a/tools/profiler/tests/chrome/chrome.toml b/tools/profiler/tests/chrome/chrome.toml new file mode 100644 index 0000000000..905d1ffff3 --- /dev/null +++ b/tools/profiler/tests/chrome/chrome.toml @@ -0,0 +1,9 @@ +[DEFAULT] +skip-if = ["tsan"] # Bug 1804081 +support-files = ["profiler_test_utils.js"] + +["test_profile_worker.html"] +skip-if = ["os == 'android' && processor == 'arm'"] # Bug 1541291 + +["test_profile_worker_bug_1428076.html"] +skip-if = ["os == 'android' && processor == 'arm'"] # Bug 1541291 diff --git a/tools/profiler/tests/chrome/profiler_test_utils.js b/tools/profiler/tests/chrome/profiler_test_utils.js new file mode 100644 index 0000000000..d2e4499b34 --- /dev/null +++ b/tools/profiler/tests/chrome/profiler_test_utils.js @@ -0,0 +1,66 @@ +"use strict"; + +(function () { + async function startProfiler(settings) { + let startPromise = Services.profiler.StartProfiler( + settings.entries, + settings.interval, + settings.features, + settings.threads, + 0, + settings.duration + ); + + info("Parent Profiler has started"); + + await startPromise; + + info("Child profilers have started"); + } + + function getProfile() { + const profile = Services.profiler.getProfileData(); + info( + "We got a profile, run the mochitest with `--keep-open true` to see the logged profile in the Web Console." + ); + + // Run the mochitest with `--keep-open true` to see the logged profile in the + // Web console. + console.log(profile); + + return profile; + } + + async function stopProfiler() { + let stopPromise = Services.profiler.StopProfiler(); + info("Parent profiler has stopped"); + await stopPromise; + info("Child profilers have stopped"); + } + + function end(error) { + if (error) { + ok(false, `We got an error: ${error}`); + } else { + ok(true, "We ran the whole process"); + } + SimpleTest.finish(); + } + + async function runTest(settings, workload) { + SimpleTest.waitForExplicitFinish(); + try { + await startProfiler(settings); + await workload(); + await getProfile(); + await stopProfiler(); + await end(); + } catch (e) { + // By catching and handling the error, we're being nice to mochitest + // runners: instead of waiting for the timeout, we fail right away. + await end(e); + } + } + + window.runTest = runTest; +})(); diff --git a/tools/profiler/tests/chrome/test_profile_worker.html b/tools/profiler/tests/chrome/test_profile_worker.html new file mode 100644 index 0000000000..8e2bae7fbd --- /dev/null +++ b/tools/profiler/tests/chrome/test_profile_worker.html @@ -0,0 +1,66 @@ +<!DOCTYPE HTML> +<html> +<!-- +https://bugzilla.mozilla.org/show_bug.cgi?id=1428076 +--> +<head> + <meta charset="utf-8"> + <title>Test for Bug 1428076</title> + <link rel="stylesheet" type="text/css" href="chrome://global/skin"/> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"/> +</head> +<body> +<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1428076">Mozilla Bug 1428076</a> + +<script src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> +<script type="application/javascript" src="profiler_test_utils.js"></script> +<script type="application/javascript"> +/* globals runTest */ + +"use strict"; + +const settings = { + entries: 1000000, // 9MB + interval: 1, // ms + features: ["js", "stackwalk", "cpu"], + threads: ["GeckoMain", "Compositor", "Worker"], // most common combination +}; + +const workerCode = ` + console.log('hello world'); + setTimeout(() => postMessage('message from worker'), 50); +`; + +function startWorker() { + // We use a Blob for the worker content to avoid an external JS file, and data + // URLs seem to be blocked in a chrome environment. + const workerContent = new Blob( + [ workerCode ], + { type: "application/javascript" } + ); + const blobURL = URL.createObjectURL(workerContent); + + // We start a worker and then terminate it right away to trigger our bug. + info("Starting the worker..."); + const myWorker = new Worker(blobURL); + return { worker: myWorker, url: blobURL }; +} + +function workload() { + const { worker, url } = startWorker(); + + return new Promise(resolve => { + worker.onmessage = () => { + info("Got a message, terminating the worker."); + worker.terminate(); + URL.revokeObjectURL(url); + resolve(); + }; + }); +} + +runTest(settings, workload); + +</script> +</body> +</html> diff --git a/tools/profiler/tests/chrome/test_profile_worker_bug_1428076.html b/tools/profiler/tests/chrome/test_profile_worker_bug_1428076.html new file mode 100644 index 0000000000..abe0e5748a --- /dev/null +++ b/tools/profiler/tests/chrome/test_profile_worker_bug_1428076.html @@ -0,0 +1,58 @@ +<!DOCTYPE HTML> +<html> +<!-- +https://bugzilla.mozilla.org/show_bug.cgi?id=1428076 +--> +<head> + <meta charset="utf-8"> + <title>Test for Bug 1428076</title> + <link rel="stylesheet" type="text/css" href="chrome://global/skin"/> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"/> +</head> +<body> +<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1428076">Mozilla Bug 1428076</a> + +<script src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> +<script type="application/javascript" src="profiler_test_utils.js"></script> +<script type="application/javascript"> +/** Test for Bug 1428076 **/ + +/* globals runTest */ + +"use strict"; + +const settings = { + entries: 1000000, // 9MB + interval: 1, // ms + features: ["js", "stackwalk"], + threads: ["GeckoMain", "Compositor", "Worker"], // most common combination +}; + +function workload() { + // We use a Blob for the worker content to avoid an external JS file, and data + // URLs seem to be blocked in a chrome environment. + const workerContent = new Blob( + [ "console.log('hello world!')" ], + { type: "application/javascript" } + ); + const blobURL = URL.createObjectURL(workerContent); + + // We start a worker and then terminate it right away to trigger our bug. + info("Starting the worker, and terminate it right away."); + const myWorker = new Worker(blobURL); + myWorker.terminate(); + + URL.revokeObjectURL(blobURL); + + // We're deferring some little time so that the worker has the time to be + // properly cleaned up and the profiler actually saves the worker data. + return new Promise(resolve => { + setTimeout(resolve, 50); + }); +} + +runTest(settings, workload); + +</script> +</body> +</html> diff --git a/tools/profiler/tests/gtest/GeckoProfiler.cpp b/tools/profiler/tests/gtest/GeckoProfiler.cpp new file mode 100644 index 0000000000..79ed7cb52a --- /dev/null +++ b/tools/profiler/tests/gtest/GeckoProfiler.cpp @@ -0,0 +1,5024 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This file tests a lot of the profiler_*() functions in GeckoProfiler.h. +// Most of the tests just check that nothing untoward (e.g. crashes, deadlocks) +// happens when calling these functions. They don't do much inspection of +// profiler internals. + +#include "mozilla/ProfilerThreadPlatformData.h" +#include "mozilla/ProfilerThreadRegistration.h" +#include "mozilla/ProfilerThreadRegistrationInfo.h" +#include "mozilla/ProfilerThreadRegistry.h" +#include "mozilla/ProfilerUtils.h" +#include "mozilla/ProgressLogger.h" +#include "mozilla/UniquePtrExtensions.h" + +#include "nsIThread.h" +#include "nsThreadUtils.h" +#include "prthread.h" + +#include "gtest/gtest.h" +#include "mozilla/gtest/MozAssertions.h" + +#include <thread> + +#if defined(_MSC_VER) || defined(__MINGW32__) +# include <processthreadsapi.h> +# include <realtimeapiset.h> +#elif defined(__APPLE__) +# include <mach/thread_act.h> +#endif + +#ifdef MOZ_GECKO_PROFILER + +# include "GeckoProfiler.h" +# include "mozilla/ProfilerMarkerTypes.h" +# include "mozilla/ProfilerMarkers.h" +# include "NetworkMarker.h" +# include "platform.h" +# include "ProfileBuffer.h" +# include "ProfilerControl.h" + +# include "js/Initialization.h" +# include "js/Printf.h" +# include "jsapi.h" +# include "json/json.h" +# include "mozilla/Atomics.h" +# include "mozilla/DataMutex.h" +# include "mozilla/ProfileBufferEntrySerializationGeckoExtensions.h" +# include "mozilla/ProfileJSONWriter.h" +# include "mozilla/ScopeExit.h" +# include "mozilla/net/HttpBaseChannel.h" +# include "nsIChannelEventSink.h" +# include "nsIThread.h" +# include "nsThreadUtils.h" + +# include <cstring> +# include <set> + +#endif // MOZ_GECKO_PROFILER + +// Note: profiler_init() has already been called in XRE_main(), so we can't +// test it here. Likewise for profiler_shutdown(), and AutoProfilerInit +// (which is just an RAII wrapper for profiler_init() and profiler_shutdown()). + +using namespace mozilla; + +TEST(GeckoProfiler, ProfilerUtils) +{ + profiler_init_main_thread_id(); + + static_assert(std::is_same_v<decltype(profiler_current_process_id()), + ProfilerProcessId>); + static_assert( + std::is_same_v<decltype(profiler_current_process_id()), + decltype(baseprofiler::profiler_current_process_id())>); + ProfilerProcessId processId = profiler_current_process_id(); + EXPECT_TRUE(processId.IsSpecified()); + EXPECT_EQ(processId, baseprofiler::profiler_current_process_id()); + + static_assert( + std::is_same_v<decltype(profiler_current_thread_id()), ProfilerThreadId>); + static_assert( + std::is_same_v<decltype(profiler_current_thread_id()), + decltype(baseprofiler::profiler_current_thread_id())>); + EXPECT_EQ(profiler_current_thread_id(), + baseprofiler::profiler_current_thread_id()); + + ProfilerThreadId mainTestThreadId = profiler_current_thread_id(); + EXPECT_TRUE(mainTestThreadId.IsSpecified()); + + ProfilerThreadId mainThreadId = profiler_main_thread_id(); + EXPECT_TRUE(mainThreadId.IsSpecified()); + + EXPECT_EQ(mainThreadId, mainTestThreadId) + << "Test should run on the main thread"; + EXPECT_TRUE(profiler_is_main_thread()); + + std::thread testThread([&]() { + EXPECT_EQ(profiler_current_process_id(), processId); + + const ProfilerThreadId testThreadId = profiler_current_thread_id(); + EXPECT_TRUE(testThreadId.IsSpecified()); + EXPECT_NE(testThreadId, mainThreadId); + EXPECT_FALSE(profiler_is_main_thread()); + + EXPECT_EQ(baseprofiler::profiler_current_process_id(), processId); + EXPECT_EQ(baseprofiler::profiler_current_thread_id(), testThreadId); + EXPECT_EQ(baseprofiler::profiler_main_thread_id(), mainThreadId); + EXPECT_FALSE(baseprofiler::profiler_is_main_thread()); + }); + testThread.join(); +} + +TEST(GeckoProfiler, ThreadRegistrationInfo) +{ + profiler_init_main_thread_id(); + + TimeStamp ts = TimeStamp::Now(); + { + profiler::ThreadRegistrationInfo trInfo{ + "name", ProfilerThreadId::FromNumber(123), false, ts}; + EXPECT_STREQ(trInfo.Name(), "name"); + EXPECT_NE(trInfo.Name(), "name") + << "ThreadRegistrationInfo should keep its own copy of the name"; + EXPECT_EQ(trInfo.RegisterTime(), ts); + EXPECT_EQ(trInfo.ThreadId(), ProfilerThreadId::FromNumber(123)); + EXPECT_EQ(trInfo.IsMainThread(), false); + } + + // Make sure the next timestamp will be different from `ts`. + while (TimeStamp::Now() == ts) { + } + + { + profiler::ThreadRegistrationInfo trInfoHere{"Here"}; + EXPECT_STREQ(trInfoHere.Name(), "Here"); + EXPECT_NE(trInfoHere.Name(), "Here") + << "ThreadRegistrationInfo should keep its own copy of the name"; + TimeStamp baseRegistrationTime = + baseprofiler::detail::GetThreadRegistrationTime(); + if (baseRegistrationTime) { + EXPECT_EQ(trInfoHere.RegisterTime(), baseRegistrationTime); + } else { + EXPECT_GT(trInfoHere.RegisterTime(), ts); + } + EXPECT_EQ(trInfoHere.ThreadId(), profiler_current_thread_id()); + EXPECT_EQ(trInfoHere.ThreadId(), profiler_main_thread_id()) + << "Gtests are assumed to run on the main thread"; + EXPECT_EQ(trInfoHere.IsMainThread(), true) + << "Gtests are assumed to run on the main thread"; + } + + { + // Sub-thread test. + // These will receive sub-thread data (to test move at thread end). + TimeStamp tsThread; + ProfilerThreadId threadThreadId; + UniquePtr<profiler::ThreadRegistrationInfo> trInfoThreadPtr; + + std::thread testThread([&]() { + profiler::ThreadRegistrationInfo trInfoThread{"Thread"}; + EXPECT_STREQ(trInfoThread.Name(), "Thread"); + EXPECT_NE(trInfoThread.Name(), "Thread") + << "ThreadRegistrationInfo should keep its own copy of the name"; + EXPECT_GT(trInfoThread.RegisterTime(), ts); + EXPECT_EQ(trInfoThread.ThreadId(), profiler_current_thread_id()); + EXPECT_NE(trInfoThread.ThreadId(), profiler_main_thread_id()); + EXPECT_EQ(trInfoThread.IsMainThread(), false); + + tsThread = trInfoThread.RegisterTime(); + threadThreadId = trInfoThread.ThreadId(); + trInfoThreadPtr = + MakeUnique<profiler::ThreadRegistrationInfo>(std::move(trInfoThread)); + }); + testThread.join(); + + ASSERT_NE(trInfoThreadPtr, nullptr); + EXPECT_STREQ(trInfoThreadPtr->Name(), "Thread"); + EXPECT_EQ(trInfoThreadPtr->RegisterTime(), tsThread); + EXPECT_EQ(trInfoThreadPtr->ThreadId(), threadThreadId); + EXPECT_EQ(trInfoThreadPtr->IsMainThread(), false) + << "Gtests are assumed to run on the main thread"; + } +} + +static constexpr ThreadProfilingFeatures scEachAndAnyThreadProfilingFeatures[] = + {ThreadProfilingFeatures::CPUUtilization, ThreadProfilingFeatures::Sampling, + ThreadProfilingFeatures::Markers, ThreadProfilingFeatures::Any}; + +TEST(GeckoProfiler, ThreadProfilingFeaturesType) +{ + ASSERT_EQ(static_cast<uint32_t>(ThreadProfilingFeatures::Any), 1u + 2u + 4u) + << "This test assumes that there are 3 binary choices 1+2+4; " + "Is this test up to date?"; + + EXPECT_EQ(Combine(ThreadProfilingFeatures::CPUUtilization, + ThreadProfilingFeatures::Sampling, + ThreadProfilingFeatures::Markers), + ThreadProfilingFeatures::Any); + + constexpr ThreadProfilingFeatures allThreadProfilingFeatures[] = { + ThreadProfilingFeatures::NotProfiled, + ThreadProfilingFeatures::CPUUtilization, + ThreadProfilingFeatures::Sampling, ThreadProfilingFeatures::Markers, + ThreadProfilingFeatures::Any}; + + for (ThreadProfilingFeatures f1 : allThreadProfilingFeatures) { + // Combine and Intersect are commutative. + for (ThreadProfilingFeatures f2 : allThreadProfilingFeatures) { + EXPECT_EQ(Combine(f1, f2), Combine(f2, f1)); + EXPECT_EQ(Intersect(f1, f2), Intersect(f2, f1)); + } + + // Combine works like OR. + EXPECT_EQ(Combine(f1, f1), f1); + EXPECT_EQ(Combine(f1, f1, f1), f1); + + // 'OR NotProfiled' doesn't change anything. + EXPECT_EQ(Combine(f1, ThreadProfilingFeatures::NotProfiled), f1); + + // 'OR Any' makes Any. + EXPECT_EQ(Combine(f1, ThreadProfilingFeatures::Any), + ThreadProfilingFeatures::Any); + + // Intersect works like AND. + EXPECT_EQ(Intersect(f1, f1), f1); + EXPECT_EQ(Intersect(f1, f1, f1), f1); + + // 'AND NotProfiled' erases anything. + EXPECT_EQ(Intersect(f1, ThreadProfilingFeatures::NotProfiled), + ThreadProfilingFeatures::NotProfiled); + + // 'AND Any' doesn't change anything. + EXPECT_EQ(Intersect(f1, ThreadProfilingFeatures::Any), f1); + } + + for (ThreadProfilingFeatures f1 : scEachAndAnyThreadProfilingFeatures) { + EXPECT_TRUE(DoFeaturesIntersect(f1, f1)); + + // NotProfiled doesn't intersect with any feature. + EXPECT_FALSE(DoFeaturesIntersect(f1, ThreadProfilingFeatures::NotProfiled)); + + // Any intersects with any feature. + EXPECT_TRUE(DoFeaturesIntersect(f1, ThreadProfilingFeatures::Any)); + } +} + +static void TestConstUnlockedConstReader( + const profiler::ThreadRegistration::UnlockedConstReader& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + EXPECT_STREQ(aData.Info().Name(), "Test thread"); + EXPECT_GE(aData.Info().RegisterTime(), aBeforeRegistration); + EXPECT_LE(aData.Info().RegisterTime(), aAfterRegistration); + EXPECT_EQ(aData.Info().ThreadId(), aThreadId); + EXPECT_FALSE(aData.Info().IsMainThread()); + +#if (defined(_MSC_VER) || defined(__MINGW32__)) && defined(MOZ_GECKO_PROFILER) + HANDLE threadHandle = aData.PlatformDataCRef().ProfiledThread(); + EXPECT_NE(threadHandle, nullptr); + EXPECT_EQ(ProfilerThreadId::FromNumber(::GetThreadId(threadHandle)), + aThreadId); + // Test calling QueryThreadCycleTime, we cannot assume that it will always + // work, but at least it shouldn't crash. + ULONG64 cycles; + (void)QueryThreadCycleTime(threadHandle, &cycles); +#elif defined(__APPLE__) && defined(MOZ_GECKO_PROFILER) + // Test calling thread_info, we cannot assume that it will always work, but at + // least it shouldn't crash. + thread_basic_info_data_t threadBasicInfo; + mach_msg_type_number_t basicCount = THREAD_BASIC_INFO_COUNT; + (void)thread_info( + aData.PlatformDataCRef().ProfiledThread(), THREAD_BASIC_INFO, + reinterpret_cast<thread_info_t>(&threadBasicInfo), &basicCount); +#elif (defined(__linux__) || defined(__ANDROID__) || defined(__FreeBSD__)) && \ + defined(MOZ_GECKO_PROFILER) + // Test calling GetClockId, we cannot assume that it will always work, but at + // least it shouldn't crash. + Maybe<clockid_t> maybeClockId = aData.PlatformDataCRef().GetClockId(); + if (maybeClockId) { + // Test calling clock_gettime, we cannot assume that it will always work, + // but at least it shouldn't crash. + timespec ts; + (void)clock_gettime(*maybeClockId, &ts); + } +#else + (void)aData.PlatformDataCRef(); +#endif + + EXPECT_GE(aData.StackTop(), aOnStackObject) + << "StackTop should be at &onStackChar, or higher on some " + "platforms"; +}; + +static void TestConstUnlockedConstReaderAndAtomicRW( + const profiler::ThreadRegistration::UnlockedConstReaderAndAtomicRW& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedConstReader(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + + (void)aData.ProfilingStackCRef(); + + EXPECT_EQ(aData.ProfilingFeatures(), ThreadProfilingFeatures::NotProfiled); + + EXPECT_FALSE(aData.IsSleeping()); +}; + +static void TestUnlockedConstReaderAndAtomicRW( + profiler::ThreadRegistration::UnlockedConstReaderAndAtomicRW& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedConstReaderAndAtomicRW(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + (void)aData.ProfilingStackRef(); + + EXPECT_FALSE(aData.IsSleeping()); + aData.SetSleeping(); + EXPECT_TRUE(aData.IsSleeping()); + aData.SetAwake(); + EXPECT_FALSE(aData.IsSleeping()); + + aData.ReinitializeOnResume(); + + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + aData.SetSleeping(); + // After sleeping, the 2nd+ calls can duplicate. + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_TRUE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_TRUE(aData.CanDuplicateLastSampleDueToSleep()); + aData.ReinitializeOnResume(); + // After reinit (and sleeping), the 2nd+ calls can duplicate. + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_TRUE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_TRUE(aData.CanDuplicateLastSampleDueToSleep()); + aData.SetAwake(); + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); +}; + +static void TestConstUnlockedRWForLockedProfiler( + const profiler::ThreadRegistration::UnlockedRWForLockedProfiler& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedConstReaderAndAtomicRW(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + // We can't create a PSAutoLock here, so just verify that the call would + // compile and return the expected type. + static_assert(std::is_same_v<decltype(aData.GetProfiledThreadData( + std::declval<PSAutoLock>())), + const ProfiledThreadData*>); +}; + +static void TestConstUnlockedReaderAndAtomicRWOnThread( + const profiler::ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& + aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedRWForLockedProfiler(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + EXPECT_EQ(aData.GetJSContext(), nullptr); +}; + +static void TestUnlockedRWForLockedProfiler( + profiler::ThreadRegistration::UnlockedRWForLockedProfiler& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedRWForLockedProfiler(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + TestUnlockedConstReaderAndAtomicRW(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + // No functions to test here. +}; + +static void TestUnlockedReaderAndAtomicRWOnThread( + profiler::ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedReaderAndAtomicRWOnThread(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + TestUnlockedRWForLockedProfiler(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + // No functions to test here. +}; + +static void TestConstLockedRWFromAnyThread( + const profiler::ThreadRegistration::LockedRWFromAnyThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedReaderAndAtomicRWOnThread(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + EXPECT_EQ(aData.GetJsFrameBuffer(), nullptr); + EXPECT_EQ(aData.GetEventTarget(), nullptr); +}; + +static void TestLockedRWFromAnyThread( + profiler::ThreadRegistration::LockedRWFromAnyThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstLockedRWFromAnyThread(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + TestUnlockedReaderAndAtomicRWOnThread(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + // We can't create a ProfiledThreadData nor PSAutoLock here, so just verify + // that the call would compile and return the expected type. + static_assert(std::is_same_v<decltype(aData.SetProfilingFeaturesAndData( + std::declval<ThreadProfilingFeatures>(), + std::declval<ProfiledThreadData*>(), + std::declval<PSAutoLock>())), + void>); + + aData.ResetMainThread(nullptr); + + TimeDuration delay = TimeDuration::FromSeconds(1); + TimeDuration running = TimeDuration::FromSeconds(1); + aData.GetRunningEventDelay(TimeStamp::Now(), delay, running); + EXPECT_TRUE(delay.IsZero()); + EXPECT_TRUE(running.IsZero()); + + aData.StartJSSampling(123u); + aData.StopJSSampling(); +}; + +static void TestConstLockedRWOnThread( + const profiler::ThreadRegistration::LockedRWOnThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstLockedRWFromAnyThread(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + + // No functions to test here. +}; + +static void TestLockedRWOnThread( + profiler::ThreadRegistration::LockedRWOnThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstLockedRWOnThread(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + TestLockedRWFromAnyThread(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + + // We don't want to really call SetJSContext here, so just verify that + // the call would compile and return the expected type. + static_assert( + std::is_same_v<decltype(aData.SetJSContext(std::declval<JSContext*>())), + void>); + aData.ClearJSContext(); + aData.PollJSSampling(); +}; + +TEST(GeckoProfiler, ThreadRegistration_DataAccess) +{ + using TR = profiler::ThreadRegistration; + + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + // Note that the main thread could already be registered, so we work in a new + // thread to test an actual registration that we control. + + std::thread testThread([&]() { + ASSERT_FALSE(TR::IsRegistered()) + << "A new std::thread should not start registered"; + EXPECT_FALSE(TR::GetOnThreadPtr()); + EXPECT_FALSE(TR::WithOnThreadRefOr([&](auto) { return true; }, false)); + + char onStackChar; + + TimeStamp beforeRegistration = TimeStamp::Now(); + TR tr{"Test thread", &onStackChar}; + TimeStamp afterRegistration = TimeStamp::Now(); + + ASSERT_TRUE(TR::IsRegistered()); + + // Note: This test will mostly be about checking the correct access to + // thread data, depending on how it's obtained. Not all the functionality + // related to that data is tested (e.g., because it involves JS or other + // external dependencies that would be difficult to control here.) + + auto TestOnThreadRef = [&](TR::OnThreadRef aOnThreadRef) { + // To test const-qualified member functions. + const TR::OnThreadRef& onThreadCRef = aOnThreadRef; + + // const UnlockedConstReader (always const) + + TestConstUnlockedConstReader(onThreadCRef.UnlockedConstReaderCRef(), + beforeRegistration, afterRegistration, + &onStackChar); + onThreadCRef.WithUnlockedConstReader( + [&](const TR::UnlockedConstReader& aData) { + TestConstUnlockedConstReader(aData, beforeRegistration, + afterRegistration, &onStackChar); + }); + + // const UnlockedConstReaderAndAtomicRW + + TestConstUnlockedConstReaderAndAtomicRW( + onThreadCRef.UnlockedConstReaderAndAtomicRWCRef(), beforeRegistration, + afterRegistration, &onStackChar); + onThreadCRef.WithUnlockedConstReaderAndAtomicRW( + [&](const TR::UnlockedConstReaderAndAtomicRW& aData) { + TestConstUnlockedConstReaderAndAtomicRW( + aData, beforeRegistration, afterRegistration, &onStackChar); + }); + + // non-const UnlockedConstReaderAndAtomicRW + + TestUnlockedConstReaderAndAtomicRW( + aOnThreadRef.UnlockedConstReaderAndAtomicRWRef(), beforeRegistration, + afterRegistration, &onStackChar); + aOnThreadRef.WithUnlockedConstReaderAndAtomicRW( + [&](TR::UnlockedConstReaderAndAtomicRW& aData) { + TestUnlockedConstReaderAndAtomicRW(aData, beforeRegistration, + afterRegistration, &onStackChar); + }); + + // const UnlockedRWForLockedProfiler + + TestConstUnlockedRWForLockedProfiler( + onThreadCRef.UnlockedRWForLockedProfilerCRef(), beforeRegistration, + afterRegistration, &onStackChar); + onThreadCRef.WithUnlockedRWForLockedProfiler( + [&](const TR::UnlockedRWForLockedProfiler& aData) { + TestConstUnlockedRWForLockedProfiler( + aData, beforeRegistration, afterRegistration, &onStackChar); + }); + + // non-const UnlockedRWForLockedProfiler + + TestUnlockedRWForLockedProfiler( + aOnThreadRef.UnlockedRWForLockedProfilerRef(), beforeRegistration, + afterRegistration, &onStackChar); + aOnThreadRef.WithUnlockedRWForLockedProfiler( + [&](TR::UnlockedRWForLockedProfiler& aData) { + TestUnlockedRWForLockedProfiler(aData, beforeRegistration, + afterRegistration, &onStackChar); + }); + + // const UnlockedReaderAndAtomicRWOnThread + + TestConstUnlockedReaderAndAtomicRWOnThread( + onThreadCRef.UnlockedReaderAndAtomicRWOnThreadCRef(), + beforeRegistration, afterRegistration, &onStackChar); + onThreadCRef.WithUnlockedReaderAndAtomicRWOnThread( + [&](const TR::UnlockedReaderAndAtomicRWOnThread& aData) { + TestConstUnlockedReaderAndAtomicRWOnThread( + aData, beforeRegistration, afterRegistration, &onStackChar); + }); + + // non-const UnlockedReaderAndAtomicRWOnThread + + TestUnlockedReaderAndAtomicRWOnThread( + aOnThreadRef.UnlockedReaderAndAtomicRWOnThreadRef(), + beforeRegistration, afterRegistration, &onStackChar); + aOnThreadRef.WithUnlockedReaderAndAtomicRWOnThread( + [&](TR::UnlockedReaderAndAtomicRWOnThread& aData) { + TestUnlockedReaderAndAtomicRWOnThread( + aData, beforeRegistration, afterRegistration, &onStackChar); + }); + + // LockedRWFromAnyThread + // Note: It cannot directly be accessed on the thread, this will be + // tested through LockedRWOnThread. + + // const LockedRWOnThread + + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + { + TR::OnThreadRef::ConstRWOnThreadWithLock constRWOnThreadWithLock = + onThreadCRef.ConstLockedRWOnThread(); + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + TestConstLockedRWOnThread(constRWOnThreadWithLock.DataCRef(), + beforeRegistration, afterRegistration, + &onStackChar); + } + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + onThreadCRef.WithConstLockedRWOnThread( + [&](const TR::LockedRWOnThread& aData) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + TestConstLockedRWOnThread(aData, beforeRegistration, + afterRegistration, &onStackChar); + }); + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + + // non-const LockedRWOnThread + + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + { + TR::OnThreadRef::RWOnThreadWithLock rwOnThreadWithLock = + aOnThreadRef.GetLockedRWOnThread(); + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + TestConstLockedRWOnThread(rwOnThreadWithLock.DataCRef(), + beforeRegistration, afterRegistration, + &onStackChar); + TestLockedRWOnThread(rwOnThreadWithLock.DataRef(), beforeRegistration, + afterRegistration, &onStackChar); + } + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + aOnThreadRef.WithLockedRWOnThread([&](TR::LockedRWOnThread& aData) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + TestLockedRWOnThread(aData, beforeRegistration, afterRegistration, + &onStackChar); + }); + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + }; + + TR::OnThreadPtr onThreadPtr = TR::GetOnThreadPtr(); + ASSERT_TRUE(onThreadPtr); + TestOnThreadRef(*onThreadPtr); + + TR::WithOnThreadRef( + [&](TR::OnThreadRef aOnThreadRef) { TestOnThreadRef(aOnThreadRef); }); + + EXPECT_TRUE(TR::WithOnThreadRefOr( + [&](TR::OnThreadRef aOnThreadRef) { + TestOnThreadRef(aOnThreadRef); + return true; + }, + false)); + }); + testThread.join(); +} + +// Thread name if registered, nullptr otherwise. +static const char* GetThreadName() { + return profiler::ThreadRegistration::WithOnThreadRefOr( + [](profiler::ThreadRegistration::OnThreadRef onThreadRef) { + return onThreadRef.WithUnlockedConstReader( + [](const profiler::ThreadRegistration::UnlockedConstReader& aData) { + return aData.Info().Name(); + }); + }, + nullptr); +} + +// Get the thread name, as registered in the PRThread, nullptr on failure. +static const char* GetPRThreadName() { + nsIThread* nsThread = NS_GetCurrentThread(); + if (!nsThread) { + return nullptr; + } + PRThread* prThread = nullptr; + if (NS_FAILED(nsThread->GetPRThread(&prThread))) { + return nullptr; + } + if (!prThread) { + return nullptr; + } + return PR_GetThreadName(prThread); +} + +TEST(GeckoProfiler, ThreadRegistration_MainThreadName) +{ + EXPECT_TRUE(profiler::ThreadRegistration::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "GeckoMain"); + + // Check that the real thread name (outside the profiler) is *not* GeckoMain. + EXPECT_STRNE(GetPRThreadName(), "GeckoMain"); +} + +TEST(GeckoProfiler, ThreadRegistration_NestedRegistrations) +{ + using TR = profiler::ThreadRegistration; + + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + // Note that the main thread could already be registered, so we work in a new + // thread to test actual registrations that we control. + + std::thread testThread([&]() { + ASSERT_FALSE(TR::IsRegistered()) + << "A new std::thread should not start registered"; + + char onStackChar; + + // Blocks {} are mostly for clarity, but some control on-stack registration + // lifetimes. + + // On-stack registration. + { + TR rt{"Test thread #1", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #1"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #1"); + } + ASSERT_FALSE(TR::IsRegistered()); + + // Off-stack registration. + { + TR::RegisterThread("Test thread #2", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #2"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #2"); + + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()); + } + + // Extra un-registration should be ignored. + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()); + + // Nested on-stack. + { + TR rt2{"Test thread #3", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #3"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #3"); + + { + TR rt3{"Test thread #4", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #3") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #3") + << "Nested registration shouldn't change the PRThread name"; + } + ASSERT_TRUE(TR::IsRegistered()) + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #3") + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #3"); + } + ASSERT_FALSE(TR::IsRegistered()); + + // Nested off-stack. + { + TR::RegisterThread("Test thread #5", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #5"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #5"); + + { + TR::RegisterThread("Test thread #6", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #5") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #5") + << "Nested registration shouldn't change the PRThread name"; + + TR::UnregisterThread(); + ASSERT_TRUE(TR::IsRegistered()) + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #5") + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #5"); + } + + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()); + } + + // Nested on- and off-stack. + { + TR rt2{"Test thread #7", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #7"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #7"); + + { + TR::RegisterThread("Test thread #8", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #7") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #7") + << "Nested registration shouldn't change the PRThread name"; + + TR::UnregisterThread(); + ASSERT_TRUE(TR::IsRegistered()) + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #7") + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #7"); + } + } + ASSERT_FALSE(TR::IsRegistered()); + + // Nested off- and on-stack. + { + TR::RegisterThread("Test thread #9", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #9"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #9"); + + { + TR rt3{"Test thread #10", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #9") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #9") + << "Nested registration shouldn't change the PRThread name"; + } + ASSERT_TRUE(TR::IsRegistered()) + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #9") + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #9"); + + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()); + } + + // Excess UnregisterThread with on-stack TR. + { + TR rt2{"Test thread #11", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #11"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #11"); + + TR::UnregisterThread(); + ASSERT_TRUE(TR::IsRegistered()) + << "On-stack thread should still be registered after off-stack " + "un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #11") + << "On-stack thread should still be registered after off-stack " + "un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #11"); + } + ASSERT_FALSE(TR::IsRegistered()); + + // Excess on-thread TR destruction with already-unregistered root off-thread + // registration. + { + TR::RegisterThread("Test thread #12", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #12"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #12"); + + { + TR rt3{"Test thread #13", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #12") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #12") + << "Nested registration shouldn't change the PRThread name"; + + // Note that we unregister the root registration, while nested `rt3` is + // still alive. + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()) + << "UnregisterThread() of the root RegisterThread() should always work"; + + // At this end of this block, `rt3` will be destroyed, but nothing + // should happen. + } + ASSERT_FALSE(TR::IsRegistered()); + } + + ASSERT_FALSE(TR::IsRegistered()); + }); + testThread.join(); +} + +TEST(GeckoProfiler, ThreadRegistry_DataAccess) +{ + using TR = profiler::ThreadRegistration; + using TRy = profiler::ThreadRegistry; + + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + // Note that the main thread could already be registered, so we work in a new + // thread to test an actual registration that we control. + + std::thread testThread([&]() { + ASSERT_FALSE(TR::IsRegistered()) + << "A new std::thread should not start registered"; + EXPECT_FALSE(TR::GetOnThreadPtr()); + EXPECT_FALSE(TR::WithOnThreadRefOr([&](auto) { return true; }, false)); + + char onStackChar; + + TimeStamp beforeRegistration = TimeStamp::Now(); + TR tr{"Test thread", &onStackChar}; + TimeStamp afterRegistration = TimeStamp::Now(); + + ASSERT_TRUE(TR::IsRegistered()); + + // Note: This test will mostly be about checking the correct access to + // thread data, depending on how it's obtained. Not all the functionality + // related to that data is tested (e.g., because it involves JS or other + // external dependencies that would be difficult to control here.) + + const ProfilerThreadId testThreadId = profiler_current_thread_id(); + + auto testThroughRegistry = [&]() { + auto TestOffThreadRef = [&](TRy::OffThreadRef aOffThreadRef) { + // To test const-qualified member functions. + const TRy::OffThreadRef& offThreadCRef = aOffThreadRef; + + // const UnlockedConstReader (always const) + + TestConstUnlockedConstReader(offThreadCRef.UnlockedConstReaderCRef(), + beforeRegistration, afterRegistration, + &onStackChar, testThreadId); + offThreadCRef.WithUnlockedConstReader( + [&](const TR::UnlockedConstReader& aData) { + TestConstUnlockedConstReader(aData, beforeRegistration, + afterRegistration, &onStackChar, + testThreadId); + }); + + // const UnlockedConstReaderAndAtomicRW + + TestConstUnlockedConstReaderAndAtomicRW( + offThreadCRef.UnlockedConstReaderAndAtomicRWCRef(), + beforeRegistration, afterRegistration, &onStackChar, testThreadId); + offThreadCRef.WithUnlockedConstReaderAndAtomicRW( + [&](const TR::UnlockedConstReaderAndAtomicRW& aData) { + TestConstUnlockedConstReaderAndAtomicRW( + aData, beforeRegistration, afterRegistration, &onStackChar, + testThreadId); + }); + + // non-const UnlockedConstReaderAndAtomicRW + + TestUnlockedConstReaderAndAtomicRW( + aOffThreadRef.UnlockedConstReaderAndAtomicRWRef(), + beforeRegistration, afterRegistration, &onStackChar, testThreadId); + aOffThreadRef.WithUnlockedConstReaderAndAtomicRW( + [&](TR::UnlockedConstReaderAndAtomicRW& aData) { + TestUnlockedConstReaderAndAtomicRW(aData, beforeRegistration, + afterRegistration, + &onStackChar, testThreadId); + }); + + // const UnlockedRWForLockedProfiler + + TestConstUnlockedRWForLockedProfiler( + offThreadCRef.UnlockedRWForLockedProfilerCRef(), beforeRegistration, + afterRegistration, &onStackChar, testThreadId); + offThreadCRef.WithUnlockedRWForLockedProfiler( + [&](const TR::UnlockedRWForLockedProfiler& aData) { + TestConstUnlockedRWForLockedProfiler(aData, beforeRegistration, + afterRegistration, + &onStackChar, testThreadId); + }); + + // non-const UnlockedRWForLockedProfiler + + TestUnlockedRWForLockedProfiler( + aOffThreadRef.UnlockedRWForLockedProfilerRef(), beforeRegistration, + afterRegistration, &onStackChar, testThreadId); + aOffThreadRef.WithUnlockedRWForLockedProfiler( + [&](TR::UnlockedRWForLockedProfiler& aData) { + TestUnlockedRWForLockedProfiler(aData, beforeRegistration, + afterRegistration, &onStackChar, + testThreadId); + }); + + // UnlockedReaderAndAtomicRWOnThread + // Note: It cannot directly be accessed off the thread, this will be + // tested through LockedRWFromAnyThread. + + // const LockedRWFromAnyThread + + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + { + TRy::OffThreadRef::ConstRWFromAnyThreadWithLock + constRWFromAnyThreadWithLock = + offThreadCRef.ConstLockedRWFromAnyThread(); + if (profiler_current_thread_id() == testThreadId) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + } + TestConstLockedRWFromAnyThread( + constRWFromAnyThreadWithLock.DataCRef(), beforeRegistration, + afterRegistration, &onStackChar, testThreadId); + } + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + offThreadCRef.WithConstLockedRWFromAnyThread( + [&](const TR::LockedRWFromAnyThread& aData) { + if (profiler_current_thread_id() == testThreadId) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + } + TestConstLockedRWFromAnyThread(aData, beforeRegistration, + afterRegistration, &onStackChar, + testThreadId); + }); + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + + // non-const LockedRWFromAnyThread + + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + { + TRy::OffThreadRef::RWFromAnyThreadWithLock rwFromAnyThreadWithLock = + aOffThreadRef.GetLockedRWFromAnyThread(); + if (profiler_current_thread_id() == testThreadId) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + } + TestLockedRWFromAnyThread(rwFromAnyThreadWithLock.DataRef(), + beforeRegistration, afterRegistration, + &onStackChar, testThreadId); + } + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + aOffThreadRef.WithLockedRWFromAnyThread( + [&](TR::LockedRWFromAnyThread& aData) { + if (profiler_current_thread_id() == testThreadId) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + } + TestLockedRWFromAnyThread(aData, beforeRegistration, + afterRegistration, &onStackChar, + testThreadId); + }); + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + + // LockedRWOnThread + // Note: It can never be accessed off the thread. + }; + + int ranTest = 0; + TRy::WithOffThreadRef(testThreadId, [&](TRy::OffThreadRef aOffThreadRef) { + TestOffThreadRef(aOffThreadRef); + ++ranTest; + }); + EXPECT_EQ(ranTest, 1); + + EXPECT_TRUE(TRy::WithOffThreadRefOr( + testThreadId, + [&](TRy::OffThreadRef aOffThreadRef) { + TestOffThreadRef(aOffThreadRef); + return true; + }, + false)); + + ranTest = 0; + EXPECT_FALSE(TRy::IsRegistryMutexLockedOnCurrentThread()); + for (TRy::OffThreadRef offThreadRef : TRy::LockedRegistry{}) { + EXPECT_TRUE(TRy::IsRegistryMutexLockedOnCurrentThread() || + !TR::IsRegistered()); + if (offThreadRef.UnlockedConstReaderCRef().Info().ThreadId() == + testThreadId) { + TestOffThreadRef(offThreadRef); + ++ranTest; + } + } + EXPECT_EQ(ranTest, 1); + EXPECT_FALSE(TRy::IsRegistryMutexLockedOnCurrentThread()); + + { + ranTest = 0; + EXPECT_FALSE(TRy::IsRegistryMutexLockedOnCurrentThread()); + TRy::LockedRegistry lockedRegistry{}; + EXPECT_TRUE(TRy::IsRegistryMutexLockedOnCurrentThread() || + !TR::IsRegistered()); + for (TRy::OffThreadRef offThreadRef : lockedRegistry) { + if (offThreadRef.UnlockedConstReaderCRef().Info().ThreadId() == + testThreadId) { + TestOffThreadRef(offThreadRef); + ++ranTest; + } + } + EXPECT_EQ(ranTest, 1); + } + EXPECT_FALSE(TRy::IsRegistryMutexLockedOnCurrentThread()); + }; + + // Test on the current thread. + testThroughRegistry(); + + // Test from another thread. + std::thread otherThread([&]() { + ASSERT_NE(profiler_current_thread_id(), testThreadId); + testThroughRegistry(); + + // Test that this unregistered thread is really not registered. + int ranTest = 0; + TRy::WithOffThreadRef( + profiler_current_thread_id(), + [&](TRy::OffThreadRef aOffThreadRef) { ++ranTest; }); + EXPECT_EQ(ranTest, 0); + + EXPECT_FALSE(TRy::WithOffThreadRefOr( + profiler_current_thread_id(), + [&](TRy::OffThreadRef aOffThreadRef) { + ++ranTest; + return true; + }, + false)); + EXPECT_EQ(ranTest, 0); + }); + otherThread.join(); + }); + testThread.join(); +} + +TEST(GeckoProfiler, ThreadRegistration_RegistrationEdgeCases) +{ + using TR = profiler::ThreadRegistration; + using TRy = profiler::ThreadRegistry; + + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + // Note that the main thread could already be registered, so we work in a new + // thread to test an actual registration that we control. + + int registrationCount = 0; + int otherThreadLoops = 0; + int otherThreadReads = 0; + + // This thread will register and unregister in a loop, with some pauses. + // Another thread will attempty to access the test thread, and lock its data. + // The main goal is to check edges cases around (un)registrations. + std::thread testThread([&]() { + const ProfilerThreadId testThreadId = profiler_current_thread_id(); + + const TimeStamp endTestAt = TimeStamp::Now() + TimeDuration::FromSeconds(1); + + std::thread otherThread([&]() { + // Initial sleep so that testThread can start its loop. + PR_Sleep(PR_MillisecondsToInterval(1)); + + while (TimeStamp::Now() < endTestAt) { + ++otherThreadLoops; + + TRy::WithOffThreadRef(testThreadId, [&](TRy::OffThreadRef + aOffThreadRef) { + if (otherThreadLoops % 1000 == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + TRy::OffThreadRef::RWFromAnyThreadWithLock rwFromAnyThreadWithLock = + aOffThreadRef.GetLockedRWFromAnyThread(); + ++otherThreadReads; + if (otherThreadReads % 1000 == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + }); + } + }); + + while (TimeStamp::Now() < endTestAt) { + ASSERT_FALSE(TR::IsRegistered()) + << "A new std::thread should not start registered"; + EXPECT_FALSE(TR::GetOnThreadPtr()); + EXPECT_FALSE(TR::WithOnThreadRefOr([&](auto) { return true; }, false)); + + char onStackChar; + + TR tr{"Test thread", &onStackChar}; + ++registrationCount; + + ASSERT_TRUE(TR::IsRegistered()); + + int ranTest = 0; + TRy::WithOffThreadRef(testThreadId, [&](TRy::OffThreadRef aOffThreadRef) { + if (registrationCount % 2000 == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + ++ranTest; + }); + EXPECT_EQ(ranTest, 1); + + if (registrationCount % 1000 == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + } + + otherThread.join(); + }); + + testThread.join(); + + // It's difficult to guess what these numbers should be, but they definitely + // should be non-zero. The main goal was to test that nothing goes wrong. + EXPECT_GT(registrationCount, 0); + EXPECT_GT(otherThreadLoops, 0); + EXPECT_GT(otherThreadReads, 0); +} + +#ifdef MOZ_GECKO_PROFILER + +// Common JSON checks. + +// Check that the given JSON string include no JSON whitespace characters +// (excluding those in property names and strings). +void JSONWhitespaceCheck(const char* aOutput) { + ASSERT_NE(aOutput, nullptr); + + enum class State { Data, String, StringEscaped }; + State state = State::Data; + size_t length = 0; + size_t whitespaces = 0; + for (const char* p = aOutput; *p != '\0'; ++p) { + ++length; + const char c = *p; + + switch (state) { + case State::Data: + if (c == '\n' || c == '\r' || c == ' ' || c == '\t') { + ++whitespaces; + } else if (c == '"') { + state = State::String; + } + break; + + case State::String: + if (c == '"') { + state = State::Data; + } else if (c == '\\') { + state = State::StringEscaped; + } + break; + + case State::StringEscaped: + state = State::String; + break; + } + } + + EXPECT_EQ(whitespaces, 0u); + EXPECT_GT(length, 0u); +} + +// Does the GETTER return a non-null TYPE? (Non-critical) +# define EXPECT_HAS_JSON(GETTER, TYPE) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).is##TYPE()) { \ + EXPECT_TRUE((GETTER).is##TYPE()) \ + << #GETTER " didn't return type " #TYPE; \ + } \ + } while (false) + +// Does the GETTER return a non-null TYPE? (Critical) +# define ASSERT_HAS_JSON(GETTER, TYPE) \ + do { \ + ASSERT_FALSE((GETTER).isNull()); \ + ASSERT_TRUE((GETTER).is##TYPE()); \ + } while (false) + +// Does the GETTER return a non-null TYPE? (Critical) +// If yes, store the reference to Json::Value into VARIABLE. +# define GET_JSON(VARIABLE, GETTER, TYPE) \ + ASSERT_HAS_JSON(GETTER, TYPE); \ + const Json::Value& VARIABLE = (GETTER) + +// Does the GETTER return a non-null TYPE? (Critical) +// If yes, store the value as `const TYPE` into VARIABLE. +# define GET_JSON_VALUE(VARIABLE, GETTER, TYPE) \ + ASSERT_HAS_JSON(GETTER, TYPE); \ + const auto VARIABLE = (GETTER).as##TYPE() + +// Non-const GET_JSON_VALUE. +# define GET_JSON_MUTABLE_VALUE(VARIABLE, GETTER, TYPE) \ + ASSERT_HAS_JSON(GETTER, TYPE); \ + auto VARIABLE = (GETTER).as##TYPE() + +// Checks that the GETTER's value is present, is of the expected TYPE, and has +// the expected VALUE. (Non-critical) +# define EXPECT_EQ_JSON(GETTER, TYPE, VALUE) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).is##TYPE()) { \ + EXPECT_TRUE((GETTER).is##TYPE()) \ + << #GETTER " didn't return type " #TYPE; \ + } else { \ + EXPECT_EQ((GETTER).as##TYPE(), (VALUE)); \ + } \ + } while (false) + +// Checks that the GETTER's value is present, and is a valid index into the +// STRINGTABLE array, pointing at the expected STRING. +# define EXPECT_EQ_STRINGTABLE(GETTER, STRINGTABLE, STRING) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).isUInt()) { \ + EXPECT_TRUE((GETTER).isUInt()) << #GETTER " didn't return an index"; \ + } else { \ + EXPECT_LT((GETTER).asUInt(), (STRINGTABLE).size()); \ + EXPECT_EQ_JSON((STRINGTABLE)[(GETTER).asUInt()], String, (STRING)); \ + } \ + } while (false) + +# define EXPECT_JSON_ARRAY_CONTAINS(GETTER, TYPE, VALUE) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).isArray()) { \ + EXPECT_TRUE((GETTER).is##TYPE()) << #GETTER " is not an array"; \ + } else if (const Json::ArrayIndex size = (GETTER).size(); size == 0u) { \ + EXPECT_NE(size, 0u) << #GETTER " is an empty array"; \ + } else { \ + bool found = false; \ + for (Json::ArrayIndex i = 0; i < size; ++i) { \ + if (!(GETTER)[i].is##TYPE()) { \ + EXPECT_TRUE((GETTER)[i].is##TYPE()) \ + << #GETTER "[" << i << "] is not " #TYPE; \ + break; \ + } \ + if ((GETTER)[i].as##TYPE() == (VALUE)) { \ + found = true; \ + break; \ + } \ + } \ + EXPECT_TRUE(found) << #GETTER " doesn't contain " #VALUE; \ + } \ + } while (false) + +# define EXPECT_JSON_ARRAY_EXCLUDES(GETTER, TYPE, VALUE) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).isArray()) { \ + EXPECT_TRUE((GETTER).is##TYPE()) << #GETTER " is not an array"; \ + } else { \ + const Json::ArrayIndex size = (GETTER).size(); \ + for (Json::ArrayIndex i = 0; i < size; ++i) { \ + if (!(GETTER)[i].is##TYPE()) { \ + EXPECT_TRUE((GETTER)[i].is##TYPE()) \ + << #GETTER "[" << i << "] is not " #TYPE; \ + break; \ + } \ + if ((GETTER)[i].as##TYPE() == (VALUE)) { \ + EXPECT_TRUE((GETTER)[i].as##TYPE() != (VALUE)) \ + << #GETTER " contains " #VALUE; \ + break; \ + } \ + } \ + } \ + } while (false) + +// Check that the given process root contains all the expected properties. +static void JSONRootCheck(const Json::Value& aRoot, + bool aWithMainThread = true) { + ASSERT_TRUE(aRoot.isObject()); + + EXPECT_HAS_JSON(aRoot["libs"], Array); + + GET_JSON(meta, aRoot["meta"], Object); + EXPECT_HAS_JSON(meta["version"], UInt); + EXPECT_HAS_JSON(meta["startTime"], Double); + EXPECT_HAS_JSON(meta["profilingStartTime"], Double); + EXPECT_HAS_JSON(meta["contentEarliestTime"], Double); + EXPECT_HAS_JSON(meta["profilingEndTime"], Double); + + EXPECT_HAS_JSON(aRoot["pages"], Array); + + EXPECT_HAS_JSON(aRoot["profilerOverhead"], Object); + + // "counters" is only present if there is any data to report. + // Test that expect "counters" should test for its presence first. + if (aRoot.isMember("counters")) { + // We have "counters", test their overall validity. + GET_JSON(counters, aRoot["counters"], Array); + for (const Json::Value& counter : counters) { + ASSERT_TRUE(counter.isObject()); + EXPECT_HAS_JSON(counter["name"], String); + EXPECT_HAS_JSON(counter["category"], String); + EXPECT_HAS_JSON(counter["description"], String); + GET_JSON(samples, counter["samples"], Object); + GET_JSON(samplesSchema, samples["schema"], Object); + EXPECT_GE(samplesSchema.size(), 3u); + GET_JSON_VALUE(samplesTime, samplesSchema["time"], UInt); + GET_JSON_VALUE(samplesNumber, samplesSchema["number"], UInt); + GET_JSON_VALUE(samplesCount, samplesSchema["count"], UInt); + GET_JSON(samplesData, samples["data"], Array); + double previousTime = 0.0; + for (const Json::Value& sample : samplesData) { + ASSERT_TRUE(sample.isArray()); + GET_JSON_VALUE(time, sample[samplesTime], Double); + EXPECT_GE(time, previousTime); + previousTime = time; + if (sample.isValidIndex(samplesNumber)) { + EXPECT_HAS_JSON(sample[samplesNumber], UInt64); + } + if (sample.isValidIndex(samplesCount)) { + EXPECT_HAS_JSON(sample[samplesCount], Int64); + } + } + } + } + + GET_JSON(threads, aRoot["threads"], Array); + const Json::ArrayIndex threadCount = threads.size(); + for (Json::ArrayIndex i = 0; i < threadCount; ++i) { + GET_JSON(thread, threads[i], Object); + EXPECT_HAS_JSON(thread["processType"], String); + EXPECT_HAS_JSON(thread["name"], String); + EXPECT_HAS_JSON(thread["registerTime"], Double); + GET_JSON(samples, thread["samples"], Object); + EXPECT_HAS_JSON(thread["markers"], Object); + EXPECT_HAS_JSON(thread["pid"], Int64); + EXPECT_HAS_JSON(thread["tid"], Int64); + GET_JSON(stackTable, thread["stackTable"], Object); + GET_JSON(frameTable, thread["frameTable"], Object); + GET_JSON(stringTable, thread["stringTable"], Array); + + GET_JSON(stackTableSchema, stackTable["schema"], Object); + EXPECT_GE(stackTableSchema.size(), 2u); + GET_JSON_VALUE(stackTablePrefix, stackTableSchema["prefix"], UInt); + GET_JSON_VALUE(stackTableFrame, stackTableSchema["frame"], UInt); + GET_JSON(stackTableData, stackTable["data"], Array); + + GET_JSON(frameTableSchema, frameTable["schema"], Object); + EXPECT_GE(frameTableSchema.size(), 1u); + GET_JSON_VALUE(frameTableLocation, frameTableSchema["location"], UInt); + GET_JSON(frameTableData, frameTable["data"], Array); + + GET_JSON(samplesSchema, samples["schema"], Object); + GET_JSON_VALUE(sampleStackIndex, samplesSchema["stack"], UInt); + GET_JSON(samplesData, samples["data"], Array); + for (const Json::Value& sample : samplesData) { + ASSERT_TRUE(sample.isArray()); + if (sample.isValidIndex(sampleStackIndex)) { + if (!sample[sampleStackIndex].isNull()) { + GET_JSON_MUTABLE_VALUE(stack, sample[sampleStackIndex], UInt); + EXPECT_TRUE(stackTableData.isValidIndex(stack)); + for (;;) { + // `stack` (from the sample, or from the callee frame's "prefix" in + // the previous loop) points into the stackTable. + GET_JSON(stackTableEntry, stackTableData[stack], Array); + GET_JSON_VALUE(frame, stackTableEntry[stackTableFrame], UInt); + + // The stackTable entry's "frame" points into the frameTable. + EXPECT_TRUE(frameTableData.isValidIndex(frame)); + GET_JSON(frameTableEntry, frameTableData[frame], Array); + GET_JSON_VALUE(location, frameTableEntry[frameTableLocation], UInt); + + // The frameTable entry's "location" points at a string. + EXPECT_TRUE(stringTable.isValidIndex(location)); + + // The stackTable entry's "prefix" is null for the root frame. + if (stackTableEntry[stackTablePrefix].isNull()) { + break; + } + // Otherwise it recursively points at the caller in the stackTable. + GET_JSON_VALUE(prefix, stackTableEntry[stackTablePrefix], UInt); + EXPECT_TRUE(stackTableData.isValidIndex(prefix)); + stack = prefix; + } + } + } + } + } + + if (aWithMainThread) { + ASSERT_GT(threadCount, 0u); + GET_JSON(thread0, threads[0], Object); + EXPECT_EQ_JSON(thread0["name"], String, "GeckoMain"); + } + + EXPECT_HAS_JSON(aRoot["pausedRanges"], Array); + + const Json::Value& processes = aRoot["processes"]; + if (!processes.isNull()) { + ASSERT_TRUE(processes.isArray()); + const Json::ArrayIndex processCount = processes.size(); + for (Json::ArrayIndex i = 0; i < processCount; ++i) { + GET_JSON(process, processes[i], Object); + JSONRootCheck(process, aWithMainThread); + } + } + + GET_JSON(profilingLog, aRoot["profilingLog"], Object); + EXPECT_EQ(profilingLog.size(), 1u); + for (auto it = profilingLog.begin(); it != profilingLog.end(); ++it) { + // The key should be a pid. + const auto key = it.name(); + for (const auto letter : key) { + EXPECT_GE(letter, '0'); + EXPECT_LE(letter, '9'); + } + // And the value should be an object. + GET_JSON(logForPid, profilingLog[key], Object); + // Its content is not defined, but we expect at least these: + EXPECT_HAS_JSON(logForPid["profilingLogBegin_TSms"], Double); + EXPECT_HAS_JSON(logForPid["profilingLogEnd_TSms"], Double); + } +} + +// Check that various expected top properties are in the JSON, and then call the +// provided `aJSONCheckFunction` with the JSON root object. +template <typename JSONCheckFunction> +void JSONOutputCheck(const char* aOutput, + JSONCheckFunction&& aJSONCheckFunction) { + ASSERT_NE(aOutput, nullptr); + + JSONWhitespaceCheck(aOutput); + + // Extract JSON. + Json::Value parsedRoot; + Json::CharReaderBuilder builder; + const std::unique_ptr<Json::CharReader> reader(builder.newCharReader()); + ASSERT_TRUE( + reader->parse(aOutput, strchr(aOutput, '\0'), &parsedRoot, nullptr)); + + JSONRootCheck(parsedRoot); + + std::forward<JSONCheckFunction>(aJSONCheckFunction)(parsedRoot); +} + +// Returns `static_cast<SamplingState>(-1)` if callback could not be installed. +static SamplingState WaitForSamplingState() { + Atomic<int> samplingState{-1}; + + if (!profiler_callback_after_sampling([&](SamplingState aSamplingState) { + samplingState = static_cast<int>(aSamplingState); + })) { + return static_cast<SamplingState>(-1); + } + + while (samplingState == -1) { + } + + return static_cast<SamplingState>(static_cast<int>(samplingState)); +} + +typedef Vector<const char*> StrVec; + +static void InactiveFeaturesAndParamsCheck() { + int entries; + Maybe<double> duration; + double interval; + uint32_t features; + StrVec filters; + uint64_t activeTabID; + + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::NativeAllocations)); + + profiler_get_start_params(&entries, &duration, &interval, &features, &filters, + &activeTabID); + + ASSERT_TRUE(entries == 0); + ASSERT_TRUE(duration == Nothing()); + ASSERT_TRUE(interval == 0); + ASSERT_TRUE(features == 0); + ASSERT_TRUE(filters.empty()); + ASSERT_TRUE(activeTabID == 0); +} + +static void ActiveParamsCheck(int aEntries, double aInterval, + uint32_t aFeatures, const char** aFilters, + size_t aFiltersLen, uint64_t aActiveTabID, + const Maybe<double>& aDuration = Nothing()) { + int entries; + Maybe<double> duration; + double interval; + uint32_t features; + StrVec filters; + uint64_t activeTabID; + + profiler_get_start_params(&entries, &duration, &interval, &features, &filters, + &activeTabID); + + ASSERT_TRUE(entries == aEntries); + ASSERT_TRUE(duration == aDuration); + ASSERT_TRUE(interval == aInterval); + ASSERT_TRUE(features == aFeatures); + ASSERT_TRUE(filters.length() == aFiltersLen); + ASSERT_TRUE(activeTabID == aActiveTabID); + for (size_t i = 0; i < aFiltersLen; i++) { + ASSERT_TRUE(strcmp(filters[i], aFilters[i]) == 0); + } +} + +TEST(GeckoProfiler, FeaturesAndParams) +{ + InactiveFeaturesAndParamsCheck(); + + // Try a couple of features and filters. + { + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + +# define PROFILER_DEFAULT_DURATION 20 /* seconds, for tests only */ + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 100, + Some(PROFILER_DEFAULT_DURATION)); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 100, + Some(PROFILER_DEFAULT_DURATION)); + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } + + // Try some different features and filters. + { + uint32_t features = + ProfilerFeature::MainThreadIO | ProfilerFeature::IPCMessages; + const char* filters[] = {"GeckoMain", "Foo", "Bar"}; + + // Testing with some arbitrary buffer size (as could be provided by + // external code), which we convert to the appropriate power of 2. + profiler_start(PowerOfTwo32(999999), 3, features, filters, + MOZ_ARRAY_LENGTH(filters), 123, Some(25.0)); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(int(PowerOfTwo32(999999).Value()), 3, features, filters, + MOZ_ARRAY_LENGTH(filters), 123, Some(25.0)); + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } + + // Try with no duration + { + uint32_t features = + ProfilerFeature::MainThreadIO | ProfilerFeature::IPCMessages; + const char* filters[] = {"GeckoMain", "Foo", "Bar"}; + + profiler_start(PowerOfTwo32(999999), 3, features, filters, + MOZ_ARRAY_LENGTH(filters), 0, Nothing()); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(int(PowerOfTwo32(999999).Value()), 3, features, filters, + MOZ_ARRAY_LENGTH(filters), 0, Nothing()); + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } + + // Try all supported features, and filters that match all threads. + { + uint32_t availableFeatures = profiler_get_available_features(); + const char* filters[] = {""}; + + profiler_start(PowerOfTwo32(88888), 10, availableFeatures, filters, + MOZ_ARRAY_LENGTH(filters), 0, Some(15.0)); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(PowerOfTwo32(88888).Value(), 10, availableFeatures, + filters, MOZ_ARRAY_LENGTH(filters), 0, Some(15.0)); + + // Don't call profiler_stop() here. + } + + // Try no features, and filters that match no threads. + { + uint32_t features = 0; + const char* filters[] = {"NoThreadWillMatchThis"}; + + // Second profiler_start() call in a row without an intervening + // profiler_stop(); this will do an implicit profiler_stop() and restart. + profiler_start(PowerOfTwo32(0), 0, features, filters, + MOZ_ARRAY_LENGTH(filters), 0, Some(0.0)); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::IPCMessages)); + + // Entries and intervals go to defaults if 0 is specified. + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 0, Nothing()); + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + + // These calls are no-ops. + profiler_stop(); + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } +} + +TEST(GeckoProfiler, EnsureStarted) +{ + InactiveFeaturesAndParamsCheck(); + + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + { + // Inactive -> Active + profiler_ensure_started(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0, + Some(PROFILER_DEFAULT_DURATION)); + + ActiveParamsCheck( + PROFILER_DEFAULT_ENTRIES.Value(), PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0, Some(PROFILER_DEFAULT_DURATION)); + } + + { + // Active -> Active with same settings + + Maybe<ProfilerBufferInfo> info0 = profiler_get_buffer_info(); + ASSERT_TRUE(info0->mRangeEnd > 0); + + // First, write some samples into the buffer. + PR_Sleep(PR_MillisecondsToInterval(500)); + + Maybe<ProfilerBufferInfo> info1 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd > info0->mRangeEnd); + + // Call profiler_ensure_started with the same settings as before. + // This operation must not clear our buffer! + profiler_ensure_started(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0, + Some(PROFILER_DEFAULT_DURATION)); + + ActiveParamsCheck( + PROFILER_DEFAULT_ENTRIES.Value(), PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0, Some(PROFILER_DEFAULT_DURATION)); + + // Check that our position in the buffer stayed the same or advanced, but + // not by much, and the range-start after profiler_ensure_started shouldn't + // have passed the range-end before. + Maybe<ProfilerBufferInfo> info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info2->mRangeEnd >= info1->mRangeEnd); + ASSERT_TRUE(info2->mRangeEnd - info1->mRangeEnd < + info1->mRangeEnd - info0->mRangeEnd); + ASSERT_TRUE(info2->mRangeStart < info1->mRangeEnd); + } + + { + // Active -> Active with *different* settings + + Maybe<ProfilerBufferInfo> info1 = profiler_get_buffer_info(); + + // Call profiler_ensure_started with a different feature set than the one + // it's currently running with. This is supposed to stop and restart the + // profiler, thereby discarding the buffer contents. + uint32_t differentFeatures = features | ProfilerFeature::CPUUtilization; + profiler_ensure_started(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + differentFeatures, filters, + MOZ_ARRAY_LENGTH(filters), 0); + + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, differentFeatures, filters, + MOZ_ARRAY_LENGTH(filters), 0); + + // Check the the buffer was cleared, so its range-start should be at/after + // its range-end before. + Maybe<ProfilerBufferInfo> info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info2->mRangeStart >= info1->mRangeEnd); + } + + { + // Active -> Inactive + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } +} + +TEST(GeckoProfiler, MultiRegistration) +{ + // This whole test only checks that function calls don't crash, they don't + // actually verify that threads get profiled or not. + + { + std::thread thread([]() { + char top; + profiler_register_thread("thread, no unreg", &top); + }); + thread.join(); + } + + { + std::thread thread([]() { profiler_unregister_thread(); }); + thread.join(); + } + + { + std::thread thread([]() { + char top; + profiler_register_thread("thread 1st", &top); + profiler_unregister_thread(); + profiler_register_thread("thread 2nd", &top); + profiler_unregister_thread(); + }); + thread.join(); + } + + { + std::thread thread([]() { + char top; + profiler_register_thread("thread once", &top); + profiler_register_thread("thread again", &top); + profiler_unregister_thread(); + }); + thread.join(); + } + + { + std::thread thread([]() { + char top; + profiler_register_thread("thread to unreg twice", &top); + profiler_unregister_thread(); + profiler_unregister_thread(); + }); + thread.join(); + } +} + +TEST(GeckoProfiler, DifferentThreads) +{ + InactiveFeaturesAndParamsCheck(); + + nsCOMPtr<nsIThread> thread; + nsresult rv = NS_NewNamedThread("GeckoProfGTest", getter_AddRefs(thread)); + ASSERT_NS_SUCCEEDED(rv); + + // Control the profiler on a background thread and verify flags on the + // main thread. + { + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_DifferentThreads_Test::TestBody"_ns, thread, + NS_NewRunnableFunction( + "GeckoProfiler_DifferentThreads_Test::TestBody", [&]() { + profiler_start(PROFILER_DEFAULT_ENTRIES, + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 0); + })); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 0); + + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_DifferentThreads_Test::TestBody"_ns, thread, + NS_NewRunnableFunction("GeckoProfiler_DifferentThreads_Test::TestBody", + [&]() { profiler_stop(); })); + + InactiveFeaturesAndParamsCheck(); + } + + // Control the profiler on the main thread and verify flags on a + // background thread. + { + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_DifferentThreads_Test::TestBody"_ns, thread, + NS_NewRunnableFunction( + "GeckoProfiler_DifferentThreads_Test::TestBody", [&]() { + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE( + !profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE( + !profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 0); + })); + + profiler_stop(); + + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_DifferentThreads_Test::TestBody"_ns, thread, + NS_NewRunnableFunction("GeckoProfiler_DifferentThreads_Test::TestBody", + [&]() { InactiveFeaturesAndParamsCheck(); })); + } + + thread->Shutdown(); +} + +TEST(GeckoProfiler, GetBacktrace) +{ + ASSERT_TRUE(!profiler_get_backtrace()); + + { + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + // These will be destroyed while the profiler is active. + static const int N = 100; + { + UniqueProfilerBacktrace u[N]; + for (int i = 0; i < N; i++) { + u[i] = profiler_get_backtrace(); + ASSERT_TRUE(u[i]); + } + } + + // These will be destroyed after the profiler stops. + UniqueProfilerBacktrace u[N]; + for (int i = 0; i < N; i++) { + u[i] = profiler_get_backtrace(); + ASSERT_TRUE(u[i]); + } + + profiler_stop(); + } + + ASSERT_TRUE(!profiler_get_backtrace()); +} + +TEST(GeckoProfiler, Pause) +{ + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test must run on the main thread"; + + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain", "Profiled GeckoProfiler.Pause"}; + + ASSERT_TRUE(!profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Ignored GeckoProfiler.Pause - before start"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - before start"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + }}.join(); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(!profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + } + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Ignored GeckoProfiler.Pause - after start"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - after start"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + }}.join(); + + // Check that we are writing samples while not paused. + Maybe<ProfilerBufferInfo> info1 = profiler_get_buffer_info(); + PR_Sleep(PR_MillisecondsToInterval(500)); + Maybe<ProfilerBufferInfo> info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd != info2->mRangeEnd); + + // Check that we are writing markers while not paused. + ASSERT_TRUE(profiler_thread_is_being_profiled_for_markers()); + ASSERT_TRUE( + profiler_thread_is_being_profiled_for_markers(ProfilerThreadId{})); + ASSERT_TRUE(profiler_thread_is_being_profiled_for_markers( + profiler_current_thread_id())); + ASSERT_TRUE( + profiler_thread_is_being_profiled_for_markers(profiler_main_thread_id())); + info1 = profiler_get_buffer_info(); + PROFILER_MARKER_UNTYPED("Not paused", OTHER, {}); + info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd != info2->mRangeEnd); + + profiler_pause(); + + ASSERT_TRUE(profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + } + ASSERT_TRUE(!profiler_thread_is_being_profiled_for_markers()); + ASSERT_TRUE( + !profiler_thread_is_being_profiled_for_markers(ProfilerThreadId{})); + ASSERT_TRUE(!profiler_thread_is_being_profiled_for_markers( + profiler_current_thread_id())); + ASSERT_TRUE(!profiler_thread_is_being_profiled_for_markers( + profiler_main_thread_id())); + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Ignored GeckoProfiler.Pause - after pause"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - after pause"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + }}.join(); + + // Check that we are not writing samples while paused. + info1 = profiler_get_buffer_info(); + PR_Sleep(PR_MillisecondsToInterval(500)); + info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd == info2->mRangeEnd); + + // Check that we are now writing markers while paused. + info1 = profiler_get_buffer_info(); + PROFILER_MARKER_UNTYPED("Paused", OTHER, {}); + info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd == info2->mRangeEnd); + PROFILER_MARKER_UNTYPED("Paused v2", OTHER, {}); + Maybe<ProfilerBufferInfo> info3 = profiler_get_buffer_info(); + ASSERT_TRUE(info2->mRangeEnd == info3->mRangeEnd); + + profiler_resume(); + + ASSERT_TRUE(!profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + } + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Ignored GeckoProfiler.Pause - after resume"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - after resume"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + }}.join(); + + profiler_stop(); + + ASSERT_TRUE(!profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + } + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD("Ignored GeckoProfiler.Pause - after stop"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - after stop"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + }}.join(); +} + +TEST(GeckoProfiler, Markers) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + PROFILER_MARKER("tracing event", OTHER, {}, Tracing, "A"); + PROFILER_MARKER("tracing start", OTHER, MarkerTiming::IntervalStart(), + Tracing, "A"); + PROFILER_MARKER("tracing end", OTHER, MarkerTiming::IntervalEnd(), Tracing, + "A"); + + auto bt = profiler_capture_backtrace(); + PROFILER_MARKER("tracing event with stack", OTHER, + MarkerStack::TakeBacktrace(std::move(bt)), Tracing, "B"); + + { AUTO_PROFILER_TRACING_MARKER("C", "auto tracing", OTHER); } + + PROFILER_MARKER_UNTYPED("M1", OTHER, {}); + PROFILER_MARKER_UNTYPED("M3", OTHER, {}); + + // Create three strings: two that are the maximum allowed length, and one that + // is one char longer. + static const size_t kMax = ProfileBuffer::kMaxFrameKeyLength; + UniquePtr<char[]> okstr1 = MakeUnique<char[]>(kMax); + UniquePtr<char[]> okstr2 = MakeUnique<char[]>(kMax); + UniquePtr<char[]> longstr = MakeUnique<char[]>(kMax + 1); + UniquePtr<char[]> longstrCut = MakeUnique<char[]>(kMax + 1); + for (size_t i = 0; i < kMax; i++) { + okstr1[i] = 'a'; + okstr2[i] = 'b'; + longstr[i] = 'c'; + longstrCut[i] = 'c'; + } + okstr1[kMax - 1] = '\0'; + okstr2[kMax - 1] = '\0'; + longstr[kMax] = '\0'; + longstrCut[kMax] = '\0'; + // Should be output as-is. + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("", LAYOUT, ""); + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("", LAYOUT, okstr1.get()); + // Should be output as label + space + okstr2. + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("okstr2", LAYOUT, okstr2.get()); + // Should be output with kMax length, ending with "...\0". + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("", LAYOUT, longstr.get()); + ASSERT_EQ(longstrCut[kMax - 4], 'c'); + longstrCut[kMax - 4] = '.'; + ASSERT_EQ(longstrCut[kMax - 3], 'c'); + longstrCut[kMax - 3] = '.'; + ASSERT_EQ(longstrCut[kMax - 2], 'c'); + longstrCut[kMax - 2] = '.'; + ASSERT_EQ(longstrCut[kMax - 1], 'c'); + longstrCut[kMax - 1] = '\0'; + + // Test basic markers 2.0. + EXPECT_TRUE(profiler_add_marker_impl( + "default-templated markers 2.0 with empty options", + geckoprofiler::category::OTHER, {})); + + PROFILER_MARKER_UNTYPED( + "default-templated markers 2.0 with option", OTHER, + MarkerStack::TakeBacktrace(profiler_capture_backtrace())); + + PROFILER_MARKER("explicitly-default-templated markers 2.0 with empty options", + OTHER, {}, NoPayload); + + EXPECT_TRUE(profiler_add_marker_impl( + "explicitly-default-templated markers 2.0 with option", + geckoprofiler::category::OTHER, {}, + ::geckoprofiler::markers::NoPayload{})); + + // Used in markers below. + TimeStamp ts1 = TimeStamp::Now(); + + // Sleep briefly to ensure a sample is taken and the pending markers are + // processed. + PR_Sleep(PR_MillisecondsToInterval(500)); + + // Used in markers below. + TimeStamp ts2 = TimeStamp::Now(); + // ts1 and ts2 should be different thanks to the sleep. + EXPECT_NE(ts1, ts2); + + // Test most marker payloads. + + // Keep this one first! (It's used to record `ts1` and `ts2`, to compare + // to serialized numbers in other markers.) + EXPECT_TRUE(profiler_add_marker_impl( + "FirstMarker", geckoprofiler::category::OTHER, + MarkerTiming::Interval(ts1, ts2), geckoprofiler::markers::TextMarker{}, + "First Marker")); + + // User-defined marker type with different properties, and fake schema. + struct GtestMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("markers-gtest"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter, int aInt, + double aDouble, const mozilla::ProfilerString8View& aText, + const mozilla::ProfilerString8View& aUniqueText, + const mozilla::TimeStamp& aTime) { + aWriter.NullProperty("null"); + aWriter.BoolProperty("bool-false", false); + aWriter.BoolProperty("bool-true", true); + aWriter.IntProperty("int", aInt); + aWriter.DoubleProperty("double", aDouble); + aWriter.StringProperty("text", aText); + aWriter.UniqueStringProperty("unique text", aUniqueText); + aWriter.UniqueStringProperty("unique text again", aUniqueText); + aWriter.TimeProperty("time", aTime); + } + static mozilla::MarkerSchema MarkerTypeDisplay() { + // Note: This is an test function that is not intended to actually output + // that correctly matches StreamJSONMarkerData data above! Instead we only + // test that it outputs the expected JSON at the end. + using MS = mozilla::MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable, + MS::Location::TimelineOverview, MS::Location::TimelineMemory, + MS::Location::TimelineIPC, MS::Location::TimelineFileIO, + MS::Location::StackChart}; + // All label functions. + schema.SetChartLabel("chart label"); + schema.SetTooltipLabel("tooltip label"); + schema.SetTableLabel("table label"); + // All data functions, all formats, all "searchable" values. + schema.AddKeyFormat("key with url", MS::Format::Url); + schema.AddKeyLabelFormat("key with label filePath", "label filePath", + MS::Format::FilePath); + schema.AddKeyFormatSearchable("key with string not-searchable", + MS::Format::String, + MS::Searchable::NotSearchable); + schema.AddKeyLabelFormatSearchable("key with label duration searchable", + "label duration", MS::Format::Duration, + MS::Searchable::Searchable); + schema.AddKeyFormat("key with time", MS::Format::Time); + schema.AddKeyFormat("key with seconds", MS::Format::Seconds); + schema.AddKeyFormat("key with milliseconds", MS::Format::Milliseconds); + schema.AddKeyFormat("key with microseconds", MS::Format::Microseconds); + schema.AddKeyFormat("key with nanoseconds", MS::Format::Nanoseconds); + schema.AddKeyFormat("key with bytes", MS::Format::Bytes); + schema.AddKeyFormat("key with percentage", MS::Format::Percentage); + schema.AddKeyFormat("key with integer", MS::Format::Integer); + schema.AddKeyFormat("key with decimal", MS::Format::Decimal); + schema.AddStaticLabelValue("static label", "static value"); + schema.AddKeyFormat("key with unique string", MS::Format::UniqueString); + return schema; + } + }; + EXPECT_TRUE(profiler_add_marker_impl( + "Gtest custom marker", geckoprofiler::category::OTHER, + MarkerTiming::Interval(ts1, ts2), GtestMarker{}, 42, 43.0, "gtest text", + "gtest unique text", ts1)); + + // User-defined marker type with no data, special frontend schema. + struct GtestSpecialMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("markers-gtest-special"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter) {} + static mozilla::MarkerSchema MarkerTypeDisplay() { + return mozilla::MarkerSchema::SpecialFrontendLocation{}; + } + }; + EXPECT_TRUE(profiler_add_marker_impl("Gtest special marker", + geckoprofiler::category::OTHER, {}, + GtestSpecialMarker{})); + + // User-defined marker type that is never used, so it shouldn't appear in the + // output. + struct GtestUnusedMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("markers-gtest-unused"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter) {} + static mozilla::MarkerSchema MarkerTypeDisplay() { + return mozilla::MarkerSchema::SpecialFrontendLocation{}; + } + }; + + // Make sure the compiler doesn't complain about this unused struct. + mozilla::Unused << GtestUnusedMarker{}; + + // Other markers in alphabetical order of payload class names. + + nsCOMPtr<nsIURI> uri; + ASSERT_TRUE( + NS_SUCCEEDED(NS_NewURI(getter_AddRefs(uri), "http://mozilla.org/"_ns))); + // The marker name will be "Load <aChannelId>: <aURI>". + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 1, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_START, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheHit, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false + /* const mozilla::net::TimingStruct* aTimings = nullptr */ + /* mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> aSource = + nullptr */ + /* const mozilla::Maybe<nsDependentCString>& aContentType = + mozilla::Nothing() */ + /* nsIURI* aRedirectURI = nullptr */ + /* uint64_t aRedirectChannelId = 0 */ + ); + + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 2, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_STOP, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe<nsDependentCString>& aContentType = + mozilla::Nothing() */ + Some(nsDependentCString("text/html")), + /* nsIURI* aRedirectURI = nullptr */ nullptr, + /* uint64_t aRedirectChannelId = 0 */ 0); + + nsCOMPtr<nsIURI> redirectURI; + ASSERT_TRUE(NS_SUCCEEDED( + NS_NewURI(getter_AddRefs(redirectURI), "http://example.com/"_ns))); + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 3, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_REDIRECT, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe<nsDependentCString>& aContentType = + mozilla::Nothing() */ + mozilla::Nothing(), + /* nsIURI* aRedirectURI = nullptr */ redirectURI, + /* uint32_t aRedirectFlags = 0 */ + nsIChannelEventSink::REDIRECT_TEMPORARY, + /* uint64_t aRedirectChannelId = 0 */ 103); + + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 4, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_REDIRECT, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe<nsDependentCString>& aContentType = + mozilla::Nothing() */ + mozilla::Nothing(), + /* nsIURI* aRedirectURI = nullptr */ redirectURI, + /* uint32_t aRedirectFlags = 0 */ + nsIChannelEventSink::REDIRECT_PERMANENT, + /* uint64_t aRedirectChannelId = 0 */ 104); + + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 5, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_REDIRECT, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe<nsDependentCString>& aContentType = + mozilla::Nothing() */ + mozilla::Nothing(), + /* nsIURI* aRedirectURI = nullptr */ redirectURI, + /* uint32_t aRedirectFlags = 0 */ nsIChannelEventSink::REDIRECT_INTERNAL, + /* uint64_t aRedirectChannelId = 0 */ 105); + + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 6, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_REDIRECT, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe<nsDependentCString>& aContentType = + mozilla::Nothing() */ + mozilla::Nothing(), + /* nsIURI* aRedirectURI = nullptr */ redirectURI, + /* uint32_t aRedirectFlags = 0 */ nsIChannelEventSink::REDIRECT_INTERNAL | + nsIChannelEventSink::REDIRECT_STS_UPGRADE, + /* uint64_t aRedirectChannelId = 0 */ 106); + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 7, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_START, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ true + /* const mozilla::net::TimingStruct* aTimings = nullptr */ + /* mozilla::UniquePtr<mozilla::ProfileChunkedBuffer> aSource = + nullptr */ + /* const mozilla::Maybe<nsDependentCString>& aContentType = + mozilla::Nothing() */ + /* nsIURI* aRedirectURI = nullptr */ + /* uint64_t aRedirectChannelId = 0 */ + ); + + EXPECT_TRUE(profiler_add_marker_impl( + "Text in main thread with stack", geckoprofiler::category::OTHER, + {MarkerStack::Capture(), MarkerTiming::Interval(ts1, ts2)}, + geckoprofiler::markers::TextMarker{}, "")); + EXPECT_TRUE(profiler_add_marker_impl( + "Text from main thread with stack", geckoprofiler::category::OTHER, + MarkerOptions(MarkerThreadId::MainThread(), MarkerStack::Capture()), + geckoprofiler::markers::TextMarker{}, "")); + + std::thread registeredThread([]() { + AUTO_PROFILER_REGISTER_THREAD("Marker test sub-thread"); + // Marker in non-profiled thread won't be stored. + EXPECT_FALSE(profiler_add_marker_impl( + "Text in registered thread with stack", geckoprofiler::category::OTHER, + MarkerStack::Capture(), geckoprofiler::markers::TextMarker{}, "")); + // Marker will be stored in main thread, with stack from registered thread. + EXPECT_TRUE(profiler_add_marker_impl( + "Text from registered thread with stack", + geckoprofiler::category::OTHER, + MarkerOptions(MarkerThreadId::MainThread(), MarkerStack::Capture()), + geckoprofiler::markers::TextMarker{}, "")); + }); + registeredThread.join(); + + std::thread unregisteredThread([]() { + // Marker in unregistered thread won't be stored. + EXPECT_FALSE(profiler_add_marker_impl( + "Text in unregistered thread with stack", + geckoprofiler::category::OTHER, MarkerStack::Capture(), + geckoprofiler::markers::TextMarker{}, "")); + // Marker will be stored in main thread, but stack cannot be captured in an + // unregistered thread. + EXPECT_TRUE(profiler_add_marker_impl( + "Text from unregistered thread with stack", + geckoprofiler::category::OTHER, + MarkerOptions(MarkerThreadId::MainThread(), MarkerStack::Capture()), + geckoprofiler::markers::TextMarker{}, "")); + }); + unregisteredThread.join(); + + EXPECT_TRUE( + profiler_add_marker_impl("Tracing", geckoprofiler::category::OTHER, {}, + geckoprofiler::markers::Tracing{}, "category")); + + EXPECT_TRUE(profiler_add_marker_impl("Text", geckoprofiler::category::OTHER, + {}, geckoprofiler::markers::TextMarker{}, + "Text text")); + + // Ensure that we evaluate to false for markers with very long texts by + // testing against a ~3mb string. A string of this size should exceed the + // available buffer chunks (max: 2) that are available and be discarded. + EXPECT_FALSE(profiler_add_marker_impl( + "Text", geckoprofiler::category::OTHER, {}, + geckoprofiler::markers::TextMarker{}, std::string(3 * 1024 * 1024, 'x'))); + + EXPECT_TRUE(profiler_add_marker_impl( + "MediaSample", geckoprofiler::category::OTHER, {}, + geckoprofiler::markers::MediaSampleMarker{}, 123, 456, 789)); + + SpliceableChunkedJSONWriter w{FailureLatchInfallibleSource::Singleton()}; + w.Start(); + EXPECT_TRUE(::profiler_stream_json_for_this_process(w).isOk()); + w.End(); + + EXPECT_FALSE(w.Failed()); + + UniquePtr<char[]> profile = w.ChunkedWriteFunc().CopyData(); + ASSERT_TRUE(!!profile.get()); + + // Expected markers, in order. + enum State { + S_tracing_event, + S_tracing_start, + S_tracing_end, + S_tracing_event_with_stack, + S_tracing_auto_tracing_start, + S_tracing_auto_tracing_end, + S_M1, + S_M3, + S_Markers2DefaultEmptyOptions, + S_Markers2DefaultWithOptions, + S_Markers2ExplicitDefaultEmptyOptions, + S_Markers2ExplicitDefaultWithOptions, + S_FirstMarker, + S_CustomMarker, + S_SpecialMarker, + S_NetworkMarkerPayload_start, + S_NetworkMarkerPayload_stop, + S_NetworkMarkerPayload_redirect_temporary, + S_NetworkMarkerPayload_redirect_permanent, + S_NetworkMarkerPayload_redirect_internal, + S_NetworkMarkerPayload_redirect_internal_sts, + S_NetworkMarkerPayload_private_browsing, + + S_TextWithStack, + S_TextToMTWithStack, + S_RegThread_TextToMTWithStack, + S_UnregThread_TextToMTWithStack, + + S_LAST, + } state = State(0); + + // These will be set when first read from S_FirstMarker, then + // compared in following markers. + // TODO: Compute these values from the timestamps. + double ts1Double = 0.0; + double ts2Double = 0.0; + + JSONOutputCheck(profile.get(), [&](const Json::Value& root) { + { + GET_JSON(threads, root["threads"], Array); + ASSERT_EQ(threads.size(), 1u); + + { + GET_JSON(thread0, threads[0], Object); + + // Keep a reference to the string table in this block, it will be used + // below. + GET_JSON(stringTable, thread0["stringTable"], Array); + ASSERT_TRUE(stringTable.isArray()); + + // Test the expected labels in the string table. + bool foundEmpty = false; + bool foundOkstr1 = false; + bool foundOkstr2 = false; + const std::string okstr2Label = std::string("okstr2 ") + okstr2.get(); + bool foundTooLong = false; + for (const auto& s : stringTable) { + ASSERT_TRUE(s.isString()); + std::string sString = s.asString(); + if (sString.empty()) { + EXPECT_FALSE(foundEmpty); + foundEmpty = true; + } else if (sString == okstr1.get()) { + EXPECT_FALSE(foundOkstr1); + foundOkstr1 = true; + } else if (sString == okstr2Label) { + EXPECT_FALSE(foundOkstr2); + foundOkstr2 = true; + } else if (sString == longstrCut.get()) { + EXPECT_FALSE(foundTooLong); + foundTooLong = true; + } else { + EXPECT_NE(sString, longstr.get()); + } + } + EXPECT_TRUE(foundEmpty); + EXPECT_TRUE(foundOkstr1); + EXPECT_TRUE(foundOkstr2); + EXPECT_TRUE(foundTooLong); + + { + GET_JSON(markers, thread0["markers"], Object); + + { + GET_JSON(data, markers["data"], Array); + + for (const Json::Value& marker : data) { + // Name the indexes into the marker tuple: + // [name, startTime, endTime, phase, category, payload] + const unsigned int NAME = 0u; + const unsigned int START_TIME = 1u; + const unsigned int END_TIME = 2u; + const unsigned int PHASE = 3u; + const unsigned int CATEGORY = 4u; + const unsigned int PAYLOAD = 5u; + + const unsigned int PHASE_INSTANT = 0; + const unsigned int PHASE_INTERVAL = 1; + const unsigned int PHASE_START = 2; + const unsigned int PHASE_END = 3; + + const unsigned int SIZE_WITHOUT_PAYLOAD = 5u; + const unsigned int SIZE_WITH_PAYLOAD = 6u; + + ASSERT_TRUE(marker.isArray()); + // The payload is optional. + ASSERT_GE(marker.size(), SIZE_WITHOUT_PAYLOAD); + ASSERT_LE(marker.size(), SIZE_WITH_PAYLOAD); + + // root.threads[0].markers.data[i] is an array with 5 or 6 + // elements. + + ASSERT_TRUE(marker[NAME].isUInt()); // name id + GET_JSON(name, stringTable[marker[NAME].asUInt()], String); + std::string nameString = name.asString(); + + EXPECT_TRUE(marker[START_TIME].isNumeric()); + EXPECT_TRUE(marker[END_TIME].isNumeric()); + EXPECT_TRUE(marker[PHASE].isUInt()); + EXPECT_TRUE(marker[PHASE].asUInt() < 4); + EXPECT_TRUE(marker[CATEGORY].isUInt()); + +# define EXPECT_TIMING_INSTANT \ + EXPECT_NE(marker[START_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_INSTANT); +# define EXPECT_TIMING_INTERVAL \ + EXPECT_NE(marker[START_TIME].asDouble(), 0); \ + EXPECT_NE(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_INTERVAL); +# define EXPECT_TIMING_START \ + EXPECT_NE(marker[START_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_START); +# define EXPECT_TIMING_END \ + EXPECT_EQ(marker[START_TIME].asDouble(), 0); \ + EXPECT_NE(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_END); + +# define EXPECT_TIMING_INSTANT_AT(t) \ + EXPECT_EQ(marker[START_TIME].asDouble(), t); \ + EXPECT_EQ(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_INSTANT); +# define EXPECT_TIMING_INTERVAL_AT(start, end) \ + EXPECT_EQ(marker[START_TIME].asDouble(), start); \ + EXPECT_EQ(marker[END_TIME].asDouble(), end); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_INTERVAL); +# define EXPECT_TIMING_START_AT(start) \ + EXPECT_EQ(marker[START_TIME].asDouble(), start); \ + EXPECT_EQ(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_START); +# define EXPECT_TIMING_END_AT(end) \ + EXPECT_EQ(marker[START_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[END_TIME].asDouble(), end); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_END); + + if (marker.size() == SIZE_WITHOUT_PAYLOAD) { + // root.threads[0].markers.data[i] is an array with 5 elements, + // so there is no payload. + if (nameString == "M1") { + ASSERT_EQ(state, S_M1); + state = State(state + 1); + } else if (nameString == "M3") { + ASSERT_EQ(state, S_M3); + state = State(state + 1); + } else if (nameString == + "default-templated markers 2.0 with empty options") { + EXPECT_EQ(state, S_Markers2DefaultEmptyOptions); + state = State(S_Markers2DefaultEmptyOptions + 1); +// TODO: Re-enable this when bug 1646714 lands, and check for stack. +# if 0 + } else if (nameString == + "default-templated markers 2.0 with option") { + EXPECT_EQ(state, S_Markers2DefaultWithOptions); + state = State(S_Markers2DefaultWithOptions + 1); +# endif + } else if (nameString == + "explicitly-default-templated markers 2.0 with " + "empty " + "options") { + EXPECT_EQ(state, S_Markers2ExplicitDefaultEmptyOptions); + state = State(S_Markers2ExplicitDefaultEmptyOptions + 1); + } else if (nameString == + "explicitly-default-templated markers 2.0 with " + "option") { + EXPECT_EQ(state, S_Markers2ExplicitDefaultWithOptions); + state = State(S_Markers2ExplicitDefaultWithOptions + 1); + } + } else { + // root.threads[0].markers.data[i] is an array with 6 elements, + // so there is a payload. + GET_JSON(payload, marker[PAYLOAD], Object); + + // root.threads[0].markers.data[i][PAYLOAD] is an object + // (payload). + + // It should at least have a "type" string. + GET_JSON(type, payload["type"], String); + std::string typeString = type.asString(); + + if (nameString == "tracing event") { + EXPECT_EQ(state, S_tracing_event); + state = State(S_tracing_event + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_INSTANT; + EXPECT_EQ_JSON(payload["category"], String, "A"); + EXPECT_TRUE(payload["stack"].isNull()); + + } else if (nameString == "tracing start") { + EXPECT_EQ(state, S_tracing_start); + state = State(S_tracing_start + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_START; + EXPECT_EQ_JSON(payload["category"], String, "A"); + EXPECT_TRUE(payload["stack"].isNull()); + + } else if (nameString == "tracing end") { + EXPECT_EQ(state, S_tracing_end); + state = State(S_tracing_end + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_END; + EXPECT_EQ_JSON(payload["category"], String, "A"); + EXPECT_TRUE(payload["stack"].isNull()); + + } else if (nameString == "tracing event with stack") { + EXPECT_EQ(state, S_tracing_event_with_stack); + state = State(S_tracing_event_with_stack + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_INSTANT; + EXPECT_EQ_JSON(payload["category"], String, "B"); + EXPECT_TRUE(payload["stack"].isObject()); + + } else if (nameString == "auto tracing") { + switch (state) { + case S_tracing_auto_tracing_start: + state = State(S_tracing_auto_tracing_start + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_START; + EXPECT_EQ_JSON(payload["category"], String, "C"); + EXPECT_TRUE(payload["stack"].isNull()); + break; + case S_tracing_auto_tracing_end: + state = State(S_tracing_auto_tracing_end + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_END; + EXPECT_EQ_JSON(payload["category"], String, "C"); + ASSERT_TRUE(payload["stack"].isNull()); + break; + default: + EXPECT_TRUE(state == S_tracing_auto_tracing_start || + state == S_tracing_auto_tracing_end); + break; + } + + } else if (nameString == + "default-templated markers 2.0 with option") { + // TODO: Remove this when bug 1646714 lands. + EXPECT_EQ(state, S_Markers2DefaultWithOptions); + state = State(S_Markers2DefaultWithOptions + 1); + EXPECT_EQ(typeString, "NoPayloadUserData"); + EXPECT_FALSE(payload["stack"].isNull()); + + } else if (nameString == "FirstMarker") { + // Record start and end times, to compare with timestamps in + // following markers. + EXPECT_EQ(state, S_FirstMarker); + ts1Double = marker[START_TIME].asDouble(); + ts2Double = marker[END_TIME].asDouble(); + state = State(S_FirstMarker + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_EQ_JSON(payload["name"], String, "First Marker"); + + } else if (nameString == "Gtest custom marker") { + EXPECT_EQ(state, S_CustomMarker); + state = State(S_CustomMarker + 1); + EXPECT_EQ(typeString, "markers-gtest"); + EXPECT_EQ(payload.size(), 1u + 9u); + EXPECT_TRUE(payload["null"].isNull()); + EXPECT_EQ_JSON(payload["bool-false"], Bool, false); + EXPECT_EQ_JSON(payload["bool-true"], Bool, true); + EXPECT_EQ_JSON(payload["int"], Int64, 42); + EXPECT_EQ_JSON(payload["double"], Double, 43.0); + EXPECT_EQ_JSON(payload["text"], String, "gtest text"); + // Unique strings can be fetched from the string table. + ASSERT_TRUE(payload["unique text"].isUInt()); + auto textIndex = payload["unique text"].asUInt(); + GET_JSON(uniqueText, stringTable[textIndex], String); + ASSERT_TRUE(uniqueText.isString()); + ASSERT_EQ(uniqueText.asString(), "gtest unique text"); + // The duplicate unique text should have the exact same index. + EXPECT_EQ_JSON(payload["unique text again"], UInt, textIndex); + EXPECT_EQ_JSON(payload["time"], Double, ts1Double); + + } else if (nameString == "Gtest special marker") { + EXPECT_EQ(state, S_SpecialMarker); + state = State(S_SpecialMarker + 1); + EXPECT_EQ(typeString, "markers-gtest-special"); + EXPECT_EQ(payload.size(), 1u) << "Only 'type' in the payload"; + + } else if (nameString == "Load 1: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_start); + state = State(S_NetworkMarkerPayload_start + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 1); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Hit"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_TRUE(payload["RedirectURI"].isNull()); + EXPECT_TRUE(payload["redirectType"].isNull()); + EXPECT_TRUE(payload["isHttpToHttpsRedirect"].isNull()); + EXPECT_TRUE(payload["redirectId"].isNull()); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 2: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_stop); + state = State(S_NetworkMarkerPayload_stop + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 2); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_TRUE(payload["RedirectURI"].isNull()); + EXPECT_TRUE(payload["redirectType"].isNull()); + EXPECT_TRUE(payload["isHttpToHttpsRedirect"].isNull()); + EXPECT_TRUE(payload["redirectId"].isNull()); + EXPECT_EQ_JSON(payload["contentType"], String, "text/html"); + + } else if (nameString == "Load 3: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_redirect_temporary); + state = State(S_NetworkMarkerPayload_redirect_temporary + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 3); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_EQ_JSON(payload["RedirectURI"], String, + "http://example.com/"); + EXPECT_EQ_JSON(payload["redirectType"], String, "Temporary"); + EXPECT_EQ_JSON(payload["isHttpToHttpsRedirect"], Bool, false); + EXPECT_EQ_JSON(payload["redirectId"], Int64, 103); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 4: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_redirect_permanent); + state = State(S_NetworkMarkerPayload_redirect_permanent + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 4); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_EQ_JSON(payload["RedirectURI"], String, + "http://example.com/"); + EXPECT_EQ_JSON(payload["redirectType"], String, "Permanent"); + EXPECT_EQ_JSON(payload["isHttpToHttpsRedirect"], Bool, false); + EXPECT_EQ_JSON(payload["redirectId"], Int64, 104); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 5: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_redirect_internal); + state = State(S_NetworkMarkerPayload_redirect_internal + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 5); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_EQ_JSON(payload["RedirectURI"], String, + "http://example.com/"); + EXPECT_EQ_JSON(payload["redirectType"], String, "Internal"); + EXPECT_EQ_JSON(payload["isHttpToHttpsRedirect"], Bool, false); + EXPECT_EQ_JSON(payload["redirectId"], Int64, 105); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 6: http://mozilla.org/") { + EXPECT_EQ(state, + S_NetworkMarkerPayload_redirect_internal_sts); + state = + State(S_NetworkMarkerPayload_redirect_internal_sts + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 6); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_EQ_JSON(payload["RedirectURI"], String, + "http://example.com/"); + EXPECT_EQ_JSON(payload["redirectType"], String, "Internal"); + EXPECT_EQ_JSON(payload["isHttpToHttpsRedirect"], Bool, true); + EXPECT_EQ_JSON(payload["redirectId"], Int64, 106); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 7: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_private_browsing); + state = State(S_NetworkMarkerPayload_private_browsing + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 7); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_EQ_JSON(payload["isPrivateBrowsing"], Bool, true); + EXPECT_TRUE(payload["RedirectURI"].isNull()); + EXPECT_TRUE(payload["redirectType"].isNull()); + EXPECT_TRUE(payload["isHttpToHttpsRedirect"].isNull()); + EXPECT_TRUE(payload["redirectId"].isNull()); + EXPECT_TRUE(payload["contentType"].isNull()); + } else if (nameString == "Text in main thread with stack") { + EXPECT_EQ(state, S_TextWithStack); + state = State(S_TextWithStack + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_FALSE(payload["stack"].isNull()); + EXPECT_TIMING_INTERVAL_AT(ts1Double, ts2Double); + EXPECT_EQ_JSON(payload["name"], String, ""); + + } else if (nameString == "Text from main thread with stack") { + EXPECT_EQ(state, S_TextToMTWithStack); + state = State(S_TextToMTWithStack + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_FALSE(payload["stack"].isNull()); + EXPECT_EQ_JSON(payload["name"], String, ""); + + } else if (nameString == + "Text in registered thread with stack") { + ADD_FAILURE() + << "Unexpected 'Text in registered thread with stack'"; + + } else if (nameString == + "Text from registered thread with stack") { + EXPECT_EQ(state, S_RegThread_TextToMTWithStack); + state = State(S_RegThread_TextToMTWithStack + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_FALSE(payload["stack"].isNull()); + EXPECT_EQ_JSON(payload["name"], String, ""); + + } else if (nameString == + "Text in unregistered thread with stack") { + ADD_FAILURE() + << "Unexpected 'Text in unregistered thread with stack'"; + + } else if (nameString == + "Text from unregistered thread with stack") { + EXPECT_EQ(state, S_UnregThread_TextToMTWithStack); + state = State(S_UnregThread_TextToMTWithStack + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_TRUE(payload["stack"].isNull()); + EXPECT_EQ_JSON(payload["name"], String, ""); + } + } // marker with payload + } // for (marker : data) + } // markers.data + } // markers + } // thread0 + } // threads + // We should have read all expected markers. + EXPECT_EQ(state, S_LAST); + + { + GET_JSON(meta, root["meta"], Object); + + { + GET_JSON(markerSchema, meta["markerSchema"], Array); + + std::set<std::string> testedSchemaNames; + + for (const Json::Value& schema : markerSchema) { + GET_JSON(name, schema["name"], String); + const std::string nameString = name.asString(); + + GET_JSON(display, schema["display"], Array); + + GET_JSON(data, schema["data"], Array); + + EXPECT_TRUE( + testedSchemaNames + .insert(std::string(nameString.data(), nameString.size())) + .second) + << "Each schema name should be unique (inserted once in the set)"; + + if (nameString == "Text") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 1u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "name"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Details"); + EXPECT_EQ_JSON(data[0u]["format"], String, "string"); + + } else if (nameString == "NoPayloadUserData") { + // TODO: Remove this when bug 1646714 lands. + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 0u); + + } else if (nameString == "FileIO") { + // These are defined in ProfilerIOInterposeObserver.cpp + + } else if (nameString == "tracing") { + EXPECT_EQ(display.size(), 3u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + EXPECT_EQ(display[2u].asString(), "timeline-overview"); + + ASSERT_EQ(data.size(), 1u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "category"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Type"); + EXPECT_EQ_JSON(data[0u]["format"], String, "string"); + + } else if (nameString == "BHR-detected hang") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 0u); + + } else if (nameString == "MainThreadLongTask") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 1u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "category"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Type"); + EXPECT_EQ_JSON(data[0u]["format"], String, "string"); + + } else if (nameString == "Log") { + EXPECT_EQ(display.size(), 1u); + EXPECT_EQ(display[0u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 2u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "module"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Module"); + EXPECT_EQ_JSON(data[0u]["format"], String, "string"); + + ASSERT_TRUE(data[1u].isObject()); + EXPECT_EQ_JSON(data[1u]["key"], String, "name"); + EXPECT_EQ_JSON(data[1u]["label"], String, "Name"); + EXPECT_EQ_JSON(data[1u]["format"], String, "string"); + + } else if (nameString == "MediaSample") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 3u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "sampleStartTimeUs"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Sample start time"); + EXPECT_EQ_JSON(data[0u]["format"], String, "microseconds"); + + ASSERT_TRUE(data[1u].isObject()); + EXPECT_EQ_JSON(data[1u]["key"], String, "sampleEndTimeUs"); + EXPECT_EQ_JSON(data[1u]["label"], String, "Sample end time"); + EXPECT_EQ_JSON(data[1u]["format"], String, "microseconds"); + + ASSERT_TRUE(data[2u].isObject()); + EXPECT_EQ_JSON(data[2u]["key"], String, "queueLength"); + EXPECT_EQ_JSON(data[2u]["label"], String, "Queue length"); + EXPECT_EQ_JSON(data[2u]["format"], String, "integer"); + + } else if (nameString == "VideoFallingBehind") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 2u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "videoFrameStartTimeUs"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Video frame start time"); + EXPECT_EQ_JSON(data[0u]["format"], String, "microseconds"); + + ASSERT_TRUE(data[1u].isObject()); + EXPECT_EQ_JSON(data[1u]["key"], String, "mediaCurrentTimeUs"); + EXPECT_EQ_JSON(data[1u]["label"], String, "Media current time"); + EXPECT_EQ_JSON(data[1u]["format"], String, "microseconds"); + + } else if (nameString == "Budget") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 0u); + + } else if (nameString == "markers-gtest") { + EXPECT_EQ(display.size(), 7u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + EXPECT_EQ(display[2u].asString(), "timeline-overview"); + EXPECT_EQ(display[3u].asString(), "timeline-memory"); + EXPECT_EQ(display[4u].asString(), "timeline-ipc"); + EXPECT_EQ(display[5u].asString(), "timeline-fileio"); + EXPECT_EQ(display[6u].asString(), "stack-chart"); + + EXPECT_EQ_JSON(schema["chartLabel"], String, "chart label"); + EXPECT_EQ_JSON(schema["tooltipLabel"], String, "tooltip label"); + EXPECT_EQ_JSON(schema["tableLabel"], String, "table label"); + + ASSERT_EQ(data.size(), 15u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "key with url"); + EXPECT_TRUE(data[0u]["label"].isNull()); + EXPECT_EQ_JSON(data[0u]["format"], String, "url"); + EXPECT_TRUE(data[0u]["searchable"].isNull()); + + ASSERT_TRUE(data[1u].isObject()); + EXPECT_EQ_JSON(data[1u]["key"], String, "key with label filePath"); + EXPECT_EQ_JSON(data[1u]["label"], String, "label filePath"); + EXPECT_EQ_JSON(data[1u]["format"], String, "file-path"); + EXPECT_TRUE(data[1u]["searchable"].isNull()); + + ASSERT_TRUE(data[2u].isObject()); + EXPECT_EQ_JSON(data[2u]["key"], String, + "key with string not-searchable"); + EXPECT_TRUE(data[2u]["label"].isNull()); + EXPECT_EQ_JSON(data[2u]["format"], String, "string"); + EXPECT_EQ_JSON(data[2u]["searchable"], Bool, false); + + ASSERT_TRUE(data[3u].isObject()); + EXPECT_EQ_JSON(data[3u]["key"], String, + "key with label duration searchable"); + EXPECT_TRUE(data[3u]["label duration"].isNull()); + EXPECT_EQ_JSON(data[3u]["format"], String, "duration"); + EXPECT_EQ_JSON(data[3u]["searchable"], Bool, true); + + ASSERT_TRUE(data[4u].isObject()); + EXPECT_EQ_JSON(data[4u]["key"], String, "key with time"); + EXPECT_TRUE(data[4u]["label"].isNull()); + EXPECT_EQ_JSON(data[4u]["format"], String, "time"); + EXPECT_TRUE(data[4u]["searchable"].isNull()); + + ASSERT_TRUE(data[5u].isObject()); + EXPECT_EQ_JSON(data[5u]["key"], String, "key with seconds"); + EXPECT_TRUE(data[5u]["label"].isNull()); + EXPECT_EQ_JSON(data[5u]["format"], String, "seconds"); + EXPECT_TRUE(data[5u]["searchable"].isNull()); + + ASSERT_TRUE(data[6u].isObject()); + EXPECT_EQ_JSON(data[6u]["key"], String, "key with milliseconds"); + EXPECT_TRUE(data[6u]["label"].isNull()); + EXPECT_EQ_JSON(data[6u]["format"], String, "milliseconds"); + EXPECT_TRUE(data[6u]["searchable"].isNull()); + + ASSERT_TRUE(data[7u].isObject()); + EXPECT_EQ_JSON(data[7u]["key"], String, "key with microseconds"); + EXPECT_TRUE(data[7u]["label"].isNull()); + EXPECT_EQ_JSON(data[7u]["format"], String, "microseconds"); + EXPECT_TRUE(data[7u]["searchable"].isNull()); + + ASSERT_TRUE(data[8u].isObject()); + EXPECT_EQ_JSON(data[8u]["key"], String, "key with nanoseconds"); + EXPECT_TRUE(data[8u]["label"].isNull()); + EXPECT_EQ_JSON(data[8u]["format"], String, "nanoseconds"); + EXPECT_TRUE(data[8u]["searchable"].isNull()); + + ASSERT_TRUE(data[9u].isObject()); + EXPECT_EQ_JSON(data[9u]["key"], String, "key with bytes"); + EXPECT_TRUE(data[9u]["label"].isNull()); + EXPECT_EQ_JSON(data[9u]["format"], String, "bytes"); + EXPECT_TRUE(data[9u]["searchable"].isNull()); + + ASSERT_TRUE(data[10u].isObject()); + EXPECT_EQ_JSON(data[10u]["key"], String, "key with percentage"); + EXPECT_TRUE(data[10u]["label"].isNull()); + EXPECT_EQ_JSON(data[10u]["format"], String, "percentage"); + EXPECT_TRUE(data[10u]["searchable"].isNull()); + + ASSERT_TRUE(data[11u].isObject()); + EXPECT_EQ_JSON(data[11u]["key"], String, "key with integer"); + EXPECT_TRUE(data[11u]["label"].isNull()); + EXPECT_EQ_JSON(data[11u]["format"], String, "integer"); + EXPECT_TRUE(data[11u]["searchable"].isNull()); + + ASSERT_TRUE(data[12u].isObject()); + EXPECT_EQ_JSON(data[12u]["key"], String, "key with decimal"); + EXPECT_TRUE(data[12u]["label"].isNull()); + EXPECT_EQ_JSON(data[12u]["format"], String, "decimal"); + EXPECT_TRUE(data[12u]["searchable"].isNull()); + + ASSERT_TRUE(data[13u].isObject()); + EXPECT_EQ_JSON(data[13u]["label"], String, "static label"); + EXPECT_EQ_JSON(data[13u]["value"], String, "static value"); + + ASSERT_TRUE(data[14u].isObject()); + EXPECT_EQ_JSON(data[14u]["key"], String, "key with unique string"); + EXPECT_TRUE(data[14u]["label"].isNull()); + EXPECT_EQ_JSON(data[14u]["format"], String, "unique-string"); + EXPECT_TRUE(data[14u]["searchable"].isNull()); + + } else if (nameString == "markers-gtest-special") { + EXPECT_EQ(display.size(), 0u); + ASSERT_EQ(data.size(), 0u); + + } else if (nameString == "markers-gtest-unused") { + ADD_FAILURE() << "Schema for GtestUnusedMarker should not be here"; + + } else { + printf("FYI: Unknown marker schema '%s'\n", nameString.c_str()); + } + } + + // Check that we've got all expected schema. + EXPECT_TRUE(testedSchemaNames.find("Text") != testedSchemaNames.end()); + EXPECT_TRUE(testedSchemaNames.find("tracing") != + testedSchemaNames.end()); + EXPECT_TRUE(testedSchemaNames.find("MediaSample") != + testedSchemaNames.end()); + } // markerSchema + } // meta + }); + + Maybe<ProfilerBufferInfo> info = profiler_get_buffer_info(); + EXPECT_TRUE(info.isSome()); + printf("Profiler buffer range: %llu .. %llu (%llu bytes)\n", + static_cast<unsigned long long>(info->mRangeStart), + static_cast<unsigned long long>(info->mRangeEnd), + // sizeof(ProfileBufferEntry) == 9 + (static_cast<unsigned long long>(info->mRangeEnd) - + static_cast<unsigned long long>(info->mRangeStart)) * + 9); + printf("Stats: min(us) .. mean(us) .. max(us) [count]\n"); + printf("- Intervals: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mIntervalsUs.min, info->mIntervalsUs.sum / info->mIntervalsUs.n, + info->mIntervalsUs.max, info->mIntervalsUs.n); + printf("- Overheads: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mOverheadsUs.min, info->mOverheadsUs.sum / info->mOverheadsUs.n, + info->mOverheadsUs.max, info->mOverheadsUs.n); + printf(" - Locking: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mLockingsUs.min, info->mLockingsUs.sum / info->mLockingsUs.n, + info->mLockingsUs.max, info->mLockingsUs.n); + printf(" - Clearning: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mCleaningsUs.min, info->mCleaningsUs.sum / info->mCleaningsUs.n, + info->mCleaningsUs.max, info->mCleaningsUs.n); + printf(" - Counters: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mCountersUs.min, info->mCountersUs.sum / info->mCountersUs.n, + info->mCountersUs.max, info->mCountersUs.n); + printf(" - Threads: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mThreadsUs.min, info->mThreadsUs.sum / info->mThreadsUs.n, + info->mThreadsUs.max, info->mThreadsUs.n); + + profiler_stop(); + + // Try to add markers while the profiler is stopped. + PROFILER_MARKER_UNTYPED("marker after profiler_stop", OTHER); + + // Warning: this could be racy + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + // This last marker shouldn't get streamed. + SpliceableChunkedJSONWriter w2{FailureLatchInfallibleSource::Singleton()}; + w2.Start(); + EXPECT_TRUE(::profiler_stream_json_for_this_process(w2).isOk()); + w2.End(); + EXPECT_FALSE(w2.Failed()); + UniquePtr<char[]> profile2 = w2.ChunkedWriteFunc().CopyData(); + ASSERT_TRUE(!!profile2.get()); + EXPECT_TRUE( + std::string_view(profile2.get()).find("marker after profiler_stop") == + std::string_view::npos); + + profiler_stop(); +} + +# define COUNTER_NAME "TestCounter" +# define COUNTER_DESCRIPTION "Test of counters in profiles" +# define COUNTER_NAME2 "Counter2" +# define COUNTER_DESCRIPTION2 "Second Test of counters in profiles" + +PROFILER_DEFINE_COUNT_TOTAL(TestCounter, COUNTER_NAME, COUNTER_DESCRIPTION); +PROFILER_DEFINE_COUNT_TOTAL(TestCounter2, COUNTER_NAME2, COUNTER_DESCRIPTION2); + +TEST(GeckoProfiler, Counters) +{ + uint32_t features = 0; + const char* filters[] = {"GeckoMain"}; + + // We will record some counter values, and check that they're present (and no + // other) when expected. + + struct NumberAndCount { + uint64_t mNumber; + int64_t mCount; + }; + + int64_t testCounters[] = {10, 7, -17}; + NumberAndCount expectedTestCounters[] = {{1u, 10}, {0u, 0}, {1u, 7}, + {0u, 0}, {0u, 0}, {1u, -17}, + {0u, 0}, {0u, 0}}; + constexpr size_t expectedTestCountersCount = + MOZ_ARRAY_LENGTH(expectedTestCounters); + + bool expectCounter2 = false; + int64_t testCounters2[] = {10}; + NumberAndCount expectedTestCounters2[] = {{1u, 10}, {0u, 0}}; + constexpr size_t expectedTestCounters2Count = + MOZ_ARRAY_LENGTH(expectedTestCounters2); + + auto checkCountersInJSON = [&](const Json::Value& aRoot) { + size_t nextExpectedTestCounter = 0u; + size_t nextExpectedTestCounter2 = 0u; + + GET_JSON(counters, aRoot["counters"], Array); + for (const Json::Value& counter : counters) { + ASSERT_TRUE(counter.isObject()); + GET_JSON_VALUE(name, counter["name"], String); + if (name == "TestCounter") { + EXPECT_EQ_JSON(counter["category"], String, COUNTER_NAME); + EXPECT_EQ_JSON(counter["description"], String, COUNTER_DESCRIPTION); + GET_JSON(samples, counter["samples"], Object); + GET_JSON(samplesSchema, samples["schema"], Object); + EXPECT_GE(samplesSchema.size(), 3u); + GET_JSON_VALUE(samplesNumber, samplesSchema["number"], UInt); + GET_JSON_VALUE(samplesCount, samplesSchema["count"], UInt); + GET_JSON(samplesData, samples["data"], Array); + for (const Json::Value& sample : samplesData) { + ASSERT_TRUE(sample.isArray()); + ASSERT_LT(nextExpectedTestCounter, expectedTestCountersCount); + EXPECT_EQ_JSON(sample[samplesNumber], UInt64, + expectedTestCounters[nextExpectedTestCounter].mNumber); + EXPECT_EQ_JSON(sample[samplesCount], Int64, + expectedTestCounters[nextExpectedTestCounter].mCount); + ++nextExpectedTestCounter; + } + } else if (name == "TestCounter2") { + EXPECT_TRUE(expectCounter2); + + EXPECT_EQ_JSON(counter["category"], String, COUNTER_NAME2); + EXPECT_EQ_JSON(counter["description"], String, COUNTER_DESCRIPTION2); + GET_JSON(samples, counter["samples"], Object); + GET_JSON(samplesSchema, samples["schema"], Object); + EXPECT_GE(samplesSchema.size(), 3u); + GET_JSON_VALUE(samplesNumber, samplesSchema["number"], UInt); + GET_JSON_VALUE(samplesCount, samplesSchema["count"], UInt); + GET_JSON(samplesData, samples["data"], Array); + for (const Json::Value& sample : samplesData) { + ASSERT_TRUE(sample.isArray()); + ASSERT_LT(nextExpectedTestCounter2, expectedTestCounters2Count); + EXPECT_EQ_JSON( + sample[samplesNumber], UInt64, + expectedTestCounters2[nextExpectedTestCounter2].mNumber); + EXPECT_EQ_JSON( + sample[samplesCount], Int64, + expectedTestCounters2[nextExpectedTestCounter2].mCount); + ++nextExpectedTestCounter2; + } + } + } + + EXPECT_EQ(nextExpectedTestCounter, expectedTestCountersCount); + if (expectCounter2) { + EXPECT_EQ(nextExpectedTestCounter2, expectedTestCounters2Count); + } + }; + + // Inactive -> Active + profiler_ensure_started(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + // Output all "TestCounter"s, with increasing delays (to test different + // number of counter samplings). + int samplingWaits = 2; + for (int64_t counter : testCounters) { + AUTO_PROFILER_COUNT_TOTAL(TestCounter, counter); + for (int i = 0; i < samplingWaits; ++i) { + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + ++samplingWaits; + } + + // Verify we got "TestCounter" in the output, but not "TestCounter2" yet. + UniquePtr<char[]> profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), checkCountersInJSON); + + // Now introduce TestCounter2. + expectCounter2 = true; + for (int64_t counter2 : testCounters2) { + AUTO_PROFILER_COUNT_TOTAL(TestCounter2, counter2); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + + // Verify we got both "TestCounter" and "TestCounter2" in the output. + profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), checkCountersInJSON); + + profiler_stop(); +} + +TEST(GeckoProfiler, Time) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + double t1 = profiler_time(); + double t2 = profiler_time(); + ASSERT_TRUE(t1 <= t2); + + // profiler_start() restarts the timer used by profiler_time(). + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + double t3 = profiler_time(); + double t4 = profiler_time(); + ASSERT_TRUE(t3 <= t4); + + profiler_stop(); + + double t5 = profiler_time(); + double t6 = profiler_time(); + ASSERT_TRUE(t4 <= t5 && t1 <= t6); +} + +TEST(GeckoProfiler, GetProfile) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!profiler_get_profile()); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + mozilla::Maybe<uint32_t> activeFeatures = profiler_features_if_active(); + ASSERT_TRUE(activeFeatures.isSome()); + // Not all platforms support stack-walking. + const bool hasStackWalk = ProfilerFeature::HasStackWalk(*activeFeatures); + + UniquePtr<char[]> profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), [&](const Json::Value& aRoot) { + GET_JSON(meta, aRoot["meta"], Object); + { + GET_JSON(configuration, meta["configuration"], Object); + { + GET_JSON(features, configuration["features"], Array); + { + EXPECT_EQ(features.size(), (hasStackWalk ? 1u : 0u)); + if (hasStackWalk) { + EXPECT_JSON_ARRAY_CONTAINS(features, String, "stackwalk"); + } + } + GET_JSON(threads, configuration["threads"], Array); + { + EXPECT_EQ(threads.size(), 1u); + EXPECT_JSON_ARRAY_CONTAINS(threads, String, "GeckoMain"); + } + } + } + }); + + profiler_stop(); + + ASSERT_TRUE(!profiler_get_profile()); +} + +TEST(GeckoProfiler, StreamJSONForThisProcess) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + SpliceableChunkedJSONWriter w{FailureLatchInfallibleSource::Singleton()}; + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Fallible()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(&w.ChunkedWriteFunc().SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT( + &std::as_const(w.ChunkedWriteFunc()).SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT(!w.Fallible()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + MOZ_RELEASE_ASSERT(&w.SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT(&std::as_const(w).SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isErr()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + w.Start(); + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isOk()); + w.End(); + + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + + UniquePtr<char[]> profile = w.ChunkedWriteFunc().CopyData(); + + JSONOutputCheck(profile.get(), [](const Json::Value&) {}); + + profiler_stop(); + + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isErr()); +} + +// Internal version of profiler_stream_json_for_this_process, which allows being +// called from a non-main thread of the parent process, at the risk of getting +// an incomplete profile. +ProfilerResult<ProfileGenerationAdditionalInformation> +do_profiler_stream_json_for_this_process( + SpliceableJSONWriter& aWriter, double aSinceTime, bool aIsShuttingDown, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger); + +TEST(GeckoProfiler, StreamJSONForThisProcessThreaded) +{ + // Same as the previous test, but calling some things on background threads. + nsCOMPtr<nsIThread> thread; + nsresult rv = NS_NewNamedThread("GeckoProfGTest", getter_AddRefs(thread)); + ASSERT_NS_SUCCEEDED(rv); + + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + SpliceableChunkedJSONWriter w{FailureLatchInfallibleSource::Singleton()}; + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Fallible()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(&w.ChunkedWriteFunc().SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT( + &std::as_const(w.ChunkedWriteFunc()).SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT(!w.Fallible()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + MOZ_RELEASE_ASSERT(&w.SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT(&std::as_const(w).SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isErr()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + + // Start the profiler on the main thread. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + // Call profiler_stream_json_for_this_process on a background thread. + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_StreamJSONForThisProcessThreaded_Test::TestBody"_ns, + thread, + NS_NewRunnableFunction( + "GeckoProfiler_StreamJSONForThisProcessThreaded_Test::TestBody", + [&]() { + w.Start(); + ASSERT_TRUE(::do_profiler_stream_json_for_this_process( + w, /* double aSinceTime */ 0.0, + /* bool aIsShuttingDown */ false, + /* ProfilerCodeAddressService* aService */ nullptr, + mozilla::ProgressLogger{}) + .isOk()); + w.End(); + })); + + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + + UniquePtr<char[]> profile = w.ChunkedWriteFunc().CopyData(); + + JSONOutputCheck(profile.get(), [](const Json::Value&) {}); + + // Stop the profiler and call profiler_stream_json_for_this_process on a + // background thread. + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_StreamJSONForThisProcessThreaded_Test::TestBody"_ns, + thread, + NS_NewRunnableFunction( + "GeckoProfiler_StreamJSONForThisProcessThreaded_Test::TestBody", + [&]() { + profiler_stop(); + ASSERT_TRUE(::do_profiler_stream_json_for_this_process( + w, /* double aSinceTime */ 0.0, + /* bool aIsShuttingDown */ false, + /* ProfilerCodeAddressService* aService */ nullptr, + mozilla::ProgressLogger{}) + .isErr()); + })); + thread->Shutdown(); + + // Call profiler_stream_json_for_this_process on the main thread. + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isErr()); +} + +TEST(GeckoProfiler, ProfilingStack) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + AUTO_PROFILER_LABEL("A::B", OTHER); + + UniqueFreePtr<char> dynamic(strdup("dynamic")); + { + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("A::C", JS, dynamic.get()); + AUTO_PROFILER_LABEL_DYNAMIC_NSCSTRING("A::C2", JS, + nsDependentCString(dynamic.get())); + AUTO_PROFILER_LABEL_DYNAMIC_LOSSY_NSSTRING( + "A::C3", JS, NS_ConvertUTF8toUTF16(dynamic.get())); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(profiler_get_backtrace()); + } + + AutoProfilerLabel label1("A", nullptr, JS::ProfilingCategoryPair::DOM); + AutoProfilerLabel label2("A", dynamic.get(), + JS::ProfilingCategoryPair::NETWORK); + ASSERT_TRUE(profiler_get_backtrace()); + + profiler_stop(); + + ASSERT_TRUE(!profiler_get_profile()); +} + +TEST(GeckoProfiler, Bug1355807) +{ + uint32_t features = ProfilerFeature::JS; + const char* manyThreadsFilter[] = {""}; + const char* fewThreadsFilter[] = {"GeckoMain"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + manyThreadsFilter, MOZ_ARRAY_LENGTH(manyThreadsFilter), 0); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + fewThreadsFilter, MOZ_ARRAY_LENGTH(fewThreadsFilter), 0); + + // In bug 1355807 this caused an assertion failure in StopJSSampling(). + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + fewThreadsFilter, MOZ_ARRAY_LENGTH(fewThreadsFilter), 0); + + profiler_stop(); +} + +class GTestStackCollector final : public ProfilerStackCollector { + public: + GTestStackCollector() : mSetIsMainThread(0), mFrames(0) {} + + virtual void SetIsMainThread() { mSetIsMainThread++; } + + virtual void CollectNativeLeafAddr(void* aAddr) { mFrames++; } + virtual void CollectJitReturnAddr(void* aAddr) { mFrames++; } + virtual void CollectWasmFrame(const char* aLabel) { mFrames++; } + virtual void CollectProfilingStackFrame( + const js::ProfilingStackFrame& aFrame) { + mFrames++; + } + + int mSetIsMainThread; + int mFrames; +}; + +void DoSuspendAndSample(ProfilerThreadId aTidToSample, + nsIThread* aSamplingThread) { + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_SuspendAndSample_Test::TestBody"_ns, aSamplingThread, + NS_NewRunnableFunction( + "GeckoProfiler_SuspendAndSample_Test::TestBody", [&]() { + uint32_t features = ProfilerFeature::CPUUtilization; + GTestStackCollector collector; + profiler_suspend_and_sample_thread(aTidToSample, features, + collector, + /* sampleNative = */ true); + + ASSERT_TRUE(collector.mSetIsMainThread == + (aTidToSample == profiler_main_thread_id())); + ASSERT_TRUE(collector.mFrames > 0); + })); +} + +TEST(GeckoProfiler, SuspendAndSample) +{ + nsCOMPtr<nsIThread> thread; + nsresult rv = NS_NewNamedThread("GeckoProfGTest", getter_AddRefs(thread)); + ASSERT_NS_SUCCEEDED(rv); + + ProfilerThreadId tid = profiler_current_thread_id(); + + ASSERT_TRUE(!profiler_is_active()); + + // Suspend and sample while the profiler is inactive. + DoSuspendAndSample(tid, thread); + + DoSuspendAndSample(ProfilerThreadId{}, thread); + + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(profiler_is_active()); + + // Suspend and sample while the profiler is active. + DoSuspendAndSample(tid, thread); + + DoSuspendAndSample(ProfilerThreadId{}, thread); + + profiler_stop(); + + ASSERT_TRUE(!profiler_is_active()); +} + +TEST(GeckoProfiler, PostSamplingCallback) +{ + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_callback_after_sampling( + [&](SamplingState) { ASSERT_TRUE(false); })); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk, filters, MOZ_ARRAY_LENGTH(filters), + 0); + { + // Stack sampling -> This label should appear at least once. + AUTO_PROFILER_LABEL("PostSamplingCallback completed", OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + UniquePtr<char[]> profileCompleted = profiler_get_profile(); + JSONOutputCheck(profileCompleted.get(), [](const Json::Value& aRoot) { + GET_JSON(threads, aRoot["threads"], Array); + { + GET_JSON(thread0, threads[0], Object); + { + EXPECT_JSON_ARRAY_CONTAINS(thread0["stringTable"], String, + "PostSamplingCallback completed"); + } + } + }); + + profiler_pause(); + { + // Paused -> This label should not appear. + AUTO_PROFILER_LABEL("PostSamplingCallback paused", OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingPaused); + } + UniquePtr<char[]> profilePaused = profiler_get_profile(); + JSONOutputCheck(profilePaused.get(), [](const Json::Value& aRoot) {}); + // This string shouldn't appear *anywhere* in the profile. + ASSERT_FALSE(strstr(profilePaused.get(), "PostSamplingCallback paused")); + + profiler_resume(); + { + // Stack sampling -> This label should appear at least once. + AUTO_PROFILER_LABEL("PostSamplingCallback resumed", OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + UniquePtr<char[]> profileResumed = profiler_get_profile(); + JSONOutputCheck(profileResumed.get(), [](const Json::Value& aRoot) { + GET_JSON(threads, aRoot["threads"], Array); + { + GET_JSON(thread0, threads[0], Object); + { + EXPECT_JSON_ARRAY_CONTAINS(thread0["stringTable"], String, + "PostSamplingCallback resumed"); + } + } + }); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk | ProfilerFeature::NoStackSampling, + filters, MOZ_ARRAY_LENGTH(filters), 0); + { + // No stack sampling -> This label should not appear. + AUTO_PROFILER_LABEL("PostSamplingCallback completed (no stacks)", OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::NoStackSamplingCompleted); + } + UniquePtr<char[]> profileNoStacks = profiler_get_profile(); + JSONOutputCheck(profileNoStacks.get(), [](const Json::Value& aRoot) {}); + // This string shouldn't appear *anywhere* in the profile. + ASSERT_FALSE(strstr(profileNoStacks.get(), + "PostSamplingCallback completed (no stacks)")); + + // Note: There is no non-racy way to test for SamplingState::JustStopped, as + // it would require coordination between `profiler_stop()` and another thread + // doing `profiler_callback_after_sampling()` at just the right moment. + + profiler_stop(); + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_callback_after_sampling( + [&](SamplingState) { ASSERT_TRUE(false); })); +} + +TEST(GeckoProfiler, ProfilingStateCallback) +{ + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!profiler_is_active()); + + struct ProfilingStateAndId { + ProfilingState mProfilingState; + int mId; + }; + DataMutex<Vector<ProfilingStateAndId>> states{"Profiling states"}; + auto CreateCallback = [&states](int id) { + return [id, &states](ProfilingState aProfilingState) { + auto lockedStates = states.Lock(); + ASSERT_TRUE( + lockedStates->append(ProfilingStateAndId{aProfilingState, id})); + }; + }; + auto CheckStatesIsEmpty = [&states]() { + auto lockedStates = states.Lock(); + EXPECT_TRUE(lockedStates->empty()); + }; + auto CheckStatesOnlyContains = [&states](ProfilingState aProfilingState, + int aId) { + auto lockedStates = states.Lock(); + EXPECT_EQ(lockedStates->length(), 1u); + if (lockedStates->length() >= 1u) { + EXPECT_EQ((*lockedStates)[0].mProfilingState, aProfilingState); + EXPECT_EQ((*lockedStates)[0].mId, aId); + } + lockedStates->clear(); + }; + + profiler_add_state_change_callback(AllProfilingStates(), CreateCallback(1), + 1); + // This is in case of error, and it also exercises the (allowed) removal of + // unknown callback ids. + auto cleanup1 = mozilla::MakeScopeExit( + []() { profiler_remove_state_change_callback(1); }); + CheckStatesIsEmpty(); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk, filters, MOZ_ARRAY_LENGTH(filters), + 0); + + CheckStatesOnlyContains(ProfilingState::Started, 1); + + profiler_add_state_change_callback(AllProfilingStates(), CreateCallback(2), + 2); + // This is in case of error, and it also exercises the (allowed) removal of + // unknown callback ids. + auto cleanup2 = mozilla::MakeScopeExit( + []() { profiler_remove_state_change_callback(2); }); + CheckStatesOnlyContains(ProfilingState::AlreadyActive, 2); + + profiler_remove_state_change_callback(2); + CheckStatesOnlyContains(ProfilingState::RemovingCallback, 2); + // Note: The actual removal is effectively tested below, by not seeing any + // more invocations of the 2nd callback. + + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + UniquePtr<char[]> profileCompleted = profiler_get_profile(); + CheckStatesOnlyContains(ProfilingState::GeneratingProfile, 1); + JSONOutputCheck(profileCompleted.get(), [](const Json::Value& aRoot) {}); + + profiler_pause(); + CheckStatesOnlyContains(ProfilingState::Pausing, 1); + UniquePtr<char[]> profilePaused = profiler_get_profile(); + CheckStatesOnlyContains(ProfilingState::GeneratingProfile, 1); + JSONOutputCheck(profilePaused.get(), [](const Json::Value& aRoot) {}); + + profiler_resume(); + CheckStatesOnlyContains(ProfilingState::Resumed, 1); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + UniquePtr<char[]> profileResumed = profiler_get_profile(); + CheckStatesOnlyContains(ProfilingState::GeneratingProfile, 1); + JSONOutputCheck(profileResumed.get(), [](const Json::Value& aRoot) {}); + + // This effectively stops the profiler before restarting it, but + // ProfilingState::Stopping is not notified. See `profiler_start` for details. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk | ProfilerFeature::NoStackSampling, + filters, MOZ_ARRAY_LENGTH(filters), 0); + CheckStatesOnlyContains(ProfilingState::Started, 1); + ASSERT_EQ(WaitForSamplingState(), SamplingState::NoStackSamplingCompleted); + UniquePtr<char[]> profileNoStacks = profiler_get_profile(); + CheckStatesOnlyContains(ProfilingState::GeneratingProfile, 1); + JSONOutputCheck(profileNoStacks.get(), [](const Json::Value& aRoot) {}); + + profiler_stop(); + CheckStatesOnlyContains(ProfilingState::Stopping, 1); + ASSERT_TRUE(!profiler_is_active()); + + profiler_remove_state_change_callback(1); + CheckStatesOnlyContains(ProfilingState::RemovingCallback, 1); + + // Note: ProfilingState::ShuttingDown cannot be tested here, and the profiler + // can only be shut down once per process. +} + +TEST(GeckoProfiler, BaseProfilerHandOff) +{ + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!baseprofiler::profiler_is_active()); + ASSERT_TRUE(!profiler_is_active()); + + BASE_PROFILER_MARKER_UNTYPED("Base marker before base profiler", OTHER, {}); + PROFILER_MARKER_UNTYPED("Gecko marker before base profiler", OTHER, {}); + + // Start the Base Profiler. + baseprofiler::profiler_start( + PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk, filters, MOZ_ARRAY_LENGTH(filters)); + + ASSERT_TRUE(baseprofiler::profiler_is_active()); + ASSERT_TRUE(!profiler_is_active()); + + // Add at least a marker, which should go straight into the buffer. + Maybe<baseprofiler::ProfilerBufferInfo> info0 = + baseprofiler::profiler_get_buffer_info(); + BASE_PROFILER_MARKER_UNTYPED("Base marker during base profiler", OTHER, {}); + Maybe<baseprofiler::ProfilerBufferInfo> info1 = + baseprofiler::profiler_get_buffer_info(); + ASSERT_GT(info1->mRangeEnd, info0->mRangeEnd); + + PROFILER_MARKER_UNTYPED("Gecko marker during base profiler", OTHER, {}); + + // Start the Gecko Profiler, which should grab the Base Profiler profile and + // stop it. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk, filters, MOZ_ARRAY_LENGTH(filters), + 0); + + ASSERT_TRUE(!baseprofiler::profiler_is_active()); + ASSERT_TRUE(profiler_is_active()); + + BASE_PROFILER_MARKER_UNTYPED("Base marker during gecko profiler", OTHER, {}); + PROFILER_MARKER_UNTYPED("Gecko marker during gecko profiler", OTHER, {}); + + // Write some Gecko Profiler samples. + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + + // Check that the Gecko Profiler profile contains at least the Base Profiler + // main thread samples. + UniquePtr<char[]> profile = profiler_get_profile(); + + profiler_stop(); + ASSERT_TRUE(!profiler_is_active()); + + BASE_PROFILER_MARKER_UNTYPED("Base marker after gecko profiler", OTHER, {}); + PROFILER_MARKER_UNTYPED("Gecko marker after gecko profiler", OTHER, {}); + + JSONOutputCheck(profile.get(), [](const Json::Value& aRoot) { + GET_JSON(threads, aRoot["threads"], Array); + { + bool found = false; + for (const Json::Value& thread : threads) { + ASSERT_TRUE(thread.isObject()); + GET_JSON(name, thread["name"], String); + if (name.asString() == "GeckoMain") { + found = true; + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Base marker before base profiler"); + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Gecko marker before base profiler"); + EXPECT_JSON_ARRAY_CONTAINS(thread["stringTable"], String, + "Base marker during base profiler"); + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Gecko marker during base profiler"); + EXPECT_JSON_ARRAY_CONTAINS(thread["stringTable"], String, + "Base marker during gecko profiler"); + EXPECT_JSON_ARRAY_CONTAINS(thread["stringTable"], String, + "Gecko marker during gecko profiler"); + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Base marker after gecko profiler"); + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Gecko marker after gecko profiler"); + break; + } + } + EXPECT_TRUE(found); + } + }); +} + +static std::string_view GetFeatureName(uint32_t feature) { + switch (feature) { +# define FEATURE_NAME(n_, str_, Name_, desc_) \ + case ProfilerFeature::Name_: \ + return str_; + + PROFILER_FOR_EACH_FEATURE(FEATURE_NAME) + +# undef FEATURE_NAME + + default: + return "?"; + } +} + +TEST(GeckoProfiler, FeatureCombinations) +{ + const char* filters[] = {"*"}; + + // List of features to test. Every combination of up to 3 of them will be + // tested, so be careful not to add too many to keep the test run at a + // reasonable time. + uint32_t featureList[] = {ProfilerFeature::JS, + ProfilerFeature::Screenshots, + ProfilerFeature::StackWalk, + ProfilerFeature::NoStackSampling, + ProfilerFeature::NativeAllocations, + ProfilerFeature::CPUUtilization, + ProfilerFeature::CPUAllThreads, + ProfilerFeature::SamplingAllThreads, + ProfilerFeature::MarkersAllThreads, + ProfilerFeature::UnregisteredThreads}; + constexpr uint32_t featureCount = uint32_t(MOZ_ARRAY_LENGTH(featureList)); + + auto testFeatures = [&](uint32_t features, + const std::string& featuresString) { + SCOPED_TRACE(featuresString.c_str()); + + ASSERT_TRUE(!profiler_is_active()); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(profiler_is_active()); + + // Write some Gecko Profiler samples. + EXPECT_EQ(WaitForSamplingState(), + (((features & ProfilerFeature::NoStackSampling) != 0) && + ((features & (ProfilerFeature::CPUUtilization | + ProfilerFeature::CPUAllThreads)) == 0)) + ? SamplingState::NoStackSamplingCompleted + : SamplingState::SamplingCompleted); + + // Check that the profile looks valid. Note that we don't test feature- + // specific changes. + UniquePtr<char[]> profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), [](const Json::Value& aRoot) {}); + + profiler_stop(); + ASSERT_TRUE(!profiler_is_active()); + }; + + testFeatures(0, "Features: (none)"); + + for (uint32_t f1 = 0u; f1 < featureCount; ++f1) { + const uint32_t features1 = featureList[f1]; + std::string features1String = "Features: "; + features1String += GetFeatureName(featureList[f1]); + + testFeatures(features1, features1String); + + for (uint32_t f2 = f1 + 1u; f2 < featureCount; ++f2) { + const uint32_t features12 = f1 | featureList[f2]; + std::string features12String = features1String + " "; + features12String += GetFeatureName(featureList[f2]); + + testFeatures(features12, features12String); + + for (uint32_t f3 = f2 + 1u; f3 < featureCount; ++f3) { + const uint32_t features123 = features12 | featureList[f3]; + std::string features123String = features12String + " "; + features123String += GetFeatureName(featureList[f3]); + + testFeatures(features123, features123String); + } + } + } +} + +static void CountCPUDeltas(const Json::Value& aThread, size_t& aOutSamplings, + uint64_t& aOutCPUDeltaSum) { + GET_JSON(samples, aThread["samples"], Object); + { + Json::ArrayIndex threadCPUDeltaIndex = 0; + GET_JSON(schema, samples["schema"], Object); + { + GET_JSON(jsonThreadCPUDeltaIndex, schema["threadCPUDelta"], UInt); + threadCPUDeltaIndex = jsonThreadCPUDeltaIndex.asUInt(); + } + + aOutSamplings = 0; + aOutCPUDeltaSum = 0; + GET_JSON(data, samples["data"], Array); + aOutSamplings = data.size(); + for (const Json::Value& sample : data) { + ASSERT_TRUE(sample.isArray()); + if (sample.isValidIndex(threadCPUDeltaIndex)) { + if (!sample[threadCPUDeltaIndex].isNull()) { + GET_JSON(cpuDelta, sample[threadCPUDeltaIndex], UInt64); + aOutCPUDeltaSum += uint64_t(cpuDelta.asUInt64()); + } + } + } + } +} + +TEST(GeckoProfiler, CPUUsage) +{ + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + const char* filters[] = {"GeckoMain", "Idle test", "Busy test"}; + + enum class TestThreadsState { + // Initial state, while constructing and starting the idle thread. + STARTING, + // Set by the idle thread just before running its main mostly-idle loop. + RUNNING1, + RUNNING2, + // Set by the main thread when it wants the idle thread to stop. + STOPPING + }; + Atomic<TestThreadsState> testThreadsState{TestThreadsState::STARTING}; + + std::thread idle([&]() { + AUTO_PROFILER_REGISTER_THREAD("Idle test"); + // Add a label to ensure that we have a non-empty stack, even if native + // stack-walking is not available. + AUTO_PROFILER_LABEL("Idle test", PROFILER); + ASSERT_TRUE(testThreadsState.compareExchange(TestThreadsState::STARTING, + TestThreadsState::RUNNING1) || + testThreadsState.compareExchange(TestThreadsState::RUNNING1, + TestThreadsState::RUNNING2)); + + while (testThreadsState != TestThreadsState::STOPPING) { + // Sleep for multiple profiler intervals, so the profiler should have + // samples with zero CPU utilization. + PR_Sleep(PR_MillisecondsToInterval(PROFILER_DEFAULT_INTERVAL * 10)); + } + }); + + std::thread busy([&]() { + AUTO_PROFILER_REGISTER_THREAD("Busy test"); + // Add a label to ensure that we have a non-empty stack, even if native + // stack-walking is not available. + AUTO_PROFILER_LABEL("Busy test", PROFILER); + ASSERT_TRUE(testThreadsState.compareExchange(TestThreadsState::STARTING, + TestThreadsState::RUNNING1) || + testThreadsState.compareExchange(TestThreadsState::RUNNING1, + TestThreadsState::RUNNING2)); + + while (testThreadsState != TestThreadsState::STOPPING) { + // Stay busy! + } + }); + + // Wait for idle thread to start running its main loop. + while (testThreadsState != TestThreadsState::RUNNING2) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + + // We want to ensure that CPU usage numbers are present whether or not we are + // collecting stack samples. + static constexpr bool scTestsWithOrWithoutStackSampling[] = {false, true}; + for (const bool testWithNoStackSampling : scTestsWithOrWithoutStackSampling) { + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_callback_after_sampling( + [&](SamplingState) { ASSERT_TRUE(false); })); + + profiler_start( + PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk | ProfilerFeature::CPUUtilization | + (testWithNoStackSampling ? ProfilerFeature::NoStackSampling : 0), + filters, MOZ_ARRAY_LENGTH(filters), 0); + // Grab a few samples, each with a different label on the stack. +# define SAMPLE_LABEL_PREFIX "CPUUsage sample label " + static constexpr const char* scSampleLabels[] = { + SAMPLE_LABEL_PREFIX "0", SAMPLE_LABEL_PREFIX "1", + SAMPLE_LABEL_PREFIX "2", SAMPLE_LABEL_PREFIX "3", + SAMPLE_LABEL_PREFIX "4", SAMPLE_LABEL_PREFIX "5", + SAMPLE_LABEL_PREFIX "6", SAMPLE_LABEL_PREFIX "7", + SAMPLE_LABEL_PREFIX "8", SAMPLE_LABEL_PREFIX "9"}; + static constexpr size_t scSampleLabelCount = + (sizeof(scSampleLabels) / sizeof(scSampleLabels[0])); + // We'll do two samplings for each label. + static constexpr size_t scMinSamplings = scSampleLabelCount * 2; + + for (const char* sampleLabel : scSampleLabels) { + AUTO_PROFILER_LABEL(sampleLabel, OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + // Note: There could have been a delay before this label above, where the + // profiler could have sampled the stack and missed the label. By forcing + // another sampling now, the label is guaranteed to be present. + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + + UniquePtr<char[]> profile = profiler_get_profile(); + + if (testWithNoStackSampling) { + // If we are testing nostacksampling, we shouldn't find this label prefix + // in the profile. + EXPECT_FALSE(strstr(profile.get(), SAMPLE_LABEL_PREFIX)); + } else { + // In normal sampling mode, we should find all labels. + for (const char* sampleLabel : scSampleLabels) { + EXPECT_TRUE(strstr(profile.get(), sampleLabel)); + } + } + + JSONOutputCheck(profile.get(), [testWithNoStackSampling]( + const Json::Value& aRoot) { + // Check that the "cpu" feature is present. + GET_JSON(meta, aRoot["meta"], Object); + { + GET_JSON(configuration, meta["configuration"], Object); + { + GET_JSON(features, configuration["features"], Array); + { EXPECT_JSON_ARRAY_CONTAINS(features, String, "cpu"); } + } + } + + { + GET_JSON(sampleUnits, meta["sampleUnits"], Object); + { + EXPECT_EQ_JSON(sampleUnits["time"], String, "ms"); + EXPECT_EQ_JSON(sampleUnits["eventDelay"], String, "ms"); +# if defined(GP_OS_windows) || defined(GP_OS_darwin) || \ + defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) + // Note: The exact string is not important here. + EXPECT_TRUE(sampleUnits["threadCPUDelta"].isString()) + << "There should be a sampleUnits.threadCPUDelta on this " + "platform"; +# else + EXPECT_FALSE(sampleUnits.isMember("threadCPUDelta")) + << "Unexpected sampleUnits.threadCPUDelta on this platform";; +# endif + } + } + + bool foundMain = false; + bool foundIdle = false; + uint64_t idleThreadCPUDeltaSum = 0u; + bool foundBusy = false; + uint64_t busyThreadCPUDeltaSum = 0u; + + // Check that the sample schema contains "threadCPUDelta". + GET_JSON(threads, aRoot["threads"], Array); + for (const Json::Value& thread : threads) { + ASSERT_TRUE(thread.isObject()); + GET_JSON(name, thread["name"], String); + if (name.asString() == "GeckoMain") { + foundMain = true; + GET_JSON(samples, thread["samples"], Object); + { + Json::ArrayIndex stackIndex = 0; + Json::ArrayIndex threadCPUDeltaIndex = 0; + GET_JSON(schema, samples["schema"], Object); + { + GET_JSON(jsonStackIndex, schema["stack"], UInt); + stackIndex = jsonStackIndex.asUInt(); + GET_JSON(jsonThreadCPUDeltaIndex, schema["threadCPUDelta"], UInt); + threadCPUDeltaIndex = jsonThreadCPUDeltaIndex.asUInt(); + } + + std::set<uint64_t> stackLeaves; // To count distinct leaves. + unsigned threadCPUDeltaCount = 0; + GET_JSON(data, samples["data"], Array); + if (testWithNoStackSampling) { + // When not sampling stacks, the first sampling loop will have no + // running times, so it won't output anything. + EXPECT_GE(data.size(), scMinSamplings - 1); + } else { + EXPECT_GE(data.size(), scMinSamplings); + } + for (const Json::Value& sample : data) { + ASSERT_TRUE(sample.isArray()); + if (sample.isValidIndex(stackIndex)) { + if (!sample[stackIndex].isNull()) { + GET_JSON(stack, sample[stackIndex], UInt64); + stackLeaves.insert(stack.asUInt64()); + } + } + if (sample.isValidIndex(threadCPUDeltaIndex)) { + if (!sample[threadCPUDeltaIndex].isNull()) { + EXPECT_TRUE(sample[threadCPUDeltaIndex].isUInt64()); + ++threadCPUDeltaCount; + } + } + } + + if (testWithNoStackSampling) { + // in nostacksampling mode, there should only be one kind of stack + // leaf (the root). + EXPECT_EQ(stackLeaves.size(), 1u); + } else { + // in normal sampling mode, there should be at least one kind of + // stack leaf for each distinct label. + EXPECT_GE(stackLeaves.size(), scSampleLabelCount); + } + +# if defined(GP_OS_windows) || defined(GP_OS_darwin) || \ + defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) + EXPECT_GE(threadCPUDeltaCount, data.size() - 1u) + << "There should be 'threadCPUDelta' values in all but 1 " + "samples"; +# else + // All "threadCPUDelta" data should be absent or null on unsupported + // platforms. + EXPECT_EQ(threadCPUDeltaCount, 0u); +# endif + } + } else if (name.asString() == "Idle test") { + foundIdle = true; + size_t samplings; + CountCPUDeltas(thread, samplings, idleThreadCPUDeltaSum); + if (testWithNoStackSampling) { + // When not sampling stacks, the first sampling loop will have no + // running times, so it won't output anything. + EXPECT_GE(samplings, scMinSamplings - 1); + } else { + EXPECT_GE(samplings, scMinSamplings); + } +# if !(defined(GP_OS_windows) || defined(GP_OS_darwin) || \ + defined(GP_OS_linux) || defined(GP_OS_android) || \ + defined(GP_OS_freebsd)) + // All "threadCPUDelta" data should be absent or null on unsupported + // platforms. + EXPECT_EQ(idleThreadCPUDeltaSum, 0u); +# endif + } else if (name.asString() == "Busy test") { + foundBusy = true; + size_t samplings; + CountCPUDeltas(thread, samplings, busyThreadCPUDeltaSum); + if (testWithNoStackSampling) { + // When not sampling stacks, the first sampling loop will have no + // running times, so it won't output anything. + EXPECT_GE(samplings, scMinSamplings - 1); + } else { + EXPECT_GE(samplings, scMinSamplings); + } +# if !(defined(GP_OS_windows) || defined(GP_OS_darwin) || \ + defined(GP_OS_linux) || defined(GP_OS_android) || \ + defined(GP_OS_freebsd)) + // All "threadCPUDelta" data should be absent or null on unsupported + // platforms. + EXPECT_EQ(busyThreadCPUDeltaSum, 0u); +# endif + } + } + + EXPECT_TRUE(foundMain); + EXPECT_TRUE(foundIdle); + EXPECT_TRUE(foundBusy); + EXPECT_LE(idleThreadCPUDeltaSum, busyThreadCPUDeltaSum); + }); + + // Note: There is no non-racy way to test for SamplingState::JustStopped, as + // it would require coordination between `profiler_stop()` and another + // thread doing `profiler_callback_after_sampling()` at just the right + // moment. + + profiler_stop(); + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_callback_after_sampling( + [&](SamplingState) { ASSERT_TRUE(false); })); + } + + testThreadsState = TestThreadsState::STOPPING; + busy.join(); + idle.join(); +} + +TEST(GeckoProfiler, AllThreads) +{ + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + ASSERT_EQ(static_cast<uint32_t>(ThreadProfilingFeatures::Any), 1u + 2u + 4u) + << "This test assumes that there are 3 binary choices 1+2+4; " + "Is this test up to date?"; + + for (uint32_t threadFeaturesBinary = 0u; + threadFeaturesBinary <= + static_cast<uint32_t>(ThreadProfilingFeatures::Any); + ++threadFeaturesBinary) { + ThreadProfilingFeatures threadFeatures = + static_cast<ThreadProfilingFeatures>(threadFeaturesBinary); + const bool threadCPU = DoFeaturesIntersect( + threadFeatures, ThreadProfilingFeatures::CPUUtilization); + const bool threadSampling = + DoFeaturesIntersect(threadFeatures, ThreadProfilingFeatures::Sampling); + const bool threadMarkers = + DoFeaturesIntersect(threadFeatures, ThreadProfilingFeatures::Markers); + + ASSERT_TRUE(!profiler_is_active()); + + uint32_t features = ProfilerFeature::StackWalk; + std::string featuresString = "Features: StackWalk Threads"; + if (threadCPU) { + features |= ProfilerFeature::CPUAllThreads; + featuresString += " CPUAllThreads"; + } + if (threadSampling) { + features |= ProfilerFeature::SamplingAllThreads; + featuresString += " SamplingAllThreads"; + } + if (threadMarkers) { + features |= ProfilerFeature::MarkersAllThreads; + featuresString += " MarkersAllThreads"; + } + + SCOPED_TRACE(featuresString.c_str()); + + const char* filters[] = {"GeckoMain", "Selected"}; + + EXPECT_FALSE(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization)); + EXPECT_FALSE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling)); + EXPECT_FALSE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers)); + EXPECT_FALSE(profiler_thread_is_being_profiled_for_markers()); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + EXPECT_TRUE(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization)); + EXPECT_TRUE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling)); + EXPECT_TRUE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers)); + EXPECT_TRUE(profiler_thread_is_being_profiled_for_markers()); + + // This will signal all threads to stop spinning. + Atomic<bool> stopThreads{false}; + + Atomic<int> selectedThreadSpins{0}; + std::thread selectedThread([&]() { + AUTO_PROFILER_REGISTER_THREAD("Selected test thread"); + // Add a label to ensure that we have a non-empty stack, even if native + // stack-walking is not available. + AUTO_PROFILER_LABEL("Selected test thread", PROFILER); + EXPECT_TRUE(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization)); + EXPECT_TRUE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling)); + EXPECT_TRUE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers)); + EXPECT_TRUE(profiler_thread_is_being_profiled_for_markers()); + while (!stopThreads) { + PROFILER_MARKER_UNTYPED("Spinning Selected!", PROFILER); + ++selectedThreadSpins; + PR_Sleep(PR_MillisecondsToInterval(1)); + } + }); + + Atomic<int> unselectedThreadSpins{0}; + std::thread unselectedThread([&]() { + AUTO_PROFILER_REGISTER_THREAD("Registered test thread"); + // Add a label to ensure that we have a non-empty stack, even if native + // stack-walking is not available. + AUTO_PROFILER_LABEL("Registered test thread", PROFILER); + // This thread is *not* selected for full profiling, but it may still be + // profiled depending on the -allthreads features. + EXPECT_EQ(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization), + threadCPU); + EXPECT_EQ( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling), + threadSampling); + EXPECT_EQ( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers), + threadMarkers); + EXPECT_EQ(profiler_thread_is_being_profiled_for_markers(), threadMarkers); + while (!stopThreads) { + PROFILER_MARKER_UNTYPED("Spinning Registered!", PROFILER); + ++unselectedThreadSpins; + PR_Sleep(PR_MillisecondsToInterval(1)); + } + }); + + Atomic<int> unregisteredThreadSpins{0}; + std::thread unregisteredThread([&]() { + // No `AUTO_PROFILER_REGISTER_THREAD` here. + EXPECT_FALSE(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization)); + EXPECT_FALSE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling)); + EXPECT_FALSE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers)); + EXPECT_FALSE(profiler_thread_is_being_profiled_for_markers()); + while (!stopThreads) { + PROFILER_MARKER_UNTYPED("Spinning Unregistered!", PROFILER); + ++unregisteredThreadSpins; + PR_Sleep(PR_MillisecondsToInterval(1)); + } + }); + + // Wait for all threads to have started at least one spin. + while (selectedThreadSpins == 0 || unselectedThreadSpins == 0 || + unregisteredThreadSpins == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + + // Wait until the sampler has done at least one loop. + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + + // Restart the spin counts, and ensure each threads will do at least one + // more spin each. Since spins are increased after PROFILER_MARKER calls, in + // the worst case, each thread will have attempted to record at least one + // marker. + selectedThreadSpins = 0; + unselectedThreadSpins = 0; + unregisteredThreadSpins = 0; + while (selectedThreadSpins < 1 && unselectedThreadSpins < 1 && + unregisteredThreadSpins < 1) { + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + + profiler_pause(); + UniquePtr<char[]> profile = profiler_get_profile(); + + profiler_stop(); + stopThreads = true; + unregisteredThread.join(); + unselectedThread.join(); + selectedThread.join(); + + JSONOutputCheck(profile.get(), [&](const Json::Value& aRoot) { + GET_JSON(threads, aRoot["threads"], Array); + int foundMain = 0; + int foundSelected = 0; + int foundSelectedMarker = 0; + int foundUnselected = 0; + int foundUnselectedMarker = 0; + for (const Json::Value& thread : threads) { + ASSERT_TRUE(thread.isObject()); + GET_JSON(stringTable, thread["stringTable"], Array); + GET_JSON(name, thread["name"], String); + if (name.asString() == "GeckoMain") { + ++foundMain; + // Don't check the main thread further in this test. + + } else if (name.asString() == "Selected test thread") { + ++foundSelected; + + GET_JSON(samples, thread["samples"], Object); + GET_JSON(samplesData, samples["data"], Array); + EXPECT_GT(samplesData.size(), 0u); + + GET_JSON(markers, thread["markers"], Object); + GET_JSON(markersData, markers["data"], Array); + for (const Json::Value& marker : markersData) { + const unsigned int NAME = 0u; + ASSERT_TRUE(marker[NAME].isUInt()); // name id + GET_JSON(name, stringTable[marker[NAME].asUInt()], String); + if (name == "Spinning Selected!") { + ++foundSelectedMarker; + } + } + } else if (name.asString() == "Registered test thread") { + ++foundUnselected; + + GET_JSON(samples, thread["samples"], Object); + GET_JSON(samplesData, samples["data"], Array); + if (threadCPU || threadSampling) { + EXPECT_GT(samplesData.size(), 0u); + } else { + EXPECT_EQ(samplesData.size(), 0u); + } + + GET_JSON(markers, thread["markers"], Object); + GET_JSON(markersData, markers["data"], Array); + for (const Json::Value& marker : markersData) { + const unsigned int NAME = 0u; + ASSERT_TRUE(marker[NAME].isUInt()); // name id + GET_JSON(name, stringTable[marker[NAME].asUInt()], String); + if (name == "Spinning Registered!") { + ++foundUnselectedMarker; + } + } + + } else { + EXPECT_STRNE(name.asString().c_str(), + "Unregistered test thread label"); + } + } + EXPECT_EQ(foundMain, 1); + EXPECT_EQ(foundSelected, 1); + EXPECT_GT(foundSelectedMarker, 0); + EXPECT_EQ(foundUnselected, + (threadCPU || threadSampling || threadMarkers) ? 1 : 0) + << "Unselected thread should only be present if at least one of the " + "allthreads feature is on"; + if (threadMarkers) { + EXPECT_GT(foundUnselectedMarker, 0); + } else { + EXPECT_EQ(foundUnselectedMarker, 0); + } + }); + } +} + +TEST(GeckoProfiler, FailureHandling) +{ + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + + // User-defined marker type that generates a failure when streaming JSON. + struct GtestFailingMarker { + static constexpr Span<const char> MarkerTypeName() { + return MakeStringSpan("markers-gtest-failing"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter) { + aWriter.SetFailure("boom!"); + } + static mozilla::MarkerSchema MarkerTypeDisplay() { + return mozilla::MarkerSchema::SpecialFrontendLocation{}; + } + }; + EXPECT_TRUE(profiler_add_marker_impl("Gtest failing marker", + geckoprofiler::category::OTHER, {}, + GtestFailingMarker{})); + + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + profiler_pause(); + + FailureLatchSource failureLatch; + SpliceableChunkedJSONWriter w{failureLatch}; + EXPECT_FALSE(w.Failed()); + ASSERT_FALSE(w.GetFailure()); + + w.Start(); + EXPECT_FALSE(w.Failed()); + ASSERT_FALSE(w.GetFailure()); + + // The marker will cause a failure during this function call. + EXPECT_FALSE(::profiler_stream_json_for_this_process(w).isOk()); + EXPECT_TRUE(w.Failed()); + ASSERT_TRUE(w.GetFailure()); + EXPECT_EQ(strcmp(w.GetFailure(), "boom!"), 0); + + // Already failed, check that we don't crash or reset the failure. + EXPECT_FALSE(::profiler_stream_json_for_this_process(w).isOk()); + EXPECT_TRUE(w.Failed()); + ASSERT_TRUE(w.GetFailure()); + EXPECT_EQ(strcmp(w.GetFailure(), "boom!"), 0); + + w.End(); + + profiler_stop(); + + EXPECT_TRUE(w.Failed()); + ASSERT_TRUE(w.GetFailure()); + EXPECT_EQ(strcmp(w.GetFailure(), "boom!"), 0); + + UniquePtr<char[]> profile = w.ChunkedWriteFunc().CopyData(); + ASSERT_EQ(profile.get(), nullptr); +} + +TEST(GeckoProfiler, NoMarkerStacks) +{ + uint32_t features = ProfilerFeature::NoMarkerStacks; + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!profiler_get_profile()); + + // Make sure that profiler_capture_backtrace returns nullptr when the profiler + // is not active. + ASSERT_TRUE(!profiler_capture_backtrace()); + + { + // Start the profiler without the NoMarkerStacks feature and make sure we + // capture stacks. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + /* features */ 0, filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(profiler_capture_backtrace()); + profiler_stop(); + } + + // Start the profiler without the NoMarkerStacks feature and make sure we + // don't capture stacks. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + // Make sure that the active features has the NoMarkerStacks feature. + mozilla::Maybe<uint32_t> activeFeatures = profiler_features_if_active(); + ASSERT_TRUE(activeFeatures.isSome()); + ASSERT_TRUE(ProfilerFeature::HasNoMarkerStacks(*activeFeatures)); + + // Make sure we don't capture stacks. + ASSERT_TRUE(!profiler_capture_backtrace()); + + // Add a marker with a stack to test. + EXPECT_TRUE(profiler_add_marker_impl( + "Text with stack", geckoprofiler::category::OTHER, MarkerStack::Capture(), + geckoprofiler::markers::TextMarker{}, "")); + + UniquePtr<char[]> profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), [&](const Json::Value& aRoot) { + // Check that the meta.configuration.features array contains + // "nomarkerstacks". + GET_JSON(meta, aRoot["meta"], Object); + { + GET_JSON(configuration, meta["configuration"], Object); + { + GET_JSON(features, configuration["features"], Array); + { + EXPECT_EQ(features.size(), 1u); + EXPECT_JSON_ARRAY_CONTAINS(features, String, "nomarkerstacks"); + } + } + } + + // Make sure that the marker we captured doesn't have a stack. + GET_JSON(threads, aRoot["threads"], Array); + { + ASSERT_EQ(threads.size(), 1u); + GET_JSON(thread0, threads[0], Object); + { + GET_JSON(markers, thread0["markers"], Object); + { + GET_JSON(data, markers["data"], Array); + { + const unsigned int NAME = 0u; + const unsigned int PAYLOAD = 5u; + bool foundMarker = false; + GET_JSON(stringTable, thread0["stringTable"], Array); + + for (const Json::Value& marker : data) { + // Even though we only added one marker, some markers like + // NotifyObservers are being added as well. Let's iterate over + // them and make sure that we have the one we added explicitly and + // check its stack doesn't exist. + GET_JSON(name, stringTable[marker[NAME].asUInt()], String); + std::string nameString = name.asString(); + + if (nameString == "Text with stack") { + // Make sure that the marker doesn't have a stack. + foundMarker = true; + EXPECT_FALSE(marker[PAYLOAD].isNull()); + EXPECT_TRUE(marker[PAYLOAD]["stack"].isNull()); + } + } + + EXPECT_TRUE(foundMarker); + } + } + } + } + }); + + profiler_stop(); + + ASSERT_TRUE(!profiler_get_profile()); +} + +#endif // MOZ_GECKO_PROFILER diff --git a/tools/profiler/tests/gtest/LulTest.cpp b/tools/profiler/tests/gtest/LulTest.cpp new file mode 100644 index 0000000000..159a366567 --- /dev/null +++ b/tools/profiler/tests/gtest/LulTest.cpp @@ -0,0 +1,51 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "mozilla/Atomics.h" +#include "LulMain.h" +#include "GeckoProfiler.h" // for TracingKind +#include "platform-linux-lul.h" // for read_procmaps + +// Set this to 0 to make LUL be completely silent during tests. +// Set it to 1 to get logging output from LUL, presumably for +// the purpose of debugging it. +#define DEBUG_LUL_TEST 0 + +// LUL needs a callback for its logging sink. +static void gtest_logging_sink_for_LulIntegration(const char* str) { + if (DEBUG_LUL_TEST == 0) { + return; + } + // Ignore any trailing \n, since LOG will add one anyway. + size_t n = strlen(str); + if (n > 0 && str[n - 1] == '\n') { + char* tmp = strdup(str); + tmp[n - 1] = 0; + fprintf(stderr, "LUL-in-gtest: %s\n", tmp); + free(tmp); + } else { + fprintf(stderr, "LUL-in-gtest: %s\n", str); + } +} + +TEST(LulIntegration, unwind_consistency) +{ + // Set up LUL and get it to read unwind info for libxul.so, which is + // all we care about here, plus (incidentally) practically every + // other object in the process too. + lul::LUL* lul = new lul::LUL(gtest_logging_sink_for_LulIntegration); + read_procmaps(lul); + + // Run unwind tests and receive information about how many there + // were and how many were successful. + lul->EnableUnwinding(); + int nTests = 0, nTestsPassed = 0; + RunLulUnitTests(&nTests, &nTestsPassed, lul); + EXPECT_TRUE(nTests == 6) << "Unexpected number of tests"; + EXPECT_EQ(nTestsPassed, nTests) << "Not all tests passed"; + + delete lul; +} diff --git a/tools/profiler/tests/gtest/LulTestDwarf.cpp b/tools/profiler/tests/gtest/LulTestDwarf.cpp new file mode 100644 index 0000000000..d6460c7597 --- /dev/null +++ b/tools/profiler/tests/gtest/LulTestDwarf.cpp @@ -0,0 +1,2733 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "LulCommonExt.h" +#include "LulDwarfExt.h" +#include "LulDwarfInt.h" +#include "LulTestInfrastructure.h" + +using lul_test::CFISection; +using lul_test::test_assembler::kBigEndian; +using lul_test::test_assembler::kLittleEndian; +using lul_test::test_assembler::Label; +using testing::_; +using testing::InSequence; +using testing::Return; +using testing::Sequence; +using testing::Test; + +#define PERHAPS_WRITE_DEBUG_FRAME_FILE(name, section) /**/ +#define PERHAPS_WRITE_EH_FRAME_FILE(name, section) /**/ + +// Set this to 0 to make LUL be completely silent during tests. +// Set it to 1 to get logging output from LUL, presumably for +// the purpose of debugging it. +#define DEBUG_LUL_TEST_DWARF 0 + +// LUL needs a callback for its logging sink. +static void gtest_logging_sink_for_LulTestDwarf(const char* str) { + if (DEBUG_LUL_TEST_DWARF == 0) { + return; + } + // Ignore any trailing \n, since LOG will add one anyway. + size_t n = strlen(str); + if (n > 0 && str[n - 1] == '\n') { + char* tmp = strdup(str); + tmp[n - 1] = 0; + fprintf(stderr, "LUL-in-gtest: %s\n", tmp); + free(tmp); + } else { + fprintf(stderr, "LUL-in-gtest: %s\n", str); + } +} + +namespace lul { + +class MockCallFrameInfoHandler : public CallFrameInfo::Handler { + public: + MOCK_METHOD6(Entry, + bool(size_t offset, uint64 address, uint64 length, uint8 version, + const std::string& augmentation, unsigned return_address)); + MOCK_METHOD2(UndefinedRule, bool(uint64 address, int reg)); + MOCK_METHOD2(SameValueRule, bool(uint64 address, int reg)); + MOCK_METHOD4(OffsetRule, + bool(uint64 address, int reg, int base_register, long offset)); + MOCK_METHOD4(ValOffsetRule, + bool(uint64 address, int reg, int base_register, long offset)); + MOCK_METHOD3(RegisterRule, bool(uint64 address, int reg, int base_register)); + MOCK_METHOD3(ExpressionRule, + bool(uint64 address, int reg, const ImageSlice& expression)); + MOCK_METHOD3(ValExpressionRule, + bool(uint64 address, int reg, const ImageSlice& expression)); + MOCK_METHOD0(End, bool()); + MOCK_METHOD2(PersonalityRoutine, bool(uint64 address, bool indirect)); + MOCK_METHOD2(LanguageSpecificDataArea, bool(uint64 address, bool indirect)); + MOCK_METHOD0(SignalHandler, bool()); +}; + +class MockCallFrameErrorReporter : public CallFrameInfo::Reporter { + public: + MockCallFrameErrorReporter() + : Reporter(gtest_logging_sink_for_LulTestDwarf, "mock filename", + "mock section") {} + MOCK_METHOD2(Incomplete, void(uint64, CallFrameInfo::EntryKind)); + MOCK_METHOD1(EarlyEHTerminator, void(uint64)); + MOCK_METHOD2(CIEPointerOutOfRange, void(uint64, uint64)); + MOCK_METHOD2(BadCIEId, void(uint64, uint64)); + MOCK_METHOD2(UnrecognizedVersion, void(uint64, int version)); + MOCK_METHOD2(UnrecognizedAugmentation, void(uint64, const string&)); + MOCK_METHOD2(InvalidPointerEncoding, void(uint64, uint8)); + MOCK_METHOD2(UnusablePointerEncoding, void(uint64, uint8)); + MOCK_METHOD2(RestoreInCIE, void(uint64, uint64)); + MOCK_METHOD3(BadInstruction, void(uint64, CallFrameInfo::EntryKind, uint64)); + MOCK_METHOD3(NoCFARule, void(uint64, CallFrameInfo::EntryKind, uint64)); + MOCK_METHOD3(EmptyStateStack, void(uint64, CallFrameInfo::EntryKind, uint64)); + MOCK_METHOD3(ClearingCFARule, void(uint64, CallFrameInfo::EntryKind, uint64)); +}; + +struct CFIFixture { + enum { kCFARegister = CallFrameInfo::Handler::kCFARegister }; + + CFIFixture() { + // Default expectations for the data handler. + // + // - Leave Entry and End without expectations, as it's probably a + // good idea to set those explicitly in each test. + // + // - Expect the *Rule functions to not be called, + // so that each test can simply list the calls they expect. + // + // I gather I could use StrictMock for this, but the manual seems + // to suggest using that only as a last resort, and this isn't so + // bad. + EXPECT_CALL(handler, UndefinedRule(_, _)).Times(0); + EXPECT_CALL(handler, SameValueRule(_, _)).Times(0); + EXPECT_CALL(handler, OffsetRule(_, _, _, _)).Times(0); + EXPECT_CALL(handler, ValOffsetRule(_, _, _, _)).Times(0); + EXPECT_CALL(handler, RegisterRule(_, _, _)).Times(0); + EXPECT_CALL(handler, ExpressionRule(_, _, _)).Times(0); + EXPECT_CALL(handler, ValExpressionRule(_, _, _)).Times(0); + EXPECT_CALL(handler, PersonalityRoutine(_, _)).Times(0); + EXPECT_CALL(handler, LanguageSpecificDataArea(_, _)).Times(0); + EXPECT_CALL(handler, SignalHandler()).Times(0); + + // Default expectations for the error/warning reporer. + EXPECT_CALL(reporter, Incomplete(_, _)).Times(0); + EXPECT_CALL(reporter, EarlyEHTerminator(_)).Times(0); + EXPECT_CALL(reporter, CIEPointerOutOfRange(_, _)).Times(0); + EXPECT_CALL(reporter, BadCIEId(_, _)).Times(0); + EXPECT_CALL(reporter, UnrecognizedVersion(_, _)).Times(0); + EXPECT_CALL(reporter, UnrecognizedAugmentation(_, _)).Times(0); + EXPECT_CALL(reporter, InvalidPointerEncoding(_, _)).Times(0); + EXPECT_CALL(reporter, UnusablePointerEncoding(_, _)).Times(0); + EXPECT_CALL(reporter, RestoreInCIE(_, _)).Times(0); + EXPECT_CALL(reporter, BadInstruction(_, _, _)).Times(0); + EXPECT_CALL(reporter, NoCFARule(_, _, _)).Times(0); + EXPECT_CALL(reporter, EmptyStateStack(_, _, _)).Times(0); + EXPECT_CALL(reporter, ClearingCFARule(_, _, _)).Times(0); + } + + MockCallFrameInfoHandler handler; + MockCallFrameErrorReporter reporter; +}; + +class LulDwarfCFI : public CFIFixture, public Test {}; + +TEST_F(LulDwarfCFI, EmptyRegion) { + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + static const char data[1] = {42}; + + ByteReader reader(ENDIANNESS_BIG); + CallFrameInfo parser(data, 0, &reader, &handler, &reporter); + EXPECT_TRUE(parser.Start()); +} + +TEST_F(LulDwarfCFI, IncompleteLength32) { + CFISection section(kBigEndian, 8); + section + // Not even long enough for an initial length. + .D16(0xa0f) + // Padding to keep valgrind happy. We subtract these off when we + // construct the parser. + .D16(0); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + EXPECT_CALL(reporter, Incomplete(_, CallFrameInfo::kUnknown)) + .WillOnce(Return()); + + string contents; + ASSERT_TRUE(section.GetContents(&contents)); + + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(8); + CallFrameInfo parser(contents.data(), contents.size() - 2, &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +TEST_F(LulDwarfCFI, IncompleteLength64) { + CFISection section(kLittleEndian, 4); + section + // An incomplete 64-bit DWARF initial length. + .D32(0xffffffff) + .D32(0x71fbaec2) + // Padding to keep valgrind happy. We subtract these off when we + // construct the parser. + .D32(0); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + EXPECT_CALL(reporter, Incomplete(_, CallFrameInfo::kUnknown)) + .WillOnce(Return()); + + string contents; + ASSERT_TRUE(section.GetContents(&contents)); + + ByteReader reader(ENDIANNESS_LITTLE); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size() - 4, &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +TEST_F(LulDwarfCFI, IncompleteId32) { + CFISection section(kBigEndian, 8); + section + .D32(3) // Initial length, not long enough for id + .D8(0xd7) + .D8(0xe5) + .D8(0xf1) // incomplete id + .CIEHeader(8727, 3983, 8889, 3, "") + .FinishEntry(); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + EXPECT_CALL(reporter, Incomplete(_, CallFrameInfo::kUnknown)) + .WillOnce(Return()); + + string contents; + ASSERT_TRUE(section.GetContents(&contents)); + + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(8); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +TEST_F(LulDwarfCFI, BadId32) { + CFISection section(kBigEndian, 8); + section + .D32(0x100) // Initial length + .D32(0xe802fade) // bogus ID + .Append(0x100 - 4, 0x42); // make the length true + section.CIEHeader(1672, 9872, 8529, 3, "").FinishEntry(); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + EXPECT_CALL(reporter, CIEPointerOutOfRange(_, 0xe802fade)).WillOnce(Return()); + + string contents; + ASSERT_TRUE(section.GetContents(&contents)); + + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(8); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +// A lone CIE shouldn't cause any handler calls. +TEST_F(LulDwarfCFI, SingleCIE) { + CFISection section(kLittleEndian, 4); + section.CIEHeader(0xffe799a8, 0x3398dcdd, 0x6e9683de, 3, ""); + section.Append(10, lul::DW_CFA_nop); + section.FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("SingleCIE", section); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_LITTLE); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// One FDE, one CIE. +TEST_F(LulDwarfCFI, OneFDE) { + CFISection section(kBigEndian, 4); + Label cie; + section.Mark(&cie) + .CIEHeader(0x4be22f75, 0x2492236e, 0x6b6efb87, 3, "") + .FinishEntry() + .FDEHeader(cie, 0x7714740d, 0x3d5a10cd) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("OneFDE", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0x7714740d, 0x3d5a10cd, 3, "", 0x6b6efb87)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// Two FDEs share a CIE. +TEST_F(LulDwarfCFI, TwoFDEsOneCIE) { + CFISection section(kBigEndian, 4); + Label cie; + section + // First FDE. readelf complains about this one because it makes + // a forward reference to its CIE. + .FDEHeader(cie, 0xa42744df, 0xa3b42121) + .FinishEntry() + // CIE. + .Mark(&cie) + .CIEHeader(0x04f7dc7b, 0x3d00c05f, 0xbd43cb59, 3, "") + .FinishEntry() + // Second FDE. + .FDEHeader(cie, 0x6057d391, 0x700f608d) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("TwoFDEsOneCIE", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0xa42744df, 0xa3b42121, 3, "", 0xbd43cb59)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0x6057d391, 0x700f608d, 3, "", 0xbd43cb59)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// Two FDEs, two CIEs. +TEST_F(LulDwarfCFI, TwoFDEsTwoCIEs) { + CFISection section(kLittleEndian, 8); + Label cie1, cie2; + section + // First CIE. + .Mark(&cie1) + .CIEHeader(0x694d5d45, 0x4233221b, 0xbf45e65a, 3, "") + .FinishEntry() + // First FDE which cites second CIE. readelf complains about + // this one because it makes a forward reference to its CIE. + .FDEHeader(cie2, 0x778b27dfe5871f05ULL, 0x324ace3448070926ULL) + .FinishEntry() + // Second FDE, which cites first CIE. + .FDEHeader(cie1, 0xf6054ca18b10bf5fULL, 0x45fdb970d8bca342ULL) + .FinishEntry() + // Second CIE. + .Mark(&cie2) + .CIEHeader(0xfba3fad7, 0x6287e1fd, 0x61d2c581, 2, "") + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("TwoFDEsTwoCIEs", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0x778b27dfe5871f05ULL, 0x324ace3448070926ULL, + 2, "", 0x61d2c581)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0xf6054ca18b10bf5fULL, 0x45fdb970d8bca342ULL, + 3, "", 0xbf45e65a)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_LITTLE); + reader.SetAddressSize(8); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// An FDE whose CIE specifies a version we don't recognize. +TEST_F(LulDwarfCFI, BadVersion) { + CFISection section(kBigEndian, 4); + Label cie1, cie2; + section.Mark(&cie1) + .CIEHeader(0xca878cf0, 0x7698ec04, 0x7b616f54, 0x52, "") + .FinishEntry() + // We should skip this entry, as its CIE specifies a version we + // don't recognize. + .FDEHeader(cie1, 0x08852292, 0x2204004a) + .FinishEntry() + // Despite the above, we should visit this entry. + .Mark(&cie2) + .CIEHeader(0x7c3ae7c9, 0xb9b9a512, 0x96cb3264, 3, "") + .FinishEntry() + .FDEHeader(cie2, 0x2094735a, 0x6e875501) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("BadVersion", section); + + EXPECT_CALL(reporter, UnrecognizedVersion(_, 0x52)).WillOnce(Return()); + + { + InSequence s; + // We should see no mention of the first FDE, but we should get + // a call to Entry for the second. + EXPECT_CALL(handler, Entry(_, 0x2094735a, 0x6e875501, 3, "", 0x96cb3264)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +// An FDE whose CIE specifies an augmentation we don't recognize. +TEST_F(LulDwarfCFI, BadAugmentation) { + CFISection section(kBigEndian, 4); + Label cie1, cie2; + section.Mark(&cie1) + .CIEHeader(0x4be22f75, 0x2492236e, 0x6b6efb87, 3, "spaniels!") + .FinishEntry() + // We should skip this entry, as its CIE specifies an + // augmentation we don't recognize. + .FDEHeader(cie1, 0x7714740d, 0x3d5a10cd) + .FinishEntry() + // Despite the above, we should visit this entry. + .Mark(&cie2) + .CIEHeader(0xf8bc4399, 0x8cf09931, 0xf2f519b2, 3, "") + .FinishEntry() + .FDEHeader(cie2, 0x7bf0fda0, 0xcbcd28d8) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("BadAugmentation", section); + + EXPECT_CALL(reporter, UnrecognizedAugmentation(_, "spaniels!")) + .WillOnce(Return()); + + { + InSequence s; + // We should see no mention of the first FDE, but we should get + // a call to Entry for the second. + EXPECT_CALL(handler, Entry(_, 0x7bf0fda0, 0xcbcd28d8, 3, "", 0xf2f519b2)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +// The return address column field is a byte in CFI version 1 +// (DWARF2), but a ULEB128 value in version 3 (DWARF3). +TEST_F(LulDwarfCFI, CIEVersion1ReturnColumn) { + CFISection section(kBigEndian, 4); + Label cie; + section + // CIE, using the version 1 format: return column is a ubyte. + .Mark(&cie) + // Use a value for the return column that is parsed differently + // as a ubyte and as a ULEB128. + .CIEHeader(0xbcdea24f, 0x5be28286, 0x9f, 1, "") + .FinishEntry() + // FDE, citing that CIE. + .FDEHeader(cie, 0xb8d347b5, 0x825e55dc) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("CIEVersion1ReturnColumn", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0xb8d347b5, 0x825e55dc, 1, "", 0x9f)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// The return address column field is a byte in CFI version 1 +// (DWARF2), but a ULEB128 value in version 3 (DWARF3). +TEST_F(LulDwarfCFI, CIEVersion3ReturnColumn) { + CFISection section(kBigEndian, 4); + Label cie; + section + // CIE, using the version 3 format: return column is a ULEB128. + .Mark(&cie) + // Use a value for the return column that is parsed differently + // as a ubyte and as a ULEB128. + .CIEHeader(0x0ab4758d, 0xc010fdf7, 0x89, 3, "") + .FinishEntry() + // FDE, citing that CIE. + .FDEHeader(cie, 0x86763f2b, 0x2a66dc23) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("CIEVersion3ReturnColumn", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0x86763f2b, 0x2a66dc23, 3, "", 0x89)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +struct CFIInsnFixture : public CFIFixture { + CFIInsnFixture() { + data_factor = 0xb6f; + return_register = 0x9be1ed9f; + version = 3; + cfa_base_register = 0x383a3aa; + cfa_offset = 0xf748; + } + + // Prepare SECTION to receive FDE instructions. + // + // - Append a stock CIE header that establishes the fixture's + // code_factor, data_factor, return_register, version, and + // augmentation values. + // - Have the CIE set up a CFA rule using cfa_base_register and + // cfa_offset. + // - Append a stock FDE header, referring to the above CIE, for the + // fde_size bytes at fde_start. Choose fde_start and fde_size + // appropriately for the section's address size. + // - Set appropriate expectations on handler in sequence s for the + // frame description entry and the CIE's CFA rule. + // + // On return, SECTION is ready to have FDE instructions appended to + // it, and its FinishEntry member called. + void StockCIEAndFDE(CFISection* section) { + // Choose appropriate constants for our address size. + if (section->AddressSize() == 4) { + fde_start = 0xc628ecfbU; + fde_size = 0x5dee04a2; + code_factor = 0x60b; + } else { + assert(section->AddressSize() == 8); + fde_start = 0x0005c57ce7806bd3ULL; + fde_size = 0x2699521b5e333100ULL; + code_factor = 0x01008e32855274a8ULL; + } + + // Create the CIE. + (*section) + .Mark(&cie_label) + .CIEHeader(code_factor, data_factor, return_register, version, "") + .D8(lul::DW_CFA_def_cfa) + .ULEB128(cfa_base_register) + .ULEB128(cfa_offset) + .FinishEntry(); + + // Create the FDE. + section->FDEHeader(cie_label, fde_start, fde_size); + + // Expect an Entry call for the FDE and a ValOffsetRule call for the + // CIE's CFA rule. + EXPECT_CALL(handler, + Entry(_, fde_start, fde_size, version, "", return_register)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, + cfa_base_register, cfa_offset)) + .InSequence(s) + .WillOnce(Return(true)); + } + + // Run the contents of SECTION through a CallFrameInfo parser, + // expecting parser.Start to return SUCCEEDS. Caller may optionally + // supply, via READER, its own ByteReader. If that's absent, a + // local one is used. + void ParseSection(CFISection* section, bool succeeds = true, + ByteReader* reader = nullptr) { + string contents; + EXPECT_TRUE(section->GetContents(&contents)); + lul::Endianness endianness; + if (section->endianness() == kBigEndian) + endianness = ENDIANNESS_BIG; + else { + assert(section->endianness() == kLittleEndian); + endianness = ENDIANNESS_LITTLE; + } + ByteReader local_reader(endianness); + ByteReader* reader_to_use = reader ? reader : &local_reader; + reader_to_use->SetAddressSize(section->AddressSize()); + CallFrameInfo parser(contents.data(), contents.size(), reader_to_use, + &handler, &reporter); + if (succeeds) + EXPECT_TRUE(parser.Start()); + else + EXPECT_FALSE(parser.Start()); + } + + Label cie_label; + Sequence s; + uint64 code_factor; + int data_factor; + unsigned return_register; + unsigned version; + unsigned cfa_base_register; + int cfa_offset; + uint64 fde_start, fde_size; +}; + +class LulDwarfCFIInsn : public CFIInsnFixture, public Test {}; + +TEST_F(LulDwarfCFIInsn, DW_CFA_set_loc) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_set_loc) + .D32(0xb1ee3e7a) + // Use DW_CFA_def_cfa to force a handler call that we can use to + // check the effect of the DW_CFA_set_loc. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x4defb431) + .ULEB128(0x6d17b0ee) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_set_loc", section); + + EXPECT_CALL(handler, + ValOffsetRule(0xb1ee3e7a, kCFARegister, 0x4defb431, 0x6d17b0ee)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_advance_loc) { + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + section + .D8(lul::DW_CFA_advance_loc | 0x2a) + // Use DW_CFA_def_cfa to force a handler call that we can use to + // check the effect of the DW_CFA_advance_loc. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x5bbb3715) + .ULEB128(0x0186c7bf) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc", section); + + EXPECT_CALL(handler, ValOffsetRule(fde_start + 0x2a * code_factor, + kCFARegister, 0x5bbb3715, 0x0186c7bf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_advance_loc1) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_advance_loc1) + .D8(0xd8) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x69d5696a) + .ULEB128(0x1eb7fc93) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc1", section); + + EXPECT_CALL(handler, ValOffsetRule((fde_start + 0xd8 * code_factor), + kCFARegister, 0x69d5696a, 0x1eb7fc93)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_advance_loc2) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_advance_loc2) + .D16(0x3adb) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x3a368bed) + .ULEB128(0x3194ee37) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc2", section); + + EXPECT_CALL(handler, ValOffsetRule((fde_start + 0x3adb * code_factor), + kCFARegister, 0x3a368bed, 0x3194ee37)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_advance_loc4) { + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_advance_loc4) + .D32(0x15813c88) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x135270c5) + .ULEB128(0x24bad7cb) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc4", section); + + EXPECT_CALL(handler, ValOffsetRule((fde_start + 0x15813c88ULL * code_factor), + kCFARegister, 0x135270c5, 0x24bad7cb)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_MIPS_advance_loc8) { + code_factor = 0x2d; + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_MIPS_advance_loc8) + .D64(0x3c4f3945b92c14ULL) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0xe17ed602) + .ULEB128(0x3d162e7f) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc8", section); + + EXPECT_CALL(handler, + ValOffsetRule((fde_start + 0x3c4f3945b92c14ULL * code_factor), + kCFARegister, 0xe17ed602, 0x3d162e7f)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa) + .ULEB128(0x4e363a85) + .ULEB128(0x815f9aa7) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_def_cfa", section); + + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x4e363a85, 0x815f9aa7)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_sf) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_sf) + .ULEB128(0x8ccb32b7) + .LEB128(0x9ea) + .D8(lul::DW_CFA_def_cfa_sf) + .ULEB128(0x9b40f5da) + .LEB128(-0x40a2) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, 0x8ccb32b7, + 0x9ea * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, 0x9b40f5da, + -0x40a2 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_register) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_register).ULEB128(0x3e7e9363).FinishEntry(); + + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x3e7e9363, cfa_offset)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +// DW_CFA_def_cfa_register should have no effect when applied to a +// non-base/offset rule. +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_registerBadRule) { + ByteReader reader(ENDIANNESS_BIG); + CFISection section(kBigEndian, 4); + ImageSlice expr("needle in a haystack"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_expression) + .Block(expr) + .D8(lul::DW_CFA_def_cfa_register) + .ULEB128(0xf1b49e49) + .FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, kCFARegister, expr)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_offset) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_offset).ULEB128(0x1e8e3b9b).FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, cfa_base_register, + 0x1e8e3b9b)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_offset_sf) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_offset_sf) + .LEB128(0x970) + .D8(lul::DW_CFA_def_cfa_offset_sf) + .LEB128(-0x2cd) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, cfa_base_register, + 0x970 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, cfa_base_register, + -0x2cd * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +// DW_CFA_def_cfa_offset should have no effect when applied to a +// non-base/offset rule. +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_offsetBadRule) { + ByteReader reader(ENDIANNESS_BIG); + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + ImageSlice expr("six ways to Sunday"); + section.D8(lul::DW_CFA_def_cfa_expression) + .Block(expr) + .D8(lul::DW_CFA_def_cfa_offset) + .ULEB128(0x1e8e3b9b) + .FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, kCFARegister, expr)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_expression) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 8); + ImageSlice expr("eating crow"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_expression).Block(expr).FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, kCFARegister, expr)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_undefined) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_undefined).ULEB128(0x300ce45d).FinishEntry(); + + EXPECT_CALL(handler, UndefinedRule(fde_start, 0x300ce45d)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_same_value) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_same_value).ULEB128(0x3865a760).FinishEntry(); + + EXPECT_CALL(handler, SameValueRule(fde_start, 0x3865a760)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_offset) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset | 0x2c).ULEB128(0x9f6).FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x2c, kCFARegister, 0x9f6 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_offset_extended) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset_extended) + .ULEB128(0x402b) + .ULEB128(0xb48) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x402b, kCFARegister, 0xb48 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_offset_extended_sf) { + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset_extended_sf) + .ULEB128(0x997c23ee) + .LEB128(0x2d00) + .D8(lul::DW_CFA_offset_extended_sf) + .ULEB128(0x9519eb82) + .LEB128(-0xa77) + .FinishEntry(); + + EXPECT_CALL(handler, OffsetRule(fde_start, 0x997c23ee, kCFARegister, + 0x2d00 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(fde_start, 0x9519eb82, kCFARegister, + -0xa77 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_val_offset) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset) + .ULEB128(0x623562fe) + .ULEB128(0x673) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x623562fe, kCFARegister, + 0x673 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_val_offset_sf) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset_sf) + .ULEB128(0x6f4f) + .LEB128(0xaab) + .D8(lul::DW_CFA_val_offset_sf) + .ULEB128(0x2483) + .LEB128(-0x8a2) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x6f4f, kCFARegister, + 0xaab * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x2483, kCFARegister, + -0x8a2 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_register) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0x278d18f9) + .ULEB128(0x1a684414) + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0x278d18f9, 0x1a684414)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_expression) { + ByteReader reader(ENDIANNESS_BIG); + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + ImageSlice expr("plus ça change, plus c'est la même chose"); + section.D8(lul::DW_CFA_expression) + .ULEB128(0xa1619fb2) + .Block(expr) + .FinishEntry(); + + EXPECT_CALL(handler, ExpressionRule(fde_start, 0xa1619fb2, expr)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_val_expression) { + ByteReader reader(ENDIANNESS_BIG); + CFISection section(kBigEndian, 4); + ImageSlice expr("he who has the gold makes the rules"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_expression) + .ULEB128(0xc5e4a9e3) + .Block(expr) + .FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, 0xc5e4a9e3, expr)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_restore) { + CFISection section(kLittleEndian, 8); + code_factor = 0x01bd188a9b1fa083ULL; + data_factor = -0x1ac8; + return_register = 0x8c35b049; + version = 2; + fde_start = 0x2d70fe998298bbb1ULL; + fde_size = 0x46ccc2e63cf0b108ULL; + Label cie; + section.Mark(&cie) + .CIEHeader(code_factor, data_factor, return_register, version, "") + // Provide a CFA rule, because register rules require them. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x6ca1d50e) + .ULEB128(0x372e38e8) + // Provide an offset(N) rule for register 0x3c. + .D8(lul::DW_CFA_offset | 0x3c) + .ULEB128(0xb348) + .FinishEntry() + // In the FDE... + .FDEHeader(cie, fde_start, fde_size) + // At a second address, provide a new offset(N) rule for register 0x3c. + .D8(lul::DW_CFA_advance_loc | 0x13) + .D8(lul::DW_CFA_offset | 0x3c) + .ULEB128(0x9a50) + // At a third address, restore the original rule for register 0x3c. + .D8(lul::DW_CFA_advance_loc | 0x01) + .D8(lul::DW_CFA_restore | 0x3c) + .FinishEntry(); + + { + InSequence s; + EXPECT_CALL(handler, + Entry(_, fde_start, fde_size, version, "", return_register)) + .WillOnce(Return(true)); + // CIE's CFA rule. + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x6ca1d50e, 0x372e38e8)) + .WillOnce(Return(true)); + // CIE's rule for register 0x3c. + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x3c, kCFARegister, 0xb348 * data_factor)) + .WillOnce(Return(true)); + // FDE's rule for register 0x3c. + EXPECT_CALL(handler, OffsetRule(fde_start + 0x13 * code_factor, 0x3c, + kCFARegister, 0x9a50 * data_factor)) + .WillOnce(Return(true)); + // Restore CIE's rule for register 0x3c. + EXPECT_CALL(handler, OffsetRule(fde_start + (0x13 + 0x01) * code_factor, + 0x3c, kCFARegister, 0xb348 * data_factor)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_restoreNoRule) { + CFISection section(kBigEndian, 4); + code_factor = 0x005f78143c1c3b82ULL; + data_factor = 0x25d0; + return_register = 0xe8; + version = 1; + fde_start = 0x4062e30f; + fde_size = 0x5302a389; + Label cie; + section.Mark(&cie) + .CIEHeader(code_factor, data_factor, return_register, version, "") + // Provide a CFA rule, because register rules require them. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x470aa334) + .ULEB128(0x099ef127) + .FinishEntry() + // In the FDE... + .FDEHeader(cie, fde_start, fde_size) + // At a second address, provide an offset(N) rule for register 0x2c. + .D8(lul::DW_CFA_advance_loc | 0x7) + .D8(lul::DW_CFA_offset | 0x2c) + .ULEB128(0x1f47) + // At a third address, restore the (missing) CIE rule for register 0x2c. + .D8(lul::DW_CFA_advance_loc | 0xb) + .D8(lul::DW_CFA_restore | 0x2c) + .FinishEntry(); + + { + InSequence s; + EXPECT_CALL(handler, + Entry(_, fde_start, fde_size, version, "", return_register)) + .WillOnce(Return(true)); + // CIE's CFA rule. + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x470aa334, 0x099ef127)) + .WillOnce(Return(true)); + // FDE's rule for register 0x2c. + EXPECT_CALL(handler, OffsetRule(fde_start + 0x7 * code_factor, 0x2c, + kCFARegister, 0x1f47 * data_factor)) + .WillOnce(Return(true)); + // Restore CIE's (missing) rule for register 0x2c. + EXPECT_CALL(handler, + SameValueRule(fde_start + (0x7 + 0xb) * code_factor, 0x2c)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_restore_extended) { + CFISection section(kBigEndian, 4); + code_factor = 0x126e; + data_factor = -0xd8b; + return_register = 0x77711787; + version = 3; + fde_start = 0x01f55a45; + fde_size = 0x452adb80; + Label cie; + section.Mark(&cie) + .CIEHeader(code_factor, data_factor, return_register, version, "", + true /* dwarf64 */) + // Provide a CFA rule, because register rules require them. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x56fa0edd) + .ULEB128(0x097f78a5) + // Provide an offset(N) rule for register 0x0f9b8a1c. + .D8(lul::DW_CFA_offset_extended) + .ULEB128(0x0f9b8a1c) + .ULEB128(0xc979) + .FinishEntry() + // In the FDE... + .FDEHeader(cie, fde_start, fde_size) + // At a second address, provide a new offset(N) rule for reg 0x0f9b8a1c. + .D8(lul::DW_CFA_advance_loc | 0x3) + .D8(lul::DW_CFA_offset_extended) + .ULEB128(0x0f9b8a1c) + .ULEB128(0x3b7b) + // At a third address, restore the original rule for register 0x0f9b8a1c. + .D8(lul::DW_CFA_advance_loc | 0x04) + .D8(lul::DW_CFA_restore_extended) + .ULEB128(0x0f9b8a1c) + .FinishEntry(); + + { + InSequence s; + EXPECT_CALL(handler, + Entry(_, fde_start, fde_size, version, "", return_register)) + .WillOnce(Return(true)); + // CIE's CFA rule. + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x56fa0edd, 0x097f78a5)) + .WillOnce(Return(true)); + // CIE's rule for register 0x0f9b8a1c. + EXPECT_CALL(handler, OffsetRule(fde_start, 0x0f9b8a1c, kCFARegister, + 0xc979 * data_factor)) + .WillOnce(Return(true)); + // FDE's rule for register 0x0f9b8a1c. + EXPECT_CALL(handler, OffsetRule(fde_start + 0x3 * code_factor, 0x0f9b8a1c, + kCFARegister, 0x3b7b * data_factor)) + .WillOnce(Return(true)); + // Restore CIE's rule for register 0x0f9b8a1c. + EXPECT_CALL(handler, + OffsetRule(fde_start + (0x3 + 0x4) * code_factor, 0x0f9b8a1c, + kCFARegister, 0xc979 * data_factor)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_remember_and_restore_state) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + + // We create a state, save it, modify it, and then restore. We + // refer to the state that is overridden the restore as the + // "outgoing" state, and the restored state the "incoming" state. + // + // Register outgoing incoming expect + // 1 offset(N) no rule new "same value" rule + // 2 register(R) offset(N) report changed rule + // 3 offset(N) offset(M) report changed offset + // 4 offset(N) offset(N) no report + // 5 offset(N) no rule new "same value" rule + section + // Create the "incoming" state, which we will save and later restore. + .D8(lul::DW_CFA_offset | 2) + .ULEB128(0x9806) + .D8(lul::DW_CFA_offset | 3) + .ULEB128(0x995d) + .D8(lul::DW_CFA_offset | 4) + .ULEB128(0x7055) + .D8(lul::DW_CFA_remember_state) + // Advance to a new instruction; an implementation could legitimately + // ignore all but the final rule for a given register at a given address. + .D8(lul::DW_CFA_advance_loc | 1) + // Create the "outgoing" state, which we will discard. + .D8(lul::DW_CFA_offset | 1) + .ULEB128(0xea1a) + .D8(lul::DW_CFA_register) + .ULEB128(2) + .ULEB128(0x1d2a3767) + .D8(lul::DW_CFA_offset | 3) + .ULEB128(0xdd29) + .D8(lul::DW_CFA_offset | 5) + .ULEB128(0xf1ce) + // At a third address, restore the incoming state. + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + uint64 addr = fde_start; + + // Expect the incoming rules to be reported. + EXPECT_CALL(handler, OffsetRule(addr, 2, kCFARegister, 0x9806 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 3, kCFARegister, 0x995d * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 4, kCFARegister, 0x7055 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + + addr += code_factor; + + // After the save, we establish the outgoing rule set. + EXPECT_CALL(handler, OffsetRule(addr, 1, kCFARegister, 0xea1a * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, RegisterRule(addr, 2, 0x1d2a3767)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 3, kCFARegister, 0xdd29 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 5, kCFARegister, 0xf1ce * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + + addr += code_factor; + + // Finally, after the restore, expect to see the differences from + // the outgoing to the incoming rules reported. + EXPECT_CALL(handler, SameValueRule(addr, 1)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 2, kCFARegister, 0x9806 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 3, kCFARegister, 0x995d * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SameValueRule(addr, 5)) + .InSequence(s) + .WillOnce(Return(true)); + + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +// Check that restoring a rule set reports changes to the CFA rule. +TEST_F(LulDwarfCFIInsn, DW_CFA_remember_and_restore_stateCFA) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + + section.D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_def_cfa_offset) + .ULEB128(0x90481102) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start + code_factor, kCFARegister, + cfa_base_register, 0x90481102)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start + code_factor * 2, kCFARegister, + cfa_base_register, cfa_offset)) + .InSequence(s) + .WillOnce(Return(true)); + + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_nop) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_nop) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x3fb8d4f1) + .ULEB128(0x078dc67b) + .D8(lul::DW_CFA_nop) + .FinishEntry(); + + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x3fb8d4f1, 0x078dc67b)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_GNU_window_save) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_GNU_window_save).FinishEntry(); + + // Don't include all the rules in any particular sequence. + + // The caller's %o0-%o7 have become the callee's %i0-%i7. This is + // the GCC register numbering. + for (int i = 8; i < 16; i++) + EXPECT_CALL(handler, RegisterRule(fde_start, i, i + 16)) + .WillOnce(Return(true)); + // The caller's %l0-%l7 and %i0-%i7 have been saved at the top of + // its frame. + for (int i = 16; i < 32; i++) + EXPECT_CALL(handler, OffsetRule(fde_start, i, kCFARegister, (i - 16) * 4)) + .WillOnce(Return(true)); + + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_GNU_args_size) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_GNU_args_size) + .ULEB128(0xeddfa520) + // Verify that we see this, meaning we parsed the above properly. + .D8(lul::DW_CFA_offset | 0x23) + .ULEB128(0x269) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x23, kCFARegister, 0x269 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_GNU_negative_offset_extended) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_GNU_negative_offset_extended) + .ULEB128(0x430cc87a) + .ULEB128(0x613) + .FinishEntry(); + + EXPECT_CALL(handler, OffsetRule(fde_start, 0x430cc87a, kCFARegister, + -0x613 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +// Three FDEs: skip the second +TEST_F(LulDwarfCFIInsn, SkipFDE) { + CFISection section(kBigEndian, 4); + Label cie; + section + // CIE, used by all FDEs. + .Mark(&cie) + .CIEHeader(0x010269f2, 0x9177, 0xedca5849, 2, "") + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x42ed390b) + .ULEB128(0x98f43aad) + .FinishEntry() + // First FDE. + .FDEHeader(cie, 0xa870ebdd, 0x60f6aa4) + .D8(lul::DW_CFA_register) + .ULEB128(0x3a860351) + .ULEB128(0x6c9a6bcf) + .FinishEntry() + // Second FDE. + .FDEHeader(cie, 0xc534f7c0, 0xf6552e9, true /* dwarf64 */) + .D8(lul::DW_CFA_register) + .ULEB128(0x1b62c234) + .ULEB128(0x26586b18) + .FinishEntry() + // Third FDE. + .FDEHeader(cie, 0xf681cfc8, 0x7e4594e) + .D8(lul::DW_CFA_register) + .ULEB128(0x26c53934) + .ULEB128(0x18eeb8a4) + .FinishEntry(); + + { + InSequence s; + + // Process the first FDE. + EXPECT_CALL(handler, Entry(_, 0xa870ebdd, 0x60f6aa4, 2, "", 0xedca5849)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + ValOffsetRule(0xa870ebdd, kCFARegister, 0x42ed390b, 0x98f43aad)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, RegisterRule(0xa870ebdd, 0x3a860351, 0x6c9a6bcf)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + // Skip the second FDE. + EXPECT_CALL(handler, Entry(_, 0xc534f7c0, 0xf6552e9, 2, "", 0xedca5849)) + .WillOnce(Return(false)); + + // Process the third FDE. + EXPECT_CALL(handler, Entry(_, 0xf681cfc8, 0x7e4594e, 2, "", 0xedca5849)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + ValOffsetRule(0xf681cfc8, kCFARegister, 0x42ed390b, 0x98f43aad)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, RegisterRule(0xf681cfc8, 0x26c53934, 0x18eeb8a4)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + ParseSection(§ion); +} + +// Quit processing in the middle of an entry's instructions. +TEST_F(LulDwarfCFIInsn, QuitMidentry) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0xe0cf850d) + .ULEB128(0x15aab431) + .D8(lul::DW_CFA_expression) + .ULEB128(0x46750aa5) + .Block("meat") + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0xe0cf850d, 0x15aab431)) + .InSequence(s) + .WillOnce(Return(false)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, false); +} + +class LulDwarfCFIRestore : public CFIInsnFixture, public Test {}; + +TEST_F(LulDwarfCFIRestore, RestoreUndefinedRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_undefined) + .ULEB128(0x0bac878e) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, UndefinedRule(fde_start, 0x0bac878e)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreUndefinedRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_undefined) + .ULEB128(0x7dedff5f) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_same_value) + .ULEB128(0x7dedff5f) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, UndefinedRule(fde_start, 0x7dedff5f)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SameValueRule(fde_start + code_factor, 0x7dedff5f)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + 2 * code_factor, 0x7dedff5f)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreSameValueRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_same_value) + .ULEB128(0xadbc9b3a) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, SameValueRule(fde_start, 0xadbc9b3a)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreSameValueRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_same_value) + .ULEB128(0x3d90dcb5) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0x3d90dcb5) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, SameValueRule(fde_start, 0x3d90dcb5)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0x3d90dcb5)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SameValueRule(fde_start + 2 * code_factor, 0x3d90dcb5)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreOffsetRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset | 0x14) + .ULEB128(0xb6f) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x14, kCFARegister, 0xb6f * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreOffsetRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset | 0x21) + .ULEB128(0xeb7) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0x21) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x21, kCFARegister, 0xeb7 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0x21)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(fde_start + 2 * code_factor, 0x21, + kCFARegister, 0xeb7 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreOffsetRuleChangedOffset) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset | 0x21) + .ULEB128(0x134) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_offset | 0x21) + .ULEB128(0xf4f) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x21, kCFARegister, 0x134 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(fde_start + code_factor, 0x21, kCFARegister, + 0xf4f * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(fde_start + 2 * code_factor, 0x21, + kCFARegister, 0x134 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreValOffsetRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset) + .ULEB128(0x829caee6) + .ULEB128(0xe4c) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x829caee6, kCFARegister, + 0xe4c * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreValOffsetRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset) + .ULEB128(0xf17c36d6) + .ULEB128(0xeb7) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0xf17c36d6) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0xf17c36d6, kCFARegister, + 0xeb7 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0xf17c36d6)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start + 2 * code_factor, 0xf17c36d6, + kCFARegister, 0xeb7 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreValOffsetRuleChangedValOffset) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset) + .ULEB128(0x2cf0ab1b) + .ULEB128(0x562) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_val_offset) + .ULEB128(0x2cf0ab1b) + .ULEB128(0xe88) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x2cf0ab1b, kCFARegister, + 0x562 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start + code_factor, 0x2cf0ab1b, + kCFARegister, 0xe88 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start + 2 * code_factor, 0x2cf0ab1b, + kCFARegister, 0x562 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreRegisterRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0x77514acc) + .ULEB128(0x464de4ce) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0x77514acc, 0x464de4ce)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreRegisterRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0xe39acce5) + .ULEB128(0x095f1559) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0xe39acce5) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0xe39acce5, 0x095f1559)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0xe39acce5)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + RegisterRule(fde_start + 2 * code_factor, 0xe39acce5, 0x095f1559)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreRegisterRuleChangedRegister) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0xd40e21b1) + .ULEB128(0x16607d6a) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_register) + .ULEB128(0xd40e21b1) + .ULEB128(0xbabb4742) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0xd40e21b1, 0x16607d6a)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + RegisterRule(fde_start + code_factor, 0xd40e21b1, 0xbabb4742)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + RegisterRule(fde_start + 2 * code_factor, 0xd40e21b1, 0x16607d6a)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreExpressionRuleUnchanged) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice dwarf("dwarf"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_expression) + .ULEB128(0x666ae152) + .Block("dwarf") + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ExpressionRule(fde_start, 0x666ae152, dwarf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreExpressionRuleChanged) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice elf("elf"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_expression) + .ULEB128(0xb5ca5c46) + .Block(elf) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0xb5ca5c46) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ExpressionRule(fde_start, 0xb5ca5c46, elf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0xb5ca5c46)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + ExpressionRule(fde_start + 2 * code_factor, 0xb5ca5c46, elf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreExpressionRuleChangedExpression) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + ImageSlice smurf("smurf"); + ImageSlice orc("orc"); + section.D8(lul::DW_CFA_expression) + .ULEB128(0x500f5739) + .Block(smurf) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_expression) + .ULEB128(0x500f5739) + .Block(orc) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ExpressionRule(fde_start, 0x500f5739, smurf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ExpressionRule(fde_start + code_factor, 0x500f5739, orc)) + .InSequence(s) + .WillOnce(Return(true)); + // Expectations are not wishes. + EXPECT_CALL(handler, + ExpressionRule(fde_start + 2 * code_factor, 0x500f5739, smurf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreValExpressionRuleUnchanged) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice hideous("hideous"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_expression) + .ULEB128(0x666ae152) + .Block(hideous) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, 0x666ae152, hideous)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreValExpressionRuleChanged) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice revolting("revolting"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_expression) + .ULEB128(0xb5ca5c46) + .Block(revolting) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0xb5ca5c46) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("RestoreValExpressionRuleChanged", section); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, 0xb5ca5c46, revolting)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0xb5ca5c46)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValExpressionRule(fde_start + 2 * code_factor, + 0xb5ca5c46, revolting)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreValExpressionRuleChangedValExpression) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice repulsive("repulsive"); + ImageSlice nauseous("nauseous"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_expression) + .ULEB128(0x500f5739) + .Block(repulsive) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_val_expression) + .ULEB128(0x500f5739) + .Block(nauseous) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("RestoreValExpressionRuleChangedValExpression", + section); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, 0x500f5739, repulsive)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + ValExpressionRule(fde_start + code_factor, 0x500f5739, nauseous)) + .InSequence(s) + .WillOnce(Return(true)); + // Expectations are not wishes. + EXPECT_CALL(handler, ValExpressionRule(fde_start + 2 * code_factor, + 0x500f5739, repulsive)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +struct EHFrameFixture : public CFIInsnFixture { + EHFrameFixture() : section(kBigEndian, 4, true) { + encoded_pointer_bases.cfi = 0x7f496cb2; + encoded_pointer_bases.text = 0x540f67b6; + encoded_pointer_bases.data = 0xe3eab768; + section.SetEncodedPointerBases(encoded_pointer_bases); + } + CFISection section; + CFISection::EncodedPointerBases encoded_pointer_bases; + + // Parse CFIInsnFixture::ParseSection, but parse the section as + // .eh_frame data, supplying stock base addresses. + void ParseEHFrameSection(CFISection* section, bool succeeds = true) { + EXPECT_TRUE(section->ContainsEHFrame()); + string contents; + EXPECT_TRUE(section->GetContents(&contents)); + lul::Endianness endianness; + if (section->endianness() == kBigEndian) + endianness = ENDIANNESS_BIG; + else { + assert(section->endianness() == kLittleEndian); + endianness = ENDIANNESS_LITTLE; + } + ByteReader reader(endianness); + reader.SetAddressSize(section->AddressSize()); + reader.SetCFIDataBase(encoded_pointer_bases.cfi, contents.data()); + reader.SetTextBase(encoded_pointer_bases.text); + reader.SetDataBase(encoded_pointer_bases.data); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter, true); + if (succeeds) + EXPECT_TRUE(parser.Start()); + else + EXPECT_FALSE(parser.Start()); + } +}; + +class LulDwarfEHFrame : public EHFrameFixture, public Test {}; + +// A simple CIE, an FDE, and a terminator. +TEST_F(LulDwarfEHFrame, Terminator) { + Label cie; + section.Mark(&cie) + .CIEHeader(9968, 2466, 67, 1, "") + .D8(lul::DW_CFA_def_cfa) + .ULEB128(3772) + .ULEB128(1372) + .FinishEntry() + .FDEHeader(cie, 0x848037a1, 0x7b30475e) + .D8(lul::DW_CFA_set_loc) + .D32(0x17713850) + .D8(lul::DW_CFA_undefined) + .ULEB128(5721) + .FinishEntry() + .D32(0) // Terminate the sequence. + // This FDE should be ignored. + .FDEHeader(cie, 0xf19629fe, 0x439fb09b) + .FinishEntry(); + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.Terminator", section); + + EXPECT_CALL(handler, Entry(_, 0x848037a1, 0x7b30475e, 1, "", 67)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(0x848037a1, kCFARegister, 3772, 1372)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(0x17713850, 5721)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + EXPECT_CALL(reporter, EarlyEHTerminator(_)).InSequence(s).WillOnce(Return()); + + ParseEHFrameSection(§ion); +} + +// The parser should recognize the Linux Standards Base 'z' augmentations. +TEST_F(LulDwarfEHFrame, SimpleFDE) { + lul::DwarfPointerEncoding lsda_encoding = lul::DwarfPointerEncoding( + lul::DW_EH_PE_indirect | lul::DW_EH_PE_datarel | lul::DW_EH_PE_sdata2); + lul::DwarfPointerEncoding fde_encoding = + lul::DwarfPointerEncoding(lul::DW_EH_PE_textrel | lul::DW_EH_PE_udata2); + + section.SetPointerEncoding(fde_encoding); + section.SetEncodedPointerBases(encoded_pointer_bases); + Label cie; + section.Mark(&cie) + .CIEHeader(4873, 7012, 100, 1, "zSLPR") + .ULEB128(7) // Augmentation data length + .D8(lsda_encoding) // LSDA pointer format + .D8(lul::DW_EH_PE_pcrel) // personality pointer format + .EncodedPointer(0x97baa00, lul::DW_EH_PE_pcrel) // and value + .D8(fde_encoding) // FDE pointer format + .D8(lul::DW_CFA_def_cfa) + .ULEB128(6706) + .ULEB128(31) + .FinishEntry() + .FDEHeader(cie, 0x540f6b56, 0xf686) + .ULEB128(2) // Augmentation data length + .EncodedPointer(0xe3eab475, lsda_encoding) // LSDA pointer, signed + .D8(lul::DW_CFA_set_loc) + .EncodedPointer(0x540fa4ce, fde_encoding) + .D8(lul::DW_CFA_undefined) + .ULEB128(0x675e) + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.SimpleFDE", section); + + EXPECT_CALL(handler, Entry(_, 0x540f6b56, 0xf686, 1, "zSLPR", 100)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, PersonalityRoutine(0x97baa00, false)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, LanguageSpecificDataArea(0xe3eab475, true)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SignalHandler()).InSequence(s).WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(0x540f6b56, kCFARegister, 6706, 31)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(0x540fa4ce, 0x675e)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +// Check that we can handle an empty 'z' augmentation. +TEST_F(LulDwarfEHFrame, EmptyZ) { + Label cie; + section.Mark(&cie) + .CIEHeader(5955, 5805, 228, 1, "z") + .ULEB128(0) // Augmentation data length + .D8(lul::DW_CFA_def_cfa) + .ULEB128(3629) + .ULEB128(247) + .FinishEntry() + .FDEHeader(cie, 0xda007738, 0xfb55c641) + .ULEB128(0) // Augmentation data length + .D8(lul::DW_CFA_advance_loc1) + .D8(11) + .D8(lul::DW_CFA_undefined) + .ULEB128(3769) + .FinishEntry(); + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.EmptyZ", section); + + EXPECT_CALL(handler, Entry(_, 0xda007738, 0xfb55c641, 1, "z", 228)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(0xda007738, kCFARegister, 3629, 247)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(0xda007738 + 11 * 5955, 3769)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +// Check that we recognize bad 'z' augmentation characters. +TEST_F(LulDwarfEHFrame, BadZ) { + Label cie; + section.Mark(&cie) + .CIEHeader(6937, 1045, 142, 1, "zQ") + .ULEB128(0) // Augmentation data length + .D8(lul::DW_CFA_def_cfa) + .ULEB128(9006) + .ULEB128(7725) + .FinishEntry() + .FDEHeader(cie, 0x1293efa8, 0x236f53f2) + .ULEB128(0) // Augmentation data length + .D8(lul::DW_CFA_advance_loc | 12) + .D8(lul::DW_CFA_register) + .ULEB128(5667) + .ULEB128(3462) + .FinishEntry(); + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.BadZ", section); + + EXPECT_CALL(reporter, UnrecognizedAugmentation(_, "zQ")).WillOnce(Return()); + + ParseEHFrameSection(§ion, false); +} + +TEST_F(LulDwarfEHFrame, zL) { + Label cie; + lul::DwarfPointerEncoding lsda_encoding = + lul::DwarfPointerEncoding(lul::DW_EH_PE_funcrel | lul::DW_EH_PE_udata2); + section.Mark(&cie) + .CIEHeader(9285, 9959, 54, 1, "zL") + .ULEB128(1) // Augmentation data length + .D8(lsda_encoding) // encoding for LSDA pointer in FDE + + .FinishEntry() + .FDEHeader(cie, 0xd40091aa, 0x9aa6e746) + .ULEB128(2) // Augmentation data length + .EncodedPointer(0xd40099cd, lsda_encoding) // LSDA pointer + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.zL", section); + + EXPECT_CALL(handler, Entry(_, 0xd40091aa, 0x9aa6e746, 1, "zL", 54)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, LanguageSpecificDataArea(0xd40099cd, false)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +TEST_F(LulDwarfEHFrame, zP) { + Label cie; + lul::DwarfPointerEncoding personality_encoding = + lul::DwarfPointerEncoding(lul::DW_EH_PE_datarel | lul::DW_EH_PE_udata2); + section.Mark(&cie) + .CIEHeader(1097, 6313, 17, 1, "zP") + .ULEB128(3) // Augmentation data length + .D8(personality_encoding) // encoding for personality routine + .EncodedPointer(0xe3eaccac, personality_encoding) // value + .FinishEntry() + .FDEHeader(cie, 0x0c8350c9, 0xbef11087) + .ULEB128(0) // Augmentation data length + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.zP", section); + + EXPECT_CALL(handler, Entry(_, 0x0c8350c9, 0xbef11087, 1, "zP", 17)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, PersonalityRoutine(0xe3eaccac, false)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +TEST_F(LulDwarfEHFrame, zR) { + Label cie; + lul::DwarfPointerEncoding pointer_encoding = + lul::DwarfPointerEncoding(lul::DW_EH_PE_textrel | lul::DW_EH_PE_sdata2); + section.SetPointerEncoding(pointer_encoding); + section.Mark(&cie) + .CIEHeader(8011, 5496, 75, 1, "zR") + .ULEB128(1) // Augmentation data length + .D8(pointer_encoding) // encoding for FDE addresses + .FinishEntry() + .FDEHeader(cie, 0x540f9431, 0xbd0) + .ULEB128(0) // Augmentation data length + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.zR", section); + + EXPECT_CALL(handler, Entry(_, 0x540f9431, 0xbd0, 1, "zR", 75)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +TEST_F(LulDwarfEHFrame, zS) { + Label cie; + section.Mark(&cie) + .CIEHeader(9217, 7694, 57, 1, "zS") + .ULEB128(0) // Augmentation data length + .FinishEntry() + .FDEHeader(cie, 0xd40091aa, 0x9aa6e746) + .ULEB128(0) // Augmentation data length + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.zS", section); + + EXPECT_CALL(handler, Entry(_, 0xd40091aa, 0x9aa6e746, 1, "zS", 57)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SignalHandler()).InSequence(s).WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +// These tests require manual inspection of the test output. +struct CFIReporterFixture { + CFIReporterFixture() + : reporter(gtest_logging_sink_for_LulTestDwarf, "test file name", + "test section name") {} + CallFrameInfo::Reporter reporter; +}; + +class LulDwarfCFIReporter : public CFIReporterFixture, public Test {}; + +TEST_F(LulDwarfCFIReporter, Incomplete) { + reporter.Incomplete(0x0102030405060708ULL, CallFrameInfo::kUnknown); +} + +TEST_F(LulDwarfCFIReporter, EarlyEHTerminator) { + reporter.EarlyEHTerminator(0x0102030405060708ULL); +} + +TEST_F(LulDwarfCFIReporter, CIEPointerOutOfRange) { + reporter.CIEPointerOutOfRange(0x0123456789abcdefULL, 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, BadCIEId) { + reporter.BadCIEId(0x0123456789abcdefULL, 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, UnrecognizedVersion) { + reporter.UnrecognizedVersion(0x0123456789abcdefULL, 43); +} + +TEST_F(LulDwarfCFIReporter, UnrecognizedAugmentation) { + reporter.UnrecognizedAugmentation(0x0123456789abcdefULL, "poodles"); +} + +TEST_F(LulDwarfCFIReporter, InvalidPointerEncoding) { + reporter.InvalidPointerEncoding(0x0123456789abcdefULL, 0x42); +} + +TEST_F(LulDwarfCFIReporter, UnusablePointerEncoding) { + reporter.UnusablePointerEncoding(0x0123456789abcdefULL, 0x42); +} + +TEST_F(LulDwarfCFIReporter, RestoreInCIE) { + reporter.RestoreInCIE(0x0123456789abcdefULL, 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, BadInstruction) { + reporter.BadInstruction(0x0123456789abcdefULL, CallFrameInfo::kFDE, + 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, NoCFARule) { + reporter.NoCFARule(0x0123456789abcdefULL, CallFrameInfo::kCIE, + 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, EmptyStateStack) { + reporter.EmptyStateStack(0x0123456789abcdefULL, CallFrameInfo::kTerminator, + 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, ClearingCFARule) { + reporter.ClearingCFARule(0x0123456789abcdefULL, CallFrameInfo::kFDE, + 0xfedcba9876543210ULL); +} +class LulDwarfExpr : public Test {}; + +class MockSummariser : public Summariser { + public: + MockSummariser() : Summariser(nullptr, 0, nullptr) {} + MOCK_METHOD2(Entry, void(uintptr_t, uintptr_t)); + MOCK_METHOD0(End, void()); + MOCK_METHOD5(Rule, void(uintptr_t, int, LExprHow, int16_t, int64_t)); + MOCK_METHOD1(AddPfxInstr, uint32_t(PfxInstr)); +}; + +TEST_F(LulDwarfExpr, SimpleTransliteration) { + MockSummariser summ; + ByteReader reader(ENDIANNESS_LITTLE); + + CFISection section(kLittleEndian, 8); + section.D8(DW_OP_lit0) + .D8(DW_OP_lit31) + .D8(DW_OP_breg0 + 17) + .LEB128(-1234) + .D8(DW_OP_const4s) + .D32(0xFEDC9876) + .D8(DW_OP_deref) + .D8(DW_OP_and) + .D8(DW_OP_plus) + .D8(DW_OP_minus) + .D8(DW_OP_shl) + .D8(DW_OP_ge); + string expr; + bool ok = section.GetContents(&expr); + EXPECT_TRUE(ok); + + { + InSequence s; + // required start marker + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Start, 0))); + // DW_OP_lit0 + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_SImm32, 0))); + // DW_OP_lit31 + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_SImm32, 31))); + // DW_OP_breg17 -1234 + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_DwReg, 17))); + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_SImm32, -1234))); + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Add))); + // DW_OP_const4s 0xFEDC9876 + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_SImm32, 0xFEDC9876))); + // DW_OP_deref + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Deref))); + // DW_OP_and + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_And))); + // DW_OP_plus + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Add))); + // DW_OP_minus + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Sub))); + // DW_OP_shl + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Shl))); + // DW_OP_ge + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_CmpGES))); + // required end marker + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_End))); + } + + int32_t ix = + parseDwarfExpr(&summ, &reader, ImageSlice(expr), false, false, false); + EXPECT_TRUE(ix >= 0); +} + +TEST_F(LulDwarfExpr, UnknownOpcode) { + MockSummariser summ; + ByteReader reader(ENDIANNESS_LITTLE); + + CFISection section(kLittleEndian, 8); + section.D8(DW_OP_lo_user - 1); + string expr; + bool ok = section.GetContents(&expr); + EXPECT_TRUE(ok); + + { + InSequence s; + // required start marker + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Start, 0))); + } + + int32_t ix = + parseDwarfExpr(&summ, &reader, ImageSlice(expr), false, false, false); + EXPECT_TRUE(ix == -1); +} + +TEST_F(LulDwarfExpr, ExpressionOverrun) { + MockSummariser summ; + ByteReader reader(ENDIANNESS_LITTLE); + + CFISection section(kLittleEndian, 8); + section.D8(DW_OP_const4s).D8(0x12).D8(0x34).D8(0x56); + string expr; + bool ok = section.GetContents(&expr); + EXPECT_TRUE(ok); + + { + InSequence s; + // required start marker + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Start, 0))); + // DW_OP_const4s followed by 3 (a.k.a. not enough) bytes + // We expect PfxInstr(PX_Simm32, not-known-for-sure-32-bit-immediate) + // Hence must use _ as the argument. + EXPECT_CALL(summ, AddPfxInstr(_)); + } + + int32_t ix = + parseDwarfExpr(&summ, &reader, ImageSlice(expr), false, false, false); + EXPECT_TRUE(ix == -1); +} + +// We'll need to mention specific Dwarf registers in the EvaluatePfxExpr tests, +// and those names are arch-specific, so a bit of macro magic is helpful. +#if defined(GP_ARCH_arm) +# define TESTED_REG_STRUCT_NAME r11 +# define TESTED_REG_DWARF_NAME DW_REG_ARM_R11 +#elif defined(GP_ARCH_arm64) +# define TESTED_REG_STRUCT_NAME x29 +# define TESTED_REG_DWARF_NAME DW_REG_AARCH64_X29 +#elif defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) +# define TESTED_REG_STRUCT_NAME xbp +# define TESTED_REG_DWARF_NAME DW_REG_INTEL_XBP +#else +# error "Unknown plat" +#endif + +struct EvaluatePfxExprFixture { + // Creates: + // initial stack, AVMA 0x12345678, at offset 4 bytes = 0xdeadbeef + // initial regs, with XBP = 0x14141356 + // initial CFA = 0x5432ABCD + EvaluatePfxExprFixture() { + // The test stack. + si.mStartAvma = 0x12345678; + si.mLen = 0; +#define XX(_byte) \ + do { \ + si.mContents[si.mLen++] = (_byte); \ + } while (0) + XX(0x55); + XX(0x55); + XX(0x55); + XX(0x55); + if (sizeof(void*) == 8) { + // le64 + XX(0xEF); + XX(0xBE); + XX(0xAD); + XX(0xDE); + XX(0); + XX(0); + XX(0); + XX(0); + } else { + // le32 + XX(0xEF); + XX(0xBE); + XX(0xAD); + XX(0xDE); + } + XX(0xAA); + XX(0xAA); + XX(0xAA); + XX(0xAA); +#undef XX + // The initial CFA. + initialCFA = TaggedUWord(0x5432ABCD); + // The initial register state. + memset(®s, 0, sizeof(regs)); + regs.TESTED_REG_STRUCT_NAME = TaggedUWord(0x14141356); + } + + StackImage si; + TaggedUWord initialCFA; + UnwindRegs regs; +}; + +class LulDwarfEvaluatePfxExpr : public EvaluatePfxExprFixture, public Test {}; + +TEST_F(LulDwarfEvaluatePfxExpr, NormalEvaluation) { + vector<PfxInstr> instrs; + // Put some junk at the start of the insn sequence. + instrs.push_back(PfxInstr(PX_End)); + instrs.push_back(PfxInstr(PX_End)); + + // Now the real sequence + // stack is empty + instrs.push_back(PfxInstr(PX_Start, 1)); + // 0x5432ABCD + instrs.push_back(PfxInstr(PX_SImm32, 0x31415927)); + // 0x5432ABCD 0x31415927 + instrs.push_back(PfxInstr(PX_DwReg, TESTED_REG_DWARF_NAME)); + // 0x5432ABCD 0x31415927 0x14141356 + instrs.push_back(PfxInstr(PX_SImm32, 42)); + // 0x5432ABCD 0x31415927 0x14141356 42 + instrs.push_back(PfxInstr(PX_Sub)); + // 0x5432ABCD 0x31415927 0x1414132c + instrs.push_back(PfxInstr(PX_Add)); + // 0x5432ABCD 0x45556c53 + instrs.push_back(PfxInstr(PX_SImm32, si.mStartAvma + 4)); + // 0x5432ABCD 0x45556c53 0x1234567c + instrs.push_back(PfxInstr(PX_Deref)); + // 0x5432ABCD 0x45556c53 0xdeadbeef + instrs.push_back(PfxInstr(PX_SImm32, 0xFE01DC23)); + // 0x5432ABCD 0x45556c53 0xdeadbeef 0xFE01DC23 + instrs.push_back(PfxInstr(PX_And)); + // 0x5432ABCD 0x45556c53 0xde019c23 + instrs.push_back(PfxInstr(PX_SImm32, 7)); + // 0x5432ABCD 0x45556c53 0xde019c23 7 + instrs.push_back(PfxInstr(PX_Shl)); + // 0x5432ABCD 0x45556c53 0x6f00ce1180 + instrs.push_back(PfxInstr(PX_SImm32, 0x7fffffff)); + // 0x5432ABCD 0x45556c53 0x6f00ce1180 7fffffff + instrs.push_back(PfxInstr(PX_And)); + // 0x5432ABCD 0x45556c53 0x00ce1180 + instrs.push_back(PfxInstr(PX_Add)); + // 0x5432ABCD 0x46237dd3 + instrs.push_back(PfxInstr(PX_Sub)); + // 0xe0f2dfa + + instrs.push_back(PfxInstr(PX_End)); + + TaggedUWord res = EvaluatePfxExpr(2 /*offset of start insn*/, ®s, + initialCFA, &si, instrs); + EXPECT_TRUE(res.Valid()); + EXPECT_TRUE(res.Value() == 0xe0f2dfa); +} + +TEST_F(LulDwarfEvaluatePfxExpr, EmptySequence) { + vector<PfxInstr> instrs; + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, BogusStartPoint) { + vector<PfxInstr> instrs; + instrs.push_back(PfxInstr(PX_SImm32, 42)); + instrs.push_back(PfxInstr(PX_SImm32, 24)); + instrs.push_back(PfxInstr(PX_SImm32, 4224)); + TaggedUWord res = EvaluatePfxExpr(1, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, MissingEndMarker) { + vector<PfxInstr> instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + instrs.push_back(PfxInstr(PX_SImm32, 24)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, StackUnderflow) { + vector<PfxInstr> instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, StackNoUnderflow) { + vector<PfxInstr> instrs; + instrs.push_back(PfxInstr(PX_Start, 1 /*push the initial CFA*/)); + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_TRUE(res.Valid()); + EXPECT_TRUE(res == initialCFA); +} + +TEST_F(LulDwarfEvaluatePfxExpr, StackOverflow) { + vector<PfxInstr> instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + for (int i = 0; i < 10 + 1; i++) { + instrs.push_back(PfxInstr(PX_SImm32, i + 100)); + } + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, StackNoOverflow) { + vector<PfxInstr> instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + for (int i = 0; i < 10 + 0; i++) { + instrs.push_back(PfxInstr(PX_SImm32, i + 100)); + } + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_TRUE(res.Valid()); + EXPECT_TRUE(res == TaggedUWord(109)); +} + +TEST_F(LulDwarfEvaluatePfxExpr, OutOfRangeShl) { + vector<PfxInstr> instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + instrs.push_back(PfxInstr(PX_SImm32, 1234)); + instrs.push_back(PfxInstr(PX_SImm32, 5678)); + instrs.push_back(PfxInstr(PX_Shl)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_TRUE(!res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, TestCmpGES) { + const int32_t argsL[6] = {0, 0, 1, -2, -1, -2}; + const int32_t argsR[6] = {0, 1, 0, -2, -2, -1}; + // expecting: t f t t t f = 101110 = 0x2E + vector<PfxInstr> instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + // The "running total" + instrs.push_back(PfxInstr(PX_SImm32, 0)); + for (unsigned int i = 0; i < sizeof(argsL) / sizeof(argsL[0]); i++) { + // Shift the "running total" at the bottom of the stack left by one bit + instrs.push_back(PfxInstr(PX_SImm32, 1)); + instrs.push_back(PfxInstr(PX_Shl)); + // Push both test args and do the comparison + instrs.push_back(PfxInstr(PX_SImm32, argsL[i])); + instrs.push_back(PfxInstr(PX_SImm32, argsR[i])); + instrs.push_back(PfxInstr(PX_CmpGES)); + // Or the result into the running total + instrs.push_back(PfxInstr(PX_Or)); + } + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_TRUE(res.Valid()); + EXPECT_TRUE(res == TaggedUWord(0x2E)); +} + +} // namespace lul diff --git a/tools/profiler/tests/gtest/LulTestInfrastructure.cpp b/tools/profiler/tests/gtest/LulTestInfrastructure.cpp new file mode 100644 index 0000000000..6d49557e9c --- /dev/null +++ b/tools/profiler/tests/gtest/LulTestInfrastructure.cpp @@ -0,0 +1,498 @@ +// Copyright (c) 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Original author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// Derived from: +// test_assembler.cc: Implementation of google_breakpad::TestAssembler. +// See test_assembler.h for details. + +// Derived from: +// cfi_assembler.cc: Implementation of google_breakpad::CFISection class. +// See cfi_assembler.h for details. + +#include "LulTestInfrastructure.h" + +#include "LulDwarfInt.h" + +#include <cassert> + +namespace lul_test { +namespace test_assembler { + +using std::back_insert_iterator; + +Label::Label() : value_(new Binding()) {} +Label::Label(uint64_t value) : value_(new Binding(value)) {} +Label::Label(const Label& label) { + value_ = label.value_; + value_->Acquire(); +} +Label::~Label() { + if (value_->Release()) delete value_; +} + +Label& Label::operator=(uint64_t value) { + value_->Set(NULL, value); + return *this; +} + +Label& Label::operator=(const Label& label) { + value_->Set(label.value_, 0); + return *this; +} + +Label Label::operator+(uint64_t addend) const { + Label l; + l.value_->Set(this->value_, addend); + return l; +} + +Label Label::operator-(uint64_t subtrahend) const { + Label l; + l.value_->Set(this->value_, -subtrahend); + return l; +} + +// When NDEBUG is #defined, assert doesn't evaluate its argument. This +// means you can't simply use assert to check the return value of a +// function with necessary side effects. +// +// ALWAYS_EVALUATE_AND_ASSERT(x) evaluates x regardless of whether +// NDEBUG is #defined; when NDEBUG is not #defined, it further asserts +// that x is true. +#ifdef NDEBUG +# define ALWAYS_EVALUATE_AND_ASSERT(x) x +#else +# define ALWAYS_EVALUATE_AND_ASSERT(x) assert(x) +#endif + +uint64_t Label::operator-(const Label& label) const { + uint64_t offset; + ALWAYS_EVALUATE_AND_ASSERT(IsKnownOffsetFrom(label, &offset)); + return offset; +} + +bool Label::IsKnownConstant(uint64_t* value_p) const { + Binding* base; + uint64_t addend; + value_->Get(&base, &addend); + if (base != NULL) return false; + if (value_p) *value_p = addend; + return true; +} + +bool Label::IsKnownOffsetFrom(const Label& label, uint64_t* offset_p) const { + Binding *label_base, *this_base; + uint64_t label_addend, this_addend; + label.value_->Get(&label_base, &label_addend); + value_->Get(&this_base, &this_addend); + // If this and label are related, Get will find their final + // common ancestor, regardless of how indirect the relation is. This + // comparison also handles the constant vs. constant case. + if (this_base != label_base) return false; + if (offset_p) *offset_p = this_addend - label_addend; + return true; +} + +Label::Binding::Binding() : base_(this), addend_(), reference_count_(1) {} + +Label::Binding::Binding(uint64_t addend) + : base_(NULL), addend_(addend), reference_count_(1) {} + +Label::Binding::~Binding() { + assert(reference_count_ == 0); + if (base_ && base_ != this && base_->Release()) delete base_; +} + +void Label::Binding::Set(Binding* binding, uint64_t addend) { + if (!base_ && !binding) { + // We're equating two constants. This could be okay. + assert(addend_ == addend); + } else if (!base_) { + // We are a known constant, but BINDING may not be, so turn the + // tables and try to set BINDING's value instead. + binding->Set(NULL, addend_ - addend); + } else { + if (binding) { + // Find binding's final value. Since the final value is always either + // completely unconstrained or a constant, never a reference to + // another variable (otherwise, it wouldn't be final), this + // guarantees we won't create cycles here, even for code like this: + // l = m, m = n, n = l; + uint64_t binding_addend; + binding->Get(&binding, &binding_addend); + addend += binding_addend; + } + + // It seems likely that setting a binding to itself is a bug + // (although I can imagine this might turn out to be helpful to + // permit). + assert(binding != this); + + if (base_ != this) { + // Set the other bindings on our chain as well. Note that this + // is sufficient even though binding relationships form trees: + // All binding operations traverse their chains to the end, and + // all bindings related to us share some tail of our chain, so + // they will see the changes we make here. + base_->Set(binding, addend - addend_); + // We're not going to use base_ any more. + if (base_->Release()) delete base_; + } + + // Adopt BINDING as our base. Note that it should be correct to + // acquire here, after the release above, even though the usual + // reference-counting rules call for acquiring first, and then + // releasing: the self-reference assertion above should have + // complained if BINDING were 'this' or anywhere along our chain, + // so we didn't release BINDING. + if (binding) binding->Acquire(); + base_ = binding; + addend_ = addend; + } +} + +void Label::Binding::Get(Binding** base, uint64_t* addend) { + if (base_ && base_ != this) { + // Recurse to find the end of our reference chain (the root of our + // tree), and then rewrite every binding along the chain to refer + // to it directly, adjusting addends appropriately. (This is why + // this member function isn't this-const.) + Binding* final_base; + uint64_t final_addend; + base_->Get(&final_base, &final_addend); + if (final_base) final_base->Acquire(); + if (base_->Release()) delete base_; + base_ = final_base; + addend_ += final_addend; + } + *base = base_; + *addend = addend_; +} + +template <typename Inserter> +static inline void InsertEndian(test_assembler::Endianness endianness, + size_t size, uint64_t number, Inserter dest) { + assert(size > 0); + if (endianness == kLittleEndian) { + for (size_t i = 0; i < size; i++) { + *dest++ = (char)(number & 0xff); + number >>= 8; + } + } else { + assert(endianness == kBigEndian); + // The loop condition is odd, but it's correct for size_t. + for (size_t i = size - 1; i < size; i--) + *dest++ = (char)((number >> (i * 8)) & 0xff); + } +} + +Section& Section::Append(Endianness endianness, size_t size, uint64_t number) { + InsertEndian(endianness, size, number, + back_insert_iterator<string>(contents_)); + return *this; +} + +Section& Section::Append(Endianness endianness, size_t size, + const Label& label) { + // If this label's value is known, there's no reason to waste an + // entry in references_ on it. + uint64_t value; + if (label.IsKnownConstant(&value)) return Append(endianness, size, value); + + // This will get caught when the references are resolved, but it's + // nicer to find out earlier. + assert(endianness != kUnsetEndian); + + references_.push_back(Reference(contents_.size(), endianness, size, label)); + contents_.append(size, 0); + return *this; +} + +#define ENDIANNESS_L kLittleEndian +#define ENDIANNESS_B kBigEndian +#define ENDIANNESS(e) ENDIANNESS_##e + +#define DEFINE_SHORT_APPEND_NUMBER_ENDIAN(e, bits) \ + Section& Section::e##bits(uint##bits##_t v) { \ + InsertEndian(ENDIANNESS(e), bits / 8, v, \ + back_insert_iterator<string>(contents_)); \ + return *this; \ + } + +#define DEFINE_SHORT_APPEND_LABEL_ENDIAN(e, bits) \ + Section& Section::e##bits(const Label& v) { \ + return Append(ENDIANNESS(e), bits / 8, v); \ + } + +// Define L16, B32, and friends. +#define DEFINE_SHORT_APPEND_ENDIAN(e, bits) \ + DEFINE_SHORT_APPEND_NUMBER_ENDIAN(e, bits) \ + DEFINE_SHORT_APPEND_LABEL_ENDIAN(e, bits) + +DEFINE_SHORT_APPEND_LABEL_ENDIAN(L, 8); +DEFINE_SHORT_APPEND_LABEL_ENDIAN(B, 8); +DEFINE_SHORT_APPEND_ENDIAN(L, 16); +DEFINE_SHORT_APPEND_ENDIAN(L, 32); +DEFINE_SHORT_APPEND_ENDIAN(L, 64); +DEFINE_SHORT_APPEND_ENDIAN(B, 16); +DEFINE_SHORT_APPEND_ENDIAN(B, 32); +DEFINE_SHORT_APPEND_ENDIAN(B, 64); + +#define DEFINE_SHORT_APPEND_NUMBER_DEFAULT(bits) \ + Section& Section::D##bits(uint##bits##_t v) { \ + InsertEndian(endianness_, bits / 8, v, \ + back_insert_iterator<string>(contents_)); \ + return *this; \ + } +#define DEFINE_SHORT_APPEND_LABEL_DEFAULT(bits) \ + Section& Section::D##bits(const Label& v) { \ + return Append(endianness_, bits / 8, v); \ + } +#define DEFINE_SHORT_APPEND_DEFAULT(bits) \ + DEFINE_SHORT_APPEND_NUMBER_DEFAULT(bits) \ + DEFINE_SHORT_APPEND_LABEL_DEFAULT(bits) + +DEFINE_SHORT_APPEND_LABEL_DEFAULT(8) +DEFINE_SHORT_APPEND_DEFAULT(16); +DEFINE_SHORT_APPEND_DEFAULT(32); +DEFINE_SHORT_APPEND_DEFAULT(64); + +Section& Section::LEB128(long long value) { + while (value < -0x40 || 0x3f < value) { + contents_ += (value & 0x7f) | 0x80; + if (value < 0) + value = (value >> 7) | ~(((unsigned long long)-1) >> 7); + else + value = (value >> 7); + } + contents_ += value & 0x7f; + return *this; +} + +Section& Section::ULEB128(uint64_t value) { + while (value > 0x7f) { + contents_ += (value & 0x7f) | 0x80; + value = (value >> 7); + } + contents_ += value; + return *this; +} + +Section& Section::Align(size_t alignment, uint8_t pad_byte) { + // ALIGNMENT must be a power of two. + assert(((alignment - 1) & alignment) == 0); + size_t new_size = (contents_.size() + alignment - 1) & ~(alignment - 1); + contents_.append(new_size - contents_.size(), pad_byte); + assert((contents_.size() & (alignment - 1)) == 0); + return *this; +} + +bool Section::GetContents(string* contents) { + // For each label reference, find the label's value, and patch it into + // the section's contents. + for (size_t i = 0; i < references_.size(); i++) { + Reference& r = references_[i]; + uint64_t value; + if (!r.label.IsKnownConstant(&value)) { + fprintf(stderr, "Undefined label #%zu at offset 0x%zx\n", i, r.offset); + return false; + } + assert(r.offset < contents_.size()); + assert(contents_.size() - r.offset >= r.size); + InsertEndian(r.endianness, r.size, value, contents_.begin() + r.offset); + } + contents->clear(); + std::swap(contents_, *contents); + references_.clear(); + return true; +} + +} // namespace test_assembler +} // namespace lul_test + +namespace lul_test { + +CFISection& CFISection::CIEHeader(uint64_t code_alignment_factor, + int data_alignment_factor, + unsigned return_address_register, + uint8_t version, const string& augmentation, + bool dwarf64) { + assert(!entry_length_); + entry_length_ = new PendingLength(); + in_fde_ = false; + + if (dwarf64) { + D32(kDwarf64InitialLengthMarker); + D64(entry_length_->length); + entry_length_->start = Here(); + D64(eh_frame_ ? kEHFrame64CIEIdentifier : kDwarf64CIEIdentifier); + } else { + D32(entry_length_->length); + entry_length_->start = Here(); + D32(eh_frame_ ? kEHFrame32CIEIdentifier : kDwarf32CIEIdentifier); + } + D8(version); + AppendCString(augmentation); + ULEB128(code_alignment_factor); + LEB128(data_alignment_factor); + if (version == 1) + D8(return_address_register); + else + ULEB128(return_address_register); + return *this; +} + +CFISection& CFISection::FDEHeader(Label cie_pointer, uint64_t initial_location, + uint64_t address_range, bool dwarf64) { + assert(!entry_length_); + entry_length_ = new PendingLength(); + in_fde_ = true; + fde_start_address_ = initial_location; + + if (dwarf64) { + D32(0xffffffff); + D64(entry_length_->length); + entry_length_->start = Here(); + if (eh_frame_) + D64(Here() - cie_pointer); + else + D64(cie_pointer); + } else { + D32(entry_length_->length); + entry_length_->start = Here(); + if (eh_frame_) + D32(Here() - cie_pointer); + else + D32(cie_pointer); + } + EncodedPointer(initial_location); + // The FDE length in an .eh_frame section uses the same encoding as the + // initial location, but ignores the base address (selected by the upper + // nybble of the encoding), as it's a length, not an address that can be + // made relative. + EncodedPointer(address_range, DwarfPointerEncoding(pointer_encoding_ & 0x0f)); + return *this; +} + +CFISection& CFISection::FinishEntry() { + assert(entry_length_); + Align(address_size_, lul::DW_CFA_nop); + entry_length_->length = Here() - entry_length_->start; + delete entry_length_; + entry_length_ = NULL; + in_fde_ = false; + return *this; +} + +CFISection& CFISection::EncodedPointer(uint64_t address, + DwarfPointerEncoding encoding, + const EncodedPointerBases& bases) { + // Omitted data is extremely easy to emit. + if (encoding == lul::DW_EH_PE_omit) return *this; + + // If (encoding & lul::DW_EH_PE_indirect) != 0, then we assume + // that ADDRESS is the address at which the pointer is stored --- in + // other words, that bit has no effect on how we write the pointer. + encoding = DwarfPointerEncoding(encoding & ~lul::DW_EH_PE_indirect); + + // Find the base address to which this pointer is relative. The upper + // nybble of the encoding specifies this. + uint64_t base; + switch (encoding & 0xf0) { + case lul::DW_EH_PE_absptr: + base = 0; + break; + case lul::DW_EH_PE_pcrel: + base = bases.cfi + Size(); + break; + case lul::DW_EH_PE_textrel: + base = bases.text; + break; + case lul::DW_EH_PE_datarel: + base = bases.data; + break; + case lul::DW_EH_PE_funcrel: + base = fde_start_address_; + break; + case lul::DW_EH_PE_aligned: + base = 0; + break; + default: + abort(); + }; + + // Make ADDRESS relative. Yes, this is appropriate even for "absptr" + // values; see gcc/unwind-pe.h. + address -= base; + + // Align the pointer, if required. + if ((encoding & 0xf0) == lul::DW_EH_PE_aligned) Align(AddressSize()); + + // Append ADDRESS to this section in the appropriate form. For the + // fixed-width forms, we don't need to differentiate between signed and + // unsigned encodings, because ADDRESS has already been extended to 64 + // bits before it was passed to us. + switch (encoding & 0x0f) { + case lul::DW_EH_PE_absptr: + Address(address); + break; + + case lul::DW_EH_PE_uleb128: + ULEB128(address); + break; + + case lul::DW_EH_PE_sleb128: + LEB128(address); + break; + + case lul::DW_EH_PE_udata2: + case lul::DW_EH_PE_sdata2: + D16(address); + break; + + case lul::DW_EH_PE_udata4: + case lul::DW_EH_PE_sdata4: + D32(address); + break; + + case lul::DW_EH_PE_udata8: + case lul::DW_EH_PE_sdata8: + D64(address); + break; + + default: + abort(); + } + + return *this; +}; + +} // namespace lul_test diff --git a/tools/profiler/tests/gtest/LulTestInfrastructure.h b/tools/profiler/tests/gtest/LulTestInfrastructure.h new file mode 100644 index 0000000000..83b239fb73 --- /dev/null +++ b/tools/profiler/tests/gtest/LulTestInfrastructure.h @@ -0,0 +1,735 @@ +// -*- mode: C++ -*- + +// Copyright (c) 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Original author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// Derived from: +// cfi_assembler.h: Define CFISection, a class for creating properly +// (and improperly) formatted DWARF CFI data for unit tests. + +// Derived from: +// test-assembler.h: interface to class for building complex binary streams. + +// To test the Breakpad symbol dumper and processor thoroughly, for +// all combinations of host system and minidump processor +// architecture, we need to be able to easily generate complex test +// data like debugging information and minidump files. +// +// For example, if we want our unit tests to provide full code +// coverage for stack walking, it may be difficult to persuade the +// compiler to generate every possible sort of stack walking +// information that we want to support; there are probably DWARF CFI +// opcodes that GCC never emits. Similarly, if we want to test our +// error handling, we will need to generate damaged minidumps or +// debugging information that (we hope) the client or compiler will +// never produce on its own. +// +// google_breakpad::TestAssembler provides a predictable and +// (relatively) simple way to generate complex formatted data streams +// like minidumps and CFI. Furthermore, because TestAssembler is +// portable, developers without access to (say) Visual Studio or a +// SPARC assembler can still work on test data for those targets. + +#ifndef LUL_TEST_INFRASTRUCTURE_H +#define LUL_TEST_INFRASTRUCTURE_H + +#include "LulDwarfExt.h" + +#include <string> +#include <vector> + +using std::string; +using std::vector; + +namespace lul_test { +namespace test_assembler { + +// A Label represents a value not yet known that we need to store in a +// section. As long as all the labels a section refers to are defined +// by the time we retrieve its contents as bytes, we can use undefined +// labels freely in that section's construction. +// +// A label can be in one of three states: +// - undefined, +// - defined as the sum of some other label and a constant, or +// - a constant. +// +// A label's value never changes, but it can accumulate constraints. +// Adding labels and integers is permitted, and yields a label. +// Subtracting a constant from a label is permitted, and also yields a +// label. Subtracting two labels that have some relationship to each +// other is permitted, and yields a constant. +// +// For example: +// +// Label a; // a's value is undefined +// Label b; // b's value is undefined +// { +// Label c = a + 4; // okay, even though a's value is unknown +// b = c + 4; // also okay; b is now a+8 +// } +// Label d = b - 2; // okay; d == a+6, even though c is gone +// d.Value(); // error: d's value is not yet known +// d - a; // is 6, even though their values are not known +// a = 12; // now b == 20, and d == 18 +// d.Value(); // 18: no longer an error +// b.Value(); // 20 +// d = 10; // error: d is already defined. +// +// Label objects' lifetimes are unconstrained: notice that, in the +// above example, even though a and b are only related through c, and +// c goes out of scope, the assignment to a sets b's value as well. In +// particular, it's not necessary to ensure that a Label lives beyond +// Sections that refer to it. +class Label { + public: + Label(); // An undefined label. + explicit Label(uint64_t value); // A label with a fixed value + Label(const Label& value); // A label equal to another. + ~Label(); + + Label& operator=(uint64_t value); + Label& operator=(const Label& value); + Label operator+(uint64_t addend) const; + Label operator-(uint64_t subtrahend) const; + uint64_t operator-(const Label& subtrahend) const; + + // We could also provide == and != that work on undefined, but + // related, labels. + + // Return true if this label's value is known. If VALUE_P is given, + // set *VALUE_P to the known value if returning true. + bool IsKnownConstant(uint64_t* value_p = NULL) const; + + // Return true if the offset from LABEL to this label is known. If + // OFFSET_P is given, set *OFFSET_P to the offset when returning true. + // + // You can think of l.KnownOffsetFrom(m, &d) as being like 'd = l-m', + // except that it also returns a value indicating whether the + // subtraction is possible given what we currently know of l and m. + // It can be possible even if we don't know l and m's values. For + // example: + // + // Label l, m; + // m = l + 10; + // l.IsKnownConstant(); // false + // m.IsKnownConstant(); // false + // uint64_t d; + // l.IsKnownOffsetFrom(m, &d); // true, and sets d to -10. + // l-m // -10 + // m-l // 10 + // m.Value() // error: m's value is not known + bool IsKnownOffsetFrom(const Label& label, uint64_t* offset_p = NULL) const; + + private: + // A label's value, or if that is not yet known, how the value is + // related to other labels' values. A binding may be: + // - a known constant, + // - constrained to be equal to some other binding plus a constant, or + // - unconstrained, and free to take on any value. + // + // Many labels may point to a single binding, and each binding may + // refer to another, so bindings and labels form trees whose leaves + // are labels, whose interior nodes (and roots) are bindings, and + // where links point from children to parents. Bindings are + // reference counted, allowing labels to be lightweight, copyable, + // assignable, placed in containers, and so on. + class Binding { + public: + Binding(); + explicit Binding(uint64_t addend); + ~Binding(); + + // Increment our reference count. + void Acquire() { reference_count_++; }; + // Decrement our reference count, and return true if it is zero. + bool Release() { return --reference_count_ == 0; } + + // Set this binding to be equal to BINDING + ADDEND. If BINDING is + // NULL, then set this binding to the known constant ADDEND. + // Update every binding on this binding's chain to point directly + // to BINDING, or to be a constant, with addends adjusted + // appropriately. + void Set(Binding* binding, uint64_t value); + + // Return what we know about the value of this binding. + // - If this binding's value is a known constant, set BASE to + // NULL, and set ADDEND to its value. + // - If this binding is not a known constant but related to other + // bindings, set BASE to the binding at the end of the relation + // chain (which will always be unconstrained), and set ADDEND to the + // value to add to that binding's value to get this binding's + // value. + // - If this binding is unconstrained, set BASE to this, and leave + // ADDEND unchanged. + void Get(Binding** base, uint64_t* addend); + + private: + // There are three cases: + // + // - A binding representing a known constant value has base_ NULL, + // and addend_ equal to the value. + // + // - A binding representing a completely unconstrained value has + // base_ pointing to this; addend_ is unused. + // + // - A binding whose value is related to some other binding's + // value has base_ pointing to that other binding, and addend_ + // set to the amount to add to that binding's value to get this + // binding's value. We only represent relationships of the form + // x = y+c. + // + // Thus, the bind_ links form a chain terminating in either a + // known constant value or a completely unconstrained value. Most + // operations on bindings do path compression: they change every + // binding on the chain to point directly to the final value, + // adjusting addends as appropriate. + Binding* base_; + uint64_t addend_; + + // The number of Labels and Bindings pointing to this binding. + // (When a binding points to itself, indicating a completely + // unconstrained binding, that doesn't count as a reference.) + int reference_count_; + }; + + // This label's value. + Binding* value_; +}; + +// Conventions for representing larger numbers as sequences of bytes. +enum Endianness { + kBigEndian, // Big-endian: the most significant byte comes first. + kLittleEndian, // Little-endian: the least significant byte comes first. + kUnsetEndian, // used internally +}; + +// A section is a sequence of bytes, constructed by appending bytes +// to the end. Sections have a convenient and flexible set of member +// functions for appending data in various formats: big-endian and +// little-endian signed and unsigned values of different sizes; +// LEB128 and ULEB128 values (see below), and raw blocks of bytes. +// +// If you need to append a value to a section that is not convenient +// to compute immediately, you can create a label, append the +// label's value to the section, and then set the label's value +// later, when it's convenient to do so. Once a label's value is +// known, the section class takes care of updating all previously +// appended references to it. +// +// Once all the labels to which a section refers have had their +// values determined, you can get a copy of the section's contents +// as a string. +// +// Note that there is no specified "start of section" label. This is +// because there are typically several different meanings for "the +// start of a section": the offset of the section within an object +// file, the address in memory at which the section's content appear, +// and so on. It's up to the code that uses the Section class to +// keep track of these explicitly, as they depend on the application. +class Section { + public: + explicit Section(Endianness endianness = kUnsetEndian) + : endianness_(endianness){}; + + // A base class destructor should be either public and virtual, + // or protected and nonvirtual. + virtual ~Section() = default; + + // Return the default endianness of this section. + Endianness endianness() const { return endianness_; } + + // Append the SIZE bytes at DATA to the end of this section. Return + // a reference to this section. + Section& Append(const string& data) { + contents_.append(data); + return *this; + }; + + // Append data from SLICE to the end of this section. Return + // a reference to this section. + Section& Append(const lul::ImageSlice& slice) { + for (size_t i = 0; i < slice.length_; i++) { + contents_.append(1, slice.start_[i]); + } + return *this; + } + + // Append data from CSTRING to the end of this section. The terminating + // zero is not included. Return a reference to this section. + Section& Append(const char* cstring) { + for (size_t i = 0; cstring[i] != '\0'; i++) { + contents_.append(1, cstring[i]); + } + return *this; + } + + // Append SIZE copies of BYTE to the end of this section. Return a + // reference to this section. + Section& Append(size_t size, uint8_t byte) { + contents_.append(size, (char)byte); + return *this; + } + + // Append NUMBER to this section. ENDIANNESS is the endianness to + // use to write the number. SIZE is the length of the number in + // bytes. Return a reference to this section. + Section& Append(Endianness endianness, size_t size, uint64_t number); + Section& Append(Endianness endianness, size_t size, const Label& label); + + // Append SECTION to the end of this section. The labels SECTION + // refers to need not be defined yet. + // + // Note that this has no effect on any Labels' values, or on + // SECTION. If placing SECTION within 'this' provides new + // constraints on existing labels' values, then it's up to the + // caller to fiddle with those labels as needed. + Section& Append(const Section& section); + + // Append the contents of DATA as a series of bytes terminated by + // a NULL character. + Section& AppendCString(const string& data) { + Append(data); + contents_ += '\0'; + return *this; + } + + // Append VALUE or LABEL to this section, with the given bit width and + // endianness. Return a reference to this section. + // + // The names of these functions have the form <ENDIANNESS><BITWIDTH>: + // <ENDIANNESS> is either 'L' (little-endian, least significant byte first), + // 'B' (big-endian, most significant byte first), or + // 'D' (default, the section's default endianness) + // <BITWIDTH> is 8, 16, 32, or 64. + // + // Since endianness doesn't matter for a single byte, all the + // <BITWIDTH>=8 functions are equivalent. + // + // These can be used to write both signed and unsigned values, as + // the compiler will properly sign-extend a signed value before + // passing it to the function, at which point the function's + // behavior is the same either way. + Section& L8(uint8_t value) { + contents_ += value; + return *this; + } + Section& B8(uint8_t value) { + contents_ += value; + return *this; + } + Section& D8(uint8_t value) { + contents_ += value; + return *this; + } + Section &L16(uint16_t), &L32(uint32_t), &L64(uint64_t), &B16(uint16_t), + &B32(uint32_t), &B64(uint64_t), &D16(uint16_t), &D32(uint32_t), + &D64(uint64_t); + Section &L8(const Label& label), &L16(const Label& label), + &L32(const Label& label), &L64(const Label& label), + &B8(const Label& label), &B16(const Label& label), + &B32(const Label& label), &B64(const Label& label), + &D8(const Label& label), &D16(const Label& label), + &D32(const Label& label), &D64(const Label& label); + + // Append VALUE in a signed LEB128 (Little-Endian Base 128) form. + // + // The signed LEB128 representation of an integer N is a variable + // number of bytes: + // + // - If N is between -0x40 and 0x3f, then its signed LEB128 + // representation is a single byte whose value is N. + // + // - Otherwise, its signed LEB128 representation is (N & 0x7f) | + // 0x80, followed by the signed LEB128 representation of N / 128, + // rounded towards negative infinity. + // + // In other words, we break VALUE into groups of seven bits, put + // them in little-endian order, and then write them as eight-bit + // bytes with the high bit on all but the last. + // + // Note that VALUE cannot be a Label (we would have to implement + // relaxation). + Section& LEB128(long long value); + + // Append VALUE in unsigned LEB128 (Little-Endian Base 128) form. + // + // The unsigned LEB128 representation of an integer N is a variable + // number of bytes: + // + // - If N is between 0 and 0x7f, then its unsigned LEB128 + // representation is a single byte whose value is N. + // + // - Otherwise, its unsigned LEB128 representation is (N & 0x7f) | + // 0x80, followed by the unsigned LEB128 representation of N / + // 128, rounded towards negative infinity. + // + // Note that VALUE cannot be a Label (we would have to implement + // relaxation). + Section& ULEB128(uint64_t value); + + // Jump to the next location aligned on an ALIGNMENT-byte boundary, + // relative to the start of the section. Fill the gap with PAD_BYTE. + // ALIGNMENT must be a power of two. Return a reference to this + // section. + Section& Align(size_t alignment, uint8_t pad_byte = 0); + + // Return the current size of the section. + size_t Size() const { return contents_.size(); } + + // Return a label representing the start of the section. + // + // It is up to the user whether this label represents the section's + // position in an object file, the section's address in memory, or + // what have you; some applications may need both, in which case + // this simple-minded interface won't be enough. This class only + // provides a single start label, for use with the Here and Mark + // member functions. + // + // Ideally, we'd provide this in a subclass that actually knows more + // about the application at hand and can provide an appropriate + // collection of start labels. But then the appending member + // functions like Append and D32 would return a reference to the + // base class, not the derived class, and the chaining won't work. + // Since the only value here is in pretty notation, that's a fatal + // flaw. + Label start() const { return start_; } + + // Return a label representing the point at which the next Appended + // item will appear in the section, relative to start(). + Label Here() const { return start_ + Size(); } + + // Set *LABEL to Here, and return a reference to this section. + Section& Mark(Label* label) { + *label = Here(); + return *this; + } + + // If there are no undefined label references left in this + // section, set CONTENTS to the contents of this section, as a + // string, and clear this section. Return true on success, or false + // if there were still undefined labels. + bool GetContents(string* contents); + + private: + // Used internally. A reference to a label's value. + struct Reference { + Reference(size_t set_offset, Endianness set_endianness, size_t set_size, + const Label& set_label) + : offset(set_offset), + endianness(set_endianness), + size(set_size), + label(set_label) {} + + // The offset of the reference within the section. + size_t offset; + + // The endianness of the reference. + Endianness endianness; + + // The size of the reference. + size_t size; + + // The label to which this is a reference. + Label label; + }; + + // The default endianness of this section. + Endianness endianness_; + + // The contents of the section. + string contents_; + + // References to labels within those contents. + vector<Reference> references_; + + // A label referring to the beginning of the section. + Label start_; +}; + +} // namespace test_assembler +} // namespace lul_test + +namespace lul_test { + +using lul::DwarfPointerEncoding; +using lul_test::test_assembler::Endianness; +using lul_test::test_assembler::Label; +using lul_test::test_assembler::Section; + +class CFISection : public Section { + public: + // CFI augmentation strings beginning with 'z', defined by the + // Linux/IA-64 C++ ABI, can specify interesting encodings for + // addresses appearing in FDE headers and call frame instructions (and + // for additional fields whose presence the augmentation string + // specifies). In particular, pointers can be specified to be relative + // to various base address: the start of the .text section, the + // location holding the address itself, and so on. These allow the + // frame data to be position-independent even when they live in + // write-protected pages. These variants are specified at the + // following two URLs: + // + // http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/dwarfext.html + // http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html + // + // CFISection leaves the production of well-formed 'z'-augmented CIEs and + // FDEs to the user, but does provide EncodedPointer, to emit + // properly-encoded addresses for a given pointer encoding. + // EncodedPointer uses an instance of this structure to find the base + // addresses it should use; you can establish a default for all encoded + // pointers appended to this section with SetEncodedPointerBases. + struct EncodedPointerBases { + EncodedPointerBases() : cfi(), text(), data() {} + + // The starting address of this CFI section in memory, for + // DW_EH_PE_pcrel. DW_EH_PE_pcrel pointers may only be used in data + // that has is loaded into the program's address space. + uint64_t cfi; + + // The starting address of this file's .text section, for DW_EH_PE_textrel. + uint64_t text; + + // The starting address of this file's .got or .eh_frame_hdr section, + // for DW_EH_PE_datarel. + uint64_t data; + }; + + // Create a CFISection whose endianness is ENDIANNESS, and where + // machine addresses are ADDRESS_SIZE bytes long. If EH_FRAME is + // true, use the .eh_frame format, as described by the Linux + // Standards Base Core Specification, instead of the DWARF CFI + // format. + CFISection(Endianness endianness, size_t address_size, bool eh_frame = false) + : Section(endianness), + address_size_(address_size), + eh_frame_(eh_frame), + pointer_encoding_(lul::DW_EH_PE_absptr), + entry_length_(NULL), + in_fde_(false) { + // The 'start', 'Here', and 'Mark' members of a CFISection all refer + // to section offsets. + start() = 0; + } + + // Return this CFISection's address size. + size_t AddressSize() const { return address_size_; } + + // Return true if this CFISection uses the .eh_frame format, or + // false if it contains ordinary DWARF CFI data. + bool ContainsEHFrame() const { return eh_frame_; } + + // Use ENCODING for pointers in calls to FDEHeader and EncodedPointer. + void SetPointerEncoding(DwarfPointerEncoding encoding) { + pointer_encoding_ = encoding; + } + + // Use the addresses in BASES as the base addresses for encoded + // pointers in subsequent calls to FDEHeader or EncodedPointer. + // This function makes a copy of BASES. + void SetEncodedPointerBases(const EncodedPointerBases& bases) { + encoded_pointer_bases_ = bases; + } + + // Append a Common Information Entry header to this section with the + // given values. If dwarf64 is true, use the 64-bit DWARF initial + // length format for the CIE's initial length. Return a reference to + // this section. You should call FinishEntry after writing the last + // instruction for the CIE. + // + // Before calling this function, you will typically want to use Mark + // or Here to make a label to pass to FDEHeader that refers to this + // CIE's position in the section. + CFISection& CIEHeader(uint64_t code_alignment_factor, + int data_alignment_factor, + unsigned return_address_register, uint8_t version = 3, + const string& augmentation = "", bool dwarf64 = false); + + // Append a Frame Description Entry header to this section with the + // given values. If dwarf64 is true, use the 64-bit DWARF initial + // length format for the CIE's initial length. Return a reference to + // this section. You should call FinishEntry after writing the last + // instruction for the CIE. + // + // This function doesn't support entries that are longer than + // 0xffffff00 bytes. (The "initial length" is always a 32-bit + // value.) Nor does it support .debug_frame sections longer than + // 0xffffff00 bytes. + CFISection& FDEHeader(Label cie_pointer, uint64_t initial_location, + uint64_t address_range, bool dwarf64 = false); + + // Note the current position as the end of the last CIE or FDE we + // started, after padding with DW_CFA_nops for alignment. This + // defines the label representing the entry's length, cited in the + // entry's header. Return a reference to this section. + CFISection& FinishEntry(); + + // Append the contents of BLOCK as a DW_FORM_block value: an + // unsigned LEB128 length, followed by that many bytes of data. + CFISection& Block(const lul::ImageSlice& block) { + ULEB128(block.length_); + Append(block); + return *this; + } + + // Append data from CSTRING as a DW_FORM_block value: an unsigned LEB128 + // length, followed by that many bytes of data. The terminating zero is not + // included. + CFISection& Block(const char* cstring) { + ULEB128(strlen(cstring)); + Append(cstring); + return *this; + } + + // Append ADDRESS to this section, in the appropriate size and + // endianness. Return a reference to this section. + CFISection& Address(uint64_t address) { + Section::Append(endianness(), address_size_, address); + return *this; + } + + // Append ADDRESS to this section, using ENCODING and BASES. ENCODING + // defaults to this section's default encoding, established by + // SetPointerEncoding. BASES defaults to this section's bases, set by + // SetEncodedPointerBases. If the DW_EH_PE_indirect bit is set in the + // encoding, assume that ADDRESS is where the true address is stored. + // Return a reference to this section. + // + // (C++ doesn't let me use default arguments here, because I want to + // refer to members of *this in the default argument expression.) + CFISection& EncodedPointer(uint64_t address) { + return EncodedPointer(address, pointer_encoding_, encoded_pointer_bases_); + } + CFISection& EncodedPointer(uint64_t address, DwarfPointerEncoding encoding) { + return EncodedPointer(address, encoding, encoded_pointer_bases_); + } + CFISection& EncodedPointer(uint64_t address, DwarfPointerEncoding encoding, + const EncodedPointerBases& bases); + + // Restate some member functions, to keep chaining working nicely. + CFISection& Mark(Label* label) { + Section::Mark(label); + return *this; + } + CFISection& D8(uint8_t v) { + Section::D8(v); + return *this; + } + CFISection& D16(uint16_t v) { + Section::D16(v); + return *this; + } + CFISection& D16(Label v) { + Section::D16(v); + return *this; + } + CFISection& D32(uint32_t v) { + Section::D32(v); + return *this; + } + CFISection& D32(const Label& v) { + Section::D32(v); + return *this; + } + CFISection& D64(uint64_t v) { + Section::D64(v); + return *this; + } + CFISection& D64(const Label& v) { + Section::D64(v); + return *this; + } + CFISection& LEB128(long long v) { + Section::LEB128(v); + return *this; + } + CFISection& ULEB128(uint64_t v) { + Section::ULEB128(v); + return *this; + } + + private: + // A length value that we've appended to the section, but is not yet + // known. LENGTH is the appended value; START is a label referring + // to the start of the data whose length was cited. + struct PendingLength { + Label length; + Label start; + }; + + // Constants used in CFI/.eh_frame data: + + // If the first four bytes of an "initial length" are this constant, then + // the data uses the 64-bit DWARF format, and the length itself is the + // subsequent eight bytes. + static const uint32_t kDwarf64InitialLengthMarker = 0xffffffffU; + + // The CIE identifier for 32- and 64-bit DWARF CFI and .eh_frame data. + static const uint32_t kDwarf32CIEIdentifier = ~(uint32_t)0; + static const uint64_t kDwarf64CIEIdentifier = ~(uint64_t)0; + static const uint32_t kEHFrame32CIEIdentifier = 0; + static const uint64_t kEHFrame64CIEIdentifier = 0; + + // The size of a machine address for the data in this section. + size_t address_size_; + + // If true, we are generating a Linux .eh_frame section, instead of + // a standard DWARF .debug_frame section. + bool eh_frame_; + + // The encoding to use for FDE pointers. + DwarfPointerEncoding pointer_encoding_; + + // The base addresses to use when emitting encoded pointers. + EncodedPointerBases encoded_pointer_bases_; + + // The length value for the current entry. + // + // Oddly, this must be dynamically allocated. Labels never get new + // values; they only acquire constraints on the value they already + // have, or assert if you assign them something incompatible. So + // each header needs truly fresh Label objects to cite in their + // headers and track their positions. The alternative is explicit + // destructor invocation and a placement new. Ick. + PendingLength* entry_length_; + + // True if we are currently emitting an FDE --- that is, we have + // called FDEHeader but have not yet called FinishEntry. + bool in_fde_; + + // If in_fde_ is true, this is its starting address. We use this for + // emitting DW_EH_PE_funcrel pointers. + uint64_t fde_start_address_; +}; + +} // namespace lul_test + +#endif // LUL_TEST_INFRASTRUCTURE_H diff --git a/tools/profiler/tests/gtest/ThreadProfileTest.cpp b/tools/profiler/tests/gtest/ThreadProfileTest.cpp new file mode 100644 index 0000000000..b8a15c39b2 --- /dev/null +++ b/tools/profiler/tests/gtest/ThreadProfileTest.cpp @@ -0,0 +1,60 @@ + +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef MOZ_GECKO_PROFILER + +# include "ProfileBuffer.h" + +# include "mozilla/PowerOfTwo.h" +# include "mozilla/ProfileBufferChunkManagerWithLocalLimit.h" +# include "mozilla/ProfileChunkedBuffer.h" + +# include "gtest/gtest.h" + +// Make sure we can record one entry and read it +TEST(ThreadProfile, InsertOneEntry) +{ + mozilla::ProfileBufferChunkManagerWithLocalLimit chunkManager( + 2 * (1 + uint32_t(sizeof(ProfileBufferEntry))) * 4, + 2 * (1 + uint32_t(sizeof(ProfileBufferEntry)))); + mozilla::ProfileChunkedBuffer profileChunkedBuffer( + mozilla::ProfileChunkedBuffer::ThreadSafety::WithMutex, chunkManager); + auto pb = mozilla::MakeUnique<ProfileBuffer>(profileChunkedBuffer); + pb->AddEntry(ProfileBufferEntry::Time(123.1)); + ProfileBufferEntry entry = pb->GetEntry(pb->BufferRangeStart()); + ASSERT_TRUE(entry.IsTime()); + ASSERT_EQ(123.1, entry.GetDouble()); +} + +// See if we can insert some entries +TEST(ThreadProfile, InsertEntriesNoWrap) +{ + mozilla::ProfileBufferChunkManagerWithLocalLimit chunkManager( + 100 * (1 + uint32_t(sizeof(ProfileBufferEntry))), + 100 * (1 + uint32_t(sizeof(ProfileBufferEntry))) / 4); + mozilla::ProfileChunkedBuffer profileChunkedBuffer( + mozilla::ProfileChunkedBuffer::ThreadSafety::WithMutex, chunkManager); + auto pb = mozilla::MakeUnique<ProfileBuffer>(profileChunkedBuffer); + const int test_size = 50; + for (int i = 0; i < test_size; i++) { + pb->AddEntry(ProfileBufferEntry::Time(i)); + } + int times = 0; + uint64_t readPos = pb->BufferRangeStart(); + while (readPos != pb->BufferRangeEnd()) { + ProfileBufferEntry entry = pb->GetEntry(readPos); + readPos++; + if (entry.GetKind() == ProfileBufferEntry::Kind::INVALID) { + continue; + } + ASSERT_TRUE(entry.IsTime()); + ASSERT_EQ(times, entry.GetDouble()); + times++; + } + ASSERT_EQ(test_size, times); +} + +#endif // MOZ_GECKO_PROFILER diff --git a/tools/profiler/tests/gtest/moz.build b/tools/profiler/tests/gtest/moz.build new file mode 100644 index 0000000000..e1c3994621 --- /dev/null +++ b/tools/profiler/tests/gtest/moz.build @@ -0,0 +1,45 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. + +if ( + CONFIG["MOZ_GECKO_PROFILER"] + and CONFIG["OS_TARGET"] in ("Android", "Linux") + and CONFIG["TARGET_CPU"] + in ( + "arm", + "aarch64", + "x86", + "x86_64", + ) +): + UNIFIED_SOURCES += [ + "LulTest.cpp", + "LulTestDwarf.cpp", + "LulTestInfrastructure.cpp", + ] + +LOCAL_INCLUDES += [ + "/netwerk/base", + "/netwerk/protocol/http", + "/toolkit/components/jsoncpp/include", + "/tools/profiler/core", + "/tools/profiler/gecko", + "/tools/profiler/lul", +] + +if CONFIG["OS_TARGET"] != "Android": + UNIFIED_SOURCES += [ + "GeckoProfiler.cpp", + "ThreadProfileTest.cpp", + ] + +USE_LIBS += [ + "jsoncpp", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul-gtest" diff --git a/tools/profiler/tests/shared-head.js b/tools/profiler/tests/shared-head.js new file mode 100644 index 0000000000..d1b2f6868a --- /dev/null +++ b/tools/profiler/tests/shared-head.js @@ -0,0 +1,591 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* globals Assert */ +/* globals info */ + +/** + * This file contains utilities that can be shared between xpcshell tests and mochitests. + */ + +// The marker phases. +const INSTANT = 0; +const INTERVAL = 1; +const INTERVAL_START = 2; +const INTERVAL_END = 3; + +// This Services declaration may shadow another from head.js, so define it as +// a var rather than a const. + +const defaultSettings = { + entries: 8 * 1024 * 1024, // 8M entries = 64MB + interval: 1, // ms + features: [], + threads: ["GeckoMain"], +}; + +// Effectively `async`: Start the profiler and return the `startProfiler` +// promise that will get resolved when all child process have started their own +// profiler. +async function startProfiler(callersSettings) { + if (Services.profiler.IsActive()) { + Assert.ok( + Services.env.exists("MOZ_PROFILER_STARTUP"), + "The profiler is active at the begining of the test, " + + "the MOZ_PROFILER_STARTUP environment variable should be set." + ); + if (Services.env.exists("MOZ_PROFILER_STARTUP")) { + // If the startup profiling environment variable exists, it is likely + // that tests are being profiled. + // Stop the profiler before starting profiler tests. + info( + "This test starts and stops the profiler and is not compatible " + + "with the use of MOZ_PROFILER_STARTUP. " + + "Stopping the profiler before starting the test." + ); + await Services.profiler.StopProfiler(); + } else { + throw new Error( + "The profiler must not be active before starting it in a test." + ); + } + } + const settings = Object.assign({}, defaultSettings, callersSettings); + return Services.profiler.StartProfiler( + settings.entries, + settings.interval, + settings.features, + settings.threads, + 0, + settings.duration + ); +} + +function startProfilerForMarkerTests() { + return startProfiler({ + features: ["nostacksampling", "js"], + threads: ["GeckoMain", "DOM Worker"], + }); +} + +/** + * This is a helper function be able to run `await wait(500)`. Unfortunately + * this is needed as the act of collecting functions relies on the periodic + * sampling of the threads. See: + * https://bugzilla.mozilla.org/show_bug.cgi?id=1529053 + * + * @param {number} time + * @returns {Promise} + */ +function wait(time) { + return new Promise(resolve => { + // eslint-disable-next-line mozilla/no-arbitrary-setTimeout + setTimeout(resolve, time); + }); +} + +/** + * Get the payloads of a type recursively, including from all subprocesses. + * + * @param {Object} profile The gecko profile. + * @param {string} type The marker payload type, e.g. "DiskIO". + * @param {Array} payloadTarget The recursive list of payloads. + * @return {Array} The final payloads. + */ +function getPayloadsOfTypeFromAllThreads(profile, type, payloadTarget = []) { + for (const { markers } of profile.threads) { + for (const markerTuple of markers.data) { + const payload = markerTuple[markers.schema.data]; + if (payload && payload.type === type) { + payloadTarget.push(payload); + } + } + } + + for (const subProcess of profile.processes) { + getPayloadsOfTypeFromAllThreads(subProcess, type, payloadTarget); + } + + return payloadTarget; +} + +/** + * Get the payloads of a type from a single thread. + * + * @param {Object} thread The thread from a profile. + * @param {string} type The marker payload type, e.g. "DiskIO". + * @return {Array} The payloads. + */ +function getPayloadsOfType(thread, type) { + const { markers } = thread; + const results = []; + for (const markerTuple of markers.data) { + const payload = markerTuple[markers.schema.data]; + if (payload && payload.type === type) { + results.push(payload); + } + } + return results; +} + +/** + * Applies the marker schema to create individual objects for each marker + * + * @param {Object} thread The thread from a profile. + * @return {InflatedMarker[]} The markers. + */ +function getInflatedMarkerData(thread) { + const { markers, stringTable } = thread; + return markers.data.map(markerTuple => { + const marker = {}; + for (const [key, tupleIndex] of Object.entries(markers.schema)) { + marker[key] = markerTuple[tupleIndex]; + if (key === "name") { + // Use the string from the string table. + marker[key] = stringTable[marker[key]]; + } + } + return marker; + }); +} + +/** + * Applies the marker schema to create individual objects for each marker, then + * keeps only the network markers that match the profiler tests. + * + * @param {Object} thread The thread from a profile. + * @return {InflatedMarker[]} The filtered network markers. + */ +function getInflatedNetworkMarkers(thread) { + const markers = getInflatedMarkerData(thread); + return markers.filter( + m => + m.data && + m.data.type === "Network" && + // We filter out network markers that aren't related to the test, to + // avoid intermittents. + m.data.URI.includes("/tools/profiler/") + ); +} + +/** + * From a list of network markers, this returns pairs of start/stop markers. + * If a stop marker can't be found for a start marker, this will return an array + * of only 1 element. + * + * @param {InflatedMarker[]} networkMarkers Network markers + * @return {InflatedMarker[][]} Pairs of network markers + */ +function getPairsOfNetworkMarkers(allNetworkMarkers) { + // For each 'start' marker we want to find the next 'stop' or 'redirect' + // marker with the same id. + const result = []; + const mapOfStartMarkers = new Map(); // marker id -> id in result array + for (const marker of allNetworkMarkers) { + const { data } = marker; + if (data.status === "STATUS_START") { + if (mapOfStartMarkers.has(data.id)) { + const previousMarker = result[mapOfStartMarkers.get(data.id)][0]; + Assert.ok( + false, + `We found 2 start markers with the same id ${data.id}, without end marker in-between.` + + `The first marker has URI ${previousMarker.data.URI}, the second marker has URI ${data.URI}.` + + ` This should not happen.` + ); + continue; + } + + mapOfStartMarkers.set(data.id, result.length); + result.push([marker]); + } else { + // STOP or REDIRECT + if (!mapOfStartMarkers.has(data.id)) { + Assert.ok( + false, + `We found an end marker without a start marker (id: ${data.id}, URI: ${data.URI}). This should not happen.` + ); + continue; + } + result[mapOfStartMarkers.get(data.id)].push(marker); + mapOfStartMarkers.delete(data.id); + } + } + + return result; +} + +/** + * It can be helpful to force the profiler to collect a JavaScript sample. This + * function spins on a while loop until at least one more sample is collected. + * + * @return {number} The index of the collected sample. + */ +function captureAtLeastOneJsSample() { + function getProfileSampleCount() { + const profile = Services.profiler.getProfileData(); + return profile.threads[0].samples.data.length; + } + + const sampleCount = getProfileSampleCount(); + // Create an infinite loop until a sample has been collected. + while (true) { + if (sampleCount < getProfileSampleCount()) { + return sampleCount; + } + } +} + +function isJSONWhitespace(c) { + return ["\n", "\r", " ", "\t"].includes(c); +} + +function verifyJSONStringIsCompact(s) { + const stateData = 0; + const stateString = 1; + const stateEscapedChar = 2; + let state = stateData; + for (let i = 0; i < s.length; ++i) { + let c = s[i]; + switch (state) { + case stateData: + if (isJSONWhitespace(c)) { + Assert.ok( + false, + `"Unexpected JSON whitespace at index ${i} in profile: <<<${s}>>>"` + ); + return; + } + if (c == '"') { + state = stateString; + } + break; + case stateString: + if (c == '"') { + state = stateData; + } else if (c == "\\") { + state = stateEscapedChar; + } + break; + case stateEscapedChar: + state = stateString; + break; + } + } +} + +/** + * This function pauses the profiler before getting the profile. Then after + * getting the data, the profiler is stopped, and all profiler data is removed. + * @returns {Promise<Profile>} + */ +async function stopNowAndGetProfile() { + // Don't await the pause, because each process will handle it before it + // receives the following `getProfileDataAsArrayBuffer()`. + Services.profiler.Pause(); + + const profileArrayBuffer = + await Services.profiler.getProfileDataAsArrayBuffer(); + await Services.profiler.StopProfiler(); + + const profileUint8Array = new Uint8Array(profileArrayBuffer); + const textDecoder = new TextDecoder("utf-8", { fatal: true }); + const profileString = textDecoder.decode(profileUint8Array); + verifyJSONStringIsCompact(profileString); + + return JSON.parse(profileString); +} + +/** + * This function ensures there's at least one sample, then pauses the profiler + * before getting the profile. Then after getting the data, the profiler is + * stopped, and all profiler data is removed. + * @returns {Promise<Profile>} + */ +async function waitSamplingAndStopAndGetProfile() { + await Services.profiler.waitOnePeriodicSampling(); + return stopNowAndGetProfile(); +} + +/** + * Verifies that a marker is an interval marker. + * + * @param {InflatedMarker} marker + * @returns {boolean} + */ +function isIntervalMarker(inflatedMarker) { + return ( + inflatedMarker.phase === 1 && + typeof inflatedMarker.startTime === "number" && + typeof inflatedMarker.endTime === "number" + ); +} + +/** + * @param {Profile} profile + * @returns {Thread[]} + */ +function getThreads(profile) { + const threads = []; + + function getThreadsRecursive(process) { + for (const thread of process.threads) { + threads.push(thread); + } + for (const subprocess of process.processes) { + getThreadsRecursive(subprocess); + } + } + + getThreadsRecursive(profile); + return threads; +} + +/** + * Find a specific marker schema from any process of a profile. + * + * @param {Profile} profile + * @param {string} name + * @returns {MarkerSchema} + */ +function getSchema(profile, name) { + { + const schema = profile.meta.markerSchema.find(s => s.name === name); + if (schema) { + return schema; + } + } + for (const subprocess of profile.processes) { + const schema = subprocess.meta.markerSchema.find(s => s.name === name); + if (schema) { + return schema; + } + } + console.error("Parent process schema", profile.meta.markerSchema); + for (const subprocess of profile.processes) { + console.error("Child process schema", subprocess.meta.markerSchema); + } + throw new Error(`Could not find a schema for "${name}".`); +} + +/** + * This escapes all characters that have a special meaning in RegExps. + * This was stolen from https://github.com/sindresorhus/escape-string-regexp and + * so it is licence MIT and: + * Copyright (c) Sindre Sorhus <sindresorhus@gmail.com> (https://sindresorhus.com). + * See the full license in https://raw.githubusercontent.com/sindresorhus/escape-string-regexp/main/license. + * @param {string} string The string to be escaped + * @returns {string} The result + */ +function escapeStringRegexp(string) { + if (typeof string !== "string") { + throw new TypeError("Expected a string"); + } + + // Escape characters with special meaning either inside or outside character + // sets. Use a simple backslash escape when it’s always valid, and a `\xnn` + // escape when the simpler form would be disallowed by Unicode patterns’ + // stricter grammar. + return string.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&").replace(/-/g, "\\x2d"); +} + +/** ------ Assertions helper ------ */ +/** + * This assert helper function makes it easy to check a lot of properties in an + * object. We augment Assert.sys.mjs to make it easier to use. + */ +Object.assign(Assert, { + /* + * It checks if the properties on the right are all present in the object on + * the left. Note that the object might still have other properties (see + * objectContainsOnly below if you want the stricter form). + * + * The basic form does basic equality on each expected property: + * + * Assert.objectContains(fixture, { + * foo: "foo", + * bar: 1, + * baz: true, + * }); + * + * But it also has a more powerful form with expectations. The available + * expectations are: + * - any(): this only checks for the existence of the property, not its value + * - number(), string(), boolean(), bigint(), function(), symbol(), object(): + * this checks if the value is of this type + * - objectContains(expected): this applies Assert.objectContains() + * recursively on this property. + * - stringContains(needle): this checks if the expected value is included in + * the property value. + * - stringMatches(regexp): this checks if the property value matches this + * regexp. The regexp can be passed as a string, to be dynamically built. + * + * example: + * + * Assert.objectContains(fixture, { + * name: Expect.stringMatches(`Load \\d+:.*${url}`), + * data: Expect.objectContains({ + * status: "STATUS_STOP", + * URI: Expect.stringContains("https://"), + * requestMethod: "GET", + * contentType: Expect.string(), + * startTime: Expect.number(), + * cached: Expect.boolean(), + * }), + * }); + * + * Each expectation will translate into one or more Assert call. Therefore if + * one expectation fails, this will be clearly visible in the test output. + * + * Expectations can also be normal functions, for example: + * + * Assert.objectContains(fixture, { + * number: value => Assert.greater(value, 5) + * }); + * + * Note that you'll need to use Assert inside this function. + */ + objectContains(object, expectedProperties) { + // Basic tests: we don't want to run other assertions if these tests fail. + if (typeof object !== "object") { + this.ok( + false, + `The first parameter should be an object, but found: ${object}.` + ); + return; + } + + if (typeof expectedProperties !== "object") { + this.ok( + false, + `The second parameter should be an object, but found: ${expectedProperties}.` + ); + return; + } + + for (const key of Object.keys(expectedProperties)) { + const expected = expectedProperties[key]; + if (!(key in object)) { + this.report( + true, + object, + expectedProperties, + `The object should contain the property "${key}", but it's missing.` + ); + continue; + } + + if (typeof expected === "function") { + // This is a function, so let's call it. + expected( + object[key], + `The object should contain the property "${key}" with an expected value and type.` + ); + } else { + // Otherwise, we check for equality. + this.equal( + object[key], + expectedProperties[key], + `The object should contain the property "${key}" with an expected value.` + ); + } + } + }, + + /** + * This is very similar to the previous `objectContains`, but this also looks + * at the number of the objects' properties. Thus this will fail if the + * objects don't have the same properties exactly. + */ + objectContainsOnly(object, expectedProperties) { + // Basic tests: we don't want to run other assertions if these tests fail. + if (typeof object !== "object") { + this.ok( + false, + `The first parameter should be an object but found: ${object}.` + ); + return; + } + + if (typeof expectedProperties !== "object") { + this.ok( + false, + `The second parameter should be an object but found: ${expectedProperties}.` + ); + return; + } + + // In objectContainsOnly, we specifically want to check if all properties + // from the fixture object are expected. + // We'll be failing a test only for the specific properties that weren't + // expected, and only fail with one message, so that the test outputs aren't + // spammed. + const extraProperties = []; + for (const fixtureKey of Object.keys(object)) { + if (!(fixtureKey in expectedProperties)) { + extraProperties.push(fixtureKey); + } + } + + if (extraProperties.length) { + // Some extra properties have been found. + this.report( + true, + object, + expectedProperties, + `These properties are present, but shouldn't: "${extraProperties.join( + '", "' + )}".` + ); + } + + // Now, let's carry on the rest of our work. + this.objectContains(object, expectedProperties); + }, +}); + +const Expect = { + any: + () => + actual => {} /* We don't check anything more than the presence of this property. */, +}; + +/* These functions are part of the Assert object, and we want to reuse them. */ +[ + "stringContains", + "stringMatches", + "objectContains", + "objectContainsOnly", +].forEach( + assertChecker => + (Expect[assertChecker] = + expected => + (actual, ...moreArgs) => + Assert[assertChecker](actual, expected, ...moreArgs)) +); + +/* These functions will only check for the type. */ +[ + "number", + "string", + "boolean", + "bigint", + "symbol", + "object", + "function", +].forEach(type => (Expect[type] = makeTypeChecker(type))); + +function makeTypeChecker(type) { + return (...unexpectedArgs) => { + if (unexpectedArgs.length) { + throw new Error( + "Type checkers expectations aren't expecting any argument." + ); + } + return (actual, message) => { + const isCorrect = typeof actual === type; + Assert.report(!isCorrect, actual, type, message, "has type"); + }; + }; +} +/* ------ End of assertion helper ------ */ diff --git a/tools/profiler/tests/xpcshell/head.js b/tools/profiler/tests/xpcshell/head.js new file mode 100644 index 0000000000..ce87b32fd5 --- /dev/null +++ b/tools/profiler/tests/xpcshell/head.js @@ -0,0 +1,244 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* import-globals-from ../shared-head.js */ + +// This Services declaration may shadow another from head.js, so define it as +// a var rather than a const. + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); +const { setTimeout } = ChromeUtils.importESModule( + "resource://gre/modules/Timer.sys.mjs" +); + +// Load the shared head +const sharedHead = do_get_file("shared-head.js", false); +if (!sharedHead) { + throw new Error("Could not load the shared head."); +} +Services.scriptloader.loadSubScript( + Services.io.newFileURI(sharedHead).spec, + this +); + +/** + * This function takes a thread, and a sample tuple from the "data" array, and + * inflates the frame to be an array of strings. + * + * @param {Object} thread - The thread from the profile. + * @param {Array} sample - The tuple from the thread.samples.data array. + * @returns {Array<string>} An array of function names. + */ +function getInflatedStackLocations(thread, sample) { + let stackTable = thread.stackTable; + let frameTable = thread.frameTable; + let stringTable = thread.stringTable; + let SAMPLE_STACK_SLOT = thread.samples.schema.stack; + let STACK_PREFIX_SLOT = stackTable.schema.prefix; + let STACK_FRAME_SLOT = stackTable.schema.frame; + let FRAME_LOCATION_SLOT = frameTable.schema.location; + + // Build the stack from the raw data and accumulate the locations in + // an array. + let stackIndex = sample[SAMPLE_STACK_SLOT]; + let locations = []; + while (stackIndex !== null) { + let stackEntry = stackTable.data[stackIndex]; + let frame = frameTable.data[stackEntry[STACK_FRAME_SLOT]]; + locations.push(stringTable[frame[FRAME_LOCATION_SLOT]]); + stackIndex = stackEntry[STACK_PREFIX_SLOT]; + } + + // The profiler tree is inverted, so reverse the array. + return locations.reverse(); +} + +/** + * This utility matches up stacks to see if they contain a certain sequence of + * stack frames. A correctly functioning profiler will have a certain sequence + * of stacks, but we can't always determine exactly which stacks will show up + * due to implementation changes, as well as memory addresses being arbitrary to + * that particular build. + * + * This function triggers a test failure with a nice debug message when it + * fails. + * + * @param {Array<string>} actualStackFrames - As generated by + * inflatedStackFrames. + * @param {Array<string | RegExp>} expectedStackFrames - Matches a subset of + * actualStackFrames + */ +function expectStackToContain( + actualStackFrames, + expectedStackFrames, + message = "The actual stack and expected stack do not match." +) { + // Log the stacks that are being passed to this assertion, as it could be + // useful for when these tests fail. + console.log("Actual stack: ", actualStackFrames); + console.log( + "Expected to contain: ", + expectedStackFrames.map(s => s.toString()) + ); + + let actualIndex = 0; + + // Start walking the expected stack and look for matches. + for ( + let expectedIndex = 0; + expectedIndex < expectedStackFrames.length; + expectedIndex++ + ) { + const expectedStackFrame = expectedStackFrames[expectedIndex]; + + while (true) { + // Make sure that we haven't run out of actual stack frames. + if (actualIndex >= actualStackFrames.length) { + info(`Could not find a match for: "${expectedStackFrame.toString()}"`); + Assert.ok(false, message); + } + + const actualStackFrame = actualStackFrames[actualIndex]; + actualIndex++; + + const itMatches = + typeof expectedStackFrame === "string" + ? expectedStackFrame === actualStackFrame + : actualStackFrame.match(expectedStackFrame); + + if (itMatches) { + // We found a match, break out of this loop. + break; + } + // Keep on looping looking for a match. + } + } + + Assert.ok(true, message); +} + +/** + * @param {Thread} thread + * @param {string} filename - The filename used to trigger FileIO. + * @returns {InflatedMarkers[]} + */ +function getInflatedFileIOMarkers(thread, filename) { + const markers = getInflatedMarkerData(thread); + return markers.filter( + marker => + marker.data?.type === "FileIO" && + marker.data?.filename?.endsWith(filename) + ); +} + +/** + * Checks properties common to all FileIO markers. + * + * @param {InflatedMarkers[]} markers + * @param {string} filename + */ +function checkInflatedFileIOMarkers(markers, filename) { + greater(markers.length, 0, "Found some markers"); + + // See IOInterposeObserver::Observation::ObservedOperationString + const validOperations = new Set([ + "write", + "fsync", + "close", + "stat", + "create/open", + "read", + ]); + const validSources = new Set(["PoisonIOInterposer", "NSPRIOInterposer"]); + + for (const marker of markers) { + try { + ok( + marker.name.startsWith("FileIO"), + "Has a marker.name that starts with FileIO" + ); + equal(marker.data.type, "FileIO", "Has a marker.data.type"); + ok(isIntervalMarker(marker), "All FileIO markers are interval markers"); + ok( + validOperations.has(marker.data.operation), + `The markers have a known operation - "${marker.data.operation}"` + ); + ok( + validSources.has(marker.data.source), + `The FileIO marker has a known source "${marker.data.source}"` + ); + ok(marker.data.filename.endsWith(filename)); + ok(Boolean(marker.data.stack), "A stack was collected"); + } catch (error) { + console.error("Failing inflated FileIO marker:", marker); + throw error; + } + } +} + +/** + * Do deep equality checks for schema, but then surface nice errors for a user to know + * what to do if the check fails. + */ +function checkSchema(actual, expected) { + const schemaName = expected.name; + info(`Checking marker schema for "${schemaName}"`); + + try { + ok( + actual, + `Schema was found for "${schemaName}". See the test output for more information.` + ); + // Check individual properties to surface easier to debug errors. + deepEqual( + expected.display, + actual.display, + `The "display" property for ${schemaName} schema matches. See the test output for more information.` + ); + if (expected.data) { + ok(actual.data, `Schema was found for "${schemaName}"`); + for (const expectedDatum of expected.data) { + const actualDatum = actual.data.find(d => d.key === expectedDatum.key); + deepEqual( + expectedDatum, + actualDatum, + `The "${schemaName}" field "${expectedDatum.key}" matches expectations. See the test output for more information.` + ); + } + equal( + expected.data.length, + actual.data.length, + "The expected and actual data have the same number of items" + ); + } + + // Finally do a true deep equal. + deepEqual(expected, actual, "The entire schema is deepEqual"); + } catch (error) { + // The test results are not very human readable. This is a bit of a hacky + // solution to make it more readable. + dump("-----------------------------------------------------\n"); + dump("The expected marker schema:\n"); + dump("-----------------------------------------------------\n"); + dump(JSON.stringify(expected, null, 2)); + dump("\n"); + dump("-----------------------------------------------------\n"); + dump("The actual marker schema:\n"); + dump("-----------------------------------------------------\n"); + dump(JSON.stringify(actual, null, 2)); + dump("\n"); + dump("-----------------------------------------------------\n"); + dump("A marker schema was not equal to expectations. If you\n"); + dump("are modifying the schema, then please copy and paste\n"); + dump("the new schema into this test.\n"); + dump("-----------------------------------------------------\n"); + dump("Copy this: " + JSON.stringify(actual)); + dump("\n"); + dump("-----------------------------------------------------\n"); + + throw error; + } +} diff --git a/tools/profiler/tests/xpcshell/test_active_configuration.js b/tools/profiler/tests/xpcshell/test_active_configuration.js new file mode 100644 index 0000000000..c4336f3f32 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_active_configuration.js @@ -0,0 +1,115 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + info( + "Checking that the profiler can fetch the information about the active " + + "configuration that is being used to power the profiler." + ); + + equal( + Services.profiler.activeConfiguration, + null, + "When the profile is off, there is no active configuration." + ); + + { + info("Start the profiler."); + const entries = 10000; + const interval = 1; + const threads = ["GeckoMain"]; + const features = ["js"]; + const activeTabID = 123; + await Services.profiler.StartProfiler( + entries, + interval, + features, + threads, + activeTabID + ); + + info("Generate the activeConfiguration."); + const { activeConfiguration } = Services.profiler; + const expectedConfiguration = { + interval, + threads, + features, + activeTabID, + // The buffer is created as a power of two that can fit all of the entires + // into it. If the ratio of entries to buffer size ever changes, this setting + // will need to be updated. + capacity: Math.pow(2, 14), + }; + + deepEqual( + activeConfiguration, + expectedConfiguration, + "The active configuration matches configuration given." + ); + + info("Get the profile."); + const profile = Services.profiler.getProfileData(); + deepEqual( + profile.meta.configuration, + expectedConfiguration, + "The configuration also matches on the profile meta object." + ); + } + + { + const entries = 20000; + const interval = 0.5; + const threads = ["GeckoMain", "DOM Worker"]; + const features = []; + const activeTabID = 111; + const duration = 20; + + info("Restart the profiler with a new configuration."); + await Services.profiler.StartProfiler( + entries, + interval, + features, + threads, + activeTabID, + // Also start it with duration, this property is optional. + duration + ); + + info("Generate the activeConfiguration."); + const { activeConfiguration } = Services.profiler; + const expectedConfiguration = { + interval, + threads, + features, + activeTabID, + duration, + // The buffer is created as a power of two that can fit all of the entires + // into it. If the ratio of entries to buffer size ever changes, this setting + // will need to be updated. + capacity: Math.pow(2, 15), + }; + + deepEqual( + activeConfiguration, + expectedConfiguration, + "The active configuration matches the new configuration." + ); + + info("Get the profile."); + const profile = Services.profiler.getProfileData(); + deepEqual( + profile.meta.configuration, + expectedConfiguration, + "The configuration also matches on the profile meta object." + ); + } + + await Services.profiler.StopProfiler(); + + equal( + Services.profiler.activeConfiguration, + null, + "When the profile is off, there is no active configuration." + ); +}); diff --git a/tools/profiler/tests/xpcshell/test_addProfilerMarker.js b/tools/profiler/tests/xpcshell/test_addProfilerMarker.js new file mode 100644 index 0000000000..b11545a41c --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_addProfilerMarker.js @@ -0,0 +1,221 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that ChromeUtils.addProfilerMarker is working correctly. + */ + +const markerNamePrefix = "test_addProfilerMarker"; +const markerText = "Text payload"; +// The same startTime will be used for all markers with a duration, +// and we store this value globally so that expectDuration and +// expectNoDuration can access it. The value isn't set here as we +// want a start time after the profiler has started +var startTime; + +function expectNoDuration(marker) { + Assert.equal( + typeof marker.startTime, + "number", + "startTime should be a number" + ); + Assert.greater( + marker.startTime, + startTime, + "startTime should be after the begining of the test" + ); + Assert.equal(typeof marker.endTime, "number", "endTime should be a number"); + Assert.equal(marker.endTime, 0, "endTime should be 0"); +} + +function expectDuration(marker) { + Assert.equal( + typeof marker.startTime, + "number", + "startTime should be a number" + ); + // Floats can cause rounding issues. We've seen up to a 4.17e-5 difference in + // intermittent failures, so we are permissive and accept up to 5e-5. + Assert.less( + Math.abs(marker.startTime - startTime), + 5e-5, + "startTime should be the expected time" + ); + Assert.equal(typeof marker.endTime, "number", "endTime should be a number"); + Assert.greater( + marker.endTime, + startTime, + "endTime should be after startTime" + ); +} + +function expectNoData(marker) { + Assert.equal( + typeof marker.data, + "undefined", + "The data property should be undefined" + ); +} + +function expectText(marker) { + Assert.equal( + typeof marker.data, + "object", + "The data property should be an object" + ); + Assert.equal(marker.data.type, "Text", "Should be a Text marker"); + Assert.equal( + marker.data.name, + markerText, + "The payload should contain the expected text" + ); +} + +function expectNoStack(marker) { + Assert.ok(!marker.data || !marker.data.stack, "There should be no stack"); +} + +function expectStack(marker, thread) { + let stack = marker.data.stack; + Assert.ok(!!stack, "There should be a stack"); + + // Marker stacks are recorded as a profile of a thread with a single sample, + // get the stack id. + stack = stack.samples.data[0][stack.samples.schema.stack]; + + const stackPrefixCol = thread.stackTable.schema.prefix; + const stackFrameCol = thread.stackTable.schema.frame; + const frameLocationCol = thread.frameTable.schema.location; + + // Get the entire stack in an array for easier processing. + let result = []; + while (stack != null) { + let stackEntry = thread.stackTable.data[stack]; + let frame = thread.frameTable.data[stackEntry[stackFrameCol]]; + result.push(thread.stringTable[frame[frameLocationCol]]); + stack = stackEntry[stackPrefixCol]; + } + + Assert.greaterOrEqual( + result.length, + 1, + "There should be at least one frame in the stack" + ); + + Assert.ok( + result.some(frame => frame.includes("testMarker")), + "the 'testMarker' function should be visible in the stack" + ); + + Assert.ok( + !result.some(frame => frame.includes("ChromeUtils.addProfilerMarker")), + "the 'ChromeUtils.addProfilerMarker' label frame should not be visible in the stack" + ); +} + +add_task(async () => { + startProfilerForMarkerTests(); + startTime = Cu.now(); + while (Cu.now() < startTime + 1) { + // Busy wait for 1ms to ensure the intentionally set start time of markers + // will be significantly different from the time at which the marker is + // recorded. + } + info("startTime used for markers with durations: " + startTime); + + /* Each call to testMarker will record a marker with a unique name. + * The testFunctions and testCases objects contain respectively test + * functions to verify that the marker found in the captured profile + * matches expectations, and a string that can be printed to describe + * in which way ChromeUtils.addProfilerMarker was called. */ + let testFunctions = {}; + let testCases = {}; + let markerId = 0; + function testMarker(args, checks) { + let name = markerNamePrefix + markerId++; + ChromeUtils.addProfilerMarker(name, ...args); + testFunctions[name] = checks; + testCases[name] = `ChromeUtils.addProfilerMarker(${[name, ...args] + .toSource() + .slice(1, -1)})`; + } + + info("Record markers without options object."); + testMarker([], m => { + expectNoDuration(m); + expectNoData(m); + }); + testMarker([startTime], m => { + expectDuration(m); + expectNoData(m); + }); + testMarker([undefined, markerText], m => { + expectNoDuration(m); + expectText(m); + }); + testMarker([startTime, markerText], m => { + expectDuration(m); + expectText(m); + }); + + info("Record markers providing the duration as the startTime property."); + testMarker([{ startTime }], m => { + expectDuration(m); + expectNoData(m); + }); + testMarker([{}, markerText], m => { + expectNoDuration(m); + expectText(m); + }); + testMarker([{ startTime }, markerText], m => { + expectDuration(m); + expectText(m); + }); + + info("Record markers to test the captureStack property."); + const captureStack = true; + testMarker([], expectNoStack); + testMarker([startTime, markerText], expectNoStack); + testMarker([{ captureStack: false }], expectNoStack); + testMarker([{ captureStack }], expectStack); + testMarker([{ startTime, captureStack }], expectStack); + testMarker([{ captureStack }, markerText], expectStack); + testMarker([{ startTime, captureStack }, markerText], expectStack); + + info("Record markers to test the category property"); + function testCategory(args, expectedCategory) { + testMarker(args, marker => { + Assert.equal(marker.category, expectedCategory); + }); + } + testCategory([], "JavaScript"); + testCategory([{ category: "Test" }], "Test"); + testCategory([{ category: "Test" }, markerText], "Test"); + testCategory([{ category: "JavaScript" }], "JavaScript"); + testCategory([{ category: "Other" }], "Other"); + testCategory([{ category: "DOM" }], "DOM"); + testCategory([{ category: "does not exist" }], "Other"); + + info("Capture the profile"); + const profile = await stopNowAndGetProfile(); + const mainThread = profile.threads.find(({ name }) => name === "GeckoMain"); + const markers = getInflatedMarkerData(mainThread).filter(m => + m.name.startsWith(markerNamePrefix) + ); + Assert.equal( + markers.length, + Object.keys(testFunctions).length, + `Found ${markers.length} test markers in the captured profile` + ); + + for (let marker of markers) { + marker.category = profile.meta.categories[marker.category].name; + info(`${testCases[marker.name]} -> ${marker.toSource()}`); + + testFunctions[marker.name](marker, mainThread); + delete testFunctions[marker.name]; + } + + Assert.equal(0, Object.keys(testFunctions).length, "all markers were found"); +}); diff --git a/tools/profiler/tests/xpcshell/test_asm.js b/tools/profiler/tests/xpcshell/test_asm.js new file mode 100644 index 0000000000..ced36ce429 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_asm.js @@ -0,0 +1,76 @@ +// Check that asm.js code shows up on the stack. +add_task(async () => { + // This test assumes that it's starting on an empty profiler stack. + // (Note that the other profiler tests also assume the profiler + // isn't already started.) + Assert.ok(!Services.profiler.IsActive()); + + let jsFuns = Cu.getJSTestingFunctions(); + if (!jsFuns.isAsmJSCompilationAvailable()) { + return; + } + + const ms = 10; + await Services.profiler.StartProfiler(10000, ms, ["js"]); + + let stack = null; + function ffi_function() { + var delayMS = 5; + while (1) { + let then = Date.now(); + do { + // do nothing + } while (Date.now() - then < delayMS); + + var thread0 = Services.profiler.getProfileData().threads[0]; + + if (delayMS > 30000) { + return; + } + + delayMS *= 2; + + if (!thread0.samples.data.length) { + continue; + } + + var lastSample = thread0.samples.data[thread0.samples.data.length - 1]; + stack = String(getInflatedStackLocations(thread0, lastSample)); + if (stack.includes("trampoline")) { + return; + } + } + } + + function asmjs_module(global, ffis) { + "use asm"; + var ffi = ffis.ffi; + function asmjs_function() { + ffi(); + } + return asmjs_function; + } + + Assert.ok(jsFuns.isAsmJSModule(asmjs_module)); + + var asmjs_function = asmjs_module(null, { ffi: ffi_function }); + Assert.ok(jsFuns.isAsmJSFunction(asmjs_function)); + + asmjs_function(); + + Assert.notEqual(stack, null); + + var i1 = stack.indexOf("entry trampoline"); + Assert.ok(i1 !== -1); + var i2 = stack.indexOf("asmjs_function"); + Assert.ok(i2 !== -1); + var i3 = stack.indexOf("exit trampoline"); + Assert.ok(i3 !== -1); + var i4 = stack.indexOf("ffi_function"); + Assert.ok(i4 !== -1); + Assert.ok(i1 < i2); + Assert.ok(i2 < i3); + Assert.ok(i3 < i4); + + await Services.profiler.StopProfiler(); +}); diff --git a/tools/profiler/tests/xpcshell/test_assertion_helper.js b/tools/profiler/tests/xpcshell/test_assertion_helper.js new file mode 100644 index 0000000000..baa4c34818 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_assertion_helper.js @@ -0,0 +1,162 @@ +add_task(function setup() { + // With the default reporter, an assertion doesn't throw if it fails, it + // merely report the result to the reporter and then go on. But in this test + // we want that a failure really throws, so that we can actually assert that + // it throws in case of failures! + // That's why we disable the default repoter here. + // I noticed that this line needs to be in an add_task (or possibly run_test) + // function. If put outside this will crash the test. + Assert.setReporter(null); +}); + +add_task(function test_objectContains() { + const fixture = { + foo: "foo", + bar: "bar", + }; + + Assert.objectContains(fixture, { foo: "foo" }, "Matches one property value"); + Assert.objectContains( + fixture, + { foo: "foo", bar: "bar" }, + "Matches both properties" + ); + Assert.objectContainsOnly( + fixture, + { foo: "foo", bar: "bar" }, + "Matches both properties" + ); + Assert.throws( + () => Assert.objectContainsOnly(fixture, { foo: "foo" }), + /AssertionError/, + "Fails if some properties are missing" + ); + Assert.throws( + () => Assert.objectContains(fixture, { foo: "bar" }), + /AssertionError/, + "Fails if the value for a present property is wrong" + ); + Assert.throws( + () => Assert.objectContains(fixture, { hello: "world" }), + /AssertionError/, + "Fails if an expected property is missing" + ); + Assert.throws( + () => Assert.objectContains(fixture, { foo: "foo", hello: "world" }), + /AssertionError/, + "Fails if some properties are present but others are missing" + ); +}); + +add_task(function test_objectContains_expectations() { + const fixture = { + foo: "foo", + bar: "bar", + num: 42, + nested: { + nestedFoo: "nestedFoo", + nestedBar: "nestedBar", + }, + }; + + Assert.objectContains( + fixture, + { + foo: Expect.stringMatches(/^fo/), + bar: Expect.stringContains("ar"), + num: Expect.number(), + nested: Expect.objectContainsOnly({ + nestedFoo: Expect.stringMatches(/[Ff]oo/), + nestedBar: Expect.stringMatches(/[Bb]ar/), + }), + }, + "Supports expectations" + ); + Assert.objectContainsOnly( + fixture, + { + foo: Expect.stringMatches(/^fo/), + bar: Expect.stringContains("ar"), + num: Expect.number(), + nested: Expect.objectContains({ + nestedFoo: Expect.stringMatches(/[Ff]oo/), + }), + }, + "Supports expectations" + ); + + Assert.objectContains(fixture, { + num: val => Assert.greater(val, 40), + }); + + // Failed expectations + Assert.throws( + () => + Assert.objectContains(fixture, { + foo: Expect.stringMatches(/bar/), + }), + /AssertionError/, + "Expect.stringMatches shouldn't match when the value is unexpected" + ); + Assert.throws( + () => + Assert.objectContains(fixture, { + foo: Expect.stringContains("bar"), + }), + /AssertionError/, + "Expect.stringContains shouldn't match when the value is unexpected" + ); + Assert.throws( + () => + Assert.objectContains(fixture, { + foo: Expect.number(), + }), + /AssertionError/, + "Expect.number shouldn't match when the value isn't a number" + ); + Assert.throws( + () => + Assert.objectContains(fixture, { + nested: Expect.objectContains({ + nestedFoo: "bar", + }), + }), + /AssertionError/, + "Expect.objectContains should throw when the value is unexpected" + ); + + Assert.throws( + () => + Assert.objectContains(fixture, { + num: val => Assert.less(val, 40), + }), + /AssertionError/, + "Expect.objectContains should throw when a function assertion fails" + ); +}); + +add_task(function test_type_expectations() { + const fixture = { + any: "foo", + string: "foo", + number: 42, + boolean: true, + bigint: 42n, + symbol: Symbol("foo"), + object: { foo: "foo" }, + function1() {}, + function2: () => {}, + }; + + Assert.objectContains(fixture, { + any: Expect.any(), + string: Expect.string(), + number: Expect.number(), + boolean: Expect.boolean(), + bigint: Expect.bigint(), + symbol: Expect.symbol(), + object: Expect.object(), + function1: Expect.function(), + function2: Expect.function(), + }); +}); diff --git a/tools/profiler/tests/xpcshell/test_enterjit_osr.js b/tools/profiler/tests/xpcshell/test_enterjit_osr.js new file mode 100644 index 0000000000..86845ddc76 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_enterjit_osr.js @@ -0,0 +1,52 @@ +// Check that the EnterJIT frame, added by the JIT trampoline and +// usable by a native unwinder to resume unwinding after encountering +// JIT code, is pushed as expected. +function run_test() { + // This test assumes that it's starting on an empty profiler stack. + // (Note that the other profiler tests also assume the profiler + // isn't already started.) + Assert.ok(!Services.profiler.IsActive()); + + const ms = 5; + Services.profiler.StartProfiler(10000, ms, ["js"]); + + function has_arbitrary_name_in_stack() { + // A frame for |arbitrary_name| has been pushed. Do a sequence of + // increasingly long spins until we get a sample. + var delayMS = 5; + while (1) { + info("loop: ms = " + delayMS); + const then = Date.now(); + do { + let n = 10000; + // eslint-disable-next-line no-empty + while (--n) {} // OSR happens here + // Spin in the hope of getting a sample. + } while (Date.now() - then < delayMS); + let profile = Services.profiler.getProfileData().threads[0]; + + // Go through all of the stacks, and search for this function name. + for (const sample of profile.samples.data) { + const stack = getInflatedStackLocations(profile, sample); + info(`The following stack was found: ${stack}`); + for (var i = 0; i < stack.length; i++) { + if (stack[i].match(/arbitrary_name/)) { + // This JS sample was correctly found. + return true; + } + } + } + + // Continue running this function with an increasingly long delay. + delayMS *= 2; + if (delayMS > 30000) { + return false; + } + } + } + Assert.ok( + has_arbitrary_name_in_stack(), + "A JS frame was found before the test timeout." + ); + Services.profiler.StopProfiler(); +} diff --git a/tools/profiler/tests/xpcshell/test_enterjit_osr_disabling.js b/tools/profiler/tests/xpcshell/test_enterjit_osr_disabling.js new file mode 100644 index 0000000000..558c9b0c3b --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_enterjit_osr_disabling.js @@ -0,0 +1,14 @@ +function run_test() { + Assert.ok(!Services.profiler.IsActive()); + + Services.profiler.StartProfiler(100, 10, ["js"]); + // The function is entered with the profiler enabled + (function () { + Services.profiler.StopProfiler(); + let n = 10000; + // eslint-disable-next-line no-empty + while (--n) {} // OSR happens here with the profiler disabled. + // An assertion will fail when this function returns, if the + // profiler stack was misbalanced. + })(); +} diff --git a/tools/profiler/tests/xpcshell/test_enterjit_osr_enabling.js b/tools/profiler/tests/xpcshell/test_enterjit_osr_enabling.js new file mode 100644 index 0000000000..313d939caf --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_enterjit_osr_enabling.js @@ -0,0 +1,14 @@ +function run_test() { + Assert.ok(!Services.profiler.IsActive()); + + // The function is entered with the profiler disabled. + (function () { + Services.profiler.StartProfiler(100, 10, ["js"]); + let n = 10000; + // eslint-disable-next-line no-empty + while (--n) {} // OSR happens here with the profiler enabled. + // An assertion will fail when this function returns, if the + // profiler stack was misbalanced. + })(); + Services.profiler.StopProfiler(); +} diff --git a/tools/profiler/tests/xpcshell/test_feature_cpufreq.js b/tools/profiler/tests/xpcshell/test_feature_cpufreq.js new file mode 100644 index 0000000000..1d8e0d9a36 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_cpufreq.js @@ -0,0 +1,102 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that the CPU Speed markers exist if and only if the cpufreq feature + * is enabled. + */ +add_task(async () => { + { + Assert.ok( + Services.profiler.GetAllFeatures().includes("cpufreq"), + "the CPU Frequency feature should exist on all platforms." + ); + const shouldBeEnabled = ["win", "linux", "android"].includes( + AppConstants.platform + ); + Assert.equal( + Services.profiler.GetFeatures().includes("cpufreq"), + shouldBeEnabled, + "the CPU Frequency feature should only be enabled on some platforms." + ); + + if (!shouldBeEnabled) { + return; + } + } + + { + const { markers, schema } = await runProfilerWithCPUSpeed(["cpufreq"]); + + checkSchema(schema, { + name: "CPUSpeed", + tableLabel: "{marker.name} Speed = {marker.data.speed}GHz", + display: ["marker-chart", "marker-table"], + data: [ + { + key: "speed", + label: "CPU Speed (GHz)", + format: "string", + }, + ], + graphs: [ + { + key: "speed", + type: "bar", + color: "ink", + }, + ], + }); + + Assert.greater(markers.length, 0, "There should be CPU Speed markers"); + const names = new Set(); + for (let marker of markers) { + names.add(marker.name); + } + let processData = await Services.sysinfo.processInfo; + equal( + names.size, + processData.count, + "We have as many CPU Speed marker names as CPU cores on the machine" + ); + } + + { + const { markers } = await runProfilerWithCPUSpeed([]); + equal( + markers.length, + 0, + "No CPUSpeed markers are found when the cpufreq feature is not turned on " + + "in the profiler." + ); + } +}); + +function getInflatedCPUFreqMarkers(thread) { + const markers = getInflatedMarkerData(thread); + return markers.filter(marker => marker.data?.type === "CPUSpeed"); +} + +/** + * Start the profiler and get CPUSpeed markers and schema. + * + * @param {Array} features The list of profiler features + * @returns {{ + * markers: InflatedMarkers[]; + * schema: MarkerSchema; + * }} + */ +async function runProfilerWithCPUSpeed(features, filename) { + const entries = 10000; + const interval = 10; + const threads = []; + await Services.profiler.StartProfiler(entries, interval, features, threads); + + const profile = await waitSamplingAndStopAndGetProfile(); + const mainThread = profile.threads.find(({ name }) => name === "GeckoMain"); + + const schema = getSchema(profile, "CPUSpeed"); + const markers = getInflatedCPUFreqMarkers(mainThread); + return { schema, markers }; +} diff --git a/tools/profiler/tests/xpcshell/test_feature_fileioall.js b/tools/profiler/tests/xpcshell/test_feature_fileioall.js new file mode 100644 index 0000000000..e5ac040b98 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_fileioall.js @@ -0,0 +1,159 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + info( + "Test that off-main thread fileio is captured for a profiled thread, " + + "and that it will be sent to the main thread." + ); + const filename = "test_marker_fileio"; + const profile = await startProfilerAndTriggerFileIO({ + features: ["fileioall"], + threadsFilter: ["GeckoMain", "BgIOThreadPool"], + filename, + }); + + const threads = getThreads(profile); + const mainThread = threads.find(thread => thread.name === "GeckoMain"); + const mainThreadFileIO = getInflatedFileIOMarkers(mainThread, filename); + let backgroundThread; + let backgroundThreadFileIO; + for (const thread of threads) { + // Check for FileIO in any of the background threads. + if (thread.name.startsWith("BgIOThreadPool")) { + const markers = getInflatedFileIOMarkers(thread, filename); + if (markers.length) { + backgroundThread = thread; + backgroundThreadFileIO = markers; + break; + } + } + } + + info("Check all of the main thread FileIO markers."); + checkInflatedFileIOMarkers(mainThreadFileIO, filename); + for (const { data, name } of mainThreadFileIO) { + equal( + name, + "FileIO (non-main thread)", + "The markers from off main thread are labeled as such." + ); + equal( + data.threadId, + backgroundThread.tid, + "The main thread FileIO markers were all sent from the background thread." + ); + } + + info("Check all of the background thread FileIO markers."); + checkInflatedFileIOMarkers(backgroundThreadFileIO, filename); + for (const { data, name } of backgroundThreadFileIO) { + equal( + name, + "FileIO", + "The markers on the thread where they were generated just say FileIO" + ); + equal( + data.threadId, + undefined, + "The background thread FileIO correctly excludes the threadId." + ); + } +}); + +add_task(async () => { + info( + "Test that off-main thread fileio is captured for a thread that is not profiled, " + + "and that it will be sent to the main thread." + ); + const filename = "test_marker_fileio"; + const profile = await startProfilerAndTriggerFileIO({ + features: ["fileioall"], + threadsFilter: ["GeckoMain"], + filename, + }); + + const threads = getThreads(profile); + const mainThread = threads.find(thread => thread.name === "GeckoMain"); + const mainThreadFileIO = getInflatedFileIOMarkers(mainThread, filename); + + info("Check all of the main thread FileIO markers."); + checkInflatedFileIOMarkers(mainThreadFileIO, filename); + for (const { data, name } of mainThreadFileIO) { + equal( + name, + "FileIO (non-profiled thread)", + "The markers from off main thread are labeled as such." + ); + equal(typeof data.threadId, "number", "A thread ID is captured."); + } +}); + +/** + * @typedef {Object} TestConfig + * @prop {Array} features The list of profiler features + * @prop {string[]} threadsFilter The list of threads to profile + * @prop {string} filename A filename to trigger a write operation + */ + +/** + * Start the profiler and get FileIO markers. + * @param {TestConfig} + * @returns {Profile} + */ +async function startProfilerAndTriggerFileIO({ + features, + threadsFilter, + filename, +}) { + const entries = 10000; + const interval = 10; + await Services.profiler.StartProfiler( + entries, + interval, + features, + threadsFilter + ); + + const path = PathUtils.join(PathUtils.tempDir, filename); + + info(`Using a temporary file to test FileIO: ${path}`); + + if (fileExists(path)) { + console.warn( + "This test is triggering FileIO by writing to a file. However, the test found an " + + "existing file at the location it was trying to write to. This could happen " + + "because a previous run of the test failed to clean up after itself. This test " + + " will now clean up that file before running the test again." + ); + await removeFile(path); + } + + info("Write to the file, but do so using a background thread."); + + // IOUtils handles file operations using a background thread. + await IOUtils.write(path, new TextEncoder().encode("Test data.")); + const exists = await fileExists(path); + ok(exists, `Created temporary file at: ${path}`); + + info("Remove the file"); + await removeFile(path); + + return stopNowAndGetProfile(); +} + +async function fileExists(file) { + try { + let { type } = await IOUtils.stat(file); + return type === "regular"; + } catch (_error) { + return false; + } +} + +async function removeFile(file) { + await IOUtils.remove(file); + const exists = await fileExists(file); + ok(!exists, `Removed temporary file: ${file}`); +} diff --git a/tools/profiler/tests/xpcshell/test_feature_java.js b/tools/profiler/tests/xpcshell/test_feature_java.js new file mode 100644 index 0000000000..e2f6879c2b --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_java.js @@ -0,0 +1,31 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that Java capturing works as expected. + */ +add_task(async () => { + info("Test that Android Java sampler works as expected."); + const entries = 10000; + const interval = 1; + const threads = []; + const features = ["java"]; + + Services.profiler.StartProfiler(entries, interval, features, threads); + Assert.ok(Services.profiler.IsActive()); + + await captureAtLeastOneJsSample(); + + info( + "Stop the profiler and check that we have successfully captured a profile" + + " with the AndroidUI thread." + ); + const profile = await stopNowAndGetProfile(); + Assert.notEqual(profile, null); + const androidUiThread = profile.threads.find( + thread => thread.name == "AndroidUI (JVM)" + ); + Assert.notEqual(androidUiThread, null); + Assert.ok(!Services.profiler.IsActive()); +}); diff --git a/tools/profiler/tests/xpcshell/test_feature_js.js b/tools/profiler/tests/xpcshell/test_feature_js.js new file mode 100644 index 0000000000..a5949e4a0c --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_js.js @@ -0,0 +1,63 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that JS capturing works as expected. + */ +add_task(async () => { + const entries = 10000; + const interval = 1; + const threads = []; + const features = ["js"]; + + await Services.profiler.StartProfiler(entries, interval, features, threads); + + // Call the following to get a nice stack in the profiler: + // functionA -> functionB -> functionC -> captureAtLeastOneJsSample + const sampleIndex = await functionA(); + + const profile = await stopNowAndGetProfile(); + + const [thread] = profile.threads; + const { samples } = thread; + + const inflatedStackFrames = getInflatedStackLocations( + thread, + samples.data[sampleIndex] + ); + + expectStackToContain( + inflatedStackFrames, + [ + "(root)", + "js::RunScript", + // The following regexes match a string similar to: + // + // "functionA (/gecko/obj/_tests/xpcshell/tools/profiler/tests/xpcshell/test_feature_js.js:47:0)" + // or + // "functionA (test_feature_js.js:47:0)" + // + // this matches the script location + // | match the line number + // | | match the column number + // v v v + /^functionA \(.*test_feature_js\.js:\d+:\d+\)$/, + /^functionB \(.*test_feature_js\.js:\d+:\d+\)$/, + /^functionC \(.*test_feature_js\.js:\d+:\d+\)$/, + ], + "The stack contains a few frame labels, as well as the JS functions that we called." + ); +}); + +function functionA() { + return functionB(); +} + +function functionB() { + return functionC(); +} + +async function functionC() { + return captureAtLeastOneJsSample(); +} diff --git a/tools/profiler/tests/xpcshell/test_feature_mainthreadio.js b/tools/profiler/tests/xpcshell/test_feature_mainthreadio.js new file mode 100644 index 0000000000..b0224e68d2 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_mainthreadio.js @@ -0,0 +1,122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +/** + * Test that the IOInterposer is working correctly to capture main thread IO. + * + * This test should not run on release or beta, as the IOInterposer is wrapped in + * an ifdef. + */ +add_task(async () => { + { + const filename = "profiler-mainthreadio-test-firstrun"; + const { markers, schema } = await runProfilerWithFileIO( + ["mainthreadio"], + filename + ); + info("Check the FileIO markers when using the mainthreadio feature"); + checkInflatedFileIOMarkers(markers, filename); + + checkSchema(schema, { + name: "FileIO", + display: ["marker-chart", "marker-table", "timeline-fileio"], + data: [ + { + key: "operation", + label: "Operation", + format: "string", + searchable: true, + }, + { key: "source", label: "Source", format: "string", searchable: true }, + { + key: "filename", + label: "Filename", + format: "file-path", + searchable: true, + }, + { + key: "threadId", + label: "Thread ID", + format: "string", + searchable: true, + }, + ], + }); + } + + { + const filename = "profiler-mainthreadio-test-no-instrumentation"; + const { markers } = await runProfilerWithFileIO([], filename); + equal( + markers.length, + 0, + "No FileIO markers are found when the mainthreadio feature is not turned on " + + "in the profiler." + ); + } + + { + const filename = "profiler-mainthreadio-test-secondrun"; + const { markers } = await runProfilerWithFileIO(["mainthreadio"], filename); + info("Check the FileIO markers when re-starting the mainthreadio feature"); + checkInflatedFileIOMarkers(markers, filename); + } +}); + +/** + * Start the profiler and get FileIO markers and schema. + * + * @param {Array} features The list of profiler features + * @param {string} filename A filename to trigger a write operation + * @returns {{ + * markers: InflatedMarkers[]; + * schema: MarkerSchema; + * }} + */ +async function runProfilerWithFileIO(features, filename) { + const entries = 10000; + const interval = 10; + const threads = []; + await Services.profiler.StartProfiler(entries, interval, features, threads); + + info("Get the file"); + const file = await IOUtils.getFile(PathUtils.tempDir, filename); + if (file.exists()) { + console.warn( + "This test is triggering FileIO by writing to a file. However, the test found an " + + "existing file at the location it was trying to write to. This could happen " + + "because a previous run of the test failed to clean up after itself. This test " + + " will now clean up that file before running the test again." + ); + file.remove(false); + } + + info( + "Generate file IO on the main thread using FileUtils.openSafeFileOutputStream." + ); + const outputStream = FileUtils.openSafeFileOutputStream(file); + + const data = "Test data."; + info("Write to the file"); + outputStream.write(data, data.length); + + info("Close the file"); + FileUtils.closeSafeFileOutputStream(outputStream); + + info("Remove the file"); + file.remove(false); + + const profile = await stopNowAndGetProfile(); + const mainThread = profile.threads.find(({ name }) => name === "GeckoMain"); + + const schema = getSchema(profile, "FileIO"); + + const markers = getInflatedFileIOMarkers(mainThread, filename); + + return { schema, markers }; +} diff --git a/tools/profiler/tests/xpcshell/test_feature_nativeallocations.js b/tools/profiler/tests/xpcshell/test_feature_nativeallocations.js new file mode 100644 index 0000000000..64398d7ef9 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_nativeallocations.js @@ -0,0 +1,158 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + if (!Services.profiler.GetFeatures().includes("nativeallocations")) { + Assert.ok( + true, + "Native allocations are not supported by this build, " + + "skip run the rest of the test." + ); + return; + } + + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + info( + "Test that the profiler can install memory hooks and collect native allocation " + + "information in the marker payloads." + ); + { + info("Start the profiler."); + await startProfiler({ + // Only instrument the main thread. + threads: ["GeckoMain"], + features: ["js", "nativeallocations"], + }); + + info( + "Do some JS work for a little bit. This will increase the amount of allocations " + + "that take place." + ); + doWork(); + + info("Get the profile data and analyze it."); + const profile = await waitSamplingAndStopAndGetProfile(); + + const { + allocationPayloads, + unmatchedAllocations, + logAllocationsAndDeallocations, + } = getAllocationInformation(profile); + + Assert.greater( + allocationPayloads.length, + 0, + "Native allocation payloads were recorded for the parent process' main thread when " + + "the Native Allocation feature was turned on." + ); + + if (unmatchedAllocations.length !== 0) { + info( + "There were unmatched allocations. Log all of the allocations and " + + "deallocations in order to aid debugging." + ); + logAllocationsAndDeallocations(); + ok( + false, + "Found a deallocation that did not have a matching allocation site. " + + "This could happen if balanced allocations is broken, or if the the " + + "buffer size of this test was too small, and some markers ended up " + + "rolling off." + ); + } + + ok(true, "All deallocation sites had matching allocations."); + } + + info("Restart the profiler, to ensure that we get no more allocations."); + { + await startProfiler({ features: ["js"] }); + info("Do some work again."); + doWork(); + info("Wait for the periodic sampling."); + const profile = await waitSamplingAndStopAndGetProfile(); + const allocationPayloads = getPayloadsOfType( + profile.threads[0], + "Native allocation" + ); + + Assert.equal( + allocationPayloads.length, + 0, + "No native allocations were collected when the feature was disabled." + ); + } +}); + +function doWork() { + this.n = 0; + for (let i = 0; i < 1e5; i++) { + this.n += Math.random(); + } +} + +/** + * Extract the allocation payloads, and find the unmatched allocations. + */ +function getAllocationInformation(profile) { + // Get all of the allocation payloads. + const allocationPayloads = getPayloadsOfType( + profile.threads[0], + "Native allocation" + ); + + // Decide what is an allocation and deallocation. + const allocations = allocationPayloads.filter( + payload => ensureIsNumber(payload.size) >= 0 + ); + const deallocations = allocationPayloads.filter( + payload => ensureIsNumber(payload.size) < 0 + ); + + // Now determine the unmatched allocations by building a set + const allocationSites = new Set( + allocations.map(({ memoryAddress }) => memoryAddress) + ); + + const unmatchedAllocations = deallocations.filter( + ({ memoryAddress }) => !allocationSites.has(memoryAddress) + ); + + // Provide a helper to log out the allocations and deallocations on failure. + function logAllocationsAndDeallocations() { + for (const { memoryAddress } of allocations) { + console.log("Allocations", formatHex(memoryAddress)); + allocationSites.add(memoryAddress); + } + + for (const { memoryAddress } of deallocations) { + console.log("Deallocations", formatHex(memoryAddress)); + } + + for (const { memoryAddress } of unmatchedAllocations) { + console.log("Deallocation with no allocation", formatHex(memoryAddress)); + } + } + + return { + allocationPayloads, + unmatchedAllocations, + logAllocationsAndDeallocations, + }; +} + +function ensureIsNumber(value) { + if (typeof value !== "number") { + throw new Error(`Expected a number: ${value}`); + } + return value; +} + +function formatHex(number) { + return `0x${number.toString(16)}`; +} diff --git a/tools/profiler/tests/xpcshell/test_feature_stackwalking.js b/tools/profiler/tests/xpcshell/test_feature_stackwalking.js new file mode 100644 index 0000000000..aa0bc86547 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_stackwalking.js @@ -0,0 +1,48 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Do a basic test to see if native frames are being collected for stackwalking. This + * test is fairly naive, as it does not attempt to check that these are valid symbols, + * only that some kind of stack walking is happening. It does this by making sure at + * least two native frames are collected. + */ +add_task(async () => { + const entries = 10000; + const interval = 1; + const threads = []; + const features = ["stackwalk"]; + + await Services.profiler.StartProfiler(entries, interval, features, threads); + const sampleIndex = await captureAtLeastOneJsSample(); + + const profile = await stopNowAndGetProfile(); + const [thread] = profile.threads; + const { samples } = thread; + + const inflatedStackFrames = getInflatedStackLocations( + thread, + samples.data[sampleIndex] + ); + const nativeStack = /^0x[0-9a-f]+$/; + + expectStackToContain( + inflatedStackFrames, + [ + "(root)", + // There are probably more native stacks here. + nativeStack, + nativeStack, + // Since this is an xpcshell test we know that JavaScript will run: + "js::RunScript", + // There are probably more native stacks here. + nativeStack, + nativeStack, + ], + "Expected native stacks to be interleaved between some frame labels. There should" + + "be more than one native stack if stack walking is working correctly. There " + + "is no attempt here to determine if the memory addresses point to the correct " + + "symbols" + ); +}); diff --git a/tools/profiler/tests/xpcshell/test_get_features.js b/tools/profiler/tests/xpcshell/test_get_features.js new file mode 100644 index 0000000000..e9bf0047c8 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_get_features.js @@ -0,0 +1,8 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + var profilerFeatures = Services.profiler.GetFeatures(); + Assert.ok(profilerFeatures != null); +} diff --git a/tools/profiler/tests/xpcshell/test_merged_stacks.js b/tools/profiler/tests/xpcshell/test_merged_stacks.js new file mode 100644 index 0000000000..7f851e8de9 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_merged_stacks.js @@ -0,0 +1,74 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we correctly merge the three stack types, JS, native, and frame labels. + */ +add_task(async () => { + const entries = 10000; + const interval = 1; + const threads = []; + const features = ["js", "stackwalk"]; + + await Services.profiler.StartProfiler(entries, interval, features, threads); + + // Call the following to get a nice stack in the profiler: + // functionA -> functionB -> functionC + const sampleIndex = await functionA(); + + const profile = await stopNowAndGetProfile(); + const [thread] = profile.threads; + const { samples } = thread; + + const inflatedStackFrames = getInflatedStackLocations( + thread, + samples.data[sampleIndex] + ); + + const nativeStack = /^0x[0-9a-f]+$/; + + expectStackToContain( + inflatedStackFrames, + [ + "(root)", + nativeStack, + nativeStack, + // There are more native stacks and frame labels here, but we know some execute + // and then the "js::RunScript" frame label runs. + "js::RunScript", + nativeStack, + nativeStack, + // The following regexes match a string similar to: + // + // "functionA (/gecko/obj/_tests/xpcshell/tools/profiler/tests/xpcshell/test_merged_stacks.js:47:0)" + // or + // "functionA (test_merged_stacks.js:47:0)" + // + // this matches the script location + // | match the line number + // | | match the column number + // v v v + /^functionA \(.*test_merged_stacks\.js:\d+:\d+\)$/, + /^functionB \(.*test_merged_stacks\.js:\d+:\d+\)$/, + /^functionC \(.*test_merged_stacks\.js:\d+:\d+\)$/, + // After the JS frames, then there are a bunch of arbitrary native stack frames + // that run. + nativeStack, + nativeStack, + ], + "The stack contains a few frame labels, as well as the JS functions that we called." + ); +}); + +async function functionA() { + return functionB(); +} + +async function functionB() { + return functionC(); +} + +async function functionC() { + return captureAtLeastOneJsSample(); +} diff --git a/tools/profiler/tests/xpcshell/test_pause.js b/tools/profiler/tests/xpcshell/test_pause.js new file mode 100644 index 0000000000..0e621fb19f --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_pause.js @@ -0,0 +1,126 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + Assert.ok(!Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + + let startPromise = Services.profiler.StartProfiler(1000, 10, []); + + // Default: Active and not paused. + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + await startPromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + // Pause everything, implicitly pauses sampling. + let pausePromise = Services.profiler.Pause(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await pausePromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // While fully paused, pause and resume sampling only, no expected changes. + let pauseSamplingPromise = Services.profiler.PauseSampling(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await pauseSamplingPromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + let resumeSamplingPromise = Services.profiler.ResumeSampling(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await resumeSamplingPromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // Resume everything. + let resumePromise = Services.profiler.Resume(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + await resumePromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + // Pause sampling only. + let pauseSampling2Promise = Services.profiler.PauseSampling(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await pauseSampling2Promise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // While sampling is paused, pause everything. + let pause2Promise = Services.profiler.Pause(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await pause2Promise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // Resume, but sampling is still paused separately. + let resume2promise = Services.profiler.Resume(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await resume2promise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // Resume sampling only. + let resumeSampling2Promise = Services.profiler.ResumeSampling(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + await resumeSampling2Promise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + let stopPromise = Services.profiler.StopProfiler(); + Assert.ok(!Services.profiler.IsActive()); + // Stopping is not pausing. + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + await stopPromise; + Assert.ok(!Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); +}); diff --git a/tools/profiler/tests/xpcshell/test_responsiveness.js b/tools/profiler/tests/xpcshell/test_responsiveness.js new file mode 100644 index 0000000000..5f57173090 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_responsiveness.js @@ -0,0 +1,50 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we can measure non-zero event delays + */ + +add_task(async () => { + const entries = 10000; + const interval = 1; + const threads = []; + const features = []; + + await Services.profiler.StartProfiler(entries, interval, features, threads); + + await functionA(); + + const profile = await stopNowAndGetProfile(); + const [thread] = profile.threads; + const { samples } = thread; + const message = "eventDelay > 0 not found."; + let SAMPLE_STACK_SLOT = thread.samples.schema.eventDelay; + + for (let i = 0; i < samples.data.length; i++) { + if (samples.data[i][SAMPLE_STACK_SLOT] > 0) { + Assert.ok(true, message); + return; + } + } + Assert.ok(false, message); +}); + +function doSyncWork(milliseconds) { + const start = Date.now(); + while (true) { + this.n = 0; + for (let i = 0; i < 1e5; i++) { + this.n += Math.random(); + } + if (Date.now() - start > milliseconds) { + return; + } + } +} + +async function functionA() { + doSyncWork(100); + return captureAtLeastOneJsSample(); +} diff --git a/tools/profiler/tests/xpcshell/test_run.js b/tools/profiler/tests/xpcshell/test_run.js new file mode 100644 index 0000000000..0e30edfd4e --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_run.js @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + Assert.ok(!Services.profiler.IsActive()); + + Services.profiler.StartProfiler(1000, 10, []); + + Assert.ok(Services.profiler.IsActive()); + + do_test_pending(); + + do_timeout(1000, function wait() { + // Check text profile format + var profileStr = Services.profiler.GetProfile(); + Assert.ok(profileStr.length > 10); + + // check json profile format + var profileObj = Services.profiler.getProfileData(); + Assert.notEqual(profileObj, null); + Assert.notEqual(profileObj.threads, null); + // We capture memory counters by default only when jemalloc is turned + // on (and it isn't for ASAN), so unless we can conditionalize for ASAN + // here we can't check that we're capturing memory counter data. + Assert.notEqual(profileObj.counters, null); + Assert.notEqual(profileObj.memory, null); + Assert.ok(profileObj.threads.length >= 1); + Assert.notEqual(profileObj.threads[0].samples, null); + // NOTE: The number of samples will be empty since we + // don't have any labels in the xpcshell code + + Services.profiler.StopProfiler(); + Assert.ok(!Services.profiler.IsActive()); + do_test_finished(); + }); +} diff --git a/tools/profiler/tests/xpcshell/test_shared_library.js b/tools/profiler/tests/xpcshell/test_shared_library.js new file mode 100644 index 0000000000..e211ca642b --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_shared_library.js @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + var libs = Services.profiler.sharedLibraries; + + Assert.equal(typeof libs, "object"); + Assert.ok(Array.isArray(libs)); + Assert.equal(typeof libs, "object"); + Assert.ok(libs.length >= 1); + Assert.equal(typeof libs[0], "object"); + Assert.equal(typeof libs[0].name, "string"); + Assert.equal(typeof libs[0].path, "string"); + Assert.equal(typeof libs[0].debugName, "string"); + Assert.equal(typeof libs[0].debugPath, "string"); + Assert.equal(typeof libs[0].arch, "string"); + Assert.equal(typeof libs[0].start, "number"); + Assert.equal(typeof libs[0].end, "number"); + Assert.ok(libs[0].start <= libs[0].end); +} diff --git a/tools/profiler/tests/xpcshell/test_start.js b/tools/profiler/tests/xpcshell/test_start.js new file mode 100644 index 0000000000..c9ae135eb8 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_start.js @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + Assert.ok(!Services.profiler.IsActive()); + + let startPromise = Services.profiler.StartProfiler(10, 100, []); + + Assert.ok(Services.profiler.IsActive()); + + await startPromise; + Assert.ok(Services.profiler.IsActive()); + + let stopPromise = Services.profiler.StopProfiler(); + + Assert.ok(!Services.profiler.IsActive()); + + await stopPromise; + Assert.ok(!Services.profiler.IsActive()); +}); diff --git a/tools/profiler/tests/xpcshell/xpcshell.toml b/tools/profiler/tests/xpcshell/xpcshell.toml new file mode 100644 index 0000000000..a04dcd6789 --- /dev/null +++ b/tools/profiler/tests/xpcshell/xpcshell.toml @@ -0,0 +1,93 @@ +[DEFAULT] +head = "head.js" +support-files = ["../shared-head.js"] + +["test_active_configuration.js"] +skip-if = ["tsan"] # Intermittent timeouts, bug 1781449 + +["test_addProfilerMarker.js"] + +["test_asm.js"] + +["test_assertion_helper.js"] + +["test_enterjit_osr.js"] + +["test_enterjit_osr_disabling.js"] +skip-if = ["!debug"] + +["test_enterjit_osr_enabling.js"] +skip-if = ["!debug"] + +["test_feature_cpufreq.js"] + +["test_feature_fileioall.js"] +skip-if = ["release_or_beta"] + +# The sanitizer checks appears to overwrite our own memory hooks in xpcshell tests, +# and no allocation markers are gathered. Skip this test in that configuration. + +["test_feature_java.js"] +skip-if = ["os != 'android'"] + +["test_feature_js.js"] +skip-if = ["tsan"] # Times out on TSan, bug 1612707 + +# See the comment on test_feature_stackwalking.js + +["test_feature_mainthreadio.js"] +skip-if = [ + "release_or_beta", + "os == 'win' && socketprocess_networking", +] + +["test_feature_nativeallocations.js"] +skip-if = [ + "os == 'android' && verify", # bug 1757528 + "asan", + "tsan", + "socketprocess_networking", +] + +# Native stackwalking is somewhat unreliable depending on the platform. +# +# We don't have frame pointers on macOS release and beta, so stack walking does not +# work. See Bug 1571216 for more details. +# +# Linux can be very unreliable when native stackwalking through JavaScript code. +# See Bug 1434402 for more details. +# +# For sanitizer builds, there were many intermittents, and we're not getting much +# additional coverage there, so it's better to be a bit more reliable. + +["test_feature_stackwalking.js"] +skip-if = [ + "os == 'mac' && release_or_beta", + "os == 'linux' && release_or_beta && !debug", + "asan", + "tsan", +] + +["test_get_features.js"] + +["test_merged_stacks.js"] +skip-if = [ + "os == 'mac' && release_or_beta", + "os == 'linux' && release_or_beta && !debug", + "asan", + "tsan", +] + +["test_pause.js"] +skip-if = ["tsan && socketprocess_networking"] # Times out on TSan and socket process, bug 1878882 + +["test_responsiveness.js"] +skip-if = ["tsan"] # Times out on TSan, bug 1612707 + +["test_run.js"] +skip-if = ["true"] + +["test_shared_library.js"] + +["test_start.js"] +skip-if = ["true"] diff --git a/tools/quitter/README.md b/tools/quitter/README.md new file mode 100644 index 0000000000..515a3c952f --- /dev/null +++ b/tools/quitter/README.md @@ -0,0 +1,6 @@ +All code for this is part of the quitter project on github: +https://github.com/mozilla-extensions/quitter + +This is within the mozilla-extensions organization so that we can sign the .xpi. + +To update code, please submit a PR to the github repo. Once that is merged in, a JIRA ticket (ex [ADDONSOPS-194](https://mozilla-hub.atlassian.net/browse/ADDONSOPS-194)) is needed to resign the .xpi. diff --git a/tools/quitter/quitter@mozilla.org.xpi b/tools/quitter/quitter@mozilla.org.xpi Binary files differnew file mode 100644 index 0000000000..5f25ca3dad --- /dev/null +++ b/tools/quitter/quitter@mozilla.org.xpi diff --git a/tools/rb/README b/tools/rb/README new file mode 100644 index 0000000000..30418602b1 --- /dev/null +++ b/tools/rb/README @@ -0,0 +1,6 @@ +This is the Refcount Balancer. See +https://firefox-source-docs.mozilla.org/performance/memory/refcount_tracing_and_balancing.html +for documentation. + +Note that the `fix_stacks.py` script is used in several other places in the +repository. diff --git a/tools/rb/filter-log.pl b/tools/rb/filter-log.pl new file mode 100755 index 0000000000..4a1f66741b --- /dev/null +++ b/tools/rb/filter-log.pl @@ -0,0 +1,44 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Filter a refcount log to show only the entries for a single object. +# Useful when manually examining refcount logs containing multiple +# objects. + +use 5.004; +use strict; +use Getopt::Long; + +GetOptions("object=s"); + +$::opt_object || + die qq{ +usage: filter-log-for.pl < logfile + --object <obj> The address of the object to examine (required) +}; + +warn "object $::opt_object\n"; + +LINE: while (<>) { + next LINE if (! /^</); + my $line = $_; + my @fields = split(/ /, $_); + + my $class = shift(@fields); + my $obj = shift(@fields); + next LINE unless ($obj eq $::opt_object); + my $sno = shift(@fields); + my $op = shift(@fields); + my $cnt = shift(@fields); + + print $line; + + # The lines in the stack trace + CALLSITE: while (<>) { + print; + last CALLSITE if (/^$/); + } +} diff --git a/tools/rb/find-comptr-leakers.pl b/tools/rb/find-comptr-leakers.pl new file mode 100755 index 0000000000..925119935c --- /dev/null +++ b/tools/rb/find-comptr-leakers.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Script loosely based on Chris Waterson's find-leakers.pl and make-tree.pl + +use 5.004; +use strict; +use Getopt::Long; + +# GetOption will create $opt_object, so ignore the +# warning that gets spit out about those vbls. +GetOptions("object=s", "list", "help"); + +# use $::opt_help twice to eliminate warning... +($::opt_help) && ($::opt_help) && die qq{ +usage: find-comptr-leakers.pl < logfile + --object <obj> Examine only object <obj> + --list Only list leaked objects + --help This message :-) +}; + +if ($::opt_object) { + warn "Examining only object $::opt_object (THIS IS BROKEN)\n"; +} else { + warn "Examining all objects\n"; +} + +my %allocs = ( ); +my %counter; +my $id = 0; + +my $accumulating = 0; +my $savedata = 0; +my $class; +my $obj; +my $sno; +my $op; +my $cnt; +my $ptr; +my $strace; + +sub save_data { + # save the data + if ($op eq 'nsCOMPtrAddRef') { + push @{ $allocs{$sno}->{$ptr} }, [ +1, $strace ]; + } + elsif ($op eq 'nsCOMPtrRelease') { + push @{ $allocs{$sno}->{$ptr} }, [ -1, $strace ]; + my $sum = 0; + my @ptrallocs = @{ $allocs{$sno}->{$ptr} }; + foreach my $alloc (@ptrallocs) { + $sum += @$alloc[0]; + } + if ( $sum == 0 ) { + delete($allocs{$sno}{$ptr}); + } + } +} + +LINE: while (<>) { + if (/^</) { + chop; # avoid \n in $ptr + my @fields = split(/ /, $_); + + ($class, $obj, $sno, $op, $cnt, $ptr) = @fields; + + $strace = ""; + + if ($::opt_list) { + save_data(); + } elsif (!($::opt_object) || ($::opt_object eq $obj)) { + $accumulating = 1; + } + } elsif ( $accumulating == 1 ) { + if ( /^$/ ) { + # if line is empty + $accumulating = 0; + save_data(); + } else { + $strace = $strace . $_; + } + } +} +if ( $accumulating == 1) { + save_data(); +} + +foreach my $serial (keys(%allocs)) { + foreach my $comptr (keys( %{$allocs{$serial}} )) { + my $sum = 0; + my @ptrallocs = @{ $allocs{$serial}->{$comptr} }; + foreach my $alloc (@ptrallocs) { + $sum += @$alloc[0]; + } + print "Object ", $serial, " held by ", $comptr, " is ", $sum, " out of balance.\n"; + unless ($::opt_list) { + print "\n"; + foreach my $alloc (@ptrallocs) { + if (@$alloc[0] == +1) { + print "Put into nsCOMPtr at:\n"; + } elsif (@$alloc[0] == -1) { + print "Released from nsCOMPtr at:\n"; + } + print @$alloc[1]; # the stack trace + print "\n"; + } + print "\n\n"; + } + } +} + diff --git a/tools/rb/find_leakers.py b/tools/rb/find_leakers.py new file mode 100755 index 0000000000..f131a2be59 --- /dev/null +++ b/tools/rb/find_leakers.py @@ -0,0 +1,116 @@ +#!/usr/bin/python +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script processes a `refcount' log, and finds out if any object leaked. +# It simply goes through the log, finds `AddRef' or `Ctor' lines, and then +# sees if they `Release' or `Dtor'. If not, it reports them as leaks. +# Please see README file in the same directory. + +import sys + +import six + + +def print_output(allocation, obj_to_class): + """Formats and prints output.""" + items = [] + for ( + obj, + count, + ) in six.iteritems(allocation): + # Adding items to a list, so we can sort them. + items.append((obj, count)) + # Sorting by count. + items.sort(key=lambda item: item[1]) + + for ( + obj, + count, + ) in items: + print( + "{obj} ({count}) @ {class_name}".format( + obj=obj, count=count, class_name=obj_to_class[obj] + ) + ) + + +def process_log(log_lines): + """Process through the log lines, and print out the result. + + @param log_lines: List of strings. + """ + allocation = {} + class_count = {} + obj_to_class = {} + + for log_line in log_lines: + if not log_line.startswith("<"): + continue + + ( + class_name, + obj, + ignore, + operation, + count, + ) = log_line.strip("\r\n").split( + " " + )[:5] + + # for AddRef/Release `count' is the refcount, + # for Ctor/Dtor it's the size. + + if (operation == "AddRef" and count == "1") or operation == "Ctor": + # Examples: + # <nsStringBuffer> 0x01AFD3B8 1 AddRef 1 + # <PStreamNotifyParent> 0x08880BD0 8 Ctor (20) + class_count[class_name] = class_count.setdefault(class_name, 0) + 1 + allocation[obj] = class_count[class_name] + obj_to_class[obj] = class_name + + elif (operation == "Release" and count == "0") or operation == "Dtor": + # Examples: + # <nsStringBuffer> 0x01AFD3B8 1 Release 0 + # <PStreamNotifyParent> 0x08880BD0 8 Dtor (20) + if obj not in allocation: + print( + "An object was released that wasn't allocated!", + ) + print(obj, "@", class_name) + else: + allocation.pop(obj) + obj_to_class.pop(obj) + + # Printing out the result. + print_output(allocation, obj_to_class) + + +def print_usage(): + print("") + print("Usage: find-leakers.py [log-file]") + print("") + print("If `log-file' provided, it will read that as the input log.") + print("Else, it will read the stdin as the input log.") + print("") + + +def main(): + """Main method of the script.""" + if len(sys.argv) == 1: + # Reading log from stdin. + process_log(sys.stdin.readlines()) + elif len(sys.argv) == 2: + # Reading log from file. + with open(sys.argv[1], "r") as log_file: + log_lines = log_file.readlines() + process_log(log_lines) + else: + print("ERROR: Invalid number of arguments") + print_usage() + + +if __name__ == "__main__": + main() diff --git a/tools/rb/fix_stacks.py b/tools/rb/fix_stacks.py new file mode 100755 index 0000000000..f30aa9944a --- /dev/null +++ b/tools/rb/fix_stacks.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# vim:sw=4:ts=4:et: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script uses `fix-stacks` to post-process the entries produced by +# MozFormatCodeAddress(). + +import atexit +import os +import platform +import re +import sys +from subprocess import PIPE, Popen + +# Matches lines produced by MozFormatCodeAddress(), e.g. +# `#01: ???[tests/example +0x43a0]`. +line_re = re.compile("#\d+: .+\[.+ \+0x[0-9A-Fa-f]+\]") + +fix_stacks = None + + +def autobootstrap(): + import buildconfig + from mozbuild.configure import ConfigureSandbox + + sandbox = ConfigureSandbox( + {}, + argv=[ + "configure", + "--help", + "--host={}".format(buildconfig.substs["HOST_ALIAS"]), + ], + ) + moz_configure = os.path.join(buildconfig.topsrcdir, "build", "moz.configure") + sandbox.include_file(os.path.join(moz_configure, "init.configure")) + # bootstrap_search_path_order has a dependency on developer_options, which + # is not defined in init.configure. Its value doesn't matter for us, though. + sandbox["developer_options"] = sandbox["always"] + sandbox.include_file(os.path.join(moz_configure, "bootstrap.configure")) + # Expand the `bootstrap_path` template for "fix-stacks", and execute the + # expanded function via `_value_for`, which will trigger autobootstrap. + sandbox._value_for(sandbox["bootstrap_path"]("fix-stacks")) + + +def initFixStacks(jsonMode, slowWarning, breakpadSymsDir, hide_errors): + # Look in MOZ_FETCHES_DIR (for automation), then in MOZBUILD_STATE_PATH + # (for a local build where the user has that set), then in ~/.mozbuild + # (for a local build with default settings). + base = os.environ.get( + "MOZ_FETCHES_DIR", + os.environ.get("MOZBUILD_STATE_PATH", os.path.expanduser("~/.mozbuild")), + ) + fix_stacks_exe = base + "/fix-stacks/fix-stacks" + if platform.system() == "Windows": + fix_stacks_exe = fix_stacks_exe + ".exe" + + if not (os.path.isfile(fix_stacks_exe) and os.access(fix_stacks_exe, os.X_OK)): + try: + autobootstrap() + except ImportError: + # We're out-of-tree (e.g. tests tasks on CI) and can't autobootstrap + # (we shouldn't anyways). + pass + + if not (os.path.isfile(fix_stacks_exe) and os.access(fix_stacks_exe, os.X_OK)): + raise Exception("cannot find `fix-stacks`; please run `./mach bootstrap`") + + args = [fix_stacks_exe] + if jsonMode: + args.append("-j") + if breakpadSymsDir: + args.append("-b") + args.append(breakpadSymsDir) + + # Sometimes we need to prevent errors from going to stderr. + stderr = open(os.devnull) if hide_errors else None + + global fix_stacks + fix_stacks = Popen( + args, stdin=PIPE, stdout=PIPE, stderr=stderr, universal_newlines=True + ) + + # Shut down the fix_stacks process on exit. We use `terminate()` + # because it is more forceful than `wait()`, and the Python docs warn + # about possible deadlocks with `wait()`. + def cleanup(fix_stacks): + for fn in [fix_stacks.stdin.close, fix_stacks.terminate]: + try: + fn() + except OSError: + pass + + atexit.register(cleanup, fix_stacks) + + if slowWarning: + print( + "Initializing stack-fixing for the first stack frame, this may take a while..." + ) + + +def fixSymbols( + line, jsonMode=False, slowWarning=False, breakpadSymsDir=None, hide_errors=False +): + is_bytes = isinstance(line, bytes) + line_str = line.decode("utf-8") if is_bytes else line + if line_re.search(line_str) is None: + return line + + if not fix_stacks: + initFixStacks(jsonMode, slowWarning, breakpadSymsDir, hide_errors) + + # Sometimes `line` is lacking a trailing newline. If we pass such a `line` + # to `fix-stacks` it will wait until it receives a newline, causing this + # script to hang. So we add a newline if one is missing and then remove it + # from the output. + is_missing_newline = not line_str.endswith("\n") + if is_missing_newline: + line_str = line_str + "\n" + fix_stacks.stdin.write(line_str) + fix_stacks.stdin.flush() + out = fix_stacks.stdout.readline() + if is_missing_newline: + out = out[:-1] + + if is_bytes and not isinstance(out, bytes): + out = out.encode("utf-8") + return out + + +if __name__ == "__main__": + bpsyms = os.environ.get("BREAKPAD_SYMBOLS_PATH", None) + for line in sys.stdin: + sys.stdout.write(fixSymbols(line, breakpadSymsDir=bpsyms)) diff --git a/tools/rb/make-tree.pl b/tools/rb/make-tree.pl new file mode 100755 index 0000000000..04f0d85341 --- /dev/null +++ b/tools/rb/make-tree.pl @@ -0,0 +1,303 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use 5.004; +use strict; +use Getopt::Long; + +$::opt_prune_depth = 0; +$::opt_subtree_size = 0; +$::opt_reverse = 0; + +# GetOption will create $opt_object & $opt_exclude, so ignore the +# warning that gets spit out about those vbls. +GetOptions("object=s", "exclude=s", "comptrs=s", "ignore-balanced", "subtree-size=i", "prune-depth=i", + "collapse-to-method", "collapse-to-class", "old-style", "reverse"); + +$::opt_object || + die qq{ +usage: leak.pl < logfile + --object <obj> The address of the object to examine (required) + --exclude <file> Exclude routines listed in <file> + --comptrs <file> Subtract all the data in the balanced COMPtr log <file> + --ignore-balanced Ignore balanced subtrees + --subtree-size <n> Print subtrees with more than <n> nodes separately + --prune-depth <depth> Prune the tree to <depth> + --collapse-to-method Aggregate data by method + --collapse-to-class Aggregate data by class (subsumes --collapse-to-method) + --reverse Reverse call stacks, showing leaves first + --old-style Old-style formatting +}; + +$::opt_prune_depth = 0 if $::opt_prune_depth < 0; +$::opt_subtree_size = 0 if $::opt_subtree_size < 0; + +warn "object $::opt_object\n"; +warn "ignoring balanced subtrees\n" if $::opt_ignore_balanced; +warn "prune depth $::opt_prune_depth\n" if $::opt_prune_depth; +warn "collapsing to class\n" if $::opt_collapse_to_class; +warn "collapsing to method\n" if $::opt_collapse_to_method && !$::opt_collapse_to_class; +warn "reversing call stacks\n" if $::opt_reverse; + + +# The 'excludes' are functions that, if detected in a particular call +# stack, will cause the _entire_ call stack to be ignored. You might, +# for example, explicitly exclude two functions that have a matching +# AddRef/Release pair. + +my %excludes; + +if ($::opt_exclude) { + open(EXCLUDE, "<".$::opt_exclude) + || die "unable to open $::opt_exclude"; + + while (<EXCLUDE>) { + chomp $_; + warn "excluding $_\n"; + $excludes{$_} = 1; + } +} + +# Each entry in the tree rooted by callGraphRoot contains the following: +# #name# This call's name+offset string +# #refcount# The net reference count of this call +# #label# The label used for this subtree; only defined for labeled nodes +# #children# List of children in alphabetical order +# zero or more children indexed by method name+offset strings. + +my $callGraphRoot; +$callGraphRoot = { '#name#' => '.root', '#refcount#' => 'n/a' }; + +# The 'imbalance' is a gross count of how balanced a particular +# callsite is. It is used to prune away callsites that are detected to +# be balanced; that is, that have matching AddRef/Release() pairs. + +my %imbalance; +$imbalance{'.root'} = 'n/a'; + +# The main read loop. + +sub read_data($$$) { + my ($INFILE, $plus, $minus) = @_; + + LINE: while (<$INFILE>) { + next LINE if (! /^</); + my @fields = split(/ /, $_); + + my $class = shift(@fields); + my $obj = shift(@fields); + my $sno = shift(@fields); + next LINE unless ($obj eq $::opt_object); + + my $op = shift(@fields); + next LINE unless ($op eq $plus || $op eq $minus); + + my $cnt = shift(@fields); + + # Collect the remaining lines to create a stack trace. We need to + # filter out the frame numbers so that frames that differ only in + # their frame number are considered equivalent. However, we need to + # keep a frame number on each line so that the fix*.py scripts can + # parse the output. So we set the frame number to 0 for every frame. + my @stack; + CALLSITE: while (<$INFILE>) { + chomp; + last CALLSITE if (/^$/); + $_ =~ s/#\d+: /#00: /; # replace frame number with 0 + $stack[++$#stack] = $_; + } + + # Reverse the remaining fields to produce the call stack, with the + # oldest frame at the front of the array. + if (! $::opt_reverse) { + @stack = reverse(@stack); + } + + my $call; + + # If any of the functions in the stack are supposed to be excluded, + # march on to the next line. + foreach $call (@stack) { + next LINE if exists($excludes{$call}); + } + + + # Add the callstack as a path through the call graph, updating + # refcounts at each node. + + my $caller = $callGraphRoot; + + foreach $call (@stack) { + + # Chop the method offset if we're 'collapsing to method' or + # 'collapsing to class'. + $call =~ s/\+0x.*$//g if ($::opt_collapse_to_method || $::opt_collapse_to_class); + + # Chop the method name if we're 'collapsing to class'. + $call =~ s/::.*$//g if ($::opt_collapse_to_class); + + my $site = $caller->{$call}; + if (!$site) { + # This is the first time we've seen this callsite. Add a + # new entry to the call tree. + + $site = { '#name#' => $call, '#refcount#' => 0 }; + $caller->{$call} = $site; + } + + if ($op eq $plus) { + ++($site->{'#refcount#'}); + ++($imbalance{$call}); + } elsif ($op eq $minus) { + --($site->{'#refcount#'}); + --($imbalance{$call}); + } else { + die "Bad operation $op"; + } + + $caller = $site; + } + } +} + +read_data(*STDIN, "AddRef", "Release"); + +if ($::opt_comptrs) { + warn "Subtracting comptr log ". $::opt_comptrs . "\n"; + open(COMPTRS, "<".$::opt_comptrs) + || die "unable to open $::opt_comptrs"; + + # read backwards to subtract + read_data(*COMPTRS, "nsCOMPtrRelease", "nsCOMPtrAddRef"); +} + +sub num_alpha { + my ($aN, $aS, $bN, $bS); + ($aN, $aS) = ($1, $2) if $a =~ /^(\d+) (.+)$/; + ($bN, $bS) = ($1, $2) if $b =~ /^(\d+) (.+)$/; + return $a cmp $b unless defined $aN && defined $bN; + return $aN <=> $bN unless $aN == $bN; + return $aS cmp $bS; +} + +# Given a subtree and its nesting level, return true if that subtree should be pruned. +# If it shouldn't be pruned, destructively attempt to prune its children. +# Also compute the #children# properties of unpruned nodes. +sub prune($$) { + my ($site, $nest) = @_; + + # If they want us to prune the tree's depth, do so here. + return 1 if ($::opt_prune_depth && $nest >= $::opt_prune_depth); + + # If the subtree is balanced, ignore it. + return 1 if ($::opt_ignore_balanced && !$site->{'#refcount#'}); + + my $name = $site->{'#name#'}; + + # If the symbol isn't imbalanced, then prune here (and warn) + if ($::opt_ignore_balanced && !$imbalance{$name}) { + warn "discarding " . $name . "\n"; +# return 1; + } + + my @children; + foreach my $child (sort num_alpha keys(%$site)) { + if (substr($child, 0, 1) ne '#') { + if (prune($site->{$child}, $nest + 1)) { + delete $site->{$child}; + } else { + push @children, $site->{$child}; + } + } + } + $site->{'#children#'} = \@children; + return 0; +} + + +# Compute the #label# properties of this subtree. +# Return the subtree's number of nodes, not counting nodes reachable +# through a labeled node. +sub createLabels($) { + my ($site) = @_; + my @children = @{$site->{'#children#'}}; + my $nChildren = @children; + my $nDescendants = 0; + + foreach my $child (@children) { + my $childDescendants = createLabels($child); + if ($nChildren > 1 && $childDescendants > $::opt_subtree_size) { + die "Internal error" if defined($child->{'#label#'}); + $child->{'#label#'} = "__label__"; + $childDescendants = 1; + } + $nDescendants += $childDescendants; + } + return $nDescendants + 1; +} + + +my $nextLabel = 0; +my @labeledSubtrees; + +sub list($$$$$) { + my ($site, $nest, $nestStr, $childrenLeft, $root) = @_; + my $label = !$root && $site->{'#label#'}; + + # Assign a unique number to the label. + if ($label) { + die unless $label eq "__label__"; + $label = "__" . ++$nextLabel . "__"; + $site->{'#label#'} = $label; + push @labeledSubtrees, $site; + } + + print $nestStr; + if ($::opt_old_style) { + print $label, " " if $label; + print $site->{'#name#'}, ": bal=", $site->{'#refcount#'}, "\n"; + } else { + my $refcount = $site->{'#refcount#'}; + my $l = 8 - length $refcount; + $l = 1 if $l < 1; + print $refcount, " " x $l; + print $label, " " if $label; + print $site->{'#name#'}, "\n"; + } + + $nestStr .= $childrenLeft && !$::opt_old_style ? "| " : " "; + if (!$label) { + my @children = @{$site->{'#children#'}}; + $childrenLeft = @children; + foreach my $child (@children) { + $childrenLeft--; + list($child, $nest + 1, $nestStr, $childrenLeft); + } + } +} + + +if (!prune($callGraphRoot, 0)) { + createLabels $callGraphRoot if ($::opt_subtree_size); + list $callGraphRoot, 0, "", 0, 1; + while (@labeledSubtrees) { + my $labeledSubtree = shift @labeledSubtrees; + print "\n------------------------------\n", +$labeledSubtree->{'#label#'}, "\n"; + list $labeledSubtree, 0, "", 0, 1; + } + print "\n------------------------------\n" if @labeledSubtrees; +} + +print qq{ +Imbalance +--------- +}; + +foreach my $call (sort num_alpha keys(%imbalance)) { + print $call . " " . $imbalance{$call} . "\n"; +} + diff --git a/tools/rewriting/Generated.txt b/tools/rewriting/Generated.txt new file mode 100644 index 0000000000..1fa9b6dffc --- /dev/null +++ b/tools/rewriting/Generated.txt @@ -0,0 +1,39 @@ +.gradle/ +build/vs/vs2019.yaml +build/vs/vs2022.yaml +browser/components/aboutwelcome/content/aboutwelcome.bundle.js +browser/components/aboutwelcome/logs/ +browser/components/aboutwelcome/node_modules/ +browser/components/asrouter/node_modules/ +browser/components/asrouter/content/asrouter-admin.bundle.js +browser/components/asrouter/logs/ +browser/components/newtab/content-src/asrouter/schemas/BackgroundTaskMessagingExperiment.schema.json +browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json +browser/components/newtab/logs/ +browser/components/newtab/node_modules/ +browser/components/storybook/storybook-static/ +browser/locales/l10n-changesets.json +browser/locales/l10n-onchange-changesets.json +devtools/client/aboutdebugging/test/jest/node_modules/ +devtools/client/application/test/components/node_modules/ +devtools/client/debugger/node_modules/ +dom/tests/ajax/jquery/ +dom/tests/ajax/mochikit/ +intl/components/src/UnicodeScriptCodes.h +intl/unicharutil/util/nsSpecialCasingData.cpp +intl/unicharutil/util/nsUnicodePropertyData.cpp +mobile/locales/l10n-changesets.json +mobile/locales/l10n-onchange-changesets.json +node_modules/ +python/mozperftest/mozperftest/tests/data/ +security/manager/tools/KnownRootHashes.json +security/manager/tools/PreloadedHPKPins.json +services/settings/dumps/ +toolkit/components/nimbus/schemas/NimbusExperiment.schema.json +toolkit/components/pdfjs/content/PdfJsDefaultPreferences.sys.mjs +toolkit/components/uniffi-js/UniFFIGeneratedScaffolding.cpp +toolkit/components/uniffi-js/UniFFIFixtureScaffolding.cpp +toolkit/components/uniffi-bindgen-gecko-js/fixtures/generated +tools/browsertime/package.json +tools/browsertime/package-lock.json +try_task_config.json diff --git a/tools/rewriting/ThirdPartyPaths.txt b/tools/rewriting/ThirdPartyPaths.txt new file mode 100644 index 0000000000..5b0309c3e5 --- /dev/null +++ b/tools/rewriting/ThirdPartyPaths.txt @@ -0,0 +1,201 @@ +browser/components/newtab/vendor/ +browser/components/pocket/content/panels/css/normalize.scss +browser/components/pocket/content/panels/js/vendor/ +browser/components/storybook/node_modules/ +browser/extensions/formautofill/content/third-party/ +browser/extensions/formautofill/test/fixtures/third_party/ +devtools/client/inspector/markup/test/lib_* +devtools/client/jsonview/lib/require.js +devtools/client/shared/build/babel.js +devtools/client/shared/source-map/ +devtools/client/shared/sourceeditor/codemirror/ +devtools/client/shared/sourceeditor/codemirror6/ +devtools/client/shared/sourceeditor/test/cm_mode_ruby.js +devtools/client/shared/sourceeditor/test/codemirror/ +devtools/client/shared/vendor/ +devtools/client/inspector/markup/test/helper_diff.js +devtools/client/debugger/src/workers/parser/utils/parse-script-tags/ +devtools/shared/acorn/ +devtools/shared/compatibility/dataset/css-properties.json +devtools/shared/heapsnapshot/CoreDump.pb.cc +devtools/shared/heapsnapshot/CoreDump.pb.h +devtools/shared/jsbeautify/ +devtools/shared/node-properties/ +devtools/shared/qrcode/decoder/ +devtools/shared/qrcode/encoder/ +devtools/shared/sprintfjs/ +devtools/shared/storage/vendor/ +dom/canvas/test/webgl-conf/checkout/ +dom/imptests/ +dom/media/gmp/rlz/ +dom/media/gmp/widevine-adapter/content_decryption_module_export.h +dom/media/gmp/widevine-adapter/content_decryption_module_ext.h +dom/media/gmp/widevine-adapter/content_decryption_module.h +dom/media/gmp/widevine-adapter/content_decryption_module_proxy.h +dom/media/platforms/ffmpeg/ffmpeg57/ +dom/media/platforms/ffmpeg/ffmpeg58/ +dom/media/platforms/ffmpeg/ffmpeg59/ +dom/media/platforms/ffmpeg/ffmpeg60/ +dom/media/platforms/ffmpeg/libav53/ +dom/media/platforms/ffmpeg/libav54/ +dom/media/platforms/ffmpeg/libav55/ +dom/media/webaudio/test/blink/ +dom/media/webrtc/tests/mochitests/helpers_from_wpt/sdp.js +dom/media/webrtc/transport/third_party/ +dom/media/webspeech/recognition/endpointer.cc +dom/media/webspeech/recognition/endpointer.h +dom/media/webspeech/recognition/energy_endpointer.cc +dom/media/webspeech/recognition/energy_endpointer.h +dom/media/webspeech/recognition/energy_endpointer_params.cc +dom/media/webspeech/recognition/energy_endpointer_params.h +dom/media/webvtt/vtt.sys.mjs +dom/tests/mochitest/ajax/ +dom/tests/mochitest/dom-level1-core/ +dom/tests/mochitest/dom-level2-core/ +dom/tests/mochitest/dom-level2-html/ +dom/u2f/tests/pkijs/ +dom/webauthn/tests/pkijs/ +dom/webgpu/tests/cts/checkout/ +editor/libeditor/tests/browserscope/lib/richtext/ +editor/libeditor/tests/browserscope/lib/richtext2/ +extensions/spellcheck/hunspell/src/ +function2/ +gfx/angle/checkout/ +gfx/cairo/ +gfx/graphite2/ +gfx/harfbuzz/ +gfx/ots/ +gfx/qcms/ +gfx/sfntly/ +gfx/skia/ +gfx/vr/service/openvr/ +gfx/vr/service/openvr/headers/openvr.h +gfx/vr/service/openvr/src/README +gfx/vr/service/openvr/src/dirtools_public.cpp +gfx/vr/service/openvr/src/dirtools_public.h +gfx/vr/service/openvr/src/envvartools_public.cpp +gfx/vr/service/openvr/src/envvartools_public.h +gfx/vr/service/openvr/src/hmderrors_public.cpp +gfx/vr/service/openvr/src/hmderrors_public.h +gfx/vr/service/openvr/src/ivrclientcore.h +gfx/vr/service/openvr/src/openvr_api_public.cpp +gfx/vr/service/openvr/src/pathtools_public.cpp +gfx/vr/service/openvr/src/pathtools_public.h +gfx/vr/service/openvr/src/sharedlibtools_public.cpp +gfx/vr/service/openvr/src/sharedlibtools_public.h +gfx/vr/service/openvr/src/strtools_public.cpp +gfx/vr/service/openvr/src/strtools_public.h +gfx/vr/service/openvr/src/vrpathregistry_public.cpp +gfx/vr/service/openvr/src/vrpathregistry_public.h +gfx/wr/ +gfx/ycbcr/ +intl/icu/ +intl/icu_capi/ +ipc/chromium/src/third_party/ +js/src/ctypes/libffi/ +js/src/dtoa.c +js/src/editline/ +js/src/jit/arm64/vixl/ +js/src/octane/ +js/src/tests/test262/ +js/src/vtune/disable_warnings.h +js/src/vtune/ittnotify_config.h +js/src/vtune/ittnotify.h +js/src/vtune/ittnotify_static.c +js/src/vtune/ittnotify_static.h +js/src/vtune/ittnotify_types.h +js/src/vtune/jitprofiling.c +js/src/vtune/jitprofiling.h +js/src/vtune/legacy/ +js/src/zydis/ +layout/docs/css-gap-decorations/ +media/ffvpx/ +media/kiss_fft/ +media/libaom/ +media/libcubeb/ +media/libdav1d/ +media/libjpeg/ +media/libmkv/ +media/libnestegg/ +media/libogg/ +media/libopus/ +media/libpng/ +media/libsoundtouch/ +media/libspeex_resampler/ +media/libtheora/ +media/libvorbis/ +media/libvpx/ +media/libwebp/ +media/libyuv/ +media/mozva/va +media/mp4parse-rust/ +media/openmax_dl/ +media/openmax_il/ +media/webrtc/signaling/gtest/MockCall.h +mfbt/double-conversion/double-conversion/ +mfbt/lz4/.* +mobile/android/exoplayer2/ +modules/brotli/ +modules/fdlibm/ +modules/freetype2/ +modules/woff2/ +modules/xz-embedded/ +modules/zlib/ +mozglue/misc/decimal/ +mozglue/tests/glibc_printf_tests/ +netwerk/dns/nsIDNKitInterface.h +netwerk/sctp/src/ +netwerk/srtp/src/ +nsprpub/ +other-licenses/ +parser/expat/ +remote/cdp/test/browser/chrome-remote-interface.js +remote/test/puppeteer/ +security/manager/tools/log_list.json +security/nss/ +security/sandbox/chromium/ +security/sandbox/chromium-shim/ +services/common/kinto-http-client.sys.mjs +services/common/kinto-offline-client.js +testing/gtest/gmock/ +testing/gtest/gtest/ +testing/mochitest/MochiKit/ +testing/mochitest/pywebsocket3/ +testing/mochitest/tests/MochiKit-1.4.2/ +testing/modules/sinon-7.2.7.js +testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/ +testing/talos/talos/tests/devtools/addon/content/pages/ +testing/talos/talos/tests/dromaeo/ +testing/talos/talos/tests/kraken/ +testing/talos/talos/tests/offscreencanvas/benchmarks/video/demuxer_mp4.js +testing/talos/talos/tests/offscreencanvas/benchmarks/video/mp4box.all.min.js +testing/talos/talos/tests/v8_7/ +testing/web-platform/tests/resources/webidl2/ +testing/web-platform/tests/tools/third_party/ +testing/web-platform/mozilla/tests/webgpu/ +testing/xpcshell/dns-packet/ +testing/xpcshell/node_ip/ +testing/xpcshell/node-http2/ +testing/xpcshell/node-ws/ +third_party/ +toolkit/components/certviewer/content/vendor/ +toolkit/components/jsoncpp/ +toolkit/components/normandy/vendor/ +toolkit/components/passwordmgr/PasswordRulesParser.sys.mjs +toolkit/components/protobuf/ +toolkit/components/reader/readability/ +toolkit/components/translation/cld2/ +toolkit/components/translations/bergamot-translator/thirdparty +toolkit/components/translations/bergamot-translator/bergamot-translator.js +toolkit/components/url-classifier/chromium/ +toolkit/components/utils/mozjexl.js +toolkit/components/viaduct/fetch_msg_types.pb.cc +toolkit/components/viaduct/fetch_msg_types.pb.h +toolkit/content/widgets/vendor/lit.all.mjs +toolkit/crashreporter/breakpad-client/ +toolkit/crashreporter/google-breakpad/ +tools/fuzzing/libfuzzer/ +tools/profiler/core/vtune/ +xpcom/build/mach_override.c +xpcom/build/mach_override.h +xpcom/io/crc32c.c diff --git a/tools/rusttests/app.mozbuild b/tools/rusttests/app.mozbuild new file mode 100644 index 0000000000..cdef0623df --- /dev/null +++ b/tools/rusttests/app.mozbuild @@ -0,0 +1,5 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +include("/toolkit/toolkit.mozbuild") diff --git a/tools/rusttests/config/mozconfigs/common b/tools/rusttests/config/mozconfigs/common new file mode 100644 index 0000000000..cac5b57ce9 --- /dev/null +++ b/tools/rusttests/config/mozconfigs/common @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file is included by all "tools/rusttests" mozconfigs + +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_PACKAGE=0 +MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0 +MOZ_AUTOMATION_UPLOAD=0 +MOZ_AUTOMATION_CHECK=0 +ac_add_options --enable-project=tools/rusttests +. "$topsrcdir/build/mozconfig.common" +. "$topsrcdir/build/mozconfig.common.override" + +unset ENABLE_CLANG_PLUGIN + +# Test geckodriver, which isn't built as part of the build jobs +ac_add_options --enable-geckodriver diff --git a/tools/rusttests/config/mozconfigs/linux32/rusttests b/tools/rusttests/config/mozconfigs/linux32/rusttests new file mode 100644 index 0000000000..fa52698f9e --- /dev/null +++ b/tools/rusttests/config/mozconfigs/linux32/rusttests @@ -0,0 +1,2 @@ +. "$topsrcdir/build/unix/mozconfig.linux32" +. "$topsrcdir/tools/rusttests/config/mozconfigs/common" diff --git a/tools/rusttests/config/mozconfigs/linux32/rusttests-debug b/tools/rusttests/config/mozconfigs/linux32/rusttests-debug new file mode 100644 index 0000000000..b207ac6b4b --- /dev/null +++ b/tools/rusttests/config/mozconfigs/linux32/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/linux32/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/linux64/rusttests b/tools/rusttests/config/mozconfigs/linux64/rusttests new file mode 100644 index 0000000000..64d502087b --- /dev/null +++ b/tools/rusttests/config/mozconfigs/linux64/rusttests @@ -0,0 +1,2 @@ +. "$topsrcdir/build/unix/mozconfig.linux" +. "$topsrcdir/tools/rusttests/config/mozconfigs/common" diff --git a/tools/rusttests/config/mozconfigs/linux64/rusttests-debug b/tools/rusttests/config/mozconfigs/linux64/rusttests-debug new file mode 100644 index 0000000000..ef7157e53b --- /dev/null +++ b/tools/rusttests/config/mozconfigs/linux64/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/linux64/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/macosx64/rusttests b/tools/rusttests/config/mozconfigs/macosx64/rusttests new file mode 100644 index 0000000000..f70026f5fd --- /dev/null +++ b/tools/rusttests/config/mozconfigs/macosx64/rusttests @@ -0,0 +1,2 @@ +. "$topsrcdir/build/macosx/mozconfig.common" +. "$topsrcdir/tools/rusttests/config/mozconfigs/common" diff --git a/tools/rusttests/config/mozconfigs/macosx64/rusttests-debug b/tools/rusttests/config/mozconfigs/macosx64/rusttests-debug new file mode 100644 index 0000000000..760d51266e --- /dev/null +++ b/tools/rusttests/config/mozconfigs/macosx64/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/macosx64/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/win32/rusttests b/tools/rusttests/config/mozconfigs/win32/rusttests new file mode 100644 index 0000000000..d953f9c0df --- /dev/null +++ b/tools/rusttests/config/mozconfigs/win32/rusttests @@ -0,0 +1,3 @@ +ac_add_options --target=i686-pc-windows-msvc +. "$topsrcdir/build/win32/mozconfig.vs-latest" +. "$topsrcdir/tools/rusttests/config/mozconfigs/windows-common" diff --git a/tools/rusttests/config/mozconfigs/win32/rusttests-debug b/tools/rusttests/config/mozconfigs/win32/rusttests-debug new file mode 100644 index 0000000000..046d2d0a1d --- /dev/null +++ b/tools/rusttests/config/mozconfigs/win32/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/win32/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/win64/rusttests b/tools/rusttests/config/mozconfigs/win64/rusttests new file mode 100644 index 0000000000..ed81e3b767 --- /dev/null +++ b/tools/rusttests/config/mozconfigs/win64/rusttests @@ -0,0 +1,3 @@ +ac_add_options --target=x86_64-pc-windows-msvc +. "$topsrcdir/build/win64/mozconfig.vs-latest" +. "$topsrcdir/tools/rusttests/config/mozconfigs/windows-common" diff --git a/tools/rusttests/config/mozconfigs/win64/rusttests-debug b/tools/rusttests/config/mozconfigs/win64/rusttests-debug new file mode 100644 index 0000000000..5f079b4d36 --- /dev/null +++ b/tools/rusttests/config/mozconfigs/win64/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/win64/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/windows-common b/tools/rusttests/config/mozconfigs/windows-common new file mode 100644 index 0000000000..afdb996920 --- /dev/null +++ b/tools/rusttests/config/mozconfigs/windows-common @@ -0,0 +1,2 @@ +. "$topsrcdir/build/mozconfig.win-common" +. "$topsrcdir/tools/rusttests/config/mozconfigs/common" diff --git a/tools/rusttests/moz.configure b/tools/rusttests/moz.configure new file mode 100644 index 0000000000..520989a8cf --- /dev/null +++ b/tools/rusttests/moz.configure @@ -0,0 +1,5 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +include("../../toolkit/moz.configure") diff --git a/tools/sanitizer/docs/asan.rst b/tools/sanitizer/docs/asan.rst new file mode 100644 index 0000000000..876c1e83f4 --- /dev/null +++ b/tools/sanitizer/docs/asan.rst @@ -0,0 +1,379 @@ +Address Sanitizer +================= + +What is Address Sanitizer? +-------------------------- + +Address Sanitizer (ASan) is a fast memory error detector that detects +use-after-free and out-of-bound bugs in C/C++ programs. It uses a +compile-time instrumentation to check all reads and writes during the +execution. In addition, the runtime part replaces the ``malloc`` and +``free`` functions to check dynamically allocated memory. More +information on how ASan works can be found on `the Address Sanitizer +wiki <https://github.com/google/sanitizers/wiki/AddressSanitizer>`__. + +A `meta bug called asan-maintenance <https://bugzilla.mozilla.org/show_bug.cgi?id=asan-maintenance>`__ +is maintained to keep track of all the bugs found with ASan. + +Downloading artifact builds +--------------------------- + +For Linux and Windows users, the easiest way to get Firefox builds with +Address Sanitizer is to download a continuous integration asan build of +mozilla-central (updated at least daily): + +- mozilla-central optimized builds: + `linux <https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.linux64-asan-opt/artifacts/public/build/target.tar.bz2>`__ + \| + `windows <https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.win64-asan-opt/artifacts/public/build/target.zip>`__ + (recommended for testing) +- mozilla-central debug builds: + `linux <https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.linux64-asan-debug/artifacts/public/build/target.tar.bz2>`__ + \| + `windows <https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.win64-asan-debug/artifacts/public/build/target.zip>`__ + (recommended for debugging if the optimized builds don't do the job) + +The fuzzing team also offers a tool called ``fuzzfetch`` to download these and many +other CI builds. It makes downloading and unpacking these builds much easier and +can be used not just for fuzzing but for all purposes that require a CI build download. + +You can install ``fuzzfetch`` from +`Github <https://github.com/MozillaSecurity/fuzzfetch>`__ or +`via pip <https://pypi.org/project/fuzzfetch/>`__. + +Afterwards, you can run e.g. + +:: + + $ python -m fuzzfetch --asan -n firefox-asan + +to get the optimized Linux ASan build mentioned above unpacked into a directory called ``firefox-asan``. +The ``--debug`` and ``--os`` switches can be used to get the other variants listed above. + +Creating Try builds +------------------- + +If for some reason you can't use the pre-built binaries mentioned in the +previous section (e.g. you want a non-Linux build or you need to test a +patch), you can either build Firefox yourself (see the following +section) or use the :ref:`try server <Pushing to Try>` to +create the customized build for you. Pushing to try requires L1 commit +access. If you don't have this access yet you can request access (see +`Becoming A Mozilla +Committer <https://www.mozilla.org/about/governance/policies/commit/>`__ +and `Mozilla Commit Access +Policy <https://www.mozilla.org/about/governance/policies/commit/access-policy/>`__ +for the requirements). + +The tree contains `several mozconfig files for creating asan +builds <https://searchfox.org/mozilla-central/search?q=&case=true&path=browser%2Fconfig%2Fmozconfigs%2F*%2F*asan*>`__ +(the "nightly-asan" files create release builds, whereas the +"debug-asan" files create debug+opt builds). For Linux builds, the +appropriate configuration file is used by the ``linux64-asan`` target. +If you want to create a macOS or Windows build, you'll need to copy the +appropriate configuration file over the regular debug configuration +before pushing to try. For example: + +:: + + cp browser/config/mozconfigs/macosx64/debug-asan browser/config/mozconfigs/macosx64/debug + +You can then `push to Try in the usual +way </tools/try/index.html#using-try>`__ +and, once the build is complete, download the appropriate build +artifact. + +Creating local builds on Windows +-------------------------------- + +On Windows, ASan is supported only in 64-bit builds. + +Run ``mach bootstrap`` to get an updated clang-cl in your +``~/.mozbuild`` directory, then use the following +:ref:`mozconfig <Configuring Build Options>`: + +:: + + ac_add_options --enable-address-sanitizer + ac_add_options --disable-jemalloc + + export LDFLAGS="clang_rt.asan_dynamic-x86_64.lib clang_rt.asan_dynamic_runtime_thunk-x86_64.lib" + CLANG_LIB_DIR="$(cd ~/.mozbuild/clang/lib/clang/*/lib/windows && pwd)" + export MOZ_CLANG_RT_ASAN_LIB_PATH="${CLANG_LIB_DIR}/clang_rt.asan_dynamic-x86_64.dll" + export PATH=$CLANG_LIB_DIR:$PATH + +If you launch an ASan build under WinDbg, you may see spurious +first-chance Access Violation exceptions. These come from ASan creating +shadow memory pages on demand, and can be ignored. Run ``sxi av`` to +ignore these exceptions. (You will still catch second-chance Access +Violation exceptions if you actually crash.) + +LeakSanitizer (LSan) is not supported on Windows. + +Creating local builds on Linux or Mac +------------------------------------- + +Build prerequisites +~~~~~~~~~~~~~~~~~~~ + +LLVM/Clang +^^^^^^^^^^ + +The ASan instrumentation is implemented as an LLVM pass and integrated +into Clang. Any clang version that is capable of compiling Firefox has +everything needed to do an ASAN build. + +Building Firefox +~~~~~~~~~~~~~~~~ + +Getting the source +^^^^^^^^^^^^^^^^^^ + +Using that or any later revision, all you need to do is to :ref:`get yourself +a clone of mozilla-central <Mercurial overview>`. + +Adjusting the build configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the build configuration file ``mozconfig`` with the following +content in your mozilla-central directory: + +:: + + # Combined .mozconfig file for ASan on Linux+Mac + + mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/objdir-ff-asan + + # Enable ASan specific code and build workarounds + ac_add_options --enable-address-sanitizer + + # These three are required by ASan + ac_add_options --disable-jemalloc + ac_add_options --disable-crashreporter + ac_add_options --disable-elf-hack + + # Keep symbols to symbolize ASan traces later + export MOZ_DEBUG_SYMBOLS=1 + ac_add_options --enable-debug-symbols + ac_add_options --disable-install-strip + + # Settings for an opt build (preferred) + # The -gline-tables-only ensures that all the necessary debug information for ASan + # is present, but the rest is stripped so the resulting binaries are smaller. + ac_add_options --enable-optimize="-O2 -gline-tables-only" + ac_add_options --disable-debug + + # Settings for a debug+opt build + #ac_add_options --enable-optimize + #ac_add_options --enable-debug + + # MacOSX only: Uncomment and adjust this path to match your SDK + # ac_add_options --with-macos-sdk=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.8.sdk + +You may also need this, as seen in +``browser/config/mozconfigs/linux64/nightly-asan`` (the config file used +for Address Sanitizer builds used for automated testing): + +:: + + # ASan specific options on Linux + ac_add_options --enable-valgrind + +Starting the build process +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now you start the build process using the regular ``./mach build`` +command. + +Starting Firefox +^^^^^^^^^^^^^^^^ + +After the build has completed, ``./mach run`` with the usual options for +running in a debugger (``gdb``, ``lldb``, ``rr``, etc.) work fine, as do +the ``--disable-e10s`` and other options. + +Building only the JavaScript shell +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to build only the JavaScript shell instead of doing a full +Firefox build, the build script below will probably help you to do so. +Execute this script in the ``js/src/`` subdirectory and pass a directory +name as the first parameter. The build will then be created in a new +subdirectory with that name. + +:: + + #! /bin/sh + + if [ -z $1 ] ; then + echo "usage: $0 <dirname>" + elif [ -d $1 ] ; then + echo "directory $1 already exists" + else + autoconf2.13 + mkdir $1 + cd $1 + CC="clang" \ + CXX="clang++" \ + CFLAGS="-fsanitize=address" \ + CXXFLAGS="-fsanitize=address" \ + LDFLAGS="-fsanitize=address" \ + ../configure --enable-debug --enable-optimize --enable-address-sanitizer --disable-jemalloc + fi + +Getting Symbols in Address Sanitizer Traces +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, ASan traces are unsymbolized and only print the +binary/library and a memory offset instead. In order to get more useful +traces, containing symbols, there are two approaches. + +Using the LLVM Symbolizer (recommended) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +LLVM ships with a symbolizer binary that ASan will readily use to +immediately output symbolized traces. To use it, just set the +environment variable ``ASAN_SYMBOLIZER_PATH`` to reflect the location of +your ``llvm-symbolizer`` binary, before running the process. This +program is usually included in an LLVM distribution. Stacks without +symbols can also be post-processed, see below. + +.. warning:: + + .. note:: + + **Warning:** On OS X, the content sandbox prevents the symbolizer + from running. To use llvm-symbolizer on ASan output from a + content process, the content sandbox must be disabled. This can be + done by setting ``MOZ_DISABLE_CONTENT_SANDBOX=1`` in your run + environment. Setting this in .mozconfig has no effect. + + +Post-Processing Traces with asan_symbolize.py +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Instead of using the llvm-symbolizer binary, you can also pipe the +output through the ``asan_symbolize.py`` script, shipped with LLVM +(``$LLVM_HOME/projects/compiler-rt/lib/asan/scripts/asan_symbolize.py``), +often included in LLVM distributions. The disadvantage is that the +script will need to use ``addr2line`` to get the symbols, which means +that every library will have to be loaded into memory +(including``libxul``, which takes a bit). + +However, in certain situations it makes sense to use this script. For +example, if you have/received an unsymbolized trace, then you can still +use the script to turn it into a symbolized trace, given that you can +get the original binaries that produced the unsymbolized trace. In order +for the script to work in such cases, you need to ensure that the paths +in the trace point to the actual binaries, or change the paths +accordingly. + +Since the output of the ``asan_symbolize.py`` script is still mangled, +you might want to pipe the output also through ``c++filt`` afterwards. + +Troubleshooting / Known problems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Cannot specify -o when generating multiple output files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you get the error +"``cannot specify -o when generating multiple output files"`` from +clang, disable ``elf-hack`` in your ``mozconfig`` to work around the +issue: + +:: + + ac_add_options --disable-elf-hack + +Optimized build +^^^^^^^^^^^^^^^ + +Since `an issue with -O2/-Os and +ASan <https://github.com/google/sanitizers/issues/20>`__ +has been resolved, the regular optimizations used by Firefox should work +without any problems. The optimized build has only a barely noticeable +speed penalty and seems to be even faster than regular debug builds. + +No "AddressSanitizer: **libc** interceptors initialized" shows after running ./mach run +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + $ ASAN_OPTIONS=verbosity=2 ./mach run + +Use the above command instead + +"An admin user name and password" is required to enter Developer Mode +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Please enable **Developer** **mode** by: + +:: + + $ /usr/sbin/DevToolsSecurity -enable + Developer mode is now enabled. + +Debugging issues that ASan finds +-------------------------------- + +When ASan discovers an issue it will simply print an error message and +exit the app. To stop the app in a debugger before ASan exits it, set a +breakpoint on ``__asan::ReportGenericError``. For more info on using +ASan and debugging issues that it uncovers, see the page `Address +sanitizer and a +debugger <https://github.com/google/sanitizers/wiki/AddressSanitizerAndDebugger>`__ +page on the upstream wiki. + +``__asan_describe_address(pointer)`` issued at the debugger prompt or +even directly in the code allows outputting lots of information about +this memory address (thread and stack of allocation, of deallocation, +whether or not it is a bit outside a known buffer, thread and stack of +allocation of this buffer, etc.). This can be useful to understand where +some buffer that is not aligned was allocated, when doing SIMD work, for +example. + +`rr <https://rr-project.org/>`__ (Linux x86 only) works great with ASan +and combined, this combo allows doing some very powerful debugging +strategies. + +LeakSanitizer +------------- + +LeakSanitizer (LSan) is a special execution mode for regular ASan. It +takes advantage of how ASan tracks the set of live blocks at any given +point to print out the allocation stack of any block that is still alive +at shutdown, but is not reachable from the stack, according to a +conservative scan. This is very useful for detecting leaks of things +such as ``char*`` that do not participate in the usual Gecko shutdown +leak detection. LSan is supported on x86_64 Linux and OS X. + +LSan is enabled by default in ASan builds, as of more recent versions of +Clang. To make an ASan build not run LSan, set the environment variable +``ASAN_OPTIONS`` to ``detect_leaks=0`` (or add it as an entry to a +``:``-separated list if it is already set to something). If you want to +enable it when it is not for some reason, set it to 1 instead of 0. If +LSan is enabled and you are using a non-debug build, you will also want +to set the environment variable ``MOZ_CC_RUN_DURING_SHUTDOWN=1``, to +ensure that we run shutdown GCs and CCs to avoid spurious leaks. + +If an object that is reported by LSan is intentionally never freed, a +symbol can be added to ``build/sanitizers/lsan_suppressions.txt`` to get +LSan to ignore it. + +For some more information on LSan, see the `Leak Sanitizer wiki +page <https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer>`__. + + +A `meta bug called lsan <https://bugzilla.mozilla.org/show_bug.cgi?id=lsan>`__ +is maintained to keep track of all the bugs found with LSan. + + + +Frequently Asked Questions about ASan +------------------------------------- + +How does ASan work exactly? +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +More information on how ASan works can be found on `the Address Sanitizer wiki <https://github.com/google/sanitizers/wiki/AddressSanitizer>`__. diff --git a/tools/sanitizer/docs/asan_nightly.rst b/tools/sanitizer/docs/asan_nightly.rst new file mode 100644 index 0000000000..9e6fdcd52e --- /dev/null +++ b/tools/sanitizer/docs/asan_nightly.rst @@ -0,0 +1,204 @@ +ASan Nightly +============ + +The **ASan Nightly Project** involves building a Firefox Nightly browser +with the popular +`AddressSanitizer <https://github.com/google/sanitizers/wiki/AddressSanitizer>`__ +tool and enhancing it with remote crash reporting capabilities for any +errors detected. + +The purpose of the project is to find subtle memory corruptions +occurring during regular browsing that would either not crash at all or +crash in a way that we cannot figure out what the exact problem is just +from the crash dump. We have a lot of inactionable crash reports and +AddressSanitizer traces are usually a lot more actionable on their own +(especially use-after-free traces). Part of this project is to figure +out if and how many actionable crash reports ASan can give us just by +surfing around. The success of the project of course also depends on the +number of participants. + +You can download the latest build using one of the links below. The +builds are self-updating daily like regular nightly builds (like with +regular builds, you can go to *"Help"* → *"About Nightly"* to force an +update check or confirm that you run the latest version). + +.. note:: + + If you came here looking for regular ASan builds (e.g. for fuzzing or + as a developer to reproduce a crash), you should probably go to the + :ref:`Address Sanitizer` doc instead. + +.. _Requirements: + +Requirements +~~~~~~~~~~~~ + +Current requirements are: + +- Windows or Linux-based Operating System +- 16 GB of RAM recommended +- Special ASan Nightly Firefox Build + + - `Linux + Download <https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.shippable.latest.firefox.linux64-asan-reporter-opt/artifacts/public/build/target.tar.bz2>`__ + - `Windows + Download <https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.shippable.latest.firefox.win64-asan-reporter-shippable-repackage-signing/artifacts/public/build/target.installer.exe>`__ + +If you are already using regular Nightly, it should be safe to share the +profile with the regular Nightly instance. If you normally use a beta or +release build (and you would like to be able to switch back to these), +you should consider using a second profile. + +.. warning:: + + **Windows Users:** Please note that the Windows builds currently show + an error during setup (see "*Known Issues*" section below), but + installation works nonetheless. We are working on the problem. + +.. note:: + + If you run in an environment with any sorts of additional security + restrictions (e.g. custom process sandboxing), please make sure that + your /tmp directory is writable and the shipped ``llvm-symbolizer`` + binary is executable from within the Firefox process. + +Preferences +~~~~~~~~~~~ + +If you wish for your crash report to be identifiable, you can go to +``about:config`` and set the **``asanreporter.clientid``** to your +**valid email address**. This isn't mandatory, you can of course report +crash traces anonymously. If you decide to send reports with your email +address and you have a Bugzilla account, consider using the same email +as your Bugzilla account uses. We will then Cc you on any bugs filed +from your crash reports. If your email does not belong to a Bugzilla +account, then we will not publish it but only use it to resolve +questions about your crash reports. + +.. note:: + + Setting this preference helps us to get back to you in case we have + questions about your setup/OS. Please consider using it so we can get + back to you if necessary. + +Bug Bounty Program +~~~~~~~~~~~~~~~~~~ + +As a special reward for participating in the program, we decided to +treat all submitted reports as if they were filed directly in Bugzilla. +This means that reports that + +- indicate a security issue of critical or high rating +- **and** that can be fixed by our developers + +are eligible for a bug bounty according to our `client bug bounty +program +rules <https://www.mozilla.org/security/client-bug-bounty/>`__. As +the report will usually not include any steps to reproduce or a test +case, it will most likely receive a lower-end bounty. Like with regular +bug reports, we would typically reward the first (identifiable) report of +an issue. + +.. warning:: + + If you would like to participate in the bounty program, make sure you + set your **``asanreporter.clientid``** preference as specified above. + We cannot reward any reports that are submitted with no email + address. + + +Known Issues +~~~~~~~~~~~~ + +This section lists all currently known limitations of the ASan Nightly +builds that are considered bugs. + +- [STRIKEOUT:Flash is currently not working] +- `Bug + 1477490 <https://bugzilla.mozilla.org/show_bug.cgi?id=1477490>`__\ [STRIKEOUT:- + Windows: Stack instrumentation disabled due to false positives] +- `Bug + 1478096 <https://bugzilla.mozilla.org/show_bug.cgi?id=1478096>`__ - + **Windows:** Error during install with maintenanceservice_tmp.exe +- It has been reported that ASan Nightly performance is particularly + bad if you run on a screen with 120hz refresh rate. Switching to 60hz + should improve performance drastically. + +Note that these bugs are **specific** to ASan Nightly as listed in the +`tracking bug dependency +list <https://bugzilla.mozilla.org/showdependencytree.cgi?id=1386297&hide_resolved=0>`__. +For the full list of bugs found by this project, see `this +list <https://bugzilla.mozilla.org/showdependencytree.cgi?id=1479399&hide_resolved=0>`__ +instead and note that some bugs might not be shown because they are +security bugs. + +If you encounter a bug not listed here, please file a bug at +`bugzilla.mozilla.org <https://bugzilla.mozilla.org/>`__ or send an +email to +`choller@mozilla.com <mailto:choller@mozilla.com?subject=%5BASan%20Nightly%20Project%5D%5BBug%20Report%5D>`__. +When filing a bug, it greatly helps if you Cc that email address and +make the bug block `bug +1386297 <https://bugzilla.mozilla.org/show_bug.cgi?id=1386297>`__. + +FAQ +~~~ + +What additional data is collected? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The project only collects ASan traces and (if you set it in the +preferences) your email address. We don't collect any other browser +data, in particular not the sites you were visiting or page contents. It +is really just crash traces submitted to a remote location. + +.. note:: + + The ASan Nightly browser also still has all the data collection + capabilities of a regular Nightly browser. The answer above only + refers to what this project collects **in addition** to what the + regular Nightly browser can collect. + +What's the performance impact? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ASan Nightly build only comes with a slight slowdown at startup and +browsing, sometimes it is not even noticeable. The RAM consumption +however is much higher than with a regular build. Be prepared to restart +your browser sometimes, especially if you use a lot of tabs at once. +Also, the updates are larger than the regular ones, so download times +for updates will be higher, especially if you have a slower internet +connection. + +.. warning:: + + If you experience performance issues, see also the *"Known Issues"* + section above, in particular the problem about screen refresh rate + slowing down Firefox. + +What about stability? +^^^^^^^^^^^^^^^^^^^^^ + +The browser is as stable as a regular Nightly build. Various people have +been surfing around with it for their daily work for weeks now and we +have barely received any crash reports. + +How do I confirm that I'm running the correct build? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you open ``about:config`` and type *"asanreporter"* into the search +field, you should see an entry called ``asanreporter.apiurl`` associated +with a URL. Do not modify this value. + +.. warning:: + + Since Firefox 64, the *"ASan Crash Reporter"* feature is no longer + listed in ``about:support`` + +Will there be support for Mac? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We are working on support for Mac, but it might take longer because we +have no ASan CI coverage on Mac due to hardware constraints. If you work +on Release Engineering and would like to help make e.g. Mac happen +earlier, feel free to `contact +me <mailto:choller@mozilla.com?subject=%5BASan%20Nightly%20Project%5D%20>`__. diff --git a/tools/sanitizer/docs/index.rst b/tools/sanitizer/docs/index.rst new file mode 100644 index 0000000000..3a1dc0806e --- /dev/null +++ b/tools/sanitizer/docs/index.rst @@ -0,0 +1,33 @@ +Sanitizer +========= + +.. toctree:: + :maxdepth: 1 + :hidden: + :glob: + + * + +**Address Sanitizer** + +Address Sanitizer (ASan) is a fast memory error detector that detects use-after-free and out-of-bound bugs in C/C++ programs. It uses a compile-time instrumentation to check all reads and writes during the execution. In addition, the runtime part replaces the malloc and free functions to check dynamically allocated memory. + +:ref:`More information <Address Sanitizer>` + +**Thread Sanitizer** + +Thread Sanitizer (TSan) is a fast data race detector for C/C++ programs. It uses a compile-time instrumentation to check all non-race-free memory access at runtime. Unlike other tools, it understands compiler-builtin atomics and synchronization and therefore provides very accurate results with no false positives (except if unsupported synchronization primitives like inline assembly or memory fences are used). + +:ref:`More information <Thread Sanitizer>` + +**Memory Sanitizer** + +Memory Sanitizer (MSan) is a fast detector used for uninitialized memory in C/C++ programs. It uses a compile-time instrumentation to ensure that all memory access at runtime uses only memory that has been initialized. + +:ref:`More information <Memory Sanitizer>` + +**ASan Nightly Project** + +The ASan Nightly Project involves building a Firefox Nightly browser with the popular AddressSanitizer tool and enhancing it with remote crash reporting capabilities for any errors detected. + +:ref:`More information <ASan Nightly>` diff --git a/tools/sanitizer/docs/memory_sanitizer.rst b/tools/sanitizer/docs/memory_sanitizer.rst new file mode 100644 index 0000000000..890d0c712b --- /dev/null +++ b/tools/sanitizer/docs/memory_sanitizer.rst @@ -0,0 +1,173 @@ +Memory Sanitizer +================ + ++--------------------------------------------------------------------+ +| This page is an import from MDN and the contents might be outdated | ++--------------------------------------------------------------------+ + +What is Memory Sanitizer? +------------------------- + +Memory Sanitizer (MSan) is a fast detector used for uninitialized memory +in C/C++ programs. It uses a compile-time instrumentation to ensure that +all memory access at runtime uses only memory that has been initialized. +Unlike most other sanitizers, MSan can easily cause false positives if +not all libraries are instrumented. This happens because MSan is +not able to observe memory initialization in uninstrumented libraries. +More information on MSan can be found on `the Memory Sanitizer +wiki <https://github.com/google/sanitizers/wiki/MemorySanitizer>`__. + +Public Builds +------------- + +**Note:** No public builds are available at this time yet. + +Manual Build +------------ + +Build prerequisites +~~~~~~~~~~~~~~~~~~~ + +**Note:** MemorySanitizer requires **64-bit Linux** to work. Other +platforms/operating systems are not supported. + +LLVM/Clang +^^^^^^^^^^ + +The MSan instrumentation is implemented as an LLVM pass and integrated +into Clang. As MSan is one of the newer sanitizers, we recommend using a +recent Clang version, such as Clang 3.7+. + +You can find precompiled binaries for LLVM/Clang on `the LLVM releases +page <https://releases.llvm.org/download.html>`__. + +Building Firefox +~~~~~~~~~~~~~~~~ + +.. warning:: + + **Warning: Running Firefox with MemorySanitizer would require all + external dependencies to be built with MemorySanitizer as well. To + our knowledge, this has never been attempted yet, so the build + configuration provided here is untested and without an appropriately + instrumented userland, it will cause false positives.** + +Getting the source +^^^^^^^^^^^^^^^^^^ + +If you don't have a source code repository clone yet, you need to :ref:`get +yourself a clone of Mozilla-central <Mercurial Overview>`. + +Adjusting the build configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the build configuration file ``.mozconfig`` with the following +content in your Mozilla-central directory: + +.. code:: bash + + mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/objdir-ff-msan + + # Enable LLVM specific code and build workarounds + ac_add_options --enable-memory-sanitizer + # If clang is already in your $PATH, then these can simply be: + # export CC=clang + # export CXX=clang++ + export CC="/path/to/clang" + export CXX="/path/to/clang++" + + # llvm-symbolizer displays much more complete backtraces when data races are detected. + # If it's not already in your $PATH, then uncomment this next line: + #export LLVM_SYMBOLIZER="/path/to/llvm-symbolizer" + + # Add MSan to our compiler flags + export CFLAGS="-fsanitize=memory" + export CXXFLAGS="-fsanitize=memory" + + # Additionally, we need the MSan flag during linking. Normally, our C/CXXFLAGS would + # be used during linking as well but there is at least one place in our build where + # our CFLAGS are not added during linking. + # Note: The use of this flag causes Clang to automatically link the MSan runtime :) + export LDFLAGS="-fsanitize=memory" + + # These three are required by MSan + ac_add_options --disable-jemalloc + ac_add_options --disable-crashreporter + ac_add_options --disable-elf-hack + + # Keep symbols to symbolize MSan traces + export MOZ_DEBUG_SYMBOLS=1 + ac_add_options --enable-debug-symbols + ac_add_options --disable-install-strip + + # Settings for an opt build + ac_add_options --enable-optimize="-O2 -gline-tables-only" + ac_add_options --disable-debug + +Starting the build process +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now you start the build process using the regular ``make -f client.mk`` +command. + +Starting Firefox +^^^^^^^^^^^^^^^^ + +After the build has completed, you can start Firefox from the ``objdir`` +as usual. + +Building the JavaScript shell +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Note:** Unlike Firefox itself, the JavaScript shell does **not** +require an instrumented userland. Calls to external libraries like +zlib are handled with special annotations inside the engine. + +.. warning:: + + **Warning: Certain technologies used inside the JavaScript engine are + incompatible with MSan and must be disabled at runtime to prevent + false positives. This includes the JITs and asm.js. Therefore always + make sure to run with + ``--no-ion --no-baseline --no-asmjs --no-native-regexp``.** + +If you want to build only the JavaScript shell instead of doing a full +Firefox build, the build script below will probably help you to do so. +Before using it, you must, of course, adjust the path name for +``LLVM_ROOT`` to match your setup. Once you have adjusted everything, +execute this script in the ``js/src/`` subdirectory and pass a directory +name as the first parameter. The build will then be created in a new +subdirectory with that name. + +.. code:: bash + + #! /bin/sh + + if [ -z $1 ] ; then + echo "usage: $0 <dirname>" + elif [ -d $1 ] ; then + echo "directory $1 already exists" + else + autoconf2.13 + mkdir $1 + cd $1 + LLVM_ROOT="/path/to/llvm" + CC="$LLVM_ROOT/build/bin/clang" \ + CXX="$LLVM_ROOT/build/bin/clang++" \ + CFLAGS="-fsanitize=memory" \ + CXXFLAGS="-fsanitize=memory" \ + LDFLAGS="-fsanitize=memory" \ + ../configure --enable-debug --enable-optimize --enable-memory-sanitizer --disable-jemalloc --enable-posix-nspr-emulation + make -j 8 + fi + +Using LLVM Symbolizer for faster/better traces +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, MSan traces are not symbolized. + +LLVM ships with the symbolizer binary ``llvm-symbolize`` that MSan will +readily use to immediately output symbolized traces if the program is +found on the ``PATH``. If your ``llvm-symbolizer`` lives outside the +``PATH``, you can set the ``MSAN_SYMBOLIZER_PATH`` environment variable +to point to your symbolizer binary. diff --git a/tools/sanitizer/docs/tsan.rst b/tools/sanitizer/docs/tsan.rst new file mode 100644 index 0000000000..77fb6c89d7 --- /dev/null +++ b/tools/sanitizer/docs/tsan.rst @@ -0,0 +1,327 @@ +Thread Sanitizer +================ + +What is Thread Sanitizer? +-------------------------- + +Thread Sanitizer (TSan) is a fast data race detector for C/C++ and Rust +programs. It uses a compile-time instrumentation to check all non-race-free +memory access at runtime. Unlike other tools, it understands compiler-builtin +atomics and synchronization and therefore provides very accurate results +with no false positives (except if unsupported synchronization primitives +like inline assembly or memory fences are used). More information on how +TSan works can be found on `the Thread Sanitizer wiki <https://github.com/google/sanitizers/wiki/ThreadSanitizerAlgorithm>`__. + +A `meta bug called tsan <https://bugzilla.mozilla.org/show_bug.cgi?id=tsan>`__ +is maintained to keep track of all the bugs found with TSan. + +A `blog post on hacks.mozilla.org <https://hacks.mozilla.org/2021/04/eliminating-data-races-in-firefox-a-technical-report/>`__ describes this project. + +Note that unlike other sanitizers, TSan is currently **only supported on Linux**. + +Downloading artifact builds +--------------------------- + +The easiest way to get Firefox builds with Thread Sanitizer is to download a +continuous integration TSan build of mozilla-central (updated at least daily): + +- mozilla-central optimized builds: + `linux <https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.linux64-tsan-opt/artifacts/public/build/target.tar.bz2>`__ + +The fuzzing team also offers a tool called ``fuzzfetch`` to download this and many +other CI builds. It makes downloading and unpacking these builds much easier and +can be used not just for fuzzing but for all purposes that require a CI build download. + +You can install ``fuzzfetch`` from +`Github <https://github.com/MozillaSecurity/fuzzfetch>`__ or +`via pip <https://pypi.org/project/fuzzfetch/>`__. + +Afterwards, you can run + +:: + + $ python -m fuzzfetch --tsan -n firefox-tsan + +to get the build mentioned above unpacked into a directory called ``firefox-tsan``. + +Creating Try builds +------------------- + +If for some reason you can't use the pre-built binaries mentioned in the +previous section (e.g. you need to test a patch), you can either build +Firefox yourself (see the following section) or use the :ref:`try server <Pushing to Try>` +to create the customized build for you. Pushing to try requires L1 commit +access. If you don't have this access yet you can request access (see +`Becoming A Mozilla +Committer <https://www.mozilla.org/about/governance/policies/commit/>`__ +and `Mozilla Commit Access +Policy <https://www.mozilla.org/about/governance/policies/commit/access-policy/>`__ +for the requirements). + +Using ``mach try fuzzy --full`` you can select the ``build-linux64-tsan/opt`` job +and related tests (if required). + +Creating local builds on Linux +------------------------------ + +Build prerequisites +~~~~~~~~~~~~~~~~~~~ + +LLVM/Clang/Rust +^^^^^^^^^^^^^^^ + +The TSan instrumentation is implemented as an LLVM pass and integrated +into Clang. We strongly recommend that you use the Clang version supplied +as part of the ``mach bootstrap`` process, as we backported several required +fixes for TSan on Firefox. + +Sanitizer support in Rust is genuinely experimental, +so our build system only works with a specially patched version of Rust +that we build in our CI. To install that specific version (or update to a newer +version), run the following in the root of your mozilla-central checkout: + +:: + + ./mach artifact toolchain --from-build linux64-rust-dev + rm -rf ~/.mozbuild/rustc-sanitizers + mv rustc ~/.mozbuild/rustc-sanitizers + rustup toolchain link gecko-sanitizers ~/.mozbuild/rustc-sanitizers + rustup override set gecko-sanitizers + +``mach artifact`` will always download the ``linux64-rust-dev`` toolchain associated +with the current mozilla central commit you have checked out. The toolchain should +mostly behave like a normal rust nightly but we don't recommend using it for anything +other than building gecko, just in case. Also note that +``~/.mozbuild/rustc-sanitizers`` is just a reasonable default location -- feel +free to "install" the toolchain wherever you please. + +Building Firefox +~~~~~~~~~~~~~~~~ + +Getting the source +^^^^^^^^^^^^^^^^^^ + +Using that or any later revision, all you need to do is to :ref:`get yourself +a clone of mozilla-central <Mercurial overview>`. + +Adjusting the build configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the build configuration file ``mozconfig`` with the following +content in your mozilla-central directory: + +:: + + # Combined .mozconfig file for TSan on Linux+Mac + + mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/objdir-ff-tsan + + # Enable ASan specific code and build workarounds + ac_add_options --enable-thread-sanitizer + + # This ensures that we also instrument Rust code. + export RUSTFLAGS="-Zsanitizer=thread" + + # rustfmt is currently missing in Rust nightly + unset RUSTFMT + + # Current Rust Nightly has warnings + ac_add_options --disable-warnings-as-errors + + # These are required by TSan + ac_add_options --disable-jemalloc + ac_add_options --disable-crashreporter + ac_add_options --disable-elf-hack + ac_add_options --disable-profiling + + # The Thread Sanitizer is not compatible with sandboxing + # (see bug 1182565) + ac_add_options --disable-sandbox + + # Keep symbols to symbolize TSan traces later + export MOZ_DEBUG_SYMBOLS=1 + ac_add_options --enable-debug-symbols + ac_add_options --disable-install-strip + + # Settings for an opt build (preferred) + # The -gline-tables-only ensures that all the necessary debug information for ASan + # is present, but the rest is stripped so the resulting binaries are smaller. + ac_add_options --enable-optimize="-O2 -gline-tables-only" + ac_add_options --disable-debug + + # Settings for a debug+opt build + #ac_add_options --enable-optimize + #ac_add_options --enable-debug + + +Starting the build process +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now you start the build process using the regular ``./mach build`` +command. + +Starting Firefox +^^^^^^^^^^^^^^^^ + +After the build has completed, ``./mach run`` with the usual options for +running in a debugger (``gdb``, ``lldb``, ``rr``, etc.) work fine, as do +the ``--disable-e10s`` and other options. + +Building only the JavaScript shell +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to build only the JavaScript shell instead of doing a full +Firefox build, the build script below will probably help you to do so. +Execute this script in the ``js/src/`` subdirectory and pass a directory +name as the first parameter. The build will then be created in a new +subdirectory with that name. + +:: + + #! /bin/sh + + if [ -z $1 ] ; then + echo "usage: $0 <dirname>" + elif [ -d $1 ] ; then + echo "directory $1 already exists" + else + autoconf2.13 + mkdir $1 + cd $1 + CC="/path/to/mozbuild/clang" \ + CXX="/path/to/mozbuild/clang++" \ + ../configure --disable-debug --enable-optimize="-O2 -gline-tables-only" --enable-thread-sanitizer --disable-jemalloc + fi + +Thread Sanitizer and Symbols +---------------------------- + +Unlike Address Sanitizer, TSan requires in-process symbolizing to work +properly in the first place, as any kind of runtime suppressions will +otherwise not work. + +Hence, it is required that you have a copy of ``llvm-symbolizer`` either +in your ``PATH`` or pointed to by the ``TSAN_SYMBOLIZER_PATH`` environment +variable. This binary is included in your local mozbuild directory, obtained +by ``./mach bootstrap``. + + +Runtime Suppressions +-------------------- + +TSan has the ability to suppress race reports at runtime. This can be used to +silence a race while a fix is developed as well as to permanently silence a +(benign) race that cannot be fixed. + +.. warning:: + **Warning**: Many races *look* benign but are indeed not. Please read + the :ref:`FAQ section <Frequently Asked Questions about TSan>` carefully + and think twice before attempting to suppress a race. + +The runtime Suppression list is directly baked into Firefox at compile-time and +located at `mozglue/build/TsanOptions.cpp <https://searchfox.org/mozilla-central/source/mozglue/build/TsanOptions.cpp>`__. + +.. warning:: + **Important**: When adding a suppression, always make sure to include + the bug number. If the suppression is supposed to be permanent, please + add the string ``permanent`` in the same line as the bug number. + +.. warning:: + **Important**: When adding a suppression for a *data race*, always make + sure to include a stack frame from **each** of the two race stacks. + Adding only one suppression for one stack can cause intermittent failures + that are later on hard to track. One exception to this rule is when suppressing + races on global variables. In that case, a single race entry with the name of + the variable is sufficient. + +Troubleshooting / Known Problems +-------------------------------- + +Known Sources of False Positives +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +TSan has a number of things that can cause false positives, namely: + + * The use of memory fences (e.g. Rust Arc) + * The use of inline assembly for synchronization + * Uninstrumented code (e.g. external libraries) using compiler-builtins for synchronization + * A lock order inversion involving only a single thread can cause a false positive deadlock + report (see also https://github.com/google/sanitizers/issues/488). + +If none of these four items are involved, you should *never* assume that TSan is reporting +a false positive to you without consulting TSan peers. It is very easy to misjudge a race +to be a false positive because races can be highly complex and totally non-obvious due to +compiler optimizations and the nature of parallel code. + +Intermittent Broken Stacks +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you intermittently see race reports where one stack is missing with a ``failed to restore the stack`` +message, this can indicate that a suppression is partially covering the race you are seeing. + +Any race where only one of the two stacks is matched by a runtime suppression will show up +if that particular stack fails to symbolize for some reason. The usual solution is to search +the suppressions for potential candidates and disable them temporarily to check if your race +report now becomes mostly consistent. + +However, there are other reasons for broken TSan stacks, in particular if they are not intermittent. +See also the ``history_size`` parameter in the `TSan flags <https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags>`__. + +Intermittent Race Reports +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unfortunately, the TSan algorithm does not guarantee, that a race is detected 100% of the +time. Intermittent failures with TSan are (to a certain degree) to be expected and the races +involved should be filed and fixed to solve the problem. + +.. _Frequently Asked Questions about TSan: + +Frequently Asked Questions about TSan +------------------------------------- + +Why fix data races? +~~~~~~~~~~~~~~~~~~~ + +Data races are undefined behavior and can cause crashes as well as correctness issues. +Compiler optimizations can cause racy code to have unpredictable and hard-to-reproduce behavior. + +At Mozilla, we have already seen several dangerous races, causing random +`use-after-free crashes <https://bugzilla.mozilla.org/show_bug.cgi?id=1580288>`__, +`intermittent test failures <https://bugzilla.mozilla.org/show_bug.cgi?id=1602009>`__, +`hangs <https://bugzilla.mozilla.org/show_bug.cgi?id=1607008>`__, +`performance issues <https://bugzilla.mozilla.org/show_bug.cgi?id=1615045>`__ and +`intermittent asserts <https://bugzilla.mozilla.org/show_bug.cgi?id=1601940>`__. Such problems do +not only decrease the quality of our code and user experience, but they also waste countless hours +of developer time. + +Since it is very hard to judge if a particular race could cause such a situation, we +have decided to fix all data races wherever possible, since doing so is often cheaper +than analyzing a race. + +My race is benign, can we ignore it? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While it is possible to add a runtime suppression to ignore the race, we *strongly* encourage +you to not do so, for two reasons: + + 1. Each suppressed race decreases the overall performance of the TSan build, as the race + has to be symbolized each time when it occurs. Since TSan is already in itself a slow + build, we need to keep the amount of suppressed races as low as possible. + + 2. Deciding if a race is truly benign is surprisingly hard. We recommend to read + `this blog post <http://software.intel.com/en-us/blogs/2013/01/06/benign-data-races-what-could-possibly-go-wrong>`__ + and `this paper <https://www.usenix.org/legacy/events/hotpar11/tech/final_files/Boehm.pdf>` + on the effects of seemingly benign races. + +Valid reasons to suppress a confirmed benign race include performance problems arising from +fixing the race or cases where fixing the race would require an unreasonable amount of work. + +Note that the use of atomics usually does not have the bad performance impact that developers +tend to associate with it. If you assume that e.g. using atomics for synchronization will +cause performance regressions, we suggest to perform a benchmark to confirm this. In many +cases, the difference is not measurable. + +How does TSan work exactly? +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +More information on how TSan works can be found on `the Thread Sanitizer wiki <https://github.com/google/sanitizers/wiki/ThreadSanitizerAlgorithm>`__. diff --git a/tools/tryselect/__init__.py b/tools/tryselect/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/cli.py b/tools/tryselect/cli.py new file mode 100644 index 0000000000..11d1867e47 --- /dev/null +++ b/tools/tryselect/cli.py @@ -0,0 +1,175 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import subprocess +import tempfile +from argparse import ArgumentParser + +from .task_config import all_task_configs + +COMMON_ARGUMENT_GROUPS = { + "push": [ + [ + ["-m", "--message"], + { + "const": "editor", + "default": "{msg}", + "nargs": "?", + "help": "Use the specified commit message, or create it in your " + "$EDITOR if blank. Defaults to computed message.", + }, + ], + [ + ["--closed-tree"], + { + "action": "store_true", + "default": False, + "help": "Push despite a closed try tree", + }, + ], + [ + ["--push-to-lando"], + { + "action": "store_true", + "default": False, + "help": "Submit changes for Lando to push to try.", + }, + ], + ], + "preset": [ + [ + ["--save"], + { + "default": None, + "help": "Save selection for future use with --preset.", + }, + ], + [ + ["--preset"], + { + "default": None, + "help": "Load a saved selection.", + }, + ], + [ + ["--list-presets"], + { + "action": "store_const", + "dest": "preset_action", + "const": "list", + "default": None, + "help": "List available preset selections.", + }, + ], + [ + ["--edit-presets"], + { + "action": "store_const", + "dest": "preset_action", + "const": "edit", + "default": None, + "help": "Edit the preset file.", + }, + ], + ], + "task": [ + [ + ["--full"], + { + "action": "store_true", + "default": False, + "help": "Use the full set of tasks as input to fzf (instead of " + "target tasks).", + }, + ], + [ + ["-p", "--parameters"], + { + "default": None, + "help": "Use the given parameters.yml to generate tasks, " + "defaults to a default set of parameters", + }, + ], + ], +} + +NO_PUSH_ARGUMENT_GROUP = [ + [ + ["--stage-changes"], + { + "dest": "stage_changes", + "action": "store_true", + "help": "Locally stage changes created by this command but do not " + "push to try.", + }, + ], + [ + ["--no-push"], + { + "dest": "dry_run", + "action": "store_true", + "help": "Do not push to try as a result of running this command (if " + "specified this command will only print calculated try " + "syntax and selection info and not change files).", + }, + ], +] + + +class BaseTryParser(ArgumentParser): + name = "try" + common_groups = ["push", "preset"] + arguments = [] + task_configs = [] + + def __init__(self, *args, **kwargs): + ArgumentParser.__init__(self, *args, **kwargs) + + group = self.add_argument_group("{} arguments".format(self.name)) + for cli, kwargs in self.arguments: + group.add_argument(*cli, **kwargs) + + for name in self.common_groups: + group = self.add_argument_group("{} arguments".format(name)) + arguments = COMMON_ARGUMENT_GROUPS[name] + + # Preset arguments are all mutually exclusive. + if name == "preset": + group = group.add_mutually_exclusive_group() + + for cli, kwargs in arguments: + group.add_argument(*cli, **kwargs) + + if name == "push": + group_no_push = group.add_mutually_exclusive_group() + arguments = NO_PUSH_ARGUMENT_GROUP + for cli, kwargs in arguments: + group_no_push.add_argument(*cli, **kwargs) + + group = self.add_argument_group("task configuration arguments") + self.task_configs = {c: all_task_configs[c]() for c in self.task_configs} + for cfg in self.task_configs.values(): + cfg.add_arguments(group) + + def validate(self, args): + if hasattr(args, "message"): + if args.message == "editor": + if "EDITOR" not in os.environ: + self.error( + "must set the $EDITOR environment variable to use blank --message" + ) + + with tempfile.NamedTemporaryFile(mode="r") as fh: + subprocess.call([os.environ["EDITOR"], fh.name]) + args.message = fh.read().strip() + + if "{msg}" not in args.message: + args.message = "{}\n\n{}".format(args.message, "{msg}") + + def parse_known_args(self, *args, **kwargs): + args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs) + self.validate(args) + return args, remainder diff --git a/tools/tryselect/docs/configuration.rst b/tools/tryselect/docs/configuration.rst new file mode 100644 index 0000000000..6743d5f385 --- /dev/null +++ b/tools/tryselect/docs/configuration.rst @@ -0,0 +1,30 @@ +Configuring Try +=============== + + +Getting Level 1 Commit Access +----------------------------- + +In order to push to try, `Level 1 Commit Access`_ is required. Please see `Becoming a Mozilla +Committer`_ for more information on how to obtain this. + + +Configuring Version Control +--------------------------- + +After you have level 1 access, you'll need to do a little bit of setup before you can push. Both +``hg`` and ``git`` are supported, move on to the appropriate section below. + + +Configuring Try with Mercurial / Git +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The recommended way to push to try is via the ``mach try`` command. This requires the +``push-to-try`` extension which can be installed by running: + +.. code-block:: shell + + $ mach vcs-setup + +.. _Level 1 Commit Access: https://www.mozilla.org/en-US/about/governance/policies/commit/access-policy/ +.. _Becoming a Mozilla Committer: https://www.mozilla.org/en-US/about/governance/policies/commit/ diff --git a/tools/tryselect/docs/img/add-new-jobs.png b/tools/tryselect/docs/img/add-new-jobs.png Binary files differnew file mode 100644 index 0000000000..513565cb14 --- /dev/null +++ b/tools/tryselect/docs/img/add-new-jobs.png diff --git a/tools/tryselect/docs/img/phab-treeherder-link.png b/tools/tryselect/docs/img/phab-treeherder-link.png Binary files differnew file mode 100644 index 0000000000..52b58b6231 --- /dev/null +++ b/tools/tryselect/docs/img/phab-treeherder-link.png diff --git a/tools/tryselect/docs/index.rst b/tools/tryselect/docs/index.rst new file mode 100644 index 0000000000..8978a72eaf --- /dev/null +++ b/tools/tryselect/docs/index.rst @@ -0,0 +1,92 @@ +Pushing to Try +============== + +"Pushing to Try" allows developers to build and test their changes on Mozilla's automation servers +without requiring their code to be reviewed and landed. + +First, :doc:`ensure that you can push to Try <configuration>`. +Try knows how to run tasks that are defined in-tree, +such as ``build-linux64/opt`` (build Firefox for Linux). To manually select some tasks for +Try to process, run the following command: + +.. code-block:: shell + + ./mach try fuzzy + +After submitting your requested tasks, you'll be given a link to your "push" in Treeherder. +It may take a few minutes for your push to appear in Treeherder! Be patient, and it will automatically +update when Try begins processing your work. + +Another very useful Try command is ``./mach try auto``, which will automatically select the tasks +that are mostly likely to be affected by your changes. +See the :doc:`selectors page <selectors/index>` to view all the other ways to select which tasks to push. + +It is possible to set environment variables, notably :doc:`MOZ_LOG </xpcom/logging>`, when pushing to +try: + +.. code-block:: shell + + ./mach try fuzzy --env MOZ_LOG=cubeb:4 + +Resolving "<Try build> is damaged and can't be opened" error +------------------------------------------------------------ + +To run a try build on macOS, you need to get around Apple's restrictions on downloaded applications. + +These restrictions differ based on your hardware: Apple Silicon machines (M1 etc.) are much stricter. + +For Apple Silicon machines, you will need to download the target.dmg artifact from the +"repackage-macosx64-shippable/opt" job. +This is a universal build (i.e. it contains both x86_64 and arm64 code), and it is signed but not notarized. +You can trigger this job using ``./mach try fuzzy --full``. + +On Intel Macs, you can run unsigned builds, once you get around the quarantining (see below), +so you can just download the "target.dmg" from a regular opt build. + +Regardless of hardware, you need to make sure that there is no quarantining attribute on +the downloaded dmg file before you attempt to run it: +Apple automatically quarantines apps that are downloaded with a browser from an untrusted +location. This "quarantine status" can be cleared by doing ``xattr -c <Try build>`` after +downloading. You can avoid this "quarantine status" by downloading the build from the command +line instead, such as by using ``curl``: + +.. code-block:: shell + + curl -L <artifact-url> -o <file-name> + +.. _attach-job-review: + +Adding Try jobs to a Phabricator patch +-------------------------------------- + +For every patch submitted for review in Phabricator, a new Try run is automatically created. +A link called ``Treeherder Jobs`` can be found in the ``Diff Detail`` section of the review in +Phabricator. + +.. image:: img/phab-treeherder-link.png + +This run is created for static analysis, linting and other tasks. Attaching new jobs to the run is +easy and doesn't require more actions from the developer. +Click on the down-arrow to access the actions menu, select the relevant jobs +and, click on ``Trigger X new jobs`` (located on the top of the job). + +.. image:: img/add-new-jobs.png + +Table of Contents +----------------- + +.. toctree:: + :maxdepth: 2 + + configuration + selectors/index + presets + tasks + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tools/tryselect/docs/presets.rst b/tools/tryselect/docs/presets.rst new file mode 100644 index 0000000000..a3368cf8b5 --- /dev/null +++ b/tools/tryselect/docs/presets.rst @@ -0,0 +1,85 @@ +Presets +======= + +Some selectors, such as ``fuzzy`` and ``syntax``, allow saving and loading presets from a file. This is a +good way to reuse a selection, either at a later date or by sharing with others. Look for a +'preset' section in ``mach <selector> --help`` to determine whether the selector supports this +functionality. + +Using Presets +------------- + +To save a preset, run: + +.. code-block:: shell + + $ mach try <selector> --save <name> <args> + +For example, to save a preset that selects all Windows mochitests: + +.. code-block:: shell + + $ mach try fuzzy --save all-windows-mochitests --query "'win 'mochitest" + preset saved, run with: --preset=all-windows-mochitests + +Then run that saved preset like this: + +.. code-block:: shell + + $ mach try --preset all-windows-mochitests + +To see a list of all available presets run: + +.. code-block:: shell + + $ mach try --list-presets + + +Editing and Sharing Presets +--------------------------- + +Presets can be defined in one of two places, in your home directory or in a file checked into +mozilla-central. + +Local Presets +~~~~~~~~~~~~~ + +These are defined in your ``$MOZBUILD_STATE_DIR``, typically ``~/.mozbuild/try_presets.yml``. +Presets defined here are your own personal collection of presets. You can modify them by running: + +.. code-block:: shell + + $ ./mach try --edit-presets + + +Shared Presets +~~~~~~~~~~~~~~ + +You can also check presets into mozilla-central in `tools/tryselect/try_presets.yml`_. These presets +will be available to all users of ``mach try``, so please be mindful when editing this file. Make +sure the name of the preset is scoped appropriately (i.e doesn't contain any team or module specific +terminology). It is good practice to prefix the preset name with the name of the team or module that +will get the most use out of it. + + +Preset Format +~~~~~~~~~~~~~ + +Presets are simple key/value objects, with the name as the key and a metadata object as the value. +For example, the preset saved above would look something like this in ``try_presets.yml``: + +.. code-block:: yaml + + all-windows-mochitests: + selector: fuzzy + description: >- + Runs all windows mochitests. + query: + - "'win 'mochitest" + +The ``selector`` key (required) allows ``mach try`` to determine which subcommand to dispatch to. +The ``description`` key (optional in user presets but required for shared presets) is a human +readable string describing what the preset selects and when to use it. All other values in the +preset are forwarded to the specified selector as is. + +.. _tools/tryselect/try_presets.yml: https://searchfox.org/mozilla-central/source/tools/tryselect/try_presets.yml diff --git a/tools/tryselect/docs/selectors/again.rst b/tools/tryselect/docs/selectors/again.rst new file mode 100644 index 0000000000..da592b7a34 --- /dev/null +++ b/tools/tryselect/docs/selectors/again.rst @@ -0,0 +1,36 @@ +Again Selector +============== + +When you push to try, the computed ``try_task_config.json`` is saved in a +history file under ``~/.mozbuild/srcdirs/<srcdir hash>/history`` (note: the +``syntax`` selector does not use ``try_task_config.json`` yet so does not save +any history). You can then use the ``again`` selector to re-push any of your +previously generated task configs. + +In the simple case, you can re-run your last try push with: + +.. code-block:: shell + + $ mach try again + +If you want to re-push a task config a little further down the history stack, +first you need to figure out its index with: + +.. code-block:: shell + + $ mach try again --list + +Then run: + +.. code-block:: shell + + $ mach try again --index <index> + +Note that index ``0`` is always the most recent ``try_task_config.json`` in the +history. You can clear your history with: + +.. code-block:: shell + + $ mach try again --purge + +Only the 10 most recent pushes will be saved in your history. diff --git a/tools/tryselect/docs/selectors/auto.rst b/tools/tryselect/docs/selectors/auto.rst new file mode 100644 index 0000000000..c5f6d4df5e --- /dev/null +++ b/tools/tryselect/docs/selectors/auto.rst @@ -0,0 +1,24 @@ +Auto Selector +============= + +This selector automatically determines the most efficient set of tests and +tasks to run against your push. It accomplishes this via a combination of +machine learning and manual heuristics. The tasks selected here should match +pretty closely to what would be scheduled if your patch were pushed to +autoland. + +It is the officially recommended selector to use when you are unsure of which +tasks should run on your push. + +To use: + +.. code-block:: bash + + $ mach try auto + +Unlike other try selectors, tasks are not chosen locally. Rather they will be +computed by the decision task. + +Like most other selectors, ``mach try auto`` supports many of the standard +templates such as ``--artifact`` or ``--env``. See ``mach try auto --help`` for +the full list of supported templates. diff --git a/tools/tryselect/docs/selectors/chooser.rst b/tools/tryselect/docs/selectors/chooser.rst new file mode 100644 index 0000000000..6c59d54009 --- /dev/null +++ b/tools/tryselect/docs/selectors/chooser.rst @@ -0,0 +1,32 @@ +Chooser Selector +================ + +When pushing to try, there are a very large amount of builds and tests to choose from. Often too +many to remember, making it easy to forget a set of tasks which should otherwise have been run. + +This selector allows you to select tasks from a web interface that lists all the possible build and +test tasks and allows you to select them from a list. It is similar in concept to the old `try +syntax chooser`_ page, except that the values are dynamically generated using the :ref:`taskgraph<TaskCluster Task-Graph Generation>` as an +input. This ensures that it will never be out of date. + +To use: + +.. code-block:: shell + + $ mach try chooser + +This will spin up a local web server (using Flask) which serves the chooser app. After making your +selection, simply press ``Push`` and the rest will be handled from there. No need to copy/paste any +syntax strings or the like. + +You can run: + +.. code-block:: shell + + $ mach try chooser --full + +To generate the interface using the full :ref:`taskgraph<TaskCluster Task-Graph Generation>` instead. This will include tasks that don't run +on mozilla-central. + + +.. _try syntax chooser: https://mozilla-releng.net/trychooser diff --git a/tools/tryselect/docs/selectors/compare.rst b/tools/tryselect/docs/selectors/compare.rst new file mode 100644 index 0000000000..a87b263030 --- /dev/null +++ b/tools/tryselect/docs/selectors/compare.rst @@ -0,0 +1,17 @@ +Compare Selector +================ + +When this command runs it pushes two identical try jobs to treeherder. The first +job is on the current commit you are on, and the second one is a commit +specified in the command line arguments. This selector is aimed at helping +engineers test performance enhancements or resolve performance regressions. + +Currently the only way you can select jobs is through fuzzy but we are +planning on expanding to other choosing frameworks also. + +You pass the commit you want to compare against as a commit hash as either +``-cc`` or ``--compare-commit``, an example is show below + +.. code-block:: shell + + $ mach try compare --compare-commit <commit-hash> diff --git a/tools/tryselect/docs/selectors/empty.rst b/tools/tryselect/docs/selectors/empty.rst new file mode 100644 index 0000000000..c3ea61b9ce --- /dev/null +++ b/tools/tryselect/docs/selectors/empty.rst @@ -0,0 +1,21 @@ +Empty Selector +============== + +The ``mach try empty`` subcommand is very simple, it won't schedule any additional tasks. You'll +still see lint tasks and python-unittest tasks if applicable, this is due to a configuration option +in taskcluster. + +Other than those, your try run will be empty. You can use treeherder's ``Add new jobs`` feature to +selectively add additional tasks after the fact. + +.. note:: + + To use ``Add new jobs`` you'll need to be logged in and have commit access level 1, just as if + you were pushing to try. + +To do this: + + 1. Click the drop-down arrow at the top right of your commit. + 2. Select ``Add new jobs`` (it may take a couple seconds to load). + 3. Choose your desired tasks by clicking them one at a time. + 4. At the top of your commit, select ``Trigger New Jobs``. diff --git a/tools/tryselect/docs/selectors/fuzzy.rst b/tools/tryselect/docs/selectors/fuzzy.rst new file mode 100644 index 0000000000..d50f801eb0 --- /dev/null +++ b/tools/tryselect/docs/selectors/fuzzy.rst @@ -0,0 +1,371 @@ +Fuzzy Selector +============== + +The fuzzy selector uses a tool called `fzf`_. It allows you to filter down all of the task labels +from a terminal based UI and an intelligent fuzzy finding algorithm. If the ``fzf`` binary is not +installed, you'll be prompted to bootstrap it on first run. + + +Understanding the Interface +--------------------------- + +When you run ``mach try fuzzy`` an interface similar to the one below will open. This is `fzf`_. + +.. image:: fzf.png + +There's a lot to unpack here, so let's examine each component a bit more closely. + + A. The set of tasks that match the currently typed-out query. In the above image only tasks that + match the query ``'linux64 mochibrochr`` are displayed. + + B. The set of selected tasks. These are the tasks that will be scheduled once you hit ``Enter``. + In other words, if the task you want does not appear here, *it won't be scheduled*. + + C. Count information of the form ``x/y (z)``, where ``x`` is the number of tasks that match the + current query, ``y`` is the total number of tasks and ``z`` is the number of tasks you have + selected. + + D. The input bar for entering queries. As you type you'll notice the list of tasks in ``A`` + starts to update immediately. In the image above, the query ``'linux64 mochibrochr`` is entered. + Correspondingly only tasks matching that query are displayed. + +In general terms, you first find tasks on the left. Then you move them over to the right by +selecting them. Once you are satisfied with your selection, press ``Enter`` to push to try. + + +Selecting Tasks +--------------- + +There are few ways you can select tasks. If you are feeling a bit overwhelmed, it might be best to +stick with the mouse to start: + + 1. Enter a query (e.g ``mochitest``) to reduce the task list a little. + 2. Scroll up and look for the task(s) you want. + 3. ``Right-Click`` as many tasks as desired to select them. + 4. Optionally delete your query, go back to step 1) and repeat. + 5. Press ``Enter`` to push (or ``Esc`` to cancel). + +.. note:: + + Dependencies are automatically filled in, so you can select a test task without needing + to select the build it depends on. + +As you ``Right-Click``, notice that a little arrow appears to the left of the task label. This +indicates that the task is selected and exists in the preview pane to the right. + +Once you are a bit more comfortable with the interface, using the keyboard is much better at quickly +selecting tasks. Here are the main shortcuts you'll need: + +.. code-block:: text + + Ctrl-K / Up => Move cursor up + Ctrl-J / Down => Move cursor down + Tab => Select task + move cursor down + Shift-Tab => Select task + move cursor up + Ctrl-A => Select all currently filtered tasks + Ctrl-T => Toggle select all currently filtered tasks + Ctrl-D => De-select all selected tasks (both filtered and not) + Alt-Bspace => Clear query from input bar + Enter => Accept selection and exit + Ctrl-C / Esc => Cancel selection and exit + ? => Toggle preview pane + + +The process for selecting tasks is otherwise the same as for a mouse. A particularly fast and +powerful way to select tasks is to: + +.. code-block:: text + + Write a precise query => Ctrl-A => Alt-Bspace => Repeat + +As before, when you are satisfied with your selection press ``Enter`` and all the tasks in the +preview pane will be pushed to try. If you change your mind you can press ``Esc`` or ``Ctrl-C`` to +exit the interface without pushing anything. + +.. note:: + + Initially ``fzf`` will automatically select whichever task is under your cursor. This is a + convenience feature for the case where you are only selecting a single task. This feature will be + turned off as soon as you *lock in* a selection with ``Right-Click``, ``Tab`` or ``Ctrl-A``. + + +Writing Queries +--------------- + +Queries are built from a series of terms, each separated by a space. Terms are logically joined by +the AND operator. For example: + +.. code-block:: text + + 'windows 'mochitest + +This query has two terms, and is the equivalent of saying: Give me all the tasks that match both the +term ``'windows'`` and the term ``'mochitest'``. In other words, this query matches all Windows +mochitest tasks. + +The single quote prefix before each term tells ``fzf`` to use exact substring matches, so only tasks +that contain both the literal string ``windows`` AND the literal string ``mochitest`` will be +matched. + +Another thing to note is that the order of the terms makes no difference, so ``'windows 'mochitest`` +and ``'mochitest 'windows`` are equivalent. + + +Fuzzy terms +~~~~~~~~~~~ + +If a term is *not* prefixed with a single quote, that makes it a fuzzy term. This means the +characters in the term need to show up in order, but not in sequence. E.g the fuzzy term ``max`` +would match the string ``mozilla firefox`` (as first there is an ``m``, then an ``a`` and finally an +``x``), but not the string ``firefox by mozilla`` (since the ``x`` is now out of order). Here's a +less contrived example: + +.. code-block:: text + + wndws mchtst + +Like the query above, this one would also select all Windows mochitest tasks. But it will +additionally select: + +.. code-block:: text + + test-macosx1014-64-shippable/opt-talos-sessionrestore-many-windows-e10s + +This is because both sequences of letters (``wndws`` and ``mchtst``) independently appear in order +somewhere in this string (remember the order of the terms makes no difference). + +At first fuzzy terms may not seem very useful, but they are actually extremely powerful! Let's use +the term from the interface image above, ``'linux64 mochibrochr``, as an example. First, just notice +how in the image ``fzf`` highlights the characters that constitute the match in green. Next, notice +how typing ``mochibrochr`` can quickly get us all mochitest browser-chrome tasks. The power of fuzzy +terms is that you don't need to memorize the exact task labels you are looking for. Just start +typing something you think is vaguely correct and chances are you'll see the task you're looking for. + + +Term Modifiers +~~~~~~~~~~~~~~ + +The following modifiers can be applied to a search term: + +.. code-block:: text + + 'word => exact match (line must contain the literal string "word") + ^word => exact prefix match (line must start with literal "word") + word$ => exact suffix match (line must end with literal "word") + !word => exact negation match (line must not contain literal "word") + 'a | 'b => OR operator (joins two exact match operators together) + +For example: + +.. code-block:: text + + ^start 'exact | 'other !ignore fuzzy end$ + +would match the string: + +.. code-block:: text + + starting to bake isn't exactly fun, but pizza is yummy in the end + +.. note:: + + The best way to learn how to write queries is to run ``mach try fuzzy --no-push`` and play + around with all of these modifiers! + + +Specifying Queries on the Command Line +-------------------------------------- + +Sometimes it's more convenient to skip the interactive interface and specify a query on the command +line with ``-q/--query``. This is equivalent to opening the interface then typing: +``<query><ctrl-a><enter>``. + +For example: + +.. code-block:: shell + + # selects all mochitest tasks + $ mach try fuzzy --query "mochitest" + +You can pass in multiple queries at once and the results of each will be joined together: + +.. code-block:: shell + + # selects all mochitest and reftest tasks + $ mach try fuzzy -q "mochitest" -q "reftest" + +If instead you want the intersection of queries, you can pass in ``-x/--and``: + +.. code-block:: shell + + # selects all windows mochitest tasks + $ mach try fuzzy --and -q "mochitest" -q "windows" + + +Modifying Presets +~~~~~~~~~~~~~~~~~ + +:doc:`Presets <../presets>` make it easy to run a pre-determined set of tasks. But sometimes you +might not want to run that set exactly as is, you may only want to use the preset as a starting +point then add or remove tasks as needed. This can be accomplished with ``-q/--query`` or +``-i/--interactive``. + +Here are some examples of adding tasks to a preset: + +.. code-block:: shell + + # selects all perf tasks plus all mochitest-chrome tasks + $ mach try fuzzy --preset perf -q "mochitest-chrome" + + # adds tasks to the perf preset interactively + $ mach try fuzzy --preset perf -i + +Similarly, ``-x/--and`` can be used to filter down a preset by taking the intersection of the two +sets: + +.. code-block:: shell + + # limits perf tasks to windows only + $ mach try fuzzy --preset perf -xq "windows" + + # limits perf tasks interactively + $ mach try fuzzy --preset perf -xi + + +Shell Conflicts +~~~~~~~~~~~~~~~ + +Unfortunately ``fzf``'s query language uses some characters (namely ``'``, ``!`` and ``$``) that can +interfere with your shell when using ``-q/--query``. Below are some tips for how to type out a query +on the command line. + +The ``!`` character is typically used for history expansion. If you don't use this feature, the +easiest way to specify queries on the command line is to disable it: + +.. code-block:: shell + + # bash + $ set +H + $ ./mach try fuzzy -q "'foo !bar" + + # zsh + $ setopt no_banghist + $ ./mach try fuzzy -q "'foo !bar" + +If using ``bash``, add ``set +H`` to your ``~/.bashrc``, ``~/.bash_profile`` or equivalent. If using +``zsh``, add ``setopt no_banghist`` to your ``~/.zshrc`` or equivalent. + +If you don't want to disable history expansion, you can escape your queries like this: + +.. code-block:: shell + + # bash + $ ./mach try fuzzy -q $'\'foo !bar' + + # zsh + $ ./mach try fuzzy -q "'foo \!bar" + + +The third option is to use ``-e/--exact`` which reverses the behaviour of the ``'`` character (see +:ref:`additional-arguments` for more details). Using this flag means you won't need to escape the +``'`` character as often and allows you to run your queries like this: + +.. code-block:: shell + + # bash and zsh + $ ./mach try fuzzy -eq 'foo !bar' + +This method is only useful if you find you almost always prefix terms with ``'`` (and rarely use +fuzzy terms). Otherwise as soon as you want to use a fuzzy match you'll run into the same problem as +before. + +.. note:: All the examples in these three approaches will select the same set of tasks. + +If you use ``fish`` shell, you won't need to escape ``!``, however you will need to escape ``$``: + +.. code-block:: shell + + # fish + $ ./mach try fuzzy -q "'foo !bar baz\$" + + +Test Paths +---------- + +One or more paths to a file or directory may be specified as positional arguments. When +specifying paths, the list of available tasks to choose from is filtered down such that +only suites that have tests in a specified path can be selected. Notably, only the first +chunk of each suite/platform appears. When the tasks are scheduled, only tests that live +under one of the specified paths will be run. + +.. note:: + + When using paths, be aware that all tests under the specified paths will run in the + same chunk. This might produce a different ordering from what gets run on production + branches, and may yield different results. + + For suites that restart the browser between each manifest (like mochitest), this + shouldn't be as big of a concern. + +Paths can be used with the interactive ``fzf`` window, or using the ``-q/--query`` argument. +For example, running: + +.. code-block:: shell + + $ mach try fuzzy layout/reftests/reftest-sanity -q "!pgo !cov !asan 'linux64" + +Would produce the following ``try_task_config.json``: + +.. code-block:: json + + { + "env":{ + "MOZHARNESS_TEST_PATHS":"{\"reftest\":\"layout/reftests/reftest-sanity\"}" + }, + "tasks":[ + "test-linux64-qr/debug-reftest-e10s-1", + "test-linux64-qr/opt-reftest-e10s-1", + "test-linux64/debug-reftest-e10s-1", + "test-linux64/debug-reftest-no-accel-e10s-1", + "test-linux64/opt-reftest-e10s-1", + "test-linux64/opt-reftest-no-accel-e10s-1", + ] + } + +Inside of these tasks, the reftest harness will only run tests that live under +``layout/reftests/reftest-sanity``. + + +.. _additional-arguments: + +Additional Arguments +-------------------- + +There are a few additional command line arguments you may wish to use: + +``-e/--exact`` +By default, ``fzf`` treats terms as a fuzzy match and prefixing a term with ``'`` turns it into an exact +match. If passing in ``--exact``, this behaviour is reversed. Non-prefixed terms become exact, and a +``'`` prefix makes a term fuzzy. + +``--full`` +By default, only target tasks (e.g tasks that would normally run on mozilla-central) +are generated. Passing in ``--full`` allows you to select from all tasks. This is useful for +things like nightly or release tasks. + +``-u/--update`` +Update the bootstrapped ``fzf`` binary to the latest version. + +For a full list of command line arguments, run: + +.. code-block:: shell + + $ mach try fuzzy --help + +For more information on using ``fzf``, run: + +.. code-block:: shell + + $ man fzf + +.. _fzf: https://github.com/junegunn/fzf diff --git a/tools/tryselect/docs/selectors/fzf.png b/tools/tryselect/docs/selectors/fzf.png Binary files differnew file mode 100644 index 0000000000..a64f4b04f3 --- /dev/null +++ b/tools/tryselect/docs/selectors/fzf.png diff --git a/tools/tryselect/docs/selectors/index.rst b/tools/tryselect/docs/selectors/index.rst new file mode 100644 index 0000000000..be618202d6 --- /dev/null +++ b/tools/tryselect/docs/selectors/index.rst @@ -0,0 +1,44 @@ +Selectors +========= + +These are the currently implemented try selectors: + +* :doc:`auto <auto>`: Have tasks chosen for you automatically. +* :doc:`fuzzy <fuzzy>`: Select tasks using a fuzzy finding algorithm and + a terminal interface. +* :doc:`chooser <chooser>`: Select tasks using a web interface. +* :doc:`again <again>`: Re-run a previous ``try_task_config.json`` based + push. +* :doc:`empty <empty>`: Don't select any tasks. Taskcluster will still run + some tasks automatically (like lint and python unittest tasks). Further tasks + can be chosen with treeherder's ``Add New Jobs`` feature. +* :doc:`syntax <syntax>`: Select tasks using classic try syntax. +* :doc:`release <release>`: Prepare a tree for doing a staging release. +* :doc:`scriptworker <scriptworker>`: Run scriptworker tasks against a recent release. +* :doc:`compare <compare>`: Push two identical try jobs, one on your current commit and another of your choice + +You can run them with: + +.. code-block:: shell + + $ mach try <selector> + +See selector specific options by running: + +.. code-block:: shell + + $ mach try <selector> --help + +.. toctree:: + :caption: Available Selectors + :maxdepth: 1 + :hidden: + + Auto <auto> + Fuzzy <fuzzy> + Chooser <chooser> + Again <again> + Empty <empty> + Syntax <syntax> + Release <release> + Scriptworker <scriptworker> diff --git a/tools/tryselect/docs/selectors/release.rst b/tools/tryselect/docs/selectors/release.rst new file mode 100644 index 0000000000..946266249b --- /dev/null +++ b/tools/tryselect/docs/selectors/release.rst @@ -0,0 +1,31 @@ +Release Selector +================ + +This command configures the tree in preparation for doing a staging release, +and pushes the result to try. The changes that that are made include: + +- Updating the version number. +- Applying the migrations that are done as part of merge day. +- Disabling repacking most locales. (This can be disabled by passing ``--no-limit-locales``). + +For staging a beta release, run the following (with an appropriate version number): + +.. code-block:: shell + + $ mach try release --version 64.0b5 --migration central-to-beta + +For staging a final release (rc or patch), run the following (with an appropriate version number) + +.. code-block:: shell + + $ mach try release --version 64.0 --migration central-to-beta --migration beta-to-release + +Once the decision task is on the push is complete, you can start the release +through `staging ship-it instance <https://shipit.staging.mozilla-releng.net/new>`_\ [#shipit]_. + +.. note:: + + If pushing from beta or release, the corresponding migration should not be + passed, as they have already been applied. + +.. [#shipit] This is only available to release engineering and release management (as of 2018-10-15). diff --git a/tools/tryselect/docs/selectors/scriptworker.rst b/tools/tryselect/docs/selectors/scriptworker.rst new file mode 100644 index 0000000000..a3cba08cbe --- /dev/null +++ b/tools/tryselect/docs/selectors/scriptworker.rst @@ -0,0 +1,31 @@ +Scriptworker Selector +===================== + +This command runs a selection of scriptworker tasks against builds from a +recent release. This is aimed at release engineering, to test changes to +scriptworker implementations. It currently requires being connected to +Mozilla's internal datacenter VPN with access to shipit\ [#shipit]_. + +There are a number of preset groups of tasks to run. To run a particular +set of tasks, pass the name of the set to ``mach try scriptworker``: + +.. code-block:: shell + + $ mach try scriptworker linux-signing + +To get the list of task sets, along with the list of tasks they will run: + +.. code-block:: shell + + $ mach try scriptworker list + +The selector defaults to using tasks from the most recent beta, to use tasks +from a different release, pass ``--release-type <release-type>``: + +.. code-block:: shell + + $ mach try scriptworker --release-type release linux-signing + + +.. [#shipit] The shipit API is not currently publicly available, and is used + to find the release graph to pull previous tasks from. diff --git a/tools/tryselect/docs/selectors/syntax.rst b/tools/tryselect/docs/selectors/syntax.rst new file mode 100644 index 0000000000..b64bb65ab7 --- /dev/null +++ b/tools/tryselect/docs/selectors/syntax.rst @@ -0,0 +1,41 @@ +Syntax Selector +=============== + +.. warning:: + + Try syntax is antiquated and hard to understand. If you aren't already + familiar with try syntax, you might want to use the ``fuzzy`` selector + instead. + +Try syntax is a command line string that goes into the commit message. Using +``mach try syntax`` will automatically create a temporary commit with your +chosen syntax and then delete it again after pushing to try. + +Try syntax can contain all kinds of different options parsed by various +places in various repos, but the majority are parsed by `try_option_syntax.py`_. +The most common arguments include: + + * ``-b/--build`` - One of ``d``, ``o`` or ``do``. This is the build type, + either opt, debug or both (required). + * ``-p/--platform`` - The platforms you want to build and/or run tests on + (required). + * ``-u/--unittests`` - The test tasks you want to run (optional). + * ``-t/--talos`` - The talos tasks you want to run (optional). + +Here are some examples: + +.. code-block:: shell + + $ mach try syntax -b do -p linux,macosx64 -u mochitest-e10s-1,crashtest -t none + $ mach try syntax -b d -p win64 -u all + $ mach try syntax -b o -p linux64 + +Unfortunately, knowing the magic strings that make it work can be a bit of a +guessing game. If you are unsure of what string will get your task to run, try +using :doc:`mach try fuzzy <fuzzy>` instead. + +While using ``mach try syntax -b do -p all -u all -t all`` will work, heavy use +of ``all`` is discouraged as it consumes a lot of unnecessary resources (some of +which are hardware constrained). + +.. _try_option_syntax.py: https://searchfox.org/mozilla-central/source/taskcluster/gecko_taskgraph/try_option_syntax.py diff --git a/tools/tryselect/docs/tasks.rst b/tools/tryselect/docs/tasks.rst new file mode 100644 index 0000000000..61de9ec9ed --- /dev/null +++ b/tools/tryselect/docs/tasks.rst @@ -0,0 +1,152 @@ +Task Generation +=============== + +Many selectors (including ``chooser``, ``coverage`` and ``fuzzy``) source their available tasks +directly from the :ref:`taskgraph <TaskCluster Task-Graph Generation>` module by building the taskgraph +locally. This means that the list of available tasks will never be stale. While this is very +powerful, it comes with a large enough performance cost to get annoying (around twenty seconds). + +The result of the taskgraph generation will be cached, so this penalty will only be incurred +whenever a file in the ``/taskcluster`` directory is modified. Unfortunately this directory changes +pretty frequently, so developers can expect to rebuild the cache each time they pull in +``mozilla-central``. Developers who regularly work on ``/taskcluster`` can expect to rebuild even +more frequently. + + +Configuring Watchman +-------------------- + +It's possible to bypass this penalty completely by using the file watching service `watchman`_. If +you use the ``fsmonitor`` mercurial extension, you already have ``watchman`` installed. + +.. note:: + + If you aren't using `fsmonitor`_ but end up installng watchman anyway, you + might as well enable it for a faster Mercurial experience. + +Otherwise, `install watchman`_. If using Linux you'll likely run into the `inotify limits`_ outlined +on that page due to the size of ``mozilla-central``. You can `read this page`_ for more information +on how to bump the limits permanently. + +Next run the following commands: + +.. code-block:: shell + + $ cd path/to/mozilla-central + $ watchman watch . + $ watchman -j < tools/tryselect/watchman.json + +You should see output like: + +.. code-block:: json + + { + "triggerid": "rebuild-taskgraph-cache", + "disposition": "created", + "version": "20200920.192359.0" + } + +That's it. Now anytime a file under ``/taskcluster`` is modified (either by your editor, or by +updating version control), the taskgraph cache will be rebuilt in the background, allowing you to +skip the wait the next time you run ``mach try``. + +.. note:: + + Watchman triggers are persistent and don't need to be added more than once. + See `Managing Triggers`_ for how to remove a trigger. + +You can test that everything is working by running these commands: + +.. code-block:: shell + + $ statedir=`mach python -c "from mach.util import get_state_dir; print(get_state_dir(specific_to_topsrcdir=True))"` + $ rm -rf $statedir/cache/taskgraph + $ touch taskcluster/mach_commands.py + # wait a minute for generation to trigger and finish + $ ls $statedir/cache/taskgraph + +If the ``target_task_set`` file exists, you are good to go. If not you can look at the ``watchman`` +log to see if there were any errors. This typically lives somewhere like +``/usr/local/var/run/watchman/$USER-state/log``. In this case please file a bug under ``Firefox +Build System :: Try`` and include the relevant portion of the log. + + +Running Watchman on Startup +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Watchman is both a client and a service all in one. When running a ``watchman`` command, the client +binary will start the service in the background if it isn't running. This means on reboot the +service won't be running and you'll need to start the service each time by invoking the client +binary (e.g by running ``watchman version``). + +If you'd like this to happen automatically, you can use your favourite platform specific way of +running commands at startup (``crontab``, ``rc.local``, etc.). Watchman stores separate state for +each user, so be sure you run the command as the user that set up the triggers. + +Setting up a systemd Service +++++++++++++++++++++++++++++ + +If ``systemd`` is an option you can create a service: + +.. code-block:: ini + + [Unit] + Description=Watchman for %i + After=network.target + + [Service] + Type=simple + User=%i + ExecStart=/usr/local/bin/watchman --log-level 1 watch-list -f + ExecStop=/usr/local/bin/watchman shutdown-server + + [Install] + WantedBy=multi-user.target + +Save this to a file called ``/etc/systemd/system/watchman@.service``. Then run: + +.. code-block:: shell + + $ sudo systemctl enable watchman@$USER.service + $ sudo systemctl start watchman@$USER.service + +The next time you reboot, the watchman service should start automatically. + + +Managing Triggers +~~~~~~~~~~~~~~~~~ + +When adding a trigger watchman writes it to disk. Typically it'll be a path similar to +``/usr/local/var/run/watchman/$USER-state/state``. While editing that file by hand would work, the +watchman binary provides an interface for managing your triggers. + +To see all directories you are currently watching: + +.. code-block:: shell + + $ watchman watch-list + +To view triggers that are active in a specified watch: + +.. code-block:: shell + + $ watchman trigger-list <path> + +To delete a trigger from a specified watch: + +.. code-block:: shell + + $ watchman trigger-del <path> <name> + +In the above two examples, replace ``<path>`` with the path of the watch, presumably +``mozilla-central``. Using ``.`` works as well if that is already your working directory. For more +information on managing triggers and a reference of other commands, see the `official docs`_. + + +.. _watchman: https://facebook.github.io/watchman/ +.. _fsmonitor: https://www.mercurial-scm.org/wiki/FsMonitorExtension +.. _install watchman: https://facebook.github.io/watchman/docs/install.html +.. _inotify limits: https://facebook.github.io/watchman/docs/install.html#linux-inotify-limits +.. _read this page: https://github.com/guard/listen/wiki/Increasing-the-amount-of-inotify-watchers +.. _this hint: https://github.com/facebook/watchman/commit/2985377eaf8c8538b28fae9add061b67991a87c2 +.. _official docs: https://facebook.github.io/watchman/docs/cmd/trigger.html diff --git a/tools/tryselect/lando.py b/tools/tryselect/lando.py new file mode 100644 index 0000000000..7abd2ddfae --- /dev/null +++ b/tools/tryselect/lando.py @@ -0,0 +1,452 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Implements Auth0 Device Code flow and Lando try submission. + +See https://auth0.com/blog/securing-a-python-cli-application-with-auth0/ for more. +""" + +from __future__ import annotations + +import base64 +import configparser +import json +import os +import time +import webbrowser +from dataclasses import ( + dataclass, + field, +) +from pathlib import Path +from typing import ( + List, + Optional, + Tuple, + Union, +) + +import requests +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozversioncontrol import ( + GitRepository, + HgRepository, +) + +from .task_config import ( + try_config_commit, +) + +TOKEN_FILE = ( + Path(get_state_dir(specific_to_topsrcdir=False)) / "lando_auth0_user_token.json" +) + +# The supported variants of `Repository` for this workflow. +SupportedVcsRepository = Union[GitRepository, HgRepository] + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + + +def convert_bytes_patch_to_base64(patch_bytes: bytes) -> str: + """Return a base64 encoded `str` representing the passed `bytes` patch.""" + return base64.b64encode(patch_bytes).decode("ascii") + + +def load_token_from_disk() -> Optional[dict]: + """Load and validate an existing Auth0 token from disk. + + Return the token as a `dict` if it can be validated, or return `None` + if any error was encountered. + """ + if not TOKEN_FILE.exists(): + print("No existing Auth0 token found.") + return None + + try: + user_token = json.loads(TOKEN_FILE.read_bytes()) + except json.JSONDecodeError: + print("Existing Auth0 token could not be decoded as JSON.") + return None + + return user_token + + +def get_stack_info(vcs: SupportedVcsRepository) -> Tuple[str, List[str]]: + """Retrieve information about the current stack for submission via Lando. + + Returns a tuple of the current public base commit as a Mercurial SHA, + and a list of ordered base64 encoded patches. + """ + base_commit = vcs.base_ref_as_hg() + if not base_commit: + raise ValueError( + "Could not determine base Mercurial commit hash for submission." + ) + print("Using", base_commit, "as the hg base commit.") + + # Reuse the base revision when on Mercurial to avoid multiple calls to `hg log`. + branch_nodes_kwargs = {} + if isinstance(vcs, HgRepository): + branch_nodes_kwargs["base_ref"] = base_commit + + nodes = vcs.get_branch_nodes(**branch_nodes_kwargs) + if not nodes: + raise ValueError("Could not find any commit hashes for submission.") + elif len(nodes) == 1: + print("Submitting a single try config commit.") + elif len(nodes) == 2: + print("Submitting 1 node and the try commit.") + else: + print("Submitting stack of", len(nodes) - 1, "nodes and the try commit.") + + patches = vcs.get_commit_patches(nodes) + base64_patches = [ + convert_bytes_patch_to_base64(patch_bytes) for patch_bytes in patches + ] + print("Patches gathered for submission.") + + return base_commit, base64_patches + + +@dataclass +class Auth0Config: + """Helper class to interact with Auth0.""" + + domain: str + client_id: str + audience: str + scope: str + algorithms: list[str] = field(default_factory=lambda: ["RS256"]) + + @property + def base_url(self) -> str: + """Auth0 base URL.""" + return f"https://{self.domain}" + + @property + def device_code_url(self) -> str: + """URL of the Device Code API endpoint.""" + return f"{self.base_url}/oauth/device/code" + + @property + def issuer(self) -> str: + """Token issuer URL.""" + return f"{self.base_url}/" + + @property + def jwks_url(self) -> str: + """URL of the JWKS file.""" + return f"{self.base_url}/.well-known/jwks.json" + + @property + def oauth_token_url(self) -> str: + """URL of the OAuth Token endpoint.""" + return f"{self.base_url}/oauth/token" + + def request_device_code(self) -> dict: + """Request authorization from Auth0 using the Device Code Flow. + + See https://auth0.com/docs/api/authentication#get-device-code for more. + """ + response = requests.post( + self.device_code_url, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data={ + "audience": self.audience, + "client_id": self.client_id, + "scope": self.scope, + }, + ) + + response.raise_for_status() + + return response.json() + + def validate_token(self, user_token: dict) -> Optional[dict]: + """Verify the given user token is valid. + + Validate the ID token, and validate the access token's expiration claim. + """ + # Import `auth0-python` here to avoid `ImportError` in tests, since + # the `python-test` site won't have `auth0-python` installed. + import jwt + from auth0.authentication.token_verifier import ( + AsymmetricSignatureVerifier, + TokenVerifier, + ) + from auth0.exceptions import ( + TokenValidationError, + ) + + signature_verifier = AsymmetricSignatureVerifier(self.jwks_url) + token_verifier = TokenVerifier( + audience=self.client_id, + issuer=self.issuer, + signature_verifier=signature_verifier, + ) + + try: + token_verifier.verify(user_token["id_token"]) + except TokenValidationError as e: + print("Could not validate existing Auth0 ID token:", str(e)) + return None + + decoded_access_token = jwt.decode( + user_token["access_token"], + algorithms=self.algorithms, + options={"verify_signature": False}, + ) + + access_token_expiration = decoded_access_token["exp"] + + # Assert that the access token isn't expired or expiring within a minute. + if time.time() > access_token_expiration + 60: + print("Access token is expired.") + return None + + user_token.update( + jwt.decode( + user_token["id_token"], + algorithms=self.algorithms, + options={"verify_signature": False}, + ) + ) + print("Auth0 token validated.") + return user_token + + def device_authorization_flow(self) -> dict: + """Perform the Device Authorization Flow. + + See https://auth0.com/docs/get-started/authentication-and-authorization-flow/device-authorization-flow + for more. + """ + start = time.perf_counter() + + device_code_data = self.request_device_code() + print( + "1. On your computer or mobile device navigate to:", + device_code_data["verification_uri_complete"], + ) + print("2. Enter the following code:", device_code_data["user_code"]) + + auth_msg = f"Auth0 token validation required at: {device_code_data['verification_uri_complete']}" + build.notify(auth_msg) + + try: + webbrowser.open(device_code_data["verification_uri_complete"]) + except webbrowser.Error: + print("Could not automatically open the web browser.") + + device_code_lifetime_s = device_code_data["expires_in"] + + # Print successive periods on the same line to avoid moving the link + # while the user is trying to click it. + print("Waiting...", end="", flush=True) + while time.perf_counter() - start < device_code_lifetime_s: + response = requests.post( + self.oauth_token_url, + data={ + "client_id": self.client_id, + "device_code": device_code_data["device_code"], + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "scope": self.scope, + }, + ) + response_data = response.json() + + if response.status_code == 200: + print("\nLogin successful.") + return response_data + + if response_data["error"] not in ("authorization_pending", "slow_down"): + raise RuntimeError(response_data["error_description"]) + + time.sleep(device_code_data["interval"]) + print(".", end="", flush=True) + + raise ValueError("Timed out waiting for Auth0 device code authentication!") + + def get_token(self) -> dict: + """Retrieve an access token for authentication. + + If a cached token is found and can be confirmed to be valid, return it. + Otherwise, perform the Device Code Flow authorization to request a new + token, validate it and save it to disk. + """ + # Load a cached token and validate it if one is available. + cached_token = load_token_from_disk() + user_token = self.validate_token(cached_token) if cached_token else None + + # Login with the Device Authorization Flow if an existing token isn't found. + if not user_token: + new_token = self.device_authorization_flow() + user_token = self.validate_token(new_token) + + if not user_token: + raise ValueError("Could not get an Auth0 token.") + + # Save token to disk. + with TOKEN_FILE.open("w") as f: + json.dump(user_token, f, indent=2, sort_keys=True) + + return user_token + + +class LandoAPIException(Exception): + """Raised when Lando throws an exception.""" + + def __init__(self, detail: Optional[str] = None): + super().__init__(detail or "") + + +@dataclass +class LandoAPI: + """Helper class to interact with Lando-API.""" + + access_token: str + api_url: str + + @property + def lando_try_api_url(self) -> str: + """URL of the Lando Try endpoint.""" + return f"https://{self.api_url}/try/patches" + + @property + def api_headers(self) -> dict[str, str]: + """Headers for use accessing and authenticating against the API.""" + return { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + @classmethod + def from_lando_config_file(cls, config_path: Path, section: str) -> LandoAPI: + """Build a `LandoConfig` from `section` in the file at `config_path`.""" + if not config_path.exists(): + raise ValueError(f"Could not find a Lando config file at `{config_path}`.") + + lando_ini_contents = config_path.read_text() + + parser = configparser.ConfigParser(delimiters="=") + parser.read_string(lando_ini_contents) + + if not parser.has_section(section): + raise ValueError(f"Lando config file does not have a {section} section.") + + auth0 = Auth0Config( + domain=parser.get(section, "auth0_domain"), + client_id=parser.get(section, "auth0_client_id"), + audience=parser.get(section, "auth0_audience"), + scope=parser.get(section, "auth0_scope"), + ) + + token = auth0.get_token() + + return LandoAPI( + api_url=parser.get(section, "api_domain"), + access_token=token["access_token"], + ) + + def post(self, url: str, body: dict) -> dict: + """Make a POST request to Lando.""" + response = requests.post(url, headers=self.api_headers, json=body) + + try: + response_json = response.json() + except json.JSONDecodeError: + # If the server didn't send back a valid JSON object, raise a stack + # trace to the terminal which includes error details. + response.raise_for_status() + + # Raise `ValueError` if the response wasn't JSON and we didn't raise + # from an invalid status. + raise LandoAPIException( + detail="Response was not valid JSON yet status was valid." + ) + + if response.status_code >= 400: + raise LandoAPIException(detail=response_json["detail"]) + + return response_json + + def post_try_push_patches( + self, + patches: List[str], + patch_format: str, + base_commit: str, + ) -> dict: + """Send try push contents to Lando. + + Send the list of base64-encoded `patches` in `patch_format` to Lando, to be applied to + the Mercurial `base_commit`, using the Auth0 `access_token` for authorization. + """ + request_json_body = { + "base_commit": base_commit, + "patch_format": patch_format, + "patches": patches, + } + + print("Submitting patches to Lando.") + response_json = self.post(self.lando_try_api_url, request_json_body) + + return response_json + + +def push_to_lando_try(vcs: SupportedVcsRepository, commit_message: str): + """Push a set of patches to Lando's try endpoint.""" + # Map `Repository` subclasses to the `patch_format` value Lando expects. + PATCH_FORMAT_STRING_MAPPING = { + GitRepository: "git-format-patch", + HgRepository: "hgexport", + } + patch_format = PATCH_FORMAT_STRING_MAPPING.get(type(vcs)) + if not patch_format: + # Other VCS types (namely `src`) are unsupported. + raise ValueError(f"Try push via Lando is not supported for `{vcs.name}`.") + + # Use Lando Prod unless the `LANDO_TRY_USE_DEV` environment variable is defined. + lando_config_section = ( + "lando-prod" if not os.getenv("LANDO_TRY_USE_DEV") else "lando-dev" + ) + + # Load Auth0 config from `.lando.ini`. + lando_ini_path = Path(vcs.path) / ".lando.ini" + lando_api = LandoAPI.from_lando_config_file(lando_ini_path, lando_config_section) + + # Get the time when the push was initiated, not including Auth0 login time. + push_start_time = time.perf_counter() + + with try_config_commit(vcs, commit_message): + try: + base_commit, patches = get_stack_info(vcs) + except ValueError as exc: + error_msg = "abort: error gathering patches for submission." + print(error_msg) + print(str(exc)) + build.notify(error_msg) + return + + try: + # Make the try request to Lando. + response_json = lando_api.post_try_push_patches( + patches, patch_format, base_commit + ) + except LandoAPIException as exc: + error_msg = "abort: error submitting patches to Lando." + print(error_msg) + print(str(exc)) + build.notify(error_msg) + return + + duration = round(time.perf_counter() - push_start_time, ndigits=2) + + job_id = response_json["id"] + success_msg = ( + f"Lando try submission success, took {duration} seconds. " + f"Landing job id: {job_id}." + ) + print(success_msg) + build.notify(success_msg) diff --git a/tools/tryselect/mach_commands.py b/tools/tryselect/mach_commands.py new file mode 100644 index 0000000000..3b74593423 --- /dev/null +++ b/tools/tryselect/mach_commands.py @@ -0,0 +1,514 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import argparse +import importlib +import os +import sys + +from mach.decorators import Command, SubCommand +from mach.util import get_state_dir +from mozbuild.base import BuildEnvironmentNotFoundException +from mozbuild.util import memoize + +CONFIG_ENVIRONMENT_NOT_FOUND = """ +No config environment detected. This means we are unable to properly +detect test files in the specified paths or tags. Please run: + + $ mach configure + +and try again. +""".lstrip() + + +class get_parser: + def __init__(self, selector): + self.selector = selector + + def __call__(self): + mod = importlib.import_module("tryselect.selectors.{}".format(self.selector)) + return getattr(mod, "{}Parser".format(self.selector.capitalize()))() + + +def generic_parser(): + from tryselect.cli import BaseTryParser + + parser = BaseTryParser() + parser.add_argument("argv", nargs=argparse.REMAINDER) + return parser + + +def init(command_context): + from tryselect import push + + push.MAX_HISTORY = command_context._mach_context.settings["try"]["maxhistory"] + + +@memoize +def presets(command_context): + from tryselect.preset import MergedHandler + + # Create our handler using both local and in-tree presets. The first + # path in this list will be treated as the 'user' file for the purposes + # of saving and editing. All subsequent paths are 'read-only'. We check + # an environment variable first for testing purposes. + if os.environ.get("MACH_TRY_PRESET_PATHS"): + preset_paths = os.environ["MACH_TRY_PRESET_PATHS"].split(os.pathsep) + else: + preset_paths = [ + os.path.join(get_state_dir(), "try_presets.yml"), + os.path.join( + command_context.topsrcdir, "tools", "tryselect", "try_presets.yml" + ), + ] + + return MergedHandler(*preset_paths) + + +def handle_presets( + command_context, preset_action=None, save=None, preset=None, **kwargs +): + """Handle preset related arguments. + + This logic lives here so that the underlying selectors don't need + special preset handling. They can all save and load presets the same + way. + """ + from tryselect.util.dicttools import merge + + user_presets = presets(command_context).handlers[0] + if preset_action == "list": + presets(command_context).list() + sys.exit() + + if preset_action == "edit": + user_presets.edit() + sys.exit() + + parser = command_context._mach_context.handler.parser + subcommand = command_context._mach_context.handler.subcommand + if "preset" not in parser.common_groups: + return kwargs + + default = parser.get_default + if save: + selector = ( + subcommand or command_context._mach_context.settings["try"]["default"] + ) + + # Only save non-default values for simplicity. + kwargs = {k: v for k, v in kwargs.items() if v != default(k)} + user_presets.save(save, selector=selector, **kwargs) + print("preset saved, run with: --preset={}".format(save)) + sys.exit() + + if preset: + if preset not in presets(command_context): + command_context._mach_context.parser.error( + "preset '{}' does not exist".format(preset) + ) + + name = preset + preset = presets(command_context)[name] + selector = preset.pop("selector") + preset.pop("description", None) # description isn't used by any selectors + + if not subcommand: + subcommand = selector + elif subcommand != selector: + print( + "error: preset '{}' exists for a different selector " + "(did you mean to run 'mach try {}' instead?)".format(name, selector) + ) + sys.exit(1) + + # Order of precedence is defaults -> presets -> cli. Configuration + # from the right overwrites configuration from the left. + defaults = {} + nondefaults = {} + for k, v in kwargs.items(): + if v == default(k): + defaults[k] = v + else: + nondefaults[k] = v + + kwargs = merge(defaults, preset, nondefaults) + + return kwargs + + +def handle_try_params(command_context, **kwargs): + from tryselect.util.dicttools import merge + + to_validate = [] + kwargs.setdefault("try_config_params", {}) + for cls in command_context._mach_context.handler.parser.task_configs.values(): + params = cls.get_parameters(**kwargs) + if params is not None: + to_validate.append(cls) + kwargs["try_config_params"] = merge(kwargs["try_config_params"], params) + + for name in cls.dests: + del kwargs[name] + + # Validate task_configs after they have all been parsed to avoid + # depending on the order they were processed. + for cls in to_validate: + cls.validate(**kwargs) + return kwargs + + +def run(command_context, **kwargs): + kwargs = handle_presets(command_context, **kwargs) + + if command_context._mach_context.handler.parser.task_configs: + kwargs = handle_try_params(command_context, **kwargs) + + mod = importlib.import_module( + "tryselect.selectors.{}".format( + command_context._mach_context.handler.subcommand + ) + ) + return mod.run(**kwargs) + + +@Command( + "try", + category="ci", + description="Push selected tasks to the try server", + parser=generic_parser, + virtualenv_name="try", +) +def try_default(command_context, argv=None, **kwargs): + """Push selected tests to the try server. + + The |mach try| command is a frontend for scheduling tasks to + run on try server using selectors. A selector is a subcommand + that provides its own set of command line arguments and are + listed below. + + If no subcommand is specified, the `auto` selector is run by + default. Run |mach try auto --help| for more information on + scheduling with the `auto` selector. + """ + init(command_context) + subcommand = command_context._mach_context.handler.subcommand + # We do special handling of presets here so that `./mach try --preset foo` + # works no matter what subcommand 'foo' was saved with. + preset = kwargs["preset"] + if preset: + if preset not in presets(command_context): + command_context._mach_context.handler.parser.error( + "preset '{}' does not exist".format(preset) + ) + + subcommand = presets(command_context)[preset]["selector"] + + sub = subcommand or command_context._mach_context.settings["try"]["default"] + return command_context._mach_context.commands.dispatch( + "try", command_context._mach_context, subcommand=sub, argv=argv, **kwargs + ) + + +@SubCommand( + "try", + "fuzzy", + description="Select tasks on try using a fuzzy finder", + parser=get_parser("fuzzy"), + virtualenv_name="try", +) +def try_fuzzy(command_context, **kwargs): + """Select which tasks to run with a fuzzy finding interface (fzf). + + When entering the fzf interface you'll be confronted by two panes. The + one on the left contains every possible task you can schedule, the one + on the right contains the list of selected tasks. In other words, the + tasks that will be scheduled once you you press <enter>. + + At first fzf will automatically select whichever task is under your + cursor, which simplifies the case when you are looking for a single + task. But normally you'll want to select many tasks. To accomplish + you'll generally start by typing a query in the search bar to filter + down the list of tasks (see Extended Search below). Then you'll either: + + A) Move the cursor to each task you want and press <tab> to select it. + Notice it now shows up in the pane to the right. + + OR + + B) Press <ctrl-a> to select every task that matches your filter. + + You can delete your query, type a new one and select further tasks as + many times as you like. Once you are happy with your selection, press + <enter> to push the selected tasks to try. + + All selected task labels and their dependencies will be scheduled. This + means you can select a test task and its build will automatically be + filled in. + + + Keyboard Shortcuts + ------------------ + + When in the fuzzy finder interface, start typing to filter down the + task list. Then use the following keyboard shortcuts to select tasks: + + Ctrl-K / Up => Move cursor up + Ctrl-J / Down => Move cursor down + Tab => Select task + move cursor down + Shift-Tab => Select task + move cursor up + Ctrl-A => Select all currently filtered tasks + Ctrl-D => De-select all currently filtered tasks + Ctrl-T => Toggle select all currently filtered tasks + Alt-Bspace => Clear query from input bar + Enter => Accept selection and exit + Ctrl-C / Esc => Cancel selection and exit + ? => Toggle preview pane + + There are many more shortcuts enabled by default, you can also define + your own shortcuts by setting `--bind` in the $FZF_DEFAULT_OPTS + environment variable. See `man fzf` for more info. + + + Extended Search + --------------- + + When typing in search terms, the following modifiers can be applied: + + 'word: exact match (line must contain the literal string "word") + ^word: exact prefix match (line must start with literal "word") + word$: exact suffix match (line must end with literal "word") + !word: exact negation match (line must not contain literal "word") + 'a | 'b: OR operator (joins two exact match operators together) + + For example: + + ^start 'exact | !ignore fuzzy end$ + + + Documentation + ------------- + + For more detailed documentation, please see: + https://firefox-source-docs.mozilla.org/tools/try/selectors/fuzzy.html + """ + init(command_context) + if kwargs.pop("interactive"): + kwargs["query"].append("INTERACTIVE") + + if kwargs.pop("intersection"): + kwargs["intersect_query"] = kwargs["query"] + del kwargs["query"] + + if kwargs.get("save") and not kwargs.get("query"): + # If saving preset without -q/--query, allow user to use the + # interface to build the query. + kwargs_copy = kwargs.copy() + kwargs_copy["dry_run"] = True + kwargs_copy["save"] = None + kwargs["query"] = run(command_context, save_query=True, **kwargs_copy) + if not kwargs["query"]: + return + + if kwargs.get("paths"): + kwargs["test_paths"] = kwargs["paths"] + + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "chooser", + description="Schedule tasks by selecting them from a web interface.", + parser=get_parser("chooser"), + virtualenv_name="try", +) +def try_chooser(command_context, **kwargs): + """Push tasks selected from a web interface to try. + + This selector will build the taskgraph and spin up a dynamically + created 'trychooser-like' web-page on the localhost. After a selection + has been made, pressing the 'Push' button will automatically push the + selection to try. + """ + init(command_context) + command_context.activate_virtualenv() + + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "auto", + description="Automatically determine which tasks to run. This runs the same " + "set of tasks that would be run on autoland. This " + "selector is EXPERIMENTAL.", + parser=get_parser("auto"), + virtualenv_name="try", +) +def try_auto(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "again", + description="Schedule a previously generated (non try syntax) push again.", + parser=get_parser("again"), + virtualenv_name="try", +) +def try_again(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "empty", + description="Push to try without scheduling any tasks.", + parser=get_parser("empty"), + virtualenv_name="try", +) +def try_empty(command_context, **kwargs): + """Push to try, running no builds or tests + + This selector does not prompt you to run anything, it just pushes + your patches to try, running no builds or tests by default. After + the push finishes, you can manually add desired jobs to your push + via Treeherder's Add New Jobs feature, located in the per-push + menu. + """ + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "syntax", + description="Select tasks on try using try syntax", + parser=get_parser("syntax"), + virtualenv_name="try", +) +def try_syntax(command_context, **kwargs): + """Push the current tree to try, with the specified syntax. + + Build options, platforms and regression tests may be selected + using the usual try options (-b, -p and -u respectively). In + addition, tests in a given directory may be automatically + selected by passing that directory as a positional argument to the + command. For example: + + mach try -b d -p linux64 dom testing/web-platform/tests/dom + + would schedule a try run for linux64 debug consisting of all + tests under dom/ and testing/web-platform/tests/dom. + + Test selection using positional arguments is available for + mochitests, reftests, xpcshell tests and web-platform-tests. + + Tests may be also filtered by passing --tag to the command, + which will run only tests marked as having the specified + tags e.g. + + mach try -b d -p win64 --tag media + + would run all tests tagged 'media' on Windows 64. + + If both positional arguments or tags and -u are supplied, the + suites in -u will be run in full. Where tests are selected by + positional argument they will be run in a single chunk. + + If no build option is selected, both debug and opt will be + scheduled. If no platform is selected a default is taken from + the AUTOTRY_PLATFORM_HINT environment variable, if set. + + The command requires either its own mercurial extension ("push-to-try", + installable from mach vcs-setup) or a git repo using git-cinnabar + (installable from mach vcs-setup). + + """ + init(command_context) + try: + if command_context.substs.get("MOZ_ARTIFACT_BUILDS"): + kwargs["local_artifact_build"] = True + except BuildEnvironmentNotFoundException: + # If we don't have a build locally, we can't tell whether + # an artifact build is desired, but we still want the + # command to succeed, if possible. + pass + + config_status = os.path.join(command_context.topobjdir, "config.status") + if (kwargs["paths"] or kwargs["tags"]) and not config_status: + print(CONFIG_ENVIRONMENT_NOT_FOUND) + sys.exit(1) + + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "coverage", + description="Select tasks on try using coverage data", + parser=get_parser("coverage"), + virtualenv_name="try", +) +def try_coverage(command_context, **kwargs): + """Select which tasks to use using coverage data.""" + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "release", + description="Push the current tree to try, configured for a staging release.", + parser=get_parser("release"), + virtualenv_name="try", +) +def try_release(command_context, **kwargs): + """Push the current tree to try, configured for a staging release.""" + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "scriptworker", + description="Run scriptworker tasks against a recent release.", + parser=get_parser("scriptworker"), + virtualenv_name="try", +) +def try_scriptworker(command_context, **kwargs): + """Run scriptworker tasks against a recent release. + + Requires VPN and shipit access. + """ + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "compare", + description="Push two try jobs, one on your current commit and another on the one you specify", + parser=get_parser("compare"), + virtualenv_name="try", +) +def try_compare(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "perf", + description="Try selector for running performance tests.", + parser=get_parser("perf"), + virtualenv_name="try", +) +def try_perf(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) diff --git a/tools/tryselect/preset.py b/tools/tryselect/preset.py new file mode 100644 index 0000000000..dc8cba5c57 --- /dev/null +++ b/tools/tryselect/preset.py @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import shlex +import subprocess + +import yaml + + +# Ensure strings with ' like fzf query strings get double-quoted: +def represent_str(self, data): + if "'" in data: + return self.represent_scalar("tag:yaml.org,2002:str", data, style='"') + return self.represent_scalar("tag:yaml.org,2002:str", data) + + +yaml.SafeDumper.add_representer(str, represent_str) + + +class PresetHandler: + def __init__(self, path): + self.path = path + self._presets = {} + + @property + def presets(self): + if not self._presets and os.path.isfile(self.path): + with open(self.path) as fh: + self._presets = yaml.safe_load(fh) or {} + + return self._presets + + def __contains__(self, name): + return name in self.presets + + def __getitem__(self, name): + return self.presets[name] + + def __len__(self): + return len(self.presets) + + def __str__(self): + if not self.presets: + return "" + return yaml.safe_dump(self.presets, default_flow_style=False) + + def list(self): + if not self.presets: + print("no presets found") + else: + print(self) + + def edit(self): + if "EDITOR" not in os.environ: + print( + "error: must set the $EDITOR environment variable to use --edit-presets" + ) + return + + subprocess.call(shlex.split(os.environ["EDITOR"]) + [self.path]) + + def save(self, name, **data): + self.presets[name] = data + + with open(self.path, "w") as fh: + fh.write(str(self)) + + +class MergedHandler: + def __init__(self, *paths): + """Helper class for dealing with multiple preset files.""" + self.handlers = [PresetHandler(p) for p in paths] + + def __contains__(self, name): + return any(name in handler for handler in self.handlers) + + def __getitem__(self, name): + for handler in self.handlers: + if name in handler: + return handler[name] + raise KeyError(name) + + def __len__(self): + return sum(len(h) for h in self.handlers) + + def __str__(self): + all_presets = { + k: v for handler in self.handlers for k, v in handler.presets.items() + } + return yaml.safe_dump(all_presets, default_flow_style=False) + + def list(self): + if len(self) == 0: + print("no presets found") + return + + for handler in self.handlers: + val = str(handler) + if val: + val = "\n ".join( + [""] + val.splitlines() + [""] + ) # indent all lines by 2 spaces + print("Presets from {}:".format(handler.path)) + print(val) diff --git a/tools/tryselect/push.py b/tools/tryselect/push.py new file mode 100644 index 0000000000..cf5e646c8c --- /dev/null +++ b/tools/tryselect/push.py @@ -0,0 +1,257 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import sys +import traceback + +import six +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozversioncontrol import MissingVCSExtension, get_repository_object + +from .lando import push_to_lando_try +from .util.estimates import duration_summary +from .util.manage_estimates import ( + download_task_history_data, + make_trimmed_taskgraph_cache, +) + +GIT_CINNABAR_NOT_FOUND = """ +Could not detect `git-cinnabar`. + +The `mach try` command requires git-cinnabar to be installed when +pushing from git. Please install it by running: + + $ ./mach vcs-setup +""".lstrip() + +HG_PUSH_TO_TRY_NOT_FOUND = """ +Could not detect `push-to-try`. + +The `mach try` command requires the push-to-try extension enabled +when pushing from hg. Please install it by running: + + $ ./mach vcs-setup +""".lstrip() + +VCS_NOT_FOUND = """ +Could not detect version control. Only `hg` or `git` are supported. +""".strip() + +UNCOMMITTED_CHANGES = """ +ERROR please commit changes before continuing +""".strip() + +MAX_HISTORY = 10 + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) +vcs = get_repository_object(build.topsrcdir) + +history_path = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "history", "try_task_configs.json" +) + + +def write_task_config(try_task_config): + config_path = os.path.join(vcs.path, "try_task_config.json") + with open(config_path, "w") as fh: + json.dump(try_task_config, fh, indent=4, separators=(",", ": "), sort_keys=True) + fh.write("\n") + return config_path + + +def write_task_config_history(msg, try_task_config): + if not os.path.isfile(history_path): + if not os.path.isdir(os.path.dirname(history_path)): + os.makedirs(os.path.dirname(history_path)) + history = [] + else: + with open(history_path) as fh: + history = fh.read().strip().splitlines() + + history.insert(0, json.dumps([msg, try_task_config])) + history = history[:MAX_HISTORY] + with open(history_path, "w") as fh: + fh.write("\n".join(history)) + + +def check_working_directory(push=True): + if not push: + return + + if not vcs.working_directory_clean(): + print(UNCOMMITTED_CHANGES) + sys.exit(1) + + +def generate_try_task_config(method, labels, params=None, routes=None): + params = params or {} + + # The user has explicitly requested a set of jobs, so run them all + # regardless of optimization (unless the selector explicitly sets this to + # True). Their dependencies can be optimized though. + params.setdefault("optimize_target_tasks", False) + + # Remove selected labels from 'existing_tasks' parameter if present + if "existing_tasks" in params: + params["existing_tasks"] = { + label: tid + for label, tid in params["existing_tasks"].items() + if label not in labels + } + + try_config = params.setdefault("try_task_config", {}) + try_config.setdefault("env", {})["TRY_SELECTOR"] = method + + try_config["tasks"] = sorted(labels) + + if routes: + try_config["routes"] = routes + + try_task_config = {"version": 2, "parameters": params} + return try_task_config + + +def task_labels_from_try_config(try_task_config): + if try_task_config["version"] == 2: + parameters = try_task_config.get("parameters", {}) + if "try_task_config" in parameters: + return parameters["try_task_config"]["tasks"] + else: + return None + elif try_task_config["version"] == 1: + return try_task_config.get("tasks", list()) + else: + return None + + +def display_push_estimates(try_task_config): + task_labels = task_labels_from_try_config(try_task_config) + if task_labels is None: + return + + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + + graph_cache = None + dep_cache = None + target_file = None + for graph_cache_file in ["target_task_graph", "full_task_graph"]: + graph_cache = os.path.join(cache_dir, graph_cache_file) + if os.path.isfile(graph_cache): + dep_cache = graph_cache.replace("task_graph", "task_dependencies") + target_file = graph_cache.replace("task_graph", "task_set") + break + + if not dep_cache: + return + + download_task_history_data(cache_dir=cache_dir) + make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_file) + + durations = duration_summary(dep_cache, task_labels, cache_dir) + + print( + "estimates: Runs {} tasks ({} selected, {} dependencies)".format( + durations["dependency_count"] + durations["selected_count"], + durations["selected_count"], + durations["dependency_count"], + ) + ) + print( + "estimates: Total task duration {}".format( + durations["dependency_duration"] + durations["selected_duration"] + ) + ) + if "percentile" in durations: + percentile = durations["percentile"] + if percentile > 50: + print("estimates: In the longest {}% of durations".format(100 - percentile)) + else: + print("estimates: In the shortest {}% of durations".format(percentile)) + print( + "estimates: Should take about {} (Finished around {})".format( + durations["wall_duration_seconds"], + durations["eta_datetime"].strftime("%Y-%m-%d %H:%M"), + ) + ) + + +def push_to_try( + method, + msg, + try_task_config=None, + stage_changes=False, + dry_run=False, + closed_tree=False, + files_to_change=None, + allow_log_capture=False, + push_to_lando=False, +): + push = not stage_changes and not dry_run + check_working_directory(push) + + if try_task_config and method not in ("auto", "empty"): + try: + display_push_estimates(try_task_config) + except Exception: + traceback.print_exc() + print("warning: unable to display push estimates") + + # Format the commit message + closed_tree_string = " ON A CLOSED TREE" if closed_tree else "" + commit_message = "{}{}\n\nPushed via `mach try {}`".format( + msg, + closed_tree_string, + method, + ) + + config_path = None + changed_files = [] + if try_task_config: + if push and method not in ("again", "auto", "empty"): + write_task_config_history(msg, try_task_config) + config_path = write_task_config(try_task_config) + changed_files.append(config_path) + + if (push or stage_changes) and files_to_change: + for path, content in files_to_change.items(): + path = os.path.join(vcs.path, path) + with open(path, "wb") as fh: + fh.write(six.ensure_binary(content)) + changed_files.append(path) + + try: + if not push: + print("Commit message:") + print(commit_message) + if config_path: + print("Calculated try_task_config.json:") + with open(config_path) as fh: + print(fh.read()) + return + + vcs.add_remove_files(*changed_files) + + try: + if push_to_lando: + push_to_lando_try(vcs, commit_message) + else: + vcs.push_to_try(commit_message, allow_log_capture=allow_log_capture) + except MissingVCSExtension as e: + if e.ext == "push-to-try": + print(HG_PUSH_TO_TRY_NOT_FOUND) + elif e.ext == "cinnabar": + print(GIT_CINNABAR_NOT_FOUND) + else: + raise + sys.exit(1) + finally: + if config_path and os.path.isfile(config_path): + os.remove(config_path) diff --git a/tools/tryselect/selectors/__init__.py b/tools/tryselect/selectors/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/selectors/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/selectors/again.py b/tools/tryselect/selectors/again.py new file mode 100644 index 0000000000..434aed7cc1 --- /dev/null +++ b/tools/tryselect/selectors/again.py @@ -0,0 +1,151 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os + +from ..cli import BaseTryParser +from ..push import history_path, push_to_try + + +class AgainParser(BaseTryParser): + name = "again" + arguments = [ + [ + ["--index"], + { + "default": 0, + "const": "list", + "nargs": "?", + "help": "Index of entry in the history to re-push, " + "where '0' is the most recent (default 0). " + "Use --index without a value to display indices.", + }, + ], + [ + ["--list"], + { + "default": False, + "action": "store_true", + "dest": "list_configs", + "help": "Display history and exit", + }, + ], + [ + ["--list-tasks"], + { + "default": 0, + "action": "count", + "dest": "list_tasks", + "help": "Like --list, but display selected tasks " + "for each history entry, up to 10. Repeat " + "to display all selected tasks.", + }, + ], + [ + ["--purge"], + { + "default": False, + "action": "store_true", + "help": "Remove all history and exit", + }, + ], + ] + common_groups = ["push"] + + +def run( + index=0, purge=False, list_configs=False, list_tasks=0, message="{msg}", **pushargs +): + if index == "list": + list_configs = True + else: + try: + index = int(index) + except ValueError: + print("error: '--index' must be an integer") + return 1 + + if purge: + os.remove(history_path) + return + + if not os.path.isfile(history_path): + print("error: history file not found: {}".format(history_path)) + return 1 + + with open(history_path) as fh: + history = fh.readlines() + + if list_configs or list_tasks > 0: + for i, data in enumerate(history): + msg, config = json.loads(data) + version = config.get("version", "1") + settings = {} + if version == 1: + tasks = config["tasks"] + settings = config + elif version == 2: + try_config = config.get("parameters", {}).get("try_task_config", {}) + tasks = try_config.get("tasks") + else: + tasks = None + + if tasks is not None: + # Select only the things that are of interest to display. + settings = settings.copy() + env = settings.pop("env", {}).copy() + env.pop("TRY_SELECTOR", None) + for name in ("tasks", "version"): + settings.pop(name, None) + + def pluralize(n, noun): + return "{n} {noun}{s}".format( + n=n, noun=noun, s="" if n == 1 else "s" + ) + + out = str(i) + ". (" + pluralize(len(tasks), "task") + if env: + out += ", " + pluralize(len(env), "env var") + if settings: + out += ", " + pluralize(len(settings), "setting") + out += ") " + msg + print(out) + + if list_tasks > 0: + indent = " " * 4 + if list_tasks > 1: + shown_tasks = tasks + else: + shown_tasks = tasks[:10] + print(indent + ("\n" + indent).join(shown_tasks)) + + num_hidden_tasks = len(tasks) - len(shown_tasks) + if num_hidden_tasks > 0: + print("{}... and {} more".format(indent, num_hidden_tasks)) + + if list_tasks and env: + for line in ("env: " + json.dumps(env, indent=2)).splitlines(): + print(" " + line) + + if list_tasks and settings: + for line in ( + "settings: " + json.dumps(settings, indent=2) + ).splitlines(): + print(" " + line) + else: + print( + "{index}. {msg}".format( + index=i, + msg=msg, + ) + ) + + return + + msg, try_task_config = json.loads(history[index]) + return push_to_try( + "again", message.format(msg=msg), try_task_config=try_task_config, **pushargs + ) diff --git a/tools/tryselect/selectors/auto.py b/tools/tryselect/selectors/auto.py new file mode 100644 index 0000000000..e7cc6c508c --- /dev/null +++ b/tools/tryselect/selectors/auto.py @@ -0,0 +1,118 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.util.python_path import find_object + +from ..cli import BaseTryParser +from ..push import push_to_try +from ..util.dicttools import merge + +TRY_AUTO_PARAMETERS = { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", # noqa + "optimize_target_tasks": True, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {}, +} + + +class AutoParser(BaseTryParser): + name = "auto" + common_groups = ["push"] + task_configs = [ + "artifact", + "env", + "chemspill-prio", + "disable-pgo", + "worker-overrides", + ] + arguments = [ + [ + ["--strategy"], + { + "default": None, + "help": "Override the default optimization strategy. Valid values " + "are the experimental strategies defined at the bottom of " + "`taskcluster/gecko_taskgraph/optimize/__init__.py`.", + }, + ], + [ + ["--tasks-regex"], + { + "default": [], + "action": "append", + "help": "Apply a regex filter to the tasks selected. Specifying " + "multiple times schedules the union of computed tasks.", + }, + ], + [ + ["--tasks-regex-exclude"], + { + "default": [], + "action": "append", + "help": "Apply a regex filter to the tasks selected. Specifying " + "multiple times excludes computed tasks matching any regex.", + }, + ], + ] + + def validate(self, args): + super().validate(args) + + if args.strategy: + if ":" not in args.strategy: + args.strategy = "gecko_taskgraph.optimize:tryselect.{}".format( + args.strategy + ) + + try: + obj = find_object(args.strategy) + except (ImportError, AttributeError): + self.error("invalid module path '{}'".format(args.strategy)) + + if not isinstance(obj, dict): + self.error("object at '{}' must be a dict".format(args.strategy)) + + +def run( + message="{msg}", + stage_changes=False, + dry_run=False, + closed_tree=False, + strategy=None, + tasks_regex=None, + tasks_regex_exclude=None, + try_config_params=None, + push_to_lando=False, + **ignored +): + msg = message.format(msg="Tasks automatically selected.") + + params = TRY_AUTO_PARAMETERS.copy() + if try_config_params: + params = merge(params, try_config_params) + + if strategy: + params["optimize_strategies"] = strategy + + if tasks_regex or tasks_regex_exclude: + params.setdefault("try_task_config", {})["tasks-regex"] = {} + params["try_task_config"]["tasks-regex"]["include"] = tasks_regex + params["try_task_config"]["tasks-regex"]["exclude"] = tasks_regex_exclude + + task_config = { + "version": 2, + "parameters": params, + } + return push_to_try( + "auto", + msg, + try_task_config=task_config, + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + push_to_lando=push_to_lando, + ) diff --git a/tools/tryselect/selectors/chooser/.eslintrc.js b/tools/tryselect/selectors/chooser/.eslintrc.js new file mode 100644 index 0000000000..861d6bafc2 --- /dev/null +++ b/tools/tryselect/selectors/chooser/.eslintrc.js @@ -0,0 +1,16 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + env: { + jquery: true, + }, + globals: { + apply: true, + applyChunks: true, + tasks: true, + }, +}; diff --git a/tools/tryselect/selectors/chooser/__init__.py b/tools/tryselect/selectors/chooser/__init__.py new file mode 100644 index 0000000000..d6a32e08d0 --- /dev/null +++ b/tools/tryselect/selectors/chooser/__init__.py @@ -0,0 +1,120 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import multiprocessing +import os +import time +import webbrowser +from threading import Timer + +from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks + +from tryselect.cli import BaseTryParser +from tryselect.push import ( + check_working_directory, + generate_try_task_config, + push_to_try, +) +from tryselect.tasks import generate_tasks + +here = os.path.abspath(os.path.dirname(__file__)) + + +class ChooserParser(BaseTryParser): + name = "chooser" + arguments = [] + common_groups = ["push", "task"] + task_configs = [ + "artifact", + "browsertime", + "chemspill-prio", + "disable-pgo", + "env", + "existing-tasks", + "gecko-profile", + "path", + "pernosco", + "rebuild", + "worker-overrides", + ] + + +def run( + update=False, + query=None, + try_config_params=None, + full=False, + parameters=None, + save=False, + preset=None, + mod_presets=False, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, + push_to_lando=False, +): + from .app import create_application + + push = not stage_changes and not dry_run + check_working_directory(push) + + tg = generate_tasks(parameters, full) + + # Remove tasks that are not to be shown unless `--full` is specified. + if not full: + excluded_tasks = [ + label + for label in tg.tasks.keys() + if not filter_by_uncommon_try_tasks(label) + ] + for task in excluded_tasks: + tg.tasks.pop(task) + + queue = multiprocessing.Queue() + + if os.environ.get("WERKZEUG_RUN_MAIN") == "true": + # we are in the reloader process, don't open the browser or do any try stuff + app = create_application(tg, queue) + app.run() + return + + # give app a second to start before opening the browser + url = "http://127.0.0.1:5000" + Timer(1, lambda: webbrowser.open(url)).start() + print("Starting trychooser on {}".format(url)) + process = multiprocessing.Process( + target=create_and_run_application, args=(tg, queue) + ) + process.start() + + selected = queue.get() + + # Allow the close page to render before terminating the process. + time.sleep(1) + process.terminate() + if not selected: + print("no tasks selected") + return + + msg = "Try Chooser Enhanced ({} tasks selected)".format(len(selected)) + return push_to_try( + "chooser", + message.format(msg=msg), + try_task_config=generate_try_task_config( + "chooser", selected, params=try_config_params + ), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + push_to_lando=push_to_lando, + ) + + +def create_and_run_application(tg, queue: multiprocessing.Queue): + from .app import create_application + + app = create_application(tg, queue) + + app.run() diff --git a/tools/tryselect/selectors/chooser/app.py b/tools/tryselect/selectors/chooser/app.py new file mode 100644 index 0000000000..99d63cd37f --- /dev/null +++ b/tools/tryselect/selectors/chooser/app.py @@ -0,0 +1,176 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import multiprocessing +from abc import ABCMeta, abstractproperty +from collections import defaultdict + +from flask import Flask, render_template, request + +SECTIONS = [] +SUPPORTED_KINDS = set() + + +def register_section(cls): + assert issubclass(cls, Section) + instance = cls() + SECTIONS.append(instance) + SUPPORTED_KINDS.update(instance.kind.split(",")) + + +class Section(object): + __metaclass__ = ABCMeta + + @abstractproperty + def name(self): + pass + + @abstractproperty + def kind(self): + pass + + @abstractproperty + def title(self): + pass + + @abstractproperty + def attrs(self): + pass + + def contains(self, task): + return task.kind in self.kind.split(",") + + def get_context(self, tasks): + labels = defaultdict(lambda: {"max_chunk": 0, "attrs": defaultdict(list)}) + + for task in tasks.values(): + if not self.contains(task): + continue + + task = task.attributes + label = labels[self.labelfn(task)] + for attr in self.attrs: + if attr in task and task[attr] not in label["attrs"][attr]: + label["attrs"][attr].append(task[attr]) + + if "test_chunk" in task: + label["max_chunk"] = max( + label["max_chunk"], int(task["test_chunk"]) + ) + + return { + "name": self.name, + "kind": self.kind, + "title": self.title, + "labels": labels, + } + + +@register_section +class Platform(Section): + name = "platform" + kind = "build" + title = "Platforms" + attrs = ["build_platform"] + + def labelfn(self, task): + return task["build_platform"] + + def contains(self, task): + if not Section.contains(self, task): + return False + + # android-stuff tasks aren't actual platforms + return task.task["tags"].get("android-stuff", False) != "true" + + +@register_section +class Test(Section): + name = "test" + kind = "test" + title = "Test Suites" + attrs = ["unittest_suite"] + + def labelfn(self, task): + suite = task["unittest_suite"].replace(" ", "-") + + if suite.endswith("-chunked"): + suite = suite[: -len("-chunked")] + + return suite + + def contains(self, task): + if not Section.contains(self, task): + return False + return task.attributes["unittest_suite"] not in ("raptor", "talos") + + +@register_section +class Perf(Section): + name = "perf" + kind = "test" + title = "Performance" + attrs = ["unittest_suite", "raptor_try_name", "talos_try_name"] + + def labelfn(self, task): + suite = task["unittest_suite"] + label = task["{}_try_name".format(suite)] + + if not label.startswith(suite): + label = "{}-{}".format(suite, label) + + if label.endswith("-e10s"): + label = label[: -len("-e10s")] + + return label + + def contains(self, task): + if not Section.contains(self, task): + return False + return task.attributes["unittest_suite"] in ("raptor", "talos") + + +@register_section +class Analysis(Section): + name = "analysis" + kind = "build,static-analysis-autotest,hazard" + title = "Analysis" + attrs = ["build_platform"] + + def labelfn(self, task): + return task["build_platform"] + + def contains(self, task): + if not Section.contains(self, task): + return False + if task.kind == "build": + return task.task["tags"].get("android-stuff", False) == "true" + return True + + +def create_application(tg, queue: multiprocessing.Queue): + tasks = {l: t for l, t in tg.tasks.items() if t.kind in SUPPORTED_KINDS} + sections = [s.get_context(tasks) for s in SECTIONS] + context = { + "tasks": {l: t.attributes for l, t in tasks.items()}, + "sections": sections, + } + + app = Flask(__name__) + app.env = "development" + app.tasks = [] + + @app.route("/", methods=["GET", "POST"]) + def chooser(): + if request.method == "GET": + return render_template("chooser.html", **context) + + if request.form["action"] == "Push": + labels = request.form["selected-tasks"].splitlines() + app.tasks.extend(labels) + + queue.put(app.tasks) + return render_template("close.html") + + return app diff --git a/tools/tryselect/selectors/chooser/static/filter.js b/tools/tryselect/selectors/chooser/static/filter.js new file mode 100644 index 0000000000..2d8731e61f --- /dev/null +++ b/tools/tryselect/selectors/chooser/static/filter.js @@ -0,0 +1,116 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const selection = $("#selection")[0]; +const count = $("#selection-count")[0]; +const pluralize = (count, noun, suffix = "s") => + `${count} ${noun}${count !== 1 ? suffix : ""}`; + +var selected = []; + +var updateLabels = () => { + $(".tab-pane.active > .filter-label").each(function (index) { + let box = $("#" + this.htmlFor)[0]; + let method = box.checked ? "add" : "remove"; + $(this)[method + "Class"]("is-checked"); + }); +}; + +var apply = () => { + let filters = {}; + let kinds = []; + + $(".filter:checked").each(function (index) { + for (let kind of this.name.split(",")) { + if (!kinds.includes(kind)) { + kinds.push(kind); + } + } + + // Checkbox element values are generated by Section.get_context() in app.py + let attrs = JSON.parse(this.value); + for (let attr in attrs) { + if (!(attr in filters)) { + filters[attr] = []; + } + + let values = attrs[attr]; + filters[attr] = filters[attr].concat(values); + } + }); + updateLabels(); + + if ( + !Object.keys(filters).length || + (Object.keys(filters).length == 1 && "build_type" in filters) + ) { + selection.value = ""; + count.innerHTML = "0 tasks selected"; + return; + } + + var taskMatches = label => { + let task = tasks[label]; + + // If no box for the given kind has been checked, this task is + // automatically not selected. + if (!kinds.includes(task.kind)) { + return false; + } + + for (let attr in filters) { + let values = filters[attr]; + if (!(attr in task) || values.includes(task[attr])) { + continue; + } + return false; + } + return true; + }; + + selected = Object.keys(tasks).filter(taskMatches); + applyChunks(); +}; + +var applyChunks = () => { + // For tasks that have a chunk filter applied, we handle that here. + let filters = {}; + $(".filter:text").each(function (index) { + let value = $(this).val(); + if (value === "") { + return; + } + + let attrs = JSON.parse(this.name); + let key = `${attrs.unittest_suite}-${attrs.unittest_flavor}`; + if (!(key in filters)) { + filters[key] = []; + } + + // Parse the chunk strings. These are formatted like printer page setups, e.g: "1,4-6,9" + for (let item of value.split(",")) { + if (!item.includes("-")) { + filters[key].push(parseInt(item)); + continue; + } + + let [start, end] = item.split("-"); + for (let i = parseInt(start); i <= parseInt(end); ++i) { + filters[key].push(i); + } + } + }); + + let chunked = selected.filter(function (label) { + let task = tasks[label]; + let key = task.unittest_suite + "-" + task.unittest_flavor; + if (key in filters && !filters[key].includes(parseInt(task.test_chunk))) { + return false; + } + return true; + }); + + selection.value = chunked.join("\n"); + count.innerText = pluralize(chunked.length, "task") + " selected"; +}; diff --git a/tools/tryselect/selectors/chooser/static/select.js b/tools/tryselect/selectors/chooser/static/select.js new file mode 100644 index 0000000000..8a315c0a52 --- /dev/null +++ b/tools/tryselect/selectors/chooser/static/select.js @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const labels = $("label.multiselect"); +const boxes = $("label.multiselect input:checkbox"); +var lastChecked = {}; + +// implements shift+click +labels.click(function (e) { + if (e.target.tagName === "INPUT") { + return; + } + + let box = $("#" + this.htmlFor)[0]; + let activeSection = $("div.tab-pane.active")[0].id; + + if (activeSection in lastChecked) { + // Bug 559506 - In Firefox shift/ctrl/alt+clicking a label doesn't check the box. + let isFirefox = navigator.userAgent.toLowerCase().indexOf("firefox") > -1; + + if (e.shiftKey) { + if (isFirefox) { + box.checked = !box.checked; + } + + let start = boxes.index(box); + let end = boxes.index(lastChecked[activeSection]); + + boxes + .slice(Math.min(start, end), Math.max(start, end) + 1) + .prop("checked", box.checked); + apply(); + } + } + + lastChecked[activeSection] = box; +}); + +function selectAll(btn) { + let checked = !!btn.value; + $("div.active label.filter-label").each(function (index) { + $(this).find("input:checkbox")[0].checked = checked; + }); + apply(); +} diff --git a/tools/tryselect/selectors/chooser/static/style.css b/tools/tryselect/selectors/chooser/static/style.css new file mode 100644 index 0000000000..6b2f96935b --- /dev/null +++ b/tools/tryselect/selectors/chooser/static/style.css @@ -0,0 +1,107 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +body { + padding-top: 70px; +} + +/* Tabs */ + +#tabbar .nav-link { + color: #009570; + font-size: 18px; + padding-bottom: 15px; + padding-top: 15px; +} + +#tabbar .nav-link.active { + color: #212529; +} + +#tabbar .nav-link:hover { + color: #0f5a3a; +} + +/* Sections */ + +.tab-content button { + font-size: 14px; + margin-bottom: 5px; + margin-top: 10px; +} + +.filter-label { + display: block; + font-size: 16px; + position: relative; + padding-left: 15px; + padding-right: 15px; + padding-top: 10px; + padding-bottom: 10px; + margin-bottom: 0; + user-select: none; + vertical-align: middle; +} + +.filter-label span { + display: flex; + min-height: 34px; + align-items: center; + justify-content: space-between; +} + +.filter-label input[type="checkbox"] { + position: absolute; + opacity: 0; + height: 0; + width: 0; +} + +.filter-label input[type="text"] { + width: 50px; +} + +.filter-label:hover { + background-color: #91a0b0; +} + +.filter-label.is-checked:hover { + background-color: #91a0b0; +} + +.filter-label.is-checked { + background-color: #404c59; + color: white; +} + +/* Preview pane */ + +#preview { + position: fixed; + height: 100vh; + margin-left: 66%; + width: 100%; +} + +#submit-tasks { + display: flex; + flex-direction: column; + height: 80%; +} + +#buttons { + display: flex; + justify-content: space-between; +} + +#push { + background-color: #00e9b7; + margin-left: 5px; + width: 100%; +} + +#selection { + height: 100%; + width: 100%; +} diff --git a/tools/tryselect/selectors/chooser/templates/chooser.html b/tools/tryselect/selectors/chooser/templates/chooser.html new file mode 100644 index 0000000000..4e009d94ac --- /dev/null +++ b/tools/tryselect/selectors/chooser/templates/chooser.html @@ -0,0 +1,78 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this file, + - You can obtain one at http://mozilla.org/MPL/2.0/. --> + +{% extends 'layout.html' %} +{% block content %} +<div class="container-fluid"> + <div class="row"> + <div class="col-8"> + <div class="form-group form-inline"> + <span class="col-form-label col-md-2 pt-1">Build Type</span> + <div class="form-check form-check-inline"> + <input id="both" class="filter form-check-input" type="radio" name="buildtype" value='{}' onchange="apply();" checked> + <label for="both" class="form-check-label">both</label> + </div> + {% for type in ["opt", "debug"] %} + <div class="form-check form-check-inline"> + <input id="{{ type }}" class="filter form-check-input" type="radio" name="buildtype" value='{"build_type": "{{ type }}"}' onchange="apply();"> + <label for={{ type }} class="form-check-label">{{ type }}</label> + </div> + {% endfor %} + </div> + <ul class="nav nav-tabs" id="tabbar" role="tablist"> + {% for section in sections %} + <li class="nav-item"> + {% if loop.first %} + <a class="nav-link active" id="{{ section.name }}-tab" data-toggle="tab" href="#{{section.name }}" role="tab" aria-controls="{{ section.name }}" aria-selected="true">{{ section.title }}</a> + {% else %} + <a class="nav-link" id="{{ section.name }}-tab" data-toggle="tab" href="#{{section.name }}" role="tab" aria-controls="{{ section.name }}" aria-selected="false">{{ section.title }}</a> + {% endif %} + </li> + {% endfor %} + </ul> + <div class="tab-content"> + <button type="button" class="btn btn-secondary" value="true" onclick="selectAll(this);">Select All</button> + <button type="button" class="btn btn-secondary" onclick="selectAll(this);">Deselect All</button> + {% for section in sections %} + {% if loop.first %} + <div class="tab-pane show active" id="{{ section.name }}" role="tabpanel" aria-labelledby="{{ section.name }}-tab"> + {% else %} + <div class="tab-pane" id="{{ section.name }}" role="tabpanel" aria-labelledby="{{ section.name }}-tab"> + {% endif %} + {% for label, meta in section.labels|dictsort %} + <label class="multiselect filter-label" for={{ label }}> + <span> + {{ label }} + <input class="filter" type="checkbox" id={{ label }} name="{{ section.kind }}" value='{{ meta.attrs|tojson|safe }}' onchange="console.log('checkbox onchange triggered');apply();"> + {% if meta.max_chunk > 1 %} + <input class="filter" type="text" pattern="[0-9][0-9,\-]*" placeholder="1-{{ meta.max_chunk }}" name='{{ meta.attrs|tojson|safe }}' oninput="applyChunks();"> + {% endif %} + </span> + </label> + {% endfor %} + </div> + {% endfor %} + </div> + </div> + <div class="col-4" id="preview"> + <form id="submit-tasks" action="" method="POST"> + <textarea id="selection" name="selected-tasks" wrap="off"></textarea> + <span id="selection-count">0 tasks selected</span><br> + <span id="buttons"> + <input id="cancel" class="btn btn-default" type="submit" name="action" value="Cancel"> + <input id="push" class="btn btn-default" type="submit" name="action" value="Push"> + </span> + </form> + </div> + </div> +</div> +{% endblock %} + +{% block scripts %} +<script> + const tasks = {{ tasks|tojson|safe }}; +</script> +<script src="{{ url_for('static', filename='filter.js') }}"></script> +<script src="{{ url_for('static', filename='select.js') }}"></script> +{% endblock %} diff --git a/tools/tryselect/selectors/chooser/templates/close.html b/tools/tryselect/selectors/chooser/templates/close.html new file mode 100644 index 0000000000..9dc0a161f3 --- /dev/null +++ b/tools/tryselect/selectors/chooser/templates/close.html @@ -0,0 +1,11 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this file, + - You can obtain one at http://mozilla.org/MPL/2.0/. --> + +{% extends 'layout.html' %} {% block content %} +<div class="container-fluid"> + <div class="alert alert-primary" role="alert"> + You may now close this page. + </div> +</div> +{% endblock %} diff --git a/tools/tryselect/selectors/chooser/templates/layout.html b/tools/tryselect/selectors/chooser/templates/layout.html new file mode 100644 index 0000000000..8553ae94df --- /dev/null +++ b/tools/tryselect/selectors/chooser/templates/layout.html @@ -0,0 +1,71 @@ +<!-- This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this file, + - You can obtain one at http://mozilla.org/MPL/2.0/. --> + +<html> + <head> + <meta charset="utf-8" /> + <title>Try Chooser Enhanced</title> + <link + rel="stylesheet" + href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css" + /> + <link + rel="stylesheet" + href="{{ url_for('static', filename='style.css') }}" + /> + </head> + <body> + <nav class="navbar navbar-default fixed-top navbar-dark bg-dark"> + <div class="container-fluid"> + <span class="navbar-brand mb-0 h1">Try Chooser Enhanced</span> + <button + class="navbar-toggler" + type="button" + data-toggle="collapse" + data-target="#navbarSupportedContent" + aria-controls="navbarSupportedContent" + aria-expanded="false" + aria-label="Toggle navigation" + > + <span class="navbar-toggler-icon"></span> + </button> + <div class="collapse navbar-collapse" id="navbarSupportedContent"> + <ul class="navbar-nav mr-auto"> + <li class="nav-item"> + <a + class="nav-link" + href="https://firefox-source-docs.mozilla.org/tools/try/index.html" + >Documentation</a + > + </li> + <li class="nav-item"> + <a + class="nav-link" + href="https://treeherder.mozilla.org/#/jobs?repo=try" + >Treeherder</a + > + </li> + </ul> + </div> + </div> + </nav> + {% block content %}{% endblock %} + <script + src="https://code.jquery.com/jquery-3.3.1.slim.min.js" + integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" + crossorigin="anonymous" + ></script> + <script + src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" + integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" + crossorigin="anonymous" + ></script> + <script + src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" + integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" + crossorigin="anonymous" + ></script> + {% block scripts %}{% endblock %} + </body> +</html> diff --git a/tools/tryselect/selectors/compare.py b/tools/tryselect/selectors/compare.py new file mode 100644 index 0000000000..ac468e0974 --- /dev/null +++ b/tools/tryselect/selectors/compare.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from mozbuild.base import MozbuildObject +from mozversioncontrol import get_repository_object + +from tryselect.cli import BaseTryParser + +from .again import run as again_run +from .fuzzy import run as fuzzy_run + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + + +class CompareParser(BaseTryParser): + name = "compare" + arguments = [ + [ + ["-cc", "--compare-commit"], + { + "default": None, + "help": "The commit that you want to compare your current revision with", + }, + ], + ] + common_groups = ["task"] + task_configs = [ + "rebuild", + ] + + def get_revisions_to_run(vcs, compare_commit): + if compare_commit is None: + compare_commit = vcs.base_ref + if vcs.branch: + current_revision_ref = vcs.branch + else: + current_revision_ref = vcs.head_ref + + return compare_commit, current_revision_ref + + +def run(compare_commit=None, **kwargs): + vcs = get_repository_object(build.topsrcdir) + compare_commit, current_revision_ref = CompareParser.get_revisions_to_run( + vcs, compare_commit + ) + print("********************************************") + print("* 2 commits are created with this command *") + print("********************************************") + + try: + fuzzy_run(**kwargs) + print("********************************************") + print("* The base commit can be found above *") + print("********************************************") + vcs.update(compare_commit) + again_run() + print("*****************************************") + print("* The compare commit can be found above *") + print("*****************************************") + finally: + vcs.update(current_revision_ref) diff --git a/tools/tryselect/selectors/coverage.py b/tools/tryselect/selectors/coverage.py new file mode 100644 index 0000000000..f396e4618c --- /dev/null +++ b/tools/tryselect/selectors/coverage.py @@ -0,0 +1,452 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import collections +import datetime +import hashlib +import json +import os +import shutil +import sqlite3 +import subprocess + +import requests +import six +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozpack.files import FileFinder +from moztest.resolve import TestResolver +from mozversioncontrol import get_repository_object + +from ..cli import BaseTryParser +from ..push import generate_try_task_config, push_to_try +from ..tasks import filter_tasks_by_paths, generate_tasks, resolve_tests_by_suite + +here = os.path.abspath(os.path.dirname(__file__)) +build = None +vcs = None +CHUNK_MAPPING_FILE = None +CHUNK_MAPPING_TAG_FILE = None + + +def setup_globals(): + # Avoid incurring expensive computation on import. + global build, vcs, CHUNK_MAPPING_TAG_FILE, CHUNK_MAPPING_FILE + build = MozbuildObject.from_environment(cwd=here) + vcs = get_repository_object(build.topsrcdir) + + root_hash = hashlib.sha256( + six.ensure_binary(os.path.abspath(build.topsrcdir)) + ).hexdigest() + cache_dir = os.path.join(get_state_dir(), "cache", root_hash, "chunk_mapping") + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + CHUNK_MAPPING_FILE = os.path.join(cache_dir, "chunk_mapping.sqlite") + CHUNK_MAPPING_TAG_FILE = os.path.join(cache_dir, "chunk_mapping_tag.json") + + +# Maps from platform names in the chunk_mapping sqlite database to respective +# substrings in task names. +PLATFORM_MAP = { + "linux": "test-linux64/opt", + "windows": "test-windows10-64/opt", +} + +# List of platform/build type combinations that are included in pushes by |mach try coverage|. +OPT_TASK_PATTERNS = [ + "macosx64/opt", + "windows10-64/opt", + "windows7-32/opt", + "linux64/opt", +] + + +class CoverageParser(BaseTryParser): + name = "coverage" + arguments = [] + common_groups = ["push", "task"] + task_configs = [ + "artifact", + "env", + "rebuild", + "chemspill-prio", + "disable-pgo", + "worker-overrides", + ] + + +def read_test_manifests(): + """Uses TestResolver to read all test manifests in the tree. + + Returns a (tests, support_files_map) tuple that describes the tests in the tree: + tests - a set of test file paths + support_files_map - a dict that maps from each support file to a list with + test files that require them it + """ + setup_globals() + test_resolver = TestResolver.from_environment(cwd=here) + file_finder = FileFinder(build.topsrcdir) + support_files_map = collections.defaultdict(list) + tests = set() + + for test in test_resolver.resolve_tests(build.topsrcdir): + tests.add(test["srcdir_relpath"]) + if "support-files" not in test: + continue + + for support_file_pattern in test["support-files"].split(): + # Get the pattern relative to topsrcdir. + if support_file_pattern.startswith("!/"): + support_file_pattern = support_file_pattern[2:] + elif support_file_pattern.startswith("/"): + support_file_pattern = support_file_pattern[1:] + else: + support_file_pattern = os.path.normpath( + os.path.join(test["dir_relpath"], support_file_pattern) + ) + + # If it doesn't have a glob, then it's a single file. + if "*" not in support_file_pattern: + # Simple case: single support file, just add it here. + support_files_map[support_file_pattern].append(test["srcdir_relpath"]) + continue + + for support_file, _ in file_finder.find(support_file_pattern): + support_files_map[support_file].append(test["srcdir_relpath"]) + + return tests, support_files_map + + +# TODO cache the output of this function +all_tests, all_support_files = read_test_manifests() + + +def download_coverage_mapping(base_revision): + try: + with open(CHUNK_MAPPING_TAG_FILE) as f: + tags = json.load(f) + if tags["target_revision"] == base_revision: + return + else: + print("Base revision changed.") + except (OSError, ValueError): + print("Chunk mapping file not found.") + + CHUNK_MAPPING_URL_TEMPLATE = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.relman.code-coverage.production.cron.{}/artifacts/public/chunk_mapping.tar.xz" # noqa + JSON_PUSHES_URL_TEMPLATE = "https://hg.mozilla.org/mozilla-central/json-pushes?version=2&tipsonly=1&startdate={}" # noqa + + # Get pushes from at most one month ago. + PUSH_HISTORY_DAYS = 30 + delta = datetime.timedelta(days=PUSH_HISTORY_DAYS) + start_time = (datetime.datetime.now() - delta).strftime("%Y-%m-%d") + pushes_url = JSON_PUSHES_URL_TEMPLATE.format(start_time) + pushes_data = requests.get(pushes_url + "&tochange={}".format(base_revision)).json() + if "error" in pushes_data: + if "unknown revision" in pushes_data["error"]: + print( + "unknown revision {}, trying with latest mozilla-central".format( + base_revision + ) + ) + pushes_data = requests.get(pushes_url).json() + + if "error" in pushes_data: + raise Exception(pushes_data["error"]) + + pushes = pushes_data["pushes"] + + print("Looking for coverage data. This might take a minute or two.") + print("Base revision:", base_revision) + for push_id in sorted(pushes.keys())[::-1]: + rev = pushes[push_id]["changesets"][0] + url = CHUNK_MAPPING_URL_TEMPLATE.format(rev) + print("push id: {},\trevision: {}".format(push_id, rev)) + + r = requests.head(url) + if not r.ok: + continue + + print("Chunk mapping found, downloading...") + r = requests.get(url, stream=True) + + CHUNK_MAPPING_ARCHIVE = os.path.join(build.topsrcdir, "chunk_mapping.tar.xz") + with open(CHUNK_MAPPING_ARCHIVE, "wb") as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + + subprocess.check_call( + [ + "tar", + "-xJf", + CHUNK_MAPPING_ARCHIVE, + "-C", + os.path.dirname(CHUNK_MAPPING_FILE), + ] + ) + os.remove(CHUNK_MAPPING_ARCHIVE) + assert os.path.isfile(CHUNK_MAPPING_FILE) + with open(CHUNK_MAPPING_TAG_FILE, "w") as f: + json.dump( + { + "target_revision": base_revision, + "chunk_mapping_revision": rev, + "download_date": start_time, + }, + f, + ) + return + raise Exception("Could not find suitable coverage data.") + + +def is_a_test(cursor, path): + """Checks the all_tests global and the chunk mapping database to see if a + given file is a test file. + """ + if path in all_tests: + return True + + cursor.execute("SELECT COUNT(*) from chunk_to_test WHERE path=?", (path,)) + if cursor.fetchone()[0]: + return True + + cursor.execute("SELECT COUNT(*) from file_to_test WHERE test=?", (path,)) + if cursor.fetchone()[0]: + return True + + return False + + +def tests_covering_file(cursor, path): + """Returns a set of tests that cover a given source file.""" + cursor.execute("SELECT test FROM file_to_test WHERE source=?", (path,)) + return {e[0] for e in cursor.fetchall()} + + +def tests_in_chunk(cursor, platform, chunk): + """Returns a set of tests that are contained in a given chunk.""" + cursor.execute( + "SELECT path FROM chunk_to_test WHERE platform=? AND chunk=?", (platform, chunk) + ) + # Because of bug 1480103, some entries in this table contain both a file name and a test name, + # separated by a space. With the split, only the file name is kept. + return {e[0].split(" ")[0] for e in cursor.fetchall()} + + +def chunks_covering_file(cursor, path): + """Returns a set of (platform, chunk) tuples with the chunks that cover a given source file.""" + cursor.execute("SELECT platform, chunk FROM file_to_chunk WHERE path=?", (path,)) + return set(cursor.fetchall()) + + +def tests_supported_by_file(path): + """Returns a set of tests that are using the given file as a support-file.""" + return set(all_support_files[path]) + + +def find_tests(changed_files): + """Finds both individual tests and test chunks that should be run to test code changes. + Argument: a list of file paths relative to the source checkout. + + Returns: a (test_files, test_chunks) tuple with two sets. + test_files - contains tests that should be run to verify changes to changed_files. + test_chunks - contains (platform, chunk) tuples with chunks that should be + run. These chunnks do not support running a subset of the tests (like + cppunit or gtest), so the whole chunk must be run. + """ + test_files = set() + test_chunks = set() + files_no_coverage = set() + + with sqlite3.connect(CHUNK_MAPPING_FILE) as conn: + c = conn.cursor() + for path in changed_files: + # If path is a test, add it to the list and continue. + if is_a_test(c, path): + test_files.add(path) + continue + + # Look at the chunk mapping and add all tests that cover this file. + tests = tests_covering_file(c, path) + chunks = chunks_covering_file(c, path) + # If we found tests covering this, then it's not a support-file, so + # save these and continue. + if tests or chunks: + test_files |= tests + test_chunks |= chunks + continue + + # Check if the path is a support-file for any test, by querying test manifests. + tests = tests_supported_by_file(path) + if tests: + test_files |= tests + continue + + # There is no coverage information for this file. + files_no_coverage.add(path) + + files_covered = set(changed_files) - files_no_coverage + test_files = {s.replace("\\", "/") for s in test_files} + + _print_found_tests(files_covered, files_no_coverage, test_files, test_chunks) + + remaining_test_chunks = set() + # For all test_chunks, try to find the tests contained by them in the + # chunk_to_test mapping. + for platform, chunk in test_chunks: + tests = tests_in_chunk(c, platform, chunk) + if tests: + for test in tests: + test_files.add(test.replace("\\", "/")) + else: + remaining_test_chunks.add((platform, chunk)) + + return test_files, remaining_test_chunks + + +def _print_found_tests(files_covered, files_no_coverage, test_files, test_chunks): + """Print a summary of what will be run to the user's terminal.""" + files_covered = sorted(files_covered) + files_no_coverage = sorted(files_no_coverage) + test_files = sorted(test_files) + test_chunks = sorted(test_chunks) + + if files_covered: + print( + "Found {} modified source files with test coverage:".format( + len(files_covered) + ) + ) + for covered in files_covered: + print("\t", covered) + + if files_no_coverage: + print( + "Found {} modified source files with no coverage:".format( + len(files_no_coverage) + ) + ) + for f in files_no_coverage: + print("\t", f) + + if not files_covered: + print("No modified source files are covered by tests.") + elif not files_no_coverage: + print("All modified source files are covered by tests.") + + if test_files: + print("Running {} individual test files.".format(len(test_files))) + else: + print("Could not find any individual tests to run.") + + if test_chunks: + print("Running {} test chunks.".format(len(test_chunks))) + for platform, chunk in test_chunks: + print("\t", platform, chunk) + else: + print("Could not find any test chunks to run.") + + +def filter_tasks_by_chunks(tasks, chunks): + """Find all tasks that will run the given chunks.""" + selected_tasks = set() + for platform, chunk in chunks: + platform = PLATFORM_MAP[platform] + + selected_task = None + for task in tasks.keys(): + if not task.startswith(platform): + continue + + if not any( + task[len(platform) + 1 :].endswith(c) for c in [chunk, chunk + "-e10s"] + ): + continue + + assert ( + selected_task is None + ), "Only one task should be selected for a given platform-chunk couple ({} - {}), {} and {} were selected".format( # noqa + platform, chunk, selected_task, task + ) + selected_task = task + + if selected_task is None: + print("Warning: no task found for chunk", platform, chunk) + else: + selected_tasks.add(selected_task) + + return list(selected_tasks) + + +def is_opt_task(task): + """True if the task runs on a supported platform and build type combination. + This is used to remove -ccov/asan/pgo tasks, along with all /debug tasks. + """ + return any(platform in task for platform in OPT_TASK_PATTERNS) + + +def run( + try_config_params={}, + full=False, + parameters=None, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, + push_to_lando=False, +): + setup_globals() + download_coverage_mapping(vcs.base_ref) + + changed_sources = vcs.get_outgoing_files() + test_files, test_chunks = find_tests(changed_sources) + if not test_files and not test_chunks: + print("ERROR Could not find any tests or chunks to run.") + return 1 + + tg = generate_tasks(parameters, full) + all_tasks = tg.tasks + + tasks_by_chunks = filter_tasks_by_chunks(all_tasks, test_chunks) + tasks_by_path = filter_tasks_by_paths(all_tasks, test_files) + tasks = filter(is_opt_task, set(tasks_by_path) | set(tasks_by_chunks)) + tasks = list(tasks) + + if not tasks: + print("ERROR Did not find any matching tasks after filtering.") + return 1 + test_count_message = ( + "{test_count} test file{test_plural} that " + + "cover{test_singular} these changes " + + "({task_count} task{task_plural} to be scheduled)" + ).format( + test_count=len(test_files), + test_plural="" if len(test_files) == 1 else "s", + test_singular="s" if len(test_files) == 1 else "", + task_count=len(tasks), + task_plural="" if len(tasks) == 1 else "s", + ) + print("Found " + test_count_message) + + # Set the test paths to be run by setting MOZHARNESS_TEST_PATHS. + path_env = { + "MOZHARNESS_TEST_PATHS": six.ensure_text( + json.dumps(resolve_tests_by_suite(test_files)) + ) + } + try_config_params.setdefault("try_task_config", {}).setdefault("env", {}).update( + path_env + ) + + # Build commit message. + msg = "try coverage - " + test_count_message + return push_to_try( + "coverage", + message.format(msg=msg), + try_task_config=generate_try_task_config("coverage", tasks, try_config_params), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + push_to_lando=push_to_lando, + ) diff --git a/tools/tryselect/selectors/empty.py b/tools/tryselect/selectors/empty.py new file mode 100644 index 0000000000..15a48fa5d2 --- /dev/null +++ b/tools/tryselect/selectors/empty.py @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from ..cli import BaseTryParser +from ..push import generate_try_task_config, push_to_try + + +class EmptyParser(BaseTryParser): + name = "empty" + common_groups = ["push"] + task_configs = [ + "artifact", + "browsertime", + "chemspill-prio", + "disable-pgo", + "env", + "gecko-profile", + "pernosco", + "routes", + "worker-overrides", + ] + + +def run( + message="{msg}", + try_config_params=None, + stage_changes=False, + dry_run=False, + closed_tree=False, + push_to_lando=False, +): + msg = 'No try selector specified, use "Add New Jobs" to select tasks.' + return push_to_try( + "empty", + message.format(msg=msg), + try_task_config=generate_try_task_config("empty", [], params=try_config_params), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + push_to_lando=push_to_lando, + ) diff --git a/tools/tryselect/selectors/fuzzy.py b/tools/tryselect/selectors/fuzzy.py new file mode 100644 index 0000000000..7a9bccc4b7 --- /dev/null +++ b/tools/tryselect/selectors/fuzzy.py @@ -0,0 +1,284 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import sys +from pathlib import PurePath + +from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks +from mach.util import get_state_dir + +from ..cli import BaseTryParser +from ..push import check_working_directory, generate_try_task_config, push_to_try +from ..tasks import filter_tasks_by_paths, generate_tasks +from ..util.fzf import ( + FZF_NOT_FOUND, + PREVIEW_SCRIPT, + format_header, + fzf_bootstrap, + fzf_shortcuts, + run_fzf, +) +from ..util.manage_estimates import ( + download_task_history_data, + make_trimmed_taskgraph_cache, +) + + +class FuzzyParser(BaseTryParser): + name = "fuzzy" + arguments = [ + [ + ["-q", "--query"], + { + "metavar": "STR", + "action": "append", + "default": [], + "help": "Use the given query instead of entering the selection " + "interface. Equivalent to typing <query><ctrl-a><enter> " + "from the interface. Specifying multiple times schedules " + "the union of computed tasks.", + }, + ], + [ + ["-i", "--interactive"], + { + "action": "store_true", + "default": False, + "help": "Force running fzf interactively even when using presets or " + "queries with -q/--query.", + }, + ], + [ + ["-x", "--and"], + { + "dest": "intersection", + "action": "store_true", + "default": False, + "help": "When specifying queries on the command line with -q/--query, " + "use the intersection of tasks rather than the union. This is " + "especially useful for post filtering presets.", + }, + ], + [ + ["-e", "--exact"], + { + "action": "store_true", + "default": False, + "help": "Enable exact match mode. Terms will use an exact match " + "by default, and terms prefixed with ' will become fuzzy.", + }, + ], + [ + ["-u", "--update"], + { + "action": "store_true", + "default": False, + "help": "Update fzf before running.", + }, + ], + [ + ["-s", "--show-estimates"], + { + "action": "store_true", + "default": False, + "help": "Show task duration estimates.", + }, + ], + [ + ["--disable-target-task-filter"], + { + "action": "store_true", + "default": False, + "help": "Some tasks run on mozilla-central but are filtered out " + "of the default list due to resource constraints. This flag " + "disables this filtering.", + }, + ], + [ + ["--show-chunk-numbers"], + { + "action": "store_true", + "default": False, + "help": "Chunk numbers are hidden to simplify the selection. This flag " + "makes them appear again.", + }, + ], + ] + common_groups = ["push", "task", "preset"] + task_configs = [ + "artifact", + "browsertime", + "chemspill-prio", + "disable-pgo", + "env", + "existing-tasks", + "gecko-profile", + "new-test-config", + "path", + "pernosco", + "rebuild", + "routes", + "worker-overrides", + ] + + +def run( + update=False, + query=None, + intersect_query=None, + full=False, + parameters=None, + try_config_params=None, + save_query=False, + stage_changes=False, + dry_run=False, + message="{msg}", + test_paths=None, + exact=False, + closed_tree=False, + show_estimates=False, + disable_target_task_filter=False, + push_to_lando=False, + show_chunk_numbers=False, + new_test_config=False, +): + fzf = fzf_bootstrap(update) + + if not fzf: + print(FZF_NOT_FOUND) + return 1 + + push = not stage_changes and not dry_run + check_working_directory(push) + tg = generate_tasks( + parameters, full=full, disable_target_task_filter=disable_target_task_filter + ) + all_tasks = tg.tasks + + # graph_Cache created by generate_tasks, recreate the path to that file. + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + if full: + graph_cache = os.path.join(cache_dir, "full_task_graph") + dep_cache = os.path.join(cache_dir, "full_task_dependencies") + target_set = os.path.join(cache_dir, "full_task_set") + else: + graph_cache = os.path.join(cache_dir, "target_task_graph") + dep_cache = os.path.join(cache_dir, "target_task_dependencies") + target_set = os.path.join(cache_dir, "target_task_set") + + if show_estimates: + download_task_history_data(cache_dir=cache_dir) + make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set) + + if not full and not disable_target_task_filter: + all_tasks = { + task_name: task + for task_name, task in all_tasks.items() + if filter_by_uncommon_try_tasks(task_name) + } + + if test_paths: + all_tasks = filter_tasks_by_paths(all_tasks, test_paths) + if not all_tasks: + return 1 + + key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()] + base_cmd = [ + fzf, + "-m", + "--bind", + ",".join(key_shortcuts), + "--header", + format_header(), + "--preview-window=right:30%", + "--print-query", + ] + + if show_estimates: + base_cmd.extend( + [ + "--preview", + '{} {} -g {} -s -c {} -t "{{+f}}"'.format( + str(PurePath(sys.executable)), PREVIEW_SCRIPT, dep_cache, cache_dir + ), + ] + ) + else: + base_cmd.extend( + [ + "--preview", + '{} {} -t "{{+f}}"'.format( + str(PurePath(sys.executable)), PREVIEW_SCRIPT + ), + ] + ) + + if exact: + base_cmd.append("--exact") + + selected = set() + queries = [] + + def get_tasks(query_arg=None, candidate_tasks=all_tasks): + cmd = base_cmd[:] + if query_arg and query_arg != "INTERACTIVE": + cmd.extend(["-f", query_arg]) + + if not show_chunk_numbers: + fzf_tasks = set(task.chunk_pattern for task in candidate_tasks.values()) + else: + fzf_tasks = set(candidate_tasks.keys()) + + query_str, tasks = run_fzf(cmd, sorted(fzf_tasks)) + queries.append(query_str) + return set(tasks) + + for q in query or []: + selected |= get_tasks(q) + + for q in intersect_query or []: + if not selected: + selected |= get_tasks(q) + else: + selected &= get_tasks( + q, + { + task_name: task + for task_name, task in all_tasks.items() + if task_name in selected or task.chunk_pattern in selected + }, + ) + + if not queries: + selected = get_tasks() + + if not selected: + print("no tasks selected") + return + + if save_query: + return queries + + # build commit message + msg = "Fuzzy" + args = ["query={}".format(q) for q in queries] + if test_paths: + args.append("paths={}".format(":".join(test_paths))) + if args: + msg = "{} {}".format(msg, "&".join(args)) + return push_to_try( + "fuzzy", + message.format(msg=msg), + try_task_config=generate_try_task_config( + "fuzzy", selected, params=try_config_params + ), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + push_to_lando=push_to_lando, + ) diff --git a/tools/tryselect/selectors/perf.py b/tools/tryselect/selectors/perf.py new file mode 100644 index 0000000000..3c59e5949c --- /dev/null +++ b/tools/tryselect/selectors/perf.py @@ -0,0 +1,1511 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import copy +import itertools +import json +import os +import pathlib +import shutil +import subprocess +from contextlib import redirect_stdout +from datetime import datetime, timedelta + +import requests +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozversioncontrol import get_repository_object + +from ..push import generate_try_task_config, push_to_try +from ..util.fzf import ( + FZF_NOT_FOUND, + build_base_cmd, + fzf_bootstrap, + run_fzf, + setup_tasks_for_fzf, +) +from .compare import CompareParser +from .perfselector.classification import ( + Apps, + ClassificationProvider, + Platforms, + Suites, + Variants, +) +from .perfselector.perfcomparators import get_comparator +from .perfselector.utils import LogProcessor + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) +cache_file = pathlib.Path(get_state_dir(), "try_perf_revision_cache.json") +PREVIEW_SCRIPT = pathlib.Path( + build.topsrcdir, "tools/tryselect/selectors/perf_preview.py" +) + +PERFHERDER_BASE_URL = ( + "https://treeherder.mozilla.org/perfherder/" + "compare?originalProject=try&originalRevision=%s&newProject=try&newRevision=%s" +) +PERFCOMPARE_BASE_URL = "https://beta--mozilla-perfcompare.netlify.app/compare-results?baseRev=%s&newRev=%s&baseRepo=try&newRepo=try" +TREEHERDER_TRY_BASE_URL = "https://treeherder.mozilla.org/jobs?repo=try&revision=%s" +TREEHERDER_ALERT_TASKS_URL = ( + "https://treeherder.mozilla.org/api/performance/alertsummary-tasks/?id=%s" +) + +# Prevent users from running more than 300 tests at once. It's possible, but +# it's more likely that a query is broken and is selecting far too much. +MAX_PERF_TASKS = 600 + +# Name of the base category with no variants applied to it +BASE_CATEGORY_NAME = "base" + +# Add environment variable for firefox-android integration. +# This will let us find the APK to upload automatically. However, +# the following option will need to be supplied: +# --browsertime-upload-apk firefox-android +# OR --mozperftest-upload-apk firefox-android +MOZ_FIREFOX_ANDROID_APK_OUTPUT = os.getenv("MOZ_FIREFOX_ANDROID_APK_OUTPUT", None) + + +class InvalidCategoryException(Exception): + """Thrown when a category is found to be invalid. + + See the `PerfParser.run_category_checks()` method for more info. + """ + + pass + + +class APKNotFound(Exception): + """Raised when a user-supplied path to an APK is invalid.""" + + pass + + +class InvalidRegressionDetectorQuery(Exception): + """Thrown when the detector query produces anything other than 1 task.""" + + pass + + +class PerfParser(CompareParser): + name = "perf" + common_groups = ["push", "task"] + task_configs = [ + "artifact", + "browsertime", + "disable-pgo", + "env", + "gecko-profile", + "path", + "rebuild", + ] + + provider = ClassificationProvider() + platforms = provider.platforms + apps = provider.apps + variants = provider.variants + suites = provider.suites + categories = provider.categories + + arguments = [ + [ + ["--show-all"], + { + "action": "store_true", + "default": False, + "help": "Show all available tasks.", + }, + ], + [ + ["--android"], + { + "action": "store_true", + "default": False, + "help": "Show android test categories (disabled by default).", + }, + ], + [ + # Bug 1866047 - Remove once monorepo changes are complete + ["--fenix"], + { + "action": "store_true", + "default": False, + "help": "Include Fenix in tasks to run (disabled by default). Must " + "be used in conjunction with --android. Fenix isn't built on mozilla-central " + "so we pull the APK being tested from the firefox-android project. This " + "means that the fenix APK being tested in the two pushes is the same, and " + "any local changes made won't impact it.", + }, + ], + [ + ["--chrome"], + { + "action": "store_true", + "default": False, + "help": "Show tests available for Chrome-based browsers " + "(disabled by default).", + }, + ], + [ + ["--custom-car"], + { + "action": "store_true", + "default": False, + "help": "Show tests available for Custom Chromium-as-Release (disabled by default). " + "Use with --android flag to select Custom CaR android tests (cstm-car-m)", + }, + ], + [ + ["--safari"], + { + "action": "store_true", + "default": False, + "help": "Show tests available for Safari (disabled by default).", + }, + ], + [ + ["--live-sites"], + { + "action": "store_true", + "default": False, + "help": "Run tasks with live sites (if possible). " + "You can also use the `live-sites` variant.", + }, + ], + [ + ["--profile"], + { + "action": "store_true", + "default": False, + "help": "Run tasks with profiling (if possible). " + "You can also use the `profiling` variant.", + }, + ], + [ + ["--single-run"], + { + "action": "store_true", + "default": False, + "help": "Run tasks without a comparison", + }, + ], + [ + ["-q", "--query"], + { + "type": str, + "default": None, + "help": "Query to run in either the perf-category selector, " + "or the fuzzy selector if --show-all is provided.", + }, + ], + [ + # Bug 1866047 - Remove once monorepo changes are complete + ["--browsertime-upload-apk"], + { + "type": str, + "default": None, + "help": "Path to an APK to upload. Note that this " + "will replace the APK installed in all Android Performance " + "tests. If the Activity, Binary Path, or Intents required " + "change at all relative to the existing GeckoView, and Fenix " + "tasks, then you will need to make fixes in the associated " + "taskcluster files (e.g. taskcluster/ci/test/browsertime-mobile.yml). " + "Alternatively, set MOZ_FIREFOX_ANDROID_APK_OUTPUT to a path to " + "an APK, and then run the command with --browsertime-upload-apk " + "firefox-android. This option will only copy the APK for browsertime, see " + "--mozperftest-upload-apk to upload APKs for startup tests.", + }, + ], + [ + # Bug 1866047 - Remove once monorepo changes are complete + ["--mozperftest-upload-apk"], + { + "type": str, + "default": None, + "help": "See --browsertime-upload-apk. This option does the same " + "thing except it's for mozperftest tests such as the startup ones. " + "Note that those tests only exist through --show-all, as they " + "aren't contained in any existing categories.", + }, + ], + [ + ["--detect-changes"], + { + "action": "store_true", + "default": False, + "help": "Adds a task that detects performance changes using MWU.", + }, + ], + [ + ["--comparator"], + { + "type": str, + "default": "BasePerfComparator", + "help": "Either a path to a file to setup a custom comparison, " + "or a builtin name. See the Firefox source docs for mach try perf for " + "examples of how to build your own, along with the interface.", + }, + ], + [ + ["--comparator-args"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "comparator_args", + "help": "Arguments provided to the base, and new revision setup stages " + "of the comparator.", + "metavar": "ARG=VALUE", + }, + ], + [ + ["--variants"], + { + "nargs": "*", + "type": str, + "default": [BASE_CATEGORY_NAME], + "dest": "requested_variants", + "choices": list(variants.keys()), + "help": "Select variants to display in the selector from: " + + ", ".join(list(variants.keys())), + "metavar": "", + }, + ], + [ + ["--platforms"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "requested_platforms", + "choices": list(platforms.keys()), + "help": "Select specific platforms to target. Android only " + "available with --android. Available platforms: " + + ", ".join(list(platforms.keys())), + "metavar": "", + }, + ], + [ + ["--apps"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "requested_apps", + "choices": list(apps.keys()), + "help": "Select specific applications to target from: " + + ", ".join(list(apps.keys())), + "metavar": "", + }, + ], + [ + ["--clear-cache"], + { + "action": "store_true", + "default": False, + "help": "Deletes the try_perf_revision_cache file", + }, + ], + [ + ["--alert"], + { + "type": str, + "default": None, + "help": "Run tests that produced this alert summary.", + }, + ], + [ + ["--extra-args"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "extra_args", + "help": "Set the extra args " + "(e.x, --extra-args verbose post-startup-delay=1)", + "metavar": "", + }, + ], + [ + ["--perfcompare-beta"], + { + "action": "store_true", + "default": False, + "help": "Use PerfCompare Beta instead of CompareView.", + }, + ], + ] + + def get_tasks(base_cmd, queries, query_arg=None, candidate_tasks=None): + cmd = base_cmd[:] + if query_arg: + cmd.extend(["-f", query_arg]) + + query_str, tasks = run_fzf(cmd, sorted(candidate_tasks)) + queries.append(query_str) + return set(tasks) + + def get_perf_tasks(base_cmd, all_tg_tasks, perf_categories, query=None): + # Convert the categories to tasks + selected_tasks = set() + queries = [] + + selected_categories = PerfParser.get_tasks( + base_cmd, queries, query, perf_categories + ) + + for category, category_info in perf_categories.items(): + if category not in selected_categories: + continue + print("Gathering tasks for %s category" % category) + + category_tasks = set() + for suite in PerfParser.suites: + # Either perform a query to get the tasks (recommended), or + # use a hardcoded task list + suite_queries = category_info["queries"].get(suite) + + category_suite_tasks = set() + if suite_queries: + print( + "Executing %s queries: %s" % (suite, ", ".join(suite_queries)) + ) + + for perf_query in suite_queries: + if not category_suite_tasks: + # Get all tasks selected with the first query + category_suite_tasks |= PerfParser.get_tasks( + base_cmd, queries, perf_query, all_tg_tasks + ) + else: + # Keep only those tasks that matched in all previous queries + category_suite_tasks &= PerfParser.get_tasks( + base_cmd, queries, perf_query, category_suite_tasks + ) + + if len(category_suite_tasks) == 0: + print("Failed to find any tasks for query: %s" % perf_query) + break + + if category_suite_tasks: + category_tasks |= category_suite_tasks + + if category_info["tasks"]: + category_tasks = set(category_info["tasks"]) & all_tg_tasks + if category_tasks != set(category_info["tasks"]): + print( + "Some expected tasks could not be found: %s" + % ", ".join(category_info["tasks"] - category_tasks) + ) + + if not category_tasks: + print("Could not find any tasks for category %s" % category) + else: + # Add the new tasks to the currently selected ones + selected_tasks |= category_tasks + + return selected_tasks, selected_categories, queries + + def _check_app(app, target): + """Checks if the app exists in the target.""" + if app.value in target: + return True + return False + + def _check_platform(platform, target): + """Checks if the platform, or it's type exists in the target.""" + if ( + platform.value in target + or PerfParser.platforms[platform.value]["platform"] in target + ): + return True + return False + + def _build_initial_decision_matrix(): + # Build first stage of matrix APPS X PLATFORMS + initial_decision_matrix = [] + for platform in Platforms: + platform_row = [] + for app in Apps: + if PerfParser._check_platform( + platform, PerfParser.apps[app.value]["platforms"] + ): + # This app can run on this platform + platform_row.append(True) + else: + platform_row.append(False) + initial_decision_matrix.append(platform_row) + return initial_decision_matrix + + def _build_intermediate_decision_matrix(): + # Second stage of matrix building applies the 2D matrix found above + # to each suite + initial_decision_matrix = PerfParser._build_initial_decision_matrix() + + intermediate_decision_matrix = [] + for suite in Suites: + suite_matrix = copy.deepcopy(initial_decision_matrix) + suite_info = PerfParser.suites[suite.value] + + # Restric the platforms for this suite now + for platform in Platforms: + for app in Apps: + runnable = False + if PerfParser._check_app( + app, suite_info["apps"] + ) and PerfParser._check_platform(platform, suite_info["platforms"]): + runnable = True + suite_matrix[platform][app] = ( + runnable and suite_matrix[platform][app] + ) + + intermediate_decision_matrix.append(suite_matrix) + return intermediate_decision_matrix + + def _build_variants_matrix(): + # Third stage is expanding the intermediate matrix + # across all the variants (non-expanded). Start with the + # intermediate matrix in the list since it provides our + # base case with no variants + intermediate_decision_matrix = PerfParser._build_intermediate_decision_matrix() + + variants_matrix = [] + for variant in Variants: + variant_matrix = copy.deepcopy(intermediate_decision_matrix) + + for suite in Suites: + if variant.value in PerfParser.suites[suite.value]["variants"]: + # Allow the variant through and set it's platforms and apps + # based on how it sets it -> only restrict, don't make allowances + # here + for platform in Platforms: + for app in Apps: + if not ( + PerfParser._check_platform( + platform, + PerfParser.variants[variant.value]["platforms"], + ) + and PerfParser._check_app( + app, PerfParser.variants[variant.value]["apps"] + ) + ): + variant_matrix[suite][platform][app] = False + else: + # This variant matrix needs to be completely False + variant_matrix[suite] = [ + [False] * len(platform_row) + for platform_row in variant_matrix[suite] + ] + + variants_matrix.append(variant_matrix) + + return variants_matrix, intermediate_decision_matrix + + def _build_decision_matrix(): + """Build the decision matrix. + + This method builds the decision matrix that is used + to determine what categories will be shown to the user. + This matrix has the following form (as lists): + - Variants + - Suites + - Platforms + - Apps + + Each element in the 4D Matrix is either True or False and tells us + whether the particular combination is "runnable" according to + the given specifications. This does not mean that the combination + exists, just that it's fully configured in this selector. + + The ("base",) variant combination found in the matrix has + no variants applied to it. At this stage, it's a catch-all for those + categories. The query it uses is reduced further in later stages. + """ + # Get the variants matrix (see methods above) and the intermediate decision + # matrix to act as the base category + ( + variants_matrix, + intermediate_decision_matrix, + ) = PerfParser._build_variants_matrix() + + # Get all possible combinations of the variants + expanded_variants = [ + variant_combination + for set_size in range(len(Variants) + 1) + for variant_combination in itertools.combinations(list(Variants), set_size) + ] + + # Final stage combines the intermediate matrix with the + # expanded variants and leaves a "base" category which + # doesn't have any variant specifications (it catches them all) + decision_matrix = {(BASE_CATEGORY_NAME,): intermediate_decision_matrix} + for variant_combination in expanded_variants: + expanded_variant_matrix = [] + + # Perform an AND operation on the combination of variants + # to determine where this particular combination can run + for suite in Suites: + suite_matrix = [] + suite_variants = PerfParser.suites[suite.value]["variants"] + + # Disable the variant combination if none of them + # are found in the suite + disable_variant = not any( + [variant.value in suite_variants for variant in variant_combination] + ) + + for platform in Platforms: + if disable_variant: + platform_row = [False for _ in Apps] + else: + platform_row = [ + all( + variants_matrix[variant][suite][platform][app] + for variant in variant_combination + if variant.value in suite_variants + ) + for app in Apps + ] + suite_matrix.append(platform_row) + + expanded_variant_matrix.append(suite_matrix) + decision_matrix[variant_combination] = expanded_variant_matrix + + return decision_matrix + + def _skip_with_restrictions(value, restrictions, requested=[]): + """Determines if we should skip an app, platform, or variant. + + We add base here since it's the base category variant that + would always be displayed and it won't affect the app, or + platform selections. + """ + if restrictions is not None and value not in restrictions + [ + BASE_CATEGORY_NAME + ]: + return True + if requested and value not in requested + [BASE_CATEGORY_NAME]: + return True + return False + + def build_category_matrix(**kwargs): + """Build a decision matrix for all the categories. + + It will have the form: + - Category + - Variants + - ... + """ + requested_variants = kwargs.get("requested_variants", [BASE_CATEGORY_NAME]) + requested_platforms = kwargs.get("requested_platforms", []) + requested_apps = kwargs.get("requested_apps", []) + + # Build the base decision matrix + decision_matrix = PerfParser._build_decision_matrix() + + # Here, the variants are further restricted by the category settings + # using the `_skip_with_restrictions` method. This part also handles + # explicitly requested platforms, apps, and variants. + category_decision_matrix = {} + for category, category_info in PerfParser.categories.items(): + category_matrix = copy.deepcopy(decision_matrix) + + for variant_combination, variant_matrix in decision_matrix.items(): + variant_runnable = True + if BASE_CATEGORY_NAME not in variant_combination: + # Make sure that all portions of the variant combination + # target at least one of the suites in the category + tmp_variant_combination = set( + [v.value for v in variant_combination] + ) + for suite in Suites: + if suite.value not in category_info["suites"]: + continue + tmp_variant_combination = tmp_variant_combination - set( + [ + variant.value + for variant in variant_combination + if variant.value + in PerfParser.suites[suite.value]["variants"] + ] + ) + if tmp_variant_combination: + # If it's not empty, then some variants + # are non-existent + variant_runnable = False + + for suite, platform, app in itertools.product(Suites, Platforms, Apps): + runnable = variant_runnable + + # Disable this combination if there are any variant + # restrictions for this suite, or if the user didn't request it + # (and did request some variants). The same is done below with + # the apps, and platforms. + if any( + PerfParser._skip_with_restrictions( + variant.value if not isinstance(variant, str) else variant, + category_info.get("variant-restrictions", {}).get( + suite.value, None + ), + requested_variants, + ) + for variant in variant_combination + ): + runnable = False + + if PerfParser._skip_with_restrictions( + platform.value, + category_info.get("platform-restrictions", None), + requested_platforms, + ): + runnable = False + + # If the platform is restricted, check if the appropriate + # flags were provided (or appropriate conditions hit). We do + # the same thing for apps below. + if ( + PerfParser.platforms[platform.value].get("restriction", None) + is not None + ): + runnable = runnable and PerfParser.platforms[platform.value][ + "restriction" + ](**kwargs) + + if PerfParser._skip_with_restrictions( + app.value, + category_info.get("app-restrictions", {}).get( + suite.value, None + ), + requested_apps, + ): + runnable = False + if PerfParser.apps[app.value].get("restriction", None) is not None: + runnable = runnable and PerfParser.apps[app.value][ + "restriction" + ](**kwargs) + + category_matrix[variant_combination][suite][platform][app] = ( + runnable and variant_matrix[suite][platform][app] + ) + + category_decision_matrix[category] = category_matrix + + return category_decision_matrix + + def _enable_restriction(restriction, **kwargs): + """Used to simplify checking a restriction.""" + return restriction is not None and restriction(**kwargs) + + def _category_suites(category_info): + """Returns all the suite enum entries in this category.""" + return [suite for suite in Suites if suite.value in category_info["suites"]] + + def _add_variant_queries( + category_info, variant_matrix, variant_combination, platform, queries, app=None + ): + """Used to add the variant queries to various categories.""" + for variant in variant_combination: + for suite in PerfParser._category_suites(category_info): + if (app is not None and variant_matrix[suite][platform][app]) or ( + app is None and any(variant_matrix[suite][platform]) + ): + queries[suite.value].append( + PerfParser.variants[variant.value]["query"] + ) + + def _build_categories(category, category_info, category_matrix): + """Builds the categories to display.""" + categories = {} + + for variant_combination, variant_matrix in category_matrix.items(): + base_category = BASE_CATEGORY_NAME in variant_combination + + for platform in Platforms: + if not any( + any(variant_matrix[suite][platform]) + for suite in PerfParser._category_suites(category_info) + ): + # There are no apps available on this platform in either + # of the requested suites + continue + + # This code has the effect of restricting all suites to + # a platform. This means categories with mixed suites will + # be available even if some suites will no longer run + # given this platform constraint. The reasoning for this is that + # it's unexpected to receive desktop tests when you explicitly + # request android. + platform_queries = { + suite: ( + category_info["query"][suite] + + [PerfParser.platforms[platform.value]["query"]] + ) + for suite in category_info["suites"] + } + + platform_category_name = f"{category} {platform.value}" + platform_category_info = { + "queries": platform_queries, + "tasks": category_info["tasks"], + "platform": platform, + "app": None, + "suites": category_info["suites"], + "base-category": base_category, + "base-category-name": category, + "description": category_info["description"], + } + for app in Apps: + if not any( + variant_matrix[suite][platform][app] + for suite in PerfParser._category_suites(category_info) + ): + # This app is not available on the given platform + # for any of the suites + continue + + # Add the queries for the app for any suites that need it and + # the variant queries if needed + app_queries = copy.deepcopy(platform_queries) + for suite in Suites: + if suite.value not in app_queries: + continue + app_queries[suite.value].append( + PerfParser.apps[app.value]["query"] + ) + if not base_category: + PerfParser._add_variant_queries( + category_info, + variant_matrix, + variant_combination, + platform, + app_queries, + app=app, + ) + + app_category_name = f"{platform_category_name} {app.value}" + if not base_category: + app_category_name = ( + f"{app_category_name} " + f"{'+'.join([v.value for v in variant_combination])}" + ) + categories[app_category_name] = { + "queries": app_queries, + "tasks": category_info["tasks"], + "platform": platform, + "app": app, + "suites": category_info["suites"], + "base-category": base_category, + "description": category_info["description"], + } + + if not base_category: + platform_category_name = ( + f"{platform_category_name} " + f"{'+'.join([v.value for v in variant_combination])}" + ) + PerfParser._add_variant_queries( + category_info, + variant_matrix, + variant_combination, + platform, + platform_queries, + ) + categories[platform_category_name] = platform_category_info + + return categories + + def _handle_variant_negations(category, category_info, **kwargs): + """Handle variant negations. + + The reason why we're negating variants here instead of where we add + them to the queries is because we need to iterate over all of the variants + but when we add them, we only look at the variants in the combination. It's + possible to combine these, but that increases the complexity of the code + by quite a bit so it's best to do it separately. + """ + for variant in Variants: + if category_info["base-category"] and variant.value in kwargs.get( + "requested_variants", [BASE_CATEGORY_NAME] + ): + # When some particular variant(s) are requested, and we are at a + # base category, don't negate it. Otherwise, if the variant + # wasn't requested negate it + continue + if variant.value in category: + # If this variant is in the category name, skip negations + continue + if not PerfParser._check_platform( + category_info["platform"], + PerfParser.variants[variant.value]["platforms"], + ): + # Make sure the variant applies to the platform + continue + + for suite in category_info["suites"]: + if variant.value not in PerfParser.suites[suite]["variants"]: + continue + category_info["queries"][suite].append( + PerfParser.variants[variant.value]["negation"] + ) + + def _handle_app_negations(category, category_info, **kwargs): + """Handle app negations. + + This is where the global chrome/safari negations get added. We use kwargs + along with the app restriction method to make this decision. + """ + for app in Apps: + if PerfParser.apps[app.value].get("negation", None) is None: + continue + elif any( + PerfParser.apps[app.value]["negation"] + in category_info["queries"][suite] + for suite in category_info["suites"] + ): + # Already added the negations + continue + if category_info.get("app", None) is not None: + # We only need to handle this for categories that + # don't specify an app + continue + + if PerfParser.apps[app.value].get("restriction", None) is None: + # If this app has no restriction flag, it means we should select it + # as much as possible and not negate it. However, if specific apps were requested, + # we should allow the negation to proceed since a `negation` field + # was provided (checked above), assuming this app was requested. + requested_apps = kwargs.get("requested_apps", []) + if requested_apps and app.value in requested_apps: + # Apps were requested, and this was is included + continue + elif not requested_apps: + # Apps were not requested, so we should keep this one + continue + + if PerfParser._enable_restriction( + PerfParser.apps[app.value].get("restriction", None), **kwargs + ): + continue + + for suite in category_info["suites"]: + if app.value not in PerfParser.suites[suite]["apps"]: + continue + category_info["queries"][suite].append( + PerfParser.apps[app.value]["negation"] + ) + + def _handle_negations(category, category_info, **kwargs): + """This method handles negations. + + This method should only include things that should be globally applied + to all the queries. The apps are included as chrome is negated if + --chrome isn't provided, and the variants are negated here too. + """ + PerfParser._handle_variant_negations(category, category_info, **kwargs) + PerfParser._handle_app_negations(category, category_info, **kwargs) + + def get_categories(**kwargs): + """Get the categories to be displayed. + + The categories are built using the decision matrices from `build_category_matrix`. + The methods above provide more detail on how this is done. Here, we use + this matrix to determine if we should show a category to a user. + + We also apply the negations for restricted apps/platforms and variants + at the end before displaying the categories. + """ + categories = {} + + # Setup the restrictions, and ease-of-use variants requested (if any) + for variant in Variants: + if PerfParser._enable_restriction( + PerfParser.variants[variant.value].get("restriction", None), **kwargs + ): + kwargs.setdefault("requested_variants", []).append(variant.value) + + category_decision_matrix = PerfParser.build_category_matrix(**kwargs) + + # Now produce the categories by finding all the entries that are True + for category, category_matrix in category_decision_matrix.items(): + categories.update( + PerfParser._build_categories( + category, PerfParser.categories[category], category_matrix + ) + ) + + # Handle the restricted app queries, and variant negations + for category, category_info in categories.items(): + PerfParser._handle_negations(category, category_info, **kwargs) + + return categories + + def inject_change_detector(base_cmd, all_tasks, selected_tasks): + query = "'perftest 'mwu 'detect" + mwu_task = PerfParser.get_tasks(base_cmd, [], query, all_tasks) + + if len(mwu_task) > 1 or len(mwu_task) == 0: + raise InvalidRegressionDetectorQuery( + f"Expected 1 task from change detector " + f"query, but found {len(mwu_task)}" + ) + + selected_tasks |= set(mwu_task) + + def check_cached_revision(selected_tasks, base_commit=None): + """ + If the base_commit parameter does not exist, remove expired cache data. + Cache data format: + { + base_commit[str]: [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-12", + "tasks": ["a-task"], + }, + { + "base_revision_treeherder": "999998888", + "date": "2023-03-12", + "tasks": ["b-task"], + }, + ] + } + + The list represents different pushes with different task selections. + + TODO: See if we can request additional tests on a given base revision. + + :param selected_tasks list: The list of tasks selected by the user + :param base_commit str: The base commit to search + :return: The base_revision_treeherder if found, else None + """ + today = datetime.now() + expired_date = (today - timedelta(weeks=2)).strftime("%Y-%m-%d") + today = today.strftime("%Y-%m-%d") + + if not cache_file.is_file(): + return + + with cache_file.open("r") as f: + cache_data = json.load(f) + + # Remove expired cache data + if base_commit is None: + for cached_base_commit in list(cache_data): + if not isinstance(cache_data[cached_base_commit], list): + # TODO: Remove in the future, this is for backwards-compatibility + # with the previous cache structure + cache_data.pop(cached_base_commit) + else: + # Go through the pushes, and expire any that are too old + new_pushes = [] + for push in cache_data[cached_base_commit]: + if push["date"] > expired_date: + new_pushes.append(push) + # If no pushes are left after expiration, expire the base commit + if new_pushes: + cache_data[cached_base_commit] = new_pushes + else: + cache_data.pop(cached_base_commit) + with cache_file.open("w") as f: + json.dump(cache_data, f, indent=4) + + cached_base_commit = cache_data.get(base_commit, None) + if cached_base_commit: + for push in cached_base_commit: + if set(selected_tasks) <= set(push["tasks"]): + return push["base_revision_treeherder"] + + def save_revision_treeherder(selected_tasks, base_commit, base_revision_treeherder): + """ + Save the base revision of treeherder to the cache. + See "check_cached_revision" for more information about the data structure. + + :param selected_tasks list: The list of tasks selected by the user + :param base_commit str: The base commit to save + :param base_revision_treeherder str: The base revision of treeherder to save + :return: None + """ + today = datetime.now().strftime("%Y-%m-%d") + new_revision = { + "base_revision_treeherder": base_revision_treeherder, + "date": today, + "tasks": list(selected_tasks), + } + cache_data = {} + + if cache_file.is_file(): + with cache_file.open("r") as f: + cache_data = json.load(f) + cache_data.setdefault(base_commit, []).append(new_revision) + else: + cache_data[base_commit] = [new_revision] + + with cache_file.open(mode="w") as f: + json.dump(cache_data, f, indent=4) + + def found_android_tasks(selected_tasks): + """ + Check if any of the selected tasks are android. + + :param selected_tasks list: List of tasks selected. + :return bool: True if android tasks were found, False otherwise. + """ + return any("android" in task for task in selected_tasks) + + def setup_try_config( + try_config_params, extra_args, selected_tasks, base_revision_treeherder=None + ): + """ + Setup the try config for a push. + + :param try_config_params dict: The current try config to be modified. + :param extra_args list: A list of extra options to add to the tasks being run. + :param selected_tasks list: List of tasks selected. Used for determining if android + tasks are selected to disable artifact mode. + :param base_revision_treeherder str: The base revision of treeherder to save + :return: None + """ + if try_config_params is None: + try_config_params = {} + + try_config = try_config_params.setdefault("try_task_config", {}) + env = try_config.setdefault("env", {}) + if extra_args: + args = " ".join(extra_args) + env["PERF_FLAGS"] = args + if base_revision_treeherder: + # Reset updated since we no longer need to worry + # about failing while we're on a base commit + env["PERF_BASE_REVISION"] = base_revision_treeherder + if PerfParser.found_android_tasks(selected_tasks) and try_config.get( + "use-artifact-builds", False + ): + # XXX: Fix artifact mode on android (no bug) + try_config["use-artifact-builds"] = False + print("Disabling artifact mode due to android task selection") + + def perf_push_to_try( + selected_tasks, + selected_categories, + queries, + try_config_params, + dry_run, + single_run, + extra_args, + comparator, + comparator_args, + alert_summary_id, + ): + """Perf-specific push to try method. + + This makes use of logic from the CompareParser to do something + very similar except with log redirection. We get the comparison + revisions, then use the repository object to update between revisions + and the LogProcessor for parsing out the revisions that are used + to build the Perfherder links. + """ + vcs = get_repository_object(build.topsrcdir) + compare_commit, current_revision_ref = PerfParser.get_revisions_to_run( + vcs, None + ) + + # Build commit message, and limit first line to 200 characters + selected_categories_msg = ", ".join(selected_categories) + if len(selected_categories_msg) > 200: + selected_categories_msg = f"{selected_categories_msg[:200]}...\n...{selected_categories_msg[200:]}" + msg = "Perf selections={} \nQueries={}".format( + selected_categories_msg, + json.dumps(queries, indent=4), + ) + if alert_summary_id: + msg = f"Perf alert summary id={alert_summary_id}" + + # Get the comparator to run + comparator_klass = get_comparator(comparator) + comparator_obj = comparator_klass( + vcs, compare_commit, current_revision_ref, comparator_args + ) + base_comparator = True + if comparator_klass.__name__ != "BasePerfComparator": + base_comparator = False + + new_revision_treeherder = "" + base_revision_treeherder = "" + try: + # redirect_stdout allows us to feed each line into + # a processor that we can use to catch the revision + # while providing real-time output + log_processor = LogProcessor() + + # Push the base revision first. This lets the new revision appear + # first in the Treeherder view, and it also lets us enhance the new + # revision with information about the base run. + base_revision_treeherder = None + if base_comparator: + # Don't cache the base revision when a custom comparison is being performed + # since the base revision is now unique and not general to all pushes + base_revision_treeherder = PerfParser.check_cached_revision( + selected_tasks, compare_commit + ) + + if not (dry_run or single_run or base_revision_treeherder): + # Setup the base revision, and try config. This lets us change the options + # we run the tests with through the PERF_FLAGS environment variable. + base_extra_args = list(extra_args) + base_try_config_params = copy.deepcopy(try_config_params) + comparator_obj.setup_base_revision(base_extra_args) + PerfParser.setup_try_config( + base_try_config_params, base_extra_args, selected_tasks + ) + + with redirect_stdout(log_processor): + # XXX Figure out if we can use the `again` selector in some way + # Right now we would need to modify it to be able to do this. + # XXX Fix up the again selector for the perf selector (if it makes sense to) + push_to_try( + "perf-again", + "{msg}".format(msg=msg), + try_task_config=generate_try_task_config( + "fuzzy", selected_tasks, params=base_try_config_params + ), + stage_changes=False, + dry_run=dry_run, + closed_tree=False, + allow_log_capture=True, + ) + + base_revision_treeherder = log_processor.revision + if base_comparator: + PerfParser.save_revision_treeherder( + selected_tasks, compare_commit, base_revision_treeherder + ) + + comparator_obj.teardown_base_revision() + + new_extra_args = list(extra_args) + comparator_obj.setup_new_revision(new_extra_args) + PerfParser.setup_try_config( + try_config_params, + new_extra_args, + selected_tasks, + base_revision_treeherder=base_revision_treeherder, + ) + + with redirect_stdout(log_processor): + push_to_try( + "perf", + "{msg}".format(msg=msg), + # XXX Figure out if changing `fuzzy` to `perf` will break something + try_task_config=generate_try_task_config( + "fuzzy", selected_tasks, params=try_config_params + ), + stage_changes=False, + dry_run=dry_run, + closed_tree=False, + allow_log_capture=True, + ) + + new_revision_treeherder = log_processor.revision + comparator_obj.teardown_new_revision() + + finally: + comparator_obj.teardown() + + return base_revision_treeherder, new_revision_treeherder + + def run( + update=False, + show_all=False, + parameters=None, + try_config_params=None, + dry_run=False, + single_run=False, + query=None, + detect_changes=False, + rebuild=1, + clear_cache=False, + **kwargs, + ): + # Setup fzf + fzf = fzf_bootstrap(update) + + if not fzf: + print(FZF_NOT_FOUND) + return 1 + + if clear_cache: + print(f"Removing cached {cache_file} file") + cache_file.unlink(missing_ok=True) + + all_tasks, dep_cache, cache_dir = setup_tasks_for_fzf( + not dry_run, + parameters, + full=True, + disable_target_task_filter=False, + ) + base_cmd = build_base_cmd( + fzf, + dep_cache, + cache_dir, + show_estimates=False, + preview_script=PREVIEW_SCRIPT, + ) + + # Perform the selection, then push to try and return the revisions + queries = [] + selected_categories = [] + alert_summary_id = kwargs.get("alert") + if alert_summary_id: + alert_tasks = requests.get( + TREEHERDER_ALERT_TASKS_URL % alert_summary_id, + headers={"User-Agent": "mozilla-central"}, + ) + if alert_tasks.status_code != 200: + print( + "\nFailed to obtain tasks from alert due to:\n" + f"Alert ID: {alert_summary_id}\n" + f"Status Code: {alert_tasks.status_code}\n" + f"Response Message: {alert_tasks.json()}\n" + ) + alert_tasks.raise_for_status() + alert_tasks = set([task for task in alert_tasks.json()["tasks"] if task]) + selected_tasks = alert_tasks & set(all_tasks) + if not selected_tasks: + raise Exception("Alert ID has no task to run.") + elif len(selected_tasks) != len(alert_tasks): + print( + "\nAll the tasks of the Alert Summary couldn't be found in the taskgraph.\n" + f"Not exist tasks: {alert_tasks - set(all_tasks)}\n" + ) + elif not show_all: + # Expand the categories first + categories = PerfParser.get_categories(**kwargs) + PerfParser.build_category_description(base_cmd, categories) + + selected_tasks, selected_categories, queries = PerfParser.get_perf_tasks( + base_cmd, all_tasks, categories, query=query + ) + else: + selected_tasks = PerfParser.get_tasks(base_cmd, queries, query, all_tasks) + + if len(selected_tasks) == 0: + print("No tasks selected") + return None + + total_task_count = len(selected_tasks) * rebuild + if total_task_count > MAX_PERF_TASKS: + print( + "\n\n----------------------------------------------------------------------------------------------\n" + f"You have selected {total_task_count} total test runs! (selected tasks({len(selected_tasks)}) * rebuild" + f" count({rebuild}) \nThese tests won't be triggered as the current maximum for a single ./mach try " + f"perf run is {MAX_PERF_TASKS}. \nIf this was unexpected, please file a bug in Testing :: Performance." + "\n----------------------------------------------------------------------------------------------\n\n" + ) + return None + + if detect_changes: + PerfParser.inject_change_detector(base_cmd, all_tasks, selected_tasks) + + return PerfParser.perf_push_to_try( + selected_tasks, + selected_categories, + queries, + try_config_params, + dry_run, + single_run, + kwargs.get("extra_args", []), + kwargs.get("comparator", "BasePerfComparator"), + kwargs.get("comparator_args", []), + alert_summary_id, + ) + + def run_category_checks(): + # XXX: Add a jsonschema check for the category definition + # Make sure the queries don't specify variants in them + variant_queries = { + suite: [ + PerfParser.variants[variant]["query"] + for variant in suite_info.get( + "variants", list(PerfParser.variants.keys()) + ) + ] + + [ + PerfParser.variants[variant]["negation"] + for variant in suite_info.get( + "variants", list(PerfParser.variants.keys()) + ) + ] + for suite, suite_info in PerfParser.suites.items() + } + + for category, category_info in PerfParser.categories.items(): + for suite, query in category_info["query"].items(): + if len(variant_queries[suite]) == 0: + # This suite has no variants + continue + if any(any(v in q for q in query) for v in variant_queries[suite]): + raise InvalidCategoryException( + f"The '{category}' category suite query for '{suite}' " + f"uses a variant in it's query '{query}'." + "If you don't want a particular variant use the " + "`variant-restrictions` field in the category." + ) + + return True + + def setup_apk_upload(framework, apk_upload_path): + """Setup the APK for uploading to test on try. + + There are two ways of performing the upload: + (1) Passing a path to an APK with: + --browsertime-upload-apk <PATH/FILE.APK> + --mozperftest-upload-apk <PATH/FILE.APK> + (2) Setting MOZ_FIREFOX_ANDROID_APK_OUTPUT to a path that will + always point to an APK (<PATH/FILE.APK>) that we can upload. + + The file is always copied to testing/raptor/raptor/user_upload.apk to + integrate with minimal changes for simpler cases when using raptor-browsertime. + + For mozperftest, the APK is always uploaded here for the same reasons: + python/mozperftest/mozperftest/user_upload.apk + """ + frameworks_to_locations = { + "browsertime": pathlib.Path( + build.topsrcdir, "testing", "raptor", "raptor", "user_upload.apk" + ), + "mozperftest": pathlib.Path( + build.topsrcdir, + "python", + "mozperftest", + "mozperftest", + "user_upload.apk", + ), + } + + print("Setting up custom APK upload") + if apk_upload_path in ("firefox-android"): + apk_upload_path = MOZ_FIREFOX_ANDROID_APK_OUTPUT + if apk_upload_path is None: + raise APKNotFound( + "MOZ_FIREFOX_ANDROID_APK_OUTPUT is not defined. It should " + "point to an APK to upload." + ) + apk_upload_path = pathlib.Path(apk_upload_path) + if not apk_upload_path.exists() or apk_upload_path.is_dir(): + raise APKNotFound( + "MOZ_FIREFOX_ANDROID_APK_OUTPUT needs to point to an APK." + ) + else: + apk_upload_path = pathlib.Path(apk_upload_path) + if not apk_upload_path.exists(): + raise APKNotFound(f"Path does not exist: {str(apk_upload_path)}") + + print("\nCopying file in-tree for upload...") + shutil.copyfile( + str(apk_upload_path), + frameworks_to_locations[framework], + ) + + hg_cmd = ["hg", "add", str(frameworks_to_locations[framework])] + print( + f"\nRunning the following hg command (RAM warnings are expected):\n" + f" {hg_cmd}" + ) + subprocess.check_output(hg_cmd) + print( + "\nAPK is setup for uploading. Please commit the changes, " + "and re-run this command. \nEnsure you supply the --android, " + "and select the correct tasks (fenix, geckoview) or use " + "--show-all for mozperftest task selection. \nFor Fenix, ensure " + "you also provide the --fenix flag." + ) + + def build_category_description(base_cmd, categories): + descriptions = {} + + for category in categories: + if categories[category].get("description"): + descriptions[category] = categories[category].get("description") + + description_file = pathlib.Path( + get_state_dir(), "try_perf_categories_info.json" + ) + with description_file.open("w") as f: + json.dump(descriptions, f, indent=4) + + preview_option = base_cmd.index("--preview") + 1 + base_cmd[preview_option] = ( + base_cmd[preview_option] + f' -d "{description_file}" -l "{{}}"' + ) + + for idx, cmd in enumerate(base_cmd): + if "--preview-window" in cmd: + base_cmd[idx] += ":wrap" + + +def get_compare_url(revisions, perfcompare_beta=False): + """Setup the comparison link.""" + if perfcompare_beta: + return PERFCOMPARE_BASE_URL % revisions + return PERFHERDER_BASE_URL % revisions + + +def run(**kwargs): + if ( + kwargs.get("browsertime_upload_apk") is not None + or kwargs.get("mozperftest_upload_apk") is not None + ): + framework = "browsertime" + upload_apk = kwargs.get("browsertime_upload_apk") + if upload_apk is None: + framework = "mozperftest" + upload_apk = kwargs.get("mozperftest_upload_apk") + + PerfParser.setup_apk_upload(framework, upload_apk) + return + + # Make sure the categories are following + # the rules we've setup + PerfParser.run_category_checks() + PerfParser.check_cached_revision([]) + + revisions = PerfParser.run( + profile=kwargs.get("try_config_params", {}) + .get("try_task_config", {}) + .get("gecko-profile", False), + rebuild=kwargs.get("try_config_params", {}) + .get("try_task_config", {}) + .get("rebuild", 1), + **kwargs, + ) + + if revisions is None: + return + + # Provide link to perfherder for comparisons now + if not kwargs.get("single_run", False): + perfcompare_url = get_compare_url( + revisions, perfcompare_beta=kwargs.get("perfcompare_beta", False) + ) + original_try_url = TREEHERDER_TRY_BASE_URL % revisions[0] + local_change_try_url = TREEHERDER_TRY_BASE_URL % revisions[1] + print( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison here " + "once the tests are complete (ensure you select the right " + "framework): %s\n" % perfcompare_url + ) + print("\n*******************************************************") + print("* 2 commits/try-runs are created... *") + print("*******************************************************") + print(f"Base revision's try run: {original_try_url}") + print(f"Local revision's try run: {local_change_try_url}\n") + print( + "If you need any help, you can find us in the #perf-help Matrix channel:\n" + "https://matrix.to/#/#perf-help:mozilla.org\n" + ) + print( + "For more information on the performance tests, see our PerfDocs here:\n" + "https://firefox-source-docs.mozilla.org/testing/perfdocs/" + ) diff --git a/tools/tryselect/selectors/perf_preview.py b/tools/tryselect/selectors/perf_preview.py new file mode 100644 index 0000000000..55219d3300 --- /dev/null +++ b/tools/tryselect/selectors/perf_preview.py @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""This script is intended to be called through fzf as a preview formatter.""" + + +import argparse +import json +import os +import pathlib +import sys + +here = os.path.abspath(os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(os.path.dirname(here), "util")) + + +def process_args(): + """Process preview arguments.""" + argparser = argparse.ArgumentParser() + argparser.add_argument( + "-t", + "--tasklist", + type=str, + default=None, + help="Path to temporary file containing the selected tasks", + ) + argparser.add_argument( + "-d", + "--description", + type=str, + default=None, + help="Path to description file containing the item description", + ) + argparser.add_argument( + "-l", + "--line", + type=str, + default=None, + help="Current line that the user is pointing", + ) + return argparser.parse_args() + + +def plain_display(taskfile, description, line): + """Original preview window display.""" + with open(taskfile) as f: + tasklist = [line.strip() for line in f] + print("\n".join(sorted(tasklist))) + + if description is None or line is None: + return + line = line.replace("'", "") + with pathlib.Path(description).open("r") as f: + description_dict = json.load(f) + if line in description_dict: + print(f"\n* Desc:\n{description_dict[line]}") + + +if __name__ == "__main__": + args = process_args() + plain_display(args.tasklist, args.description, args.line) diff --git a/tools/tryselect/selectors/perfselector/__init__.py b/tools/tryselect/selectors/perfselector/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/selectors/perfselector/classification.py b/tools/tryselect/selectors/perfselector/classification.py new file mode 100644 index 0000000000..cabf2a323e --- /dev/null +++ b/tools/tryselect/selectors/perfselector/classification.py @@ -0,0 +1,387 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import enum + + +class ClassificationEnum(enum.Enum): + """This class provides the ability to use Enums as array indices.""" + + @property + def value(self): + return self._value_["value"] + + def __index__(self): + return self._value_["index"] + + def __int__(self): + return self._value_["index"] + + +class Platforms(ClassificationEnum): + ANDROID_A51 = {"value": "android-a51", "index": 0} + ANDROID = {"value": "android", "index": 1} + WINDOWS = {"value": "windows", "index": 2} + LINUX = {"value": "linux", "index": 3} + MACOSX = {"value": "macosx", "index": 4} + DESKTOP = {"value": "desktop", "index": 5} + + +class Apps(ClassificationEnum): + FIREFOX = {"value": "firefox", "index": 0} + CHROME = {"value": "chrome", "index": 1} + CHROMIUM = {"value": "chromium", "index": 2} + GECKOVIEW = {"value": "geckoview", "index": 3} + FENIX = {"value": "fenix", "index": 4} + CHROME_M = {"value": "chrome-m", "index": 5} + SAFARI = {"value": "safari", "index": 6} + CHROMIUM_RELEASE = {"value": "custom-car", "index": 7} + CHROMIUM_RELEASE_M = {"value": "cstm-car-m", "index": 8} + + +class Suites(ClassificationEnum): + RAPTOR = {"value": "raptor", "index": 0} + TALOS = {"value": "talos", "index": 1} + AWSY = {"value": "awsy", "index": 2} + + +class Variants(ClassificationEnum): + FISSION = {"value": "fission", "index": 0} + BYTECODE_CACHED = {"value": "bytecode-cached", "index": 1} + LIVE_SITES = {"value": "live-sites", "index": 2} + PROFILING = {"value": "profiling", "index": 3} + SWR = {"value": "swr", "index": 4} + + +""" +The following methods and constants are used for restricting +certain platforms and applications such as chrome, safari, and +android tests. These all require a flag such as --android to +enable (see build_category_matrix for more info). +""" + + +def check_for_android(android=False, **kwargs): + return android + + +def check_for_fenix(fenix=False, **kwargs): + return fenix or ("fenix" in kwargs.get("requested_apps", [])) + + +def check_for_chrome(chrome=False, **kwargs): + return chrome + + +def check_for_custom_car(custom_car=False, **kwargs): + return custom_car + + +def check_for_safari(safari=False, **kwargs): + return safari + + +def check_for_live_sites(live_sites=False, **kwargs): + return live_sites + + +def check_for_profile(profile=False, **kwargs): + return profile + + +class ClassificationProvider: + @property + def platforms(self): + return { + Platforms.ANDROID_A51.value: { + "query": "'android 'a51 'shippable 'aarch64", + "restriction": check_for_android, + "platform": Platforms.ANDROID.value, + }, + Platforms.ANDROID.value: { + # The android, and android-a51 queries are expected to be the same, + # we don't want to run the tests on other mobile platforms. + "query": "'android 'a51 'shippable 'aarch64", + "restriction": check_for_android, + "platform": Platforms.ANDROID.value, + }, + Platforms.WINDOWS.value: { + "query": "!-32 'windows 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.LINUX.value: { + "query": "!clang 'linux 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.MACOSX.value: { + "query": "'osx 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.DESKTOP.value: { + "query": "!android 'shippable !-32 !clang", + "platform": Platforms.DESKTOP.value, + }, + } + + @property + def apps(self): + return { + Apps.FIREFOX.value: { + "query": "!chrom !geckoview !fenix !safari !m-car", + "platforms": [Platforms.DESKTOP.value], + }, + Apps.CHROME.value: { + "query": "'chrome", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.DESKTOP.value], + }, + Apps.CHROMIUM.value: { + "query": "'chromium", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.DESKTOP.value], + }, + Apps.GECKOVIEW.value: { + "query": "'geckoview", + "negation": "!geckoview", + "platforms": [Platforms.ANDROID.value], + }, + Apps.FENIX.value: { + "query": "'fenix", + "negation": "!fenix", + "restriction": check_for_fenix, + "platforms": [Platforms.ANDROID.value], + }, + Apps.CHROME_M.value: { + "query": "'chrome-m", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.ANDROID.value], + }, + Apps.SAFARI.value: { + "query": "'safari", + "negation": "!safari", + "restriction": check_for_safari, + "platforms": [Platforms.MACOSX.value], + }, + Apps.CHROMIUM_RELEASE.value: { + "query": "'m-car", + "negation": "!m-car", + "restriction": check_for_custom_car, + "platforms": [ + Platforms.LINUX.value, + Platforms.WINDOWS.value, + Platforms.MACOSX.value, + ], + }, + Apps.CHROMIUM_RELEASE_M.value: { + "query": "'m-car", + "negation": "!m-car", + "restriction": check_for_custom_car, + "platforms": [Platforms.ANDROID.value], + }, + } + + @property + def variants(self): + return { + Variants.FISSION.value: { + "query": "!nofis", + "negation": "'nofis", + "platforms": [Platforms.ANDROID.value], + "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value], + }, + Variants.BYTECODE_CACHED.value: { + "query": "'bytecode", + "negation": "!bytecode", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + Variants.LIVE_SITES.value: { + "query": "'live", + "negation": "!live", + "restriction": check_for_live_sites, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [ # XXX No live CaR tests + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + Apps.SAFARI.value, + ], + }, + Variants.PROFILING.value: { + "query": "'profil", + "negation": "!profil", + "restriction": check_for_profile, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value], + }, + Variants.SWR.value: { + "query": "'swr", + "negation": "!swr", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + } + + @property + def suites(self): + return { + Suites.RAPTOR.value: { + "apps": list(self.apps.keys()), + "platforms": list(self.platforms.keys()), + "variants": [ + Variants.FISSION.value, + Variants.LIVE_SITES.value, + Variants.PROFILING.value, + Variants.BYTECODE_CACHED.value, + ], + }, + Suites.TALOS.value: { + "apps": [Apps.FIREFOX.value], + "platforms": [Platforms.DESKTOP.value], + "variants": [ + Variants.PROFILING.value, + Variants.SWR.value, + ], + }, + Suites.AWSY.value: { + "apps": [Apps.FIREFOX.value], + "platforms": [Platforms.DESKTOP.value], + "variants": [], + }, + } + + """ + Here you can find the base categories that are defined for the perf + selector. The following fields are available: + * query: Set the queries to use for each suite you need. + * suites: The suites that are needed for this category. + * tasks: A hard-coded list of tasks to select. + * platforms: The platforms that it can run on. + * app-restrictions: A list of apps that the category can run. + * variant-restrictions: A list of variants available for each suite. + + Note that setting the App/Variant-Restriction fields should be used to + restrict the available apps and variants, not expand them. + """ + + @property + def categories(self): + return { + "Pageload": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6 !tp6-bench"], + }, + "suites": [Suites.RAPTOR.value], + "tasks": [], + "description": "A group of tests that measures various important pageload metrics. More information " + "can about what is exactly measured can found here:" + " https://firefox-source-docs.mozilla.org/testing/perfdocs/raptor.html#desktop", + }, + "Speedometer 3": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'speedometer3"], + }, + "variant-restrictions": {Suites.RAPTOR.value: [Variants.FISSION.value]}, + "suites": [Suites.RAPTOR.value], + "app-restrictions": {}, + "tasks": [], + "description": "A group of Speedometer3 tests on various platforms and architectures, speedometer3 is" + "currently the best benchmark we have for a baseline on real-world web performance", + }, + "Responsiveness": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'responsive"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + ], + }, + "tasks": [], + "description": "A group of tests that ensure that the interactive part of the browser stays fast and" + "responsive", + }, + "Benchmarks": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'benchmark !tp6-bench"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + "description": "A group of tests that benchmark how the browser performs in various categories. " + "More information about what exact benchmarks we run can be found here: " + "https://firefox-source-docs.mozilla.org/testing/perfdocs/raptor.html#benchmarks", + }, + "DAMP (Devtools)": { + "query": { + Suites.TALOS.value: ["'talos 'damp"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + "description": "The DAMP tests are a group of tests that measure the performance of the browsers " + "devtools under certain conditiones. More information on the DAMP tests can be found" + " here: https://firefox-source-docs.mozilla.org/devtools/tests/performance-tests" + "-damp.html#what-does-it-do", + }, + "Talos PerfTests": { + "query": { + Suites.TALOS.value: ["'talos"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + "description": "This selects all of the talos performance tests. More information can be found here: " + "https://firefox-source-docs.mozilla.org/testing/perfdocs/talos.html#test-types", + }, + "Resource Usage": { + "query": { + Suites.TALOS.value: ["'talos 'xperf | 'tp5"], + Suites.RAPTOR.value: ["'power 'osx"], + Suites.AWSY.value: ["'awsy"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value], + "platform-restrictions": [Platforms.DESKTOP.value], + "variant-restrictions": { + Suites.RAPTOR.value: [], + Suites.TALOS.value: [], + }, + "app-restrictions": { + Suites.RAPTOR.value: [Apps.FIREFOX.value], + Suites.TALOS.value: [Apps.FIREFOX.value], + }, + "tasks": [], + "description": "A group of tests that monitor resource usage of various metrics like power, CPU, and" + "memory", + }, + "Graphics, & Media Playback": { + "query": { + # XXX This might not be an exhaustive list for talos atm + Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"], + Suites.RAPTOR.value: ["'browsertime 'youtube-playback"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: [Variants.FISSION.value]}, + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + ], + }, + "tasks": [], + "description": "A group of tests that monitor key graphics and media metrics to keep the browser fast", + }, + } diff --git a/tools/tryselect/selectors/perfselector/perfcomparators.py b/tools/tryselect/selectors/perfselector/perfcomparators.py new file mode 100644 index 0000000000..fce35fe562 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/perfcomparators.py @@ -0,0 +1,258 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import importlib +import inspect +import pathlib + +BUILTIN_COMPARATORS = {} + + +class ComparatorNotFound(Exception): + """Raised when we can't find the specified comparator. + + Triggered when either the comparator name is incorrect for a builtin one, + or when a path to a specified comparator cannot be found. + """ + + pass + + +class GithubRequestFailure(Exception): + """Raised when we hit a failure during PR link parsing.""" + + pass + + +class BadComparatorArgs(Exception): + """Raised when the args given to the comparator are incorrect.""" + + pass + + +def comparator(comparator_klass): + BUILTIN_COMPARATORS[comparator_klass.__name__] = comparator_klass + return comparator_klass + + +@comparator +class BasePerfComparator: + def __init__(self, vcs, compare_commit, current_revision_ref, comparator_args): + """Initialize the standard/default settings for Comparators. + + :param vcs object: Used for updating the local repo. + :param compare_commit str: The base revision found for the local repo. + :param current_revision_ref str: The current revision of the local repo. + :param comparator_args list: List of comparator args in the format NAME=VALUE. + """ + self.vcs = vcs + self.compare_commit = compare_commit + self.current_revision_ref = current_revision_ref + self.comparator_args = comparator_args + + # Used to ensure that the local repo gets cleaned up appropriately on failures + self._updated = False + + def setup_base_revision(self, extra_args): + """Setup the base try run/revision. + + In this case, we update to the repo to the base revision and + push that to try. The extra_args can be used to set additional + arguments for Raptor (not available for other harnesses). + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + self.vcs.update(self.compare_commit) + self._updated = True + + def teardown_base_revision(self): + """Teardown the setup for the base revision.""" + if self._updated: + self.vcs.update(self.current_revision_ref) + self._updated = False + + def setup_new_revision(self, extra_args): + """Setup the new try run/revision. + + Note that the extra_args are reset between the base, and new revision runs. + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + pass + + def teardown_new_revision(self): + """Teardown the new run/revision setup.""" + pass + + def teardown(self): + """Teardown for failures. + + This method can be used for ensuring that the repo is cleaned up + when a failure is hit at any point in the process of doing the + new/base revision setups, or the pushes to try. + """ + self.teardown_base_revision() + + +def get_github_pull_request_info(link): + """Returns information about a PR link. + + This method accepts a Github link in either of these formats: + https://github.com/mozilla-mobile/firefox-android/pull/1627, + https://github.com/mozilla-mobile/firefox-android/pull/1876/commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365 #noqa + + and returns the Github link, branch, and revision of the commit. + """ + from urllib.parse import urlparse + + import requests + + # Parse the url, and get all the necessary info + parsed_url = urlparse(link) + path_parts = parsed_url.path.strip("/").split("/") + owner, repo = path_parts[0], path_parts[1] + pr_number = path_parts[-1] + + if "/pull/" not in parsed_url.path: + raise GithubRequestFailure( + f"Link for Github PR is invalid (missing /pull/): {link}" + ) + + # Get the commit being targeted in the PR + pr_commit = None + if "/commits/" in parsed_url.path: + pr_commit = path_parts[-1] + pr_number = path_parts[-3] + + # Make the request, and get the PR info, otherwise, + # raise an exception if the response code is not 200 + api_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" + response = requests.get(api_url) + if response.status_code == 200: + link_info = response.json() + return ( + link_info["head"]["repo"]["html_url"], + pr_commit if pr_commit else link_info["head"]["sha"], + link_info["head"]["ref"], + ) + + raise GithubRequestFailure( + f"The following url returned a non-200 status code: {api_url}" + ) + + +@comparator +class BenchmarkComparator(BasePerfComparator): + def _get_benchmark_info(self, arg_prefix): + # Get the flag from the comparator args + benchmark_info = {"repo": None, "branch": None, "revision": None, "link": None} + for arg in self.comparator_args: + if arg.startswith(arg_prefix): + _, settings = arg.split(arg_prefix) + setting, val = settings.split("=") + if setting not in benchmark_info: + raise BadComparatorArgs( + f"Unknown argument provided `{setting}`. Only the following " + f"are available (prefixed with `{arg_prefix}`): " + f"{list(benchmark_info.keys())}" + ) + benchmark_info[setting] = val + + # Parse the link for any required information + if benchmark_info.get("link", None) is not None: + ( + benchmark_info["repo"], + benchmark_info["revision"], + benchmark_info["branch"], + ) = get_github_pull_request_info(benchmark_info["link"]) + + return benchmark_info + + def _setup_benchmark_args(self, extra_args, benchmark_info): + # Setup the arguments for Raptor + extra_args.append(f"benchmark-repository={benchmark_info['repo']}") + extra_args.append(f"benchmark-revision={benchmark_info['revision']}") + + if benchmark_info.get("branch", None): + extra_args.append(f"benchmark-branch={benchmark_info['branch']}") + + def setup_base_revision(self, extra_args): + """Sets up the options for a base benchmark revision run. + + Checks for a `base-link` in the + command and adds the appropriate commands to the extra_args + which will be added to the PERF_FLAGS environment variable. + + If that isn't provided, then you must provide the repo, branch, + and revision directly through these (branch is optional): + + base-repo=https://github.com/mozilla-mobile/firefox-android + base-branch=main + base-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365 + + Otherwise, we'll use the default mach try perf + base behaviour. + + TODO: Get the information automatically from a commit link. Github + API doesn't provide the branch name from a link like that. + """ + base_info = self._get_benchmark_info("base-") + + # If no options were provided, use the default BasePerfComparator behaviour + if not any(v is not None for v in base_info.values()): + raise BadComparatorArgs( + f"Could not find the correct base-revision arguments in: {self.comparator_args}" + ) + + self._setup_benchmark_args(extra_args, base_info) + + def setup_new_revision(self, extra_args): + """Sets up the options for a new benchmark revision run. + + Same as `setup_base_revision`, except it uses + `new-` as the prefix instead of `base-`. + """ + new_info = self._get_benchmark_info("new-") + + # If no options were provided, use the default BasePerfComparator behaviour + if not any(v is not None for v in new_info.values()): + raise BadComparatorArgs( + f"Could not find the correct new-revision arguments in: {self.comparator_args}" + ) + + self._setup_benchmark_args(extra_args, new_info) + + +def get_comparator(comparator): + if comparator in BUILTIN_COMPARATORS: + return BUILTIN_COMPARATORS[comparator] + + file = pathlib.Path(comparator) + if not file.exists(): + raise ComparatorNotFound( + f"Expected either a path to a file containing a comparator, or a " + f"builtin comparator from this list: {BUILTIN_COMPARATORS.keys()}" + ) + + # Importing a source file directly + spec = importlib.util.spec_from_file_location(name=file.name, location=comparator) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + members = inspect.getmembers( + module, + lambda c: inspect.isclass(c) + and issubclass(c, BasePerfComparator) + and c != BasePerfComparator, + ) + + if not members: + raise ComparatorNotFound( + f"The path {comparator} was found but it was not a valid comparator. " + f"Ensure it is a subclass of BasePerfComparator and optionally contains the " + f"following methods: " + f"{', '.join(inspect.getmembers(BasePerfComparator, predicate=inspect.ismethod))}" + ) + + return members[0][-1] diff --git a/tools/tryselect/selectors/perfselector/utils.py b/tools/tryselect/selectors/perfselector/utils.py new file mode 100644 index 0000000000..105d003091 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/utils.py @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +import sys + +REVISION_MATCHER = re.compile(r"remote:.*/try/rev/([\w]*)[ \t]*$") + + +class LogProcessor: + def __init__(self): + self.buf = "" + self.stdout = sys.__stdout__ + self._revision = None + + @property + def revision(self): + return self._revision + + def write(self, buf): + while buf: + try: + newline_index = buf.index("\n") + except ValueError: + # No newline, wait for next call + self.buf += buf + break + + # Get data up to next newline and combine with previously buffered data + data = self.buf + buf[: newline_index + 1] + buf = buf[newline_index + 1 :] + + # Reset buffer then output line + self.buf = "" + if data.strip() == "": + continue + self.stdout.write(data.strip("\n") + "\n") + + # Check if a temporary commit wa created + match = REVISION_MATCHER.match(data) + if match: + # Last line found is the revision we want + self._revision = match.group(1) diff --git a/tools/tryselect/selectors/preview.py b/tools/tryselect/selectors/preview.py new file mode 100644 index 0000000000..1d232af9e0 --- /dev/null +++ b/tools/tryselect/selectors/preview.py @@ -0,0 +1,102 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""This script is intended to be called through fzf as a preview formatter.""" + + +import argparse +import os +import sys + +here = os.path.abspath(os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(os.path.dirname(here), "util")) +from estimates import duration_summary + + +def process_args(): + """Process preview arguments.""" + argparser = argparse.ArgumentParser() + argparser.add_argument( + "-s", + "--show-estimates", + action="store_true", + help="Show task duration estimates (default: False)", + ) + argparser.add_argument( + "-g", + "--graph-cache", + type=str, + default=None, + help="Filename of task graph dependencies", + ) + argparser.add_argument( + "-c", + "--cache_dir", + type=str, + default=None, + help="Path to cache directory containing task durations", + ) + argparser.add_argument( + "-t", + "--tasklist", + type=str, + default=None, + help="Path to temporary file containing the selected tasks", + ) + return argparser.parse_args() + + +def plain_display(taskfile): + """Original preview window display.""" + with open(taskfile) as f: + tasklist = [line.strip() for line in f] + print("\n".join(sorted(tasklist))) + + +def duration_display(graph_cache_file, taskfile, cache_dir): + """Preview window display with task durations + metadata.""" + with open(taskfile) as f: + tasklist = [line.strip() for line in f] + + durations = duration_summary(graph_cache_file, tasklist, cache_dir) + output = "" + max_columns = int(os.environ["FZF_PREVIEW_COLUMNS"]) + + output += "\nSelected tasks take {}\n".format(durations["selected_duration"]) + output += "+{} dependencies, total {}\n".format( + durations["dependency_count"], + durations["selected_duration"] + durations["dependency_duration"], + ) + + if durations.get("percentile"): + output += "This is in the top {}% of requests\n".format( + 100 - durations["percentile"] + ) + + output += "Estimated finish in {} at {}".format( + durations["wall_duration_seconds"], durations["eta_datetime"].strftime("%H:%M") + ) + + duration_width = 5 # show five numbers at most. + output += "{:>{width}}\n".format("Duration", width=max_columns) + for task in tasklist: + duration = durations["task_durations"].get(task, 0.0) + output += "{:{align}{width}} {:{nalign}{nwidth}}s\n".format( + task, + duration, + align="<", + width=max_columns - (duration_width + 2), # 2: space and 's' + nalign=">", + nwidth=duration_width, + ) + + print(output) + + +if __name__ == "__main__": + args = process_args() + if args.show_estimates and os.path.isdir(args.cache_dir): + duration_display(args.graph_cache, args.tasklist, args.cache_dir) + else: + plain_display(args.tasklist) diff --git a/tools/tryselect/selectors/release.py b/tools/tryselect/selectors/release.py new file mode 100644 index 0000000000..994bbe644d --- /dev/null +++ b/tools/tryselect/selectors/release.py @@ -0,0 +1,159 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os + +import attr +import yaml +from mozilla_version.gecko import FirefoxVersion + +from ..cli import BaseTryParser +from ..push import push_to_try, vcs + +TARGET_TASKS = { + "staging": "staging_release_builds", + "release-sim": "release_simulation", +} + + +def read_file(path): + with open(path) as fh: + return fh.read() + + +class ReleaseParser(BaseTryParser): + name = "release" + arguments = [ + [ + ["-v", "--version"], + { + "metavar": "STR", + "required": True, + "action": "store", + "type": FirefoxVersion.parse, + "help": "The version number to use for the staging release.", + }, + ], + [ + ["--migration"], + { + "metavar": "STR", + "action": "append", + "dest": "migrations", + "choices": [ + "central-to-beta", + "beta-to-release", + "early-to-late-beta", + "release-to-esr", + ], + "help": "Migration to run for the release (can be specified multiple times).", + }, + ], + [ + ["--no-limit-locales"], + { + "action": "store_false", + "dest": "limit_locales", + "help": "Don't build a limited number of locales in the staging release.", + }, + ], + [ + ["--tasks"], + { + "choices": TARGET_TASKS.keys(), + "default": "staging", + "help": "Which tasks to run on-push.", + }, + ], + ] + common_groups = ["push"] + task_configs = ["disable-pgo", "worker-overrides"] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.set_defaults(migrations=[]) + + +def run( + version, + migrations, + limit_locales, + tasks, + try_config_params=None, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, + push_to_lando=False, +): + app_version = attr.evolve(version, beta_number=None, is_esr=False) + + files_to_change = { + "browser/config/version.txt": "{}\n".format(app_version), + "browser/config/version_display.txt": "{}\n".format(version), + "config/milestone.txt": "{}\n".format(app_version), + } + with open("browser/config/version.txt") as f: + current_version = FirefoxVersion.parse(f.read()) + format_options = { + "current_major_version": current_version.major_number, + "next_major_version": version.major_number, + "current_weave_version": current_version.major_number + 2, + "next_weave_version": version.major_number + 2, + } + + if "beta-to-release" in migrations and "early-to-late-beta" not in migrations: + migrations.append("early-to-late-beta") + + release_type = version.version_type.name.lower() + if release_type not in ("beta", "release", "esr"): + raise Exception( + "Can't do staging release for version: {} type: {}".format( + version, version.version_type + ) + ) + elif release_type == "esr": + release_type += str(version.major_number) + task_config = {"version": 2, "parameters": try_config_params or {}} + task_config["parameters"].update( + { + "target_tasks_method": TARGET_TASKS[tasks], + "optimize_target_tasks": True, + "release_type": release_type, + } + ) + + with open(os.path.join(vcs.path, "taskcluster/ci/config.yml")) as f: + migration_configs = yaml.safe_load(f) + for migration in migrations: + migration_config = migration_configs["merge-automation"]["behaviors"][migration] + for path, from_, to in migration_config["replacements"]: + if path in files_to_change: + contents = files_to_change[path] + else: + contents = read_file(path) + from_ = from_.format(**format_options) + to = to.format(**format_options) + files_to_change[path] = contents.replace(from_, to) + + if limit_locales: + files_to_change["browser/locales/l10n-changesets.json"] = read_file( + os.path.join(vcs.path, "browser/locales/l10n-onchange-changesets.json") + ) + files_to_change["browser/locales/shipped-locales"] = "en-US\n" + read_file( + os.path.join(vcs.path, "browser/locales/onchange-locales") + ) + + msg = "staging release: {}".format(version) + return push_to_try( + "release", + message.format(msg=msg), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + try_task_config=task_config, + files_to_change=files_to_change, + push_to_lando=push_to_lando, + ) diff --git a/tools/tryselect/selectors/scriptworker.py b/tools/tryselect/selectors/scriptworker.py new file mode 100644 index 0000000000..08020390c2 --- /dev/null +++ b/tools/tryselect/selectors/scriptworker.py @@ -0,0 +1,174 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import sys + +import requests +from gecko_taskgraph.util.taskgraph import find_existing_tasks +from taskgraph.parameters import Parameters +from taskgraph.util.taskcluster import find_task_id, get_artifact, get_session + +from ..cli import BaseTryParser +from ..push import push_to_try + +TASK_TYPES = { + "linux-signing": [ + "build-signing-linux-shippable/opt", + "build-signing-linux64-shippable/opt", + "build-signing-win64-shippable/opt", + "build-signing-win32-shippable/opt", + "repackage-signing-win64-shippable/opt", + "repackage-signing-win32-shippable/opt", + "repackage-signing-msi-win32-shippable/opt", + "repackage-signing-msi-win64-shippable/opt", + "mar-signing-linux64-shippable/opt", + ], + "linux-signing-partial": ["partials-signing-linux64-shippable/opt"], + "mac-signing": ["build-signing-macosx64-shippable/opt"], + "beetmover-candidates": ["beetmover-repackage-linux64-shippable/opt"], + "bouncer-submit": ["release-bouncer-sub-firefox"], + "balrog-submit": [ + "release-balrog-submit-toplevel-firefox", + "balrog-linux64-shippable/opt", + ], + "tree": ["release-early-tagging-firefox", "release-version-bump-firefox"], +} + +RELEASE_TO_BRANCH = { + "beta": "releases/mozilla-beta", + "release": "releases/mozilla-release", +} + + +class ScriptworkerParser(BaseTryParser): + name = "scriptworker" + arguments = [ + [ + ["task_type"], + { + "choices": ["list"] + list(TASK_TYPES.keys()), + "metavar": "TASK-TYPE", + "help": "Scriptworker task types to run. (Use `list` to show possibilities)", + }, + ], + [ + ["--release-type"], + { + "choices": ["nightly"] + list(RELEASE_TO_BRANCH.keys()), + "default": "beta", + "help": "Release type to run", + }, + ], + ] + + common_groups = ["push"] + task_configs = ["worker-overrides", "routes"] + + +def get_releases(branch): + response = requests.get( + "https://shipitapi-public.services.mozilla.com/releases", + params={"product": "firefox", "branch": branch, "status": "shipped"}, + headers={"Accept": "application/json"}, + ) + response.raise_for_status() + return response.json() + + +def get_release_graph(release): + for phase in release["phases"]: + if phase["name"] in ("ship_firefox",): + return phase["actionTaskId"] + raise Exception("No ship phase.") + + +def get_nightly_graph(): + return find_task_id( + "gecko.v2.mozilla-central.latest.taskgraph.decision-nightly-desktop" + ) + + +def print_available_task_types(): + print("Available task types:") + for task_type, tasks in TASK_TYPES.items(): + print(" " * 4 + "{}:".format(task_type)) + for task in tasks: + print(" " * 8 + "- {}".format(task)) + + +def get_hg_file(parameters, path): + session = get_session() + response = session.get(parameters.file_url(path)) + response.raise_for_status() + return response.content + + +def run( + task_type, + release_type, + try_config_params=None, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, + push_to_lando=False, +): + if task_type == "list": + print_available_task_types() + sys.exit(0) + + if release_type == "nightly": + previous_graph = get_nightly_graph() + else: + release = get_releases(RELEASE_TO_BRANCH[release_type])[-1] + previous_graph = get_release_graph(release) + existing_tasks = find_existing_tasks([previous_graph]) + + previous_parameters = Parameters( + strict=False, **get_artifact(previous_graph, "public/parameters.yml") + ) + + # Copy L10n configuration from the commit the release we are using was + # based on. This *should* ensure that the chunking of L10n tasks is the + # same between graphs. + files_to_change = { + path: get_hg_file(previous_parameters, path) + for path in [ + "browser/locales/l10n-changesets.json", + "browser/locales/shipped-locales", + ] + } + + task_config = {"version": 2, "parameters": try_config_params or {}} + task_config["parameters"]["optimize_target_tasks"] = True + task_config["parameters"]["existing_tasks"] = existing_tasks + for param in ( + "app_version", + "build_number", + "next_version", + "release_history", + "release_product", + "release_type", + "version", + ): + task_config["parameters"][param] = previous_parameters[param] + + try_config = task_config["parameters"].setdefault("try_task_config", {}) + try_config["tasks"] = TASK_TYPES[task_type] + for label in try_config["tasks"]: + if label in existing_tasks: + del existing_tasks[label] + + msg = "scriptworker tests: {}".format(task_type) + return push_to_try( + "scriptworker", + message.format(msg=msg), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + try_task_config=task_config, + files_to_change=files_to_change, + push_to_lando=push_to_lando, + ) diff --git a/tools/tryselect/selectors/syntax.py b/tools/tryselect/selectors/syntax.py new file mode 100644 index 0000000000..29b80f519a --- /dev/null +++ b/tools/tryselect/selectors/syntax.py @@ -0,0 +1,708 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import re +import sys +from collections import defaultdict + +import mozpack.path as mozpath +from moztest.resolve import TestResolver + +from ..cli import BaseTryParser +from ..push import build, push_to_try + +here = os.path.abspath(os.path.dirname(__file__)) + + +class SyntaxParser(BaseTryParser): + name = "syntax" + arguments = [ + [ + ["paths"], + { + "nargs": "*", + "default": [], + "help": "Paths to search for tests to run on try.", + }, + ], + [ + ["-b", "--build"], + { + "dest": "builds", + "default": "do", + "help": "Build types to run (d for debug, o for optimized).", + }, + ], + [ + ["-p", "--platform"], + { + "dest": "platforms", + "action": "append", + "help": "Platforms to run (required if not found in the environment as " + "AUTOTRY_PLATFORM_HINT).", + }, + ], + [ + ["-u", "--unittests"], + { + "dest": "tests", + "action": "append", + "help": "Test suites to run in their entirety.", + }, + ], + [ + ["-t", "--talos"], + { + "action": "append", + "help": "Talos suites to run.", + }, + ], + [ + ["-j", "--jobs"], + { + "action": "append", + "help": "Job tasks to run.", + }, + ], + [ + ["--tag"], + { + "dest": "tags", + "action": "append", + "help": "Restrict tests to the given tag (may be specified multiple times).", + }, + ], + [ + ["--and"], + { + "action": "store_true", + "dest": "intersection", + "help": "When -u and paths are supplied run only the intersection of the " + "tests specified by the two arguments.", + }, + ], + [ + ["--no-artifact"], + { + "action": "store_true", + "help": "Disable artifact builds even if --enable-artifact-builds is set " + "in the mozconfig.", + }, + ], + [ + ["-v", "--verbose"], + { + "dest": "verbose", + "action": "store_true", + "default": False, + "help": "Print detailed information about the resulting test selection " + "and commands performed.", + }, + ], + ] + + # Arguments we will accept on the command line and pass through to try + # syntax with no further intervention. The set is taken from + # http://trychooser.pub.build.mozilla.org with a few additions. + # + # Note that the meaning of store_false and store_true arguments is + # not preserved here, as we're only using these to echo the literal + # arguments to another consumer. Specifying either store_false or + # store_true here will have an equivalent effect. + pass_through_arguments = { + "--rebuild": { + "action": "store", + "dest": "rebuild", + "help": "Re-trigger all test jobs (up to 20 times)", + }, + "--rebuild-talos": { + "action": "store", + "dest": "rebuild_talos", + "help": "Re-trigger all talos jobs", + }, + "--interactive": { + "action": "store_true", + "dest": "interactive", + "help": "Allow ssh-like access to running test containers", + }, + "--no-retry": { + "action": "store_true", + "dest": "no_retry", + "help": "Do not retrigger failed tests", + }, + "--setenv": { + "action": "append", + "dest": "setenv", + "help": "Set the corresponding variable in the test environment for " + "applicable harnesses.", + }, + "-f": { + "action": "store_true", + "dest": "failure_emails", + "help": "Request failure emails only", + }, + "--failure-emails": { + "action": "store_true", + "dest": "failure_emails", + "help": "Request failure emails only", + }, + "-e": { + "action": "store_true", + "dest": "all_emails", + "help": "Request all emails", + }, + "--all-emails": { + "action": "store_true", + "dest": "all_emails", + "help": "Request all emails", + }, + "--artifact": { + "action": "store_true", + "dest": "artifact", + "help": "Force artifact builds where possible.", + }, + "--upload-xdbs": { + "action": "store_true", + "dest": "upload_xdbs", + "help": "Upload XDB compilation db files generated by hazard build", + }, + } + task_configs = [] + + def __init__(self, *args, **kwargs): + BaseTryParser.__init__(self, *args, **kwargs) + + group = self.add_argument_group("pass-through arguments") + for arg, opts in self.pass_through_arguments.items(): + group.add_argument(arg, **opts) + + +class TryArgumentTokenizer: + symbols = [ + ("separator", ","), + ("list_start", r"\["), + ("list_end", r"\]"), + ("item", r"([^,\[\]\s][^,\[\]]+)"), + ("space", r"\s+"), + ] + token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols)) + + def tokenize(self, data): + for match in self.token_re.finditer(data): + symbol = match.lastgroup + data = match.group(symbol) + if symbol == "space": + pass + else: + yield symbol, data + + +class TryArgumentParser: + """Simple three-state parser for handling expressions + of the from "foo[sub item, another], bar,baz". This takes + input from the TryArgumentTokenizer and runs through a small + state machine, returning a dictionary of {top-level-item:[sub_items]} + i.e. the above would result in + {"foo":["sub item", "another"], "bar": [], "baz": []} + In the case of invalid input a ValueError is raised.""" + + EOF = object() + + def __init__(self): + self.reset() + + def reset(self): + self.tokens = None + self.current_item = None + self.data = {} + self.token = None + self.state = None + + def parse(self, tokens): + self.reset() + self.tokens = tokens + self.consume() + self.state = self.item_state + while self.token[0] != self.EOF: + self.state() + return self.data + + def consume(self): + try: + self.token = next(self.tokens) + except StopIteration: + self.token = (self.EOF, None) + + def expect(self, *types): + if self.token[0] not in types: + raise ValueError( + "Error parsing try string, unexpected %s" % (self.token[0]) + ) + + def item_state(self): + self.expect("item") + value = self.token[1].strip() + if value not in self.data: + self.data[value] = [] + self.current_item = value + self.consume() + if self.token[0] == "separator": + self.consume() + elif self.token[0] == "list_start": + self.consume() + self.state = self.subitem_state + elif self.token[0] == self.EOF: + pass + else: + raise ValueError + + def subitem_state(self): + self.expect("item") + value = self.token[1].strip() + self.data[self.current_item].append(value) + self.consume() + if self.token[0] == "separator": + self.consume() + elif self.token[0] == "list_end": + self.consume() + self.state = self.after_list_end_state + else: + raise ValueError + + def after_list_end_state(self): + self.expect("separator") + self.consume() + self.state = self.item_state + + +def parse_arg(arg): + tokenizer = TryArgumentTokenizer() + parser = TryArgumentParser() + return parser.parse(tokenizer.tokenize(arg)) + + +class AutoTry: + # Maps from flavors to the job names needed to run that flavour + flavor_jobs = { + "mochitest": ["mochitest-1", "mochitest-e10s-1"], + "xpcshell": ["xpcshell"], + "chrome": ["mochitest-o"], + "browser-a11y": ["mochitest-ba"], + "browser-media": ["mochitest-bmda"], + "browser-chrome": [ + "mochitest-browser-chrome-1", + "mochitest-e10s-browser-chrome-1", + "mochitest-browser-chrome-e10s-1", + ], + "devtools-chrome": [ + "mochitest-devtools-chrome-1", + "mochitest-e10s-devtools-chrome-1", + "mochitest-devtools-chrome-e10s-1", + ], + "crashtest": ["crashtest", "crashtest-e10s"], + "reftest": ["reftest", "reftest-e10s"], + "remote": ["mochitest-remote"], + "web-platform-tests": ["web-platform-tests-1"], + } + + flavor_suites = { + "mochitest": "mochitests", + "xpcshell": "xpcshell", + "chrome": "mochitest-o", + "browser-chrome": "mochitest-bc", + "browser-a11y": "mochitest-ba", + "browser-media": "mochitest-bmda", + "devtools-chrome": "mochitest-dt", + "crashtest": "crashtest", + "reftest": "reftest", + "web-platform-tests": "web-platform-tests", + } + + compiled_suites = [ + "cppunit", + "gtest", + "jittest", + ] + + common_suites = [ + "cppunit", + "crashtest", + "firefox-ui-functional", + "geckoview", + "geckoview-junit", + "gtest", + "jittest", + "jsreftest", + "marionette", + "marionette-e10s", + "mochitests", + "reftest", + "robocop", + "web-platform-tests", + "xpcshell", + ] + + def __init__(self): + self.topsrcdir = build.topsrcdir + self._resolver = None + + @property + def resolver(self): + if self._resolver is None: + self._resolver = TestResolver.from_environment(cwd=here) + return self._resolver + + @classmethod + def split_try_string(cls, data): + return re.findall(r"(?:\[.*?\]|\S)+", data) + + def paths_by_flavor(self, paths=None, tags=None): + paths_by_flavor = defaultdict(set) + + if not (paths or tags): + return dict(paths_by_flavor) + + tests = list(self.resolver.resolve_tests(paths=paths, tags=tags)) + + for t in tests: + if t["flavor"] in self.flavor_suites: + flavor = t["flavor"] + if "subsuite" in t and t["subsuite"] == "devtools": + flavor = "devtools-chrome" + + if "subsuite" in t and t["subsuite"] == "a11y": + flavor = "browser-a11y" + + if "subsuite" in t and t["subsuite"] == "media-bc": + flavor = "browser-media" + + if flavor in ["crashtest", "reftest"]: + manifest_relpath = os.path.relpath(t["manifest"], self.topsrcdir) + paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath)) + elif "dir_relpath" in t: + paths_by_flavor[flavor].add(t["dir_relpath"]) + else: + file_relpath = os.path.relpath(t["path"], self.topsrcdir) + dir_relpath = os.path.dirname(file_relpath) + paths_by_flavor[flavor].add(dir_relpath) + + for flavor, path_set in paths_by_flavor.items(): + paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths) + + return dict(paths_by_flavor) + + def deduplicate_prefixes(self, path_set, input_paths): + # Removes paths redundant to test selection in the given path set. + # If a path was passed on the commandline that is the prefix of a + # path in our set, we only need to include the specified prefix to + # run the intended tests (every test in "layout/base" will run if + # "layout" is passed to the reftest harness). + removals = set() + additions = set() + + for path in path_set: + full_path = path + while path: + path, _ = os.path.split(path) + if path in input_paths: + removals.add(full_path) + additions.add(path) + + return additions | (path_set - removals) + + def remove_duplicates(self, paths_by_flavor, tests): + rv = {} + for item in paths_by_flavor: + if self.flavor_suites[item] not in tests: + rv[item] = paths_by_flavor[item].copy() + return rv + + def calc_try_syntax( + self, + platforms, + tests, + talos, + jobs, + builds, + paths_by_flavor, + tags, + extras, + intersection, + ): + parts = ["try:"] + + if platforms: + parts.extend(["-b", builds, "-p", ",".join(platforms)]) + + suites = tests if not intersection else {} + paths = set() + for flavor, flavor_tests in paths_by_flavor.items(): + suite = self.flavor_suites[flavor] + if suite not in suites and (not intersection or suite in tests): + for job_name in self.flavor_jobs[flavor]: + for test in flavor_tests: + paths.add("{}:{}".format(flavor, test)) + suites[job_name] = tests.get(suite, []) + + # intersection implies tests are expected + if intersection and not suites: + raise ValueError("No tests found matching filters") + + if extras.get("artifact") and any([p.endswith("-nightly") for p in platforms]): + print( + 'You asked for |--artifact| but "-nightly" platforms don\'t have artifacts. ' + "Running without |--artifact| instead." + ) + del extras["artifact"] + + if extras.get("artifact"): + rejected = [] + for suite in suites.keys(): + if any([suite.startswith(c) for c in self.compiled_suites]): + rejected.append(suite) + if rejected: + raise ValueError( + "You can't run {} with " + "--artifact option.".format(", ".join(rejected)) + ) + + if extras.get("artifact") and "all" in suites.keys(): + non_compiled_suites = set(self.common_suites) - set(self.compiled_suites) + message = ( + "You asked for |-u all| with |--artifact| but compiled-code tests ({tests})" + " can't run against an artifact build. Running (-u {non_compiled_suites}) " + "instead." + ) + string_format = { + "tests": ",".join(self.compiled_suites), + "non_compiled_suites": ",".join(non_compiled_suites), + } + print(message.format(**string_format)) + del suites["all"] + suites.update({suite_name: None for suite_name in non_compiled_suites}) + + if suites: + parts.append("-u") + parts.append( + ",".join( + "{}{}".format(k, "[%s]" % ",".join(v) if v else "") + for k, v in sorted(suites.items()) + ) + ) + + if talos: + parts.append("-t") + parts.append( + ",".join( + "{}{}".format(k, "[%s]" % ",".join(v) if v else "") + for k, v in sorted(talos.items()) + ) + ) + + if jobs: + parts.append("-j") + parts.append(",".join(jobs)) + + if tags: + parts.append(" ".join("--tag %s" % t for t in tags)) + + if paths: + parts.append("--try-test-paths %s" % " ".join(sorted(paths))) + + args_by_dest = { + v["dest"]: k for k, v in SyntaxParser.pass_through_arguments.items() + } + for dest, value in extras.items(): + assert dest in args_by_dest + arg = args_by_dest[dest] + action = SyntaxParser.pass_through_arguments[arg]["action"] + if action == "store": + parts.append(arg) + parts.append(value) + if action == "append": + for e in value: + parts.append(arg) + parts.append(e) + if action in ("store_true", "store_false"): + parts.append(arg) + + return " ".join(parts) + + def normalise_list(self, items, allow_subitems=False): + rv = defaultdict(list) + for item in items: + parsed = parse_arg(item) + for key, values in parsed.items(): + rv[key].extend(values) + + if not allow_subitems: + if not all(item == [] for item in rv.values()): + raise ValueError("Unexpected subitems in argument") + return rv.keys() + else: + return rv + + def validate_args(self, **kwargs): + tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"] + if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected): + if "AUTOTRY_PLATFORM_HINT" in os.environ: + kwargs["platforms"] = [os.environ["AUTOTRY_PLATFORM_HINT"]] + elif tests_selected: + print("Must specify platform when selecting tests.") + sys.exit(1) + else: + print( + "Either platforms or jobs must be specified as an argument to autotry." + ) + sys.exit(1) + + try: + platforms = ( + self.normalise_list(kwargs["platforms"]) if kwargs["platforms"] else {} + ) + except ValueError as e: + print("Error parsing -p argument:\n%s" % e) + sys.exit(1) + + try: + tests = ( + self.normalise_list(kwargs["tests"], allow_subitems=True) + if kwargs["tests"] + else {} + ) + except ValueError as e: + print("Error parsing -u argument ({}):\n{}".format(kwargs["tests"], e)) + sys.exit(1) + + try: + talos = ( + self.normalise_list(kwargs["talos"], allow_subitems=True) + if kwargs["talos"] + else [] + ) + except ValueError as e: + print("Error parsing -t argument:\n%s" % e) + sys.exit(1) + + try: + jobs = self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {} + except ValueError as e: + print("Error parsing -j argument:\n%s" % e) + sys.exit(1) + + paths = [] + for p in kwargs["paths"]: + p = mozpath.normpath(os.path.abspath(p)) + if not (os.path.isdir(p) and p.startswith(self.topsrcdir)): + print( + 'Specified path "%s" is not a directory under the srcdir,' + " unable to specify tests outside of the srcdir" % p + ) + sys.exit(1) + if len(p) <= len(self.topsrcdir): + print( + 'Specified path "%s" is at the top of the srcdir and would' + " select all tests." % p + ) + sys.exit(1) + paths.append(os.path.relpath(p, self.topsrcdir)) + + try: + tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else [] + except ValueError as e: + print("Error parsing --tags argument:\n%s" % e) + sys.exit(1) + + extra_values = {k["dest"] for k in SyntaxParser.pass_through_arguments.values()} + extra_args = {k: v for k, v in kwargs.items() if k in extra_values and v} + + return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args + + def run(self, **kwargs): + if not any(kwargs[item] for item in ("paths", "tests", "tags")): + kwargs["paths"] = set() + kwargs["tags"] = set() + + builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args( + **kwargs + ) + + if paths or tags: + paths = [ + os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir) + for item in paths + ] + paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags) + + if not paths_by_flavor and not tests: + print( + "No tests were found when attempting to resolve paths:\n\n\t%s" + % paths + ) + sys.exit(1) + + if not kwargs["intersection"]: + paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests) + else: + paths_by_flavor = {} + + # No point in dealing with artifacts if we aren't running any builds + local_artifact_build = False + if platforms: + local_artifact_build = kwargs.get("local_artifact_build", False) + + # Add --artifact if --enable-artifact-builds is set ... + if local_artifact_build: + extra["artifact"] = True + # ... unless --no-artifact is explicitly given. + if kwargs["no_artifact"]: + if "artifact" in extra: + del extra["artifact"] + + try: + msg = self.calc_try_syntax( + platforms, + tests, + talos, + jobs, + builds, + paths_by_flavor, + tags, + extra, + kwargs["intersection"], + ) + except ValueError as e: + print(e) + sys.exit(1) + + if local_artifact_build and not kwargs["no_artifact"]: + print( + "mozconfig has --enable-artifact-builds; including " + "--artifact flag in try syntax (use --no-artifact " + "to override)" + ) + + if kwargs["verbose"] and paths_by_flavor: + print("The following tests will be selected: ") + for flavor, paths in paths_by_flavor.items(): + print("{}: {}".format(flavor, ",".join(paths))) + + if kwargs["verbose"]: + print("The following try syntax was calculated:\n%s" % msg) + + push_to_try( + "syntax", + kwargs["message"].format(msg=msg), + stage_changes=kwargs["stage_changes"], + dry_run=kwargs["dry_run"], + closed_tree=kwargs["closed_tree"], + push_to_lando=kwargs["push_to_lando"], + ) + + +def run(**kwargs): + at = AutoTry() + return at.run(**kwargs) diff --git a/tools/tryselect/task_config.py b/tools/tryselect/task_config.py new file mode 100644 index 0000000000..f7a78cbfbf --- /dev/null +++ b/tools/tryselect/task_config.py @@ -0,0 +1,642 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Templates provide a way of modifying the task definition of selected tasks. +They are added to 'try_task_config.json' and processed by the transforms. +""" + + +import json +import os +import pathlib +import subprocess +import sys +from abc import ABCMeta, abstractmethod, abstractproperty +from argparse import SUPPRESS, Action +from contextlib import contextmanager +from textwrap import dedent + +import mozpack.path as mozpath +import requests +import six +from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject +from mozversioncontrol import Repository +from taskgraph.util import taskcluster + +from .tasks import resolve_tests_by_suite +from .util.ssh import get_ssh_user + +here = pathlib.Path(__file__).parent +build = MozbuildObject.from_environment(cwd=str(here)) + + +@contextmanager +def try_config_commit(vcs: Repository, commit_message: str): + """Context manager that creates and removes a try config commit.""" + # Add the `try_task_config.json` file if it exists. + try_task_config_path = pathlib.Path(build.topsrcdir) / "try_task_config.json" + if try_task_config_path.exists(): + vcs.add_remove_files("try_task_config.json") + + try: + # Create a try config commit. + vcs.create_try_commit(commit_message) + + yield + finally: + # Revert the try config commit. + vcs.remove_current_commit() + + +class ParameterConfig: + __metaclass__ = ABCMeta + + def __init__(self): + self.dests = set() + + def add_arguments(self, parser): + for cli, kwargs in self.arguments: + action = parser.add_argument(*cli, **kwargs) + self.dests.add(action.dest) + + @abstractproperty + def arguments(self): + pass + + @abstractmethod + def get_parameters(self, **kwargs) -> dict: + pass + + def validate(self, **kwargs): + pass + + +class TryConfig(ParameterConfig): + @abstractmethod + def try_config(self, **kwargs) -> dict: + pass + + def get_parameters(self, **kwargs): + result = self.try_config(**kwargs) + if result is None: + return None + return {"try_task_config": result} + + +class Artifact(TryConfig): + arguments = [ + [ + ["--artifact"], + {"action": "store_true", "help": "Force artifact builds where possible."}, + ], + [ + ["--no-artifact"], + { + "action": "store_true", + "help": "Disable artifact builds even if being used locally.", + }, + ], + ] + + def add_arguments(self, parser): + group = parser.add_mutually_exclusive_group() + return super().add_arguments(group) + + @classmethod + def is_artifact_build(cls): + try: + return build.substs.get("MOZ_ARTIFACT_BUILDS", False) + except BuildEnvironmentNotFoundException: + return False + + def try_config(self, artifact, no_artifact, **kwargs): + if artifact: + return {"use-artifact-builds": True, "disable-pgo": True} + + if no_artifact: + return + + if self.is_artifact_build(): + print("Artifact builds enabled, pass --no-artifact to disable") + return {"use-artifact-builds": True, "disable-pgo": True} + + +class Pernosco(TryConfig): + arguments = [ + [ + ["--pernosco"], + { + "action": "store_true", + "default": None, + "help": "Opt-in to analysis by the Pernosco debugging service.", + }, + ], + [ + ["--no-pernosco"], + { + "dest": "pernosco", + "action": "store_false", + "default": None, + "help": "Opt-out of the Pernosco debugging service (if you are on the include list).", + }, + ], + ] + + def add_arguments(self, parser): + group = parser.add_mutually_exclusive_group() + return super().add_arguments(group) + + def try_config(self, pernosco, **kwargs): + if pernosco is None: + return + + if pernosco: + try: + # The Pernosco service currently requires a Mozilla e-mail address to + # log in. Prevent people with non-Mozilla addresses from using this + # flag so they don't end up consuming time and resources only to + # realize they can't actually log in and see the reports. + address = get_ssh_user() + if not address.endswith("@mozilla.com"): + print( + dedent( + """\ + Pernosco requires a Mozilla e-mail address to view its reports. Please + push to try with an @mozilla.com address to use --pernosco. + + Current user: {} + """.format( + address + ) + ) + ) + sys.exit(1) + + except (subprocess.CalledProcessError, IndexError): + print("warning: failed to detect current user for 'hg.mozilla.org'") + print("Pernosco requires a Mozilla e-mail address to view its reports.") + while True: + answer = input( + "Do you have an @mozilla.com address? [Y/n]: " + ).lower() + if answer == "n": + sys.exit(1) + elif answer == "y": + break + + return { + "env": { + "PERNOSCO": str(int(pernosco)), + } + } + + def validate(self, **kwargs): + try_config = kwargs["try_config_params"].get("try_task_config") or {} + if try_config.get("use-artifact-builds"): + print( + "Pernosco does not support artifact builds at this time. " + "Please try again with '--no-artifact'." + ) + sys.exit(1) + + +class Path(TryConfig): + arguments = [ + [ + ["paths"], + { + "nargs": "*", + "default": [], + "help": "Run tasks containing tests under the specified path(s).", + }, + ], + ] + + def try_config(self, paths, **kwargs): + if not paths: + return + + for p in paths: + if not os.path.exists(p): + print("error: '{}' is not a valid path.".format(p), file=sys.stderr) + sys.exit(1) + + paths = [ + mozpath.relpath(mozpath.join(os.getcwd(), p), build.topsrcdir) + for p in paths + ] + return { + "env": { + "MOZHARNESS_TEST_PATHS": six.ensure_text( + json.dumps(resolve_tests_by_suite(paths)) + ), + } + } + + +class Environment(TryConfig): + arguments = [ + [ + ["--env"], + { + "action": "append", + "default": None, + "help": "Set an environment variable, of the form FOO=BAR. " + "Can be passed in multiple times.", + }, + ], + ] + + def try_config(self, env, **kwargs): + if not env: + return + return { + "env": dict(e.split("=", 1) for e in env), + } + + +class ExistingTasks(ParameterConfig): + TREEHERDER_PUSH_ENDPOINT = ( + "https://treeherder.mozilla.org/api/project/try/push/?count=1&author={user}" + ) + TREEHERDER_PUSH_URL = ( + "https://treeherder.mozilla.org/jobs?repo={branch}&revision={revision}" + ) + + arguments = [ + [ + ["-E", "--use-existing-tasks"], + { + "const": "last_try_push", + "default": None, + "nargs": "?", + "help": """ + Use existing tasks from a previous push. Without args this + uses your most recent try push. You may also specify + `rev=<revision>` where <revision> is the head revision of the + try push or `task-id=<task id>` where <task id> is the Decision + task id of the push. This last method even works for non-try + branches. + """, + }, + ] + ] + + def find_decision_task(self, use_existing_tasks): + branch = "try" + if use_existing_tasks == "last_try_push": + # Use existing tasks from user's previous try push. + user = get_ssh_user() + url = self.TREEHERDER_PUSH_ENDPOINT.format(user=user) + res = requests.get(url, headers={"User-Agent": "gecko-mach-try/1.0"}) + res.raise_for_status() + data = res.json() + if data["meta"]["count"] == 0: + raise Exception(f"Could not find a try push for '{user}'!") + revision = data["results"][0]["revision"] + + elif use_existing_tasks.startswith("rev="): + revision = use_existing_tasks[len("rev=") :] + + else: + raise Exception("Unable to parse '{use_existing_tasks}'!") + + url = self.TREEHERDER_PUSH_URL.format(branch=branch, revision=revision) + print(f"Using existing tasks from: {url}") + index_path = f"gecko.v2.{branch}.revision.{revision}.taskgraph.decision" + return taskcluster.find_task_id(index_path) + + def get_parameters(self, use_existing_tasks, **kwargs): + if not use_existing_tasks: + return + + if use_existing_tasks.startswith("task-id="): + tid = use_existing_tasks[len("task-id=") :] + else: + tid = self.find_decision_task(use_existing_tasks) + + label_to_task_id = taskcluster.get_artifact(tid, "public/label-to-taskid.json") + return {"existing_tasks": label_to_task_id} + + +class RangeAction(Action): + def __init__(self, min, max, *args, **kwargs): + self.min = min + self.max = max + kwargs["metavar"] = "[{}-{}]".format(self.min, self.max) + super().__init__(*args, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + name = option_string or self.dest + if values < self.min: + parser.error("{} can not be less than {}".format(name, self.min)) + if values > self.max: + parser.error("{} can not be more than {}".format(name, self.max)) + setattr(namespace, self.dest, values) + + +class Rebuild(TryConfig): + arguments = [ + [ + ["--rebuild"], + { + "action": RangeAction, + "min": 2, + "max": 20, + "default": None, + "type": int, + "help": "Rebuild all selected tasks the specified number of times.", + }, + ], + ] + + def try_config(self, rebuild, **kwargs): + if not rebuild: + return + + if ( + not kwargs.get("new_test_config", False) + and kwargs.get("full") + and rebuild > 3 + ): + print( + "warning: limiting --rebuild to 3 when using --full. " + "Use custom push actions to add more." + ) + rebuild = 3 + + return { + "rebuild": rebuild, + } + + +class Routes(TryConfig): + arguments = [ + [ + ["--route"], + { + "action": "append", + "dest": "routes", + "help": ( + "Additional route to add to the tasks " + "(note: these will not be added to the decision task)" + ), + }, + ], + ] + + def try_config(self, routes, **kwargs): + if routes: + return { + "routes": routes, + } + + +class ChemspillPrio(TryConfig): + arguments = [ + [ + ["--chemspill-prio"], + { + "action": "store_true", + "help": "Run at a higher priority than most try jobs (chemspills only).", + }, + ], + ] + + def try_config(self, chemspill_prio, **kwargs): + if chemspill_prio: + return {"chemspill-prio": True} + + +class GeckoProfile(TryConfig): + arguments = [ + [ + ["--gecko-profile"], + { + "dest": "profile", + "action": "store_true", + "default": False, + "help": "Create and upload a gecko profile during talos/raptor tasks.", + }, + ], + [ + ["--gecko-profile-interval"], + { + "dest": "gecko_profile_interval", + "type": float, + "help": "How frequently to take samples (ms)", + }, + ], + [ + ["--gecko-profile-entries"], + { + "dest": "gecko_profile_entries", + "type": int, + "help": "How many samples to take with the profiler", + }, + ], + [ + ["--gecko-profile-features"], + { + "dest": "gecko_profile_features", + "type": str, + "default": None, + "help": "Set the features enabled for the profiler.", + }, + ], + [ + ["--gecko-profile-threads"], + { + "dest": "gecko_profile_threads", + "type": str, + "help": "Comma-separated list of threads to sample.", + }, + ], + # For backwards compatibility + [ + ["--talos-profile"], + { + "dest": "profile", + "action": "store_true", + "default": False, + "help": SUPPRESS, + }, + ], + # This is added for consistency with the 'syntax' selector + [ + ["--geckoProfile"], + { + "dest": "profile", + "action": "store_true", + "default": False, + "help": SUPPRESS, + }, + ], + ] + + def try_config( + self, + profile, + gecko_profile_interval, + gecko_profile_entries, + gecko_profile_features, + gecko_profile_threads, + **kwargs, + ): + if profile or not all( + s is None for s in (gecko_profile_features, gecko_profile_threads) + ): + cfg = { + "gecko-profile": True, + "gecko-profile-interval": gecko_profile_interval, + "gecko-profile-entries": gecko_profile_entries, + "gecko-profile-features": gecko_profile_features, + "gecko-profile-threads": gecko_profile_threads, + } + return {key: value for key, value in cfg.items() if value is not None} + + +class Browsertime(TryConfig): + arguments = [ + [ + ["--browsertime"], + { + "action": "store_true", + "help": "Use browsertime during Raptor tasks.", + }, + ], + ] + + def try_config(self, browsertime, **kwargs): + if browsertime: + return { + "browsertime": True, + } + + +class DisablePgo(TryConfig): + arguments = [ + [ + ["--disable-pgo"], + { + "action": "store_true", + "help": "Don't run PGO builds", + }, + ], + ] + + def try_config(self, disable_pgo, **kwargs): + if disable_pgo: + return { + "disable-pgo": True, + } + + +class NewConfig(TryConfig): + arguments = [ + [ + ["--new-test-config"], + { + "action": "store_true", + "help": "When a test fails (mochitest only) restart the browser and start from the next test", + }, + ], + ] + + def try_config(self, new_test_config, **kwargs): + if new_test_config: + return { + "new-test-config": True, + } + + +class WorkerOverrides(TryConfig): + arguments = [ + [ + ["--worker-override"], + { + "action": "append", + "dest": "worker_overrides", + "help": ( + "Override the worker pool used for a given taskgraph worker alias. " + "The argument should be `<alias>=<worker-pool>`. " + "Can be specified multiple times." + ), + }, + ], + [ + ["--worker-suffix"], + { + "action": "append", + "dest": "worker_suffixes", + "help": ( + "Override the worker pool used for a given taskgraph worker alias, " + "by appending a suffix to the work-pool. " + "The argument should be `<alias>=<suffix>`. " + "Can be specified multiple times." + ), + }, + ], + ] + + def try_config(self, worker_overrides, worker_suffixes, **kwargs): + from gecko_taskgraph.util.workertypes import get_worker_type + from taskgraph.config import load_graph_config + + overrides = {} + if worker_overrides: + for override in worker_overrides: + alias, worker_pool = override.split("=", 1) + if alias in overrides: + print( + "Can't override worker alias {alias} more than once. " + "Already set to use {previous}, but also asked to use {new}.".format( + alias=alias, previous=overrides[alias], new=worker_pool + ) + ) + sys.exit(1) + overrides[alias] = worker_pool + + if worker_suffixes: + root = build.topsrcdir + root = os.path.join(root, "taskcluster", "ci") + graph_config = load_graph_config(root) + for worker_suffix in worker_suffixes: + alias, suffix = worker_suffix.split("=", 1) + if alias in overrides: + print( + "Can't override worker alias {alias} more than once. " + "Already set to use {previous}, but also asked " + "to add suffix {suffix}.".format( + alias=alias, previous=overrides[alias], suffix=suffix + ) + ) + sys.exit(1) + provisioner, worker_type = get_worker_type( + graph_config, worker_type=alias, parameters={"level": "1"} + ) + overrides[alias] = "{provisioner}/{worker_type}{suffix}".format( + provisioner=provisioner, worker_type=worker_type, suffix=suffix + ) + + if overrides: + return {"worker-overrides": overrides} + + +all_task_configs = { + "artifact": Artifact, + "browsertime": Browsertime, + "chemspill-prio": ChemspillPrio, + "disable-pgo": DisablePgo, + "env": Environment, + "existing-tasks": ExistingTasks, + "gecko-profile": GeckoProfile, + "new-test-config": NewConfig, + "path": Path, + "pernosco": Pernosco, + "rebuild": Rebuild, + "routes": Routes, + "worker-overrides": WorkerOverrides, +} diff --git a/tools/tryselect/tasks.py b/tools/tryselect/tasks.py new file mode 100644 index 0000000000..fa3eebc161 --- /dev/null +++ b/tools/tryselect/tasks.py @@ -0,0 +1,209 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import re +import sys +from collections import defaultdict + +import mozpack.path as mozpath +import taskgraph +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozpack.files import FileFinder +from moztest.resolve import TestManifestLoader, TestResolver, get_suite_definition +from taskgraph.generator import TaskGraphGenerator +from taskgraph.parameters import ParameterMismatch, parameters_loader +from taskgraph.taskgraph import TaskGraph + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + +PARAMETER_MISMATCH = """ +ERROR - The parameters being used to generate tasks differ from those expected +by your working copy: + + {} + +To fix this, either rebase onto the latest mozilla-central or pass in +-p/--parameters. For more information on how to define parameters, see: +https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters +""" + + +def invalidate(cache): + try: + cmod = os.path.getmtime(cache) + except OSError as e: + # File does not exist. We catch OSError rather than use `isfile` + # because the recommended watchman hook could possibly invalidate the + # cache in-between the check to `isfile` and the call to `getmtime` + # below. + if e.errno == 2: + return + raise + + tc_dir = os.path.join(build.topsrcdir, "taskcluster") + tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir)) + + if tmod > cmod: + os.remove(cache) + + +def cache_key(attr, params, disable_target_task_filter): + key = attr + if params and params["project"] not in ("autoland", "mozilla-central"): + key += f"-{params['project']}" + + if disable_target_task_filter and "full" not in attr: + key += "-uncommon" + return key + + +def generate_tasks(params=None, full=False, disable_target_task_filter=False): + attr = "full_task_set" if full else "target_task_set" + target_tasks_method = ( + "try_select_tasks" + if not disable_target_task_filter + else "try_select_tasks_uncommon" + ) + params = parameters_loader( + params, + strict=False, + overrides={ + "try_mode": "try_select", + "target_tasks_method": target_tasks_method, + }, + ) + root = os.path.join(build.topsrcdir, "taskcluster", "ci") + taskgraph.fast = True + generator = TaskGraphGenerator(root_dir=root, parameters=params) + + def add_chunk_patterns(tg): + for task_name, task in tg.tasks.items(): + chunk_index = -1 + if task_name.endswith("-cf"): + chunk_index = -2 + + chunks = task.task.get("extra", {}).get("chunks", {}) + if isinstance(chunks, int): + task.chunk_pattern = "{}-*/{}".format( + "-".join(task_name.split("-")[:chunk_index]), chunks + ) + else: + assert isinstance(chunks, dict) + if chunks.get("total", 1) == 1: + task.chunk_pattern = task_name + else: + task.chunk_pattern = "{}-*".format( + "-".join(task_name.split("-")[:chunk_index]) + ) + return tg + + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + key = cache_key(attr, generator.parameters, disable_target_task_filter) + cache = os.path.join(cache_dir, key) + + invalidate(cache) + if os.path.isfile(cache): + with open(cache) as fh: + return add_chunk_patterns(TaskGraph.from_json(json.load(fh))[1]) + + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + + print("Task configuration changed, generating {}".format(attr.replace("_", " "))) + + cwd = os.getcwd() + os.chdir(build.topsrcdir) + + def generate(attr): + try: + tg = getattr(generator, attr) + except ParameterMismatch as e: + print(PARAMETER_MISMATCH.format(e.args[0])) + sys.exit(1) + + # write cache + key = cache_key(attr, generator.parameters, disable_target_task_filter) + with open(os.path.join(cache_dir, key), "w") as fh: + json.dump(tg.to_json(), fh) + return add_chunk_patterns(tg) + + # Cache both full_task_set and target_task_set regardless of whether or not + # --full was requested. Caching is cheap and can potentially save a lot of + # time. + tg_full = generate("full_task_set") + tg_target = generate("target_task_set") + + # discard results from these, we only need cache. + if full: + generate("full_task_graph") + generate("target_task_graph") + + os.chdir(cwd) + if full: + return tg_full + return tg_target + + +def filter_tasks_by_paths(tasks, paths): + resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) + run_suites, run_tests = resolver.resolve_metadata(paths) + flavors = {(t["flavor"], t.get("subsuite")) for t in run_tests} + + task_regexes = set() + for flavor, subsuite in flavors: + _, suite = get_suite_definition(flavor, subsuite, strict=True) + if "task_regex" not in suite: + print( + "warning: no tasks could be resolved from flavor '{}'{}".format( + flavor, " and subsuite '{}'".format(subsuite) if subsuite else "" + ) + ) + continue + + task_regexes.update(suite["task_regex"]) + + def match_task(task): + return any(re.search(pattern, task) for pattern in task_regexes) + + return { + task_name: task for task_name, task in tasks.items() if match_task(task_name) + } + + +def resolve_tests_by_suite(paths): + resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) + _, run_tests = resolver.resolve_metadata(paths) + + suite_to_tests = defaultdict(list) + + # A dictionary containing all the input paths that we haven't yet + # assigned to a specific test flavor. + remaining_paths_by_suite = defaultdict(lambda: set(paths)) + + for test in run_tests: + key, _ = get_suite_definition(test["flavor"], test.get("subsuite"), strict=True) + + test_path = test.get("srcdir_relpath") + if test_path is None: + continue + found_path = None + manifest_relpath = None + if "manifest_relpath" in test: + manifest_relpath = mozpath.normpath(test["manifest_relpath"]) + for path in remaining_paths_by_suite[key]: + if test_path.startswith(path) or manifest_relpath == path: + found_path = path + break + if found_path: + suite_to_tests[key].append(found_path) + remaining_paths_by_suite[key].remove(found_path) + + return suite_to_tests diff --git a/tools/tryselect/test/conftest.py b/tools/tryselect/test/conftest.py new file mode 100644 index 0000000000..d9cb7daee3 --- /dev/null +++ b/tools/tryselect/test/conftest.py @@ -0,0 +1,106 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from unittest.mock import MagicMock + +import pytest +import yaml +from moztest.resolve import TestResolver +from responses import RequestsMock +from taskgraph.graph import Graph +from taskgraph.task import Task +from taskgraph.taskgraph import TaskGraph +from tryselect import push + + +@pytest.fixture +def responses(): + with RequestsMock() as rsps: + yield rsps + + +@pytest.fixture +def tg(request): + if not hasattr(request.module, "TASKS"): + pytest.fail( + "'tg' fixture used from a module that didn't define the TASKS variable" + ) + + tasks = request.module.TASKS + for task in tasks: + task.setdefault("task", {}) + task["task"].setdefault("tags", {}) + + tasks = {t["label"]: Task(**t) for t in tasks} + return TaskGraph(tasks, Graph(tasks.keys(), set())) + + +@pytest.fixture +def patch_resolver(monkeypatch): + def inner(suites, tests): + def fake_test_metadata(*args, **kwargs): + return suites, tests + + monkeypatch.setattr(TestResolver, "resolve_metadata", fake_test_metadata) + + return inner + + +@pytest.fixture(autouse=True) +def patch_vcs(monkeypatch): + attrs = { + "path": push.vcs.path, + } + mock = MagicMock() + mock.configure_mock(**attrs) + monkeypatch.setattr(push, "vcs", mock) + + +@pytest.fixture(scope="session") +def run_mach(): + import mach_initialize + from tryselect.tasks import build + + mach = mach_initialize.initialize(build.topsrcdir) + + def inner(args): + return mach.run(args) + + return inner + + +def pytest_generate_tests(metafunc): + if all( + fixture in metafunc.fixturenames + for fixture in ("task_config", "args", "expected") + ): + + def load_tests(): + for task_config, tests in metafunc.module.TASK_CONFIG_TESTS.items(): + for args, expected in tests: + yield (task_config, args, expected) + + tests = list(load_tests()) + ids = ["{} {}".format(t[0], " ".join(t[1])).strip() for t in tests] + metafunc.parametrize("task_config,args,expected", tests, ids=ids) + + elif all( + fixture in metafunc.fixturenames for fixture in ("shared_name", "shared_preset") + ): + preset_path = os.path.join( + push.build.topsrcdir, "tools", "tryselect", "try_presets.yml" + ) + with open(preset_path, "r") as fh: + presets = list(yaml.safe_load(fh).items()) + + ids = [p[0] for p in presets] + + # Mark fuzzy presets on Windows xfail due to fzf not being installed. + if os.name == "nt": + for i, preset in enumerate(presets): + if preset[1]["selector"] == "fuzzy": + presets[i] = pytest.param(*preset, marks=pytest.mark.xfail) + + metafunc.parametrize("shared_name,shared_preset", presets, ids=ids) diff --git a/tools/tryselect/test/cram.toml b/tools/tryselect/test/cram.toml new file mode 100644 index 0000000000..5dd8c41b4e --- /dev/null +++ b/tools/tryselect/test/cram.toml @@ -0,0 +1,5 @@ +["test_auto.t"] +["test_empty.t"] +["test_fuzzy.t"] +["test_message.t"] +["test_preset.t"] diff --git a/tools/tryselect/test/python.toml b/tools/tryselect/test/python.toml new file mode 100644 index 0000000000..f88156f69b --- /dev/null +++ b/tools/tryselect/test/python.toml @@ -0,0 +1,31 @@ +[DEFAULT] +subsuite = "try" + +["test_again.py"] + +["test_auto.py"] + +["test_chooser.py"] + +["test_fuzzy.py"] + +["test_mozharness_integration.py"] + +["test_perf.py"] + +["test_perfcomparators.py"] + +["test_presets.py"] +# Modifies "task_duration_history.json" in .mozbuild. Since other tests depend on this file, this test +# shouldn't be run in parallel with those other tests. +sequential = true + +["test_push.py"] + +["test_release.py"] + +["test_scriptworker.py"] + +["test_task_configs.py"] + +["test_tasks.py"] diff --git a/tools/tryselect/test/setup.sh b/tools/tryselect/test/setup.sh new file mode 100644 index 0000000000..c883ec6e8a --- /dev/null +++ b/tools/tryselect/test/setup.sh @@ -0,0 +1,101 @@ +export topsrcdir=$TESTDIR/../../../ +export MOZBUILD_STATE_PATH=$TMP/mozbuild +export MACH_TRY_PRESET_PATHS=$MOZBUILD_STATE_PATH/try_presets.yml + +# This helps to find fzf when running these tests locally, since normally fzf +# would be found via MOZBUILD_STATE_PATH pointing to $HOME/.mozbuild +export PATH="$PATH:$HOME/.mozbuild/fzf/bin" + +export MACHRC=$TMP/machrc +cat > $MACHRC << EOF +[try] +default=syntax +EOF + +cmd="$topsrcdir/mach python -c 'from mach.util import get_state_dir; print(get_state_dir(specific_to_topsrcdir=True))'" +# First run local state dir generation so it doesn't affect test output. +eval $cmd > /dev/null 2>&1 +# Now run it again to get the actual directory. +cachedir=$(eval $cmd)/cache/taskgraph +mkdir -p $cachedir +# Run `mach try --help` to generate virtualenv. +eval "$topsrcdir/mach try --help" > /dev/null 2>&1 + +cat > $cachedir/target_task_set << EOF +{ + "test/foo-opt": { + "kind": "test", + "label": "test/foo-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/foo-debug": { + "kind": "test", + "label": "test/foo-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "build-baz": { + "kind": "build", + "label": "build-baz", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + } +} +EOF + +cat > $cachedir/full_task_set << EOF +{ + "test/foo-opt": { + "kind": "test", + "label": "test/foo-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/foo-debug": { + "kind": "test", + "label": "test/foo-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/bar-opt": { + "kind": "test", + "label": "test/bar-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/bar-debug": { + "kind": "test", + "label": "test/bar-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "build-baz": { + "kind": "build", + "label": "build-baz", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + } +} +EOF + +# set mtime to the future so we don't re-generate tasks +find $cachedir -type f -exec touch -d "next day" {} + + +export testargs="--no-push --no-artifact" diff --git a/tools/tryselect/test/test_again.py b/tools/tryselect/test/test_again.py new file mode 100644 index 0000000000..3c6b87cfdf --- /dev/null +++ b/tools/tryselect/test/test_again.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import mozunit +import pytest +from six.moves import reload_module as reload +from tryselect import push +from tryselect.selectors import again + + +@pytest.fixture(autouse=True) +def patch_history_path(tmpdir, monkeypatch): + monkeypatch.setattr(push, "history_path", tmpdir.join("history.json").strpath) + reload(again) + + +def test_try_again(monkeypatch): + push.push_to_try( + "fuzzy", + "Fuzzy message", + try_task_config=push.generate_try_task_config( + "fuzzy", + ["foo", "bar"], + {"try_task_config": {"use-artifact-builds": True}}, + ), + ) + + assert os.path.isfile(push.history_path) + with open(push.history_path, "r") as fh: + assert len(fh.readlines()) == 1 + + def fake_push_to_try(*args, **kwargs): + return args, kwargs + + monkeypatch.setattr(push, "push_to_try", fake_push_to_try) + reload(again) + + args, kwargs = again.run() + + assert args[0] == "again" + assert args[1] == "Fuzzy message" + + try_task_config = kwargs["try_task_config"]["parameters"].pop("try_task_config") + assert sorted(try_task_config.get("tasks")) == sorted(["foo", "bar"]) + assert try_task_config.get("env") == {"TRY_SELECTOR": "fuzzy"} + assert try_task_config.get("use-artifact-builds") + + with open(push.history_path, "r") as fh: + assert len(fh.readlines()) == 1 + + +def test_no_push_does_not_generate_history(tmpdir): + assert not os.path.isfile(push.history_path) + + push.push_to_try( + "fuzzy", + "Fuzzy", + try_task_config=push.generate_try_task_config( + "fuzzy", + ["foo", "bar"], + {"use-artifact-builds": True}, + ), + dry_run=True, + ) + assert not os.path.isfile(push.history_path) + assert again.run() == 1 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_auto.py b/tools/tryselect/test/test_auto.py new file mode 100644 index 0000000000..63f0fe6bd7 --- /dev/null +++ b/tools/tryselect/test/test_auto.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest +from tryselect.selectors.auto import AutoParser + + +def test_strategy_validation(): + parser = AutoParser() + args = parser.parse_args(["--strategy", "relevant_tests"]) + assert args.strategy == "gecko_taskgraph.optimize:tryselect.relevant_tests" + + args = parser.parse_args( + ["--strategy", "gecko_taskgraph.optimize:experimental.relevant_tests"] + ) + assert args.strategy == "gecko_taskgraph.optimize:experimental.relevant_tests" + + with pytest.raises(SystemExit): + parser.parse_args(["--strategy", "gecko_taskgraph.optimize:tryselect"]) + + with pytest.raises(SystemExit): + parser.parse_args(["--strategy", "foo"]) + + with pytest.raises(SystemExit): + parser.parse_args(["--strategy", "foo:bar"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_auto.t b/tools/tryselect/test/test_auto.t new file mode 100644 index 0000000000..c3fe797949 --- /dev/null +++ b/tools/tryselect/test/test_auto.t @@ -0,0 +1,61 @@ + + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test auto selector + + $ ./mach try auto $testargs + Commit message: + Tasks automatically selected. + + Pushed via `mach try auto` + Calculated try_task_config.json: + { + "parameters": { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", + "optimize_target_tasks": true, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {} + }, + "version": 2 + } + + + $ ./mach try auto $testargs --closed-tree + Commit message: + Tasks automatically selected. ON A CLOSED TREE + + Pushed via `mach try auto` + Calculated try_task_config.json: + { + "parameters": { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", + "optimize_target_tasks": true, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {} + }, + "version": 2 + } + + $ ./mach try auto $testargs --closed-tree -m "foo {msg} bar" + Commit message: + foo Tasks automatically selected. bar ON A CLOSED TREE + + Pushed via `mach try auto` + Calculated try_task_config.json: + { + "parameters": { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", + "optimize_target_tasks": true, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {} + }, + "version": 2 + } + diff --git a/tools/tryselect/test/test_chooser.py b/tools/tryselect/test/test_chooser.py new file mode 100644 index 0000000000..3d60a0f8d4 --- /dev/null +++ b/tools/tryselect/test/test_chooser.py @@ -0,0 +1,84 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import multiprocessing + +import mozunit +import pytest +from tryselect.selectors.chooser.app import create_application + +TASKS = [ + { + "kind": "build", + "label": "build-windows", + "attributes": { + "build_platform": "windows", + }, + }, + { + "kind": "test", + "label": "test-windows-mochitest-e10s", + "attributes": { + "unittest_suite": "mochitest-browser-chrome", + "mochitest_try_name": "mochitest-browser-chrome", + }, + }, +] + + +@pytest.fixture +def queue(): + return multiprocessing.Queue() + + +@pytest.fixture +def app(tg, queue): + app = create_application(tg, queue) + app.config["TESTING"] = True + + ctx = app.app_context() + ctx.push() + yield app + ctx.pop() + + +def test_try_chooser(app, queue: multiprocessing.Queue): + client = app.test_client() + + response = client.get("/") + assert response.status_code == 200 + + expected_output = [ + b"""<title>Try Chooser Enhanced</title>""", + b"""<input class="filter" type="checkbox" id=windows name="build" value='{"build_platform": ["windows"]}' onchange="console.log('checkbox onchange triggered');apply();">""", # noqa + b"""<input class="filter" type="checkbox" id=mochitest-browser-chrome name="test" value='{"unittest_suite": ["mochitest-browser-chrome"]}' onchange="console.log('checkbox onchange triggered');apply();">""", # noqa + ] + + for expected in expected_output: + assert expected in response.data + + response = client.post("/", data={"action": "Cancel"}) + assert response.status_code == 200 + assert b"You may now close this page" in response.data + assert queue.get() == [] + + response = client.post("/", data={"action": "Push", "selected-tasks": ""}) + assert response.status_code == 200 + assert b"You may now close this page" in response.data + assert queue.get() == [] + + response = client.post( + "/", + data={ + "action": "Push", + "selected-tasks": "build-windows\ntest-windows-mochitest-e10s", + }, + ) + assert response.status_code == 200 + assert b"You may now close this page" in response.data + assert set(queue.get()) == set(["build-windows", "test-windows-mochitest-e10s"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_empty.t b/tools/tryselect/test/test_empty.t new file mode 100644 index 0000000000..d7e9c22618 --- /dev/null +++ b/tools/tryselect/test/test_empty.t @@ -0,0 +1,62 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test empty selector + + $ ./mach try empty --no-push + Commit message: + No try selector specified, use "Add New Jobs" to select tasks. + + Pushed via `mach try empty` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "empty" + }, + "tasks": [] + } + }, + "version": 2 + } + + $ ./mach try empty --no-push --closed-tree + Commit message: + No try selector specified, use "Add New Jobs" to select tasks. ON A CLOSED TREE + + Pushed via `mach try empty` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "empty" + }, + "tasks": [] + } + }, + "version": 2 + } + + $ ./mach try empty --no-push --closed-tree -m "foo {msg} bar" + Commit message: + foo No try selector specified, use "Add New Jobs" to select tasks. bar ON A CLOSED TREE + + Pushed via `mach try empty` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "empty" + }, + "tasks": [] + } + }, + "version": 2 + } + diff --git a/tools/tryselect/test/test_fuzzy.py b/tools/tryselect/test/test_fuzzy.py new file mode 100644 index 0000000000..9ff1b386af --- /dev/null +++ b/tools/tryselect/test/test_fuzzy.py @@ -0,0 +1,125 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os + +import mozunit +import pytest + + +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +@pytest.mark.parametrize("show_chunk_numbers", [True, False]) +def test_query_paths(run_mach, capfd, show_chunk_numbers): + cmd = [ + "try", + "fuzzy", + "--no-push", + "-q", + "^test-linux '64-qr/debug-mochitest-chrome-1proc-", + "caps/tests/mochitest/test_addonMayLoad.html", + ] + chunk = "*" + if show_chunk_numbers: + cmd.append("--show-chunk-numbers") + chunk = "1" + + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + delim = "Calculated try_task_config.json:" + index = output.find(delim) + result = json.loads(output[index + len(delim) :]) + + # If there are more than one tasks here, it means that something went wrong + # with the path filtering. + tasks = result["parameters"]["try_task_config"]["tasks"] + assert tasks == ["test-linux1804-64-qr/debug-mochitest-chrome-1proc-%s" % chunk] + + +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +@pytest.mark.parametrize("show_chunk_numbers", [True, False]) +def test_query_paths_no_chunks(run_mach, capfd, show_chunk_numbers): + cmd = [ + "try", + "fuzzy", + "--no-push", + "-q", + "^test-linux '64-qr/debug-cppunittest", + ] + if show_chunk_numbers: + cmd.append("--show-chunk-numbers") + + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + delim = "Calculated try_task_config.json:" + index = output.find(delim) + result = json.loads(output[index + len(delim) :]) + + # If there are more than one tasks here, it means that something went wrong + # with the path filtering. + tasks = result["parameters"]["try_task_config"]["tasks"] + assert tasks == ["test-linux1804-64-qr/debug-cppunittest-1proc"] + + +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +@pytest.mark.parametrize("variant", ["", "spi-nw"]) +def test_query_paths_variants(run_mach, capfd, variant): + if variant: + variant = "-%s" % variant + + cmd = [ + "try", + "fuzzy", + "--no-push", + "-q", + "^test-linux '64-qr/debug-mochitest-browser-chrome%s-" % variant, + ] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + if variant: + expected = ["test-linux1804-64-qr/debug-mochitest-browser-chrome%s-*" % variant] + else: + expected = [ + "test-linux1804-64-qr/debug-mochitest-browser-chrome-spi-nw-*", + "test-linux1804-64-qr/debug-mochitest-browser-chrome-swr-*", + ] + + delim = "Calculated try_task_config.json:" + index = output.find(delim) + result = json.loads(output[index + len(delim) :]) + tasks = result["parameters"]["try_task_config"]["tasks"] + assert tasks == expected + + +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +@pytest.mark.parametrize("full", [True, False]) +def test_query(run_mach, capfd, full): + cmd = ["try", "fuzzy", "--no-push", "-q", "'source-test-python-taskgraph-tests-py3"] + if full: + cmd.append("--full") + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + delim = "Calculated try_task_config.json:" + index = output.find(delim) + result = json.loads(output[index + len(delim) :]) + + # Should only ever mach one task exactly. + tasks = result["parameters"]["try_task_config"]["tasks"] + assert tasks == ["source-test-python-taskgraph-tests-py3"] + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_fuzzy.t b/tools/tryselect/test/test_fuzzy.t new file mode 100644 index 0000000000..843b053e08 --- /dev/null +++ b/tools/tryselect/test/test_fuzzy.t @@ -0,0 +1,252 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test fuzzy selector + + $ ./mach try fuzzy $testargs -q "'foo" + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + + + $ ./mach try fuzzy $testargs -q "'bar" + no tasks selected + $ ./mach try fuzzy $testargs --full -q "'bar" + Commit message: + Fuzzy query='bar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/bar-debug", + "test/bar-opt" + ] + } + }, + "version": 2 + } + + +Test multiple selectors + + $ ./mach try fuzzy $testargs --full -q "'foo" -q "'bar" + Commit message: + Fuzzy query='foo&query='bar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/bar-debug", + "test/bar-opt", + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + +Test query intersection + + $ ./mach try fuzzy $testargs --and -q "'foo" -q "'opt" + Commit message: + Fuzzy query='foo&query='opt + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-opt" + ] + } + }, + "version": 2 + } + + +Test intersection with preset containing multiple queries + + $ ./mach try fuzzy --save foo -q "'test" -q "'opt" + preset saved, run with: --preset=foo + + $ ./mach try fuzzy $testargs --preset foo -xq "'test" + Commit message: + Fuzzy query='test&query='opt&query='test + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + $ ./mach try $testargs --preset foo -xq "'test" + Commit message: + Fuzzy query='test&query='opt&query='test + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + +Test exact match + + $ ./mach try fuzzy $testargs --full -q "testfoo | 'testbar" + Commit message: + Fuzzy query=testfoo | 'testbar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + $ ./mach try fuzzy $testargs --full --exact -q "testfoo | 'testbar" + Commit message: + Fuzzy query=testfoo | 'testbar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/bar-debug", + "test/bar-opt" + ] + } + }, + "version": 2 + } + + +Test task config + + $ ./mach try fuzzy --no-push --artifact -q "'foo" + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "disable-pgo": true, + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "use-artifact-builds": true + } + }, + "version": 2 + } + + $ ./mach try fuzzy $testargs --env FOO=1 --env BAR=baz -q "'foo" + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "BAR": "baz", + "FOO": "1", + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + diff --git a/tools/tryselect/test/test_message.t b/tools/tryselect/test/test_message.t new file mode 100644 index 0000000000..a707e410fb --- /dev/null +++ b/tools/tryselect/test/test_message.t @@ -0,0 +1,73 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test custom commit messages with fuzzy selector + + $ ./mach try fuzzy $testargs -q foo --message "Foobar" + Commit message: + Foobar + + Fuzzy query=foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + $ ./mach try fuzzy $testargs -q foo -m "Foobar: {msg}" + Commit message: + Foobar: Fuzzy query=foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + $ unset EDITOR + $ ./mach try fuzzy $testargs -q foo -m > /dev/null 2>&1 + [2] + + +Test custom commit messages with syntax selector + + $ ./mach try syntax $testargs -p linux -u mochitests --message "Foobar" + Commit message: + Foobar + + try: -b do -p linux -u mochitests + + Pushed via `mach try syntax` + $ ./mach try syntax $testargs -p linux -u mochitests -m "Foobar: {msg}" + Commit message: + Foobar: try: -b do -p linux -u mochitests + + Pushed via `mach try syntax` + $ unset EDITOR + $ ./mach try syntax $testargs -p linux -u mochitests -m > /dev/null 2>&1 + [2] diff --git a/tools/tryselect/test/test_mozharness_integration.py b/tools/tryselect/test/test_mozharness_integration.py new file mode 100644 index 0000000000..abeaaf370e --- /dev/null +++ b/tools/tryselect/test/test_mozharness_integration.py @@ -0,0 +1,145 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os + +import mozunit +import pytest +from mozfile import load_source +from tryselect.tasks import build, resolve_tests_by_suite + +MOZHARNESS_SCRIPTS = { + "android_emulator_unittest": { + "class_name": "AndroidEmulatorTest", + "configs": [ + "android/android_common.py", + ], + "xfail": [ + "cppunittest", + "crashtest-qr", + "gtest", + "geckoview-junit", + "jittest", + "jsreftest", + "reftest-qr", + ], + }, + "desktop_unittest": { + "class_name": "DesktopUnittest", + "configs": [ + "unittests/linux_unittest.py", + "unittests/mac_unittest.py", + "unittests/win_unittest.py", + ], + "xfail": [ + "cppunittest", + "gtest", + "jittest", + "jittest-chunked", + "jittest1", + "jittest2", + "jsreftest", + "mochitest-valgrind-plain", + "reftest-no-accel", + "reftest-snapshot", + "xpcshell-msix", + ], + }, +} +"""A suite being listed in a script's `xfail` list means it won't work +properly with MOZHARNESS_TEST_PATHS (the mechanism |mach try fuzzy <path>| +uses). +""" + + +def get_mozharness_test_paths(name): + scriptdir = os.path.join(build.topsrcdir, "testing", "mozharness") + mod = load_source( + "scripts." + name, os.path.join(scriptdir, "scripts", name + ".py") + ) + + class_name = MOZHARNESS_SCRIPTS[name]["class_name"] + cls = getattr(mod, class_name) + return cls(require_config_file=False)._get_mozharness_test_paths + + +@pytest.fixture(scope="module") +def all_suites(): + from moztest.resolve import _test_flavors, _test_subsuites + + all_suites = [] + for flavor in _test_flavors: + all_suites.append({"flavor": flavor, "srcdir_relpath": "test"}) + + for flavor, subsuite in _test_subsuites: + all_suites.append( + {"flavor": flavor, "subsuite": subsuite, "srcdir_relpath": "test"} + ) + + return all_suites + + +def generate_suites_from_config(path): + parent, name = os.path.split(path) + name = os.path.splitext(name)[0] + + configdir = os.path.join( + build.topsrcdir, "testing", "mozharness", "configs", parent + ) + + mod = load_source(name, os.path.join(configdir, name + ".py")) + + config = mod.config + + for category in sorted(config["suite_definitions"]): + key = "all_{}_suites".format(category) + if key not in config: + yield category, + continue + + for suite in sorted(config["all_{}_suites".format(category)]): + yield category, suite + + +def generate_suites(): + for name, script in MOZHARNESS_SCRIPTS.items(): + seen = set() + + for path in script["configs"]: + for suite in generate_suites_from_config(path): + if suite in seen: + continue + seen.add(suite) + + item = (name, suite) + + if suite[-1] in script["xfail"]: + item = pytest.param(item, marks=pytest.mark.xfail) + + yield item + + +def idfn(item): + name, suite = item + return "{}/{}".format(name, suite[-1]) + + +@pytest.mark.parametrize("item", generate_suites(), ids=idfn) +def test_suites(item, patch_resolver, all_suites): + """An integration test to make sure the suites returned by + `tasks.resolve_tests_by_suite` match up with the names defined in + mozharness. + """ + patch_resolver([], all_suites) + suites = resolve_tests_by_suite(["test"]) + os.environ["MOZHARNESS_TEST_PATHS"] = json.dumps(suites) + + name, suite = item + func = get_mozharness_test_paths(name) + assert func(*suite) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_perf.py b/tools/tryselect/test/test_perf.py new file mode 100644 index 0000000000..0db45df83e --- /dev/null +++ b/tools/tryselect/test/test_perf.py @@ -0,0 +1,1425 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import pathlib +import shutil +import tempfile +from unittest import mock + +import mozunit +import pytest +from tryselect.selectors.perf import ( + MAX_PERF_TASKS, + Apps, + InvalidCategoryException, + InvalidRegressionDetectorQuery, + PerfParser, + Platforms, + Suites, + Variants, + run, +) +from tryselect.selectors.perf_preview import plain_display +from tryselect.selectors.perfselector.classification import ( + check_for_live_sites, + check_for_profile, +) + +TASKS = [ + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-animometer", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-optimizing", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-webaudio", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-speedometer", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-jetstream2", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-ares6", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-optimizing", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-sunspider", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-matrix-react-bench", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-baseline", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-twitch-animation", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-assorted-dom", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-stylebench", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-baseline", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-htmlsuite", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-unity-webgl", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot", +] + +# The TEST_VARIANTS, and TEST_CATEGORIES are used to force +# a particular set of categories to show up in testing. Otherwise, +# every time someone adds a category, or a variant, we'll need +# to redo all the category counts. The platforms, and apps are +# not forced because they change infrequently. +TEST_VARIANTS = { + # Bug 1837058 - Switch this back to Variants.NO_FISSION when + # the default flips to fission on android + Variants.FISSION.value: { + "query": "'nofis", + "negation": "!nofis", + "platforms": [Platforms.ANDROID.value], + "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value], + }, + Variants.BYTECODE_CACHED.value: { + "query": "'bytecode", + "negation": "!bytecode", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + Variants.LIVE_SITES.value: { + "query": "'live", + "negation": "!live", + "restriction": check_for_live_sites, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": list(PerfParser.apps.keys()), + }, + Variants.PROFILING.value: { + "query": "'profil", + "negation": "!profil", + "restriction": check_for_profile, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value], + }, + Variants.SWR.value: { + "query": "'swr", + "negation": "!swr", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, +} + +TEST_CATEGORIES = { + "Pageload": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6"], + }, + "suites": [Suites.RAPTOR.value], + "tasks": [], + "description": "", + }, + "Pageload (essential)": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6 'essential"], + }, + "variant-restrictions": {Suites.RAPTOR.value: [Variants.FISSION.value]}, + "suites": [Suites.RAPTOR.value], + "tasks": [], + "description": "", + }, + "Responsiveness": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'responsive"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + "description": "", + }, + "Benchmarks": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'benchmark"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + "description": "", + }, + "DAMP (Devtools)": { + "query": { + Suites.TALOS.value: ["'talos 'damp"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + "description": "", + }, + "Talos PerfTests": { + "query": { + Suites.TALOS.value: ["'talos"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + "description": "", + }, + "Resource Usage": { + "query": { + Suites.TALOS.value: ["'talos 'xperf | 'tp5"], + Suites.RAPTOR.value: ["'power 'osx"], + Suites.AWSY.value: ["'awsy"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value], + "platform-restrictions": [Platforms.DESKTOP.value], + "variant-restrictions": { + Suites.RAPTOR.value: [], + Suites.TALOS.value: [], + }, + "app-restrictions": { + Suites.RAPTOR.value: [Apps.FIREFOX.value], + Suites.TALOS.value: [Apps.FIREFOX.value], + }, + "tasks": [], + "description": "", + }, + "Graphics, & Media Playback": { + "query": { + # XXX This might not be an exhaustive list for talos atm + Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"], + Suites.RAPTOR.value: ["'browsertime 'youtube-playback"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: [Variants.FISSION.value]}, + "tasks": [], + "description": "", + }, +} + + +@pytest.mark.parametrize( + "category_options, expected_counts, unique_categories, missing", + [ + # Default should show the premade live category, but no chrome or android + # The benchmark desktop category should be visible in all configurations + # except for when there are requested apps/variants/platforms + ( + {}, + 58, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!fenix", + "!safari", + "!m-car", + ] + }, + "Pageload macosx": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!fenix", + "!safari", + "!m-car", + ] + }, + "Resource Usage desktop": { + "awsy": ["'awsy", "!android 'shippable !-32 !clang"], + "raptor": [ + "'power 'osx", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!fenix", + "!safari", + "!m-car", + ], + "talos": [ + "'talos 'xperf | 'tp5", + "!android 'shippable !-32 !clang", + "!profil", + "!swr", + ], + }, + }, + [ + "Responsiveness android-p2 geckoview", + "Benchmarks desktop chromium", + ], + ), # Default settings + ( + {"live_sites": True}, + 66, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!profil", + "!chrom", + "!fenix", + "!safari", + "!m-car", + ] + }, + "Pageload macosx": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "!bytecode", + "!profil", + "!chrom", + "!fenix", + "!safari", + "!m-car", + ] + }, + "Pageload macosx live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'live", + "!bytecode", + "!profil", + "!chrom", + "!fenix", + "!safari", + "!m-car", + ], + }, + }, + [ + "Responsiveness android-p2 geckoview", + "Benchmarks desktop chromium", + "Benchmarks desktop firefox profiling", + "Talos desktop live-sites", + "Talos desktop profiling+swr", + "Benchmarks desktop firefox live-sites+profiling" + "Benchmarks desktop firefox live-sites", + ], + ), + ( + {"live_sites": True, "safari": True}, + 72, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!profil", + "!chrom", + "!fenix", + "!m-car", + ] + }, + "Pageload macosx safari": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'safari", + "!bytecode", + "!profil", + ] + }, + "Pageload macosx safari live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'safari", + "'live", + "!bytecode", + "!profil", + ], + }, + }, + [ + "Pageload linux safari", + "Pageload desktop safari", + ], + ), + ( + {"live_sites": True, "chrome": True}, + 114, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!profil", + "!fenix", + "!safari", + "!m-car", + ] + }, + "Pageload macosx live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'live", + "!bytecode", + "!profil", + "!fenix", + "!safari", + "!m-car", + ], + }, + "Benchmarks desktop chromium": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "'chromium", + "!bytecode", + "!profil", + ], + }, + }, + [ + "Responsiveness android-p2 geckoview", + "Firefox Pageload linux chrome", + "Talos PerfTests desktop swr", + ], + ), + ( + {"android": True}, + 78, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!fenix", + "!safari", + "!m-car", + ], + }, + "Responsiveness android-a51 geckoview": { + "raptor": [ + "'browsertime 'responsive", + "'android 'a51 'shippable 'aarch64", + "'geckoview", + "!nofis", + "!live", + "!profil", + ], + }, + }, + [ + "Responsiveness android-a51 chrome-m", + "Firefox Pageload android", + "Pageload android-a51 fenix", + ], + ), + ( + {"android": True, "chrome": True}, + 128, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!fenix", + "!safari", + "!m-car", + ], + }, + "Responsiveness android-a51 chrome-m": { + "raptor": [ + "'browsertime 'responsive", + "'android 'a51 'shippable 'aarch64", + "'chrome-m", + "!nofis", + "!live", + "!profil", + ], + }, + }, + ["Responsiveness android-p2 chrome-m", "Resource Usage android"], + ), + ( + {"android": True, "chrome": True, "profile": True}, + 164, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!fenix", + "!safari", + "!m-car", + ] + }, + "Talos PerfTests desktop profiling": { + "talos": [ + "'talos", + "!android 'shippable !-32 !clang", + "'profil", + "!swr", + ] + }, + }, + [ + "Resource Usage desktop profiling", + "DAMP (Devtools) desktop chrome", + "Resource Usage android", + "Resource Usage windows chromium", + ], + ), + ( + {"android": True, "fenix": True}, + 88, + { + "Pageload android-a51": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "!nofis", + "!live", + "!profil", + "!chrom", + "!safari", + "!m-car", + ] + }, + "Pageload android-a51 fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!nofis", + "!live", + "!profil", + ] + }, + }, + [ + "Resource Usage desktop profiling", + "DAMP (Devtools) desktop chrome", + "Resource Usage android", + "Resource Usage windows chromium", + ], + ), + # Show all available windows tests, no other platform should exist + # including the desktop catgeory + ( + {"requested_platforms": ["windows"]}, + 14, + { + "Benchmarks windows firefox": { + "raptor": [ + "'browsertime 'benchmark", + "!-32 'windows 'shippable", + "!chrom !geckoview !fenix !safari !m-car", + "!bytecode", + "!live", + "!profil", + ] + }, + }, + [ + "Resource Usage desktop", + "Benchmarks desktop", + "Benchmarks linux firefox bytecode-cached+profiling", + ], + ), + # Can't have fenix on the windows platform + ( + {"requested_platforms": ["windows"], "requested_apps": ["fenix"]}, + 0, + {}, + ["Benchmarks desktop"], + ), + # Android flag also needs to be supplied + ( + {"requested_platforms": ["android"], "requested_apps": ["fenix"]}, + 0, + {}, + ["Benchmarks desktop"], + ), + # There should be no global categories available, only fenix + ( + { + "requested_platforms": ["android"], + "requested_apps": ["fenix"], + "android": True, + }, + 10, + { + "Pageload android fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!nofis", + "!live", + "!profil", + ], + } + }, + ["Benchmarks desktop", "Pageload (live) android"], + ), + # Test with multiple apps + ( + { + "requested_platforms": ["android"], + "requested_apps": ["fenix", "geckoview"], + "android": True, + }, + 15, + { + "Benchmarks android geckoview": { + "raptor": [ + "'browsertime 'benchmark", + "'android 'a51 'shippable 'aarch64", + "'geckoview", + "!nofis", + "!live", + "!profil", + ], + }, + "Pageload android fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!nofis", + "!live", + "!profil", + ], + }, + }, + [ + "Benchmarks desktop", + "Pageload android no-fission", + "Pageload android fenix live-sites", + ], + ), + # Variants are inclusive, so we'll see the variant alongside the + # base here for fenix + ( + { + "requested_variants": ["fission"], + "requested_apps": ["fenix"], + "android": True, + }, + 32, + { + "Pageload android-a51 fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!live", + "!profil", + ], + }, + "Pageload android-a51 fenix fission": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + "Pageload (essential) android fenix fission": { + "raptor": [ + "'browsertime 'tp6 'essential", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + }, + [ + "Benchmarks desktop", + "Pageload (live) android", + "Pageload android-p2 fenix live-sites", + ], + ), + # With multiple variants, we'll see the base variant (with no combinations) + # for each of them + ( + { + "requested_variants": ["fission", "live-sites"], + "requested_apps": ["fenix"], + "android": True, + }, + 40, + { + "Pageload android-a51 fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!profil", + ], + }, + "Pageload android-a51 fenix fission": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + "Pageload android-a51 fenix live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'live", + "!nofis", + "!profil", + ], + }, + "Pageload (essential) android fenix fission": { + "raptor": [ + "'browsertime 'tp6 'essential", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + "Pageload android fenix fission+live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "'live", + "!profil", + ], + }, + }, + [ + "Benchmarks desktop", + "Pageload (live) android", + "Pageload android-p2 fenix live-sites", + "Pageload (essential) android fenix no-fission+live-sites", + ], + ), + # Make sure that no no-fission tasks are selected when a variant cannot + # run on a requested platform + ( + { + "requested_variants": ["no-fission"], + "requested_platforms": ["windows"], + }, + 14, + { + "Responsiveness windows firefox": { + "raptor": [ + "'browsertime 'responsive", + "!-32 'windows 'shippable", + "!chrom !geckoview !fenix !safari !m-car", + "!bytecode", + "!live", + "!profil", + ], + }, + }, + ["Benchmarks desktop", "Responsiveness windows firefox no-fisson"], + ), + # We should only see the base and the live-site variants here for windows + ( + { + "requested_variants": ["no-fission", "live-sites"], + "requested_platforms": ["windows"], + "android": True, + }, + 16, + { + "Responsiveness windows firefox": { + "raptor": [ + "'browsertime 'responsive", + "!-32 'windows 'shippable", + "!chrom !geckoview !fenix !safari !m-car", + "!bytecode", + "!profil", + ], + }, + "Pageload windows live-sites": { + "raptor": [ + "'browsertime 'tp6", + "!-32 'windows 'shippable", + "'live", + "!bytecode", + "!profil", + "!chrom", + "!fenix", + "!safari", + "!m-car", + ], + }, + "Graphics, & Media Playback windows": { + "raptor": [ + "'browsertime 'youtube-playback", + "!-32 'windows 'shippable", + "!bytecode", + "!profil", + "!chrom", + "!fenix", + "!safari", + "!m-car", + ], + "talos": [ + "'talos 'svgr | 'bcv | 'webgl", + "!-32 'windows 'shippable", + "!profil", + "!swr", + ], + }, + }, + [ + "Benchmarks desktop", + "Responsiveness windows firefox no-fisson", + "Pageload (live) android", + "Talos desktop live-sites", + "Talos android", + "Graphics, & Media Playback windows live-sites", + "Graphics, & Media Playback android no-fission", + ], + ), + ], +) +def test_category_expansion( + category_options, expected_counts, unique_categories, missing +): + # Set the categories, and variants to expand + PerfParser.categories = TEST_CATEGORIES + PerfParser.variants = TEST_VARIANTS + + # Expand the categories, then either check if the unique_categories, + # exist or are missing from the categories + expanded_cats = PerfParser.get_categories(**category_options) + + assert len(expanded_cats) == expected_counts + assert not any([expanded_cats.get(ucat, None) is not None for ucat in missing]) + assert all( + [expanded_cats.get(ucat, None) is not None for ucat in unique_categories.keys()] + ) + + # Ensure that the queries are as expected + for cat_name, cat_query in unique_categories.items(): + # Don't use get here because these fields should always exist + assert cat_query == expanded_cats[cat_name]["queries"] + + +@pytest.mark.parametrize( + "options, call_counts, log_ind, expected_log_message", + [ + ( + {}, + [10, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"query": "'Pageload 'linux 'firefox"}, + [10, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"cached_revision": "cached_base_revision"}, + [10, 1, 1, 10, 2, 0], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=cached_base_revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"dry_run": True}, + [10, 1, 1, 10, 2, 0], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=&newProject=try&newRevision=revision\n" + ), + ), + ( + {"show_all": True}, + [1, 2, 2, 8, 2, 1], + 0, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"show_all": True, "query": "'shippable !32 speedometer 'firefox"}, + [1, 2, 2, 8, 2, 1], + 0, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"single_run": True}, + [10, 1, 1, 4, 2, 0], + 2, + ( + "If you need any help, you can find us in the #perf-help Matrix channel:\n" + "https://matrix.to/#/#perf-help:mozilla.org\n" + ), + ), + ( + {"detect_changes": True}, + [11, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"perfcompare_beta": True}, + [10, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://beta--mozilla-perfcompare.netlify.app/compare-results?" + "baseRev=revision&newRev=revision&baseRepo=try&newRepo=try\n" + ), + ), + ], +) +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_full_run(options, call_counts, log_ind, expected_log_message): + with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( + "tryselect.selectors.perf.run_fzf" + ) as fzf, mock.patch( + "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() + ), mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ) as logger, mock.patch( + "tryselect.selectors.perf.PerfParser.check_cached_revision", + ) as ccr, mock.patch( + "tryselect.selectors.perf.PerfParser.save_revision_treeherder" + ) as srt, mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print: + fzf_side_effects = [ + ["", ["Benchmarks linux"]], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", ["Perftest Change Detector"]], + ] + # Number of side effects for fzf should always be greater than + # or equal to the number of calls expected + assert len(fzf_side_effects) >= call_counts[0] + + fzf.side_effect = fzf_side_effects + ccr.return_value = options.get("cached_revision", "") + + run(**options) + + assert fzf.call_count == call_counts[0] + assert ptt.call_count == call_counts[1] + assert logger.call_count == call_counts[2] + assert perf_print.call_count == call_counts[3] + assert ccr.call_count == call_counts[4] + assert srt.call_count == call_counts[5] + assert perf_print.call_args_list[log_ind][0][0] == expected_log_message + + +@pytest.mark.parametrize( + "options, call_counts, log_ind, expected_log_message, expected_failure", + [ + ( + {"detect_changes": True}, + [11, 0, 0, 2, 1], + 1, + ( + "Executing raptor queries: 'browsertime 'benchmark, !clang 'linux " + "'shippable, !bytecode, !live, !profil, !chrom, !fenix, !safari, !m-car" + ), + InvalidRegressionDetectorQuery, + ), + ], +) +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_change_detection_task_injection_failure( + options, + call_counts, + log_ind, + expected_log_message, + expected_failure, +): + with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( + "tryselect.selectors.perf.run_fzf" + ) as fzf, mock.patch( + "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() + ), mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ) as logger, mock.patch( + "tryselect.selectors.perf.PerfParser.check_cached_revision" + ) as ccr, mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print: + fzf_side_effects = [ + ["", ["Benchmarks linux"]], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ] + assert len(fzf_side_effects) >= call_counts[0] + + fzf.side_effect = fzf_side_effects + + with pytest.raises(expected_failure): + run(**options) + + assert fzf.call_count == call_counts[0] + assert ptt.call_count == call_counts[1] + assert logger.call_count == call_counts[2] + assert perf_print.call_count == call_counts[3] + assert ccr.call_count == call_counts[4] + assert perf_print.call_args_list[log_ind][0][0] == expected_log_message + + +@pytest.mark.parametrize( + "query, should_fail", + [ + ( + { + "query": { + # Raptor has all variants available so it + # should fail on this category + "raptor": ["browsertime 'live 'no-fission"], + } + }, + True, + ), + ( + { + "query": { + # Awsy has no variants defined so it shouldn't fail + # on a query like this + "awsy": ["browsertime 'live 'no-fission"], + } + }, + False, + ), + ], +) +def test_category_rules(query, should_fail): + # Set the categories, and variants to expand + PerfParser.categories = {"test-live": query} + PerfParser.variants = TEST_VARIANTS + + if should_fail: + with pytest.raises(InvalidCategoryException): + PerfParser.run_category_checks() + else: + assert PerfParser.run_category_checks() + + # Reset the categories, and variants to expand + PerfParser.categories = TEST_CATEGORIES + PerfParser.variants = TEST_VARIANTS + + +@pytest.mark.parametrize( + "apk_name, apk_content, should_fail, failure_message", + [ + ( + "real-file", + "file-content", + False, + None, + ), + ("bad-file", None, True, "Path does not exist:"), + ], +) +def test_apk_upload(apk_name, apk_content, should_fail, failure_message): + with mock.patch("tryselect.selectors.perf.subprocess") as _, mock.patch( + "tryselect.selectors.perf.shutil" + ) as _: + temp_dir = None + try: + temp_dir = tempfile.mkdtemp() + sample_apk = pathlib.Path(temp_dir, apk_name) + if apk_content is not None: + with sample_apk.open("w") as f: + f.write(apk_content) + + if should_fail: + with pytest.raises(Exception) as exc_info: + PerfParser.setup_apk_upload("browsertime", str(sample_apk)) + assert failure_message in str(exc_info) + else: + PerfParser.setup_apk_upload("browsertime", str(sample_apk)) + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir) + + +@pytest.mark.parametrize( + "args, load_data, return_value, call_counts, exists_cache_file", + [ + ( + ( + [], + "base_commit", + ), + { + "base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": [], + }, + ], + }, + "2b04563b5", + [1, 0], + True, + ), + ( + ( + ["task-a"], + "subset_base_commit", + ), + { + "subset_base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": ["task-a", "task-b"], + }, + ], + }, + "2b04563b5", + [1, 0], + True, + ), + ( + ([], "not_exist_cached_base_commit"), + { + "base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": [], + }, + ], + }, + None, + [1, 0], + True, + ), + ( + ( + ["task-a", "task-b"], + "superset_base_commit", + ), + { + "superset_base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": ["task-a"], + }, + ], + }, + None, + [1, 0], + True, + ), + ( + ([], None), + {}, + None, + [1, 1], + True, + ), + ( + ([], None), + {}, + None, + [0, 0], + False, + ), + ], +) +def test_check_cached_revision( + args, load_data, return_value, call_counts, exists_cache_file +): + with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( + "tryselect.selectors.perf.json.dump" + ) as dump, mock.patch( + "tryselect.selectors.perf.pathlib.Path.is_file" + ) as is_file, mock.patch( + "tryselect.selectors.perf.pathlib.Path.open" + ): + load.return_value = load_data + is_file.return_value = exists_cache_file + result = PerfParser.check_cached_revision(*args) + + assert load.call_count == call_counts[0] + assert dump.call_count == call_counts[1] + assert result == return_value + + +@pytest.mark.parametrize( + "args, call_counts, exists_cache_file", + [ + ( + ["base_commit", "base_revision_treeherder"], + [0, 1], + False, + ), + ( + ["base_commit", "base_revision_treeherder"], + [1, 1], + True, + ), + ], +) +def test_save_revision_treeherder(args, call_counts, exists_cache_file): + with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( + "tryselect.selectors.perf.json.dump" + ) as dump, mock.patch( + "tryselect.selectors.perf.pathlib.Path.is_file" + ) as is_file, mock.patch( + "tryselect.selectors.perf.pathlib.Path.open" + ): + is_file.return_value = exists_cache_file + PerfParser.save_revision_treeherder(TASKS, args[0], args[1]) + + assert load.call_count == call_counts[0] + assert dump.call_count == call_counts[1] + + +@pytest.mark.parametrize( + "total_tasks, options, call_counts, expected_log_message, expected_failure", + [ + ( + MAX_PERF_TASKS + 1, + {}, + [1, 0, 0, 1], + ( + "\n\n----------------------------------------------------------------------------------------------\n" + f"You have selected {MAX_PERF_TASKS+1} total test runs! (selected tasks({MAX_PERF_TASKS+1}) * rebuild" + f" count(1) \nThese tests won't be triggered as the current maximum for a single ./mach try " + f"perf run is {MAX_PERF_TASKS}. \nIf this was unexpected, please file a bug in Testing :: Performance." + "\n----------------------------------------------------------------------------------------------\n\n" + ), + True, + ), + ( + MAX_PERF_TASKS, + {"show_all": True}, + [9, 0, 0, 8], + ( + "For more information on the performance tests, see our " + "PerfDocs here:\nhttps://firefox-source-docs.mozilla.org/testing/perfdocs/" + ), + False, + ), + ( + int((MAX_PERF_TASKS + 2) / 2), + { + "show_all": True, + "try_config_params": {"try_task_config": {"rebuild": 2}}, + }, + [1, 0, 0, 1], + ( + "\n\n----------------------------------------------------------------------------------------------\n" + f"You have selected {int((MAX_PERF_TASKS + 2) / 2) * 2} total test runs! (selected tasks(" + f"{int((MAX_PERF_TASKS + 2) / 2)}) * rebuild" + f" count(2) \nThese tests won't be triggered as the current maximum for a single ./mach try " + f"perf run is {MAX_PERF_TASKS}. \nIf this was unexpected, please file a bug in Testing :: Performance." + "\n----------------------------------------------------------------------------------------------\n\n" + ), + True, + ), + (0, {}, [1, 0, 0, 1], ("No tasks selected"), True), + ], +) +def test_max_perf_tasks( + total_tasks, + options, + call_counts, + expected_log_message, + expected_failure, +): + # Set the categories, and variants to expand + PerfParser.categories = TEST_CATEGORIES + PerfParser.variants = TEST_VARIANTS + + with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print, mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ), mock.patch( + "tryselect.selectors.perf.PerfParser.perf_push_to_try", + new_callable=mock.MagicMock, + return_value=("revision1", "revision2"), + ) as perf_push_to_try_mock, mock.patch( + "tryselect.selectors.perf.PerfParser.get_perf_tasks" + ) as get_perf_tasks_mock, mock.patch( + "tryselect.selectors.perf.PerfParser.get_tasks" + ) as get_tasks_mock, mock.patch( + "tryselect.selectors.perf.run_fzf" + ) as fzf, mock.patch( + "tryselect.selectors.perf.fzf_bootstrap", return_value=mock.MagicMock() + ): + tasks = ["a-task"] * total_tasks + get_tasks_mock.return_value = tasks + get_perf_tasks_mock.return_value = tasks, [], [] + + run(**options) + + assert perf_push_to_try_mock.call_count == 0 if expected_failure else 1 + assert ptt.call_count == call_counts[1] + assert perf_print.call_count == call_counts[3] + assert fzf.call_count == 0 + assert perf_print.call_args_list[-1][0][0] == expected_log_message + + +@pytest.mark.parametrize( + "try_config, selected_tasks, expected_try_config", + [ + ( + {"use-artifact-builds": True}, + ["some-android-task"], + {"use-artifact-builds": False}, + ), + ( + {"use-artifact-builds": True}, + ["some-desktop-task"], + {"use-artifact-builds": True}, + ), + ( + {"use-artifact-builds": False}, + ["some-android-task"], + {"use-artifact-builds": False}, + ), + ( + {"use-artifact-builds": True}, + ["some-desktop-task", "some-android-task"], + {"use-artifact-builds": False}, + ), + ], +) +def test_artifact_mode_autodisable(try_config, selected_tasks, expected_try_config): + PerfParser.setup_try_config({"try_task_config": try_config}, [], selected_tasks) + assert ( + try_config["use-artifact-builds"] == expected_try_config["use-artifact-builds"] + ) + + +def test_build_category_description(): + base_cmd = ["--preview", '-t "{+f}"'] + + with mock.patch("tryselect.selectors.perf.json.dump") as dump: + PerfParser.build_category_description(base_cmd, "") + + assert dump.call_count == 1 + assert str(base_cmd).count("-d") == 1 + assert str(base_cmd).count("-l") == 1 + + +@pytest.mark.parametrize( + "options, call_count", + [ + ({}, [1, 1, 2]), + ({"show_all": True}, [0, 0, 1]), + ], +) +def test_preview_description(options, call_count): + with mock.patch("tryselect.selectors.perf.PerfParser.perf_push_to_try"), mock.patch( + "tryselect.selectors.perf.fzf_bootstrap" + ), mock.patch( + "tryselect.selectors.perf.PerfParser.get_perf_tasks" + ) as get_perf_tasks, mock.patch( + "tryselect.selectors.perf.PerfParser.get_tasks" + ), mock.patch( + "tryselect.selectors.perf.PerfParser.build_category_description" + ) as bcd: + get_perf_tasks.return_value = [], [], [] + + run(**options) + + assert bcd.call_count == call_count[0] + + base_cmd = ["--preview", '-t "{+f}"'] + option = base_cmd[base_cmd.index("--preview") + 1].split(" ") + description, line = None, None + if call_count[0] == 1: + PerfParser.build_category_description(base_cmd, "") + option = base_cmd[base_cmd.index("--preview") + 1].split(" ") + description = option[option.index("-d") + 1] + line = "Current line" + + taskfile = option[option.index("-t") + 1] + + with mock.patch("tryselect.selectors.perf_preview.open"), mock.patch( + "tryselect.selectors.perf_preview.pathlib.Path.open" + ), mock.patch("tryselect.selectors.perf_preview.json.load") as load, mock.patch( + "tryselect.selectors.perf_preview.print" + ) as preview_print: + load.return_value = {line: "test description"} + + plain_display(taskfile, description, line) + + assert load.call_count == call_count[1] + assert preview_print.call_count == call_count[2] + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_perfcomparators.py b/tools/tryselect/test/test_perfcomparators.py new file mode 100644 index 0000000000..51f0bdb287 --- /dev/null +++ b/tools/tryselect/test/test_perfcomparators.py @@ -0,0 +1,150 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import tempfile +from unittest import mock + +import mozunit +import pytest +from tryselect.selectors.perfselector.perfcomparators import ( + BadComparatorArgs, + BenchmarkComparator, + ComparatorNotFound, + get_comparator, +) + + +@pytest.mark.parametrize( + "test_link", + [ + "https://github.com/mozilla-mobile/firefox-android/pull/1627", + "https://github.com/mozilla-mobile/firefox-android/pull/1876/" + "commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365", + ], +) +def test_benchmark_comparator(test_link): + def _verify_extra_args(extra_args): + assert len(extra_args) == 3 + if "commit" in test_link: + assert ( + "benchmark-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365" + in extra_args + ) + else: + assert "benchmark-revision=sha-for-link" in extra_args + assert "benchmark-repository=url-for-link" in extra_args + assert "benchmark-branch=ref-for-link" in extra_args + + comparator = BenchmarkComparator( + None, None, None, [f"base-link={test_link}", f"new-link={test_link}"] + ) + + with mock.patch("requests.get") as mocked_get: + magic_get = mock.MagicMock() + magic_get.json.return_value = { + "head": { + "repo": { + "html_url": "url-for-link", + }, + "sha": "sha-for-link", + "ref": "ref-for-link", + } + } + magic_get.status_code = 200 + mocked_get.return_value = magic_get + + extra_args = [] + comparator.setup_base_revision(extra_args) + _verify_extra_args(extra_args) + + extra_args = [] + comparator.setup_new_revision(extra_args) + _verify_extra_args(extra_args) + + +def test_benchmark_comparator_no_pr_links(): + def _verify_extra_args(extra_args): + assert len(extra_args) == 3 + assert "benchmark-revision=rev" in extra_args + assert "benchmark-repository=link" in extra_args + assert "benchmark-branch=fake" in extra_args + + comparator = BenchmarkComparator( + None, + None, + None, + [ + "base-repo=link", + "base-branch=fake", + "base-revision=rev", + "new-repo=link", + "new-branch=fake", + "new-revision=rev", + ], + ) + + with mock.patch("requests.get") as mocked_get: + magic_get = mock.MagicMock() + magic_get.json.return_value = { + "head": { + "repo": { + "html_url": "url-for-link", + }, + "sha": "sha-for-link", + "ref": "ref-for-link", + } + } + magic_get.status_code = 200 + mocked_get.return_value = magic_get + + extra_args = [] + comparator.setup_base_revision(extra_args) + _verify_extra_args(extra_args) + + extra_args = [] + comparator.setup_new_revision(extra_args) + _verify_extra_args(extra_args) + + +def test_benchmark_comparator_bad_args(): + comparator = BenchmarkComparator( + None, + None, + None, + [ + "base-bad-args=val", + ], + ) + + with pytest.raises(BadComparatorArgs): + comparator.setup_base_revision([]) + + +def test_get_comparator_bad_name(): + with pytest.raises(ComparatorNotFound): + get_comparator("BadName") + + +def test_get_comparator_bad_script(): + with pytest.raises(ComparatorNotFound): + with tempfile.NamedTemporaryFile() as tmpf: + tmpf.close() + get_comparator(tmpf.name) + + +def test_get_comparator_benchmark_name(): + comparator_klass = get_comparator("BenchmarkComparator") + assert comparator_klass.__name__ == "BenchmarkComparator" + + +def test_get_comparator_benchmark_script(): + # If the get_comparator method is working for scripts, then + # it should find the first defined class in this file, or the + # first imported class that matches it + comparator_klass = get_comparator(__file__) + assert comparator_klass.__name__ == "BenchmarkComparator" + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_preset.t b/tools/tryselect/test/test_preset.t new file mode 100644 index 0000000000..13e6946d32 --- /dev/null +++ b/tools/tryselect/test/test_preset.t @@ -0,0 +1,390 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test preset with no subcommand + + $ ./mach try $testargs --save foo -b do -p linux -u mochitests -t none --tag foo + preset saved, run with: --preset=foo + + $ ./mach try $testargs --preset foo + Commit message: + try: -b do -p linux -u mochitests -t none --tag foo + + Pushed via `mach try syntax` + + $ ./mach try syntax $testargs --preset foo + Commit message: + try: -b do -p linux -u mochitests -t none --tag foo + + Pushed via `mach try syntax` + + $ ./mach try $testargs --list-presets + Presets from */mozbuild/try_presets.yml: (glob) + + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + + $ unset EDITOR + $ ./mach try $testargs --edit-presets + error: must set the $EDITOR environment variable to use --edit-presets + $ export EDITOR=cat + $ ./mach try $testargs --edit-presets + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + +Test preset with syntax subcommand + + $ ./mach try syntax $testargs --save bar -b do -p win32 -u none -t all --tag bar + preset saved, run with: --preset=bar + + $ ./mach try syntax $testargs --preset bar + Commit message: + try: -b do -p win32 -u none -t all --tag bar + + Pushed via `mach try syntax` + + $ ./mach try $testargs --preset bar + Commit message: + try: -b do -p win32 -u none -t all --tag bar + + Pushed via `mach try syntax` + + $ ./mach try syntax $testargs --list-presets + Presets from */mozbuild/try_presets.yml: (glob) + + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + + $ ./mach try syntax $testargs --edit-presets + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + +Test preset with fuzzy subcommand + + $ ./mach try fuzzy $testargs --save baz -q "'foo" --rebuild 5 + preset saved, run with: --preset=baz + + $ ./mach try fuzzy $testargs --preset baz + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + + $ ./mach try $testargs --preset baz + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + +Queries can be appended to presets + + $ ./mach try fuzzy $testargs --preset baz -q "'build" + Commit message: + Fuzzy query='foo&query='build + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "build-baz", + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + + $ ./mach try $testargs --preset baz -xq "'opt" + Commit message: + Fuzzy query='foo&query='opt + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "test/foo-opt" + ] + } + }, + "version": 2 + } + + + $ ./mach try fuzzy $testargs --list-presets + Presets from */mozbuild/try_presets.yml: (glob) + + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + baz: + dry_run: true + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + + $ ./mach try fuzzy $testargs --edit-presets + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + baz: + dry_run: true + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + +Test gecko-profile argument handling. Add in profiling to a preset. + + $ ./mach try fuzzy $testargs --preset baz --gecko-profile-features=nostacksampling,cpu + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "gecko-profile": true, + "gecko-profile-features": "nostacksampling,cpu", + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + +Check whether the gecko-profile flags can be used from a preset, and check +dashes vs underscores (presets save with underscores to match ArgumentParser +settings; everything else uses dashes.) + + $ ./mach try fuzzy $testargs --save profile -q "'foo" --rebuild 5 --gecko-profile-features=nostacksampling,cpu + preset saved, run with: --preset=profile + + $ ./mach try fuzzy $testargs --preset profile + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": false, + "try_task_config": { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "gecko-profile": true, + "gecko-profile-features": "nostacksampling,cpu", + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ] + } + }, + "version": 2 + } + + $ EDITOR=cat ./mach try fuzzy $testargs --edit-preset profile + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + baz: + dry_run: true + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + profile: + dry_run: true + gecko_profile_features: nostacksampling,cpu + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + + $ rm $MOZBUILD_STATE_PATH/try_presets.yml diff --git a/tools/tryselect/test/test_presets.py b/tools/tryselect/test/test_presets.py new file mode 100644 index 0000000000..89cc810808 --- /dev/null +++ b/tools/tryselect/test/test_presets.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest + +TASKS = [ + { + "kind": "build", + "label": "build-windows", + "attributes": { + "build_platform": "windows", + }, + }, + { + "kind": "test", + "label": "test-windows-mochitest-e10s", + "attributes": { + "unittest_suite": "mochitest", + "unittest_flavor": "browser-chrome", + "mochitest_try_name": "mochitest", + }, + }, +] + + +@pytest.fixture(autouse=True) +def skip_taskgraph_generation(monkeypatch, tg): + def fake_generate_tasks(*args, **kwargs): + return tg + + from tryselect import tasks + + monkeypatch.setattr(tasks, "generate_tasks", fake_generate_tasks) + + +@pytest.mark.xfail( + strict=False, reason="Bug 1635204: " "test_shared_presets[sample-suites] is flaky" +) +def test_shared_presets(run_mach, shared_name, shared_preset): + """This test makes sure that we don't break any of the in-tree presets when + renaming/removing variables in any of the selectors. + """ + assert "description" in shared_preset + assert "selector" in shared_preset + + selector = shared_preset["selector"] + if selector == "fuzzy": + assert "query" in shared_preset + assert isinstance(shared_preset["query"], list) + + # Run the preset and assert there were no exceptions. + assert run_mach(["try", "--no-push", "--preset", shared_name]) == 0 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_push.py b/tools/tryselect/test/test_push.py new file mode 100644 index 0000000000..97f2e047d7 --- /dev/null +++ b/tools/tryselect/test/test_push.py @@ -0,0 +1,54 @@ +import mozunit +import pytest +from tryselect import push + + +@pytest.mark.parametrize( + "method,labels,params,routes,expected", + ( + pytest.param( + "fuzzy", + ["task-foo", "task-bar"], + None, + None, + { + "parameters": { + "optimize_target_tasks": False, + "try_task_config": { + "env": {"TRY_SELECTOR": "fuzzy"}, + "tasks": ["task-bar", "task-foo"], + }, + }, + "version": 2, + }, + id="basic", + ), + pytest.param( + "fuzzy", + ["task-foo"], + {"existing_tasks": {"task-foo": "123", "task-bar": "abc"}}, + None, + { + "parameters": { + "existing_tasks": {"task-bar": "abc"}, + "optimize_target_tasks": False, + "try_task_config": { + "env": {"TRY_SELECTOR": "fuzzy"}, + "tasks": ["task-foo"], + }, + }, + "version": 2, + }, + id="existing_tasks", + ), + ), +) +def test_generate_try_task_config(method, labels, params, routes, expected): + assert ( + push.generate_try_task_config(method, labels, params=params, routes=routes) + == expected + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_release.py b/tools/tryselect/test/test_release.py new file mode 100644 index 0000000000..a1a0d348b2 --- /dev/null +++ b/tools/tryselect/test/test_release.py @@ -0,0 +1,43 @@ +# Any copyright is dedicated to the Public Domain. +# https://creativecommons.org/publicdomain/zero/1.0/ + +from textwrap import dedent + +import mozunit + + +def test_release(run_mach, capfd): + cmd = [ + "try", + "release", + "--no-push", + "--version=97.0", + ] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + expected = dedent( + """ + Commit message: + staging release: 97.0 + + Pushed via `mach try release` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": true, + "release_type": "release", + "target_tasks_method": "staging_release_builds" + }, + "version": 2 + } + + """ + ).lstrip() + assert expected in output + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_scriptworker.py b/tools/tryselect/test/test_scriptworker.py new file mode 100644 index 0000000000..e25279ace4 --- /dev/null +++ b/tools/tryselect/test/test_scriptworker.py @@ -0,0 +1,39 @@ +# Any copyright is dedicated to the Public Domain. +# https://creativecommons.org/publicdomain/zero/1.0/ + +import re +from textwrap import dedent + +import mozunit + + +def test_release(run_mach, capfd): + cmd = [ + "try", + "scriptworker", + "--no-push", + "tree", + ] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + expected = re.compile( + dedent( + r""" + Pushed via `mach try scriptworker` + Calculated try_task_config.json: + { + "parameters": { + "app_version": "\d+\.\d+", + "build_number": \d+, + """ + ).lstrip(), + re.MULTILINE, + ) + assert expected.search(output) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_task_configs.py b/tools/tryselect/test/test_task_configs.py new file mode 100644 index 0000000000..afa21bfabf --- /dev/null +++ b/tools/tryselect/test/test_task_configs.py @@ -0,0 +1,257 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import inspect +from argparse import ArgumentParser +from textwrap import dedent + +import mozunit +import pytest +from tryselect.task_config import Pernosco, all_task_configs + +TC_URL = "https://firefox-ci-tc.services.mozilla.com" +TH_URL = "https://treeherder.mozilla.org" + +# task configs have a list of tests of the form (input, expected) +TASK_CONFIG_TESTS = { + "artifact": [ + (["--no-artifact"], None), + ( + ["--artifact"], + {"try_task_config": {"use-artifact-builds": True, "disable-pgo": True}}, + ), + ], + "chemspill-prio": [ + ([], None), + (["--chemspill-prio"], {"try_task_config": {"chemspill-prio": True}}), + ], + "env": [ + ([], None), + ( + ["--env", "foo=bar", "--env", "num=10"], + {"try_task_config": {"env": {"foo": "bar", "num": "10"}}}, + ), + ], + "path": [ + ([], None), + ( + ["dom/indexedDB"], + { + "try_task_config": { + "env": {"MOZHARNESS_TEST_PATHS": '{"xpcshell": ["dom/indexedDB"]}'} + } + }, + ), + ( + ["dom/indexedDB", "testing"], + { + "try_task_config": { + "env": { + "MOZHARNESS_TEST_PATHS": '{"xpcshell": ["dom/indexedDB", "testing"]}' + } + } + }, + ), + (["invalid/path"], SystemExit), + ], + "pernosco": [ + ([], None), + ], + "rebuild": [ + ([], None), + (["--rebuild", "10"], {"try_task_config": {"rebuild": 10}}), + (["--rebuild", "1"], SystemExit), + (["--rebuild", "21"], SystemExit), + ], + "worker-overrides": [ + ([], None), + ( + ["--worker-override", "alias=worker/pool"], + {"try_task_config": {"worker-overrides": {"alias": "worker/pool"}}}, + ), + ( + [ + "--worker-override", + "alias=worker/pool", + "--worker-override", + "alias=other/pool", + ], + SystemExit, + ), + ( + ["--worker-suffix", "b-linux=-dev"], + { + "try_task_config": { + "worker-overrides": {"b-linux": "gecko-1/b-linux-dev"} + } + }, + ), + ( + [ + "--worker-override", + "b-linux=worker/pool" "--worker-suffix", + "b-linux=-dev", + ], + SystemExit, + ), + ], + "new-test-config": [ + ([], None), + (["--new-test-config"], {"try_task_config": {"new-test-config": True}}), + ], +} + + +@pytest.fixture +def config_patch_resolver(patch_resolver): + def inner(paths): + patch_resolver( + [], [{"flavor": "xpcshell", "srcdir_relpath": path} for path in paths] + ) + + return inner + + +def test_task_configs(config_patch_resolver, task_config, args, expected): + parser = ArgumentParser() + + cfg = all_task_configs[task_config]() + cfg.add_arguments(parser) + + if inspect.isclass(expected) and issubclass(expected, BaseException): + with pytest.raises(expected): + args = parser.parse_args(args) + if task_config == "path": + config_patch_resolver(**vars(args)) + + cfg.get_parameters(**vars(args)) + else: + args = parser.parse_args(args) + if task_config == "path": + config_patch_resolver(**vars(args)) + + params = cfg.get_parameters(**vars(args)) + assert params == expected + + +@pytest.fixture +def patch_ssh_user(mocker): + def inner(user): + mock_stdout = mocker.Mock() + mock_stdout.stdout = dedent( + f""" + key1 foo + user {user} + key2 bar + """ + ) + return mocker.patch( + "tryselect.util.ssh.subprocess.run", return_value=mock_stdout + ) + + return inner + + +def test_pernosco(patch_ssh_user): + patch_ssh_user("user@mozilla.com") + parser = ArgumentParser() + + cfg = Pernosco() + cfg.add_arguments(parser) + args = parser.parse_args(["--pernosco"]) + params = cfg.get_parameters(**vars(args)) + assert params == {"try_task_config": {"env": {"PERNOSCO": "1"}}} + + +def test_exisiting_tasks(responses, patch_ssh_user): + parser = ArgumentParser() + cfg = all_task_configs["existing-tasks"]() + cfg.add_arguments(parser) + + user = "user@example.com" + rev = "a" * 40 + task_id = "abc" + label_to_taskid = {"task-foo": "123", "task-bar": "456"} + + args = ["--use-existing-tasks"] + args = parser.parse_args(args) + + responses.add( + responses.GET, + f"{TH_URL}/api/project/try/push/?count=1&author={user}", + json={"meta": {"count": 1}, "results": [{"revision": rev}]}, + ) + + responses.add( + responses.GET, + f"{TC_URL}/api/index/v1/task/gecko.v2.try.revision.{rev}.taskgraph.decision", + json={"taskId": task_id}, + ) + + responses.add( + responses.GET, + f"{TC_URL}/api/queue/v1/task/{task_id}/artifacts/public/label-to-taskid.json", + json=label_to_taskid, + ) + + m = patch_ssh_user(user) + params = cfg.get_parameters(**vars(args)) + assert params == {"existing_tasks": label_to_taskid} + + m.assert_called_once_with( + ["ssh", "-G", "hg.mozilla.org"], text=True, check=True, capture_output=True + ) + + +def test_exisiting_tasks_task_id(responses): + parser = ArgumentParser() + cfg = all_task_configs["existing-tasks"]() + cfg.add_arguments(parser) + + task_id = "abc" + label_to_taskid = {"task-foo": "123", "task-bar": "456"} + + args = ["--use-existing-tasks", f"task-id={task_id}"] + args = parser.parse_args(args) + + responses.add( + responses.GET, + f"{TC_URL}/api/queue/v1/task/{task_id}/artifacts/public/label-to-taskid.json", + json=label_to_taskid, + ) + + params = cfg.get_parameters(**vars(args)) + assert params == {"existing_tasks": label_to_taskid} + + +def test_exisiting_tasks_rev(responses): + parser = ArgumentParser() + cfg = all_task_configs["existing-tasks"]() + cfg.add_arguments(parser) + + rev = "aaaaaa" + task_id = "abc" + label_to_taskid = {"task-foo": "123", "task-bar": "456"} + + args = ["--use-existing-tasks", f"rev={rev}"] + args = parser.parse_args(args) + + responses.add( + responses.GET, + f"{TC_URL}/api/index/v1/task/gecko.v2.try.revision.{rev}.taskgraph.decision", + json={"taskId": task_id}, + ) + + responses.add( + responses.GET, + f"{TC_URL}/api/queue/v1/task/{task_id}/artifacts/public/label-to-taskid.json", + json=label_to_taskid, + ) + + params = cfg.get_parameters(**vars(args)) + assert params == {"existing_tasks": label_to_taskid} + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_tasks.py b/tools/tryselect/test/test_tasks.py new file mode 100644 index 0000000000..2e99c72d8b --- /dev/null +++ b/tools/tryselect/test/test_tasks.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import mozunit +import pytest +from tryselect.tasks import cache_key, filter_tasks_by_paths, resolve_tests_by_suite + + +def test_filter_tasks_by_paths(patch_resolver): + tasks = {"foobar/xpcshell-1": {}, "foobar/mochitest": {}, "foobar/xpcshell": {}} + + patch_resolver(["xpcshell"], {}) + assert list(filter_tasks_by_paths(tasks, "dummy")) == [] + + patch_resolver([], [{"flavor": "xpcshell"}]) + assert list(filter_tasks_by_paths(tasks, "dummy")) == [ + "foobar/xpcshell-1", + "foobar/xpcshell", + ] + + +@pytest.mark.parametrize( + "input, tests, expected", + ( + pytest.param( + ["xpcshell.js"], + [{"flavor": "xpcshell", "srcdir_relpath": "xpcshell.js"}], + {"xpcshell": ["xpcshell.js"]}, + id="single test", + ), + pytest.param( + ["xpcshell.ini"], + [ + { + "flavor": "xpcshell", + "srcdir_relpath": "xpcshell.js", + "manifest_relpath": "xpcshell.ini", + }, + ], + {"xpcshell": ["xpcshell.ini"]}, + id="single manifest", + ), + pytest.param( + ["xpcshell.js", "mochitest.js"], + [ + {"flavor": "xpcshell", "srcdir_relpath": "xpcshell.js"}, + {"flavor": "mochitest", "srcdir_relpath": "mochitest.js"}, + ], + { + "xpcshell": ["xpcshell.js"], + "mochitest-plain": ["mochitest.js"], + }, + id="two tests", + ), + pytest.param( + ["test/xpcshell.ini"], + [ + { + "flavor": "xpcshell", + "srcdir_relpath": "test/xpcshell.js", + "manifest_relpath": os.path.join("test", "xpcshell.ini"), + }, + ], + {"xpcshell": ["test/xpcshell.ini"]}, + id="mismatched path separators", + ), + ), +) +def test_resolve_tests_by_suite(patch_resolver, input, tests, expected): + patch_resolver([], tests) + assert resolve_tests_by_suite(input) == expected + + +@pytest.mark.parametrize( + "attr,params,disable_target_task_filter,expected", + ( + ("target_task_set", None, False, "target_task_set"), + ("target_task_set", {"project": "autoland"}, False, "target_task_set"), + ("target_task_set", {"project": "mozilla-central"}, False, "target_task_set"), + ("target_task_set", None, True, "target_task_set-uncommon"), + ("full_task_set", {"project": "pine"}, False, "full_task_set-pine"), + ("full_task_set", None, True, "full_task_set"), + ), +) +def test_cache_key(attr, params, disable_target_task_filter, expected): + assert cache_key(attr, params, disable_target_task_filter) == expected + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/try_presets.yml b/tools/tryselect/try_presets.yml new file mode 100644 index 0000000000..ebdc94aa03 --- /dev/null +++ b/tools/tryselect/try_presets.yml @@ -0,0 +1,298 @@ +--- +# Presets defined here will be available to all users. Run them with: +# $ mach try --preset <name> +# +# If editing this file, make sure to run: +# $ mach python-test tools/tryselect/test/test_presets.py +# +# Descriptions are required. Please keep this in alphabetical order. + +# yamllint disable rule:line-length + +builds: + selector: fuzzy + description: >- + Run builds without any of the extras. + query: + - "^build- !fuzzing !notarization !reproduced !rusttests !signing !upload-symbols" + +builds-debug: + selector: fuzzy + description: >- + Run the bare minimum of debug build jobs to ensure builds work on + all tier-1 platforms. + query: + - "^build- 'debug !fuzzing !rusttests !signing !plain !asan !tsan !noopt !toolchain !upload-symbols" + +builds-debugopt: + selector: fuzzy + description: >- + Run the bare minimum of debug and opt build jobs to ensure builds work on + all tier-1 platforms. + query: + - "^build- !fuzzing !rusttests !signing !plain !asan !tsan !noopt !toolchain !upload-symbols" + +desktop-frontend: + description: >- + Run mochitest-browser, xpcshell, mochitest-chrome, mochitest-a11y, + marionette, firefox-ui-functional on all desktop platforms. + Excludes non-shipped/duplicate configurations like asan/tsan/msix + to reduce the runtime of the push as well as infra load. + Use with --artifact to speed up your trypush. + If this is green, you can be 99% sure that any frontend change will + stick on central. + selector: fuzzy + query: + # Runs 64-bit frontend-tests, plus win7. Tries to avoid running + # asan/tsan because they're not available as artifact builds, and + # rarely offer different results from debug/opt. It also avoids running + # msix/swr/a11y-checks/gpu/nofis/headless variants of otherwise + # identical tests, as again those are unlikely to show different + # results for frontend-only changes. + # This won't run 32-bit debug tests, which seems an acceptable + # trade-off for query complexity + runtime on infrastructure. + - "'browser-chrome 'windows7 | '64 !spi !asan !tsan !msix !a11y !swr | 'linux" + - "'mochitest-chrome 'windows7 | '64 !spi !asan !tsan !swr !gpu" + - "'xpcshell 'windows7 | '64 !spi !asan !tsan !msix !nofis !condprof" + - "'browser-a11y | 'mochitest-a11y 'windows7 | '64 !spi !asan !tsan !no-cache !swr" + - "'marionette 'windows7 | '64 !asan !source !headless !swr" + - "'firefox-ui-functional 'windows7 | '64 !asan !tsan" + +devtools: + selector: fuzzy + description: >- + Runs the tests relevant to the Firefox Devtools + query: + - "'node-debugger | 'node-devtools" + - "'mozlint-eslint" + # Windows: skip jobs on asan and 32 bits platforms + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'windows !asan !-32" + # macos: no extra platform to filter out + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'macosx" + # Linux is being named "linux1804" and may change over time, so use a more flexible search + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'linux '64-qr/ !swr" + - "'xpcshell 'linux !nofis '64-qr/" + +devtools-linux: + selector: fuzzy + description: >- + Runs the tests relevant to the Firefox Devtools, on Linux only. + query: + - "'node-debugger | 'node-devtools" + - "'mozlint-eslint" + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'linux '64-qr/ !swr" + - "'xpcshell 'linux !nofis '64-qr/" + +fpush-linux-android: + selector: fuzzy + description: >- + Runs correctness test suites on Linux and Android emulator platforms, as + well as builds across main platforms. The resulting jobs on TreeHerder + used to end up looking like a "F" shape (not so much these days) and so + this is typically referred to as an F-push. This is useful to do as a + general sanity check on changes to cross-platform Gecko code where you + unsure of what tests might be affected. Linux and Android (emulator) + test coverage are relatively cheap to run and cover a lot of the + codebase, while the builds on other platforms catch compilation problems + that might result from only building locally on one platform. + query: + - "'test-linux1804 'debug- !-shippable !-asan" + - "'test-android-em 'debug" + - "^build !-shippable !-signing !-asan !-fuzzing !-rusttests !-base-toolchain !-aar-" + +geckodriver: + selector: fuzzy + description: >- + Runs the tests relevant to geckodriver, which implements the WebDriver + specification. This preset can be filtered down further to limit it to + a specific platform or other tasks only. For example: + |mach try --preset geckodriver -xq "'linux"| + query: + - "'rusttests" + - "'platform 'wdspec 'debug 'nofis" + - "'browsertime 'amazon 'shippable 'firefox 'nofis" + +layout: + selector: fuzzy + description: >- + Runs the tests most relevant to layout. + This preset can be filtered down further to limit it to + a specific platform or build configuration. For example: + |mach try --preset layout -xq "linux64 'opt"| + query: + # Most mochitest + reftest + crashtest + wpt + - "!asan !tsan !jsreftest !shippable !webgl !condprof !media !webgpu 'mochitest | 'web-platform | 'crashtest | 'reftest" + # Style system unit tests + - "'rusttests" + +media-full: + selector: fuzzy + description: >- + Runs tests that exercise media playback and WebRTC code. + query: + - "mochitest-media !dfpi !nofis" + - "mochitest-media android !spi !swr !lite" + - "mochitest-browser-chrome !dfpi !nofis !a11y" + - "mochitest-browser-media" + - "web-platform-tests !dfpi !nofis !shippable" + - "web-platform-tests android !wdspec !spi !swr !lite" + - "crashtest !wdspec !nofis" + - "crashtest android !wdspec !spi !swr !lite" + - "'gtest" + +mochitest-bc: + description: >- + Runs mochitest-browser-chrome on all Desktop platforms in both opt + and debug. Excludes jobs that require non-artifact builds (asan, + tsan, msix, etc.) and some non-default configurations. For frontend + only changes, use this with --artifact to speed up your trypushes. + query: + - "'browser-chrome 'windows7 | '64 !tsan !asan !msix !spi !a11y !swr | 'linux" + selector: fuzzy + +perf: + selector: fuzzy + description: >- + Runs all performance (raptor and talos) tasks across all platforms. + This preset can be filtered down further (e.g to limit it to a specific + platform) via |mach try --preset perf -xq "'windows"|. + + Android hardware platforms are excluded due to resource limitations. + query: + - "^test- !android-hw 'raptor | 'talos" + rebuild: 5 + +perf-chrome: + description: >- + Runs the talos tests most likely to change when making a change to + the browser chrome. This skips a number of talos jobs that are unlikely + to be affected in order to conserve resources. + query: + - "opt-talos- 'chrome | 'svg | 'session | 'tabswitch | 'other | 'g5" + rebuild: 6 + selector: fuzzy + +remote-protocol: + selector: fuzzy + description: >- + Runs the tests relevant to the Remote protocol, which underpins + many test harnesses as well as our CDP and WebDriver implementations. + This preset can be filtered down further to limit it to a specific + platform or to opt/debug tasks only. For example: + |mach try --preset remote-protocol -xq "'linux 'opt"| + query: + - "'awsy-base" + - "'firefox-ui" + - "'marionette !swr | harness" + - "'mochitest-browser !spi !swr !nofis '-1$" + - "'mochitest-remote !spi !swr" + - "'platform 'reftest !swr !nofis | 'android !-lite -1$" + - "'platform 'wdspec !swr" + - "'platform !reftest !wdspec !swr !nofis | 'android !-lite -1$" + - "'puppeteer" + - "'reftest !platform !gpu !swr !no-accel !nofis | 'android !-lite -1$" + - "'xpcshell !spi !tsan !-lite" + +sample-suites: + selector: fuzzy + description: >- + Runs one chunk of every test suite plus all suites that aren't chunked. + It is useful for testing infrastructure changes that can affect the + harnesses themselves but are unlikely to break specific tests. + query: + - ^test- -1$ + # Only run a single talos + raptor suite per platform + - ^test- !1$ !2$ !3$ !4$ !5$ !6$ !7$ !8$ !9$ !0$ !raptor !talos + - ^test- 'raptor-speedometer | 'talos-g1 + +sm-shell-all: + selector: fuzzy + description: <- + Runs a set of tests aimed to give a reasonable level of confidence for + basic SpiderMonkey changes (shell only), all platforms + query: + - "'spidermonkey | 'shell-haz" + - "!shippable !android 'jittest" # macosx64 jittests + +sm-shell: + selector: fuzzy + description: <- + Runs a set of tests aimed to give a reasonable level of confidence for + basic SpiderMonkey changes (shell only) (linux only) + query: + - "!win !osx 'spidermonkey | 'shell-haz" + + +sm-all: + selector: fuzzy + description: <- + Runs a set of tests aimed to give a reasonable level of confidence for + basic SpiderMonkey changes, including those that would require a + browser build. + query: + - "'spidermonkey | 'hazard" + - "!android !asan !shippable 'xpcshell" + - "!android !asan !shippable 'jsreftest" + - "!shippable !android 'jittest" # macosx64 jittests + +webextensions: + selector: fuzzy + description: <- + Runs most of the unit tests of WebExtension code across all desktop + platforms and Android, including mochitests, xpcshell and test-verify. + GeckoView JUnit tests are NOT run. + paths: # must be duplicate of test_paths, see bug 1556445 + - browser/components/extensions/test/ + - mobile/android/components/extensions/test/ + - toolkit/components/extensions/test/ + - toolkit/mozapps/extensions/test/ + test_paths: # must be duplicate of paths, see bug 1556445 + - browser/components/extensions/test/ + - mobile/android/components/extensions/test/ + - toolkit/components/extensions/test/ + - toolkit/mozapps/extensions/test/ + query: + - "'64-qr/ | 'windows11-64-2009-qr !wpt !gpu !msix" + +webgpu: + selector: fuzzy + description: >- + Runs the tests relevant to WebGPU. + query: + - "'webgpu" + - "source-test-mozlint-updatebot" + - "source-test-vendor-rust" + +webrender: + selector: fuzzy + description: >- + Runs the conformance tests relevant to WebRender. + query: + - "!talos !raptor !shippable !asan '-qr" + - "^webrender-" + +webrender-reftests: + selector: fuzzy + description: >- + Runs the reftests relevant to WebRender. + query: + - "!talos !raptor !shippable !asan !nofis 'reftest" + +webrender-reftests-linux: + selector: fuzzy + description: >- + Runs the reftests relevant to WebRender on linux only. + query: + - "!talos !raptor !shippable !asan !nofis 'linux 'reftest" + +webrender-perf: + selector: fuzzy + description: >- + Runs the performance tests relevant to WebRender. + query: + - "'-qr 'svgr" + - "'-qr 'g1" + - "'-qr 'g4" + - "'-qr 'tp5" + - "'-qr 'talos-webgl" + - "'-qr 'motionmark-animometer" diff --git a/tools/tryselect/util/__init__.py b/tools/tryselect/util/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/util/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/util/dicttools.py b/tools/tryselect/util/dicttools.py new file mode 100644 index 0000000000..465e4a43de --- /dev/null +++ b/tools/tryselect/util/dicttools.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import copy + + +def merge_to(source, dest): + """ + Merge dict and arrays (override scalar values) + + Keys from source override keys from dest, and elements from lists in source + are appended to lists in dest. + + :param dict source: to copy from + :param dict dest: to copy to (modified in place) + """ + + for key, value in source.items(): + # Override mismatching or empty types + if type(value) != type(dest.get(key)): # noqa + dest[key] = source[key] + continue + + # Merge dict + if isinstance(value, dict): + merge_to(value, dest[key]) + continue + + if isinstance(value, list): + dest[key] = dest[key] + source[key] + continue + + dest[key] = source[key] + + return dest + + +def merge(*objects): + """ + Merge the given objects, using the semantics described for merge_to, with + objects later in the list taking precedence. From an inheritance + perspective, "parents" should be listed before "children". + + Returns the result without modifying any arguments. + """ + if len(objects) == 1: + return copy.deepcopy(objects[0]) + return merge_to(objects[-1], merge(*objects[:-1])) diff --git a/tools/tryselect/util/estimates.py b/tools/tryselect/util/estimates.py new file mode 100644 index 0000000000..a15ad72831 --- /dev/null +++ b/tools/tryselect/util/estimates.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +from datetime import datetime, timedelta + +TASK_DURATION_CACHE = "task_duration_history.json" +GRAPH_QUANTILE_CACHE = "graph_quantile_cache.csv" +TASK_DURATION_TAG_FILE = "task_duration_tag.json" + + +def find_all_dependencies(graph, tasklist): + all_dependencies = dict() + + def find_dependencies(task): + dependencies = set() + if task in all_dependencies: + return all_dependencies[task] + if task not in graph: + # Don't add tasks (and so durations) for + # things optimised out. + return dependencies + dependencies.add(task) + for dep in graph.get(task, list()): + all_dependencies[dep] = find_dependencies(dep) + dependencies.update(all_dependencies[dep]) + return dependencies + + full_deps = set() + for task in tasklist: + full_deps.update(find_dependencies(task)) + + # Since these have been asked for, they're not inherited dependencies. + return sorted(full_deps - set(tasklist)) + + +def find_longest_path(graph, tasklist, duration_data): + dep_durations = dict() + + def find_dependency_durations(task): + if task in dep_durations: + return dep_durations[task] + + durations = [find_dependency_durations(dep) for dep in graph.get(task, list())] + durations.append(0.0) + md = max(durations) + duration_data.get(task, 0.0) + dep_durations[task] = md + return md + + longest_paths = [find_dependency_durations(task) for task in tasklist] + # Default in case there are no tasks + if longest_paths: + return max(longest_paths) + else: + return 0 + + +def determine_percentile(quantiles_file, duration): + duration = duration.total_seconds() + + with open(quantiles_file) as f: + f.readline() # skip header + boundaries = [float(l.strip()) for l in f.readlines()] + + boundaries.sort() + for i, v in enumerate(boundaries): + if duration < v: + break + # Estimate percentile from len(boundaries)-quantile + return int(100 * i / len(boundaries)) + + +def task_duration_data(cache_dir): + with open(os.path.join(cache_dir, TASK_DURATION_CACHE)) as f: + return json.load(f) + + +def duration_summary(graph_cache_file, tasklist, cache_dir): + durations = task_duration_data(cache_dir) + + graph = dict() + if graph_cache_file: + with open(graph_cache_file) as f: + graph = json.load(f) + dependencies = find_all_dependencies(graph, tasklist) + longest_path = find_longest_path(graph, tasklist, durations) + dependency_duration = 0.0 + for task in dependencies: + dependency_duration += int(durations.get(task, 0.0)) + + total_requested_duration = 0.0 + for task in tasklist: + duration = int(durations.get(task, 0.0)) + total_requested_duration += duration + output = dict() + + total_requested_duration = timedelta(seconds=total_requested_duration) + total_dependency_duration = timedelta(seconds=dependency_duration) + + output["selected_duration"] = total_requested_duration + output["dependency_duration"] = total_dependency_duration + output["dependency_count"] = len(dependencies) + output["selected_count"] = len(tasklist) + + percentile = None + graph_quantile_cache = os.path.join(cache_dir, GRAPH_QUANTILE_CACHE) + if os.path.isfile(graph_quantile_cache): + percentile = determine_percentile( + graph_quantile_cache, total_dependency_duration + total_requested_duration + ) + if percentile: + output["percentile"] = percentile + + output["wall_duration_seconds"] = timedelta(seconds=int(longest_path)) + output["eta_datetime"] = datetime.now() + timedelta(seconds=longest_path) + + output["task_durations"] = { + task: int(durations.get(task, 0.0)) for task in tasklist + } + + return output diff --git a/tools/tryselect/util/fzf.py b/tools/tryselect/util/fzf.py new file mode 100644 index 0000000000..63318fce18 --- /dev/null +++ b/tools/tryselect/util/fzf.py @@ -0,0 +1,424 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import platform +import shutil +import subprocess +import sys + +import mozfile +import six +from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks +from mach.util import get_state_dir +from mozboot.util import http_download_and_save +from mozbuild.base import MozbuildObject +from mozterm import Terminal +from packaging.version import Version + +from ..push import check_working_directory +from ..tasks import generate_tasks +from ..util.manage_estimates import ( + download_task_history_data, + make_trimmed_taskgraph_cache, +) + +terminal = Terminal() + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + +PREVIEW_SCRIPT = os.path.join(build.topsrcdir, "tools/tryselect/selectors/preview.py") + +FZF_MIN_VERSION = "0.20.0" +FZF_CURRENT_VERSION = "0.29.0" + +# It would make more sense to have the full filename be the key; but that makes +# the line too long and ./mach lint and black can't agree about what to about that. +# You can get these from the github release, e.g. +# https://github.com/junegunn/fzf/releases/download/0.24.1/fzf_0.24.1_checksums.txt +# However the darwin releases may not be included, so double check you have everything +FZF_CHECKSUMS = { + "linux_armv5.tar.gz": "61d3c2aa77b977ba694836fd1134da9272bd97ee490ececaf87959b985820111", + "linux_armv6.tar.gz": "db6b30fcbbd99ac4cf7e3ff6c5db1d3c0afcbe37d10ec3961bdc43e8c4f2e4f9", + "linux_armv7.tar.gz": "ed86f0e91e41d2cea7960a78e3eb175dc2a5fc1510380c195d0c3559bfdc701c", + "linux_arm64.tar.gz": "47988d8b68905541cbc26587db3ed1cfa8bc3aa8da535120abb4229b988f259e", + "linux_amd64.tar.gz": "0106f458b933be65edb0e8f0edb9a16291a79167836fd26a77ff5496269b5e9a", + "windows_armv5.zip": "08eaac45b3600d82608d292c23e7312696e7e11b6278b292feba25e8eb91c712", + "windows_armv6.zip": "8b6618726a9d591a45120fddebc29f4164e01ce6639ed9aa8fc79ab03eefcfed", + "windows_armv7.zip": "c167117b4c08f4f098446291115871ce5f14a8a8b22f0ca70e1b4342452ab5d7", + "windows_arm64.zip": "0cda7bf68850a3e867224a05949612405e63a4421d52396c1a6c9427d4304d72", + "windows_amd64.zip": "f0797ceee089017108c80b09086c71b8eec43d4af11ce939b78b1d5cfd202540", + "darwin_arm64.zip": "2571b4d381f1fc691e7603bbc8113a67116da2404751ebb844818d512dd62b4b", + "darwin_amd64.zip": "bc541e8ae0feb94efa96424bfe0b944f746db04e22f5cccfe00709925839a57f", + "openbsd_amd64.tar.gz": "b62343827ff83949c09d5e2c8ca0c1198d05f733c9a779ec37edd840541ccdab", + "freebsd_amd64.tar.gz": "f0367f2321c070d103589c7c7eb6a771bc7520820337a6c2fbb75be37ff783a9", +} + +FZF_INSTALL_MANUALLY = """ +The `mach try fuzzy` command depends on fzf. Please install it following the +appropriate instructions for your platform: + + https://github.com/junegunn/fzf#installation + +Only the binary is required, if you do not wish to install the shell and +editor integrations, download the appropriate binary and put it on your $PATH: + + https://github.com/junegunn/fzf/releases +""".lstrip() + +FZF_COULD_NOT_DETERMINE_PLATFORM = ( + """ +Could not automatically obtain the `fzf` binary because we could not determine +your Operating System. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_COULD_NOT_DETERMINE_MACHINE = ( + """ +Could not automatically obtain the `fzf` binary because we could not determine +your machine type. It's reported as '%s' and we don't handle that case; but fzf +may still be available as a prebuilt binary. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_NOT_SUPPORTED_X86 = ( + """ +We don't believe that a prebuilt binary for `fzf` if available on %s, but we +could be wrong. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_NOT_FOUND = ( + """ +Could not find the `fzf` binary. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_VERSION_FAILED = ( + """ +Could not obtain the 'fzf' version; we require version > 0.20.0 for some of +the features. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_INSTALL_FAILED = ( + """ +Failed to install fzf. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_HEADER = """ +For more shortcuts, see {t.italic_white}mach help try fuzzy{t.normal} and {t.italic_white}man fzf +{shortcuts} +""".strip() + +fzf_shortcuts = { + "ctrl-a": "select-all", + "ctrl-d": "deselect-all", + "ctrl-t": "toggle-all", + "alt-bspace": "beginning-of-line+kill-line", + "?": "toggle-preview", +} + +fzf_header_shortcuts = [ + ("select", "tab"), + ("accept", "enter"), + ("cancel", "ctrl-c"), + ("select-all", "ctrl-a"), + ("cursor-up", "up"), + ("cursor-down", "down"), +] + + +def get_fzf_platform(): + if platform.machine() in ["i386", "i686"]: + print(FZF_NOT_SUPPORTED_X86 % platform.machine()) + sys.exit(1) + + if platform.system().lower() == "windows": + if platform.machine().lower() in ["x86_64", "amd64"]: + return "windows_amd64.zip" + elif platform.machine().lower() == "arm64": + return "windows_arm64.zip" + else: + print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine()) + sys.exit(1) + elif platform.system().lower() == "darwin": + if platform.machine().lower() in ["x86_64", "amd64"]: + return "darwin_amd64.zip" + elif platform.machine().lower() == "arm64": + return "darwin_arm64.zip" + else: + print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine()) + sys.exit(1) + elif platform.system().lower() == "linux": + if platform.machine().lower() in ["x86_64", "amd64"]: + return "linux_amd64.tar.gz" + elif platform.machine().lower() == "arm64": + return "linux_arm64.tar.gz" + else: + print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine()) + sys.exit(1) + else: + print(FZF_COULD_NOT_DETERMINE_PLATFORM) + sys.exit(1) + + +def get_fzf_state_dir(): + return os.path.join(get_state_dir(), "fzf") + + +def get_fzf_filename(): + return "fzf-%s-%s" % (FZF_CURRENT_VERSION, get_fzf_platform()) + + +def get_fzf_download_link(): + return "https://github.com/junegunn/fzf/releases/download/%s/%s" % ( + FZF_CURRENT_VERSION, + get_fzf_filename(), + ) + + +def clean_up_state_dir(): + """ + We used to have a checkout of fzf that we would update. + Now we only download the bin and cpin the hash; so if + we find the old git checkout, wipe it + """ + + fzf_path = os.path.join(get_state_dir(), "fzf") + git_path = os.path.join(fzf_path, ".git") + if os.path.isdir(git_path): + shutil.rmtree(fzf_path, ignore_errors=True) + + # Also delete any existing fzf binary + fzf_bin = shutil.which("fzf", path=fzf_path) + if fzf_bin: + mozfile.remove(fzf_bin) + + # Make sure the state dir is present + if not os.path.isdir(fzf_path): + os.makedirs(fzf_path) + + +def download_and_install_fzf(): + clean_up_state_dir() + download_link = get_fzf_download_link() + download_file = get_fzf_filename() + download_destination_path = get_fzf_state_dir() + download_destination_file = os.path.join(download_destination_path, download_file) + http_download_and_save( + download_link, download_destination_file, FZF_CHECKSUMS[get_fzf_platform()] + ) + + mozfile.extract(download_destination_file, download_destination_path) + mozfile.remove(download_destination_file) + + +def get_fzf_version(fzf_bin): + cmd = [fzf_bin, "--version"] + try: + fzf_version = subprocess.check_output(cmd) + except subprocess.CalledProcessError: + print(FZF_VERSION_FAILED) + sys.exit(1) + + # Some fzf versions have extra, e.g 0.18.0 (ff95134) + fzf_version = six.ensure_text(fzf_version.split()[0]) + + return fzf_version + + +def should_force_fzf_update(fzf_bin): + fzf_version = get_fzf_version(fzf_bin) + + # 0.20.0 introduced passing selections through a temporary file, + # which is good for large ctrl-a actions. + if Version(fzf_version) < Version(FZF_MIN_VERSION): + print("fzf version is old, you must update to use ./mach try fuzzy.") + return True + return False + + +def fzf_bootstrap(update=False): + """ + Bootstrap fzf if necessary and return path to the executable. + + This function is a bit complicated. We fetch a new version of fzf if: + 1) an existing fzf is too outdated + 2) the user says --update and we are behind the recommended version + 3) no fzf can be found and + 3a) user passes --update + 3b) user agrees to a prompt + + """ + fzf_path = get_fzf_state_dir() + + fzf_bin = shutil.which("fzf") + if not fzf_bin: + fzf_bin = shutil.which("fzf", path=fzf_path) + + if fzf_bin and should_force_fzf_update(fzf_bin): # Case (1) + update = True + + if fzf_bin and not update: + return fzf_bin + + elif fzf_bin and update: + # Case 2 + fzf_version = get_fzf_version(fzf_bin) + if Version(fzf_version) < Version(FZF_CURRENT_VERSION) and update: + # Bug 1623197: We only want to run fzf's `install` if it's not in the $PATH + # Swap to os.path.commonpath when we're not on Py2 + if fzf_bin and update and not fzf_bin.startswith(fzf_path): + print( + "fzf installed somewhere other than {}, please update manually".format( + fzf_path + ) + ) + sys.exit(1) + + download_and_install_fzf() + print("Updated fzf to {}".format(FZF_CURRENT_VERSION)) + else: + print("fzf is the recommended version and does not need an update") + + else: # not fzf_bin: + if not update: + # Case 3b + install = input("Could not detect fzf, install it now? [y/n]: ") + if install.lower() != "y": + return + + # Case 3a and 3b-fall-through + download_and_install_fzf() + fzf_bin = shutil.which("fzf", path=fzf_path) + print("Installed fzf to {}".format(fzf_path)) + + return fzf_bin + + +def format_header(): + shortcuts = [] + for action, key in fzf_header_shortcuts: + shortcuts.append( + "{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}".format( + t=terminal, action=action, key=key + ) + ) + return FZF_HEADER.format(shortcuts=", ".join(shortcuts), t=terminal) + + +def run_fzf(cmd, tasks): + env = dict(os.environ) + env.update( + {"PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p])} + ) + # Make sure fzf uses Windows' shell rather than MozillaBuild bash or + # whatever our caller uses, since it doesn't quote the arguments properly + # and thus windows paths like: C:\moz\foo end up as C:mozfoo... + if platform.system() == "Windows": + env["SHELL"] = env["COMSPEC"] + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + env=env, + universal_newlines=True, + ) + out = proc.communicate("\n".join(tasks))[0].splitlines() + + selected = [] + query = None + if out: + query = out[0] + selected = out[1:] + return query, selected + + +def setup_tasks_for_fzf( + push, + parameters, + full=False, + disable_target_task_filter=False, + show_estimates=True, +): + check_working_directory(push) + tg = generate_tasks( + parameters, full=full, disable_target_task_filter=disable_target_task_filter + ) + all_tasks = sorted(tg.tasks.keys()) + + # graph_Cache created by generate_tasks, recreate the path to that file. + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + if full: + graph_cache = os.path.join(cache_dir, "full_task_graph") + dep_cache = os.path.join(cache_dir, "full_task_dependencies") + target_set = os.path.join(cache_dir, "full_task_set") + else: + graph_cache = os.path.join(cache_dir, "target_task_graph") + dep_cache = os.path.join(cache_dir, "target_task_dependencies") + target_set = os.path.join(cache_dir, "target_task_set") + + if show_estimates: + download_task_history_data(cache_dir=cache_dir) + make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set) + + if not full and not disable_target_task_filter: + # Put all_tasks into a list because it's used multiple times, and "filter()" + # returns a consumable iterator. + all_tasks = list(filter(filter_by_uncommon_try_tasks, all_tasks)) + + return all_tasks, dep_cache, cache_dir + + +def build_base_cmd( + fzf, dep_cache, cache_dir, show_estimates=True, preview_script=PREVIEW_SCRIPT +): + key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()] + base_cmd = [ + fzf, + "-m", + "--bind", + ",".join(key_shortcuts), + "--header", + format_header(), + "--preview-window=right:30%", + "--print-query", + ] + + if show_estimates: + base_cmd.extend( + [ + "--preview", + '{} {} -g {} -s -c {} -t "{{+f}}"'.format( + sys.executable, preview_script, dep_cache, cache_dir + ), + ] + ) + else: + base_cmd.extend( + [ + "--preview", + '{} {} -t "{{+f}}"'.format(sys.executable, preview_script), + ] + ) + + return base_cmd diff --git a/tools/tryselect/util/manage_estimates.py b/tools/tryselect/util/manage_estimates.py new file mode 100644 index 0000000000..23fa481228 --- /dev/null +++ b/tools/tryselect/util/manage_estimates.py @@ -0,0 +1,132 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +from datetime import datetime, timedelta + +import requests +import six + +TASK_DURATION_URL = ( + "https://storage.googleapis.com/mozilla-mach-data/task_duration_history.json" +) +GRAPH_QUANTILES_URL = ( + "https://storage.googleapis.com/mozilla-mach-data/machtry_quantiles.csv" +) +from .estimates import GRAPH_QUANTILE_CACHE, TASK_DURATION_CACHE, TASK_DURATION_TAG_FILE + + +def check_downloaded_history(tag_file, duration_cache, quantile_cache): + if not os.path.isfile(tag_file): + return False + + try: + with open(tag_file) as f: + duration_tags = json.load(f) + download_date = datetime.strptime( + duration_tags.get("download_date"), "%Y-%M-%d" + ) + if download_date < datetime.now() - timedelta(days=7): + return False + except (OSError, ValueError): + return False + + if not os.path.isfile(duration_cache): + return False + # Check for old format version of file. + with open(duration_cache) as f: + data = json.load(f) + if isinstance(data, list): + return False + if not os.path.isfile(quantile_cache): + return False + + return True + + +def download_task_history_data(cache_dir): + """Fetch task duration data exported from BigQuery.""" + task_duration_cache = os.path.join(cache_dir, TASK_DURATION_CACHE) + task_duration_tag_file = os.path.join(cache_dir, TASK_DURATION_TAG_FILE) + graph_quantile_cache = os.path.join(cache_dir, GRAPH_QUANTILE_CACHE) + + if check_downloaded_history( + task_duration_tag_file, task_duration_cache, graph_quantile_cache + ): + return + + try: + os.unlink(task_duration_tag_file) + os.unlink(task_duration_cache) + os.unlink(graph_quantile_cache) + except OSError: + print("No existing task history to clean up.") + + try: + r = requests.get(TASK_DURATION_URL, stream=True) + r.raise_for_status() + except requests.exceptions.RequestException as exc: + # This is fine, the durations just won't be in the preview window. + print( + "Error fetching task duration cache from {}: {}".format( + TASK_DURATION_URL, exc + ) + ) + return + + # The data retrieved from google storage is a newline-separated + # list of json entries, which Python's json module can't parse. + duration_data = list() + for line in r.text.splitlines(): + duration_data.append(json.loads(line)) + + # Reformat duration data to avoid list of dicts, as this is slow in the preview window + duration_data = {d["name"]: d["mean_duration_seconds"] for d in duration_data} + + with open(task_duration_cache, "w") as f: + json.dump(duration_data, f, indent=4) + + try: + r = requests.get(GRAPH_QUANTILES_URL, stream=True) + r.raise_for_status() + except requests.exceptions.RequestException as exc: + # This is fine, the percentile just won't be in the preview window. + print( + "Error fetching task group percentiles from {}: {}".format( + GRAPH_QUANTILES_URL, exc + ) + ) + return + + with open(graph_quantile_cache, "w") as f: + f.write(six.ensure_text(r.content)) + + with open(task_duration_tag_file, "w") as f: + json.dump({"download_date": datetime.now().strftime("%Y-%m-%d")}, f, indent=4) + + +def make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=None): + """Trim the taskgraph cache used for dependencies. + + Speeds up the fzf preview window to less human-perceptible + ranges.""" + if not os.path.isfile(graph_cache): + return + + target_task_set = set() + if target_file and os.path.isfile(target_file): + with open(target_file) as f: + target_task_set = set(json.load(f).keys()) + + with open(graph_cache) as f: + graph = json.load(f) + graph = { + name: list(defn["dependencies"].values()) + for name, defn in graph.items() + if name in target_task_set + } + with open(dep_cache, "w") as f: + json.dump(graph, f, indent=4) diff --git a/tools/tryselect/util/ssh.py b/tools/tryselect/util/ssh.py new file mode 100644 index 0000000000..7682306bc7 --- /dev/null +++ b/tools/tryselect/util/ssh.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import subprocess + + +def get_ssh_user(host="hg.mozilla.org"): + ssh_config = subprocess.run( + ["ssh", "-G", host], + text=True, + check=True, + capture_output=True, + ).stdout + + lines = [l.strip() for l in ssh_config.splitlines()] + for line in lines: + if not line: + continue + key, value = line.split(" ", 1) + if key.lower() == "user": + return value + + raise Exception(f"Could not detect ssh user for '{host}'!") diff --git a/tools/tryselect/watchman.json b/tools/tryselect/watchman.json new file mode 100644 index 0000000000..a41b1829d7 --- /dev/null +++ b/tools/tryselect/watchman.json @@ -0,0 +1,15 @@ +[ + "trigger", + ".", + { + "name": "rebuild-taskgraph-cache", + "expression": ["match", "taskcluster/**", "wholename"], + "command": [ + "./mach", + "python", + "-c", + "from tryselect.tasks import generate_tasks; generate_tasks()" + ], + "append_files": false + } +] diff --git a/tools/update-packaging/README b/tools/update-packaging/README new file mode 100644 index 0000000000..7029ffb8e5 --- /dev/null +++ b/tools/update-packaging/README @@ -0,0 +1,4 @@ +This directory contains a tool for generating update packages for +the update system described here: + + http://wiki.mozilla.org/Software_Update diff --git a/tools/update-packaging/app.mozbuild b/tools/update-packaging/app.mozbuild new file mode 100644 index 0000000000..92fd8822e7 --- /dev/null +++ b/tools/update-packaging/app.mozbuild @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + "/modules/libmar/src", + "/modules/libmar/tool", + "/other-licenses/bsdiff", +] diff --git a/tools/update-packaging/common.sh b/tools/update-packaging/common.sh new file mode 100755 index 0000000000..397ed21e25 --- /dev/null +++ b/tools/update-packaging/common.sh @@ -0,0 +1,201 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Code shared by update packaging scripts. +# Author: Darin Fisher +# + +# ----------------------------------------------------------------------------- +QUIET=0 + +# By default just assume that these tools exist on our path +MAR=${MAR:-mar} +MBSDIFF=${MBSDIFF:-mbsdiff} +XZ=${XZ:-xz} +$XZ --version > /dev/null 2>&1 +if [ $? -ne 0 ]; then + # If $XZ is not set and not found on the path then this is probably + # running on a windows buildbot. Some of the Windows build systems have + # xz.exe in topsrcdir/xz/. Look in the places this would be in both a + # mozilla-central and comm-central build. + XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe" + $XZ --version > /dev/null 2>&1 + if [ $? -ne 0 ]; then + XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe" + $XZ --version > /dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "xz was not found on this system!" + echo "exiting" + exit 1 + fi + fi +fi +# Ensure that we're always using the right compression settings +export XZ_OPT="-T1 -7e" + +# ----------------------------------------------------------------------------- +# Helper routines + +notice() { + echo "$*" 1>&2 +} + +verbose_notice() { + if [ $QUIET -eq 0 ]; then + notice "$*" + fi +} + +get_file_size() { + info=($(ls -ln "$1")) + echo ${info[4]} +} + +copy_perm() { + reference="$1" + target="$2" + + if [ -x "$reference" ]; then + chmod 0755 "$target" + else + chmod 0644 "$target" + fi +} + +make_add_instruction() { + f="$1" + filev3="$2" + + # Used to log to the console + if [ $4 ]; then + forced=" (forced)" + else + forced= + fi + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + verbose_notice " add-if \"$testdir\" \"$f\"" + echo "add-if \"$testdir\" \"$f\"" >> "$filev3" + else + verbose_notice " add \"$f\"$forced" + echo "add \"$f\"" >> "$filev3" + fi +} + +check_for_add_if_not_update() { + add_if_not_file_chk="$1" + + if [ "$(basename "$add_if_not_file_chk")" = "channel-prefs.js" -o \ + "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +make_add_if_not_instruction() { + f="$1" + filev3="$2" + + verbose_notice " add-if-not \"$f\" \"$f\"" + echo "add-if-not \"$f\" \"$f\"" >> "$filev3" +} + +make_patch_instruction() { + f="$1" + filev3="$2" + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + verbose_notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3" + else + verbose_notice " patch \"$f.patch\" \"$f\"" + echo "patch \"$f.patch\" \"$f\"" >> "$filev3" + fi +} + +append_remove_instructions() { + dir="$1" + filev3="$2" + + if [ -f "$dir/removed-files" ]; then + listfile="$dir/removed-files" + elif [ -f "$dir/Contents/Resources/removed-files" ]; then + listfile="$dir/Contents/Resources/removed-files" + fi + if [ -n "$listfile" ]; then + # Map spaces to pipes so that we correctly handle filenames with spaces. + files=($(cat "$listfile" | tr " " "|" | sort -r)) + num_files=${#files[*]} + for ((i=0; $i<$num_files; i=$i+1)); do + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. + if [ -n "$f" ]; then + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + verbose_notice " rmdir \"$f\"" + echo "rmdir \"$f\"" >> "$filev3" + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + verbose_notice " rmrfdir \"$f\"" + echo "rmrfdir \"$f\"" >> "$filev3" + else + verbose_notice " remove \"$f\"" + echo "remove \"$f\"" >> "$filev3" + fi + fi + fi + done + fi +} + +# List all files in the current directory, stripping leading "./" +# Pass a variable name and it will be filled as an array. +list_files() { + count=0 + temp_filelist=$(mktemp) + find . -type f \ + ! -name "update.manifest" \ + ! -name "updatev2.manifest" \ + ! -name "updatev3.manifest" \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "${temp_filelist}" + while read file; do + eval "${1}[$count]=\"$file\"" + (( count++ )) + done < "${temp_filelist}" + rm "${temp_filelist}" +} + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + temp_dirlist=$(mktemp) + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "${temp_dirlist}" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "${temp_dirlist}" + rm "${temp_dirlist}" +} diff --git a/tools/update-packaging/make_full_update.sh b/tools/update-packaging/make_full_update.sh new file mode 100755 index 0000000000..db2c5898ef --- /dev/null +++ b/tools/update-packaging/make_full_update.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates full update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +if [ $1 = -h ]; then + print_usage + notice "" + notice "The contents of DIRECTORY will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice " -q be less verbose" + notice "" + exit 1 +fi + +if [ $1 = -q ]; then + QUIET=1 + export QUIET + shift +fi + +# ----------------------------------------------------------------------------- + +mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}" + +archive="$1" +targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi +workdir="$targetdir.work" +updatemanifestv3="$workdir/updatev3.manifest" +targetfiles="updatev3.manifest" + +mkdir -p "$workdir" + +# Generate a list of all files in the target directory. +pushd "$targetdir" +if test $? -ne 0 ; then + exit 1 +fi + +if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" + exit 1 + fi +fi + +list_files files + +popd + +# Add the type of update to the beginning of the update manifests. +> "$updatemanifestv3" +notice "" +notice "Adding type instruction to update manifests" +notice " type complete" +echo "type \"complete\"" >> "$updatemanifestv3" + +notice "" +notice "Adding file add instructions to update manifests" +num_files=${#files[*]} + +for ((i=0; $i<$num_files; i=$i+1)); do + f="${files[$i]}" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + else + make_add_instruction "$f" "$updatemanifestv3" + fi + + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" +done + +# Append remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv3" + +$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" + +mar_command="$mar_command -C \"$workdir\" -c output.mar" +eval "$mar_command $targetfiles" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/tools/update-packaging/make_incremental_update.sh b/tools/update-packaging/make_incremental_update.sh new file mode 100755 index 0000000000..31b74813e5 --- /dev/null +++ b/tools/update-packaging/make_incremental_update.sh @@ -0,0 +1,313 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates incremental update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" + notice "" + notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice " -f clobber this file in the installation" + notice " Must be a path to a file to clobber in the partial update." + notice " -q be less verbose" + notice "" +} + +check_for_forced_update() { + force_list="$1" + forced_file_chk="$2" + + local f + + if [ "$forced_file_chk" = "precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + # notarization ticket + if [ "$forced_file_chk" = "Contents/CodeResources" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "${forced_file_chk##*.}" = "chk" ]; then + ## "true" *giggle* + return 0; + fi + + for f in $force_list; do + #echo comparing $forced_file_chk to $f + if [ "$forced_file_chk" = "$f" ]; then + ## "true" *giggle* + return 0; + fi + done + ## 'false'... because this is bash. Oh yay! + return 1; +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +requested_forced_updates='Contents/MacOS/firefox' + +while getopts "hqf:" flag +do + case "$flag" in + h) print_usage; exit 0 + ;; + q) QUIET=1 + ;; + f) requested_forced_updates="$requested_forced_updates $OPTARG" + ;; + ?) print_usage; exit 1 + ;; + esac +done + +# ----------------------------------------------------------------------------- + +mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}" + +let arg_start=$OPTIND-1 +shift $arg_start + +archive="$1" +olddir="$2" +newdir="$3" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then + # Remove the / + newdir=$(echo "$newdir" | sed -e 's:\/$::') +fi +workdir="$(mktemp -d)" +updatemanifestv3="$workdir/updatev3.manifest" +archivefiles="updatev3.manifest" + +mkdir -p "$workdir" + +# Generate a list of all files in the target directory. +pushd "$olddir" +if test $? -ne 0 ; then + exit 1 +fi + +list_files oldfiles +list_dirs olddirs + +popd + +pushd "$newdir" +if test $? -ne 0 ; then + exit 1 +fi + +if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" + exit 1 + fi +fi + +list_dirs newdirs +list_files newfiles + +popd + +# Add the type of update to the beginning of the update manifests. +notice "" +notice "Adding type instruction to update manifests" +> $updatemanifestv3 +notice " type partial" +echo "type \"partial\"" >> $updatemanifestv3 + +notice "" +notice "Adding file patch and add instructions to update manifests" + +num_oldfiles=${#oldfiles[*]} +remove_array= +num_removes=0 + +for ((i=0; $i<$num_oldfiles; i=$i+1)); do + f="${oldfiles[$i]}" + + # If this file exists in the new directory as well, then check if it differs. + if [ -f "$newdir/$f" ]; then + + if check_for_add_if_not_update "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p "$(dirname "$workdir/$f")" + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_if_not_instruction "$f" "$updatemanifestv3" + archivefiles="$archivefiles \"$f\"" + continue 1 + fi + + if check_for_forced_update "$requested_forced_updates" "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p "$(dirname "$workdir/$f")" + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_instruction "$f" "$updatemanifestv3" 1 + archivefiles="$archivefiles \"$f\"" + continue 1 + fi + + if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then + # Compute both the compressed binary diff and the compressed file, and + # compare the sizes. Then choose the smaller of the two to package. + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + verbose_notice "diffing \"$f\"" + # MBSDIFF_HOOK represents the communication interface with funsize and, + # if enabled, caches the intermediate patches for future use and + # compute avoidance + # + # An example of MBSDIFF_HOOK env variable could look like this: + # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user" + # where myscript.sh has the following usage: + # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ + # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL + # + # Note: patches are bzipped or xz stashed in funsize to gain more speed + + # if service is not enabled then default to old behavior + if [ -z "$MBSDIFF_HOOK" ]; then + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" + else + # if service enabled then check patch existence for retrieval + if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then + verbose_notice "file \"$f\" found in funsize, diffing skipped" + else + # if not found already - compute it and cache it for future use + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" + $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz" + fi + fi + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + patchfile="$workdir/$f.patch.xz" + patchsize=$(get_file_size "$patchfile") + fullsize=$(get_file_size "$workdir/$f") + + if [ $patchsize -lt $fullsize ]; then + make_patch_instruction "$f" "$updatemanifestv3" + mv -f "$patchfile" "$workdir/$f.patch" + rm -f "$workdir/$f" + archivefiles="$archivefiles \"$f.patch\"" + else + make_add_instruction "$f" "$updatemanifestv3" + rm -f "$patchfile" + archivefiles="$archivefiles \"$f\"" + fi + fi + else + # remove instructions are added after add / patch instructions for + # consistency with make_incremental_updates.py + remove_array[$num_removes]=$f + (( num_removes++ )) + fi +done + +# Newly added files +notice "" +notice "Adding file add instructions to update manifests" +num_newfiles=${#newfiles[*]} + +for ((i=0; $i<$num_newfiles; i=$i+1)); do + f="${newfiles[$i]}" + + # If we've already tested this file, then skip it + for ((j=0; $j<$num_oldfiles; j=$j+1)); do + if [ "$f" = "${oldfiles[j]}" ]; then + continue 2 + fi + done + + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + else + make_add_instruction "$f" "$updatemanifestv3" + fi + + + archivefiles="$archivefiles \"$f\"" +done + +notice "" +notice "Adding file remove instructions to update manifests" +for ((i=0; $i<$num_removes; i=$i+1)); do + f="${remove_array[$i]}" + verbose_notice " remove \"$f\"" + echo "remove \"$f\"" >> $updatemanifestv3 +done + +# Add remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$newdir" "$updatemanifestv3" + +notice "" +notice "Adding directory remove instructions for directories that no longer exist" +num_olddirs=${#olddirs[*]} + +for ((i=0; $i<$num_olddirs; i=$i+1)); do + f="${olddirs[$i]}" + # If this dir doesn't exist in the new directory remove it. + if [ ! -d "$newdir/$f" ]; then + verbose_notice " rmdir $f/" + echo "rmdir \"$f/\"" >> $updatemanifestv3 + fi +done + +$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" + +mar_command="$mar_command -C \"$workdir\" -c output.mar" +eval "$mar_command $archivefiles" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/tools/update-packaging/moz.build b/tools/update-packaging/moz.build new file mode 100644 index 0000000000..568f361a54 --- /dev/null +++ b/tools/update-packaging/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/update-packaging/moz.configure b/tools/update-packaging/moz.configure new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/update-packaging/moz.configure diff --git a/tools/update-packaging/test/buildrefmars.sh b/tools/update-packaging/test/buildrefmars.sh new file mode 100755 index 0000000000..fd2d5384d7 --- /dev/null +++ b/tools/update-packaging/test/buildrefmars.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Builds all the reference mars + +if [ -f "ref.mar" ]; then + rm "ref.mar" +fi +if [ -f "ref-mac.mar" ]; then + rm "ref-mac.mar" +fi + + ../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to + ../make_incremental_update.sh ref-mac.mar `pwd`/from-mac `pwd`/to-mac + +if [ -f "product-1.0.lang.platform.complete.mar" ]; then + rm "product-1.0.lang.platform.complete.mar" +fi +if [ -f "product-2.0.lang.platform.complete.mar" ]; then + rm "product-2.0.lang.platform.complete.mar" +fi +if [ -f "product-2.0.lang.mac.complete.mar" ]; then + rm "product-2.0.lang.mac.complete.mar" +fi + +./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from" +./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to" +./make_full_update.sh product-1.0.lang.mac.complete.mar "`pwd`/from-mac" +./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac" diff --git a/tools/update-packaging/test/common.sh b/tools/update-packaging/test/common.sh new file mode 100755 index 0000000000..0e1857c8b2 --- /dev/null +++ b/tools/update-packaging/test/common.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Code shared by update packaging scripts. +# Author: Darin Fisher +# +# In here to use the local common.sh to allow the full mars to have unfiltered files + +# ----------------------------------------------------------------------------- +# By default just assume that these tools exist on our path +MAR=${MAR:-mar} +XZ=${XZ:-xz} +MBSDIFF=${MBSDIFF:-mbsdiff} + +# ----------------------------------------------------------------------------- +# Helper routines + +notice() { + echo "$*" 1>&2 +} + +get_file_size() { + info=($(ls -ln "$1")) + echo ${info[4]} +} + +copy_perm() { + reference="$1" + target="$2" + + if [ -x "$reference" ]; then + chmod 0755 "$target" + else + chmod 0644 "$target" + fi +} + +make_add_instruction() { + f="$1" + filev2="$2" + # The third param will be an empty string when a file add instruction is only + # needed in the version 2 manifest. This only happens when the file has an + # add-if-not instruction in the version 3 manifest. This is due to the + # precomplete file prior to the version 3 manifest having a remove instruction + # for this file so the file is removed before applying a complete update. + filev3="$3" + + # Used to log to the console + if [ $4 ]; then + forced=" (forced)" + else + forced= + fi + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " add-if \"$testdir\" \"$f\"" + echo "add-if \"$testdir\" \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add-if \"$testdir\" \"$f\"" >> $filev3 + fi + else + notice " add \"$f\"$forced" + echo "add \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add \"$f\"" >> $filev3 + fi + fi +} + +check_for_add_if_not_update() { + add_if_not_file_chk="$1" + + if [ "$(basename "$add_if_not_file_chk")" = "channel-prefs.js" -o \ + "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +check_for_add_to_manifestv2() { + add_if_not_file_chk="$1" + + if [ "$(basename "$add_if_not_file_chk")" = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +make_add_if_not_instruction() { + f="$1" + filev3="$2" + + notice " add-if-not \"$f\" \"$f\"" + echo "add-if-not \"$f\" \"$f\"" >> $filev3 +} + +make_patch_instruction() { + f="$1" + filev2="$2" + filev3="$3" + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3 + else + notice " patch \"$f.patch\" \"$f\"" + echo "patch \"$f.patch\" \"$f\"" >> $filev2 + echo "patch \"$f.patch\" \"$f\"" >> $filev3 + fi +} + +append_remove_instructions() { + dir="$1" + filev2="$2" + filev3="$3" + + if [ -f "$dir/removed-files" ]; then + listfile="$dir/removed-files" + elif [ -f "$dir/Contents/Resources/removed-files" ]; then + listfile="$dir/Contents/Resources/removed-files" + fi + if [ -n "$listfile" ]; then + # Map spaces to pipes so that we correctly handle filenames with spaces. + files=($(cat "$listfile" | tr " " "|" | sort -r)) + num_files=${#files[*]} + for ((i=0; $i<$num_files; i=$i+1)); do + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. + if [ -n "$f" ]; then + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + notice " rmdir \"$f\"" + echo "rmdir \"$f\"" >> $filev2 + echo "rmdir \"$f\"" >> $filev3 + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + notice " rmrfdir \"$f\"" + echo "rmrfdir \"$f\"" >> $filev2 + echo "rmrfdir \"$f\"" >> $filev3 + else + notice " remove \"$f\"" + echo "remove \"$f\"" >> $filev2 + echo "remove \"$f\"" >> $filev3 + fi + fi + fi + done + fi +} + +# List all files in the current directory, stripping leading "./" +# Pass a variable name and it will be filled as an array. +list_files() { + count=0 + + # Removed the exclusion cases here to allow for generation of testing mars + find . -type f \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "$workdir/temp-filelist" + while read file; do + eval "${1}[$count]=\"$file\"" + (( count++ )) + done < "$workdir/temp-filelist" + rm "$workdir/temp-filelist" +} + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "$workdir/temp-dirlist" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "$workdir/temp-dirlist" + rm "$workdir/temp-dirlist" +} diff --git a/tools/update-packaging/test/diffmar.sh b/tools/update-packaging/test/diffmar.sh new file mode 100755 index 0000000000..4b16ffb77a --- /dev/null +++ b/tools/update-packaging/test/diffmar.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Compares two mars + +marA="$1" +marB="$2" +testDir="$3" +workdir="/tmp/diffmar/$testDir" +fromdir="$workdir/0" +todir="$workdir/1" + +# On Windows, creation time can be off by a second or more between the files in +# the fromdir and todir due to them being extracted synchronously so use +# time-style and exclude seconds from the creation time. +lsargs="-algR" +unamestr=`uname` +if [ ! "$unamestr" = 'Darwin' ]; then + unamestr=`uname -o` + if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then + lsargs="-algR --time-style=+%Y-%m-%d-%H:%M" + fi +fi + +rm -rf "$workdir" +mkdir -p "$fromdir" +mkdir -p "$todir" + +cp "$1" "$fromdir" +cp "$2" "$todir" + +cd "$fromdir" +mar -x "$1" +rm "$1" +rm -f updatev2.manifest # Older files may contain this +mv updatev3.manifest updatev3.manifest.xz +xz -d updatev3.manifest.xz +ls $lsargs > files.txt + +cd "$todir" +mar -x "$2" +rm "$2" +mv updatev3.manifest updatev3.manifest.xz +xz -d updatev3.manifest.xz +ls $lsargs > files.txt + +echo "diffing $fromdir and $todir" +echo "on linux shell sort and python sort return different results" +echo "which can cause differences in the manifest files" +diff -ru "$fromdir" "$todir" diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/force.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/same.bin b/tools/update-packaging/test/from-mac/Contents/MacOS/same.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/same.bin diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest b/tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest new file mode 100644 index 0000000000..d3e8ed851b --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt new file mode 100644 index 0000000000..d7c40c63cd --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt @@ -0,0 +1 @@ +This from file should be ignored diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest new file mode 100644 index 0000000000..d3e8ed851b --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/application.ini b/tools/update-packaging/test/from-mac/Contents/Resources/application.ini new file mode 100644 index 0000000000..942e91a16a --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=1 +BuildID=20120101010101 diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/chrome.manifest b/tools/update-packaging/test/from-mac/Contents/Resources/chrome.manifest new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/chrome.manifest diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/precomplete b/tools/update-packaging/test/from-mac/Contents/Resources/precomplete new file mode 100644 index 0000000000..2d9068d372 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/precomplete @@ -0,0 +1,26 @@ +remove "Contents/MacOS/{foodir/update.manifest" +remove "Contents/MacOS/{foodir/same.txt" +remove "Contents/MacOS/{foodir/same.bin" +remove "Contents/MacOS/{foodir/removed.txt" +remove "Contents/MacOS/{foodir/readme.txt" +remove "Contents/MacOS/{foodir/force.txt" +remove "Contents/MacOS/{foodir/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/update.manifest" +remove "Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/same.txt" +remove "Contents/MacOS/same.bin" +remove "Contents/MacOS/removed.txt" +remove "Contents/MacOS/readme.txt" +remove "Contents/MacOS/force.txt" +remove "Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/application.ini" +remove "Contents/Resources/precomplete" +rmdir "Contents/MacOS/{foodir/" +rmdir "Contents/MacOS/searchplugins/diff/" +rmdir "Contents/MacOS/searchplugins/" +rmdir "Contents/MacOS/extensions/diff/" +rmdir "Contents/MacOS/extensions/" +rmdir "Contents/MacOS/" +rmdir "Contents/Resources/" +rmdir "Contents/" diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/readme.txt b/tools/update-packaging/test/from-mac/Contents/Resources/readme.txt new file mode 100644 index 0000000000..b1a96f1fea --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/readme.txt @@ -0,0 +1,2 @@ +This from file should be ignored + diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/removed-files b/tools/update-packaging/test/from-mac/Contents/Resources/removed-files new file mode 100644 index 0000000000..5bbdac6f62 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/removed-files @@ -0,0 +1,8 @@ +Contents/Resources/removed1.txt +Contents/MacOS/removed2.bin +Contents/MacOS/recursivedir/meh/* +Contents/Resources/dir/ +Contents/MacOS/this file has spaces + + +Contents/MacOS/extra-spaces diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/removed.txt b/tools/update-packaging/test/from-mac/Contents/Resources/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/same.txt b/tools/update-packaging/test/from-mac/Contents/Resources/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini b/tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/application.ini b/tools/update-packaging/test/from/application.ini new file mode 100644 index 0000000000..942e91a16a --- /dev/null +++ b/tools/update-packaging/test/from/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=1 +BuildID=20120101010101 diff --git a/tools/update-packaging/test/from/chrome.manifest b/tools/update-packaging/test/from/chrome.manifest new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/update-packaging/test/from/chrome.manifest diff --git a/tools/update-packaging/test/from/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/force.txt b/tools/update-packaging/test/from/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/precomplete b/tools/update-packaging/test/from/precomplete new file mode 100644 index 0000000000..e27f4fc31c --- /dev/null +++ b/tools/update-packaging/test/from/precomplete @@ -0,0 +1,23 @@ +remove "{foodir/update.manifest" +remove "{foodir/same.txt" +remove "{foodir/same.bin" +remove "{foodir/removed.txt" +remove "{foodir/readme.txt" +remove "{foodir/force.txt" +remove "{foodir/diff-patch-larger-than-file.txt" +remove "update.manifest" +remove "searchplugins/diff/diff-patch-larger-than-file.txt" +remove "same.txt" +remove "same.bin" +remove "removed.txt" +remove "readme.txt" +remove "precomplete" +remove "force.txt" +remove "extensions/diff/diff-patch-larger-than-file.txt" +remove "diff-patch-larger-than-file.txt" +remove "application.ini" +rmdir "{foodir/" +rmdir "searchplugins/diff/" +rmdir "searchplugins/" +rmdir "extensions/diff/" +rmdir "extensions/" diff --git a/tools/update-packaging/test/from/readme.txt b/tools/update-packaging/test/from/readme.txt new file mode 100644 index 0000000000..b1a96f1fea --- /dev/null +++ b/tools/update-packaging/test/from/readme.txt @@ -0,0 +1,2 @@ +This from file should be ignored + diff --git a/tools/update-packaging/test/from/removed-files b/tools/update-packaging/test/from/removed-files new file mode 100644 index 0000000000..73b348d9c4 --- /dev/null +++ b/tools/update-packaging/test/from/removed-files @@ -0,0 +1,8 @@ +removed1.txt +removed2.bin +recursivedir/meh/* +dir/ +this file has spaces + + +extra-spaces diff --git a/tools/update-packaging/test/from/removed.txt b/tools/update-packaging/test/from/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from/same.bin b/tools/update-packaging/test/from/same.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/from/same.bin diff --git a/tools/update-packaging/test/from/same.txt b/tools/update-packaging/test/from/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/update-settings.ini b/tools/update-packaging/test/from/update-settings.ini new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/from/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from/update.manifest b/tools/update-packaging/test/from/update.manifest new file mode 100644 index 0000000000..d3e8ed851b --- /dev/null +++ b/tools/update-packaging/test/from/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/from/{foodir/channel-prefs.js b/tools/update-packaging/test/from/{foodir/channel-prefs.js new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/{foodir/force.txt b/tools/update-packaging/test/from/{foodir/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/{foodir/readme.txt b/tools/update-packaging/test/from/{foodir/readme.txt new file mode 100644 index 0000000000..d7c40c63cd --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/readme.txt @@ -0,0 +1 @@ +This from file should be ignored diff --git a/tools/update-packaging/test/from/{foodir/removed.txt b/tools/update-packaging/test/from/{foodir/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from/{foodir/same.bin b/tools/update-packaging/test/from/{foodir/same.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/same.bin diff --git a/tools/update-packaging/test/from/{foodir/same.txt b/tools/update-packaging/test/from/{foodir/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/{foodir/update.manifest b/tools/update-packaging/test/from/{foodir/update.manifest new file mode 100644 index 0000000000..d3e8ed851b --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/make_full_update.sh b/tools/update-packaging/test/make_full_update.sh new file mode 100755 index 0000000000..cdcdae3c3b --- /dev/null +++ b/tools/update-packaging/test/make_full_update.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates full update packages for the update system. +# Author: Darin Fisher +# +# In here to use the local common.sh to allow the full mars to have unfiltered files + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +if [ $1 = -h ]; then + print_usage + notice "" + notice "The contents of DIRECTORY will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice "" + exit 1 +fi + +# ----------------------------------------------------------------------------- + +archive="$1" +targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi +workdir="$targetdir.work" +updatemanifestv3="$workdir/updatev3.manifest" +targetfiles="updatev3.manifest" + +mkdir -p "$workdir" + +# Generate a list of all files in the target directory. +pushd "$targetdir" +if test $? -ne 0 ; then + exit 1 +fi + +if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" + exit 1 + fi +fi + +list_files files + +popd + +# Add the type of update to the beginning of the update manifests. +> $updatemanifestv3 +notice "" +notice "Adding type instruction to update manifests" +notice " type complete" +echo "type \"complete\"" >> $updatemanifestv3 + +notice "" +notice "Adding file add instructions to update manifests" +num_files=${#files[*]} + +for ((i=0; $i<$num_files; i=$i+1)); do + f="${files[$i]}" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + else + make_add_instruction "$f" "$updatemanifestv3" + fi + + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $XZ $XZ_OPT --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" +done + +# Append remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv3" + +$XZ $XZ_OPT --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" + +eval "$MAR -C \"$workdir\" -c output.mar $targetfiles" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/tools/update-packaging/test/runtests.sh b/tools/update-packaging/test/runtests.sh new file mode 100755 index 0000000000..c66e54eaa6 --- /dev/null +++ b/tools/update-packaging/test/runtests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo "testing make_incremental_updates.py" +python ../make_incremental_updates.py -f testpatchfile.txt + +echo "" +echo "diffing ref.mar and test.mar" +./diffmar.sh ref.mar test.mar test + +echo "" +echo "diffing ref-mac.mar and test-mac.mar" +./diffmar.sh ref-mac.mar test-mac.mar test-mac diff --git a/tools/update-packaging/test/testpatchfile.txt b/tools/update-packaging/test/testpatchfile.txt new file mode 100644 index 0000000000..a19c831eb4 --- /dev/null +++ b/tools/update-packaging/test/testpatchfile.txt @@ -0,0 +1,2 @@ +product-1.0.lang.platform.complete.mar,product-2.0.lang.platform.complete.mar,test.mar,"" +product-1.0.lang.mac.complete.mar,product-2.0.lang.mac.complete.mar,test-mac.mar,"" diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js b/tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js new file mode 100644 index 0000000000..3b2aed8e02 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js @@ -0,0 +1 @@ +this is a new file diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/added.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/added.txt new file mode 100644 index 0000000000..b242c36062 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..a61ffbb5e0 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/force.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/same.bin b/tools/update-packaging/test/to-mac/Contents/MacOS/same.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/same.bin diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest b/tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest new file mode 100644 index 0000000000..73364fdca1 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt new file mode 100644 index 0000000000..b242c36062 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..a61ffbb5e0 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt new file mode 100644 index 0000000000..b5f7004cc9 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest new file mode 100644 index 0000000000..73364fdca1 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/application.ini b/tools/update-packaging/test/to-mac/Contents/Resources/application.ini new file mode 100644 index 0000000000..7bdc78819f --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=2 +BuildID=20130101010101 diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/chrome.manifest b/tools/update-packaging/test/to-mac/Contents/Resources/chrome.manifest new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/chrome.manifest diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/precomplete b/tools/update-packaging/test/to-mac/Contents/Resources/precomplete new file mode 100644 index 0000000000..7af8bfd769 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/precomplete @@ -0,0 +1,33 @@ +remove "Contents/MacOS/{foodir/update.manifest" +remove "Contents/MacOS/{foodir/same.txt" +remove "Contents/MacOS/{foodir/same.bin" +remove "Contents/MacOS/{foodir/readme.txt" +remove "Contents/MacOS/{foodir/force.txt" +remove "Contents/MacOS/{foodir/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/{foodir/added.txt" +remove "Contents/MacOS/update.manifest" +remove "Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/searchplugins/added/file.txt" +remove "Contents/MacOS/same.txt" +remove "Contents/MacOS/same.bin" +remove "Contents/MacOS/removed-files" +remove "Contents/MacOS/readme.txt" +remove "Contents/MacOS/force.txt" +remove "Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/extensions/added/file.txt" +remove "Contents/MacOS/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/diff-patch-larger-than-file.bin" +remove "Contents/MacOS/application.ini" +remove "Contents/MacOS/added.txt" +remove "Contents/MacOS/addFeedPrefs.js" +remove "Contents/Resources/precomplete" +rmdir "Contents/MacOS/{foodir/" +rmdir "Contents/MacOS/searchplugins/diff/" +rmdir "Contents/MacOS/searchplugins/added/" +rmdir "Contents/MacOS/searchplugins/" +rmdir "Contents/MacOS/extensions/diff/" +rmdir "Contents/MacOS/extensions/added/" +rmdir "Contents/MacOS/extensions/" +rmdir "Contents/MacOS/" +rmdir "Contents/Resources/" +rmdir "Contents/" diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/readme.txt b/tools/update-packaging/test/to-mac/Contents/Resources/readme.txt new file mode 100644 index 0000000000..b5f7004cc9 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/removed-files b/tools/update-packaging/test/to-mac/Contents/Resources/removed-files new file mode 100644 index 0000000000..a756cc560b --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/removed-files @@ -0,0 +1,14 @@ +Contents/Resources/removed1.txt +Contents/MacOS/removed2.bin +Contents/MacOS/recursivedir/meh/* +Contents/MacOS/removed3-foo.txt +Contents/Resources/dir/ +Contents/MacOS/this file has spaces +Contents/MacOS/notherdir/ + + +Contents/Resources/extra-spaces + +Contents/MacOS/lastFile + + diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/same.txt b/tools/update-packaging/test/to-mac/Contents/Resources/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini b/tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js new file mode 100644 index 0000000000..d6ada4591d --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from partial file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/addFeedPrefs.js b/tools/update-packaging/test/to/addFeedPrefs.js new file mode 100644 index 0000000000..3b2aed8e02 --- /dev/null +++ b/tools/update-packaging/test/to/addFeedPrefs.js @@ -0,0 +1 @@ +this is a new file diff --git a/tools/update-packaging/test/to/added.txt b/tools/update-packaging/test/to/added.txt new file mode 100644 index 0000000000..b242c36062 --- /dev/null +++ b/tools/update-packaging/test/to/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to/application.ini b/tools/update-packaging/test/to/application.ini new file mode 100644 index 0000000000..7bdc78819f --- /dev/null +++ b/tools/update-packaging/test/to/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=2 +BuildID=20130101010101 diff --git a/tools/update-packaging/test/to/chrome.manifest b/tools/update-packaging/test/to/chrome.manifest new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/update-packaging/test/to/chrome.manifest diff --git a/tools/update-packaging/test/to/diff-patch-larger-than-file.bin b/tools/update-packaging/test/to/diff-patch-larger-than-file.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/to/diff-patch-larger-than-file.bin diff --git a/tools/update-packaging/test/to/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..a61ffbb5e0 --- /dev/null +++ b/tools/update-packaging/test/to/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to/distribution/extensions/added/file.txt b/tools/update-packaging/test/to/distribution/extensions/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to/distribution/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to/extensions/added/file.txt b/tools/update-packaging/test/to/extensions/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to/force.txt b/tools/update-packaging/test/to/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/precomplete b/tools/update-packaging/test/to/precomplete new file mode 100644 index 0000000000..c2700dd972 --- /dev/null +++ b/tools/update-packaging/test/to/precomplete @@ -0,0 +1,30 @@ +remove "{foodir/update.manifest" +remove "{foodir/same.txt" +remove "{foodir/same.bin" +remove "{foodir/readme.txt" +remove "{foodir/force.txt" +remove "{foodir/diff-patch-larger-than-file.txt" +remove "{foodir/added.txt" +remove "update.manifest" +remove "searchplugins/diff/diff-patch-larger-than-file.txt" +remove "searchplugins/added/file.txt" +remove "same.txt" +remove "same.bin" +remove "removed-files" +remove "readme.txt" +remove "precomplete" +remove "force.txt" +remove "extensions/diff/diff-patch-larger-than-file.txt" +remove "extensions/added/file.txt" +remove "diff-patch-larger-than-file.txt" +remove "diff-patch-larger-than-file.bin" +remove "application.ini" +remove "added.txt" +remove "addFeedPrefs.js" +rmdir "{foodir/" +rmdir "searchplugins/diff/" +rmdir "searchplugins/added/" +rmdir "searchplugins/" +rmdir "extensions/diff/" +rmdir "extensions/added/" +rmdir "extensions/" diff --git a/tools/update-packaging/test/to/readme.txt b/tools/update-packaging/test/to/readme.txt new file mode 100644 index 0000000000..b5f7004cc9 --- /dev/null +++ b/tools/update-packaging/test/to/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to/removed-files b/tools/update-packaging/test/to/removed-files new file mode 100644 index 0000000000..4fdfff7fd5 --- /dev/null +++ b/tools/update-packaging/test/to/removed-files @@ -0,0 +1,14 @@ +removed1.txt +removed2.bin +recursivedir/meh/* +removed3-foo.txt +dir/ +this file has spaces +notherdir/ + + +extra-spaces + +lastFile + + diff --git a/tools/update-packaging/test/to/same.bin b/tools/update-packaging/test/to/same.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/to/same.bin diff --git a/tools/update-packaging/test/to/same.txt b/tools/update-packaging/test/to/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/searchplugins/added/file.txt b/tools/update-packaging/test/to/searchplugins/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to/searchplugins/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to/update-settings.ini b/tools/update-packaging/test/to/update-settings.ini new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/to/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/to/update.manifest b/tools/update-packaging/test/to/update.manifest new file mode 100644 index 0000000000..73364fdca1 --- /dev/null +++ b/tools/update-packaging/test/to/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test/to/{foodir/added.txt b/tools/update-packaging/test/to/{foodir/added.txt new file mode 100644 index 0000000000..b242c36062 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to/{foodir/channel-prefs.js b/tools/update-packaging/test/to/{foodir/channel-prefs.js new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..a61ffbb5e0 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to/{foodir/force.txt b/tools/update-packaging/test/to/{foodir/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/{foodir/readme.txt b/tools/update-packaging/test/to/{foodir/readme.txt new file mode 100644 index 0000000000..b5f7004cc9 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to/{foodir/same.bin b/tools/update-packaging/test/to/{foodir/same.bin Binary files differnew file mode 100644 index 0000000000..a9ee7258cc --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/same.bin diff --git a/tools/update-packaging/test/to/{foodir/same.txt b/tools/update-packaging/test/to/{foodir/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/{foodir/update.manifest b/tools/update-packaging/test/to/{foodir/update.manifest new file mode 100644 index 0000000000..73364fdca1 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-programs/README b/tools/update-programs/README new file mode 100644 index 0000000000..35de17eb8f --- /dev/null +++ b/tools/update-programs/README @@ -0,0 +1,19 @@ +This directory defines a build project for focused work on the "update +programs": programs owned or maintained by the Install/Update team +that are standalone binaries (i.e., not part of the Firefox binary +proper). + +To use this build project, prepare a minimal mozconfig with +``` +ac_add_options --enable-project=tools/update-programs +``` + +Depending on the mozconfig options and host and target OS, some of the +following will be built: + +1. the maintenance service (when `--enable-maintenance-service`); +2. the updater binary (when `MOZ_UPDATER=1`); +3. the Windows Default Browser Agent (when `--enable-default-browser-agent`); + +Packaging the installer and uninstaller is not yet supported: instead, +use an (artifact) build with `--enable-project=browser`. diff --git a/tools/update-programs/app.mozbuild b/tools/update-programs/app.mozbuild new file mode 100644 index 0000000000..f14be4af89 --- /dev/null +++ b/tools/update-programs/app.mozbuild @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +if ( + CONFIG["MOZ_MAINTENANCE_SERVICE"] + or CONFIG["MOZ_UPDATE_AGENT"] + or CONFIG["MOZ_UPDATER"] +): + DIRS += [ + "/toolkit/mozapps/update/common", + ] + +if CONFIG["MOZ_DEFAULT_BROWSER_AGENT"]: + DIRS += [ + "/toolkit/components/jsoncpp/src/lib_json", + "/toolkit/mozapps/defaultagent", + ] + +if CONFIG["MOZ_MAINTENANCE_SERVICE"]: + DIRS += ["/toolkit/components/maintenanceservice"] + +if CONFIG["MOZ_UPDATER"]: + # NSS (and NSPR). + DIRS += [ + "/modules/xz-embedded", + "/config/external/nspr", + "/config/external/sqlite", + "/config/external/zlib", + "/memory", + "/mfbt", + "/mozglue", + "/security", + ] + + # The signing related bits of libmar depend on NSS. + DIRS += [ + "/modules/libmar", + "/other-licenses/bsdiff", + "/toolkit/mozapps/update/updater/bspatch", + "/toolkit/mozapps/update/updater", + ] + +# Expose specific non-XPCOM headers when building standalone. +if not CONFIG["MOZ_UPDATER"]: + # When building the updater, we build /mozglue, which includes this. + EXPORTS.mozilla += [ + "/mozglue/misc/DynamicallyLinkedFunctionPtr.h", + ] + +EXPORTS.mozilla += [ + "/toolkit/xre/CmdLineAndEnvUtils.h", + "/widget/windows/WinHeaderOnlyUtils.h", +] + +EXPORTS += [ + "/xpcom/base/nsAutoRef.h", + "/xpcom/base/nsWindowsHelpers.h", + "/xpcom/string/nsCharTraits.h", + "/xpcom/string/nsUTF8Utils.h", +] diff --git a/tools/update-programs/confvars.sh b/tools/update-programs/confvars.sh new file mode 100644 index 0000000000..66e2a80605 --- /dev/null +++ b/tools/update-programs/confvars.sh @@ -0,0 +1,12 @@ +#! /bin/sh +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +MOZ_APP_VENDOR=Mozilla + +MOZ_BRANDING_DIRECTORY=browser/branding/unofficial +MOZ_APP_ID={ec8030f7-c20a-464f-9b0e-13a3a9e97384} + +# Build the updater by default. Use --disable-updater to not. +MOZ_UPDATER=1 diff --git a/tools/update-programs/moz.configure b/tools/update-programs/moz.configure new file mode 100644 index 0000000000..1ce4a4ade9 --- /dev/null +++ b/tools/update-programs/moz.configure @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +# Spoof a stub of `js/moz.configure` for the included scripts. +@dependable +def js_standalone(): + return False + + +@depends(target) +def fold_libs(target): + return target.os in ("WINNT", "OSX", "Android") + + +set_config("MOZ_FOLD_LIBS", fold_libs) + + +include("../../build/moz.configure/rust.configure") +include("../../build/moz.configure/nspr.configure") +include("../../build/moz.configure/nss.configure") +include("../../build/moz.configure/update-programs.configure") diff --git a/tools/update-verify/README.md b/tools/update-verify/README.md new file mode 100644 index 0000000000..14eb2a5f9a --- /dev/null +++ b/tools/update-verify/README.md @@ -0,0 +1,118 @@ +Mozilla Build Verification Scripts +================================== + +Contents +-------- + +updates -> AUS and update verification + +l10n -> l10n vs. en-US verification + +common -> useful utility scripts + +Update Verification +------------------- + +`verify.sh` + +> Does a low-level check of all advertised MAR files. Expects to have a +> file named all-locales, but does not (yet) handle platform exceptions, so +> these should be removed from the locales file. +> +> Prints errors on both STDOUT and STDIN, the intention is to run the +> script with STDOUT redirected to an output log. If there is not output +> on the console and an exit code of 0 then all tests pass; otherwise one +> or more tests failed. +> +> Does the following: +> +> 1) download update.xml from AUS for a particular release +> 2) download the partial and full mar advertised +> 3) check that the partial and full match the advertised size and sha1sum +> 4) downloads the latest release, and an older release +> 5) applies MAR to the older release, and compares the two releases. +> +> Step 5 is repeated for both the complete and partial MAR. +> +> Expects to have an updates.cfg file, describing all releases to try updating +> from. + +Valid Platforms for AUS +----------------------- +- Linux_x86-gcc3 +- Darwin_Universal-gcc3 +- Linux_x86-gcc3 +- WINNT_x86-msvc +- Darwin_ppc-gcc3 + +--- +Running it locally +================== + +Requirements: +------------- + +- [Docker](https://docs.docker.com/get-docker/) +- [optional | Mac] zstd (`brew install zst`) + +Docker Image +------------ + +1. [Ship-it](https://shipit.mozilla-releng.net/recent) holds the latest builds. +1. Clicking on "Ship task" of latest build will open the task group in +Taskcluster. +1. On the "Name contains" lookup box, search for `release-update-verify-firefox` +and open a `update-verify` task +1. Make note of the `CHANNEL` under Payload. ie: `beta-localtest` +1. Click "See more" under Task Details and open the `docker-image-update-verify` +task. + +Download the image artifact from *docker-image-update-verify* task and load it +manually +``` +zstd -d image.tar.zst +docker image load -i image.tar +``` + +**OR** + +Load docker image using mach and a task +``` +# Replace TASK-ID with the ID of a docker-image-update-verify task +./mach taskcluster-load-image --task-id=<TASK-ID> +``` + +Update Verify Config +-------------------- + +1. Open Taskcluster Task Group +1. Search for `update-verify-config` and open the task +1. Under Artifacts, download `update-verify.cfg` file + +Run Docker +---------- + +To run the container interactively: +> Replace `<MOZ DIRECTORY>` with gecko repository path on local host <br /> +> Replace `<UVC PATH>` with path to `update-verify.cfg` file on local host. +ie.: `~/Downloads/update-verify.cfg` +> Replace `<CHANNEL>` with value from `update-verify` task (Docker steps) + +``` +docker run \ + -it \ + --rm \ + -e CHANNEL=beta-localtest \ + -e MOZ_FETCHES_DIR=/builds/worker/fetches \ + -e MOZBUILD_STATE_PATH=/builds/worker/.mozbuild \ + -v <UVC PATH>:/builds/worker/fetches/update-verify.cfg + -v <MOZ DIRECTORY>:/builds/worker/checkouts/gecko \ + -w /builds/worker/checkouts/gecko \ + update-verify +``` +> Note that `MOZ_FETCHES_DIR` here is different from what is used in production. + +`total-chunks` and `this-chunk` refer to the number of lines in `update-verify.cfg` +``` +./tools/update-verify/scripts/chunked-verify.sh --total-chunks=228 --this-chunk=4 +``` diff --git a/tools/update-verify/python/util/__init__.py b/tools/update-verify/python/util/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/update-verify/python/util/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/update-verify/python/util/commands.py b/tools/update-verify/python/util/commands.py new file mode 100644 index 0000000000..e53464e6f8 --- /dev/null +++ b/tools/update-verify/python/util/commands.py @@ -0,0 +1,57 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Functions for running commands""" + +import logging +import os +import subprocess +import time + +import six + +log = logging.getLogger(__name__) + + +# timeout message, used in TRANSIENT_HG_ERRORS and in tests. +TERMINATED_PROCESS_MSG = "timeout, process terminated" + + +def log_cmd(cmd, **kwargs): + # cwd is special in that we always want it printed, even if it's not + # explicitly chosen + kwargs = kwargs.copy() + if "cwd" not in kwargs: + kwargs["cwd"] = os.getcwd() + log.info("command: START") + log.info("command: %s" % subprocess.list2cmdline(cmd)) + for key, value in six.iteritems(kwargs): + log.info("command: %s: %s", key, str(value)) + + +def merge_env(env): + new_env = os.environ.copy() + new_env.update(env) + return new_env + + +def run_cmd(cmd, **kwargs): + """Run cmd (a list of arguments). Raise subprocess.CalledProcessError if + the command exits with non-zero. If the command returns successfully, + return 0.""" + log_cmd(cmd, **kwargs) + # We update this after logging because we don't want all of the inherited + # env vars muddling up the output + if "env" in kwargs: + kwargs["env"] = merge_env(kwargs["env"]) + try: + t = time.monotonic() + log.info("command: output:") + return subprocess.check_call(cmd, **kwargs) + except subprocess.CalledProcessError: + log.info("command: ERROR", exc_info=True) + raise + finally: + elapsed = time.monotonic() - t + log.info("command: END (%.2fs elapsed)\n", elapsed) diff --git a/tools/update-verify/release/common/cached_download.sh b/tools/update-verify/release/common/cached_download.sh new file mode 100644 index 0000000000..7cb3c42f8d --- /dev/null +++ b/tools/update-verify/release/common/cached_download.sh @@ -0,0 +1,40 @@ +# this library works like a wrapper around wget, to allow downloads to be cached +# so that if later the same url is retrieved, the entry from the cache will be +# returned. + +pushd `dirname $0` &>/dev/null +cache_dir="$(pwd)/cache" +popd &>/dev/null + +# Deletes all files in the cache directory +# We don't support folders or .dot(hidden) files +# By not deleting the cache directory, it allows us to use Docker tmpfs mounts, +# which are the only workaround to poor mount r/w performance on MacOS +# Reference: https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288 +clear_cache () { + rm -rf "${cache_dir}/*" +} + +# download method - you pass a filename to save the file under, and the url to call +cached_download () { + local output_file="${1}" + local url="${2}" + + if fgrep -x "${url}" "${cache_dir}/urls.list" >/dev/null; then + echo "Retrieving '${url}' from cache..." + local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')" + cp "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" "${output_file}" + else + echo "Downloading '${url}' and placing in cache..." + rm -f "${output_file}" + $retry wget -O "${output_file}" --progress=dot:giga --server-response "${url}" 2>&1 + local exit_code=$? + if [ "${exit_code}" == 0 ]; then + echo "${url}" >> "${cache_dir}/urls.list" + local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')" + cp "${output_file}" "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" + else + return "${exit_code}" + fi + fi +} diff --git a/tools/update-verify/release/common/check_updates.sh b/tools/update-verify/release/common/check_updates.sh new file mode 100644 index 0000000000..acf06d8b4e --- /dev/null +++ b/tools/update-verify/release/common/check_updates.sh @@ -0,0 +1,125 @@ +check_updates () { + # called with 10 args - platform, source package, target package, update package, old updater boolean, + # a path to the updater binary to use for the tests, a file to write diffs to, the update channel, + # update-settings.ini values, and a flag to indicate the target is dep-signed + update_platform=$1 + source_package=$2 + target_package=$3 + locale=$4 + use_old_updater=$5 + updater=$6 + diff_file=$7 + channel=$8 + mar_channel_IDs=$9 + update_to_dep=${10} + + # cleanup + rm -rf source/* + rm -rf target/* + + unpack_build $update_platform source "$source_package" $locale '' $mar_channel_IDs + if [ "$?" != "0" ]; then + echo "FAILED: cannot unpack_build $update_platform source $source_package" + return 1 + fi + unpack_build $update_platform target "$target_package" $locale + if [ "$?" != "0" ]; then + echo "FAILED: cannot unpack_build $update_platform target $target_package" + return 1 + fi + + case $update_platform in + Darwin_ppc-gcc | Darwin_Universal-gcc3 | Darwin_x86_64-gcc3 | Darwin_x86-gcc3-u-ppc-i386 | Darwin_x86-gcc3-u-i386-x86_64 | Darwin_x86_64-gcc3-u-i386-x86_64 | Darwin_aarch64-gcc3) + platform_dirname="*.app" + ;; + WINNT*) + platform_dirname="bin" + ;; + Linux_x86-gcc | Linux_x86-gcc3 | Linux_x86_64-gcc3) + platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'` + ;; + esac + + if [ -f update/update.status ]; then rm update/update.status; fi + if [ -f update/update.log ]; then rm update/update.log; fi + + if [ -d source/$platform_dirname ]; then + if [ `uname | cut -c-5` == "MINGW" ]; then + # windows + # change /c/path/to/pwd to c:\\path\\to\\pwd + four_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\\\,g') + two_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\,g') + cwd="$two_backslash_pwd\\source\\$platform_dirname" + update_abspath="$two_backslash_pwd\\update" + else + # not windows + # use ls here, because mac uses *.app, and we need to expand it + cwd=$(ls -d $PWD/source/$platform_dirname) + update_abspath="$PWD/update" + fi + + cd_dir=$(ls -d ${PWD}/source/${platform_dirname}) + cd "${cd_dir}" + set -x + "$updater" "$update_abspath" "$cwd" "$cwd" 0 + set +x + cd ../.. + else + echo "TEST-UNEXPECTED-FAIL: no dir in source/$platform_dirname" + return 1 + fi + + cat update/update.log + update_status=`cat update/update.status` + + if [ "$update_status" != "succeeded" ] + then + echo "TEST-UNEXPECTED-FAIL: update status was not successful: $update_status" + return 1 + fi + + # If we were testing an OS X mar on Linux, the unpack step copied the + # precomplete file from Contents/Resources to the root of the install + # to ensure the Linux updater binary could find it. However, only the + # precomplete file in Contents/Resources was updated, which means + # the copied version in the root of the install will usually have some + # differences between the source and target. To prevent this false + # positive from failing the tests, we simply remove it before diffing. + # The precomplete file in Contents/Resources is still diffed, so we + # don't lose any coverage by doing this. + cd `echo "source/$platform_dirname"` + if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]] + then + rm "precomplete" + fi + cd ../.. + cd `echo "target/$platform_dirname"` + if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]] + then + rm "precomplete" + fi + cd ../.. + + # If we are testing an OSX mar to update from a production-signed/notarized + # build to a dep-signed one, ignore Contents/CodeResources which won't be + # present in the target, to avoid spurious failures + # Same applies to provisioning profiles, since we don't have them outside of prod + if ${update_to_dep}; then + ignore_coderesources="--ignore-missing=Contents/CodeResources --ignore-missing=Contents/embedded.provisionprofile" + else + ignore_coderesources= + fi + + ../compare-directories.py source/${platform_dirname} target/${platform_dirname} ${channel} ${ignore_coderesources} > "${diff_file}" + diffErr=$? + cat "${diff_file}" + if [ $diffErr == 2 ] + then + echo "TEST-UNEXPECTED-FAIL: differences found after update" + return 1 + elif [ $diffErr != 0 ] + then + echo "TEST-UNEXPECTED-FAIL: unknown error from diff: $diffErr" + return 3 + fi +} diff --git a/tools/update-verify/release/common/download_builds.sh b/tools/update-verify/release/common/download_builds.sh new file mode 100644 index 0000000000..e279c808db --- /dev/null +++ b/tools/update-verify/release/common/download_builds.sh @@ -0,0 +1,36 @@ +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3" + +download_builds() { + # cleanup + mkdir -p downloads/ + rm -rf downloads/* + + source_url="$1" + target_url="$2" + + if [ -z "$source_url" ] || [ -z "$target_url" ] + then + "download_builds usage: <source_url> <target_url>" + exit 1 + fi + + for url in "$source_url" "$target_url" + do + source_file=`basename "$url"` + if [ -f "$source_file" ]; then rm "$source_file"; fi + cd downloads + if [ -f "$source_file" ]; then rm "$source_file"; fi + cached_download "${source_file}" "${url}" + status=$? + if [ $status != 0 ]; then + echo "TEST-UNEXPECTED-FAIL: Could not download source $source_file from $url" + echo "skipping.." + cd ../ + return $status + fi + cd ../ + done +} diff --git a/tools/update-verify/release/common/download_mars.sh b/tools/update-verify/release/common/download_mars.sh new file mode 100644 index 0000000000..d2dab107d2 --- /dev/null +++ b/tools/update-verify/release/common/download_mars.sh @@ -0,0 +1,105 @@ +download_mars () { + update_url="$1" + only="$2" + test_only="$3" + to_build_id="$4" + to_app_version="$5" + to_display_version="$6" + + max_tries=5 + try=1 + # retrying until we get offered an update + while [ "$try" -le "$max_tries" ]; do + echo "Using $update_url" + # retrying until AUS gives us any response at all + cached_download update.xml "${update_url}" + + echo "Got this response:" + cat update.xml + # If the first line after <updates> is </updates> then we have an + # empty snippet. Otherwise we're done + if [ "$(grep -A1 '<updates>' update.xml | tail -1)" != "</updates>" ]; then + break; + fi + echo "Empty response, sleeping" + sleep 5 + try=$(($try+1)) + done + + echo; echo; # padding + + update_line=`fgrep "<update " update.xml` + grep_rv=$? + if [ 0 -ne $grep_rv ]; then + echo "TEST-UNEXPECTED-FAIL: no <update/> found for $update_url" + return 1 + fi + command=`echo $update_line | sed -e 's/^.*<update //' -e 's:>.*$::' -e 's:\&:\&:g'` + eval "export $command" + + if [ ! -z "$to_build_id" -a "$buildID" != "$to_build_id" ]; then + echo "TEST-UNEXPECTED-FAIL: expected buildID $to_build_id does not match actual $buildID" + return 1 + fi + + if [ ! -z "$to_display_version" -a "$displayVersion" != "$to_display_version" ]; then + echo "TEST-UNEXPECTED-FAIL: expected displayVersion $to_display_version does not match actual $displayVersion" + return 1 + fi + + if [ ! -z "$to_app_version" -a "$appVersion" != "$to_app_version" ]; then + echo "TEST-UNEXPECTED-FAIL: expected appVersion $to_app_version does not match actual $appVersion" + return 1 + fi + + mkdir -p update/ + if [ -z $only ]; then + only="partial complete" + fi + for patch_type in $only + do + line=`fgrep "patch type=\"$patch_type" update.xml` + grep_rv=$? + + if [ 0 -ne $grep_rv ]; then + echo "TEST-UNEXPECTED-FAIL: no $patch_type update found for $update_url" + return 1 + fi + + command=`echo $line | sed -e 's/^.*<patch //' -e 's:/>.*$::' -e 's:\&:\&:g'` + eval "export $command" + + if [ "$test_only" == "1" ] + then + echo "Testing $URL" + curl -s -I -L $URL + return + else + cached_download "update/${patch_type}.mar" "${URL}" + fi + if [ "$?" != 0 ]; then + echo "Could not download $patch_type!" + echo "from: $URL" + fi + actual_size=`perl -e "printf \"%d\n\", (stat(\"update/$patch_type.mar\"))[7]"` + actual_hash=`openssl dgst -$hashFunction update/$patch_type.mar | sed -e 's/^.*= //'` + + if [ $actual_size != $size ]; then + echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong size" + echo "TEST-UNEXPECTED-FAIL: update.xml size: $size" + echo "TEST-UNEXPECTED-FAIL: actual size: $actual_size" + return 1 + fi + + if [ $actual_hash != $hashValue ]; then + echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong hash" + echo "TEST-UNEXPECTED-FAIL: update.xml hash: $hashValue" + echo "TEST-UNEXPECTED-FAIL: actual hash: $actual_hash" + return 1 + fi + + cp update/$patch_type.mar update/update.mar + echo $actual_size > update/$patch_type.size + + done +} diff --git a/tools/update-verify/release/common/installdmg.ex b/tools/update-verify/release/common/installdmg.ex new file mode 100755 index 0000000000..08bcf9a201 --- /dev/null +++ b/tools/update-verify/release/common/installdmg.ex @@ -0,0 +1,45 @@ +#!/usr/bin/expect +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is Mozilla Corporation Code. +# +# The Initial Developer of the Original Code is +# Clint Talbert. +# Portions created by the Initial Developer are Copyright (C) 2007 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Armen Zambrano Gasparnian <armenzg@mozilla.com> +# Axel Hecht <l10n@mozilla.com> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +#send_user $argv +spawn hdiutil attach -readonly -mountroot /tmp -private -noautoopen [lindex $argv 0] +expect { +"byte" {send "G"; exp_continue} +"END" {send "\r"; exp_continue} +"Y/N?" {send "Y\r"; exp_continue} +} diff --git a/tools/update-verify/release/common/unpack-diskimage.sh b/tools/update-verify/release/common/unpack-diskimage.sh new file mode 100755 index 0000000000..b647a69c4d --- /dev/null +++ b/tools/update-verify/release/common/unpack-diskimage.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the installdmg.sh script from taols utilities +# +# The Initial Developer of the Original Code is +# Mozilla Corporation. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Chris AtLee <catlee@mozilla.com> +# Robert Kaiser <kairo@kairo.at> +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +# Unpack a disk image to a specified target folder +# +# Usage: unpack-diskimage <image_file> +# <mountpoint> +# <target_path> + +DMG_PATH=$1 +MOUNTPOINT=$2 +TARGETPATH=$3 +LOGFILE=unpack.output + +# How long to wait before giving up waiting for the mount to fininsh +TIMEOUT=90 + +# If the mount point already exists, then the previous run may not have cleaned +# up properly. We should try to umount and remove the its directory. +if [ -d $MOUNTPOINT ]; then + echo "$MOUNTPOINT already exists, trying to clean up" + hdiutil detach $MOUNTPOINT -force + rm -rdfv $MOUNTPOINT +fi + +# Install an on-exit handler that will unmount and remove the '$MOUNTPOINT' directory +trap "{ if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; rm -rdfv $MOUNTPOINT; fi; }" EXIT + +mkdir -p $MOUNTPOINT + +hdiutil attach -verbose -noautoopen -mountpoint $MOUNTPOINT "$DMG_PATH" &> $LOGFILE +# Wait for files to show up +# hdiutil uses a helper process, diskimages-helper, which isn't always done its +# work by the time hdiutil exits. So we wait until something shows up in the +# mount point directory. +i=0 +while [ "$(echo $MOUNTPOINT/*)" == "$MOUNTPOINT/*" ]; do + if [ $i -gt $TIMEOUT ]; then + echo "No files found, exiting" + exit 1 + fi + sleep 1 + i=$(expr $i + 1) +done +# Now we can copy everything out of the $MOUNTPOINT directory into the target directory +rsync -av $MOUNTPOINT/* $MOUNTPOINT/.DS_Store $MOUNTPOINT/.background $MOUNTPOINT/.VolumeIcon.icns $TARGETPATH/ > $LOGFILE +# sometimes hdiutil fails with "Resource busy" +hdiutil detach $MOUNTPOINT || { sleep 10; \ + if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; fi; } +i=0 +while [ "$(echo $MOUNTPOINT/*)" != "$MOUNTPOINT/*" ]; do + if [ $i -gt $TIMEOUT ]; then + echo "Cannot umount, exiting" + exit 1 + fi + sleep 1 + i=$(expr $i + 1) +done +rm -rdf $MOUNTPOINT diff --git a/tools/update-verify/release/common/unpack.sh b/tools/update-verify/release/common/unpack.sh new file mode 100755 index 0000000000..3249936493 --- /dev/null +++ b/tools/update-verify/release/common/unpack.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +function cleanup() { + hdiutil detach ${DEV_NAME} || + { sleep 5 && hdiutil detach ${DEV_NAME} -force; }; + return $1 && $?; +}; + +unpack_build () { + unpack_platform="$1" + dir_name="$2" + pkg_file="$3" + locale=$4 + unpack_jars=$5 + update_settings_string=$6 + + if [ ! -f "$pkg_file" ]; then + return 1 + fi + mkdir -p $dir_name + pushd $dir_name > /dev/null + case $unpack_platform in + # $unpack_platform is either + # - a balrog platform name (from testing/mozharness/scripts/release/update-verify-config-creator.py) + # - a simple platform name (from tools/update-verify/release/updates/verify.sh) + mac|Darwin_*) + os=`uname` + # How we unpack a dmg differs depending on which platform we're on. + if [[ "$os" == "Darwin" ]] + then + cd ../ + echo "installing $pkg_file" + ../common/unpack-diskimage.sh "$pkg_file" mnt $dir_name + else + 7z x ../"$pkg_file" > /dev/null + if [ `ls -1 | wc -l` -ne 1 ] + then + echo "Couldn't find .app package" + return 1 + fi + unpack_dir=$(ls -1) + unpack_dir=$(ls -d "${unpack_dir}") + mv "${unpack_dir}"/*.app . + rm -rf "${unpack_dir}" + appdir=$(ls -1) + appdir=$(ls -d *.app) + # The updater guesses the location of these files based on + # its own target architecture, not the mar. If we're not + # unpacking mac-on-mac, we need to copy them so it can find + # them. It's important to copy (and not move), because when + # we diff the installer vs updated build afterwards, the + # installer version will have them in their original place. + cp "${appdir}/Contents/Resources/update-settings.ini" "${appdir}/update-settings.ini" + cp "${appdir}/Contents/Resources/precomplete" "${appdir}/precomplete" + fi + update_settings_file="${appdir}/update-settings.ini" + ;; + win32|WINNT_*) + 7z x ../"$pkg_file" > /dev/null + if [ -d localized ] + then + mkdir bin/ + cp -rp nonlocalized/* bin/ + cp -rp localized/* bin/ + rm -rf nonlocalized + rm -rf localized + if [ $(find optional/ | wc -l) -gt 1 ] + then + cp -rp optional/* bin/ + rm -rf optional + fi + elif [ -d core ] + then + mkdir bin/ + cp -rp core/* bin/ + rm -rf core + else + for file in *.xpi + do + unzip -o $file > /dev/null + done + unzip -o ${locale}.xpi > /dev/null + fi + update_settings_file='bin/update-settings.ini' + ;; + linux|Linux_*) + if `echo $pkg_file | grep -q "tar.gz"` + then + tar xfz ../"$pkg_file" > /dev/null + elif `echo $pkg_file | grep -q "tar.bz2"` + then + tar xfj ../"$pkg_file" > /dev/null + else + echo "Unknown package type for file: $pkg_file" + exit 1 + fi + update_settings_file=`echo $product | tr '[A-Z]' '[a-z]'`'/update-settings.ini' + ;; + *) + echo "Unknown platform to unpack: $unpack_platform" + exit 1 + esac + + if [ ! -z $unpack_jars ]; then + for f in `find . -name '*.jar' -o -name '*.ja'`; do + unzip -o "$f" -d "$f.dir" > /dev/null + done + fi + + if [ ! -z $update_settings_string ]; then + echo "Modifying update-settings.ini" + cat "${update_settings_file}" | sed -e "s/^ACCEPTED_MAR_CHANNEL_IDS.*/ACCEPTED_MAR_CHANNEL_IDS=${update_settings_string}/" > "${update_settings_file}.new" + diff -u "${update_settings_file}" "${update_settings_file}.new" + echo " " + rm "${update_settings_file}" + mv "${update_settings_file}.new" "${update_settings_file}" + fi + + popd > /dev/null + +} diff --git a/tools/update-verify/release/compare-directories.py b/tools/update-verify/release/compare-directories.py new file mode 100755 index 0000000000..ea70d79b31 --- /dev/null +++ b/tools/update-verify/release/compare-directories.py @@ -0,0 +1,276 @@ +#! /usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import difflib +import hashlib +import logging +import os +import sys + +""" Define the transformations needed to make source + update == target + +Required: +The files list describes the files which a transform may be used on. +The 'side' is one of ('source', 'target') and defines where each transform is applied +The 'channel_prefix' list controls which channels a transform may be used for, where a value of +'beta' means all of beta, beta-localtest, beta-cdntest, etc. + +One or more: +A 'deletion' specifies a start of line to match on, removing the whole line +A 'substitution' is a list of full string to match and its replacement +""" +TRANSFORMS = [ + # channel-prefs.js + { + # preprocessor comments, eg //@line 6 "/builds/worker/workspace/... + # this can be removed once each channel has a watershed above 59.0b2 (from bug 1431342) + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["aurora", "beta", "release", "esr"], + "side": "source", + "deletion": '//@line 6 "', + }, + { + # updates from a beta to an RC build, the latter specifies the release channel + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["beta"], + "side": "target", + "substitution": [ + 'pref("app.update.channel", "release");\n', + 'pref("app.update.channel", "beta");\n', + ], + }, + { + # updates from an RC to a beta build + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["beta"], + "side": "source", + "substitution": [ + 'pref("app.update.channel", "release");\n', + 'pref("app.update.channel", "beta");\n', + ], + }, + { + # Warning comments from bug 1576546 + # When updating from a pre-70.0 build to 70.0+ this removes the new comments in + # the target side. In the 70.0+ --> 70.0+ case with a RC we won't need this, and + # the channel munging above will make channel-prefs.js identical, allowing the code + # to break before applying this transform. + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["aurora", "beta", "release", "esr"], + "side": "target", + "deletion": "//", + }, + # update-settings.ini + { + # updates from a beta to an RC build, the latter specifies the release channel + # on mac, we actually have both files. The second location is the real + # one but we copy to the first to run the linux64 updater + "files": ["update-settings.ini", "Contents/Resources/update-settings.ini"], + "channel_prefix": ["beta"], + "side": "target", + "substitution": [ + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n", + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n", + ], + }, + { + # updates from an RC to a beta build + # on mac, we only need to modify the legit file this time. unpack_build + # handles the copy for the updater in both source and target + "files": ["Contents/Resources/update-settings.ini"], + "channel_prefix": ["beta"], + "side": "source", + "substitution": [ + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n", + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n", + ], + }, +] + + +def walk_dir(path): + all_files = [] + all_dirs = [] + + for root, dirs, files in os.walk(path): + all_dirs.extend([os.path.join(root, d) for d in dirs]) + all_files.extend([os.path.join(root, f) for f in files]) + + # trim off directory prefix for easier comparison + all_dirs = [d[len(path) + 1 :] for d in all_dirs] + all_files = [f[len(path) + 1 :] for f in all_files] + + return all_dirs, all_files + + +def compare_listings( + source_list, target_list, label, source_dir, target_dir, ignore_missing=None +): + obj1 = set(source_list) + obj2 = set(target_list) + difference_found = False + ignore_missing = ignore_missing or () + + if ignore_missing: + logging.warning("ignoring paths: {}".format(ignore_missing)) + + left_diff = obj1 - obj2 + if left_diff: + if left_diff - set(ignore_missing): + _log = logging.error + difference_found = True + else: + _log = logging.warning + _log("Ignoring missing files due to ignore_missing") + + _log("{} only in {}:".format(label, source_dir)) + for d in sorted(left_diff): + _log(" {}".format(d)) + + right_diff = obj2 - obj1 + if right_diff: + logging.error("{} only in {}:".format(label, target_dir)) + for d in sorted(right_diff): + logging.error(" {}".format(d)) + difference_found = True + + return difference_found + + +def hash_file(filename): + h = hashlib.sha256() + with open(filename, "rb", buffering=0) as f: + for b in iter(lambda: f.read(128 * 1024), b""): + h.update(b) + return h.hexdigest() + + +def compare_common_files(files, channel, source_dir, target_dir): + difference_found = False + for filename in files: + source_file = os.path.join(source_dir, filename) + target_file = os.path.join(target_dir, filename) + + if os.stat(source_file).st_size != os.stat(target_file).st_size or hash_file( + source_file + ) != hash_file(target_file): + logging.info("Difference found in {}".format(filename)) + file_contents = { + "source": open(source_file).readlines(), + "target": open(target_file).readlines(), + } + + transforms = [ + t + for t in TRANSFORMS + if filename in t["files"] + and channel.startswith(tuple(t["channel_prefix"])) + ] + logging.debug( + "Got {} transform(s) to consider for {}".format( + len(transforms), filename + ) + ) + for transform in transforms: + side = transform["side"] + + if "deletion" in transform: + d = transform["deletion"] + logging.debug( + "Trying deleting lines starting {} from {}".format(d, side) + ) + file_contents[side] = [ + l for l in file_contents[side] if not l.startswith(d) + ] + + if "substitution" in transform: + r = transform["substitution"] + logging.debug("Trying replacement for {} in {}".format(r, side)) + file_contents[side] = [ + l.replace(r[0], r[1]) for l in file_contents[side] + ] + + if file_contents["source"] == file_contents["target"]: + logging.info("Transforms removed all differences") + break + + if file_contents["source"] != file_contents["target"]: + difference_found = True + logging.error( + "{} still differs after transforms, residual diff:".format(filename) + ) + for l in difflib.unified_diff( + file_contents["source"], file_contents["target"] + ): + logging.error(l.rstrip()) + + return difference_found + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + "Compare two directories recursively, with transformations for expected diffs" + ) + parser.add_argument("source", help="Directory containing updated Firefox") + parser.add_argument("target", help="Directory containing expected Firefox") + parser.add_argument("channel", help="Update channel used") + parser.add_argument( + "--verbose", "-v", action="store_true", help="Enable verbose logging" + ) + parser.add_argument( + "--ignore-missing", + action="append", + metavar="<path>", + help="Ignore absence of <path> in the target", + ) + + args = parser.parse_args() + level = logging.INFO + if args.verbose: + level = logging.DEBUG + logging.basicConfig(level=level, format="%(message)s", stream=sys.stdout) + + source = args.source + target = args.target + if not os.path.exists(source) or not os.path.exists(target): + logging.error("Source and/or target directory doesn't exist") + sys.exit(3) + + logging.info("Comparing {} with {}...".format(source, target)) + source_dirs, source_files = walk_dir(source) + target_dirs, target_files = walk_dir(target) + + dir_list_diff = compare_listings( + source_dirs, target_dirs, "Directories", source, target + ) + file_list_diff = compare_listings( + source_files, target_files, "Files", source, target, args.ignore_missing + ) + file_diff = compare_common_files( + set(source_files) & set(target_files), args.channel, source, target + ) + + if file_diff: + # Use status of 2 since python will use 1 if there is an error running the script + sys.exit(2) + elif dir_list_diff or file_list_diff: + # this has traditionally been a WARN, but we don't have files on one + # side anymore so lets FAIL + sys.exit(2) + else: + logging.info("No differences found") diff --git a/tools/update-verify/release/final-verification.sh b/tools/update-verify/release/final-verification.sh new file mode 100755 index 0000000000..879c64697f --- /dev/null +++ b/tools/update-verify/release/final-verification.sh @@ -0,0 +1,519 @@ +#!/bin/bash + +function usage { + log "In the updates subdirectory of the directory this script is in," + log "there are a bunch of config files. You should call this script," + log "passing the names of one or more of those files as parameters" + log "to this script." + log "" + log "This will validate that the update.xml files all exist for the" + log "given config file, and that they report the correct file sizes" + log "for the associated mar files, and that the associated mar files" + log "are available on the update servers." + log "" + log "This script will spawn multiple curl processes to query the" + log "snippets (update.xml file downloads) and the download urls in" + log "parallel. The number of parallel curl processes can be managed" + log "with the -p MAX_PROCS option." + log "" + log "Only the first three bytes of the mar files are downloaded" + log "using curl -r 0-2 option to save time. GET requests are issued" + log "rather than HEAD requests, since Akamai (one of our CDN" + log "partners) caches GET and HEAD requests separately - therefore" + log "they can be out-of-sync, and it is important that we validate" + log "that the GET requests return the expected results." + log "" + log "Please note this script can run on linux and OS X. It has not" + log "been tested on Windows, but may also work. It can be run" + log "locally, and does not require access to the mozilla vpn or" + log "any other special network, since the update servers are" + log "available over the internet. However, it does require an" + log "up-to-date checkout of the tools repository, as the updates/" + log "subfolder changes over time, and reflects the currently" + log "available updates. It makes no changes to the update servers" + log "so there is no harm in running it. It simply generates a" + log "report. However, please try to avoid hammering the update" + log "servers aggressively, e.g. with thousands of parallel" + log "processes. For example, feel free to run the examples below," + log "first making sure that your source code checkout is up-to-" + log "date on your own machine, to get the latest configs in the" + log "updates/ subdirectory." + log "" + log "Usage:" + log " $(basename "${0}") [-p MAX_PROCS] config1 [config2 config3 config4 ...]" + log " $(basename "${0}") -h" + log "" + log "Examples:" + log " 1. $(basename "${0}") -p 128 mozBeta-thunderbird-linux.cfg mozBeta-thunderbird-linux64.cfg" + log " 2. $(basename "${0}") mozBeta-thunderbird-linux64.cfg" +} + +function log { + echo "$(date): ${1}" +} + +# subprocesses don't log in real time, due to synchronisation +# issues which can cause log entries to overwrite each other. +# therefore this function outputs log entries written to +# temporary files on disk, and then deletes them. +function flush_logs { + ls -1rt "${TMPDIR}" | grep '^log\.' | while read LOG + do + cat "${TMPDIR}/${LOG}" + rm "${TMPDIR}/${LOG}" + done +} + +# this function takes an update.xml url as an argument +# and then logs a list of config files and their line +# numbers, that led to this update.xml url being tested +function show_cfg_file_entries { + local update_xml_url="${1}" + cat "${update_xml_urls}" | cut -f1 -d' ' | grep -Fn "${update_xml_url}" | sed 's/:.*//' | while read match_line_no + do + cfg_file="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f3 -d' ')" + cfg_line_no="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f4 -d' ')" + log " ${cfg_file} line ${cfg_line_no}: $(sed -n -e "${cfg_line_no}p" "${cfg_file}")" + done +} + +# this function takes a mar url as an argument and then +# logs information about which update.xml urls referenced +# this mar url, and which config files referenced those +# mar urls - so you have a full understanding of why this +# mar url was ever tested +function show_update_xml_entries { + local mar_url="${1}" + grep -Frl "${mar_url}" "${TMPDIR}" | grep '/update_xml_to_mar\.' | while read update_xml_to_mar + do + mar_size="$(cat "${update_xml_to_mar}" | cut -f2 -d' ')" + update_xml_url="$(cat "${update_xml_to_mar}" | cut -f3 -d' ')" + patch_type="$(cat "${update_xml_to_mar}" | cut -f4 -d' ')" + update_xml_actual_url="$(cat "${update_xml_to_mar}" | cut -f5 -d' ')" + log " ${update_xml_url}" + [ -n "${update_xml_actual_url}" ] && log " which redirected to: ${update_xml_actual_url}" + log " This contained an entry for:" + log " patch type: ${patch_type}" + log " mar size: ${mar_size}" + log " mar url: ${mar_url}" + log " The update.xml url above was retrieved because of the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" | sed 's/ / /' + done +} + +echo -n "$(date): Command called:" +for ((INDEX=0; INDEX<=$#; INDEX+=1)) +do + echo -n " '${!INDEX}'" +done +echo '' +log "From directory: '$(pwd)'" +log '' +log "Parsing arguments..." + +# Max procs lowered in bug 894368 to try to avoid spurious failures +MAX_PROCS=48 +BAD_ARG=0 +BAD_FILE=0 +while getopts p:h OPT +do + case "${OPT}" in + p) MAX_PROCS="${OPTARG}";; + h) usage + exit;; + *) BAD_ARG=1;; + esac +done +shift "$((OPTIND - 1))" + +# invalid option specified +[ "${BAD_ARG}" == 1 ] && exit 66 + +log "Checking one or more config files have been specified..." +if [ $# -lt 1 ] +then + usage + log "ERROR: You must specify one or more config files" + exit 64 +fi + +log "Checking whether MAX_PROCS is a number..." +if ! let x=MAX_PROCS 2>/dev/null +then + usage + log "ERROR: MAX_PROCS must be a number (-p option); you specified '${MAX_PROCS}' - this is not a number." + exit 65 +fi + +# config files are in updates subdirectory below this script +if ! cd "$(dirname "${0}")/updates" 2>/dev/null +then + log "ERROR: Cannot cd into '$(dirname "${0}")/updates' from '$(pwd)'" + exit 68 +fi + +log "Checking specified config files (and downloading them if necessary):" +log '' +configs=() +for file in "${@}" +do + if [[ ${file} == http* ]] + then + log " Downloading config file '${file}'" + cfg=$(mktemp) + curl -fL --retry 5 --compressed "${file}" > "$cfg" + if [ "$?" != 0 ]; then + log "Error downloading config file '${file}'" + BAD_FILE=1 + else + log " * '${file}' ok, downloaded to '${cfg}'" + configs+=($cfg) + fi + elif [ -f "${file}" ] + then + log " * '${file}' ok" + configs+=(${file}) + else + log " * '${file}' missing" + BAD_FILE=1 + fi +done +log '' + +# invalid config specified +if [ "${BAD_FILE}" == 1 ] +then + log "ERROR: Unable to download config file(s) or config files are missing from repo - see above" + exit 67 +fi + +log "All checks completed successfully." +log '' +log "Starting stopwatch..." +log '' +log "Please be aware output will now be buffered up, and only displayed after completion." +log "Therefore do not be alarmed if you see no output for several minutes." +log "See https://bugzilla.mozilla.org/show_bug.cgi?id=863602#c5 for details". +log '' + +START_TIME="$(date +%s)" + +# Create a temporary directory for all temp files, that can easily be +# deleted afterwards. See https://bugzilla.mozilla.org/show_bug.cgi?id=863602 +# to understand why we write everything in distinct temporary files rather +# than writing to standard error/standard out or files shared across +# processes. +# Need to unset TMPDIR first since it affects mktemp behaviour on next line +unset TMPDIR +export TMPDIR="$(mktemp -d -t final_verification.XXXXXXXXXX)" + +# this temporary file will list all update urls that need to be checked, in this format: +# <update url> <comma separated list of patch types> <cfg file that requests it> <line number of config file> +# e.g. +# https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zh-TW/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64-major.cfg 3 +# https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zu/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64.cfg 7 +# https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/ach/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64-major.cfg 11 +# https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/af/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64.cfg 17 +update_xml_urls="$(mktemp -t update_xml_urls.XXXXXXXXXX)" + +#################################################################################### +# And now a summary of all temp files that will get generated during this process... +# +# 1) mktemp -t failure.XXXXXXXXXX +# +# Each failure will generate a one line temp file, which is a space separated +# output of the error code, and the instance data for the failure. +# e.g. +# +# PATCH_TYPE_MISSING https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/update.xml?force=1 complete https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/default/default/default/update.xml?force=1 +# +# 2) mktemp -t update_xml_to_mar.XXXXXXXXXX +# +# For each mar url referenced in an update.xml file, a temp file will be created to store the +# association between update.xml url and mar url. This is later used (e.g. in function +# show_update_xml_entries) to trace back the update.xml url(s) that led to a mar url being +# tested. It is also used to keep a full list of mar urls to test. +# e.g. +# +# <mar url> <mar size> <update.xml url> <patch type> <update.xml redirection url, if HTTP 302 returned> +# +# 3) mktemp -t log.XXXXXXXXXX +# +# For each log message logged by a subprocesses, we will create a temp log file with the +# contents of the log message, since we cannot safely output the log message from the subprocess +# and guarantee that it will be correctly output. By buffering log output in individual log files +# we guarantee that log messages will not interfere with each other. We then flush them when all +# forked subprocesses have completed. +# +# 4) mktemp -t mar_headers.XXXXXXXXXX +# +# We keep a copy of the mar url http headers retrieved in one file per mar url. +# +# 5) mktemp -t update.xml.headers.XXXXXXXXXX +# +# We keep a copy of the update.xml http headers retrieved in one file per update.xml url. +# +# 6) mktemp -t update.xml.XXXXXXXXXX +# +# We keep a copy of each update.xml file retrieved in individual files. +#################################################################################### + + +# generate full list of update.xml urls, followed by patch types, +# as defined in the specified config files - and write into "${update_xml_urls}" file +aus_server="https://aus5.mozilla.org" +for cfg_file in "${configs[@]}" +do + line_no=0 + sed -e 's/localtest/cdntest/' "${cfg_file}" | while read config_line + do + let line_no++ + # to avoid contamination between iterations, reset variables + # each loop in case they are not declared + # aus_server is not "cleared" each iteration - to be consistent with previous behaviour of old + # final-verification.sh script - might be worth reviewing if we really want this behaviour + release="" product="" platform="" build_id="" locales="" channel="" from="" patch_types="complete" + eval "${config_line}" + for locale in ${locales} + do + echo "${aus_server}/update/3/$product/$release/$build_id/$platform/$locale/$channel/default/default/default/update.xml?force=1" "${patch_types// /,}" "${cfg_file}" "${line_no}" + done + done +done > "${update_xml_urls}" + +# download update.xml files and grab the mar urls from downloaded file for each patch type required +cat "${update_xml_urls}" | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../get-update-xml.sh +if [ "$?" != 0 ]; then + flush_logs + log "Error generating update requests" + exit 70 +fi + +flush_logs + +# download http header for each mar url +find "${TMPDIR}" -name 'update_xml_to_mar.*' -type f | xargs cat | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../test-mar-url.sh +if [ "$?" != 0 ]; then + flush_logs + log "Error HEADing mar urls" + exit 71 +fi + +flush_logs + +log '' +log 'Stopping stopwatch...' +STOP_TIME="$(date +%s)" + +number_of_failures="$(find "${TMPDIR}" -name 'failure.*' -type f | wc -l | sed 's/ //g')" +number_of_update_xml_urls="$(cat "${update_xml_urls}" | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')" +number_of_mar_urls="$(find "${TMPDIR}" -name "update_xml_to_mar.*" | xargs cat | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')" + +if [ "${number_of_failures}" -eq 0 ] +then + log + log "All tests passed successfully." + log + exit_code=0 +else + log '' + log '====================================' + [ "${number_of_failures}" -gt 1 ] && log "${number_of_failures} FAILURES" || log '1 FAILURE' + failure=0 + ls -1tr "${TMPDIR}" | grep '^failure\.' | while read failure_file + do + while read failure_code entry1 entry2 entry3 entry4 entry5 entry6 entry7 + do + log '====================================' + log '' + case "${failure_code}" in + + UPDATE_XML_UNAVAILABLE) + update_xml_url="${entry1}" + update_xml="${entry2}" + update_xml_headers="${entry3}" + update_xml_debug="${entry4}" + update_xml_curl_exit_code="${entry5}" + log "FAILURE $((++failure)): Update xml file not available" + log "" + log " Download url: ${update_xml_url}" + log " Curl returned exit code: ${update_xml_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_headers}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_debug}" + log "" + log " The returned update.xml file was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml}" + log "" + log " This url was tested because of the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" + log "" + + ;; + + UPDATE_XML_REDIRECT_FAILED) + update_xml_url="${entry1}" + update_xml_actual_url="${entry2}" + update_xml="${entry3}" + update_xml_headers="${entry4}" + update_xml_debug="${entry5}" + update_xml_curl_exit_code="${entry6}" + log "FAILURE $((++failure)): Update xml file not available at *redirected* location" + log "" + log " Download url: ${update_xml_url}" + log " Redirected to: ${update_xml_actual_url}" + log " It could not be downloaded from this url." + log " Curl returned exit code: ${update_xml_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_headers}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_debug}" + log "" + log " The returned update.xml file was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml}" + log "" + log " This url was tested because of the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" + log "" + ;; + + PATCH_TYPE_MISSING) + update_xml_url="${entry1}" + patch_type="${entry2}" + update_xml="${entry3}" + update_xml_headers="${entry4}" + update_xml_debug="${entry5}" + update_xml_actual_url="${entry6}" + log "FAILURE $((++failure)): Patch type '${patch_type}' not present in the downloaded update.xml file." + log "" + log " Update xml file downloaded from: ${update_xml_url}" + [ -n "${update_xml_actual_url}" ] && log " This redirected to the download url: ${update_xml_actual_url}" + log " Curl returned exit code: 0 (success)" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_headers}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_debug}" + log "" + log " The returned update.xml file was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml}" + log "" + log " This url and patch type combination was tested due to the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" + log "" + ;; + + NO_MAR_FILE) + mar_url="${entry1}" + mar_headers_file="${entry2}" + mar_headers_debug_file="${entry3}" + mar_file_curl_exit_code="${entry4}" + mar_actual_url="${entry5}" + log "FAILURE $((++failure)): Could not retrieve mar file" + log "" + log " Mar file url: ${mar_url}" + [ -n "${mar_actual_url}" ] && log " This redirected to: ${mar_actual_url}" + log " The mar file could not be downloaded from this location." + log " Curl returned exit code: ${mar_file_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_file}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_debug_file}" + log "" + log " The mar download was tested because it was referenced in the following update xml file(s):" + show_update_xml_entries "${mar_url}" + log "" + ;; + + MAR_FILE_WRONG_SIZE) + mar_url="${entry1}" + mar_required_size="${entry2}" + mar_actual_size="${entry3}" + mar_headers_file="${entry4}" + mar_headers_debug_file="${entry5}" + mar_file_curl_exit_code="${entry6}" + mar_actual_url="${entry7}" + log "FAILURE $((++failure)): Mar file is wrong size" + log "" + log " Mar file url: ${mar_url}" + [ -n "${mar_actual_url}" ] && log " This redirected to: ${mar_actual_url}" + log " The http header of the mar file url says that the mar file is ${mar_actual_size} bytes." + log " One or more of the following update.xml file(s) says that the file should be ${mar_required_size} bytes." + log "" + log " These are the update xml file(s) that referenced this mar:" + show_update_xml_entries "${mar_url}" + log "" + log " Curl returned exit code: ${mar_file_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_file}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_debug_file}" + log "" + ;; + + BAD_HTTP_RESPONSE_CODE_FOR_MAR) + mar_url="${entry1}" + mar_headers_file="${entry2}" + mar_headers_debug_file="${entry3}" + mar_file_curl_exit_code="${entry4}" + mar_actual_url="${entry5}" + http_response_code="$(sed -e "s/$(printf '\r')//" -n -e '/^HTTP\//p' "${mar_headers_file}" | tail -1)" + log "FAILURE $((++failure)): '${http_response_code}' for mar file" + log "" + log " Mar file url: ${mar_url}" + [ -n "${mar_actual_url}" ] && log " This redirected to: ${mar_actual_url}" + log "" + log " These are the update xml file(s) that referenced this mar:" + show_update_xml_entries "${mar_url}" + log "" + log " Curl returned exit code: ${mar_file_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_file}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_debug_file}" + log "" + ;; + + *) + log "ERROR: Unknown failure code - '${failure_code}'" + log "ERROR: This is a serious bug in this script." + log "ERROR: Only known failure codes are: UPDATE_XML_UNAVAILABLE, UPDATE_XML_REDIRECT_FAILED, PATCH_TYPE_MISSING, NO_MAR_FILE, MAR_FILE_WRONG_SIZE, BAD_HTTP_RESPONSE_CODE_FOR_MAR" + log "" + log "FAILURE $((++failure)): Data from failure is: ${entry1} ${entry2} ${entry3} ${entry4} ${entry5} ${entry6}" + log "" + ;; + + esac + done < "${TMPDIR}/${failure_file}" + done + exit_code=1 +fi + + +log '' +log '====================================' +log 'KEY STATS' +log '====================================' +log '' +log "Config files scanned: ${#@}" +log "Update xml files downloaded and parsed: ${number_of_update_xml_urls}" +log "Unique mar urls found: ${number_of_mar_urls}" +log "Failures: ${number_of_failures}" +log "Parallel processes used (maximum limit): ${MAX_PROCS}" +log "Execution time: $((STOP_TIME-START_TIME)) seconds" +log '' + +rm -rf "${TMPDIR}" +exit ${exit_code} diff --git a/tools/update-verify/release/get-update-xml.sh b/tools/update-verify/release/get-update-xml.sh new file mode 100755 index 0000000000..4c1fa724a8 --- /dev/null +++ b/tools/update-verify/release/get-update-xml.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +update_xml_url="${1}" +patch_types="${2}" +update_xml="$(mktemp -t update.xml.XXXXXXXXXX)" +update_xml_headers="$(mktemp -t update.xml.headers.XXXXXXXXXX)" +update_xml_debug="$(mktemp -t update.xml.debug.XXXXXXXXXX)" +curl --retry 50 --retry-max-time 300 -s --show-error -D "${update_xml_headers}" -L -v -H "Cache-Control: max-stale=0" "${update_xml_url}" > "${update_xml}" 2>"${update_xml_debug}" +update_xml_curl_exit_code=$? +if [ "${update_xml_curl_exit_code}" == 0 ] +then + update_xml_actual_url="$(sed -e "s/$(printf '\r')//" -n -e 's/^Location: //p' "${update_xml_headers}" | tail -1)" + [ -n "${update_xml_actual_url}" ] && update_xml_url_with_redirects="${update_xml_url} => ${update_xml_actual_url}" || update_xml_url_with_redirects="${update_xml_url}" + echo "$(date): Downloaded update.xml file from ${update_xml_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)" + for patch_type in ${patch_types//,/ } + do + mar_url_and_size="$(sed -e 's/\&/\&/g' -n -e 's/.*<patch .*type="'"${patch_type}"'".* URL="\([^"]*\)".*size="\([^"]*\)".*/\1 \2/p' "${update_xml}" | tail -1)" + if [ -z "${mar_url_and_size}" ] + then + echo "$(date): FAILURE: No patch type '${patch_type}' found in update.xml from ${update_xml_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)" + echo "PATCH_TYPE_MISSING ${update_xml_url} ${patch_type} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" + else + echo "$(date): Mar url and file size for patch type '${patch_type}' extracted from ${update_xml_url_with_redirects} (${mar_url_and_size})" > "$(mktemp -t log.XXXXXXXXXX)" + echo "${mar_url_and_size} ${update_xml_url} ${patch_type} ${update_xml_actual_url}" > "$(mktemp -t update_xml_to_mar.XXXXXXXXXX)" + fi + done +else + if [ -z "${update_xml_actual_url}" ] + then + echo "$(date): FAILURE: Could not retrieve update.xml from ${update_xml_url} for patch type(s) '${patch_types}'" > "$(mktemp -t log.XXXXXXXXXX)" + echo "UPDATE_XML_UNAVAILABLE ${update_xml_url} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_curl_exit_code}" > "$(mktemp -t failure.XXXXXXXXXX)" + else + echo "$(date): FAILURE: update.xml from ${update_xml_url} redirected to ${update_xml_actual_url} but could not retrieve update.xml from here" > "$(mktemp -t log.XXXXXXXXXX)" + echo "UPDATE_XML_REDIRECT_FAILED ${update_xml_url} ${update_xml_actual_url} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_curl_exit_code}" > "$(mktemp -t failure.XXXXXXXXXX)" + fi +fi diff --git a/tools/update-verify/release/mar_certs/README b/tools/update-verify/release/mar_certs/README new file mode 100644 index 0000000000..dd931ef1d3 --- /dev/null +++ b/tools/update-verify/release/mar_certs/README @@ -0,0 +1,29 @@ +These certificates are imported from mozilla-central (https://hg.mozilla.org/mozilla-central/file/tip/toolkit/mozapps/update/updater) +and used to support staging update verify jobs. These jobs end up replacing the certificates within the binaries +(through a binary search and replace), and must all be the same length for this to work correctly. If we recreate +these certificates, and the resulting public certificates are not the same length anymore, the commonName may be +changed to line them up again. https://github.com/google/der-ascii is a useful tool for doing this. For example: + +To convert the certificate to ascii: +der2ascii -i dep1.der -o dep1.ascii + +Then use your favourite editor to change the commonName field. That block will look something like: + SEQUENCE { + SET { + SEQUENCE { + # commonName + OBJECT_IDENTIFIER { 2.5.4.3 } + PrintableString { "CI MAR signing key 1" } + } + } + } + +You can pad the PrintableString with spaces to increase the length of the cert (1 space = 1 byte). + +Then, convert back to der: +ascii2der -i dep1.ascii -o newdep1.der + +The certificats in the sha1 subdirectory are from +https://hg.mozilla.org/mozilla-central/file/0fcbe72581bc/toolkit/mozapps/update/updater +which are the SHA-1 certs from before they where updated in Bug 1105689. They only include the release +certs, since the nightly certs are different length, and we only care about updates from old ESRs. diff --git a/tools/update-verify/release/mar_certs/dep1.der b/tools/update-verify/release/mar_certs/dep1.der Binary files differnew file mode 100644 index 0000000000..5320f41dfa --- /dev/null +++ b/tools/update-verify/release/mar_certs/dep1.der diff --git a/tools/update-verify/release/mar_certs/dep2.der b/tools/update-verify/release/mar_certs/dep2.der Binary files differnew file mode 100644 index 0000000000..f3eb568425 --- /dev/null +++ b/tools/update-verify/release/mar_certs/dep2.der diff --git a/tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der b/tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der Binary files differnew file mode 100644 index 0000000000..44fd95dcff --- /dev/null +++ b/tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der diff --git a/tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der b/tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der Binary files differnew file mode 100644 index 0000000000..90f8e6e82c --- /dev/null +++ b/tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der diff --git a/tools/update-verify/release/mar_certs/release_primary.der b/tools/update-verify/release/mar_certs/release_primary.der Binary files differnew file mode 100644 index 0000000000..1d94f88ad7 --- /dev/null +++ b/tools/update-verify/release/mar_certs/release_primary.der diff --git a/tools/update-verify/release/mar_certs/release_secondary.der b/tools/update-verify/release/mar_certs/release_secondary.der Binary files differnew file mode 100644 index 0000000000..474706c4b7 --- /dev/null +++ b/tools/update-verify/release/mar_certs/release_secondary.der diff --git a/tools/update-verify/release/mar_certs/sha1/dep1.der b/tools/update-verify/release/mar_certs/sha1/dep1.der Binary files differnew file mode 100644 index 0000000000..ec8ce6184d --- /dev/null +++ b/tools/update-verify/release/mar_certs/sha1/dep1.der diff --git a/tools/update-verify/release/mar_certs/sha1/dep2.der b/tools/update-verify/release/mar_certs/sha1/dep2.der Binary files differnew file mode 100644 index 0000000000..4d0f244df2 --- /dev/null +++ b/tools/update-verify/release/mar_certs/sha1/dep2.der diff --git a/tools/update-verify/release/mar_certs/sha1/release_primary.der b/tools/update-verify/release/mar_certs/sha1/release_primary.der Binary files differnew file mode 100644 index 0000000000..11417c35e7 --- /dev/null +++ b/tools/update-verify/release/mar_certs/sha1/release_primary.der diff --git a/tools/update-verify/release/mar_certs/sha1/release_secondary.der b/tools/update-verify/release/mar_certs/sha1/release_secondary.der Binary files differnew file mode 100644 index 0000000000..16a7ef6d91 --- /dev/null +++ b/tools/update-verify/release/mar_certs/sha1/release_secondary.der diff --git a/tools/update-verify/release/mar_certs/xpcshellCertificate.der b/tools/update-verify/release/mar_certs/xpcshellCertificate.der Binary files differnew file mode 100644 index 0000000000..ea1fd47faa --- /dev/null +++ b/tools/update-verify/release/mar_certs/xpcshellCertificate.der diff --git a/tools/update-verify/release/replace-updater-certs.py b/tools/update-verify/release/replace-updater-certs.py new file mode 100644 index 0000000000..9e981fbfe0 --- /dev/null +++ b/tools/update-verify/release/replace-updater-certs.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os.path +import sys + +cert_dir = sys.argv[1] +# Read twice, because strings cannot be copied +updater_data = open(sys.argv[2], "rb").read() +new_updater = open(sys.argv[2], "rb").read() +outfile = sys.argv[3] + +cert_pairs = sys.argv[4:] + +if (len(cert_pairs) % 2) != 0: + print("Certs must be provided in pairs") + sys.exit(1) + +for find_cert, replace_cert in zip(*[iter(cert_pairs)] * 2): + find = open(os.path.join(cert_dir, find_cert), "rb").read() + replace = open(os.path.join(cert_dir, replace_cert), "rb").read() + print("Looking for {}...".format(find_cert)) + if find in new_updater: + print("Replacing {} with {}".format(find_cert, replace_cert)) + new_updater = new_updater.replace(find, replace) + else: + print("Didn't find {}...".format(find_cert)) + +if len(updater_data) != len(new_updater): + print( + "WARNING: new updater is not the same length as the old one (old: {}, new: {})".format( + len(updater_data), len(new_updater) + ) + ) + +if updater_data == new_updater: + print("WARNING: updater is unchanged") + +with open(outfile, "wb+") as f: + f.write(new_updater) diff --git a/tools/update-verify/release/test-mar-url.sh b/tools/update-verify/release/test-mar-url.sh new file mode 100755 index 0000000000..217c03d72a --- /dev/null +++ b/tools/update-verify/release/test-mar-url.sh @@ -0,0 +1,46 @@ +#!/bin/bash +mar_url="${1}" +mar_required_size="${2}" + +mar_headers_file="$(mktemp -t mar_headers.XXXXXXXXXX)" +mar_headers_debug_file="$(mktemp -t mar_headers_debug.XXXXXXXXXX)" +curl --retry 50 --retry-max-time 300 -s -i -r 0-2 -L -v "${mar_url}" > "${mar_headers_file}" 2>"${mar_headers_debug_file}" +mar_file_curl_exit_code=$? + +# Bug 894368 - HTTP 408's are not handled by the "curl --retry" mechanism; in this case retry in bash +attempts=1 +while [ "$((++attempts))" -lt 50 ] && grep 'HTTP/1\.1 408 Request Timeout' "${mar_headers_file}" &>/dev/null +do + sleep 1 + curl --retry 50 --retry-max-time 300 -s -i -r 0-2 -L -v "${mar_url}" > "${mar_headers_file}" 2>"${mar_headers_debug_file}" + mar_file_curl_exit_code=$? +done + +# check file size matches what was written in update.xml +# strip out dos line returns from header if they occur +# note: below, using $(printf '\r') for Darwin compatibility, rather than simple '\r' +# (i.e. shell interprets '\r' rather than sed interpretting '\r') +mar_actual_size="$(sed -e "s/$(printf '\r')//" -n -e 's/^Content-Range: bytes 0-2\///ip' "${mar_headers_file}" | tail -1)" +mar_actual_url="$(sed -e "s/$(printf '\r')//" -n -e 's/^Location: //p' "${mar_headers_file}" | tail -1)" +# note: below, sed -n '/^HTTP\//p' acts as grep '^HTTP/', but requires less overhead as sed already running +http_response_code="$(sed -e "s/$(printf '\r')//" -n -e '/^HTTP\//p' "${mar_headers_file}" | tail -1)" + +[ -n "${mar_actual_url}" ] && mar_url_with_redirects="${mar_url} => ${mar_actual_url}" || mar_url_with_redirects="${mar_url}" + +if [ "${mar_actual_size}" == "${mar_required_size}" ] +then + echo "$(date): Mar file ${mar_url_with_redirects} available with correct size (${mar_actual_size} bytes)" > "$(mktemp -t log.XXXXXXXXXX)" +elif [ -z "${mar_actual_size}" ] +then + echo "$(date): FAILURE: Could not retrieve http header for mar file from ${mar_url}" > "$(mktemp -t log.XXXXXXXXXX)" + echo "NO_MAR_FILE ${mar_url} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" + # If we get a response code (i.e. not an empty string), it better contain "206 Partial Content" or we should report on it. + # If response code is empty, this should be caught by a different block to this one (e.g. "could not retrieve http header"). +elif [ -n "${http_response_code}" ] && [ "${http_response_code}" == "${http_response_code/206 Partial Content/}" ] +then + echo "$(date): FAILURE: received a '${http_response_code}' response for mar file from ${mar_url} (expected HTTP 206 Partial Content)" > "$(mktemp -t log.XXXXXXXXXX)" + echo "BAD_HTTP_RESPONSE_CODE_FOR_MAR ${mar_url} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" +else + echo "$(date): FAILURE: Mar file incorrect size - should be ${mar_required_size} bytes, but is ${mar_actual_size} bytes - ${mar_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)" + echo "MAR_FILE_WRONG_SIZE ${mar_url} ${mar_required_size} ${mar_actual_size} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" +fi diff --git a/tools/update-verify/release/updates/verify.sh b/tools/update-verify/release/updates/verify.sh new file mode 100755 index 0000000000..3f8556b424 --- /dev/null +++ b/tools/update-verify/release/updates/verify.sh @@ -0,0 +1,292 @@ +#!/bin/bash +#set -x + +. ../common/cached_download.sh +. ../common/unpack.sh +. ../common/download_mars.sh +. ../common/download_builds.sh +. ../common/check_updates.sh + +# Cache init being handled by new async_download.py +# clear_cache +# create_cache + +ftp_server_to="http://stage.mozilla.org/pub/mozilla.org" +ftp_server_from="http://stage.mozilla.org/pub/mozilla.org" +aus_server="https://aus4.mozilla.org" +to="" +to_build_id="" +to_app_version="" +to_display_version="" +override_certs="" +diff_summary_log=${DIFF_SUMMARY_LOG:-"$PWD/diff-summary.log"} +if [ -e ${diff_summary_log} ]; then + rm ${diff_summary_log} +fi +touch ${diff_summary_log} + +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3" +cert_replacer="$MY_DIR/../replace-updater-certs.py" + +dep_overrides="nightly_aurora_level3_primary.der dep1.der nightly_aurora_level3_secondary.der dep2.der release_primary.der dep1.der release_secondary.der dep2.der sha1/release_primary.der sha1/dep1.der sha1/release_secondary.der sha1/dep2.der" +nightly_overrides="dep1.der nightly_aurora_level3_primary.der dep2.der nightly_aurora_level3_secondary.der release_primary.der nightly_aurora_level3_primary.der release_secondary.der nightly_aurora_level3_secondary.der" +release_overrides="dep1.der release_primary.der dep2.der release_secondary.der nightly_aurora_level3_primary.der release_primary.der nightly_aurora_level3_secondary.der release_secondary.der" + +runmode=0 +config_file="updates.cfg" +UPDATE_ONLY=1 +TEST_ONLY=2 +MARS_ONLY=3 +COMPLETE=4 + +usage() +{ + echo "Usage: verify.sh [OPTION] [CONFIG_FILE]" + echo " -u, --update-only only download update.xml" + echo " -t, --test-only only test that MARs exist" + echo " -m, --mars-only only test MARs" + echo " -c, --complete complete upgrade test" +} + +if [ -z "$*" ] +then + usage + exit 0 +fi + +pass_arg_count=0 +while [ "$#" -gt "$pass_arg_count" ] +do + case "$1" in + -u | --update-only) + runmode=$UPDATE_ONLY + shift + ;; + -t | --test-only) + runmode=$TEST_ONLY + shift + ;; + -m | --mars-only) + runmode=$MARS_ONLY + shift + ;; + -c | --complete) + runmode=$COMPLETE + shift + ;; + *) + # Move the unrecognized arg to the end of the list + arg="$1" + shift + set -- "$@" "$arg" + pass_arg_count=`expr $pass_arg_count + 1` + esac +done + +if [ -n "$arg" ] +then + config_file=$arg + echo "Using config file $config_file" +else + echo "Using default config file $config_file" +fi + +if [ "$runmode" == "0" ] +then + usage + exit 0 +fi + +while read entry +do + # initialize all config variables + release="" + product="" + platform="" + build_id="" + locales="" + channel="" + from="" + patch_types="complete" + use_old_updater=0 + mar_channel_IDs="" + updater_package="" + eval $entry + + # the arguments for updater changed in Gecko 34/SeaMonkey 2.31 + major_version=`echo $release | cut -f1 -d.` + if [[ "$product" == "seamonkey" ]]; then + minor_version=`echo $release | cut -f2 -d.` + if [[ $major_version -le 2 && $minor_version -lt 31 ]]; then + use_old_updater=1 + fi + elif [[ $major_version -lt 34 ]]; then + use_old_updater=1 + fi + + # Note: cross platform tests seem to work for everything except Mac-on-Windows. + # We probably don't care about this use case. + if [[ "$updater_package" == "" ]]; then + updater_package="$from" + fi + + for locale in $locales + do + rm -f update/partial.size update/complete.size + for patch_type in $patch_types + do + update_path="${product}/${release}/${build_id}/${platform}/${locale}/${channel}/default/default/default" + if [ "$runmode" == "$MARS_ONLY" ] || [ "$runmode" == "$COMPLETE" ] || + [ "$runmode" == "$TEST_ONLY" ] + then + if [ "$runmode" == "$TEST_ONLY" ] + then + download_mars "${aus_server}/update/3/${update_path}/default/update.xml?force=1" ${patch_type} 1 \ + "${to_build_id}" "${to_app_version}" "${to_display_version}" + err=$? + else + download_mars "${aus_server}/update/3/${update_path}/update.xml?force=1" ${patch_type} 0 \ + "${to_build_id}" "${to_app_version}" "${to_display_version}" + err=$? + fi + if [ "$err" != "0" ]; then + echo "TEST-UNEXPECTED-FAIL: [${release} ${locale} ${patch_type}] download_mars returned non-zero exit code: ${err}" + continue + fi + else + mkdir -p updates/${update_path}/complete + mkdir -p updates/${update_path}/partial + $retry wget -q -O ${patch_type} updates/${update_path}/${patch_type}/update.xml "${aus_server}/update/3/${update_path}/update.xml?force=1" + + fi + if [ "$runmode" == "$COMPLETE" ] + then + if [ -z "$from" ] || [ -z "$to" ] + then + continue + fi + + updater_platform="" + updater_package_url=`echo "${ftp_server_from}${updater_package}" | sed "s/%locale%/${locale}/"` + updater_package_filename=`basename "$updater_package_url"` + case $updater_package_filename in + *dmg) + platform_dirname="*.app" + updater_bins="Contents/MacOS/updater.app/Contents/MacOS/updater Contents/MacOS/updater.app/Contents/MacOS/org.mozilla.updater" + updater_platform="mac" + ;; + *exe) + updater_package_url=`echo "${updater_package_url}" | sed "s/ja-JP-mac/ja/"` + platform_dirname="bin" + updater_bins="updater.exe" + updater_platform="win32" + ;; + *bz2) + updater_package_url=`echo "${updater_package_url}" | sed "s/ja-JP-mac/ja/"` + platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'` + updater_bins="updater" + updater_platform="linux" + ;; + *) + echo "Couldn't detect updater platform" + exit 1 + ;; + esac + + rm -rf updater/* + cached_download "${updater_package_filename}" "${updater_package_url}" + unpack_build "$updater_platform" updater "$updater_package_filename" "$locale" + + # Even on Windows, we want Unix-style paths for the updater, because of MSYS. + cwd=$(\ls -d $PWD/updater/$platform_dirname) + # Bug 1209376. Linux updater linked against other libraries in the installation directory + export LD_LIBRARY_PATH=$cwd + updater="null" + for updater_bin in $updater_bins; do + if [ -e "$cwd/$updater_bin" ]; then + echo "Found updater at $updater_bin" + updater="$cwd/$updater_bin" + break + fi + done + + update_to_dep=false + if [ ! -z "$override_certs" ]; then + echo "Replacing certs in updater binary" + cp "${updater}" "${updater}.orig" + case ${override_certs} in + dep) + overrides=${dep_overrides} + update_to_dep=true + ;; + nightly) + overrides=${nightly_overrides} + ;; + release) + overrides=${release_overrides} + ;; + *) + echo "Unknown override cert - skipping" + ;; + esac + python3 "${cert_replacer}" "${MY_DIR}/../mar_certs" "${updater}.orig" "${updater}" ${overrides} + else + echo "override_certs is '${override_certs}', not replacing any certificates" + fi + + if [ "$updater" == "null" ]; then + echo "Couldn't find updater binary" + continue + fi + + from_path=`echo $from | sed "s/%locale%/${locale}/"` + to_path=`echo $to | sed "s/%locale%/${locale}/"` + download_builds "${ftp_server_from}${from_path}" "${ftp_server_to}${to_path}" + err=$? + if [ "$err" != "0" ]; then + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] download_builds returned non-zero exit code: $err" + continue + fi + source_file=`basename "$from_path"` + target_file=`basename "$to_path"` + diff_file="results.diff" + if [ -e ${diff_file} ]; then + rm ${diff_file} + fi + check_updates "${platform}" "downloads/${source_file}" "downloads/${target_file}" ${locale} ${use_old_updater} ${updater} ${diff_file} ${channel} "${mar_channel_IDs}" ${update_to_dep} + err=$? + if [ "$err" == "0" ]; then + continue + elif [ "$err" == "1" ]; then + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] check_updates returned failure for $platform downloads/$source_file vs. downloads/$target_file: $err" + elif [ "$err" == "2" ]; then + echo "WARN: [$release $locale $patch_type] check_updates returned warning for $platform downloads/$source_file vs. downloads/$target_file: $err" + else + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] check_updates returned unknown error for $platform downloads/$source_file vs. downloads/$target_file: $err" + fi + + if [ -s ${diff_file} ]; then + echo "Found diffs for ${patch_type} update from ${aus_server}/update/3/${update_path}/update.xml?force=1" >> ${diff_summary_log} + cat ${diff_file} >> ${diff_summary_log} + echo "" >> ${diff_summary_log} + fi + fi + done + if [ -f update/partial.size ] && [ -f update/complete.size ]; then + partial_size=`cat update/partial.size` + complete_size=`cat update/complete.size` + if [ $partial_size -gt $complete_size ]; then + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] partial updates are larger than complete updates" + elif [ $partial_size -eq $complete_size ]; then + echo "WARN: [$release $locale $patch_type] partial updates are the same size as complete updates, this should only happen for major updates" + else + echo "SUCCESS: [$release $locale $patch_type] partial updates are smaller than complete updates, all is well in the universe" + fi + fi + done +done < $config_file + +clear_cache diff --git a/tools/update-verify/scripts/async_download.py b/tools/update-verify/scripts/async_download.py new file mode 100644 index 0000000000..efedc8295f --- /dev/null +++ b/tools/update-verify/scripts/async_download.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import asyncio +import glob +import logging +import os +import sys +import xml.etree.ElementTree as ET +from os import path + +import aiohttp + +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") +log = logging.getLogger(__name__) + +UV_CACHE_PATH = os.getenv( + "UV_CACHE_PATH", os.path.join(path.dirname(__file__), "../release/updates/cache/") +) +UV_PARALLEL_DOWNLOADS = os.getenv("UV_PARALLEL_DOWNLOADS", 20) + +FTP_SERVER_TO = os.getenv("ftp_server_to", "http://stage.mozilla.org/pub/mozilla.org") +FTP_SERVER_FROM = os.getenv( + "ftp_server_from", "http://stage.mozilla.org/pub/mozilla.org" +) +AUS_SERVER = os.getenv("aus_server", "https://aus5.mozilla.org") + + +def create_cache(): + if not os.path.isdir(UV_CACHE_PATH): + os.mkdir(UV_CACHE_PATH) + + +def remove_cache(): + """ + Removes all files in the cache folder + We don't support folders or .dot(hidden) files + By not deleting the cache directory, it allows us to use Docker tmpfs mounts, + which are the only workaround to poor mount r/w performance on MacOS + Bug Reference: + https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288 + """ + files = glob.glob(f"{UV_CACHE_PATH}/*") + for f in files: + os.remove(f) + + +def _cachepath(i, ext): + # Helper function: given an index, return a cache file path + return path.join(UV_CACHE_PATH, f"obj_{i:0>5}.{ext}") + + +async def fetch_url(url, path, connector): + """ + Fetch/download a file to a specific path + + Parameters + ---------- + url : str + URL to be fetched + + path : str + Path to save binary + + Returns + ------- + dict + Request result. If error result['error'] is True + """ + + def _result(response, error=False): + data = { + "headers": dict(response.headers), + "status": response.status, + "reason": response.reason, + "_request_info": str(response._request_info), + "url": url, + "path": path, + "error": error, + } + return data + + # Set connection timeout to 15 minutes + timeout = aiohttp.ClientTimeout(total=900) + + try: + async with aiohttp.ClientSession( + connector=connector, connector_owner=False, timeout=timeout + ) as session: + log.info(f"Retrieving {url}") + async with session.get( + url, headers={"Cache-Control": "max-stale=0"} + ) as response: + # Any response code > 299 means something went wrong + if response.status > 299: + log.info(f"Failed to download {url} with status {response.status}") + return _result(response, True) + + with open(path, "wb") as fd: + while True: + chunk = await response.content.read() + if not chunk: + break + fd.write(chunk) + result = _result(response) + log.info(f'Finished downloading {url}\n{result["headers"]}') + return result + + except ( + UnicodeDecodeError, # Data parsing + asyncio.TimeoutError, # Async timeout + aiohttp.ClientError, # aiohttp error + ) as e: + log.error("=============") + log.error(f"Error downloading {url}") + log.error(e) + log.error("=============") + return {"path": path, "url": url, "error": True} + + +async def download_multi(targets, sourceFunc): + """ + Download list of targets + + Parameters + ---------- + targets : list + List of urls to download + + sourceFunc : str + Source function name (for filename) + + Returns + ------- + tuple + List of responses (Headers) + """ + + targets = set(targets) + amount = len(targets) + + connector = aiohttp.TCPConnector( + limit=UV_PARALLEL_DOWNLOADS, # Simultaneous connections, per host + ttl_dns_cache=600, # Cache DNS for 10 mins + ) + + log.info(f"\nDownloading {amount} files ({UV_PARALLEL_DOWNLOADS} async limit)") + + # Transform targets into {url, path} objects + payloads = [ + {"url": url, "path": _cachepath(i, sourceFunc)} + for (i, url) in enumerate(targets) + ] + + downloads = [] + + fetches = [fetch_url(t["url"], t["path"], connector) for t in payloads] + + downloads.extend(await asyncio.gather(*fetches)) + connector.close() + + results = [] + # Remove file if download failed + for fetch in downloads: + # If there's an error, try to remove the file, but keep going if file not present + if fetch["error"]: + try: + os.unlink(fetch.get("path", None)) + except (TypeError, FileNotFoundError) as e: + log.info(f"Unable to cleanup error file: {e} continuing...") + continue + + results.append(fetch) + + return results + + +async def download_builds(verifyConfig): + """ + Given UpdateVerifyConfig, download and cache all necessary updater files + Include "to" and "from"/"updater_pacakge" + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + Chunked config + + Returns + ------- + list : List of file paths and urls to each updater file + """ + + updaterUrls = set() + for release in verifyConfig.releases: + ftpServerFrom = release["ftp_server_from"] + ftpServerTo = release["ftp_server_to"] + + for locale in release["locales"]: + toUri = verifyConfig.to + if toUri is not None and ftpServerTo is not None: + toUri = toUri.replace("%locale%", locale) + updaterUrls.add(f"{ftpServerTo}{toUri}") + + for reference in ("updater_package", "from"): + uri = release.get(reference, None) + if uri is None: + continue + uri = uri.replace("%locale%", locale) + # /ja-JP-mac/ locale is replaced with /ja/ for updater packages + uri = uri.replace("ja-JP-mac", "ja") + updaterUrls.add(f"{ftpServerFrom}{uri}") + + log.info(f"About to download {len(updaterUrls)} updater packages") + + updaterResults = await download_multi(list(updaterUrls), "updater.async.cache") + return updaterResults + + +def get_mar_urls_from_update(path): + """ + Given an update.xml file, return MAR URLs + + If update.xml doesn't have URLs, returns empty list + + Parameters + ---------- + path : str + Path to update.xml file + + Returns + ------- + list : List of URLs + """ + + result = [] + root = ET.parse(path).getroot() + for patch in root.findall("update/patch"): + url = patch.get("URL") + if url: + result.append(url) + return result + + +async def download_mars(updatePaths): + """ + Given list of update.xml paths, download MARs for each + + Parameters + ---------- + update_paths : list + List of paths to update.xml files + """ + + patchUrls = set() + for updatePath in updatePaths: + for url in get_mar_urls_from_update(updatePath): + patchUrls.add(url) + + log.info(f"About to download {len(patchUrls)} MAR packages") + marResults = await download_multi(list(patchUrls), "mar.async.cache") + return marResults + + +async def download_update_xml(verifyConfig): + """ + Given UpdateVerifyConfig, download and cache all necessary update.xml files + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + Chunked config + + Returns + ------- + list : List of file paths and urls to each update.xml file + """ + + xmlUrls = set() + product = verifyConfig.product + urlTemplate = ( + "{server}/update/3/{product}/{release}/{build}/{platform}/" + "{locale}/{channel}/default/default/default/update.xml?force=1" + ) + + for release in verifyConfig.releases: + for locale in release["locales"]: + xmlUrls.add( + urlTemplate.format( + server=AUS_SERVER, + product=product, + release=release["release"], + build=release["build_id"], + platform=release["platform"], + locale=locale, + channel=verifyConfig.channel, + ) + ) + + log.info(f"About to download {len(xmlUrls)} update.xml files") + xmlResults = await download_multi(list(xmlUrls), "update.xml.async.cache") + return xmlResults + + +async def _download_from_config(verifyConfig): + """ + Given an UpdateVerifyConfig object, download all necessary files to cache + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + The config - already chunked + """ + remove_cache() + create_cache() + + downloadList = [] + ################## + # Download files # + ################## + xmlFiles = await download_update_xml(verifyConfig) + downloadList.extend(xmlFiles) + downloadList += await download_mars(x["path"] for x in xmlFiles) + downloadList += await download_builds(verifyConfig) + + ##################### + # Create cache.list # + ##################### + cacheLinks = [] + + # Rename files and add to cache_links + for download in downloadList: + cacheLinks.append(download["url"]) + fileIndex = len(cacheLinks) + os.rename(download["path"], _cachepath(fileIndex, "cache")) + + cacheIndexPath = path.join(UV_CACHE_PATH, "urls.list") + with open(cacheIndexPath, "w") as cache: + cache.writelines(f"{l}\n" for l in cacheLinks) + + # Log cache + log.info("Cache index urls.list contents:") + with open(cacheIndexPath, "r") as cache: + for ln, url in enumerate(cache.readlines()): + line = url.replace("\n", "") + log.info(f"Line {ln+1}: {line}") + + return None + + +def download_from_config(verifyConfig): + """ + Given an UpdateVerifyConfig object, download all necessary files to cache + (sync function that calls the async one) + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + The config - already chunked + """ + return asyncio.run(_download_from_config(verifyConfig)) diff --git a/tools/update-verify/scripts/chunked-verify.py b/tools/update-verify/scripts/chunked-verify.py new file mode 100644 index 0000000000..8c4320d4cc --- /dev/null +++ b/tools/update-verify/scripts/chunked-verify.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import sys +from os import path +from tempfile import mkstemp + +sys.path.append(path.join(path.dirname(__file__), "../python")) +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") +log = logging.getLogger(__name__) + +from async_download import download_from_config +from mozrelease.update_verify import UpdateVerifyConfig +from util.commands import run_cmd + +UPDATE_VERIFY_COMMAND = ["bash", "verify.sh", "-c"] +UPDATE_VERIFY_DIR = path.join(path.dirname(__file__), "../release/updates") + + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser("") + + parser.set_defaults( + chunks=None, + thisChunk=None, + ) + parser.add_argument("--verify-config", required=True, dest="verifyConfig") + parser.add_argument("--verify-channel", required=True, dest="verify_channel") + parser.add_argument("--chunks", required=True, dest="chunks", type=int) + parser.add_argument("--this-chunk", required=True, dest="thisChunk", type=int) + parser.add_argument("--diff-summary", required=True, type=str) + + options = parser.parse_args() + assert options.chunks and options.thisChunk, "chunks and this-chunk are required" + assert path.isfile(options.verifyConfig), "Update verify config must exist!" + verifyConfigFile = options.verifyConfig + + fd, configFile = mkstemp() + # Needs to be opened in "bytes" mode because we perform relative seeks on it + fh = os.fdopen(fd, "wb") + try: + verifyConfig = UpdateVerifyConfig() + verifyConfig.read(path.join(UPDATE_VERIFY_DIR, verifyConfigFile)) + myVerifyConfig = verifyConfig.getChunk(options.chunks, options.thisChunk) + # override the channel if explicitly set + if options.verify_channel: + myVerifyConfig.channel = options.verify_channel + myVerifyConfig.write(fh) + fh.close() + run_cmd(["cat", configFile]) + + # Before verifying, we want to download and cache all required files + download_from_config(myVerifyConfig) + + run_cmd( + UPDATE_VERIFY_COMMAND + [configFile], + cwd=UPDATE_VERIFY_DIR, + env={"DIFF_SUMMARY_LOG": path.abspath(options.diff_summary)}, + ) + finally: + if path.exists(configFile): + os.unlink(configFile) diff --git a/tools/update-verify/scripts/chunked-verify.sh b/tools/update-verify/scripts/chunked-verify.sh new file mode 100755 index 0000000000..ad6af19080 --- /dev/null +++ b/tools/update-verify/scripts/chunked-verify.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -ex +set -o pipefail +# This ugly hack is a cross-platform (Linux/Mac/Windows+MSYS) way to get the +# absolute path to the directory containing this script +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +SCRIPTS_DIR="$MY_DIR/.." +PYTHON='./mach python' +VERIFY_CONFIG="$MOZ_FETCHES_DIR/update-verify.cfg" + +while [ "$#" -gt 0 ]; do + case $1 in + # Parse total-chunks + --total-chunks=*) chunks="${1#*=}"; shift 1;; + --total-chunks) chunks="${2}"; shift 2;; + + # Parse this-chunk + --this-chunk=*) thisChunk="${1#*=}"; shift 1;; + --this-chunk) thisChunk="${2}"; shift 2;; + + # Stop if other parameters are sent + *) echo "Unknown parameter: ${1}"; exit 1;; + esac +done + +# Validate parameters +if [ -z "${chunks}" ]; then echo "Required parameter: --total-chunks"; exit 1; fi +if [ -z "${thisChunk}" ]; then echo "Required parameter: --this-chunk"; exit 1; fi + +# release promotion +if [ -n "$CHANNEL" ]; then + EXTRA_PARAMS="--verify-channel $CHANNEL" +else + EXTRA_PARAMS="" +fi +$PYTHON $MY_DIR/chunked-verify.py --chunks $chunks --this-chunk $thisChunk \ +--verify-config $VERIFY_CONFIG --diff-summary $PWD/diff-summary.log $EXTRA_PARAMS \ +2>&1 | tee $SCRIPTS_DIR/../verify_log.txt + +print_failed_msg() +{ + echo "-------------------------" + echo "This run has failed, see the above log" + echo + return 1 +} + +print_warning_msg() +{ + echo "-------------------------" + echo "This run has warnings, see the above log" + echo + return 2 +} + +set +x + +echo "Scanning log for failures and warnings" +echo "--------------------------------------" + +# Test for a failure, note we are set -e. +# Grep returns 0 on a match and 1 on no match +# Testing for failures first is important because it's OK to to mark as failed +# when there's failures+warnings, but not OK to mark as warnings in the same +# situation. +( ! grep 'TEST-UNEXPECTED-FAIL:' $SCRIPTS_DIR/../verify_log.txt ) || print_failed_msg +( ! grep 'WARN:' $SCRIPTS_DIR/../verify_log.txt ) || print_warning_msg + +echo "-------------------------" +echo "All is well" diff --git a/tools/vcs/mach_commands.py b/tools/vcs/mach_commands.py new file mode 100644 index 0000000000..4623a23634 --- /dev/null +++ b/tools/vcs/mach_commands.py @@ -0,0 +1,242 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os +import re +import subprocess +import sys + +import mozpack.path as mozpath +from mach.decorators import Command, CommandArgument + +GITHUB_ROOT = "https://github.com/" +PR_REPOSITORIES = { + "webrender": { + "github": "servo/webrender", + "path": "gfx/wr", + "bugzilla_product": "Core", + "bugzilla_component": "Graphics: WebRender", + }, + "webgpu": { + "github": "gfx-rs/wgpu", + "path": "gfx/wgpu", + "bugzilla_product": "Core", + "bugzilla_component": "Graphics: WebGPU", + }, + "debugger": { + "github": "firefox-devtools/debugger", + "path": "devtools/client/debugger", + "bugzilla_product": "DevTools", + "bugzilla_component": "Debugger", + }, +} + + +@Command( + "import-pr", + category="misc", + description="Import a pull request from Github to the local repo.", +) +@CommandArgument("-b", "--bug-number", help="Bug number to use in the commit messages.") +@CommandArgument( + "-t", + "--bugzilla-token", + help="Bugzilla API token used to file a new bug if no bug number is provided.", +) +@CommandArgument("-r", "--reviewer", help="Reviewer nick to apply to commit messages.") +@CommandArgument( + "pull_request", + help="URL to the pull request to import (e.g. " + "https://github.com/servo/webrender/pull/3665).", +) +def import_pr( + command_context, + pull_request, + bug_number=None, + bugzilla_token=None, + reviewer=None, +): + import requests + + pr_number = None + repository = None + for r in PR_REPOSITORIES.values(): + if pull_request.startswith(GITHUB_ROOT + r["github"] + "/pull/"): + # sanitize URL, dropping anything after the PR number + pr_number = int(re.search("/pull/([0-9]+)", pull_request).group(1)) + pull_request = GITHUB_ROOT + r["github"] + "/pull/" + str(pr_number) + repository = r + break + + if repository is None: + command_context.log( + logging.ERROR, + "unrecognized_repo", + {}, + "The pull request URL was not recognized; add it to the list of " + "recognized repos in PR_REPOSITORIES in %s" % __file__, + ) + sys.exit(1) + + command_context.log( + logging.INFO, + "import_pr", + {"pr_url": pull_request}, + "Attempting to import {pr_url}", + ) + dirty = [ + f + for f in command_context.repository.get_changed_files(mode="all") + if f.startswith(repository["path"]) + ] + if dirty: + command_context.log( + logging.ERROR, + "dirty_tree", + repository, + "Local {path} tree is dirty; aborting!", + ) + sys.exit(1) + target_dir = mozpath.join( + command_context.topsrcdir, os.path.normpath(repository["path"]) + ) + + if bug_number is None: + if bugzilla_token is None: + command_context.log( + logging.WARNING, + "no_token", + {}, + "No bug number or bugzilla API token provided; bug number will not " + "be added to commit messages.", + ) + else: + bug_number = _file_bug( + command_context, bugzilla_token, repository, pr_number + ) + elif bugzilla_token is not None: + command_context.log( + logging.WARNING, + "too_much_bug", + {}, + "Providing a bugzilla token is unnecessary when a bug number is provided. " + "Using bug number; ignoring token.", + ) + + pr_patch = requests.get(pull_request + ".patch") + pr_patch.raise_for_status() + for patch in _split_patches(pr_patch.content, bug_number, pull_request, reviewer): + command_context.log( + logging.INFO, + "commit_msg", + patch, + "Processing commit [{commit_summary}] by [{author}] at [{date}]", + ) + patch_cmd = subprocess.Popen( + ["patch", "-p1", "-s"], stdin=subprocess.PIPE, cwd=target_dir + ) + patch_cmd.stdin.write(patch["diff"].encode("utf-8")) + patch_cmd.stdin.close() + patch_cmd.wait() + if patch_cmd.returncode != 0: + command_context.log( + logging.ERROR, + "commit_fail", + {}, + 'Error applying diff from commit via "patch -p1 -s". Aborting...', + ) + sys.exit(patch_cmd.returncode) + command_context.repository.commit( + patch["commit_msg"], patch["author"], patch["date"], [target_dir] + ) + command_context.log(logging.INFO, "commit_pass", {}, "Committed successfully.") + + +def _file_bug(command_context, token, repo, pr_number): + import requests + + bug = requests.post( + "https://bugzilla.mozilla.org/rest/bug?api_key=%s" % token, + json={ + "product": repo["bugzilla_product"], + "component": repo["bugzilla_component"], + "summary": "Land %s#%s in mozilla-central" % (repo["github"], pr_number), + "version": "unspecified", + }, + ) + bug.raise_for_status() + command_context.log(logging.DEBUG, "new_bug", {}, bug.content) + bugnumber = json.loads(bug.content)["id"] + command_context.log( + logging.INFO, "new_bug", {"bugnumber": bugnumber}, "Filed bug {bugnumber}" + ) + return bugnumber + + +def _split_patches(patchfile, bug_number, pull_request, reviewer): + INITIAL = 0 + HEADERS = 1 + STAT_AND_DIFF = 2 + + patch = b"" + state = INITIAL + for line in patchfile.splitlines(): + if state == INITIAL: + if line.startswith(b"From "): + state = HEADERS + elif state == HEADERS: + patch += line + b"\n" + if line == b"---": + state = STAT_AND_DIFF + elif state == STAT_AND_DIFF: + if line.startswith(b"From "): + yield _parse_patch(patch, bug_number, pull_request, reviewer) + patch = b"" + state = HEADERS + else: + patch += line + b"\n" + if len(patch) > 0: + yield _parse_patch(patch, bug_number, pull_request, reviewer) + return + + +def _parse_patch(patch, bug_number, pull_request, reviewer): + import email + from email import header, policy + + parse_policy = policy.compat32.clone(max_line_length=None) + parsed_mail = email.message_from_bytes(patch, policy=parse_policy) + + def header_as_unicode(key): + decoded = header.decode_header(parsed_mail[key]) + return str(header.make_header(decoded)) + + author = header_as_unicode("From") + date = header_as_unicode("Date") + commit_summary = header_as_unicode("Subject") + email_body = parsed_mail.get_payload(decode=True).decode("utf-8") + (commit_body, diff) = ("\n" + email_body).rsplit("\n---\n", 1) + + bug_prefix = "" + if bug_number is not None: + bug_prefix = "Bug %s - " % bug_number + commit_summary = re.sub(r"^\[PATCH[0-9 /]*\] ", bug_prefix, commit_summary) + if reviewer is not None: + commit_summary += " r=" + reviewer + + commit_msg = commit_summary + "\n" + if len(commit_body) > 0: + commit_msg += commit_body + "\n" + commit_msg += "\n[import_pr] From " + pull_request + "\n" + + patch_obj = { + "author": author, + "date": date, + "commit_summary": commit_summary, + "commit_msg": commit_msg, + "diff": diff, + } + return patch_obj |