From 9e3c08db40b8916968b9f30096c7be3f00ce9647 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 21 Apr 2024 13:44:51 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- tools/bloatview/bloatdiff.pl | 372 + tools/bloatview/bloattable.pl | 590 ++ tools/browsertime/README.md | 15 + tools/browsertime/mach_commands.py | 689 ++ tools/browsertime/moz.build | 8 + tools/browsertime/package-lock.json | 5715 ++++++++++++++ tools/browsertime/package.json | 1 + tools/clang-tidy/config.yaml | 327 + .../clang-tidy/test/bugprone-argument-comment.cpp | 6 + .../clang-tidy/test/bugprone-argument-comment.json | 13 + .../test/bugprone-assert-side-effect.cpp | 8 + .../test/bugprone-assert-side-effect.json | 8 + .../bugprone-bool-pointer-implicit-conversion.cpp | 20 + .../bugprone-bool-pointer-implicit-conversion.json | 8 + .../bugprone-forward-declaration-namespace.cpp | 3 + .../bugprone-forward-declaration-namespace.json | 8 + .../test/bugprone-incorrect-roundings.cpp | 7 + .../test/bugprone-incorrect-roundings.json | 8 + .../clang-tidy/test/bugprone-integer-division.cpp | 5 + .../clang-tidy/test/bugprone-integer-division.json | 8 + .../clang-tidy/test/bugprone-macro-parentheses.cpp | 5 + .../test/bugprone-macro-parentheses.json | 28 + .../test/bugprone-macro-repeated-side-effects.cpp | 28 + .../test/bugprone-macro-repeated-side-effects.json | 8 + .../test/bugprone-misplaced-widening-cast.cpp | 3 + .../test/bugprone-misplaced-widening-cast.json | 8 + .../test/bugprone-move-forwarding-reference.cpp | 25 + .../test/bugprone-move-forwarding-reference.json | 8 + .../test/bugprone-multiple-statement-macro.cpp | 10 + .../test/bugprone-multiple-statement-macro.json | 7 + .../clang-tidy/test/bugprone-sizeof-expression.cpp | 3 + .../test/bugprone-sizeof-expression.json | 8 + .../test/bugprone-string-constructor.cpp | 17 + .../test/bugprone-string-constructor.json | 18 + .../test/bugprone-string-integer-assignment.cpp | 28 + .../test/bugprone-string-integer-assignment.json | 13 + .../test/bugprone-suspicious-memset-usage.cpp | 22 + .../test/bugprone-suspicious-memset-usage.json | 18 + .../test/bugprone-suspicious-missing-comma.cpp | 9 + .../test/bugprone-suspicious-missing-comma.json | 8 + .../test/bugprone-suspicious-semicolon.cpp | 8 + .../test/bugprone-suspicious-semicolon.json | 8 + .../test/bugprone-suspicious-string-compare.cpp | 8 + .../test/bugprone-suspicious-string-compare.json | 8 + .../clang-tidy/test/bugprone-swapped-arguments.cpp | 26 + .../test/bugprone-swapped-arguments.json | 23 + .../test/bugprone-too-small-loop-variable.cpp | 4 + .../test/bugprone-too-small-loop-variable.json | 8 + tools/clang-tidy/test/bugprone-unused-raii.cpp | 25 + tools/clang-tidy/test/bugprone-unused-raii.json | 8 + tools/clang-tidy/test/bugprone-use-after-move.cpp | 62 + tools/clang-tidy/test/bugprone-use-after-move.json | 4 + .../test/clang-analyzer-core.CallAndMessage.cpp | 10 + .../test/clang-analyzer-core.CallAndMessage.json | 8 + .../test/clang-analyzer-core.DivideZero.cpp | 4 + .../test/clang-analyzer-core.DivideZero.json | 4 + .../clang-analyzer-core.NonNullParamChecker.cpp | 6 + .../clang-analyzer-core.NonNullParamChecker.json | 8 + .../test/clang-analyzer-core.NullDereference.cpp | 9 + .../test/clang-analyzer-core.NullDereference.json | 8 + ...analyzer-core.UndefinedBinaryOperatorResult.cpp | 4 + ...nalyzer-core.UndefinedBinaryOperatorResult.json | 8 + .../clang-analyzer-core.uninitialized.Assign.cpp | 4 + .../clang-analyzer-core.uninitialized.Assign.json | 8 + .../clang-analyzer-core.uninitialized.Branch.cpp | 6 + .../clang-analyzer-core.uninitialized.Branch.json | 8 + .../test/clang-analyzer-cplusplus.Move.cpp | 10 + .../test/clang-analyzer-cplusplus.Move.json | 8 + .../test/clang-analyzer-cplusplus.NewDelete.cpp | 50 + .../test/clang-analyzer-cplusplus.NewDelete.json | 23 + .../clang-analyzer-cplusplus.NewDeleteLeaks.cpp | 6 + .../clang-analyzer-cplusplus.NewDeleteLeaks.json | 8 + .../test/clang-analyzer-deadcode.DeadStores.cpp | 6 + .../test/clang-analyzer-deadcode.DeadStores.json | 8 + .../clang-analyzer-optin.performance.Padding.cpp | 6 + .../clang-analyzer-optin.performance.Padding.json | 8 + .../clang-analyzer-security.FloatLoopCounter.cpp | 3 + .../clang-analyzer-security.FloatLoopCounter.json | 8 + ...alyzer-security.insecureAPI.UncheckedReturn.cpp | 5 + ...lyzer-security.insecureAPI.UncheckedReturn.json | 8 + .../clang-analyzer-security.insecureAPI.bcmp.cpp | 5 + .../clang-analyzer-security.insecureAPI.bcmp.json | 8 + .../clang-analyzer-security.insecureAPI.bcopy.cpp | 5 + .../clang-analyzer-security.insecureAPI.bcopy.json | 8 + .../clang-analyzer-security.insecureAPI.bzero.cpp | 5 + .../clang-analyzer-security.insecureAPI.bzero.json | 8 + .../clang-analyzer-security.insecureAPI.getpw.cpp | 6 + .../clang-analyzer-security.insecureAPI.getpw.json | 8 + .../clang-analyzer-security.insecureAPI.gets.cpp | 6 + .../clang-analyzer-security.insecureAPI.gets.json | 1 + ...clang-analyzer-security.insecureAPI.mkstemp.cpp | 5 + ...lang-analyzer-security.insecureAPI.mkstemp.json | 8 + .../clang-analyzer-security.insecureAPI.mktemp.cpp | 5 + ...clang-analyzer-security.insecureAPI.mktemp.json | 8 + .../clang-analyzer-security.insecureAPI.rand.cpp | 4 + .../clang-analyzer-security.insecureAPI.rand.json | 1 + .../clang-analyzer-security.insecureAPI.strcpy.cpp | 7 + ...clang-analyzer-security.insecureAPI.strcpy.json | 1 + .../clang-analyzer-security.insecureAPI.vfork.cpp | 5 + .../clang-analyzer-security.insecureAPI.vfork.json | 8 + .../clang-tidy/test/clang-analyzer-unix.Malloc.cpp | 37 + .../test/clang-analyzer-unix.Malloc.json | 20 + .../clang-analyzer-unix.cstring.BadSizeArg.cpp | 9 + .../clang-analyzer-unix.cstring.BadSizeArg.json | 8 + .../test/clang-analyzer-unix.cstring.NullArg.cpp | 14 + .../test/clang-analyzer-unix.cstring.NullArg.json | 8 + .../cppcoreguidelines-narrowing-conversions.cpp | 4 + .../cppcoreguidelines-narrowing-conversions.json | 8 + .../cppcoreguidelines-pro-type-member-init.cpp | 7 + .../cppcoreguidelines-pro-type-member-init.json | 8 + .../clang-tidy/test/misc-non-copyable-objects.cpp | 5 + .../clang-tidy/test/misc-non-copyable-objects.json | 8 + .../clang-tidy/test/misc-redundant-expression.cpp | 3 + .../clang-tidy/test/misc-redundant-expression.json | 8 + tools/clang-tidy/test/misc-unused-alias-decls.cpp | 18 + tools/clang-tidy/test/misc-unused-alias-decls.json | 13 + tools/clang-tidy/test/misc-unused-using-decls.cpp | 4 + tools/clang-tidy/test/misc-unused-using-decls.json | 4 + tools/clang-tidy/test/modernize-avoid-bind.cpp | 6 + tools/clang-tidy/test/modernize-avoid-bind.json | 4 + .../test/modernize-concat-nested-namespaces.cpp | 5 + .../test/modernize-concat-nested-namespaces.json | 8 + .../test/modernize-deprecated-ios-base-aliases.cpp | 18 + .../modernize-deprecated-ios-base-aliases.json | 8 + tools/clang-tidy/test/modernize-loop-convert.cpp | 7 + tools/clang-tidy/test/modernize-loop-convert.json | 4 + .../test/modernize-raw-string-literal.cpp | 1 + .../test/modernize-raw-string-literal.json | 8 + .../test/modernize-redundant-void-arg.cpp | 5 + .../test/modernize-redundant-void-arg.json | 1 + tools/clang-tidy/test/modernize-shrink-to-fit.cpp | 10 + tools/clang-tidy/test/modernize-shrink-to-fit.json | 13 + tools/clang-tidy/test/modernize-use-auto.cpp | 11 + tools/clang-tidy/test/modernize-use-auto.json | 1 + .../test/modernize-use-bool-literals.cpp | 5 + .../test/modernize-use-bool-literals.json | 23 + .../test/modernize-use-equals-default.cpp | 6 + .../test/modernize-use-equals-default.json | 13 + .../test/modernize-use-equals-delete.cpp | 7 + .../test/modernize-use-equals-delete.json | 23 + tools/clang-tidy/test/modernize-use-nullptr.cpp | 5 + tools/clang-tidy/test/modernize-use-nullptr.json | 1 + tools/clang-tidy/test/modernize-use-override.cpp | 8 + tools/clang-tidy/test/modernize-use-override.json | 1 + tools/clang-tidy/test/modernize-use-using.cpp | 5 + tools/clang-tidy/test/modernize-use-using.json | 5 + .../test/performance-faster-string-find.cpp | 6 + .../test/performance-faster-string-find.json | 8 + .../clang-tidy/test/performance-for-range-copy.cpp | 30 + .../test/performance-for-range-copy.json | 8 + .../performance-implicit-conversion-in-loop.cpp | 43 + .../performance-implicit-conversion-in-loop.json | 8 + .../test/performance-inefficient-algorithm.cpp | 30 + .../test/performance-inefficient-algorithm.json | 8 + ...erformance-inefficient-string-concatenation.cpp | 13 + ...rformance-inefficient-string-concatenation.json | 8 + .../performance-inefficient-vector-operation.cpp | 10 + .../performance-inefficient-vector-operation.json | 8 + .../clang-tidy/test/performance-move-const-arg.cpp | 33 + .../test/performance-move-const-arg.json | 8 + .../test/performance-move-constructor-init.cpp | 11 + .../test/performance-move-constructor-init.json | 8 + .../test/performance-noexcept-move-constructor.cpp | 4 + .../performance-noexcept-move-constructor.json | 13 + .../test/performance-type-promotion-in-math-fn.cpp | 7 + .../performance-type-promotion-in-math-fn.json | 8 + ...performance-unnecessary-copy-initialization.cpp | 7 + ...erformance-unnecessary-copy-initialization.json | 8 + .../test/performance-unnecessary-value-param.cpp | 4 + .../test/performance-unnecessary-value-param.json | 8 + .../test/readability-braces-around-statements.cpp | 38 + .../test/readability-braces-around-statements.json | 13 + .../test/readability-const-return-type.cpp | 5 + .../test/readability-const-return-type.json | 8 + .../test/readability-container-size-empty.cpp | 7 + .../test/readability-container-size-empty.json | 8 + .../test/readability-delete-null-pointer.cpp | 6 + .../test/readability-delete-null-pointer.json | 8 + .../test/readability-else-after-return.cpp | 10 + .../test/readability-else-after-return.json | 8 + .../test/readability-implicit-bool-conversion.cpp | 55 + .../test/readability-implicit-bool-conversion.json | 102 + ...ity-inconsistent-declaration-parameter-name.cpp | 6 + ...ty-inconsistent-declaration-parameter-name.json | 8 + .../test/readability-isolate-declaration.cpp | 7 + .../test/readability-isolate-declaration.json | 8 + .../clang-tidy/test/readability-magic-numbers.cpp | 4 + .../clang-tidy/test/readability-magic-numbers.json | 8 + .../test/readability-misleading-indentation.cpp | 20 + .../test/readability-misleading-indentation.json | 8 + .../test/readability-non-const-parameter.cpp | 5 + .../test/readability-non-const-parameter.json | 8 + .../clang-tidy/test/readability-qualified-auto.cpp | 6 + .../test/readability-qualified-auto.json | 8 + .../test/readability-redundant-control-flow.cpp | 5 + .../test/readability-redundant-control-flow.json | 8 + .../test/readability-redundant-preprocessor.cpp | 5 + .../test/readability-redundant-preprocessor.json | 8 + .../test/readability-redundant-smartptr-get.cpp | 19 + .../test/readability-redundant-smartptr-get.json | 8 + .../test/readability-redundant-string-cstr.cpp | 7 + .../test/readability-redundant-string-cstr.json | 4 + .../test/readability-redundant-string-init.cpp | 5 + .../test/readability-redundant-string-init.json | 8 + .../test/readability-simplify-boolean-expr.cpp | 2 + .../test/readability-simplify-boolean-expr.json | 8 + ...eadability-static-accessed-through-instance.cpp | 11 + ...adability-static-accessed-through-instance.json | 8 + .../test/readability-uniqueptr-delete-release.cpp | 6 + .../test/readability-uniqueptr-delete-release.json | 8 + tools/clang-tidy/test/structures.h | 108 + tools/code-coverage/CodeCoverageHandler.cpp | 158 + tools/code-coverage/CodeCoverageHandler.h | 38 + tools/code-coverage/PerTestCoverageUtils.sys.mjs | 90 + tools/code-coverage/components.conf | 14 + tools/code-coverage/docs/index.rst | 209 + tools/code-coverage/moz.build | 39 + tools/code-coverage/nsCodeCoverage.cpp | 96 + tools/code-coverage/nsCodeCoverage.h | 22 + tools/code-coverage/nsICodeCoverage.idl | 24 + tools/code-coverage/tests/mochitest/mochitest.ini | 3 + .../mochitest/test_coverage_specialpowers.html | 38 + tools/code-coverage/tests/xpcshell/head.js | 100 + tools/code-coverage/tests/xpcshell/support.js | 5 + tools/code-coverage/tests/xpcshell/test_basic.js | 115 + .../tests/xpcshell/test_basic_child_and_parent.js | 120 + tools/code-coverage/tests/xpcshell/xpcshell.ini | 7 + tools/compare-locales/mach_commands.py | 409 + tools/compare-locales/requirements.in | 2 + tools/crashreporter/injector/app.mozbuild | 14 + tools/crashreporter/injector/injector.cc | 40 + tools/crashreporter/injector/moz.build | 36 + tools/crashreporter/injector/moz.configure | 3 + .../system-symbols/mac/PackageSymbolDumper.py | 392 + .../system-symbols/mac/get_update_packages.py | 154 + .../system-symbols/mac/list-packages.py | 59 + tools/crashreporter/system-symbols/mac/run.sh | 59 + .../system-symbols/mac/scrapesymbols/__init__.py | 0 .../mac/scrapesymbols/gathersymbols.py | 212 + tools/crashreporter/system-symbols/win/LICENSE | 202 + .../system-symbols/win/known-microsoft-symbols.txt | 17 + tools/crashreporter/system-symbols/win/run.sh | 11 + .../system-symbols/win/scrape-report.py | 75 + .../crashreporter/system-symbols/win/skiplist.txt | 0 .../system-symbols/win/symsrv-fetch.py | 526 ++ tools/esmify/import-to-import_esmodule.js | 476 ++ tools/esmify/is-esmified.js | 81 + tools/esmify/mach_commands.py | 910 +++ tools/esmify/map.json | 1154 +++ tools/esmify/package-lock.json | 7895 ++++++++++++++++++++ tools/esmify/package.json | 9 + tools/esmify/static-import.js | 147 + tools/esmify/use-import-export-declarations.js | 208 + tools/esmify/utils.js | 165 + tools/fuzzing/common/FuzzingMutate.cpp | 32 + tools/fuzzing/common/FuzzingMutate.h | 24 + tools/fuzzing/common/FuzzingTraits.cpp | 41 + tools/fuzzing/common/FuzzingTraits.h | 117 + tools/fuzzing/common/moz.build | 11 + tools/fuzzing/docs/fuzzing_interface.rst | 496 ++ tools/fuzzing/docs/index.rst | 438 ++ tools/fuzzing/interface/FuzzingInterface.cpp | 30 + tools/fuzzing/interface/FuzzingInterface.h | 115 + tools/fuzzing/interface/FuzzingInterfaceStream.cpp | 54 + tools/fuzzing/interface/FuzzingInterfaceStream.h | 90 + tools/fuzzing/interface/harness/FuzzerRunner.cpp | 91 + tools/fuzzing/interface/harness/FuzzerRunner.h | 24 + .../fuzzing/interface/harness/FuzzerTestHarness.h | 256 + tools/fuzzing/interface/harness/moz.build | 16 + tools/fuzzing/interface/moz.build | 32 + tools/fuzzing/ipc/IPCFuzzController.cpp | 848 +++ tools/fuzzing/ipc/IPCFuzzController.h | 193 + tools/fuzzing/ipc/moz.build | 27 + tools/fuzzing/libfuzzer-config.mozbuild | 13 + tools/fuzzing/libfuzzer-flags.mozbuild | 11 + tools/fuzzing/libfuzzer/FuzzerBuiltins.h | 35 + tools/fuzzing/libfuzzer/FuzzerBuiltinsMsvc.h | 72 + tools/fuzzing/libfuzzer/FuzzerCommand.h | 178 + tools/fuzzing/libfuzzer/FuzzerCorpus.h | 533 ++ tools/fuzzing/libfuzzer/FuzzerCrossOver.cpp | 51 + tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp | 295 + tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h | 135 + tools/fuzzing/libfuzzer/FuzzerDefs.h | 75 + tools/fuzzing/libfuzzer/FuzzerDictionary.h | 118 + tools/fuzzing/libfuzzer/FuzzerDriver.cpp | 888 +++ tools/fuzzing/libfuzzer/FuzzerExtFunctions.def | 50 + tools/fuzzing/libfuzzer/FuzzerExtFunctions.h | 34 + .../fuzzing/libfuzzer/FuzzerExtFunctionsDlsym.cpp | 51 + tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp | 54 + .../libfuzzer/FuzzerExtFunctionsWindows.cpp | 82 + tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp | 42 + tools/fuzzing/libfuzzer/FuzzerFlags.def | 169 + tools/fuzzing/libfuzzer/FuzzerFork.cpp | 427 ++ tools/fuzzing/libfuzzer/FuzzerFork.h | 24 + tools/fuzzing/libfuzzer/FuzzerIO.cpp | 165 + tools/fuzzing/libfuzzer/FuzzerIO.h | 106 + tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp | 181 + tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp | 417 ++ tools/fuzzing/libfuzzer/FuzzerInterceptors.cpp | 235 + tools/fuzzing/libfuzzer/FuzzerInterface.h | 79 + tools/fuzzing/libfuzzer/FuzzerInternal.h | 175 + tools/fuzzing/libfuzzer/FuzzerLoop.cpp | 901 +++ tools/fuzzing/libfuzzer/FuzzerMain.cpp | 21 + tools/fuzzing/libfuzzer/FuzzerMerge.cpp | 419 ++ tools/fuzzing/libfuzzer/FuzzerMerge.h | 87 + tools/fuzzing/libfuzzer/FuzzerMutate.cpp | 562 ++ tools/fuzzing/libfuzzer/FuzzerMutate.h | 156 + tools/fuzzing/libfuzzer/FuzzerOptions.h | 85 + tools/fuzzing/libfuzzer/FuzzerPlatform.h | 163 + tools/fuzzing/libfuzzer/FuzzerRandom.h | 38 + tools/fuzzing/libfuzzer/FuzzerSHA1.cpp | 223 + tools/fuzzing/libfuzzer/FuzzerSHA1.h | 32 + tools/fuzzing/libfuzzer/FuzzerTracePC.cpp | 657 ++ tools/fuzzing/libfuzzer/FuzzerTracePC.h | 289 + tools/fuzzing/libfuzzer/FuzzerUtil.cpp | 236 + tools/fuzzing/libfuzzer/FuzzerUtil.h | 111 + tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp | 170 + tools/fuzzing/libfuzzer/FuzzerUtilFuchsia.cpp | 565 ++ tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp | 41 + tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp | 185 + tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp | 221 + tools/fuzzing/libfuzzer/FuzzerValueBitMap.h | 73 + tools/fuzzing/libfuzzer/LICENSE.TXT | 68 + tools/fuzzing/libfuzzer/clone_libfuzzer.sh | 36 + tools/fuzzing/libfuzzer/moz.build | 55 + .../fuzzing/libfuzzer/patches/10-ef-runtime.patch | 31 + .../fuzzing/libfuzzer/patches/11-callback-rv.patch | 132 + .../libfuzzer/patches/12-custom-mutator-fail.patch | 53 + .../libfuzzer/patches/13-unused-write.patch | 88 + .../libfuzzer/patches/14-explicit-allocator.patch | 30 + .../libfuzzer/patches/15-return-to-exit.patch | 968 +++ .../messagemanager/MessageManagerFuzzer.cpp | 327 + .../fuzzing/messagemanager/MessageManagerFuzzer.h | 63 + tools/fuzzing/messagemanager/moz.build | 11 + tools/fuzzing/moz.build | 33 + tools/fuzzing/nyx/Nyx.cpp | 206 + tools/fuzzing/nyx/Nyx.h | 54 + tools/fuzzing/nyx/NyxWrapper.h | 33 + tools/fuzzing/nyx/moz.build | 16 + tools/fuzzing/registry/FuzzerRegistry.cpp | 26 + tools/fuzzing/registry/FuzzerRegistry.h | 44 + tools/fuzzing/registry/moz.build | 20 + tools/fuzzing/rust/Cargo.toml | 11 + tools/fuzzing/rust/RustFuzzingTargets.cpp | 15 + tools/fuzzing/rust/RustFuzzingTargets.h | 26 + tools/fuzzing/rust/moz.build | 16 + tools/fuzzing/rust/src/lib.rs | 341 + tools/fuzzing/shmem/SharedMemoryFuzzer.cpp | 122 + tools/fuzzing/shmem/SharedMemoryFuzzer.h | 38 + tools/fuzzing/shmem/moz.build | 11 + tools/fuzzing/smoke/grizzly_requirements.in | 2 + tools/fuzzing/smoke/grizzly_requirements.txt | 127 + tools/fuzzing/smoke/js.py | 26 + tools/fuzzing/smoke/python.ini | 5 + tools/fuzzing/smoke/smoke.py | 71 + tools/fuzzing/smoke/test_grizzly.py | 42 + tools/fuzzing/smoke/tests.py | 34 + tools/github-sync/converter.py | 481 ++ tools/github-sync/read-json.py | 42 + tools/github-sync/readme.md | 106 + tools/github-sync/sync-to-github.sh | 159 + tools/jprof/README.html | 330 + tools/jprof/bfd.cpp | 221 + tools/jprof/coff.cpp | 93 + tools/jprof/elf.cpp | 128 + tools/jprof/intcnt.cpp | 69 + tools/jprof/intcnt.h | 39 + tools/jprof/jprofsig | 46 + tools/jprof/leaky.cpp | 827 ++ tools/jprof/leaky.h | 124 + tools/jprof/moz.build | 28 + tools/jprof/split-profile.py | 156 + tools/jprof/strset.cpp | 37 + tools/jprof/strset.h | 19 + tools/jprof/stub/Makefile.in | 8 + tools/jprof/stub/config.h | 18 + tools/jprof/stub/jprof.h | 17 + tools/jprof/stub/libmalloc.cpp | 740 ++ tools/jprof/stub/libmalloc.h | 45 + tools/jprof/stub/moz.build | 17 + tools/leak-gauge/leak-gauge.html | 320 + tools/leak-gauge/leak-gauge.pl | 239 + tools/lint/android-api-lint.yml | 15 + tools/lint/android-checkstyle.yml | 15 + tools/lint/android-format.yml | 15 + tools/lint/android-javadoc.yml | 15 + tools/lint/android-lint.yml | 15 + tools/lint/android-test.yml | 15 + tools/lint/android/__init__.py | 0 tools/lint/android/lints.py | 421 ++ tools/lint/black.yml | 18 + tools/lint/clang-format.yml | 12 + tools/lint/clang-format/__init__.py | 237 + tools/lint/clippy.yml | 107 + tools/lint/clippy/__init__.py | 113 + tools/lint/codespell.yml | 95 + tools/lint/cpp/__init__.py | 3 + tools/lint/cpp/mingw-capitalization.py | 37 + tools/lint/cpp/mingw-headers.txt | 1452 ++++ tools/lint/eslint.yml | 32 + tools/lint/eslint/.eslintrc.js | 30 + tools/lint/eslint/__init__.py | 280 + tools/lint/eslint/eslint-plugin-mozilla/.npmignore | 8 + tools/lint/eslint/eslint-plugin-mozilla/LICENSE | 363 + tools/lint/eslint/eslint-plugin-mozilla/README.md | 56 + .../eslint-plugin-mozilla/lib/configs/.eslintrc.js | 8 + .../lib/configs/browser-test.js | 91 + .../lib/configs/chrome-test.js | 61 + .../lib/configs/mochitest-test.js | 59 + .../lib/configs/recommended.js | 356 + .../lib/configs/require-jsdoc.js | 32 + .../lib/configs/valid-jsdoc.js | 25 + .../lib/configs/xpcshell-test.js | 50 + .../lib/environments/browser-window.js | 119 + .../lib/environments/chrome-script.js | 28 + .../lib/environments/chrome-worker.js | 25 + .../lib/environments/frame-script.js | 39 + .../eslint-plugin-mozilla/lib/environments/jsm.js | 31 + .../lib/environments/privileged.js | 803 ++ .../lib/environments/process-script.js | 38 + .../lib/environments/remote-page.js | 43 + .../lib/environments/simpletest.js | 35 + .../eslint-plugin-mozilla/lib/environments/sjs.js | 30 + .../lib/environments/special-powers-sandbox.js | 46 + .../lib/environments/specific.js | 31 + .../lib/environments/utils.js | 62 + .../lib/environments/xpcshell.js | 59 + .../eslint/eslint-plugin-mozilla/lib/globals.js | 665 ++ .../eslint/eslint-plugin-mozilla/lib/helpers.js | 805 ++ .../lint/eslint/eslint-plugin-mozilla/lib/index.js | 96 + .../lib/rules/avoid-Date-timing.js | 61 + .../lib/rules/avoid-removeChild.js | 70 + .../lib/rules/balanced-listeners.js | 149 + .../lib/rules/balanced-observers.js | 121 + .../lib/rules/consistent-if-bracing.js | 54 + .../lib/rules/import-browser-window-globals.js | 50 + .../lib/rules/import-content-task-globals.js | 73 + .../lib/rules/import-globals.js | 21 + .../lib/rules/import-headjs-globals.js | 51 + .../lib/rules/lazy-getter-object-name.js | 48 + .../lib/rules/mark-exported-symbols-as-used.js | 90 + .../lib/rules/mark-test-function-used.js | 44 + .../eslint-plugin-mozilla/lib/rules/no-aArgs.js | 57 + .../lib/rules/no-addtask-setup.js | 57 + .../lib/rules/no-arbitrary-setTimeout.js | 65 + .../rules/no-compare-against-boolean-literals.js | 40 + .../lib/rules/no-cu-reportError.js | 130 + .../lib/rules/no-define-cc-etc.js | 57 + .../lib/rules/no-redeclare-with-import-autofix.js | 160 + .../lib/rules/no-throw-cr-literal.js | 101 + .../lib/rules/no-useless-parameters.js | 156 + .../lib/rules/no-useless-removeEventListener.js | 69 + .../lib/rules/no-useless-run-test.js | 76 + .../lib/rules/prefer-boolean-length-check.js | 129 + .../lib/rules/prefer-formatValues.js | 97 + .../lib/rules/reject-addtask-only.js | 48 + .../lib/rules/reject-chromeutils-import-params.js | 62 + .../rules/reject-eager-module-in-lazy-getter.js | 103 + .../lib/rules/reject-global-this.js | 40 + .../lib/rules/reject-globalThis-modification.js | 70 + .../reject-import-system-module-from-non-system.js | 36 + .../lib/rules/reject-importGlobalProperties.js | 94 + .../lib/rules/reject-lazy-imports-into-globals.js | 76 + .../lib/rules/reject-mixing-eager-and-lazy.js | 153 + .../lib/rules/reject-multiple-getters-calls.js | 81 + .../lib/rules/reject-relative-requires.js | 39 + .../lib/rules/reject-scriptableunicodeconverter.js | 41 + .../lib/rules/reject-some-requires.js | 41 + .../lib/rules/reject-top-level-await.js | 45 + .../lib/rules/rejects-requires-await.js | 47 + .../eslint-plugin-mozilla/lib/rules/use-cc-etc.js | 52 + .../lib/rules/use-chromeutils-generateqi.js | 105 + .../lib/rules/use-chromeutils-import.js | 78 + .../lib/rules/use-default-preference-values.js | 52 + .../lib/rules/use-includes-instead-of-indexOf.js | 50 + .../lib/rules/use-isInstance.js | 154 + .../lib/rules/use-ownerGlobal.js | 40 + .../lib/rules/use-returnValue.js | 41 + .../lib/rules/use-services.js | 104 + .../lib/rules/use-static-import.js | 87 + .../lib/rules/valid-ci-uses.js | 167 + .../eslint-plugin-mozilla/lib/rules/valid-lazy.js | 221 + .../lib/rules/valid-services-property.js | 126 + .../lib/rules/valid-services.js | 62 + .../lib/rules/var-only-at-top-level.js | 39 + .../eslint/eslint-plugin-mozilla/lib/services.json | 62 + .../lint/eslint/eslint-plugin-mozilla/manifest.tt | 10 + .../eslint/eslint-plugin-mozilla/package-lock.json | 6924 +++++++++++++++++ .../lint/eslint/eslint-plugin-mozilla/package.json | 53 + .../reporters/mozilla-format.js | 57 + .../eslint-plugin-mozilla/scripts/createExports.js | 77 + .../tests/avoid-Date-timing.js | 34 + .../tests/avoid-removeChild.js | 37 + .../tests/balanced-listeners.js | 100 + .../tests/balanced-observers.js | 80 + .../tests/consistent-if-bracing.js | 39 + .../eslint/eslint-plugin-mozilla/tests/globals.js | 161 + .../tests/lazy-getter-object-name.js | 51 + .../tests/mark-exported-symbols-as-used.js | 45 + .../tests/no-addtask-setup.js | 69 + .../tests/no-arbitrary-setTimeout.js | 39 + .../tests/no-compare-against-boolean-literals.js | 59 + .../tests/no-cu-reportError.js | 100 + .../tests/no-define-cc-etc.js | 63 + .../tests/no-redeclare-with-import-autofix.js | 77 + .../tests/no-throw-cr-literal.js | 62 + .../tests/no-useless-parameters.js | 175 + .../tests/no-useless-removeEventListener.js | 96 + .../tests/no-useless-run-test.js | 126 + .../tests/prefer-boolean-length-check.js | 96 + .../tests/prefer-formatValues.js | 70 + .../tests/reject-addtask-only.js | 56 + .../tests/reject-chromeutils-import-params.js | 67 + .../tests/reject-eager-module-in-lazy-getter.js | 80 + .../tests/reject-global-this.js | 49 + .../tests/reject-globalThis-modification.js | 90 + .../reject-import-system-module-from-non-system.js | 33 + .../tests/reject-importGlobalProperties.js | 87 + .../tests/reject-lazy-imports-into-globals.js | 56 + .../tests/reject-mixing-eager-and-lazy.js | 124 + .../tests/reject-multiple-getters-calls.js | 60 + .../tests/reject-relative-requires.js | 57 + .../tests/reject-scriptableunicodeconverter.js | 33 + .../tests/reject-some-requires.js | 49 + .../tests/reject-top-level-await.js | 36 + .../tests/rejects-requires-await.js | 32 + .../eslint-plugin-mozilla/tests/use-cc-etc.js | 60 + .../tests/use-chromeutils-generateqi.js | 81 + .../tests/use-chromeutils-import.js | 65 + .../tests/use-default-preference-values.js | 41 + .../tests/use-includes-instead-of-indexOf.js | 38 + .../eslint-plugin-mozilla/tests/use-isInstance.js | 130 + .../eslint-plugin-mozilla/tests/use-ownerGlobal.js | 35 + .../eslint-plugin-mozilla/tests/use-returnValue.js | 39 + .../eslint-plugin-mozilla/tests/use-services.js | 60 + .../tests/use-static-import.js | 80 + .../eslint-plugin-mozilla/tests/valid-ci-uses.js | 58 + .../eslint-plugin-mozilla/tests/valid-lazy.js | 176 + .../tests/valid-services-property.js | 42 + .../eslint-plugin-mozilla/tests/valid-services.js | 33 + .../eslint-plugin-mozilla/tests/xpidl/docshell.xpt | 6077 +++++++++++++++ .../tests/xpidl/xpcom_base.xpt | 3197 ++++++++ tools/lint/eslint/eslint-plugin-mozilla/update.sh | 61 + .../eslint/eslint-plugin-spidermonkey-js/LICENSE | 363 + .../lib/environments/self-hosted.js | 180 + .../eslint-plugin-spidermonkey-js/lib/index.js | 20 + .../lib/processors/self-hosted.js | 128 + .../eslint-plugin-spidermonkey-js/package.json | 28 + tools/lint/eslint/manifest.tt | 10 + tools/lint/eslint/setup_helper.py | 432 ++ tools/lint/eslint/update.sh | 71 + tools/lint/file-perm.yml | 53 + tools/lint/file-perm/__init__.py | 45 + tools/lint/file-whitespace.yml | 160 + tools/lint/file-whitespace/__init__.py | 124 + tools/lint/fluent-lint.yml | 13 + tools/lint/fluent-lint/__init__.py | 470 ++ tools/lint/fluent-lint/exclusions.yml | 198 + tools/lint/hooks.py | 63 + tools/lint/hooks_clang_format.py | 99 + tools/lint/hooks_js_format.py | 83 + tools/lint/l10n.yml | 37 + tools/lint/libpref/__init__.py | 112 + tools/lint/license.yml | 94 + tools/lint/license/__init__.py | 195 + tools/lint/license/valid-licenses.txt | 30 + tools/lint/lintpref.yml | 17 + tools/lint/mach_commands.py | 189 + tools/lint/mingw-capitalization.yml | 12 + tools/lint/mscom-init.yml | 46 + tools/lint/perfdocs.yml | 17 + tools/lint/perfdocs/__init__.py | 13 + tools/lint/perfdocs/doc_helpers.py | 85 + tools/lint/perfdocs/framework_gatherers.py | 571 ++ tools/lint/perfdocs/gatherer.py | 156 + tools/lint/perfdocs/generator.py | 281 + tools/lint/perfdocs/logger.py | 81 + tools/lint/perfdocs/perfdocs.py | 95 + tools/lint/perfdocs/templates/index.rst | 86 + tools/lint/perfdocs/utils.py | 157 + tools/lint/perfdocs/verifier.py | 601 ++ tools/lint/python/__init__.py | 3 + tools/lint/python/black.py | 179 + tools/lint/python/black_requirements.in | 4 + tools/lint/python/black_requirements.txt | 117 + tools/lint/python/l10n_lint.py | 171 + tools/lint/python/ruff.py | 187 + tools/lint/python/ruff_requirements.in | 1 + tools/lint/python/ruff_requirements.txt | 25 + tools/lint/rejected-words.yml | 335 + tools/lint/rst.yml | 11 + tools/lint/rst/__init__.py | 116 + tools/lint/rst/requirements.in | 19 + tools/lint/rst/requirements.txt | 210 + tools/lint/ruff.yml | 17 + tools/lint/rust/__init__.py | 180 + tools/lint/rustfmt.yml | 20 + tools/lint/shell/__init__.py | 148 + tools/lint/shellcheck.yml | 15 + tools/lint/spell/__init__.py | 168 + tools/lint/spell/codespell_requirements.in | 1 + tools/lint/spell/codespell_requirements.txt | 10 + tools/lint/spell/exclude-list.txt | 23 + tools/lint/stylelint.yml | 15 + tools/lint/stylelint/__init__.py | 187 + tools/lint/test-manifest-alpha.yml | 19 + tools/lint/test-manifest-alpha/__init__.py | 77 + .../test-manifest-alpha/error-level-manifests.yml | 7 + tools/lint/test-manifest-disable.yml | 16 + tools/lint/test-manifest-skip-if.yml | 18 + tools/lint/test/conftest.py | 305 + tools/lint/test/files/android-format/Bad.java | 8 + tools/lint/test/files/android-format/Main.kt | 7 + tools/lint/test/files/android-format/build.gradle | 1 + tools/lint/test/files/black/bad.py | 6 + tools/lint/test/files/black/invalid.py | 4 + tools/lint/test/files/clang-format/bad/bad.cpp | 6 + tools/lint/test/files/clang-format/bad/bad2.c | 8 + tools/lint/test/files/clang-format/bad/bad2.h | 1 + tools/lint/test/files/clang-format/bad/good.cpp | 1 + tools/lint/test/files/clang-format/good/foo.cpp | 1 + tools/lint/test/files/clippy/test1/Cargo.toml | 17 + tools/lint/test/files/clippy/test1/bad.rs | 14 + tools/lint/test/files/clippy/test1/bad2.rs | 14 + tools/lint/test/files/clippy/test1/good.rs | 6 + tools/lint/test/files/clippy/test2/Cargo.lock | 5 + tools/lint/test/files/clippy/test2/Cargo.toml | 8 + tools/lint/test/files/clippy/test2/src/bad_1.rs | 15 + tools/lint/test/files/clippy/test2/src/bad_2.rs | 17 + tools/lint/test/files/codespell/ignore.rst | 5 + tools/lint/test/files/eslint/good.js | 0 tools/lint/test/files/eslint/import/bad_import.js | 1 + tools/lint/test/files/eslint/nolint/foo.txt | 0 tools/lint/test/files/eslint/subdir/bad.js | 2 + tools/lint/test/files/eslint/testprettierignore | 1 + .../lint/test/files/file-perm/maybe-shebang/bad.js | 2 + .../test/files/file-perm/maybe-shebang/good.js | 5 + .../test/files/file-perm/no-shebang/bad-shebang.c | 2 + tools/lint/test/files/file-perm/no-shebang/bad.c | 1 + tools/lint/test/files/file-perm/no-shebang/bad.png | Bin 0 -> 49 bytes tools/lint/test/files/file-perm/no-shebang/good.c | 1 + .../lint/test/files/file-whitespace/bad-newline.c | 3 + .../lint/test/files/file-whitespace/bad-windows.c | 3 + tools/lint/test/files/file-whitespace/bad.c | 3 + tools/lint/test/files/file-whitespace/bad.js | 3 + tools/lint/test/files/file-whitespace/good.c | 1 + tools/lint/test/files/file-whitespace/good.js | 5 + tools/lint/test/files/fluent-lint/bad.ftl | 44 + .../files/fluent-lint/brand-names-excluded.ftl | 2 + tools/lint/test/files/fluent-lint/brand-names.ftl | 30 + .../lint/test/files/fluent-lint/comment-group1.ftl | 35 + .../lint/test/files/fluent-lint/comment-group2.ftl | 15 + .../test/files/fluent-lint/comment-resource1.ftl | 11 + .../test/files/fluent-lint/comment-resource2.ftl | 6 + .../test/files/fluent-lint/comment-resource3.ftl | 6 + .../test/files/fluent-lint/comment-resource4.ftl | 8 + .../test/files/fluent-lint/comment-resource5.ftl | 8 + .../test/files/fluent-lint/comment-resource6.ftl | 4 + .../test/files/fluent-lint/comment-variables1.ftl | 60 + .../test/files/fluent-lint/comment-variables2.ftl | 27 + tools/lint/test/files/fluent-lint/excluded.ftl | 6 + tools/lint/test/files/fluent-lint/test-brands.ftl | 5 + .../tools/lint/fluent-lint/exclusions.yml | 17 + tools/lint/test/files/license/.eslintrc.js | 5 + tools/lint/test/files/license/bad.c | 1 + tools/lint/test/files/license/bad.js | 6 + tools/lint/test/files/license/good-other.h | 9 + tools/lint/test/files/license/good.c | 8 + tools/lint/test/files/license/good.js | 7 + tools/lint/test/files/lintpref/bad.js | 2 + tools/lint/test/files/lintpref/good.js | 6 + tools/lint/test/files/rst/.dotfile.rst | 11 + tools/lint/test/files/rst/bad.rst | 20 + tools/lint/test/files/rst/bad2.rst | 4 + tools/lint/test/files/rst/bad3.rst | 6 + tools/lint/test/files/rst/good.rst | 11 + tools/lint/test/files/ruff/bad.py | 4 + tools/lint/test/files/ruff/ruff.toml | 1 + tools/lint/test/files/rustfmt/subdir/bad.rs | 16 + tools/lint/test/files/rustfmt/subdir/bad2.rs | 17 + tools/lint/test/files/rustfmt/subdir/good.rs | 6 + tools/lint/test/files/shellcheck/bad.sh | 3 + tools/lint/test/files/shellcheck/good.sh | 2 + tools/lint/test/files/stylelint/nolint/foo.txt | 0 tools/lint/test/files/stylelint/subdir/bad.css | 5 + .../test-manifest-alpha/mochitest-in-order.ini | 29 + .../mochitest-mostly-in-order.ini | 30 + .../mochitest-very-out-of-order.ini | 29 + .../other-ini-very-out-of-order.ini | 29 + tools/lint/test/files/trojan-source/README | 5 + .../test/files/trojan-source/commenting-out.cpp | 9 + .../lint/test/files/trojan-source/early-return.py | 9 + .../test/files/trojan-source/invisible-function.rs | 15 + tools/lint/test/files/updatebot/.yamllint | 6 + .../lint/test/files/updatebot/cargo-mismatch.yaml | 44 + tools/lint/test/files/updatebot/good1.yaml | 44 + tools/lint/test/files/updatebot/good2.yaml | 74 + tools/lint/test/files/updatebot/no-revision.yaml | 43 + tools/lint/test/files/yaml/.yamllint | 6 + tools/lint/test/files/yaml/bad.yml | 8 + tools/lint/test/files/yaml/good.yml | 6 + tools/lint/test/python.ini | 35 + tools/lint/test/test_android_format.py | 38 + tools/lint/test/test_black.py | 53 + tools/lint/test/test_clang_format.py | 139 + tools/lint/test/test_codespell.py | 37 + tools/lint/test/test_eslint.py | 99 + tools/lint/test/test_file_license.py | 23 + tools/lint/test/test_file_perm.py | 35 + tools/lint/test/test_file_whitespace.py | 51 + tools/lint/test/test_fluent_lint.py | 156 + tools/lint/test/test_lintpref.py | 16 + tools/lint/test/test_manifest_alpha.py | 33 + tools/lint/test/test_perfdocs.py | 858 +++ tools/lint/test/test_perfdocs_generation.py | 297 + tools/lint/test/test_perfdocs_helpers.py | 206 + tools/lint/test/test_rst.py | 25 + tools/lint/test/test_ruff.py | 39 + tools/lint/test/test_rustfmt.py | 70 + tools/lint/test/test_shellcheck.py | 26 + tools/lint/test/test_stylelint.py | 65 + tools/lint/test/test_trojan_source.py | 25 + tools/lint/test/test_updatebot.py | 44 + tools/lint/test/test_yaml.py | 28 + tools/lint/tox/tox_requirements.txt | 7 + tools/lint/trojan-source.yml | 27 + tools/lint/trojan-source/__init__.py | 67 + tools/lint/updatebot.yml | 9 + tools/lint/updatebot/__init__.py | 3 + tools/lint/updatebot/validate_yaml.py | 52 + tools/lint/wpt.yml | 10 + tools/lint/wpt/__init__.py | 3 + tools/lint/wpt/wpt.py | 59 + tools/lint/yaml.yml | 19 + tools/lint/yamllint_/__init__.py | 101 + tools/mach_commands.py | 594 ++ tools/moz.build | 79 + tools/moztreedocs/__init__.py | 244 + tools/moztreedocs/docs/adding-documentation.rst | 30 + tools/moztreedocs/docs/architecture.rst | 51 + tools/moztreedocs/docs/index.rst | 24 + tools/moztreedocs/docs/jsdoc-support.rst | 16 + tools/moztreedocs/docs/mdn-import.rst | 34 + tools/moztreedocs/docs/mermaid-integration.rst | 104 + tools/moztreedocs/docs/nested-docs.rst | 14 + tools/moztreedocs/docs/redirect.rst | 11 + tools/moztreedocs/docs/rstlint.rst | 12 + tools/moztreedocs/docs/run-try-job.rst | 27 + tools/moztreedocs/docs/server-synchronization.rst | 5 + tools/moztreedocs/mach_commands.py | 533 ++ tools/moztreedocs/package.py | 29 + tools/moztreedocs/upload.py | 172 + tools/performance/PerfStats.cpp | 317 + tools/performance/PerfStats.h | 161 + tools/performance/moz.build | 17 + tools/phabricator/mach_commands.py | 135 + tools/power/mach_commands.py | 149 + tools/power/moz.build | 26 + tools/power/rapl.cpp | 874 +++ tools/profiler/core/EHABIStackWalk.cpp | 597 ++ tools/profiler/core/EHABIStackWalk.h | 28 + tools/profiler/core/MicroGeckoProfiler.cpp | 203 + tools/profiler/core/PageInformation.cpp | 44 + tools/profiler/core/PageInformation.h | 68 + tools/profiler/core/PlatformMacros.h | 130 + tools/profiler/core/PowerCounters-linux.cpp | 287 + tools/profiler/core/PowerCounters-mac-amd64.cpp | 419 ++ tools/profiler/core/PowerCounters-mac-arm64.cpp | 47 + tools/profiler/core/PowerCounters-win.cpp | 342 + tools/profiler/core/PowerCounters.h | 52 + .../profiler/core/ProfileAdditionalInformation.cpp | 102 + tools/profiler/core/ProfileBuffer.cpp | 243 + tools/profiler/core/ProfileBuffer.h | 260 + tools/profiler/core/ProfileBufferEntry.cpp | 2321 ++++++ tools/profiler/core/ProfileBufferEntry.h | 532 ++ tools/profiler/core/ProfiledThreadData.cpp | 455 ++ tools/profiler/core/ProfiledThreadData.h | 250 + tools/profiler/core/ProfilerBacktrace.cpp | 101 + tools/profiler/core/ProfilerBacktrace.h | 184 + tools/profiler/core/ProfilerBindings.cpp | 386 + tools/profiler/core/ProfilerCodeAddressService.cpp | 75 + tools/profiler/core/ProfilerMarkers.cpp | 32 + tools/profiler/core/ProfilerThreadRegistration.cpp | 198 + .../core/ProfilerThreadRegistrationData.cpp | 303 + tools/profiler/core/ProfilerThreadRegistry.cpp | 40 + tools/profiler/core/ProfilerUtils.cpp | 118 + tools/profiler/core/VTuneProfiler.cpp | 80 + tools/profiler/core/VTuneProfiler.h | 78 + tools/profiler/core/memory_hooks.cpp | 632 ++ tools/profiler/core/memory_hooks.h | 25 + tools/profiler/core/platform-linux-android.cpp | 636 ++ tools/profiler/core/platform-macos.cpp | 297 + tools/profiler/core/platform-win32.cpp | 496 ++ tools/profiler/core/platform.cpp | 7067 ++++++++++++++++++ tools/profiler/core/platform.h | 381 + tools/profiler/core/shared-libraries-linux.cc | 280 + tools/profiler/core/shared-libraries-macos.cc | 211 + tools/profiler/core/shared-libraries-win32.cc | 167 + tools/profiler/core/vtune/ittnotify.h | 4123 ++++++++++ tools/profiler/docs/buffer.rst | 70 + tools/profiler/docs/code-overview.rst | 1494 ++++ tools/profiler/docs/fissionprofiler-20200424.png | Bin 0 -> 131301 bytes tools/profiler/docs/fissionprofiler.umlet.uxf | 546 ++ tools/profiler/docs/index.rst | 37 + tools/profiler/docs/instrumenting-javascript.rst | 60 + tools/profiler/docs/instrumenting-rust.rst | 433 ++ tools/profiler/docs/markers-guide.rst | 485 ++ tools/profiler/docs/memory.rst | 46 + tools/profiler/docs/profilerclasses-20220913.png | Bin 0 -> 727313 bytes tools/profiler/docs/profilerclasses.umlet.uxf | 811 ++ .../docs/profilerthreadregistration-20220913.png | Bin 0 -> 383738 bytes .../docs/profilerthreadregistration.umlet.uxf | 710 ++ tools/profiler/gecko/ChildProfilerController.cpp | 170 + tools/profiler/gecko/PProfiler.ipdl | 44 + tools/profiler/gecko/ProfilerChild.cpp | 565 ++ .../profiler/gecko/ProfilerIOInterposeObserver.cpp | 216 + tools/profiler/gecko/ProfilerIOInterposeObserver.h | 32 + tools/profiler/gecko/ProfilerParent.cpp | 1002 +++ tools/profiler/gecko/ProfilerTypes.ipdlh | 43 + tools/profiler/gecko/components.conf | 17 + tools/profiler/gecko/nsIProfiler.idl | 208 + tools/profiler/gecko/nsProfiler.cpp | 1487 ++++ tools/profiler/gecko/nsProfiler.h | 117 + tools/profiler/gecko/nsProfilerCIID.h | 16 + tools/profiler/gecko/nsProfilerStartParams.cpp | 65 + tools/profiler/gecko/nsProfilerStartParams.h | 36 + tools/profiler/lul/AutoObjectMapper.cpp | 79 + tools/profiler/lul/AutoObjectMapper.h | 64 + tools/profiler/lul/LulCommon.cpp | 100 + tools/profiler/lul/LulCommonExt.h | 509 ++ tools/profiler/lul/LulDwarf.cpp | 2538 +++++++ tools/profiler/lul/LulDwarfExt.h | 1312 ++++ tools/profiler/lul/LulDwarfInt.h | 193 + tools/profiler/lul/LulDwarfSummariser.cpp | 549 ++ tools/profiler/lul/LulDwarfSummariser.h | 64 + tools/profiler/lul/LulElf.cpp | 887 +++ tools/profiler/lul/LulElfExt.h | 69 + tools/profiler/lul/LulElfInt.h | 218 + tools/profiler/lul/LulMain.cpp | 2079 ++++++ tools/profiler/lul/LulMain.h | 378 + tools/profiler/lul/LulMainInt.h | 631 ++ tools/profiler/lul/platform-linux-lul.cpp | 75 + tools/profiler/lul/platform-linux-lul.h | 19 + tools/profiler/moz.build | 227 + tools/profiler/public/ChildProfilerController.h | 71 + tools/profiler/public/GeckoProfiler.h | 435 ++ tools/profiler/public/GeckoProfilerReporter.h | 26 + tools/profiler/public/GeckoTraceEvent.h | 1060 +++ tools/profiler/public/MicroGeckoProfiler.h | 130 + .../profiler/public/ProfileAdditionalInformation.h | 90 + ...rofileBufferEntrySerializationGeckoExtensions.h | 160 + tools/profiler/public/ProfileJSONWriter.h | 19 + tools/profiler/public/ProfilerBindings.h | 162 + tools/profiler/public/ProfilerChild.h | 106 + tools/profiler/public/ProfilerCodeAddressService.h | 52 + tools/profiler/public/ProfilerControl.h | 190 + tools/profiler/public/ProfilerCounts.h | 296 + tools/profiler/public/ProfilerLabels.h | 268 + tools/profiler/public/ProfilerMarkerTypes.h | 41 + tools/profiler/public/ProfilerMarkers.h | 355 + tools/profiler/public/ProfilerMarkersDetail.h | 31 + .../profiler/public/ProfilerMarkersPrerequisites.h | 31 + tools/profiler/public/ProfilerParent.h | 119 + tools/profiler/public/ProfilerRunnable.h | 68 + tools/profiler/public/ProfilerRustBindings.h | 12 + tools/profiler/public/ProfilerState.h | 399 + tools/profiler/public/ProfilerThreadPlatformData.h | 80 + tools/profiler/public/ProfilerThreadRegistration.h | 367 + .../public/ProfilerThreadRegistrationData.h | 537 ++ .../public/ProfilerThreadRegistrationInfo.h | 64 + tools/profiler/public/ProfilerThreadRegistry.h | 321 + tools/profiler/public/ProfilerThreadSleep.h | 58 + tools/profiler/public/ProfilerThreadState.h | 128 + tools/profiler/public/ProfilerUtils.h | 32 + tools/profiler/public/shared-libraries.h | 213 + tools/profiler/rust-api/Cargo.toml | 23 + tools/profiler/rust-api/README.md | 5 + tools/profiler/rust-api/build.rs | 118 + tools/profiler/rust-api/cbindgen.toml | 15 + tools/profiler/rust-api/extra-bindgen-flags.in | 1 + tools/profiler/rust-api/macros/Cargo.toml | 13 + tools/profiler/rust-api/macros/src/lib.rs | 65 + tools/profiler/rust-api/src/gecko_bindings/glue.rs | 53 + tools/profiler/rust-api/src/gecko_bindings/mod.rs | 21 + .../src/gecko_bindings/profiling_categories.rs | 32 + tools/profiler/rust-api/src/json_writer.rs | 86 + tools/profiler/rust-api/src/label.rs | 137 + tools/profiler/rust-api/src/lib.rs | 29 + .../rust-api/src/marker/deserializer_tags_state.rs | 116 + tools/profiler/rust-api/src/marker/mod.rs | 284 + tools/profiler/rust-api/src/marker/options.rs | 138 + tools/profiler/rust-api/src/marker/schema.rs | 233 + tools/profiler/rust-api/src/profiler_state.rs | 78 + tools/profiler/rust-api/src/thread.rs | 23 + tools/profiler/rust-api/src/time.rs | 71 + tools/profiler/rust-helper/Cargo.toml | 23 + .../rust-helper/src/compact_symbol_table.rs | 40 + tools/profiler/rust-helper/src/elf.rs | 101 + tools/profiler/rust-helper/src/lib.rs | 107 + tools/profiler/tests/browser/browser.ini | 102 + .../browser/browser_test_feature_ipcmessages.js | 100 + .../browser/browser_test_feature_jsallocations.js | 74 + .../browser_test_feature_nostacksampling.js | 72 + .../browser/browser_test_marker_network_cancel.js | 71 + ...browser_test_marker_network_private_browsing.js | 91 + .../browser_test_marker_network_redirect.js | 341 + ...est_marker_network_serviceworker_cache_first.js | 378 + ...arker_network_serviceworker_no_fetch_handler.js | 218 + ...erviceworker_no_respondWith_in_fetch_handler.js | 294 + ...r_network_serviceworker_synthetized_response.js | 480 ++ .../browser/browser_test_marker_network_simple.js | 81 + .../browser/browser_test_marker_network_sts.js | 130 + .../tests/browser/browser_test_markers_gc_cc.js | 49 + .../browser/browser_test_markers_parent_process.js | 37 + .../browser_test_markers_preferencereads.js | 73 + .../browser/browser_test_profile_capture_by_pid.js | 199 + .../tests/browser/browser_test_profile_fission.js | 191 + .../browser_test_profile_multi_frame_page_info.js | 83 + .../browser_test_profile_single_frame_page_info.js | 132 + .../browser/browser_test_profile_slow_capture.js | 104 + tools/profiler/tests/browser/do_work_500ms.html | 41 + .../tests/browser/firefox-logo-nightly.svg | 1 + tools/profiler/tests/browser/head.js | 159 + tools/profiler/tests/browser/multi_frame.html | 11 + .../tests/browser/page_with_resources.html | 11 + tools/profiler/tests/browser/redirect.sjs | 8 + .../serviceworkers/firefox-logo-nightly.svg | 1 + .../browser/serviceworkers/serviceworker-utils.js | 39 + .../serviceworkers/serviceworker_cache_first.js | 34 + .../serviceworker_no_fetch_handler.js | 4 + ...erviceworker_no_respondWith_in_fetch_handler.js | 9 + .../browser/serviceworkers/serviceworker_page.html | 10 + .../serviceworkers/serviceworker_register.html | 9 + .../serviceworkers/serviceworker_simple.html | 9 + .../serviceworker_synthetized_response.js | 27 + tools/profiler/tests/browser/simple.html | 9 + tools/profiler/tests/browser/single_frame.html | 10 + tools/profiler/tests/chrome/chrome.ini | 8 + tools/profiler/tests/chrome/profiler_test_utils.js | 66 + .../profiler/tests/chrome/test_profile_worker.html | 66 + .../chrome/test_profile_worker_bug_1428076.html | 58 + tools/profiler/tests/gtest/GeckoProfiler.cpp | 5099 +++++++++++++ tools/profiler/tests/gtest/LulTest.cpp | 51 + tools/profiler/tests/gtest/LulTestDwarf.cpp | 2733 +++++++ .../profiler/tests/gtest/LulTestInfrastructure.cpp | 498 ++ tools/profiler/tests/gtest/LulTestInfrastructure.h | 736 ++ tools/profiler/tests/gtest/ThreadProfileTest.cpp | 60 + tools/profiler/tests/gtest/moz.build | 45 + tools/profiler/tests/shared-head.js | 591 ++ tools/profiler/tests/xpcshell/head.js | 244 + .../tests/xpcshell/test_active_configuration.js | 115 + .../tests/xpcshell/test_addProfilerMarker.js | 221 + tools/profiler/tests/xpcshell/test_asm.js | 76 + .../tests/xpcshell/test_assertion_helper.js | 162 + tools/profiler/tests/xpcshell/test_enterjit_osr.js | 52 + .../tests/xpcshell/test_enterjit_osr_disabling.js | 14 + .../tests/xpcshell/test_enterjit_osr_enabling.js | 14 + .../tests/xpcshell/test_feature_fileioall.js | 159 + tools/profiler/tests/xpcshell/test_feature_java.js | 31 + tools/profiler/tests/xpcshell/test_feature_js.js | 63 + .../tests/xpcshell/test_feature_mainthreadio.js | 122 + .../xpcshell/test_feature_nativeallocations.js | 158 + .../tests/xpcshell/test_feature_stackwalking.js | 48 + tools/profiler/tests/xpcshell/test_get_features.js | 8 + .../profiler/tests/xpcshell/test_merged_stacks.js | 74 + tools/profiler/tests/xpcshell/test_pause.js | 126 + .../profiler/tests/xpcshell/test_responsiveness.js | 50 + tools/profiler/tests/xpcshell/test_run.js | 37 + .../profiler/tests/xpcshell/test_shared_library.js | 21 + tools/profiler/tests/xpcshell/test_start.js | 21 + tools/profiler/tests/xpcshell/xpcshell.ini | 72 + tools/quitter/README.md | 6 + tools/quitter/quitter@mozilla.org.xpi | Bin 0 -> 16237 bytes tools/rb/README | 6 + tools/rb/filter-log.pl | 44 + tools/rb/find-comptr-leakers.pl | 114 + tools/rb/find_leakers.py | 110 + tools/rb/fix_stacks.py | 135 + tools/rb/make-tree.pl | 303 + tools/rewriting/Generated.txt | 31 + tools/rewriting/ThirdPartyPaths.txt | 195 + tools/rusttests/app.mozbuild | 5 + tools/rusttests/config/mozconfigs/common | 19 + .../rusttests/config/mozconfigs/linux32/rusttests | 2 + .../config/mozconfigs/linux32/rusttests-debug | 2 + .../rusttests/config/mozconfigs/linux64/rusttests | 2 + .../config/mozconfigs/linux64/rusttests-debug | 2 + .../rusttests/config/mozconfigs/macosx64/rusttests | 2 + .../config/mozconfigs/macosx64/rusttests-debug | 2 + tools/rusttests/config/mozconfigs/win32/rusttests | 3 + .../config/mozconfigs/win32/rusttests-debug | 2 + tools/rusttests/config/mozconfigs/win64/rusttests | 3 + .../config/mozconfigs/win64/rusttests-debug | 2 + tools/rusttests/config/mozconfigs/windows-common | 2 + tools/rusttests/moz.configure | 5 + tools/sanitizer/docs/asan.rst | 378 + tools/sanitizer/docs/asan_nightly.rst | 204 + tools/sanitizer/docs/index.rst | 33 + tools/sanitizer/docs/memory_sanitizer.rst | 173 + tools/sanitizer/docs/tsan.rst | 327 + tools/tryselect/__init__.py | 3 + tools/tryselect/cli.py | 167 + tools/tryselect/docs/configuration.rst | 30 + tools/tryselect/docs/img/add-new-jobs.png | Bin 0 -> 33561 bytes tools/tryselect/docs/img/phab-treeherder-link.png | Bin 0 -> 35922 bytes tools/tryselect/docs/index.rst | 71 + tools/tryselect/docs/presets.rst | 85 + tools/tryselect/docs/selectors/again.rst | 36 + tools/tryselect/docs/selectors/auto.rst | 24 + tools/tryselect/docs/selectors/chooser.rst | 32 + tools/tryselect/docs/selectors/compare.rst | 17 + tools/tryselect/docs/selectors/empty.rst | 21 + tools/tryselect/docs/selectors/fuzzy.rst | 371 + tools/tryselect/docs/selectors/fzf.png | Bin 0 -> 44467 bytes tools/tryselect/docs/selectors/index.rst | 44 + tools/tryselect/docs/selectors/release.rst | 31 + tools/tryselect/docs/selectors/scriptworker.rst | 31 + tools/tryselect/docs/selectors/syntax.rst | 41 + tools/tryselect/docs/tasks.rst | 152 + tools/tryselect/mach_commands.py | 528 ++ tools/tryselect/preset.py | 107 + tools/tryselect/push.py | 239 + tools/tryselect/selectors/__init__.py | 3 + tools/tryselect/selectors/again.py | 151 + tools/tryselect/selectors/auto.py | 115 + tools/tryselect/selectors/chooser/.eslintrc.js | 16 + tools/tryselect/selectors/chooser/__init__.py | 97 + tools/tryselect/selectors/chooser/app.py | 177 + tools/tryselect/selectors/chooser/requirements.txt | 44 + tools/tryselect/selectors/chooser/static/filter.js | 116 + tools/tryselect/selectors/chooser/static/select.js | 46 + tools/tryselect/selectors/chooser/static/style.css | 107 + .../selectors/chooser/templates/chooser.html | 78 + .../selectors/chooser/templates/close.html | 11 + .../selectors/chooser/templates/layout.html | 71 + tools/tryselect/selectors/compare.py | 66 + tools/tryselect/selectors/coverage.py | 447 ++ tools/tryselect/selectors/empty.py | 24 + tools/tryselect/selectors/fuzzy.py | 255 + tools/tryselect/selectors/perf.py | 1335 ++++ tools/tryselect/selectors/perfselector/__init__.py | 3 + .../selectors/perfselector/classification.py | 374 + .../selectors/perfselector/perfcomparators.py | 258 + tools/tryselect/selectors/perfselector/utils.py | 44 + tools/tryselect/selectors/preview.py | 102 + tools/tryselect/selectors/release.py | 159 + tools/tryselect/selectors/scriptworker.py | 177 + tools/tryselect/selectors/syntax.py | 708 ++ tools/tryselect/task_config.py | 530 ++ tools/tryselect/tasks.py | 186 + tools/tryselect/test/conftest.py | 101 + tools/tryselect/test/cram.ini | 5 + tools/tryselect/test/python.ini | 19 + tools/tryselect/test/setup.sh | 99 + tools/tryselect/test/test_again.py | 73 + tools/tryselect/test/test_auto.py | 31 + tools/tryselect/test/test_auto.t | 61 + tools/tryselect/test/test_chooser.py | 77 + tools/tryselect/test/test_empty.t | 47 + tools/tryselect/test/test_fuzzy.py | 54 + tools/tryselect/test/test_fuzzy.t | 201 + tools/tryselect/test/test_message.t | 63 + .../tryselect/test/test_mozharness_integration.py | 143 + tools/tryselect/test/test_perf.py | 1245 +++ tools/tryselect/test/test_perfcomparators.py | 150 + tools/tryselect/test/test_preset.t | 360 + tools/tryselect/test/test_presets.py | 58 + tools/tryselect/test/test_release.py | 43 + tools/tryselect/test/test_scriptworker.py | 39 + tools/tryselect/test/test_task_configs.py | 144 + tools/tryselect/test/test_tasks.py | 93 + tools/tryselect/try_presets.yml | 266 + tools/tryselect/util/__init__.py | 3 + tools/tryselect/util/dicttools.py | 50 + tools/tryselect/util/estimates.py | 126 + tools/tryselect/util/fzf.py | 423 ++ tools/tryselect/util/manage_estimates.py | 132 + tools/tryselect/watchman.json | 15 + tools/update-packaging/README | 4 + tools/update-packaging/app.mozbuild | 9 + tools/update-packaging/common.sh | 201 + tools/update-packaging/make_full_update.sh | 121 + tools/update-packaging/make_incremental_update.sh | 313 + tools/update-packaging/moz.build | 5 + tools/update-packaging/moz.configure | 0 tools/update-packaging/test/buildrefmars.sh | 27 + tools/update-packaging/test/common.sh | 202 + tools/update-packaging/test/diffmar.sh | 48 + .../Contents/MacOS/diff-patch-larger-than-file.txt | 1 + .../test/from-mac/Contents/MacOS/force.txt | 1 + .../test/from-mac/Contents/MacOS/removed.txt | 1 + .../test/from-mac/Contents/MacOS/same.bin | Bin 0 -> 200 bytes .../test/from-mac/Contents/MacOS/update.manifest | 1 + .../MacOS/{foodir/diff-patch-larger-than-file.txt | 1 + .../from-mac/Contents/MacOS/{foodir/readme.txt | 1 + .../from-mac/Contents/MacOS/{foodir/removed.txt | 1 + .../test/from-mac/Contents/MacOS/{foodir/same.bin | Bin 0 -> 200 bytes .../test/from-mac/Contents/MacOS/{foodir/same.txt | 1 + .../Contents/MacOS/{foodir/update.manifest | 1 + .../from-mac/Contents/Resources/application.ini | 5 + .../from-mac/Contents/Resources/chrome.manifest | 0 .../diff/diff-patch-larger-than-file.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + .../test/from-mac/Contents/Resources/precomplete | 26 + .../test/from-mac/Contents/Resources/readme.txt | 2 + .../test/from-mac/Contents/Resources/removed-files | 8 + .../test/from-mac/Contents/Resources/removed.txt | 1 + .../test/from-mac/Contents/Resources/same.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + .../Contents/Resources/update-settings.ini | 1 + .../Contents/Resources/{foodir/channel-prefs.js | 1 + .../from-mac/Contents/Resources/{foodir/force.txt | 1 + tools/update-packaging/test/from/application.ini | 5 + tools/update-packaging/test/from/chrome.manifest | 0 .../test/from/diff-patch-larger-than-file.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + tools/update-packaging/test/from/force.txt | 1 + tools/update-packaging/test/from/precomplete | 23 + tools/update-packaging/test/from/readme.txt | 2 + tools/update-packaging/test/from/removed-files | 8 + tools/update-packaging/test/from/removed.txt | 1 + tools/update-packaging/test/from/same.bin | Bin 0 -> 200 bytes tools/update-packaging/test/from/same.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + .../update-packaging/test/from/update-settings.ini | 1 + tools/update-packaging/test/from/update.manifest | 1 + .../test/from/{foodir/channel-prefs.js | 1 + .../from/{foodir/diff-patch-larger-than-file.txt | 1 + tools/update-packaging/test/from/{foodir/force.txt | 1 + .../update-packaging/test/from/{foodir/readme.txt | 1 + .../update-packaging/test/from/{foodir/removed.txt | 1 + tools/update-packaging/test/from/{foodir/same.bin | Bin 0 -> 200 bytes tools/update-packaging/test/from/{foodir/same.txt | 1 + .../test/from/{foodir/update.manifest | 1 + tools/update-packaging/test/make_full_update.sh | 112 + tools/update-packaging/test/runtests.sh | 12 + tools/update-packaging/test/testpatchfile.txt | 2 + .../test/to-mac/Contents/MacOS/addFeedPrefs.js | 1 + .../test/to-mac/Contents/MacOS/added.txt | 1 + .../Contents/MacOS/diff-patch-larger-than-file.bin | Bin 0 -> 200 bytes .../Contents/MacOS/diff-patch-larger-than-file.txt | 1 + .../test/to-mac/Contents/MacOS/force.txt | 1 + .../test/to-mac/Contents/MacOS/same.bin | Bin 0 -> 200 bytes .../test/to-mac/Contents/MacOS/update.manifest | 1 + .../test/to-mac/Contents/MacOS/{foodir/added.txt | 1 + .../MacOS/{foodir/diff-patch-larger-than-file.txt | 1 + .../test/to-mac/Contents/MacOS/{foodir/readme.txt | 1 + .../test/to-mac/Contents/MacOS/{foodir/same.bin | Bin 0 -> 200 bytes .../test/to-mac/Contents/MacOS/{foodir/same.txt | 1 + .../to-mac/Contents/MacOS/{foodir/update.manifest | 1 + .../test/to-mac/Contents/Resources/application.ini | 5 + .../test/to-mac/Contents/Resources/chrome.manifest | 0 .../distribution/extensions/added/file.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + .../Contents/Resources/extensions/added/file.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + .../test/to-mac/Contents/Resources/precomplete | 33 + .../test/to-mac/Contents/Resources/readme.txt | 1 + .../test/to-mac/Contents/Resources/removed-files | 14 + .../test/to-mac/Contents/Resources/same.txt | 1 + .../Resources/searchplugins/added/file.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + .../to-mac/Contents/Resources/update-settings.ini | 1 + .../Contents/Resources/{foodir/channel-prefs.js | 1 + .../to-mac/Contents/Resources/{foodir/force.txt | 1 + tools/update-packaging/test/to/addFeedPrefs.js | 1 + tools/update-packaging/test/to/added.txt | 1 + tools/update-packaging/test/to/application.ini | 5 + tools/update-packaging/test/to/chrome.manifest | 0 .../test/to/diff-patch-larger-than-file.bin | Bin 0 -> 200 bytes .../test/to/diff-patch-larger-than-file.txt | 1 + .../test/to/distribution/extensions/added/file.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + .../test/to/extensions/added/file.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + tools/update-packaging/test/to/force.txt | 1 + tools/update-packaging/test/to/precomplete | 30 + tools/update-packaging/test/to/readme.txt | 1 + tools/update-packaging/test/to/removed-files | 14 + tools/update-packaging/test/to/same.bin | Bin 0 -> 200 bytes tools/update-packaging/test/to/same.txt | 1 + .../test/to/searchplugins/added/file.txt | 1 + .../diff/diff-patch-larger-than-file.txt | 1 + tools/update-packaging/test/to/update-settings.ini | 1 + tools/update-packaging/test/to/update.manifest | 1 + tools/update-packaging/test/to/{foodir/added.txt | 1 + .../test/to/{foodir/channel-prefs.js | 1 + .../to/{foodir/diff-patch-larger-than-file.txt | 1 + tools/update-packaging/test/to/{foodir/force.txt | 1 + tools/update-packaging/test/to/{foodir/readme.txt | 1 + tools/update-packaging/test/to/{foodir/same.bin | Bin 0 -> 200 bytes tools/update-packaging/test/to/{foodir/same.txt | 1 + .../test/to/{foodir/update.manifest | 1 + tools/update-programs/README | 19 + tools/update-programs/app.mozbuild | 61 + tools/update-programs/confvars.sh | 12 + tools/update-programs/moz.configure | 22 + tools/update-verify/README.md | 118 + tools/update-verify/python/util/__init__.py | 3 + tools/update-verify/python/util/commands.py | 57 + .../release/common/cached_download.sh | 40 + .../update-verify/release/common/check_updates.sh | 124 + .../release/common/download_builds.sh | 36 + .../update-verify/release/common/download_mars.sh | 105 + tools/update-verify/release/common/installdmg.ex | 45 + .../release/common/unpack-diskimage.sh | 95 + tools/update-verify/release/common/unpack.sh | 121 + tools/update-verify/release/compare-directories.py | 273 + tools/update-verify/release/final-verification.sh | 519 ++ tools/update-verify/release/get-update-xml.sh | 36 + tools/update-verify/release/mar_certs/README | 29 + tools/update-verify/release/mar_certs/dep1.der | Bin 0 -> 1225 bytes tools/update-verify/release/mar_certs/dep2.der | Bin 0 -> 1225 bytes .../mar_certs/nightly_aurora_level3_primary.der | Bin 0 -> 1225 bytes .../mar_certs/nightly_aurora_level3_secondary.der | Bin 0 -> 1225 bytes .../release/mar_certs/release_primary.der | Bin 0 -> 1225 bytes .../release/mar_certs/release_secondary.der | Bin 0 -> 1225 bytes .../update-verify/release/mar_certs/sha1/dep1.der | Bin 0 -> 709 bytes .../update-verify/release/mar_certs/sha1/dep2.der | Bin 0 -> 713 bytes .../release/mar_certs/sha1/release_primary.der | Bin 0 -> 709 bytes .../release/mar_certs/sha1/release_secondary.der | Bin 0 -> 713 bytes .../release/mar_certs/xpcshellCertificate.der | Bin 0 -> 1189 bytes .../update-verify/release/replace-updater-certs.py | 41 + tools/update-verify/release/test-mar-url.sh | 46 + tools/update-verify/release/updates/verify.sh | 292 + tools/update-verify/scripts/async_download.py | 362 + tools/update-verify/scripts/chunked-verify.py | 68 + tools/update-verify/scripts/chunked-verify.sh | 72 + tools/vcs/mach_commands.py | 242 + 1230 files changed, 173798 insertions(+) create mode 100755 tools/bloatview/bloatdiff.pl create mode 100755 tools/bloatview/bloattable.pl create mode 100644 tools/browsertime/README.md create mode 100644 tools/browsertime/mach_commands.py create mode 100644 tools/browsertime/moz.build create mode 100644 tools/browsertime/package-lock.json create mode 100644 tools/browsertime/package.json create mode 100644 tools/clang-tidy/config.yaml create mode 100644 tools/clang-tidy/test/bugprone-argument-comment.cpp create mode 100644 tools/clang-tidy/test/bugprone-argument-comment.json create mode 100644 tools/clang-tidy/test/bugprone-assert-side-effect.cpp create mode 100644 tools/clang-tidy/test/bugprone-assert-side-effect.json create mode 100644 tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.cpp create mode 100644 tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.json create mode 100644 tools/clang-tidy/test/bugprone-forward-declaration-namespace.cpp create mode 100644 tools/clang-tidy/test/bugprone-forward-declaration-namespace.json create mode 100644 tools/clang-tidy/test/bugprone-incorrect-roundings.cpp create mode 100644 tools/clang-tidy/test/bugprone-incorrect-roundings.json create mode 100644 tools/clang-tidy/test/bugprone-integer-division.cpp create mode 100644 tools/clang-tidy/test/bugprone-integer-division.json create mode 100644 tools/clang-tidy/test/bugprone-macro-parentheses.cpp create mode 100644 tools/clang-tidy/test/bugprone-macro-parentheses.json create mode 100644 tools/clang-tidy/test/bugprone-macro-repeated-side-effects.cpp create mode 100644 tools/clang-tidy/test/bugprone-macro-repeated-side-effects.json create mode 100644 tools/clang-tidy/test/bugprone-misplaced-widening-cast.cpp create mode 100644 tools/clang-tidy/test/bugprone-misplaced-widening-cast.json create mode 100644 tools/clang-tidy/test/bugprone-move-forwarding-reference.cpp create mode 100644 tools/clang-tidy/test/bugprone-move-forwarding-reference.json create mode 100644 tools/clang-tidy/test/bugprone-multiple-statement-macro.cpp create mode 100644 tools/clang-tidy/test/bugprone-multiple-statement-macro.json create mode 100644 tools/clang-tidy/test/bugprone-sizeof-expression.cpp create mode 100644 tools/clang-tidy/test/bugprone-sizeof-expression.json create mode 100644 tools/clang-tidy/test/bugprone-string-constructor.cpp create mode 100644 tools/clang-tidy/test/bugprone-string-constructor.json create mode 100644 tools/clang-tidy/test/bugprone-string-integer-assignment.cpp create mode 100644 tools/clang-tidy/test/bugprone-string-integer-assignment.json create mode 100644 tools/clang-tidy/test/bugprone-suspicious-memset-usage.cpp create mode 100644 tools/clang-tidy/test/bugprone-suspicious-memset-usage.json create mode 100644 tools/clang-tidy/test/bugprone-suspicious-missing-comma.cpp create mode 100644 tools/clang-tidy/test/bugprone-suspicious-missing-comma.json create mode 100644 tools/clang-tidy/test/bugprone-suspicious-semicolon.cpp create mode 100644 tools/clang-tidy/test/bugprone-suspicious-semicolon.json create mode 100644 tools/clang-tidy/test/bugprone-suspicious-string-compare.cpp create mode 100644 tools/clang-tidy/test/bugprone-suspicious-string-compare.json create mode 100644 tools/clang-tidy/test/bugprone-swapped-arguments.cpp create mode 100644 tools/clang-tidy/test/bugprone-swapped-arguments.json create mode 100644 tools/clang-tidy/test/bugprone-too-small-loop-variable.cpp create mode 100644 tools/clang-tidy/test/bugprone-too-small-loop-variable.json create mode 100644 tools/clang-tidy/test/bugprone-unused-raii.cpp create mode 100644 tools/clang-tidy/test/bugprone-unused-raii.json create mode 100644 tools/clang-tidy/test/bugprone-use-after-move.cpp create mode 100644 tools/clang-tidy/test/bugprone-use-after-move.json create mode 100644 tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.json create mode 100644 tools/clang-tidy/test/clang-analyzer-core.DivideZero.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-core.DivideZero.json create mode 100644 tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.json create mode 100644 tools/clang-tidy/test/clang-analyzer-core.NullDereference.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-core.NullDereference.json create mode 100644 tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.json create mode 100644 tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.json create mode 100644 tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.json create mode 100644 tools/clang-tidy/test/clang-analyzer-cplusplus.Move.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-cplusplus.Move.json create mode 100644 tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.json create mode 100644 tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.json create mode 100644 tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.json create mode 100644 tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.json create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.json create mode 100644 tools/clang-tidy/test/clang-analyzer-unix.Malloc.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-unix.Malloc.json create mode 100644 tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.json create mode 100644 tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.cpp create mode 100644 tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.json create mode 100644 tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.cpp create mode 100644 tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.json create mode 100644 tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.cpp create mode 100644 tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.json create mode 100644 tools/clang-tidy/test/misc-non-copyable-objects.cpp create mode 100644 tools/clang-tidy/test/misc-non-copyable-objects.json create mode 100644 tools/clang-tidy/test/misc-redundant-expression.cpp create mode 100644 tools/clang-tidy/test/misc-redundant-expression.json create mode 100644 tools/clang-tidy/test/misc-unused-alias-decls.cpp create mode 100644 tools/clang-tidy/test/misc-unused-alias-decls.json create mode 100644 tools/clang-tidy/test/misc-unused-using-decls.cpp create mode 100644 tools/clang-tidy/test/misc-unused-using-decls.json create mode 100644 tools/clang-tidy/test/modernize-avoid-bind.cpp create mode 100644 tools/clang-tidy/test/modernize-avoid-bind.json create mode 100644 tools/clang-tidy/test/modernize-concat-nested-namespaces.cpp create mode 100644 tools/clang-tidy/test/modernize-concat-nested-namespaces.json create mode 100644 tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.cpp create mode 100644 tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.json create mode 100644 tools/clang-tidy/test/modernize-loop-convert.cpp create mode 100644 tools/clang-tidy/test/modernize-loop-convert.json create mode 100644 tools/clang-tidy/test/modernize-raw-string-literal.cpp create mode 100644 tools/clang-tidy/test/modernize-raw-string-literal.json create mode 100644 tools/clang-tidy/test/modernize-redundant-void-arg.cpp create mode 100644 tools/clang-tidy/test/modernize-redundant-void-arg.json create mode 100644 tools/clang-tidy/test/modernize-shrink-to-fit.cpp create mode 100644 tools/clang-tidy/test/modernize-shrink-to-fit.json create mode 100644 tools/clang-tidy/test/modernize-use-auto.cpp create mode 100644 tools/clang-tidy/test/modernize-use-auto.json create mode 100644 tools/clang-tidy/test/modernize-use-bool-literals.cpp create mode 100644 tools/clang-tidy/test/modernize-use-bool-literals.json create mode 100644 tools/clang-tidy/test/modernize-use-equals-default.cpp create mode 100644 tools/clang-tidy/test/modernize-use-equals-default.json create mode 100644 tools/clang-tidy/test/modernize-use-equals-delete.cpp create mode 100644 tools/clang-tidy/test/modernize-use-equals-delete.json create mode 100644 tools/clang-tidy/test/modernize-use-nullptr.cpp create mode 100644 tools/clang-tidy/test/modernize-use-nullptr.json create mode 100644 tools/clang-tidy/test/modernize-use-override.cpp create mode 100644 tools/clang-tidy/test/modernize-use-override.json create mode 100644 tools/clang-tidy/test/modernize-use-using.cpp create mode 100644 tools/clang-tidy/test/modernize-use-using.json create mode 100644 tools/clang-tidy/test/performance-faster-string-find.cpp create mode 100644 tools/clang-tidy/test/performance-faster-string-find.json create mode 100644 tools/clang-tidy/test/performance-for-range-copy.cpp create mode 100644 tools/clang-tidy/test/performance-for-range-copy.json create mode 100644 tools/clang-tidy/test/performance-implicit-conversion-in-loop.cpp create mode 100644 tools/clang-tidy/test/performance-implicit-conversion-in-loop.json create mode 100644 tools/clang-tidy/test/performance-inefficient-algorithm.cpp create mode 100644 tools/clang-tidy/test/performance-inefficient-algorithm.json create mode 100644 tools/clang-tidy/test/performance-inefficient-string-concatenation.cpp create mode 100644 tools/clang-tidy/test/performance-inefficient-string-concatenation.json create mode 100644 tools/clang-tidy/test/performance-inefficient-vector-operation.cpp create mode 100644 tools/clang-tidy/test/performance-inefficient-vector-operation.json create mode 100644 tools/clang-tidy/test/performance-move-const-arg.cpp create mode 100644 tools/clang-tidy/test/performance-move-const-arg.json create mode 100644 tools/clang-tidy/test/performance-move-constructor-init.cpp create mode 100644 tools/clang-tidy/test/performance-move-constructor-init.json create mode 100644 tools/clang-tidy/test/performance-noexcept-move-constructor.cpp create mode 100644 tools/clang-tidy/test/performance-noexcept-move-constructor.json create mode 100644 tools/clang-tidy/test/performance-type-promotion-in-math-fn.cpp create mode 100644 tools/clang-tidy/test/performance-type-promotion-in-math-fn.json create mode 100644 tools/clang-tidy/test/performance-unnecessary-copy-initialization.cpp create mode 100644 tools/clang-tidy/test/performance-unnecessary-copy-initialization.json create mode 100644 tools/clang-tidy/test/performance-unnecessary-value-param.cpp create mode 100644 tools/clang-tidy/test/performance-unnecessary-value-param.json create mode 100644 tools/clang-tidy/test/readability-braces-around-statements.cpp create mode 100644 tools/clang-tidy/test/readability-braces-around-statements.json create mode 100644 tools/clang-tidy/test/readability-const-return-type.cpp create mode 100644 tools/clang-tidy/test/readability-const-return-type.json create mode 100644 tools/clang-tidy/test/readability-container-size-empty.cpp create mode 100644 tools/clang-tidy/test/readability-container-size-empty.json create mode 100644 tools/clang-tidy/test/readability-delete-null-pointer.cpp create mode 100644 tools/clang-tidy/test/readability-delete-null-pointer.json create mode 100644 tools/clang-tidy/test/readability-else-after-return.cpp create mode 100644 tools/clang-tidy/test/readability-else-after-return.json create mode 100644 tools/clang-tidy/test/readability-implicit-bool-conversion.cpp create mode 100644 tools/clang-tidy/test/readability-implicit-bool-conversion.json create mode 100644 tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.cpp create mode 100644 tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.json create mode 100644 tools/clang-tidy/test/readability-isolate-declaration.cpp create mode 100644 tools/clang-tidy/test/readability-isolate-declaration.json create mode 100644 tools/clang-tidy/test/readability-magic-numbers.cpp create mode 100644 tools/clang-tidy/test/readability-magic-numbers.json create mode 100644 tools/clang-tidy/test/readability-misleading-indentation.cpp create mode 100644 tools/clang-tidy/test/readability-misleading-indentation.json create mode 100644 tools/clang-tidy/test/readability-non-const-parameter.cpp create mode 100644 tools/clang-tidy/test/readability-non-const-parameter.json create mode 100644 tools/clang-tidy/test/readability-qualified-auto.cpp create mode 100644 tools/clang-tidy/test/readability-qualified-auto.json create mode 100644 tools/clang-tidy/test/readability-redundant-control-flow.cpp create mode 100644 tools/clang-tidy/test/readability-redundant-control-flow.json create mode 100644 tools/clang-tidy/test/readability-redundant-preprocessor.cpp create mode 100644 tools/clang-tidy/test/readability-redundant-preprocessor.json create mode 100644 tools/clang-tidy/test/readability-redundant-smartptr-get.cpp create mode 100644 tools/clang-tidy/test/readability-redundant-smartptr-get.json create mode 100644 tools/clang-tidy/test/readability-redundant-string-cstr.cpp create mode 100644 tools/clang-tidy/test/readability-redundant-string-cstr.json create mode 100644 tools/clang-tidy/test/readability-redundant-string-init.cpp create mode 100644 tools/clang-tidy/test/readability-redundant-string-init.json create mode 100644 tools/clang-tidy/test/readability-simplify-boolean-expr.cpp create mode 100644 tools/clang-tidy/test/readability-simplify-boolean-expr.json create mode 100644 tools/clang-tidy/test/readability-static-accessed-through-instance.cpp create mode 100644 tools/clang-tidy/test/readability-static-accessed-through-instance.json create mode 100644 tools/clang-tidy/test/readability-uniqueptr-delete-release.cpp create mode 100644 tools/clang-tidy/test/readability-uniqueptr-delete-release.json create mode 100644 tools/clang-tidy/test/structures.h create mode 100644 tools/code-coverage/CodeCoverageHandler.cpp create mode 100644 tools/code-coverage/CodeCoverageHandler.h create mode 100644 tools/code-coverage/PerTestCoverageUtils.sys.mjs create mode 100644 tools/code-coverage/components.conf create mode 100644 tools/code-coverage/docs/index.rst create mode 100644 tools/code-coverage/moz.build create mode 100644 tools/code-coverage/nsCodeCoverage.cpp create mode 100644 tools/code-coverage/nsCodeCoverage.h create mode 100644 tools/code-coverage/nsICodeCoverage.idl create mode 100644 tools/code-coverage/tests/mochitest/mochitest.ini create mode 100644 tools/code-coverage/tests/mochitest/test_coverage_specialpowers.html create mode 100644 tools/code-coverage/tests/xpcshell/head.js create mode 100644 tools/code-coverage/tests/xpcshell/support.js create mode 100644 tools/code-coverage/tests/xpcshell/test_basic.js create mode 100644 tools/code-coverage/tests/xpcshell/test_basic_child_and_parent.js create mode 100644 tools/code-coverage/tests/xpcshell/xpcshell.ini create mode 100644 tools/compare-locales/mach_commands.py create mode 100644 tools/compare-locales/requirements.in create mode 100644 tools/crashreporter/injector/app.mozbuild create mode 100644 tools/crashreporter/injector/injector.cc create mode 100644 tools/crashreporter/injector/moz.build create mode 100644 tools/crashreporter/injector/moz.configure create mode 100755 tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py create mode 100644 tools/crashreporter/system-symbols/mac/get_update_packages.py create mode 100755 tools/crashreporter/system-symbols/mac/list-packages.py create mode 100755 tools/crashreporter/system-symbols/mac/run.sh create mode 100644 tools/crashreporter/system-symbols/mac/scrapesymbols/__init__.py create mode 100644 tools/crashreporter/system-symbols/mac/scrapesymbols/gathersymbols.py create mode 100644 tools/crashreporter/system-symbols/win/LICENSE create mode 100644 tools/crashreporter/system-symbols/win/known-microsoft-symbols.txt create mode 100755 tools/crashreporter/system-symbols/win/run.sh create mode 100644 tools/crashreporter/system-symbols/win/scrape-report.py create mode 100644 tools/crashreporter/system-symbols/win/skiplist.txt create mode 100644 tools/crashreporter/system-symbols/win/symsrv-fetch.py create mode 100644 tools/esmify/import-to-import_esmodule.js create mode 100644 tools/esmify/is-esmified.js create mode 100644 tools/esmify/mach_commands.py create mode 100644 tools/esmify/map.json create mode 100644 tools/esmify/package-lock.json create mode 100644 tools/esmify/package.json create mode 100644 tools/esmify/static-import.js create mode 100644 tools/esmify/use-import-export-declarations.js create mode 100644 tools/esmify/utils.js create mode 100644 tools/fuzzing/common/FuzzingMutate.cpp create mode 100644 tools/fuzzing/common/FuzzingMutate.h create mode 100644 tools/fuzzing/common/FuzzingTraits.cpp create mode 100644 tools/fuzzing/common/FuzzingTraits.h create mode 100644 tools/fuzzing/common/moz.build create mode 100644 tools/fuzzing/docs/fuzzing_interface.rst create mode 100644 tools/fuzzing/docs/index.rst create mode 100644 tools/fuzzing/interface/FuzzingInterface.cpp create mode 100644 tools/fuzzing/interface/FuzzingInterface.h create mode 100644 tools/fuzzing/interface/FuzzingInterfaceStream.cpp create mode 100644 tools/fuzzing/interface/FuzzingInterfaceStream.h create mode 100644 tools/fuzzing/interface/harness/FuzzerRunner.cpp create mode 100644 tools/fuzzing/interface/harness/FuzzerRunner.h create mode 100644 tools/fuzzing/interface/harness/FuzzerTestHarness.h create mode 100644 tools/fuzzing/interface/harness/moz.build create mode 100644 tools/fuzzing/interface/moz.build create mode 100644 tools/fuzzing/ipc/IPCFuzzController.cpp create mode 100644 tools/fuzzing/ipc/IPCFuzzController.h create mode 100644 tools/fuzzing/ipc/moz.build create mode 100644 tools/fuzzing/libfuzzer-config.mozbuild create mode 100644 tools/fuzzing/libfuzzer-flags.mozbuild create mode 100644 tools/fuzzing/libfuzzer/FuzzerBuiltins.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerBuiltinsMsvc.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerCommand.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerCorpus.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerCrossOver.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerDefs.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerDictionary.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerDriver.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerExtFunctions.def create mode 100644 tools/fuzzing/libfuzzer/FuzzerExtFunctions.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsym.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerExtFunctionsWindows.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerFlags.def create mode 100644 tools/fuzzing/libfuzzer/FuzzerFork.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerFork.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerIO.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerIO.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerInterceptors.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerInterface.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerInternal.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerLoop.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerMain.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerMerge.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerMerge.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerMutate.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerMutate.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerOptions.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerPlatform.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerRandom.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerSHA1.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerSHA1.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerTracePC.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerTracePC.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerUtil.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerUtil.h create mode 100644 tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerUtilFuchsia.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp create mode 100644 tools/fuzzing/libfuzzer/FuzzerValueBitMap.h create mode 100644 tools/fuzzing/libfuzzer/LICENSE.TXT create mode 100755 tools/fuzzing/libfuzzer/clone_libfuzzer.sh create mode 100644 tools/fuzzing/libfuzzer/moz.build create mode 100644 tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch create mode 100644 tools/fuzzing/libfuzzer/patches/11-callback-rv.patch create mode 100644 tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch create mode 100644 tools/fuzzing/libfuzzer/patches/13-unused-write.patch create mode 100644 tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch create mode 100644 tools/fuzzing/libfuzzer/patches/15-return-to-exit.patch create mode 100644 tools/fuzzing/messagemanager/MessageManagerFuzzer.cpp create mode 100644 tools/fuzzing/messagemanager/MessageManagerFuzzer.h create mode 100644 tools/fuzzing/messagemanager/moz.build create mode 100644 tools/fuzzing/moz.build create mode 100644 tools/fuzzing/nyx/Nyx.cpp create mode 100644 tools/fuzzing/nyx/Nyx.h create mode 100644 tools/fuzzing/nyx/NyxWrapper.h create mode 100644 tools/fuzzing/nyx/moz.build create mode 100644 tools/fuzzing/registry/FuzzerRegistry.cpp create mode 100644 tools/fuzzing/registry/FuzzerRegistry.h create mode 100644 tools/fuzzing/registry/moz.build create mode 100644 tools/fuzzing/rust/Cargo.toml create mode 100644 tools/fuzzing/rust/RustFuzzingTargets.cpp create mode 100644 tools/fuzzing/rust/RustFuzzingTargets.h create mode 100644 tools/fuzzing/rust/moz.build create mode 100644 tools/fuzzing/rust/src/lib.rs create mode 100644 tools/fuzzing/shmem/SharedMemoryFuzzer.cpp create mode 100644 tools/fuzzing/shmem/SharedMemoryFuzzer.h create mode 100644 tools/fuzzing/shmem/moz.build create mode 100644 tools/fuzzing/smoke/grizzly_requirements.in create mode 100644 tools/fuzzing/smoke/grizzly_requirements.txt create mode 100755 tools/fuzzing/smoke/js.py create mode 100644 tools/fuzzing/smoke/python.ini create mode 100644 tools/fuzzing/smoke/smoke.py create mode 100644 tools/fuzzing/smoke/test_grizzly.py create mode 100644 tools/fuzzing/smoke/tests.py create mode 100755 tools/github-sync/converter.py create mode 100755 tools/github-sync/read-json.py create mode 100644 tools/github-sync/readme.md create mode 100755 tools/github-sync/sync-to-github.sh create mode 100644 tools/jprof/README.html create mode 100644 tools/jprof/bfd.cpp create mode 100644 tools/jprof/coff.cpp create mode 100644 tools/jprof/elf.cpp create mode 100644 tools/jprof/intcnt.cpp create mode 100644 tools/jprof/intcnt.h create mode 100755 tools/jprof/jprofsig create mode 100644 tools/jprof/leaky.cpp create mode 100644 tools/jprof/leaky.h create mode 100644 tools/jprof/moz.build create mode 100755 tools/jprof/split-profile.py create mode 100644 tools/jprof/strset.cpp create mode 100644 tools/jprof/strset.h create mode 100644 tools/jprof/stub/Makefile.in create mode 100644 tools/jprof/stub/config.h create mode 100644 tools/jprof/stub/jprof.h create mode 100644 tools/jprof/stub/libmalloc.cpp create mode 100644 tools/jprof/stub/libmalloc.h create mode 100644 tools/jprof/stub/moz.build create mode 100644 tools/leak-gauge/leak-gauge.html create mode 100755 tools/leak-gauge/leak-gauge.pl create mode 100644 tools/lint/android-api-lint.yml create mode 100644 tools/lint/android-checkstyle.yml create mode 100644 tools/lint/android-format.yml create mode 100644 tools/lint/android-javadoc.yml create mode 100644 tools/lint/android-lint.yml create mode 100644 tools/lint/android-test.yml create mode 100644 tools/lint/android/__init__.py create mode 100644 tools/lint/android/lints.py create mode 100644 tools/lint/black.yml create mode 100644 tools/lint/clang-format.yml create mode 100644 tools/lint/clang-format/__init__.py create mode 100644 tools/lint/clippy.yml create mode 100644 tools/lint/clippy/__init__.py create mode 100644 tools/lint/codespell.yml create mode 100644 tools/lint/cpp/__init__.py create mode 100644 tools/lint/cpp/mingw-capitalization.py create mode 100644 tools/lint/cpp/mingw-headers.txt create mode 100644 tools/lint/eslint.yml create mode 100644 tools/lint/eslint/.eslintrc.js create mode 100644 tools/lint/eslint/__init__.py create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/.npmignore create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/LICENSE create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/README.md create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/configs/.eslintrc.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/configs/browser-test.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/configs/chrome-test.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/configs/mochitest-test.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/configs/require-jsdoc.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/configs/valid-jsdoc.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/configs/xpcshell-test.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/browser-window.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-script.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-worker.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/frame-script.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/jsm.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/privileged.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/process-script.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/remote-page.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/simpletest.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/sjs.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/special-powers-sandbox.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/specific.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/utils.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/environments/xpcshell.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/helpers.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/index.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/avoid-Date-timing.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/avoid-removeChild.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-listeners.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-observers.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/consistent-if-bracing.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-browser-window-globals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-content-task-globals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-globals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-headjs-globals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/lazy-getter-object-name.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-exported-symbols-as-used.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-test-function-used.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-aArgs.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-addtask-setup.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-arbitrary-setTimeout.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-compare-against-boolean-literals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-cu-reportError.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-define-cc-etc.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-redeclare-with-import-autofix.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-throw-cr-literal.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-parameters.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-removeEventListener.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-useless-run-test.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/prefer-boolean-length-check.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/prefer-formatValues.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-addtask-only.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-chromeutils-import-params.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-eager-module-in-lazy-getter.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-global-this.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-globalThis-modification.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-import-system-module-from-non-system.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-importGlobalProperties.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-lazy-imports-into-globals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-mixing-eager-and-lazy.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-multiple-getters-calls.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-relative-requires.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-scriptableunicodeconverter.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-some-requires.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-top-level-await.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/rejects-requires-await.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-cc-etc.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-generateqi.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-chromeutils-import.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-default-preference-values.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-includes-instead-of-indexOf.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-isInstance.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-ownerGlobal.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-returnValue.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-services.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/use-static-import.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-ci-uses.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-lazy.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-services-property.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/valid-services.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/rules/var-only-at-top-level.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/lib/services.json create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/manifest.tt create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/package-lock.json create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/package.json create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/reporters/mozilla-format.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/scripts/createExports.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/avoid-Date-timing.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/avoid-removeChild.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/balanced-listeners.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/balanced-observers.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/consistent-if-bracing.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/globals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/lazy-getter-object-name.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/mark-exported-symbols-as-used.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-addtask-setup.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-arbitrary-setTimeout.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-compare-against-boolean-literals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-cu-reportError.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-define-cc-etc.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-redeclare-with-import-autofix.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-throw-cr-literal.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-parameters.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-removeEventListener.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/no-useless-run-test.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/prefer-boolean-length-check.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/prefer-formatValues.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-addtask-only.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-chromeutils-import-params.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-eager-module-in-lazy-getter.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-global-this.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-globalThis-modification.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-import-system-module-from-non-system.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-importGlobalProperties.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-lazy-imports-into-globals.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-mixing-eager-and-lazy.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-multiple-getters-calls.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-relative-requires.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-scriptableunicodeconverter.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-some-requires.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/reject-top-level-await.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/rejects-requires-await.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-cc-etc.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-generateqi.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-chromeutils-import.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-default-preference-values.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-includes-instead-of-indexOf.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-isInstance.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-ownerGlobal.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-returnValue.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-services.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/use-static-import.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/valid-ci-uses.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/valid-lazy.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/valid-services-property.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/valid-services.js create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/xpidl/docshell.xpt create mode 100644 tools/lint/eslint/eslint-plugin-mozilla/tests/xpidl/xpcom_base.xpt create mode 100755 tools/lint/eslint/eslint-plugin-mozilla/update.sh create mode 100644 tools/lint/eslint/eslint-plugin-spidermonkey-js/LICENSE create mode 100644 tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/environments/self-hosted.js create mode 100644 tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/index.js create mode 100644 tools/lint/eslint/eslint-plugin-spidermonkey-js/lib/processors/self-hosted.js create mode 100644 tools/lint/eslint/eslint-plugin-spidermonkey-js/package.json create mode 100644 tools/lint/eslint/manifest.tt create mode 100644 tools/lint/eslint/setup_helper.py create mode 100755 tools/lint/eslint/update.sh create mode 100644 tools/lint/file-perm.yml create mode 100644 tools/lint/file-perm/__init__.py create mode 100644 tools/lint/file-whitespace.yml create mode 100644 tools/lint/file-whitespace/__init__.py create mode 100644 tools/lint/fluent-lint.yml create mode 100644 tools/lint/fluent-lint/__init__.py create mode 100644 tools/lint/fluent-lint/exclusions.yml create mode 100755 tools/lint/hooks.py create mode 100755 tools/lint/hooks_clang_format.py create mode 100755 tools/lint/hooks_js_format.py create mode 100644 tools/lint/l10n.yml create mode 100644 tools/lint/libpref/__init__.py create mode 100644 tools/lint/license.yml create mode 100644 tools/lint/license/__init__.py create mode 100644 tools/lint/license/valid-licenses.txt create mode 100644 tools/lint/lintpref.yml create mode 100644 tools/lint/mach_commands.py create mode 100644 tools/lint/mingw-capitalization.yml create mode 100644 tools/lint/mscom-init.yml create mode 100644 tools/lint/perfdocs.yml create mode 100644 tools/lint/perfdocs/__init__.py create mode 100644 tools/lint/perfdocs/doc_helpers.py create mode 100644 tools/lint/perfdocs/framework_gatherers.py create mode 100644 tools/lint/perfdocs/gatherer.py create mode 100644 tools/lint/perfdocs/generator.py create mode 100644 tools/lint/perfdocs/logger.py create mode 100644 tools/lint/perfdocs/perfdocs.py create mode 100644 tools/lint/perfdocs/templates/index.rst create mode 100644 tools/lint/perfdocs/utils.py create mode 100644 tools/lint/perfdocs/verifier.py create mode 100644 tools/lint/python/__init__.py create mode 100644 tools/lint/python/black.py create mode 100644 tools/lint/python/black_requirements.in create mode 100644 tools/lint/python/black_requirements.txt create mode 100644 tools/lint/python/l10n_lint.py create mode 100644 tools/lint/python/ruff.py create mode 100644 tools/lint/python/ruff_requirements.in create mode 100644 tools/lint/python/ruff_requirements.txt create mode 100644 tools/lint/rejected-words.yml create mode 100644 tools/lint/rst.yml create mode 100644 tools/lint/rst/__init__.py create mode 100644 tools/lint/rst/requirements.in create mode 100644 tools/lint/rst/requirements.txt create mode 100644 tools/lint/ruff.yml create mode 100644 tools/lint/rust/__init__.py create mode 100644 tools/lint/rustfmt.yml create mode 100644 tools/lint/shell/__init__.py create mode 100644 tools/lint/shellcheck.yml create mode 100644 tools/lint/spell/__init__.py create mode 100644 tools/lint/spell/codespell_requirements.in create mode 100644 tools/lint/spell/codespell_requirements.txt create mode 100644 tools/lint/spell/exclude-list.txt create mode 100644 tools/lint/stylelint.yml create mode 100644 tools/lint/stylelint/__init__.py create mode 100644 tools/lint/test-manifest-alpha.yml create mode 100644 tools/lint/test-manifest-alpha/__init__.py create mode 100644 tools/lint/test-manifest-alpha/error-level-manifests.yml create mode 100644 tools/lint/test-manifest-disable.yml create mode 100644 tools/lint/test-manifest-skip-if.yml create mode 100644 tools/lint/test/conftest.py create mode 100644 tools/lint/test/files/android-format/Bad.java create mode 100644 tools/lint/test/files/android-format/Main.kt create mode 100644 tools/lint/test/files/android-format/build.gradle create mode 100644 tools/lint/test/files/black/bad.py create mode 100644 tools/lint/test/files/black/invalid.py create mode 100644 tools/lint/test/files/clang-format/bad/bad.cpp create mode 100644 tools/lint/test/files/clang-format/bad/bad2.c create mode 100644 tools/lint/test/files/clang-format/bad/bad2.h create mode 100644 tools/lint/test/files/clang-format/bad/good.cpp create mode 100644 tools/lint/test/files/clang-format/good/foo.cpp create mode 100644 tools/lint/test/files/clippy/test1/Cargo.toml create mode 100644 tools/lint/test/files/clippy/test1/bad.rs create mode 100644 tools/lint/test/files/clippy/test1/bad2.rs create mode 100644 tools/lint/test/files/clippy/test1/good.rs create mode 100644 tools/lint/test/files/clippy/test2/Cargo.lock create mode 100644 tools/lint/test/files/clippy/test2/Cargo.toml create mode 100644 tools/lint/test/files/clippy/test2/src/bad_1.rs create mode 100644 tools/lint/test/files/clippy/test2/src/bad_2.rs create mode 100644 tools/lint/test/files/codespell/ignore.rst create mode 100644 tools/lint/test/files/eslint/good.js create mode 100644 tools/lint/test/files/eslint/import/bad_import.js create mode 100644 tools/lint/test/files/eslint/nolint/foo.txt create mode 100644 tools/lint/test/files/eslint/subdir/bad.js create mode 100644 tools/lint/test/files/eslint/testprettierignore create mode 100755 tools/lint/test/files/file-perm/maybe-shebang/bad.js create mode 100755 tools/lint/test/files/file-perm/maybe-shebang/good.js create mode 100755 tools/lint/test/files/file-perm/no-shebang/bad-shebang.c create mode 100755 tools/lint/test/files/file-perm/no-shebang/bad.c create mode 100755 tools/lint/test/files/file-perm/no-shebang/bad.png create mode 100644 tools/lint/test/files/file-perm/no-shebang/good.c create mode 100644 tools/lint/test/files/file-whitespace/bad-newline.c create mode 100644 tools/lint/test/files/file-whitespace/bad-windows.c create mode 100644 tools/lint/test/files/file-whitespace/bad.c create mode 100644 tools/lint/test/files/file-whitespace/bad.js create mode 100644 tools/lint/test/files/file-whitespace/good.c create mode 100644 tools/lint/test/files/file-whitespace/good.js create mode 100644 tools/lint/test/files/fluent-lint/bad.ftl create mode 100644 tools/lint/test/files/fluent-lint/brand-names-excluded.ftl create mode 100644 tools/lint/test/files/fluent-lint/brand-names.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-group1.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-group2.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-resource1.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-resource2.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-resource3.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-resource4.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-resource5.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-resource6.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-variables1.ftl create mode 100644 tools/lint/test/files/fluent-lint/comment-variables2.ftl create mode 100644 tools/lint/test/files/fluent-lint/excluded.ftl create mode 100644 tools/lint/test/files/fluent-lint/test-brands.ftl create mode 100644 tools/lint/test/files/fluent-lint/tools/lint/fluent-lint/exclusions.yml create mode 100644 tools/lint/test/files/license/.eslintrc.js create mode 100644 tools/lint/test/files/license/bad.c create mode 100644 tools/lint/test/files/license/bad.js create mode 100644 tools/lint/test/files/license/good-other.h create mode 100644 tools/lint/test/files/license/good.c create mode 100644 tools/lint/test/files/license/good.js create mode 100644 tools/lint/test/files/lintpref/bad.js create mode 100644 tools/lint/test/files/lintpref/good.js create mode 100644 tools/lint/test/files/rst/.dotfile.rst create mode 100644 tools/lint/test/files/rst/bad.rst create mode 100644 tools/lint/test/files/rst/bad2.rst create mode 100644 tools/lint/test/files/rst/bad3.rst create mode 100644 tools/lint/test/files/rst/good.rst create mode 100644 tools/lint/test/files/ruff/bad.py create mode 100644 tools/lint/test/files/ruff/ruff.toml create mode 100644 tools/lint/test/files/rustfmt/subdir/bad.rs create mode 100644 tools/lint/test/files/rustfmt/subdir/bad2.rs create mode 100644 tools/lint/test/files/rustfmt/subdir/good.rs create mode 100644 tools/lint/test/files/shellcheck/bad.sh create mode 100644 tools/lint/test/files/shellcheck/good.sh create mode 100644 tools/lint/test/files/stylelint/nolint/foo.txt create mode 100644 tools/lint/test/files/stylelint/subdir/bad.css create mode 100644 tools/lint/test/files/test-manifest-alpha/mochitest-in-order.ini create mode 100644 tools/lint/test/files/test-manifest-alpha/mochitest-mostly-in-order.ini create mode 100644 tools/lint/test/files/test-manifest-alpha/mochitest-very-out-of-order.ini create mode 100644 tools/lint/test/files/test-manifest-alpha/other-ini-very-out-of-order.ini create mode 100644 tools/lint/test/files/trojan-source/README create mode 100644 tools/lint/test/files/trojan-source/commenting-out.cpp create mode 100644 tools/lint/test/files/trojan-source/early-return.py create mode 100644 tools/lint/test/files/trojan-source/invisible-function.rs create mode 100644 tools/lint/test/files/updatebot/.yamllint create mode 100644 tools/lint/test/files/updatebot/cargo-mismatch.yaml create mode 100644 tools/lint/test/files/updatebot/good1.yaml create mode 100644 tools/lint/test/files/updatebot/good2.yaml create mode 100644 tools/lint/test/files/updatebot/no-revision.yaml create mode 100644 tools/lint/test/files/yaml/.yamllint create mode 100644 tools/lint/test/files/yaml/bad.yml create mode 100644 tools/lint/test/files/yaml/good.yml create mode 100644 tools/lint/test/python.ini create mode 100644 tools/lint/test/test_android_format.py create mode 100644 tools/lint/test/test_black.py create mode 100644 tools/lint/test/test_clang_format.py create mode 100644 tools/lint/test/test_codespell.py create mode 100644 tools/lint/test/test_eslint.py create mode 100644 tools/lint/test/test_file_license.py create mode 100644 tools/lint/test/test_file_perm.py create mode 100644 tools/lint/test/test_file_whitespace.py create mode 100644 tools/lint/test/test_fluent_lint.py create mode 100644 tools/lint/test/test_lintpref.py create mode 100644 tools/lint/test/test_manifest_alpha.py create mode 100644 tools/lint/test/test_perfdocs.py create mode 100644 tools/lint/test/test_perfdocs_generation.py create mode 100644 tools/lint/test/test_perfdocs_helpers.py create mode 100644 tools/lint/test/test_rst.py create mode 100644 tools/lint/test/test_ruff.py create mode 100644 tools/lint/test/test_rustfmt.py create mode 100644 tools/lint/test/test_shellcheck.py create mode 100644 tools/lint/test/test_stylelint.py create mode 100644 tools/lint/test/test_trojan_source.py create mode 100644 tools/lint/test/test_updatebot.py create mode 100644 tools/lint/test/test_yaml.py create mode 100644 tools/lint/tox/tox_requirements.txt create mode 100644 tools/lint/trojan-source.yml create mode 100644 tools/lint/trojan-source/__init__.py create mode 100644 tools/lint/updatebot.yml create mode 100644 tools/lint/updatebot/__init__.py create mode 100644 tools/lint/updatebot/validate_yaml.py create mode 100644 tools/lint/wpt.yml create mode 100644 tools/lint/wpt/__init__.py create mode 100644 tools/lint/wpt/wpt.py create mode 100644 tools/lint/yaml.yml create mode 100644 tools/lint/yamllint_/__init__.py create mode 100644 tools/mach_commands.py create mode 100644 tools/moz.build create mode 100644 tools/moztreedocs/__init__.py create mode 100644 tools/moztreedocs/docs/adding-documentation.rst create mode 100644 tools/moztreedocs/docs/architecture.rst create mode 100644 tools/moztreedocs/docs/index.rst create mode 100644 tools/moztreedocs/docs/jsdoc-support.rst create mode 100644 tools/moztreedocs/docs/mdn-import.rst create mode 100644 tools/moztreedocs/docs/mermaid-integration.rst create mode 100644 tools/moztreedocs/docs/nested-docs.rst create mode 100644 tools/moztreedocs/docs/redirect.rst create mode 100644 tools/moztreedocs/docs/rstlint.rst create mode 100644 tools/moztreedocs/docs/run-try-job.rst create mode 100644 tools/moztreedocs/docs/server-synchronization.rst create mode 100644 tools/moztreedocs/mach_commands.py create mode 100644 tools/moztreedocs/package.py create mode 100644 tools/moztreedocs/upload.py create mode 100644 tools/performance/PerfStats.cpp create mode 100644 tools/performance/PerfStats.h create mode 100644 tools/performance/moz.build create mode 100644 tools/phabricator/mach_commands.py create mode 100644 tools/power/mach_commands.py create mode 100644 tools/power/moz.build create mode 100644 tools/power/rapl.cpp create mode 100644 tools/profiler/core/EHABIStackWalk.cpp create mode 100644 tools/profiler/core/EHABIStackWalk.h create mode 100644 tools/profiler/core/MicroGeckoProfiler.cpp create mode 100644 tools/profiler/core/PageInformation.cpp create mode 100644 tools/profiler/core/PageInformation.h create mode 100644 tools/profiler/core/PlatformMacros.h create mode 100644 tools/profiler/core/PowerCounters-linux.cpp create mode 100644 tools/profiler/core/PowerCounters-mac-amd64.cpp create mode 100644 tools/profiler/core/PowerCounters-mac-arm64.cpp create mode 100644 tools/profiler/core/PowerCounters-win.cpp create mode 100644 tools/profiler/core/PowerCounters.h create mode 100644 tools/profiler/core/ProfileAdditionalInformation.cpp create mode 100644 tools/profiler/core/ProfileBuffer.cpp create mode 100644 tools/profiler/core/ProfileBuffer.h create mode 100644 tools/profiler/core/ProfileBufferEntry.cpp create mode 100644 tools/profiler/core/ProfileBufferEntry.h create mode 100644 tools/profiler/core/ProfiledThreadData.cpp create mode 100644 tools/profiler/core/ProfiledThreadData.h create mode 100644 tools/profiler/core/ProfilerBacktrace.cpp create mode 100644 tools/profiler/core/ProfilerBacktrace.h create mode 100644 tools/profiler/core/ProfilerBindings.cpp create mode 100644 tools/profiler/core/ProfilerCodeAddressService.cpp create mode 100644 tools/profiler/core/ProfilerMarkers.cpp create mode 100644 tools/profiler/core/ProfilerThreadRegistration.cpp create mode 100644 tools/profiler/core/ProfilerThreadRegistrationData.cpp create mode 100644 tools/profiler/core/ProfilerThreadRegistry.cpp create mode 100644 tools/profiler/core/ProfilerUtils.cpp create mode 100644 tools/profiler/core/VTuneProfiler.cpp create mode 100644 tools/profiler/core/VTuneProfiler.h create mode 100644 tools/profiler/core/memory_hooks.cpp create mode 100644 tools/profiler/core/memory_hooks.h create mode 100644 tools/profiler/core/platform-linux-android.cpp create mode 100644 tools/profiler/core/platform-macos.cpp create mode 100644 tools/profiler/core/platform-win32.cpp create mode 100644 tools/profiler/core/platform.cpp create mode 100644 tools/profiler/core/platform.h create mode 100644 tools/profiler/core/shared-libraries-linux.cc create mode 100644 tools/profiler/core/shared-libraries-macos.cc create mode 100644 tools/profiler/core/shared-libraries-win32.cc create mode 100644 tools/profiler/core/vtune/ittnotify.h create mode 100644 tools/profiler/docs/buffer.rst create mode 100644 tools/profiler/docs/code-overview.rst create mode 100644 tools/profiler/docs/fissionprofiler-20200424.png create mode 100644 tools/profiler/docs/fissionprofiler.umlet.uxf create mode 100644 tools/profiler/docs/index.rst create mode 100644 tools/profiler/docs/instrumenting-javascript.rst create mode 100644 tools/profiler/docs/instrumenting-rust.rst create mode 100644 tools/profiler/docs/markers-guide.rst create mode 100644 tools/profiler/docs/memory.rst create mode 100644 tools/profiler/docs/profilerclasses-20220913.png create mode 100644 tools/profiler/docs/profilerclasses.umlet.uxf create mode 100644 tools/profiler/docs/profilerthreadregistration-20220913.png create mode 100644 tools/profiler/docs/profilerthreadregistration.umlet.uxf create mode 100644 tools/profiler/gecko/ChildProfilerController.cpp create mode 100644 tools/profiler/gecko/PProfiler.ipdl create mode 100644 tools/profiler/gecko/ProfilerChild.cpp create mode 100644 tools/profiler/gecko/ProfilerIOInterposeObserver.cpp create mode 100644 tools/profiler/gecko/ProfilerIOInterposeObserver.h create mode 100644 tools/profiler/gecko/ProfilerParent.cpp create mode 100644 tools/profiler/gecko/ProfilerTypes.ipdlh create mode 100644 tools/profiler/gecko/components.conf create mode 100644 tools/profiler/gecko/nsIProfiler.idl create mode 100644 tools/profiler/gecko/nsProfiler.cpp create mode 100644 tools/profiler/gecko/nsProfiler.h create mode 100644 tools/profiler/gecko/nsProfilerCIID.h create mode 100644 tools/profiler/gecko/nsProfilerStartParams.cpp create mode 100644 tools/profiler/gecko/nsProfilerStartParams.h create mode 100644 tools/profiler/lul/AutoObjectMapper.cpp create mode 100644 tools/profiler/lul/AutoObjectMapper.h create mode 100644 tools/profiler/lul/LulCommon.cpp create mode 100644 tools/profiler/lul/LulCommonExt.h create mode 100644 tools/profiler/lul/LulDwarf.cpp create mode 100644 tools/profiler/lul/LulDwarfExt.h create mode 100644 tools/profiler/lul/LulDwarfInt.h create mode 100644 tools/profiler/lul/LulDwarfSummariser.cpp create mode 100644 tools/profiler/lul/LulDwarfSummariser.h create mode 100644 tools/profiler/lul/LulElf.cpp create mode 100644 tools/profiler/lul/LulElfExt.h create mode 100644 tools/profiler/lul/LulElfInt.h create mode 100644 tools/profiler/lul/LulMain.cpp create mode 100644 tools/profiler/lul/LulMain.h create mode 100644 tools/profiler/lul/LulMainInt.h create mode 100644 tools/profiler/lul/platform-linux-lul.cpp create mode 100644 tools/profiler/lul/platform-linux-lul.h create mode 100644 tools/profiler/moz.build create mode 100644 tools/profiler/public/ChildProfilerController.h create mode 100644 tools/profiler/public/GeckoProfiler.h create mode 100644 tools/profiler/public/GeckoProfilerReporter.h create mode 100644 tools/profiler/public/GeckoTraceEvent.h create mode 100644 tools/profiler/public/MicroGeckoProfiler.h create mode 100644 tools/profiler/public/ProfileAdditionalInformation.h create mode 100644 tools/profiler/public/ProfileBufferEntrySerializationGeckoExtensions.h create mode 100644 tools/profiler/public/ProfileJSONWriter.h create mode 100644 tools/profiler/public/ProfilerBindings.h create mode 100644 tools/profiler/public/ProfilerChild.h create mode 100644 tools/profiler/public/ProfilerCodeAddressService.h create mode 100644 tools/profiler/public/ProfilerControl.h create mode 100644 tools/profiler/public/ProfilerCounts.h create mode 100644 tools/profiler/public/ProfilerLabels.h create mode 100644 tools/profiler/public/ProfilerMarkerTypes.h create mode 100644 tools/profiler/public/ProfilerMarkers.h create mode 100644 tools/profiler/public/ProfilerMarkersDetail.h create mode 100644 tools/profiler/public/ProfilerMarkersPrerequisites.h create mode 100644 tools/profiler/public/ProfilerParent.h create mode 100644 tools/profiler/public/ProfilerRunnable.h create mode 100644 tools/profiler/public/ProfilerRustBindings.h create mode 100644 tools/profiler/public/ProfilerState.h create mode 100644 tools/profiler/public/ProfilerThreadPlatformData.h create mode 100644 tools/profiler/public/ProfilerThreadRegistration.h create mode 100644 tools/profiler/public/ProfilerThreadRegistrationData.h create mode 100644 tools/profiler/public/ProfilerThreadRegistrationInfo.h create mode 100644 tools/profiler/public/ProfilerThreadRegistry.h create mode 100644 tools/profiler/public/ProfilerThreadSleep.h create mode 100644 tools/profiler/public/ProfilerThreadState.h create mode 100644 tools/profiler/public/ProfilerUtils.h create mode 100644 tools/profiler/public/shared-libraries.h create mode 100644 tools/profiler/rust-api/Cargo.toml create mode 100644 tools/profiler/rust-api/README.md create mode 100644 tools/profiler/rust-api/build.rs create mode 100644 tools/profiler/rust-api/cbindgen.toml create mode 100644 tools/profiler/rust-api/extra-bindgen-flags.in create mode 100644 tools/profiler/rust-api/macros/Cargo.toml create mode 100644 tools/profiler/rust-api/macros/src/lib.rs create mode 100644 tools/profiler/rust-api/src/gecko_bindings/glue.rs create mode 100644 tools/profiler/rust-api/src/gecko_bindings/mod.rs create mode 100644 tools/profiler/rust-api/src/gecko_bindings/profiling_categories.rs create mode 100644 tools/profiler/rust-api/src/json_writer.rs create mode 100644 tools/profiler/rust-api/src/label.rs create mode 100644 tools/profiler/rust-api/src/lib.rs create mode 100644 tools/profiler/rust-api/src/marker/deserializer_tags_state.rs create mode 100644 tools/profiler/rust-api/src/marker/mod.rs create mode 100644 tools/profiler/rust-api/src/marker/options.rs create mode 100644 tools/profiler/rust-api/src/marker/schema.rs create mode 100644 tools/profiler/rust-api/src/profiler_state.rs create mode 100644 tools/profiler/rust-api/src/thread.rs create mode 100644 tools/profiler/rust-api/src/time.rs create mode 100644 tools/profiler/rust-helper/Cargo.toml create mode 100644 tools/profiler/rust-helper/src/compact_symbol_table.rs create mode 100644 tools/profiler/rust-helper/src/elf.rs create mode 100644 tools/profiler/rust-helper/src/lib.rs create mode 100644 tools/profiler/tests/browser/browser.ini create mode 100644 tools/profiler/tests/browser/browser_test_feature_ipcmessages.js create mode 100644 tools/profiler/tests/browser/browser_test_feature_jsallocations.js create mode 100644 tools/profiler/tests/browser/browser_test_feature_nostacksampling.js create mode 100644 tools/profiler/tests/browser/browser_test_marker_network_cancel.js create mode 100644 tools/profiler/tests/browser/browser_test_marker_network_private_browsing.js create mode 100644 tools/profiler/tests/browser/browser_test_marker_network_redirect.js create mode 100644 tools/profiler/tests/browser/browser_test_marker_network_serviceworker_cache_first.js create mode 100644 tools/profiler/tests/browser/browser_test_marker_network_serviceworker_no_fetch_handler.js create mode 100644 tools/profiler/tests/browser/browser_test_marker_network_serviceworker_no_respondWith_in_fetch_handler.js create mode 100644 tools/profiler/tests/browser/browser_test_marker_network_serviceworker_synthetized_response.js create mode 100644 tools/profiler/tests/browser/browser_test_marker_network_simple.js create mode 100644 tools/profiler/tests/browser/browser_test_marker_network_sts.js create mode 100644 tools/profiler/tests/browser/browser_test_markers_gc_cc.js create mode 100644 tools/profiler/tests/browser/browser_test_markers_parent_process.js create mode 100644 tools/profiler/tests/browser/browser_test_markers_preferencereads.js create mode 100644 tools/profiler/tests/browser/browser_test_profile_capture_by_pid.js create mode 100644 tools/profiler/tests/browser/browser_test_profile_fission.js create mode 100644 tools/profiler/tests/browser/browser_test_profile_multi_frame_page_info.js create mode 100644 tools/profiler/tests/browser/browser_test_profile_single_frame_page_info.js create mode 100644 tools/profiler/tests/browser/browser_test_profile_slow_capture.js create mode 100644 tools/profiler/tests/browser/do_work_500ms.html create mode 100644 tools/profiler/tests/browser/firefox-logo-nightly.svg create mode 100644 tools/profiler/tests/browser/head.js create mode 100644 tools/profiler/tests/browser/multi_frame.html create mode 100644 tools/profiler/tests/browser/page_with_resources.html create mode 100644 tools/profiler/tests/browser/redirect.sjs create mode 100644 tools/profiler/tests/browser/serviceworkers/firefox-logo-nightly.svg create mode 100644 tools/profiler/tests/browser/serviceworkers/serviceworker-utils.js create mode 100644 tools/profiler/tests/browser/serviceworkers/serviceworker_cache_first.js create mode 100644 tools/profiler/tests/browser/serviceworkers/serviceworker_no_fetch_handler.js create mode 100644 tools/profiler/tests/browser/serviceworkers/serviceworker_no_respondWith_in_fetch_handler.js create mode 100644 tools/profiler/tests/browser/serviceworkers/serviceworker_page.html create mode 100644 tools/profiler/tests/browser/serviceworkers/serviceworker_register.html create mode 100644 tools/profiler/tests/browser/serviceworkers/serviceworker_simple.html create mode 100644 tools/profiler/tests/browser/serviceworkers/serviceworker_synthetized_response.js create mode 100644 tools/profiler/tests/browser/simple.html create mode 100644 tools/profiler/tests/browser/single_frame.html create mode 100644 tools/profiler/tests/chrome/chrome.ini create mode 100644 tools/profiler/tests/chrome/profiler_test_utils.js create mode 100644 tools/profiler/tests/chrome/test_profile_worker.html create mode 100644 tools/profiler/tests/chrome/test_profile_worker_bug_1428076.html create mode 100644 tools/profiler/tests/gtest/GeckoProfiler.cpp create mode 100644 tools/profiler/tests/gtest/LulTest.cpp create mode 100644 tools/profiler/tests/gtest/LulTestDwarf.cpp create mode 100644 tools/profiler/tests/gtest/LulTestInfrastructure.cpp create mode 100644 tools/profiler/tests/gtest/LulTestInfrastructure.h create mode 100644 tools/profiler/tests/gtest/ThreadProfileTest.cpp create mode 100644 tools/profiler/tests/gtest/moz.build create mode 100644 tools/profiler/tests/shared-head.js create mode 100644 tools/profiler/tests/xpcshell/head.js create mode 100644 tools/profiler/tests/xpcshell/test_active_configuration.js create mode 100644 tools/profiler/tests/xpcshell/test_addProfilerMarker.js create mode 100644 tools/profiler/tests/xpcshell/test_asm.js create mode 100644 tools/profiler/tests/xpcshell/test_assertion_helper.js create mode 100644 tools/profiler/tests/xpcshell/test_enterjit_osr.js create mode 100644 tools/profiler/tests/xpcshell/test_enterjit_osr_disabling.js create mode 100644 tools/profiler/tests/xpcshell/test_enterjit_osr_enabling.js create mode 100644 tools/profiler/tests/xpcshell/test_feature_fileioall.js create mode 100644 tools/profiler/tests/xpcshell/test_feature_java.js create mode 100644 tools/profiler/tests/xpcshell/test_feature_js.js create mode 100644 tools/profiler/tests/xpcshell/test_feature_mainthreadio.js create mode 100644 tools/profiler/tests/xpcshell/test_feature_nativeallocations.js create mode 100644 tools/profiler/tests/xpcshell/test_feature_stackwalking.js create mode 100644 tools/profiler/tests/xpcshell/test_get_features.js create mode 100644 tools/profiler/tests/xpcshell/test_merged_stacks.js create mode 100644 tools/profiler/tests/xpcshell/test_pause.js create mode 100644 tools/profiler/tests/xpcshell/test_responsiveness.js create mode 100644 tools/profiler/tests/xpcshell/test_run.js create mode 100644 tools/profiler/tests/xpcshell/test_shared_library.js create mode 100644 tools/profiler/tests/xpcshell/test_start.js create mode 100644 tools/profiler/tests/xpcshell/xpcshell.ini create mode 100644 tools/quitter/README.md create mode 100644 tools/quitter/quitter@mozilla.org.xpi create mode 100644 tools/rb/README create mode 100755 tools/rb/filter-log.pl create mode 100755 tools/rb/find-comptr-leakers.pl create mode 100755 tools/rb/find_leakers.py create mode 100755 tools/rb/fix_stacks.py create mode 100755 tools/rb/make-tree.pl create mode 100644 tools/rewriting/Generated.txt create mode 100644 tools/rewriting/ThirdPartyPaths.txt create mode 100644 tools/rusttests/app.mozbuild create mode 100644 tools/rusttests/config/mozconfigs/common create mode 100644 tools/rusttests/config/mozconfigs/linux32/rusttests create mode 100644 tools/rusttests/config/mozconfigs/linux32/rusttests-debug create mode 100644 tools/rusttests/config/mozconfigs/linux64/rusttests create mode 100644 tools/rusttests/config/mozconfigs/linux64/rusttests-debug create mode 100644 tools/rusttests/config/mozconfigs/macosx64/rusttests create mode 100644 tools/rusttests/config/mozconfigs/macosx64/rusttests-debug create mode 100644 tools/rusttests/config/mozconfigs/win32/rusttests create mode 100644 tools/rusttests/config/mozconfigs/win32/rusttests-debug create mode 100644 tools/rusttests/config/mozconfigs/win64/rusttests create mode 100644 tools/rusttests/config/mozconfigs/win64/rusttests-debug create mode 100644 tools/rusttests/config/mozconfigs/windows-common create mode 100644 tools/rusttests/moz.configure create mode 100644 tools/sanitizer/docs/asan.rst create mode 100644 tools/sanitizer/docs/asan_nightly.rst create mode 100644 tools/sanitizer/docs/index.rst create mode 100644 tools/sanitizer/docs/memory_sanitizer.rst create mode 100644 tools/sanitizer/docs/tsan.rst create mode 100644 tools/tryselect/__init__.py create mode 100644 tools/tryselect/cli.py create mode 100644 tools/tryselect/docs/configuration.rst create mode 100644 tools/tryselect/docs/img/add-new-jobs.png create mode 100644 tools/tryselect/docs/img/phab-treeherder-link.png create mode 100644 tools/tryselect/docs/index.rst create mode 100644 tools/tryselect/docs/presets.rst create mode 100644 tools/tryselect/docs/selectors/again.rst create mode 100644 tools/tryselect/docs/selectors/auto.rst create mode 100644 tools/tryselect/docs/selectors/chooser.rst create mode 100644 tools/tryselect/docs/selectors/compare.rst create mode 100644 tools/tryselect/docs/selectors/empty.rst create mode 100644 tools/tryselect/docs/selectors/fuzzy.rst create mode 100644 tools/tryselect/docs/selectors/fzf.png create mode 100644 tools/tryselect/docs/selectors/index.rst create mode 100644 tools/tryselect/docs/selectors/release.rst create mode 100644 tools/tryselect/docs/selectors/scriptworker.rst create mode 100644 tools/tryselect/docs/selectors/syntax.rst create mode 100644 tools/tryselect/docs/tasks.rst create mode 100644 tools/tryselect/mach_commands.py create mode 100644 tools/tryselect/preset.py create mode 100644 tools/tryselect/push.py create mode 100644 tools/tryselect/selectors/__init__.py create mode 100644 tools/tryselect/selectors/again.py create mode 100644 tools/tryselect/selectors/auto.py create mode 100644 tools/tryselect/selectors/chooser/.eslintrc.js create mode 100644 tools/tryselect/selectors/chooser/__init__.py create mode 100644 tools/tryselect/selectors/chooser/app.py create mode 100644 tools/tryselect/selectors/chooser/requirements.txt create mode 100644 tools/tryselect/selectors/chooser/static/filter.js create mode 100644 tools/tryselect/selectors/chooser/static/select.js create mode 100644 tools/tryselect/selectors/chooser/static/style.css create mode 100644 tools/tryselect/selectors/chooser/templates/chooser.html create mode 100644 tools/tryselect/selectors/chooser/templates/close.html create mode 100644 tools/tryselect/selectors/chooser/templates/layout.html create mode 100644 tools/tryselect/selectors/compare.py create mode 100644 tools/tryselect/selectors/coverage.py create mode 100644 tools/tryselect/selectors/empty.py create mode 100644 tools/tryselect/selectors/fuzzy.py create mode 100644 tools/tryselect/selectors/perf.py create mode 100644 tools/tryselect/selectors/perfselector/__init__.py create mode 100644 tools/tryselect/selectors/perfselector/classification.py create mode 100644 tools/tryselect/selectors/perfselector/perfcomparators.py create mode 100644 tools/tryselect/selectors/perfselector/utils.py create mode 100644 tools/tryselect/selectors/preview.py create mode 100644 tools/tryselect/selectors/release.py create mode 100644 tools/tryselect/selectors/scriptworker.py create mode 100644 tools/tryselect/selectors/syntax.py create mode 100644 tools/tryselect/task_config.py create mode 100644 tools/tryselect/tasks.py create mode 100644 tools/tryselect/test/conftest.py create mode 100644 tools/tryselect/test/cram.ini create mode 100644 tools/tryselect/test/python.ini create mode 100644 tools/tryselect/test/setup.sh create mode 100644 tools/tryselect/test/test_again.py create mode 100644 tools/tryselect/test/test_auto.py create mode 100644 tools/tryselect/test/test_auto.t create mode 100644 tools/tryselect/test/test_chooser.py create mode 100644 tools/tryselect/test/test_empty.t create mode 100644 tools/tryselect/test/test_fuzzy.py create mode 100644 tools/tryselect/test/test_fuzzy.t create mode 100644 tools/tryselect/test/test_message.t create mode 100644 tools/tryselect/test/test_mozharness_integration.py create mode 100644 tools/tryselect/test/test_perf.py create mode 100644 tools/tryselect/test/test_perfcomparators.py create mode 100644 tools/tryselect/test/test_preset.t create mode 100644 tools/tryselect/test/test_presets.py create mode 100644 tools/tryselect/test/test_release.py create mode 100644 tools/tryselect/test/test_scriptworker.py create mode 100644 tools/tryselect/test/test_task_configs.py create mode 100644 tools/tryselect/test/test_tasks.py create mode 100644 tools/tryselect/try_presets.yml create mode 100644 tools/tryselect/util/__init__.py create mode 100644 tools/tryselect/util/dicttools.py create mode 100644 tools/tryselect/util/estimates.py create mode 100644 tools/tryselect/util/fzf.py create mode 100644 tools/tryselect/util/manage_estimates.py create mode 100644 tools/tryselect/watchman.json create mode 100644 tools/update-packaging/README create mode 100644 tools/update-packaging/app.mozbuild create mode 100755 tools/update-packaging/common.sh create mode 100755 tools/update-packaging/make_full_update.sh create mode 100755 tools/update-packaging/make_incremental_update.sh create mode 100644 tools/update-packaging/moz.build create mode 100644 tools/update-packaging/moz.configure create mode 100755 tools/update-packaging/test/buildrefmars.sh create mode 100755 tools/update-packaging/test/common.sh create mode 100755 tools/update-packaging/test/diffmar.sh create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/force.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/same.bin create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/application.ini create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/chrome.manifest create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/precomplete create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/readme.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/removed-files create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/removed.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/same.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js create mode 100644 tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt create mode 100644 tools/update-packaging/test/from/application.ini create mode 100644 tools/update-packaging/test/from/chrome.manifest create mode 100644 tools/update-packaging/test/from/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from/force.txt create mode 100644 tools/update-packaging/test/from/precomplete create mode 100644 tools/update-packaging/test/from/readme.txt create mode 100644 tools/update-packaging/test/from/removed-files create mode 100644 tools/update-packaging/test/from/removed.txt create mode 100644 tools/update-packaging/test/from/same.bin create mode 100644 tools/update-packaging/test/from/same.txt create mode 100644 tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from/update-settings.ini create mode 100644 tools/update-packaging/test/from/update.manifest create mode 100644 tools/update-packaging/test/from/{foodir/channel-prefs.js create mode 100644 tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/from/{foodir/force.txt create mode 100644 tools/update-packaging/test/from/{foodir/readme.txt create mode 100644 tools/update-packaging/test/from/{foodir/removed.txt create mode 100644 tools/update-packaging/test/from/{foodir/same.bin create mode 100644 tools/update-packaging/test/from/{foodir/same.txt create mode 100644 tools/update-packaging/test/from/{foodir/update.manifest create mode 100755 tools/update-packaging/test/make_full_update.sh create mode 100755 tools/update-packaging/test/runtests.sh create mode 100644 tools/update-packaging/test/testpatchfile.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/added.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/force.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/same.bin create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/application.ini create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/chrome.manifest create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/precomplete create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/readme.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/removed-files create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/same.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js create mode 100644 tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt create mode 100644 tools/update-packaging/test/to/addFeedPrefs.js create mode 100644 tools/update-packaging/test/to/added.txt create mode 100644 tools/update-packaging/test/to/application.ini create mode 100644 tools/update-packaging/test/to/chrome.manifest create mode 100644 tools/update-packaging/test/to/diff-patch-larger-than-file.bin create mode 100644 tools/update-packaging/test/to/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to/distribution/extensions/added/file.txt create mode 100644 tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to/extensions/added/file.txt create mode 100644 tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to/force.txt create mode 100644 tools/update-packaging/test/to/precomplete create mode 100644 tools/update-packaging/test/to/readme.txt create mode 100644 tools/update-packaging/test/to/removed-files create mode 100644 tools/update-packaging/test/to/same.bin create mode 100644 tools/update-packaging/test/to/same.txt create mode 100644 tools/update-packaging/test/to/searchplugins/added/file.txt create mode 100644 tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to/update-settings.ini create mode 100644 tools/update-packaging/test/to/update.manifest create mode 100644 tools/update-packaging/test/to/{foodir/added.txt create mode 100644 tools/update-packaging/test/to/{foodir/channel-prefs.js create mode 100644 tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt create mode 100644 tools/update-packaging/test/to/{foodir/force.txt create mode 100644 tools/update-packaging/test/to/{foodir/readme.txt create mode 100644 tools/update-packaging/test/to/{foodir/same.bin create mode 100644 tools/update-packaging/test/to/{foodir/same.txt create mode 100644 tools/update-packaging/test/to/{foodir/update.manifest create mode 100644 tools/update-programs/README create mode 100644 tools/update-programs/app.mozbuild create mode 100644 tools/update-programs/confvars.sh create mode 100644 tools/update-programs/moz.configure create mode 100644 tools/update-verify/README.md create mode 100644 tools/update-verify/python/util/__init__.py create mode 100644 tools/update-verify/python/util/commands.py create mode 100644 tools/update-verify/release/common/cached_download.sh create mode 100644 tools/update-verify/release/common/check_updates.sh create mode 100644 tools/update-verify/release/common/download_builds.sh create mode 100644 tools/update-verify/release/common/download_mars.sh create mode 100755 tools/update-verify/release/common/installdmg.ex create mode 100755 tools/update-verify/release/common/unpack-diskimage.sh create mode 100755 tools/update-verify/release/common/unpack.sh create mode 100755 tools/update-verify/release/compare-directories.py create mode 100755 tools/update-verify/release/final-verification.sh create mode 100755 tools/update-verify/release/get-update-xml.sh create mode 100644 tools/update-verify/release/mar_certs/README create mode 100644 tools/update-verify/release/mar_certs/dep1.der create mode 100644 tools/update-verify/release/mar_certs/dep2.der create mode 100644 tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der create mode 100644 tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der create mode 100644 tools/update-verify/release/mar_certs/release_primary.der create mode 100644 tools/update-verify/release/mar_certs/release_secondary.der create mode 100644 tools/update-verify/release/mar_certs/sha1/dep1.der create mode 100644 tools/update-verify/release/mar_certs/sha1/dep2.der create mode 100644 tools/update-verify/release/mar_certs/sha1/release_primary.der create mode 100644 tools/update-verify/release/mar_certs/sha1/release_secondary.der create mode 100644 tools/update-verify/release/mar_certs/xpcshellCertificate.der create mode 100644 tools/update-verify/release/replace-updater-certs.py create mode 100755 tools/update-verify/release/test-mar-url.sh create mode 100755 tools/update-verify/release/updates/verify.sh create mode 100644 tools/update-verify/scripts/async_download.py create mode 100644 tools/update-verify/scripts/chunked-verify.py create mode 100755 tools/update-verify/scripts/chunked-verify.sh create mode 100644 tools/vcs/mach_commands.py (limited to 'tools') diff --git a/tools/bloatview/bloatdiff.pl b/tools/bloatview/bloatdiff.pl new file mode 100755 index 0000000000..8c93ad2b00 --- /dev/null +++ b/tools/bloatview/bloatdiff.pl @@ -0,0 +1,372 @@ +#!/usr/bin/perl -w +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +################################################################################ + +sub usage() { + print < a.out +# **make change** +# firefox-bin -P default resource:///res/bloatcycle.html > b.out +# bloatdiff.pl a.out b.out + +EOUSAGE +} + +$OLDFILE = $ARGV[0]; +$NEWFILE = $ARGV[1]; +#$LABEL = $ARGV[2]; + +if (!$OLDFILE or + ! -e $OLDFILE or + -z $OLDFILE) { + print "\nError: Previous log file not specified, does not exist, or is empty.\n\n"; + &usage(); + exit 1; +} + +if (!$NEWFILE or + ! -e $NEWFILE or + -z $NEWFILE) { + print "\nError: Current log file not specified, does not exist, or is empty.\n\n"; + &usage(); + exit 1; +} + +sub processFile { + my ($filename, $map, $prevMap) = @_; + open(FH, $filename); + while () { + if (m{ + ^\s*(\d+)\s # Line number + ([\w:]+)\s+ # Name + (-?\d+)\s+ # Size + (-?\d+)\s+ # Leaked + (-?\d+)\s+ # Objects Total + (-?\d+)\s+ # Objects Rem + \(\s*(-?[\d.]+)\s+ # Objects Mean + \+/-\s+ + ([\w.]+)\)\s+ # Objects StdDev + (-?\d+)\s+ # Reference Total + (-?\d+)\s+ # Reference Rem + \(\s*(-?[\d.]+)\s+ # Reference Mean + \+/-\s+ + ([\w\.]+)\) # Reference StdDev + }x) { + $$map{$2} = { name => $2, + size => $3, + leaked => $4, + objTotal => $5, + objRem => $6, + objMean => $7, + objStdDev => $8, + refTotal => $9, + refRem => $10, + refMean => $11, + refStdDev => $12, + bloat => $3 * $5 # size * objTotal + }; + } else { +# print "failed to parse: $_\n"; + } + } + close(FH); +} + +%oldMap = (); +processFile($OLDFILE, \%oldMap); + +%newMap = (); +processFile($NEWFILE, \%newMap); + +################################################################################ + +$inf = 9999999.99; + +sub getLeaksDelta { + my ($key) = @_; + my $oldLeaks = $oldMap{$key}{leaked} || 0; + my $newLeaks = $newMap{$key}{leaked}; + my $percentLeaks = 0; + if ($oldLeaks == 0) { + if ($newLeaks != 0) { + # there weren't any leaks before, but now there are! + $percentLeaks = $inf; + } + } + else { + $percentLeaks = ($newLeaks - $oldLeaks) / $oldLeaks * 100; + } + # else we had no record of this class before + return ($newLeaks - $oldLeaks, $percentLeaks); +} + +################################################################################ + +sub getBloatDelta { + my ($key) = @_; + my $newBloat = $newMap{$key}{bloat}; + my $percentBloat = 0; + my $oldSize = $oldMap{$key}{size} || 0; + my $oldTotal = $oldMap{$key}{objTotal} || 0; + my $oldBloat = $oldTotal * $oldSize; + if ($oldBloat == 0) { + if ($newBloat != 0) { + # this class wasn't used before, but now it is + $percentBloat = $inf; + } + } + else { + $percentBloat = ($newBloat - $oldBloat) / $oldBloat * 100; + } + # else we had no record of this class before + return ($newBloat - $oldBloat, $percentBloat); +} + +################################################################################ + +foreach $key (keys %newMap) { + my ($newLeaks, $percentLeaks) = getLeaksDelta($key); + my ($newBloat, $percentBloat) = getBloatDelta($key); + $newMap{$key}{leakDelta} = $newLeaks; + $newMap{$key}{leakPercent} = $percentLeaks; + $newMap{$key}{bloatDelta} = $newBloat; + $newMap{$key}{bloatPercent} = $percentBloat; +} + +################################################################################ + +# Print a value of bytes out in a reasonable +# KB, MB, or GB form. Copied from build-seamonkey-util.pl, sorry. -mcafee +sub PrintSize($) { + + # print a number with 3 significant figures + sub PrintNum($) { + my ($num) = @_; + my $rv; + if ($num < 1) { + $rv = sprintf "%.3f", ($num); + } elsif ($num < 10) { + $rv = sprintf "%.2f", ($num); + } elsif ($num < 100) { + $rv = sprintf "%.1f", ($num); + } else { + $rv = sprintf "%d", ($num); + } + } + + my ($size) = @_; + my $rv; + if ($size > 1000000000) { + $rv = PrintNum($size / 1000000000.0) . "G"; + } elsif ($size > 1000000) { + $rv = PrintNum($size / 1000000.0) . "M"; + } elsif ($size > 1000) { + $rv = PrintNum($size / 1000.0) . "K"; + } else { + $rv = PrintNum($size); + } +} + + +print "Bloat/Leak Delta Report\n"; +print "--------------------------------------------------------------------------------------\n"; +print "Current file: $NEWFILE\n"; +print "Previous file: $OLDFILE\n"; +print "----------------------------------------------leaks------leaks%------bloat------bloat%\n"; + + if (! $newMap{"TOTAL"} or + ! $newMap{"TOTAL"}{bloat}) { + # It's OK if leaked or leakPercent are 0 (in fact, that would be good). + # If bloatPercent is zero, it is also OK, because we may have just had + # two runs exactly the same or with no new bloat. + print "\nError: unable to calculate bloat/leak data.\n"; + print "There is no data present.\n\n"; + print "HINT - Did your test run complete successfully?\n"; + print "HINT - Are you pointing at the right log files?\n\n"; + &usage(); + exit 1; + } + +printf "%-40s %10s %10.2f%% %10s %10.2f%%\n", + ("TOTAL", + $newMap{"TOTAL"}{leaked}, $newMap{"TOTAL"}{leakPercent}, + $newMap{"TOTAL"}{bloat}, $newMap{"TOTAL"}{bloatPercent}); + +################################################################################ + +sub percentStr { + my ($p) = @_; + if ($p == $inf) { + return "-"; + } + else { + return sprintf "%10.2f%%", $p; + } +} + +# NEW LEAKS +@keys = sort { $newMap{$b}{leakPercent} <=> $newMap{$a}{leakPercent} } keys %newMap; +my $needsHeading = 1; +my $total = 0; +foreach $key (@keys) { + my $percentLeaks = $newMap{$key}{leakPercent}; + my $leaks = $newMap{$key}{leaked}; + if ($percentLeaks > 0 && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--NEW-LEAKS-----------------------------------leaks------leaks%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, percentStr($percentLeaks)); + $total += $leaks; + } +} +if (!$needsHeading) { + printf "%-40s %10s\n", ("TOTAL", $total); +} + +# FIXED LEAKS +@keys = sort { $newMap{$b}{leakPercent} <=> $newMap{$a}{leakPercent} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $percentLeaks = $newMap{$key}{leakPercent}; + my $leaks = $newMap{$key}{leaked}; + if ($percentLeaks < 0 && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--FIXED-LEAKS---------------------------------leaks------leaks%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, percentStr($percentLeaks)); + $total += $leaks; + } +} +if (!$needsHeading) { + printf "%-40s %10s\n", ("TOTAL", $total); +} + +# NEW BLOAT +@keys = sort { $newMap{$b}{bloatPercent} <=> $newMap{$a}{bloatPercent} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $percentBloat = $newMap{$key}{bloatPercent}; + my $bloat = $newMap{$key}{bloat}; + if ($percentBloat > 0 && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--NEW-BLOAT-----------------------------------bloat------bloat%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $bloat, percentStr($percentBloat)); + $total += $bloat; + } +} +if (!$needsHeading) { + printf "%-40s %10s\n", ("TOTAL", $total); +} + +# ALL LEAKS +@keys = sort { $newMap{$b}{leaked} <=> $newMap{$a}{leaked} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $leaks = $newMap{$key}{leaked}; + my $percentLeaks = $newMap{$key}{leakPercent}; + if ($leaks > 0) { + if ($needsHeading) { + printf "--ALL-LEAKS-----------------------------------leaks------leaks%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, percentStr($percentLeaks)); + if ($key !~ /TOTAL/) { + $total += $leaks; + } + } +} +if (!$needsHeading) { +# printf "%-40s %10s\n", ("TOTAL", $total); +} + +# ALL BLOAT +@keys = sort { $newMap{$b}{bloat} <=> $newMap{$a}{bloat} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $bloat = $newMap{$key}{bloat}; + my $percentBloat = $newMap{$key}{bloatPercent}; + if ($bloat > 0) { + if ($needsHeading) { + printf "--ALL-BLOAT-----------------------------------bloat------bloat%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $bloat, percentStr($percentBloat)); + if ($key !~ /TOTAL/) { + $total += $bloat; + } + } +} +if (!$needsHeading) { +# printf "%-40s %10s\n", ("TOTAL", $total); +} + +# NEW CLASSES +@keys = sort { $newMap{$b}{bloatDelta} <=> $newMap{$a}{bloatDelta} } keys %newMap; +$needsHeading = 1; +my $ltotal = 0; +my $btotal = 0; +foreach $key (@keys) { + my $leaks = $newMap{$key}{leaked}; + my $bloat = $newMap{$key}{bloat}; + my $percentBloat = $newMap{$key}{bloatPercent}; + if ($percentBloat == $inf && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--CLASSES-NOT-REPORTED-LAST-TIME--------------leaks------bloat------------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, $bloat); + if ($key !~ /TOTAL/) { + $ltotal += $leaks; + $btotal += $bloat; + } + } +} +if (!$needsHeading) { + printf "%-40s %10s %10s\n", ("TOTAL", $ltotal, $btotal); +} + +# OLD CLASSES +@keys = sort { ($oldMap{$b}{bloat} || 0) <=> ($oldMap{$a}{bloat} || 0) } keys %oldMap; +$needsHeading = 1; +$ltotal = 0; +$btotal = 0; +foreach $key (@keys) { + if (!defined($newMap{$key})) { + my $leaks = $oldMap{$key}{leaked}; + my $bloat = $oldMap{$key}{bloat}; + if ($needsHeading) { + printf "--CLASSES-THAT-WENT-AWAY----------------------leaks------bloat------------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, $bloat); + if ($key !~ /TOTAL/) { + $ltotal += $leaks; + $btotal += $bloat; + } + } +} +if (!$needsHeading) { + printf "%-40s %10s %10s\n", ("TOTAL", $ltotal, $btotal); +} + +print "--------------------------------------------------------------------------------------\n"; diff --git a/tools/bloatview/bloattable.pl b/tools/bloatview/bloattable.pl new file mode 100755 index 0000000000..e8acfabed3 --- /dev/null +++ b/tools/bloatview/bloattable.pl @@ -0,0 +1,590 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# bloattable [-debug] [-source] [-byte n|-obj n|-ref n] ... > +# +# file1, file2, ... filen should be successive BloatView files generated from the same run. +# Summarize them in an HTML table. Output the HTML to the standard output. +# +# If -debug is set, create a slightly larger html file which is more suitable for debugging this script. +# If -source is set, create an html file that prints the html source as the output +# If -byte n, -obj n, or -ref n is given, make the page default to showing byte, object, or reference statistics, +# respectively, and sort by the nth column (n is zero-based, so the first column has n==0). +# +# See http://lxr.mozilla.org/mozilla/source/xpcom/doc/MemoryTools.html + +use 5.004; +use strict; +use diagnostics; +use File::Basename; +use Getopt::Long; + +# The generated HTML is almost entirely generated by a script. Only the , , and elements are explicit +# because a \n"; + print "\n\n"; + print "\n"; + if ($source) { + print "

"; + print quoteHTML "\n"; + print quoteHTML "\n"; + print quoteHTML "\n"; + print "\n"; + print quoteHTML "\n\n"; + print quoteHTML "\n"; + print "\n"; + print quoteHTML "\n"; + print quoteHTML "\n"; + print "

\n"; + } else { + print "\n"; + } + print "\n"; + print "\n"; +} + + + +# Read the bloat file into hash table $h. The hash table is indexed by class names; +# each entry is a list with the following elements: +# bytesAlloc Total number of bytes allocated +# bytesNet Total number of bytes allocated but not deallocated +# objectsAlloc Total number of objects allocated +# objectsNet Total number of objects allocated but not deallocated +# refsAlloc Total number of references AddRef'd +# refsNet Total number of references AddRef'd but not Released +# Except for TOTAL, all hash table entries refer to mutually exclusive data. +# $sizes is a hash table indexed by class names. Each entry of that table contains the class's instance size. +sub readBloatFile($\%\%) { + my ($file, $h, $sizes) = @_; + local $_; # Needed for 'while ()' below. + + my $readSomething = 0; + open FILE, $file; + while () { + if (my ($name, $size, $bytesNet, $objectsAlloc, $objectsNet, $refsAlloc, $refsNet) = + /^\s*(?:\d+)\s+([\w:]+)\s+(\d+)\s+(-?\d+)\s+(\d+)\s+(-?\d+)\s*\([^()]*\)\s*(\d+)\s+(-?\d+)\s*\([^()]*\)\s*$/) { + my $bytesAlloc; + if ($name eq "TOTAL") { + $size = "undefined"; + $bytesAlloc = "undefined"; + } else { + $bytesAlloc = $objectsAlloc * $size; + if ($bytesNet != $objectsNet * $size) { + print STDERR "In '$file', class $name bytesNet != objectsNet * size: $bytesNet != $objectsNet * $size\n"; + } + } + print STDERR "Duplicate entry $name in '$file'\n" if $$h{$name}; + $$h{$name} = [$bytesAlloc, $bytesNet, $objectsAlloc, $objectsNet, $refsAlloc, $refsNet]; + + my $oldSize = $$sizes{$name}; + print STDERR "Mismatch of sizes of class $name: $oldSize and $size\n" if defined($oldSize) && $size ne $oldSize; + $$sizes{$name} = $size; + $readSomething = 1; + } elsif (/^\s*(?:\d+)\s+([\w:]+)\s/) { + print STDERR "Unable to parse '$file' line: $_"; + } + } + close FILE; + print STDERR "No data in '$file'\n" unless $readSomething; + return $h; +} + + +my %sizes; # => +my %tables; # => ; see readBloatFile for format of + +# Generate the JavaScript source code for the row named $c. $l can contain the initial entries of the row. +sub genTableRowSource($$) { + my ($l, $c) = @_; + my $lastE; + foreach (@ARGV) { + my $e = $tables{$_}{$c}; + if (defined($lastE) && !defined($e)) { + $e = [0,0,0,0,0,0]; + print STDERR "Class $c is defined in an earlier file but not in '$_'\n"; + } + if (defined $e) { + if (defined $lastE) { + for (my $i = 0; $i <= $#$e; $i++) { + my $n = $$e[$i]; + $l .= ($n eq "undefined" ? "undefined" : $n - $$lastE[$i]) . ","; + } + $l .= " "; + } else { + $l .= join(",", @$e) . ", "; + } + $lastE = $e; + } else { + $l .= "0,0,0,0,0,0, "; + } + } + $l .= join(",", @$lastE); + return "[$l]"; +} + + + +my $debug; +my $source; +my $showMode; +my $sortColumn; +my @modeOptions; + +GetOptions("debug" => \$debug, "source" => \$source, "byte=i" => \$modeOptions[0], "obj=i" => \$modeOptions[1], "ref=i" => \$modeOptions[2]); +for (my $i = 0; $i != 3; $i++) { + my $modeOption = $modeOptions[$i]; + if ($modeOption) { + die "Only one of -byte, -obj, or -ref may be given" if defined $showMode; + my $nFileColumns = scalar(@ARGV) + 1; + die "-byte, -obj, or -ref column number out of range" if $modeOption < 0 || $modeOption >= 2 + 2*$nFileColumns; + $showMode = $i; + if ($modeOption >= 2) { + $modeOption -= 2; + $sortColumn = 2 + $showMode*2; + if ($modeOption >= $nFileColumns) { + $sortColumn++; + $modeOption -= $nFileColumns; + } + $sortColumn += $modeOption*6; + } else { + $sortColumn = $modeOption; + } + } +} +unless (defined $showMode) { + $showMode = 0; + $sortColumn = 0; +} + +# Read all of the bloat files. +foreach (@ARGV) { + unless ($tables{$_}) { + my $f = $_; + my %table; + + readBloatFile $_, %table, %sizes; + $tables{$_} = \%table; + } +} +die "No input" unless %sizes; + +my @scriptData; # JavaScript source for the tables passed to JavaScript. Each entry is one line of JavaScript. +my @persistentScriptData; # Same as @scriptData, but persists the page reloads itself. + +# Print a list of bloat file names. +push @persistentScriptData, "var nFiles = " . scalar(@ARGV) . ";"; +push @persistentScriptData, "var fileTags = [" . join(", ", map {singleQuoteString substr(fileCoreName($_), -10)} @ARGV) . "];"; +push @persistentScriptData, "var fileNames = [" . join(", ", map {singleQuoteString $_} @ARGV) . "];"; +push @persistentScriptData, "var fileDates = [" . join(", ", map {singleQuoteString localtime fileModDate $_} @ARGV) . "];"; + +# Print the bloat tables. +push @persistentScriptData, "var totals = " . genTableRowSource('"TOTAL", undefined, ', "TOTAL") . ";"; +push @scriptData, "var classTables = ["; +delete $sizes{"TOTAL"}; +my @classes = sort(keys %sizes); +for (my $i = 0; $i <= $#classes; $i++) { + my $c = $classes[$i]; + push @scriptData, genTableRowSource(doubleQuoteString($c).", ".$sizes{$c}.", ", $c) . ($i == $#classes ? "];" : ","); +} + +generate(@scriptData, @persistentScriptData, $debug, $source, $showMode, $sortColumn); +1; + + +# The source of the eval'd JavaScript follows. +# Comments starting with // that are alone on a line are stripped by the Perl script. +__END__ + +// showMode: 0=bytes, 1=objects, 2=references +var showMode; +var modeName; +var modeNameUpper; + +var sortColumn; + +// Sort according to the sortColumn. Column 0 is sorted alphabetically in ascending order. +// All other columns are sorted numerically in descending order, with column 0 used for a secondary sort. +// Undefined is always listed last. +function sortCompare(x, y) { + if (sortColumn) { + var xc = x[sortColumn]; + var yc = y[sortColumn]; + if (xc < yc || xc === undefined && yc !== undefined) return 1; + if (yc < xc || yc === undefined && xc !== undefined) return -1; + } + + var x0 = x[0]; + var y0 = y[0]; + if (x0 > y0 || x0 === undefined && y0 !== undefined) return 1; + if (y0 > x0 || y0 === undefined && x0 !== undefined) return -1; + return 0; +} + + +// Quote special HTML characters in the string. +function quoteHTML(s) { + s = s.replace(/&/g, '&'); + // Can't use //g, '>'); + s = s.replace(/ /g, ' '); + return s; +} + + +function writeFileTable(d) { + d.writeln(''); + d.writeln('\n\n\n\n'); + for (var i = 0; i < nFiles; i++) + d.writeln('\n\n\n\n'); + d.writeln('
NameFileDate
'+quoteHTML(fileTags[i])+''+quoteHTML(fileNames[i])+''+quoteHTML(fileDates[i])+'
'); +} + + +function writeReloadLink(d, column, s, rowspan) { + d.write(rowspan == 1 ? '' : ''); + if (column != sortColumn) + d.write(''); + d.write(s); + if (column != sortColumn) + d.write(''); + d.writeln(''); +} + +function writeClassTableRow(d, row, base, modeName) { + if (modeName) { + d.writeln('\n'+modeName+''); + } else { + d.writeln('\n'+quoteHTML(row[0])+''); + var v = row[1]; + d.writeln(''+(v === undefined ? '' : v)+''); + } + for (var i = 0; i != 2; i++) { + var c = base + i; + for (var j = 0; j <= nFiles; j++) { + v = row[c]; + var style = 'num'; + if (j != nFiles) + if (v > 0) { + style = 'pos'; + v = '+'+v; + } else + style = 'neg'; + d.writeln(''+(v === undefined ? '' : v)+''); + c += 6; + } + } + d.writeln(''); +} + +function writeClassTable(d) { + var base = 2 + showMode*2; + + // Make a copy because a sort is destructive. + var table = classTables.concat(); + table.sort(sortCompare); + + d.writeln(''); + + d.writeln(''); + writeReloadLink(d, 0, 'Class Name', 2); + writeReloadLink(d, 1, 'Instance
Size', 2); + d.writeln(''); + d.writeln('\n'); + d.writeln(''); + for (var i = 0; i != 2; i++) { + var c = base + i; + for (var j = 0; j <= nFiles; j++) { + writeReloadLink(d, c, j == nFiles ? 'Total' : quoteHTML(fileTags[j]), 1); + c += 6; + } + } + d.writeln(''); + + writeClassTableRow(d, totals, base, 0); + for (var r = 0; r < table.length; r++) + writeClassTableRow(d, table[r], base, 0); + + d.writeln('
'+modeNameUpper+'s allocated'+modeNameUpper+'s allocated but not freed
'); +} + + +var modeNames = ["byte", "object", "reference"]; +var modeNamesUpper = ["Byte", "Object", "Reference"]; +var styleSheet = ''; + + +function showHead(d) { + modeName = modeNames[showMode]; + modeNameUpper = modeNamesUpper[showMode]; + d.writeln(''+modeNameUpper+' Bloats'); + d.writeln(styleSheet); +} + +function showBody(d) { + d.writeln('

'+modeNameUpper+' Bloats

'); + writeFileTable(d); + d.write('
'); + for (var i = 0; i != 3; i++) + if (i != showMode) { + var newSortColumn = sortColumn; + if (sortColumn >= 2) + newSortColumn = sortColumn + (i-showMode)*2; + d.write(''); + } + d.writeln('
'); + d.writeln('

The numbers do not include malloc\'d data such as string contents.

'); + d.writeln('

Click on a column heading to sort by that column. Click on a class name to see details for that class.

'); + writeClassTable(d); +} + + +function showRowDetail(rowName) { + var row; + var i; + + if (rowName == "TOTAL") + row = totals; + else { + for (i = 0; i < classTables.length; i++) + if (rowName == classTables[i][0]) { + row = classTables[i]; + break; + } + } + if (row) { + var w = window.open("", "ClassTableRowDetails"); + var d = w.document; + d.open(); + d.writeln(''); + d.writeln('\n\n'+quoteHTML(rowName)+' bloat details'); + d.writeln(styleSheet); + d.writeln('\n\n'); + d.writeln('

'+quoteHTML(rowName)+'

'); + if (row[1] !== undefined) + d.writeln('

Each instance has '+row[1]+' bytes.

'); + + d.writeln(''); + d.writeln('\n\n'); + d.writeln('\n'); + d.writeln('\n'); + for (i = 0; i != 2; i++) + for (var j = 0; j <= nFiles; j++) + d.writeln(''); + d.writeln(''); + + for (i = 0; i != 3; i++) + writeClassTableRow(d, row, 2+i*2, modeNamesUpper[i]+'s'); + + d.writeln('
AllocatedAllocated but not freed
'+(j == nFiles ? 'Total' : quoteHTML(fileTags[j]))+'
\n\n'); + d.close(); + } + return undefined; +} + + +function stringSource(s) { + s = s.replace(/\\/g, '\\\\'); + s = s.replace(/"/g, '\\"'); + s = s.replace(/<\//g, '<\\/'); + return '"'+s+'"'; +} + +function reloadSelf(n,m) { + // Need to cache these because globals go away on document.open(). + var sa = srcArray; + var ss = stringSource; + var ct = classTables; + var i; + + document.open(); + // Uncomment this and comment the document.open() line above to see the reloaded page's source. + //var w = window.open("", "NewDoc"); + //var d = w.document; + //var document = new Object; + //document.write = function () { + // for (var i = 0; i < arguments.length; i++) { + // var s = arguments[i].toString(); + // s = s.replace(/&/g, '&'); + // s = s.replace(/\x3C/g, '<'); + // s = s.replace(/>/g, '>'); + // s = s.replace(/ /g, ' '); + // d.write(s); + // } + //}; + //document.writeln = function () { + // for (var i = 0; i < arguments.length; i++) { + // var s = arguments[i].toString(); + // s = s.replace(/&/g, '&'); + // s = s.replace(/\x3C/g, '<'); + // s = s.replace(/>/g, '>'); + // s = s.replace(/ /g, ' '); + // d.write(s); + // } + // d.writeln('
'); + //}; + + document.writeln(''); + document.writeln('\n\n\n\n\n\n\n\n'); + document.close(); + return undefined; +} diff --git a/tools/browsertime/README.md b/tools/browsertime/README.md new file mode 100644 index 0000000000..bd76cf65be --- /dev/null +++ b/tools/browsertime/README.md @@ -0,0 +1,15 @@ +=========== +Browsertime +=========== + +This code is obtained from `https://github.com/sitespeedio/browsertime` and all +changes to the code must be made there. It's installed through `./mach browsertime --setup` +and it's used when running `./mach browsertime` or through raptor with +`./mach raptor --browsertime`. + +You can update this code by running `./mach browsertime --update-upstream-url `, where +URL is a link to a github tarball of a particular commit in the aforementioned repository. + +For more information, you can consult the wiki `https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Browsertime`. +If you have any questions about this code, or browsertime, you can find us in +`#browsertime`, or `#perftest` in Riot. diff --git a/tools/browsertime/mach_commands.py b/tools/browsertime/mach_commands.py new file mode 100644 index 0000000000..4850c82b75 --- /dev/null +++ b/tools/browsertime/mach_commands.py @@ -0,0 +1,689 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +r"""Make it easy to install and run [browsertime](https://github.com/sitespeedio/browsertime). + +Browsertime is a harness for running performance tests, similar to +Mozilla's Raptor testing framework. Browsertime is written in Node.js +and uses Selenium WebDriver to drive multiple browsers including +Chrome, Chrome for Android, Firefox, and (pending the resolution of +[Bug 1525126](https://bugzilla.mozilla.org/show_bug.cgi?id=1525126) +and similar tickets) Firefox for Android and GeckoView-based vehicles. + +Right now a custom version of browsertime and the underlying +geckodriver binary are needed to support GeckoView-based vehicles; +this module accommodates those in-progress custom versions. + +To get started, run +``` +./mach browsertime --setup [--clobber] +``` +This will populate `tools/browsertime/node_modules`. + +To invoke browsertime, run +``` +./mach browsertime [ARGS] +``` +All arguments are passed through to browsertime. +""" + +import argparse +import collections +import contextlib +import json +import logging +import os +import platform +import re +import stat +import subprocess +import sys +import time + +import mozpack.path as mozpath +from mach.decorators import Command, CommandArgument +from mozbuild.base import BinaryNotFoundException, MachCommandBase +from mozbuild.util import mkdir +from six import StringIO + +AUTOMATION = "MOZ_AUTOMATION" in os.environ +BROWSERTIME_ROOT = os.path.dirname(__file__) + +PILLOW_VERSION = "8.4.0" # version 8.4.0 currently supports python 3.6 to 3.10 +PYSSIM_VERSION = "0.4" +SCIPY_VERSION = "1.2.3" +NUMPY_VERSION = "1.16.1" +OPENCV_VERSION = "4.5.4.60" + +py3_minor = sys.version_info.minor +if py3_minor > 7: + SCIPY_VERSION = "1.7.3" + NUMPY_VERSION = "1.22.0" + PILLOW_VERSION = "9.0.0" + +MIN_NODE_VERSION = "16.0.0" + +IS_APPLE_SILICON = sys.platform.startswith( + "darwin" +) and platform.processor().startswith("arm") + + +@contextlib.contextmanager +def silence(): + oldout, olderr = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = StringIO(), StringIO() + yield + finally: + sys.stdout, sys.stderr = oldout, olderr + + +def node_path(command_context): + import platform + from distutils.version import StrictVersion + + from mozbuild.nodeutil import find_node_executable + + state_dir = command_context._mach_context.state_dir + cache_path = os.path.join(state_dir, "browsertime", "node-16") + + NODE_FAILURE_MSG = ( + "Could not locate a node binary that is at least version {}. ".format( + MIN_NODE_VERSION + ) + + "Please run `./mach raptor --browsertime -t amazon` to install it " + + "from the Taskcluster Toolchain artifacts." + ) + + # Check standard locations first + node_exe = find_node_executable(min_version=StrictVersion(MIN_NODE_VERSION)) + if node_exe and (node_exe[0] is not None): + return os.path.abspath(node_exe[0]) + if not os.path.exists(cache_path): + raise Exception(NODE_FAILURE_MSG) + + # Check the browsertime-specific node location next + node_name = "node" + if platform.system() == "Windows": + node_name = "node.exe" + node_exe_path = os.path.join( + state_dir, + "browsertime", + "node-16", + "node", + ) + else: + node_exe_path = os.path.join( + state_dir, + "browsertime", + "node-16", + "node", + "bin", + ) + + node_exe = os.path.join(node_exe_path, node_name) + if not os.path.exists(node_exe): + raise Exception(NODE_FAILURE_MSG) + + return os.path.abspath(node_exe) + + +def package_path(): + """The path to the `browsertime` directory. + + Override the default with the `BROWSERTIME` environment variable.""" + override = os.environ.get("BROWSERTIME", None) + if override: + return override + + return mozpath.join(BROWSERTIME_ROOT, "node_modules", "browsertime") + + +def browsertime_path(): + """The path to the `browsertime.js` script.""" + # On Windows, invoking `node_modules/.bin/browsertime{.cmd}` + # doesn't work when invoked as an argument to our specific + # binary. Since we want our version of node, invoke the + # actual script directly. + return mozpath.join(package_path(), "bin", "browsertime.js") + + +def visualmetrics_path(): + """The path to the `visualmetrics.py` script.""" + return mozpath.join(package_path(), "browsertime", "visualmetrics-portable.py") + + +def host_platform(): + is_64bits = sys.maxsize > 2 ** 32 + + if sys.platform.startswith("win"): + if is_64bits: + return "win64" + elif sys.platform.startswith("linux"): + if is_64bits: + return "linux64" + elif sys.platform.startswith("darwin"): + return "darwin" + + raise ValueError("sys.platform is not yet supported: {}".format(sys.platform)) + + +# Map from `host_platform()` to a `fetch`-like syntax. +host_fetches = { + "darwin": { + "ffmpeg": { + "type": "static-url", + "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-macos.zip", # noqa + # An extension to `fetch` syntax. + "path": "ffmpeg-macos", + }, + }, + "linux64": { + "ffmpeg": { + "type": "static-url", + "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-i686-static.tar.xz", # noqa + # An extension to `fetch` syntax. + "path": "ffmpeg-4.4.1-i686-static", + }, + }, + "win64": { + "ffmpeg": { + "type": "static-url", + "url": "https://github.com/mozilla/perf-automation/releases/download/FFMPEG-v4.4.1/ffmpeg-4.4.1-full_build.zip", # noqa + # An extension to `fetch` syntax. + "path": "ffmpeg-4.4.1-full_build", + }, + }, +} + + +def artifact_cache_path(command_context): + r"""Downloaded artifacts will be kept here.""" + # The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE. + return mozpath.join(command_context._mach_context.state_dir, "cache", "browsertime") + + +def state_path(command_context): + r"""Unpacked artifacts will be kept here.""" + # The convention is $MOZBUILD_STATE_PATH/$FEATURE. + return mozpath.join(command_context._mach_context.state_dir, "browsertime") + + +def setup_prerequisites(command_context): + r"""Install browsertime and visualmetrics.py prerequisites.""" + + from mozbuild.action.tooltool import unpack_file + from mozbuild.artifact_cache import ArtifactCache + + # Download the visualmetrics-portable.py requirements. + artifact_cache = ArtifactCache( + artifact_cache_path(command_context), + log=command_context.log, + skip_cache=False, + ) + + fetches = host_fetches[host_platform()] + for tool, fetch in sorted(fetches.items()): + archive = artifact_cache.fetch(fetch["url"]) + # TODO: assert type, verify sha256 (and size?). + + if fetch.get("unpack", True): + cwd = os.getcwd() + try: + mkdir(state_path(command_context)) + os.chdir(state_path(command_context)) + command_context.log( + logging.INFO, + "browsertime", + {"path": archive}, + "Unpacking temporary location {path}", + ) + + unpack_file(archive) + + # Make sure the expected path exists after extraction + path = os.path.join(state_path(command_context), fetch.get("path")) + if not os.path.exists(path): + raise Exception("Cannot find an extracted directory: %s" % path) + + try: + # Some archives provide binaries that don't have the + # executable bit set so we need to set it here + for root, dirs, files in os.walk(path): + for edir in dirs: + loc_to_change = os.path.join(root, edir) + st = os.stat(loc_to_change) + os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC) + for efile in files: + loc_to_change = os.path.join(root, efile) + st = os.stat(loc_to_change) + os.chmod(loc_to_change, st.st_mode | stat.S_IEXEC) + except Exception as e: + raise Exception( + "Could not set executable bit in %s, error: %s" % (path, str(e)) + ) + finally: + os.chdir(cwd) + + +def setup_browsertime( + command_context, + should_clobber=False, + new_upstream_url="", + install_vismet_reqs=False, +): + r"""Install browsertime and visualmetrics.py prerequisites and the Node.js package.""" + + sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint")) + import setup_helper + + if not new_upstream_url: + setup_prerequisites(command_context) + + if new_upstream_url: + package_json_path = os.path.join(BROWSERTIME_ROOT, "package.json") + + command_context.log( + logging.INFO, + "browsertime", + { + "new_upstream_url": new_upstream_url, + "package_json_path": package_json_path, + }, + "Updating browsertime node module version in {package_json_path} " + "to {new_upstream_url}", + ) + + if not re.search("/tarball/[a-f0-9]{40}$", new_upstream_url): + raise ValueError( + "New upstream URL does not end with /tarball/[a-f0-9]{40}: '%s'" + % new_upstream_url + ) + + with open(package_json_path) as f: + existing_body = json.loads( + f.read(), object_pairs_hook=collections.OrderedDict + ) + + existing_body["devDependencies"]["browsertime"] = new_upstream_url + + updated_body = json.dumps(existing_body) + + with open(package_json_path, "w") as f: + f.write(updated_body) + + # Install the browsertime Node.js requirements. + if not setup_helper.check_node_executables_valid(): + return 1 + + # To use a custom `geckodriver`, set + # os.environ[b"GECKODRIVER_BASE_URL"] = bytes(url) + # to an endpoint with binaries named like + # https://github.com/sitespeedio/geckodriver/blob/master/install.js#L31. + if AUTOMATION: + os.environ["CHROMEDRIVER_SKIP_DOWNLOAD"] = "true" + os.environ["GECKODRIVER_SKIP_DOWNLOAD"] = "true" + + if install_vismet_reqs: + # Hide this behind a flag so we don't install them by default in Raptor + command_context.log( + logging.INFO, "browsertime", {}, "Installing python requirements" + ) + activate_browsertime_virtualenv(command_context) + + command_context.log( + logging.INFO, + "browsertime", + {"package_json": mozpath.join(BROWSERTIME_ROOT, "package.json")}, + "Installing browsertime node module from {package_json}", + ) + + # Add the mozbuild Node binary path to the OS environment in Apple Silicon. + # During the browesertime installation, it seems installation of sitespeed.io + # sub dependencies look for a global Node rather than the mozbuild Node binary. + # Normally `--scripts-prepend-node-path` should prevent this but it seems to still + # look for a system Node in the environment. This logic ensures the same Node is used. + node_dir = os.path.dirname(node_path(command_context)) + if IS_APPLE_SILICON and node_dir not in os.environ["PATH"]: + os.environ["PATH"] += os.pathsep + node_dir + + status = setup_helper.package_setup( + BROWSERTIME_ROOT, + "browsertime", + should_update=new_upstream_url != "", + should_clobber=should_clobber, + no_optional=new_upstream_url or AUTOMATION, + ) + + if status: + return status + if new_upstream_url or AUTOMATION: + return 0 + if install_vismet_reqs: + return check(command_context) + + return 0 + + +def node(command_context, args): + r"""Invoke node (interactively) with the given arguments.""" + return command_context.run_process( + [node_path(command_context)] + args, + append_env=append_env(command_context), + pass_thru=True, # Allow user to run Node interactively. + ensure_exit_code=False, # Don't throw on non-zero exit code. + cwd=mozpath.join(command_context.topsrcdir), + ) + + +def append_env(command_context, append_path=True): + fetches = host_fetches[host_platform()] + + # Ensure that `ffmpeg` is found and added to the environment + path = os.environ.get("PATH", "").split(os.pathsep) if append_path else [] + path_to_ffmpeg = mozpath.join( + state_path(command_context), fetches["ffmpeg"]["path"] + ) + + path.insert( + 0, + path_to_ffmpeg + if host_platform().startswith("linux") + else mozpath.join(path_to_ffmpeg, "bin"), + ) # noqa + + # Ensure that bare `node` and `npm` in scripts, including post-install + # scripts, finds the binary we're invoking with. Without this, it's + # easy for compiled extensions to get mismatched versions of the Node.js + # extension API. + node_dir = os.path.dirname(node_path(command_context)) + path = [node_dir] + path + + append_env = { + "PATH": os.pathsep.join(path), + # Bug 1560193: The JS library browsertime uses to execute commands + # (execa) will muck up the PATH variable and put the directory that + # node is in first in path. If this is globally-installed node, + # that means `/usr/bin` will be inserted first which means that we + # will get `/usr/bin/python` for `python`. + # + # Our fork of browsertime supports a `PYTHON` environment variable + # that points to the exact python executable to use. + "PYTHON": command_context.virtualenv_manager.python_path, + } + + return append_env + + +def _need_install(command_context, package): + from pip._internal.req.constructors import install_req_from_line + + req = install_req_from_line(package) + req.check_if_exists(use_user_site=False) + if req.satisfied_by is None: + return True + venv_site_lib = os.path.abspath( + os.path.join(command_context.virtualenv_manager.bin_path, "..", "lib") + ) + site_packages = os.path.abspath(req.satisfied_by.location) + return not site_packages.startswith(venv_site_lib) + + +def activate_browsertime_virtualenv(command_context, *args, **kwargs): + r"""Activates virtualenv. + + This function will also install Pillow and pyssim if needed. + It will raise an error in case the install failed. + """ + # TODO: Remove `./mach browsertime` completely, as a follow up to Bug 1758990 + MachCommandBase.activate_virtualenv(command_context, *args, **kwargs) + + # installing Python deps on the fly + for dep in ( + "Pillow==%s" % PILLOW_VERSION, + "pyssim==%s" % PYSSIM_VERSION, + "scipy==%s" % SCIPY_VERSION, + "numpy==%s" % NUMPY_VERSION, + "opencv-python==%s" % OPENCV_VERSION, + ): + if _need_install(command_context, dep): + subprocess.check_call( + [ + command_context.virtualenv_manager.python_path, + "-m", + "pip", + "install", + dep, + ] + ) + + +def check(command_context): + r"""Run `visualmetrics.py --check`.""" + command_context.activate_virtualenv() + + args = ["--check"] + status = command_context.run_process( + [command_context.virtualenv_manager.python_path, visualmetrics_path()] + args, + # For --check, don't allow user's path to interfere with path testing except on Linux + append_env=append_env( + command_context, append_path=host_platform().startswith("linux") + ), + pass_thru=True, + ensure_exit_code=False, # Don't throw on non-zero exit code. + cwd=mozpath.join(command_context.topsrcdir), + ) + + sys.stdout.flush() + sys.stderr.flush() + + if status: + return status + + # Avoid logging the command (and, on Windows, the environment). + command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL) + print("browsertime version:", end=" ") + + sys.stdout.flush() + sys.stderr.flush() + + return node(command_context, [browsertime_path()] + ["--version"]) + + +def extra_default_args(command_context, args=[]): + # Add Mozilla-specific default arguments. This is tricky because browsertime is quite + # loose about arguments; repeat arguments are generally accepted but then produce + # difficult to interpret type errors. + + def extract_browser_name(args): + "Extracts the browser name if any" + # These are BT arguments, it's BT job to check them + # here we just want to extract the browser name + res = re.findall("(--browser|-b)[= ]([\w]+)", " ".join(args)) + if res == []: + return None + return res[0][-1] + + def matches(args, *flags): + "Return True if any argument matches any of the given flags (maybe with an argument)." + for flag in flags: + if flag in args or any(arg.startswith(flag + "=") for arg in args): + return True + return False + + extra_args = [] + + # Default to Firefox. Override with `-b ...` or `--browser=...`. + specifies_browser = matches(args, "-b", "--browser") + if not specifies_browser: + extra_args.extend(("-b", "firefox")) + + # Default to not collect HAR. Override with `--skipHar=false`. + specifies_har = matches(args, "--har", "--skipHar", "--gzipHar") + if not specifies_har: + extra_args.append("--skipHar") + + if not matches(args, "--android"): + # If --firefox.binaryPath is not specified, default to the objdir binary + # Note: --firefox.release is not a real browsertime option, but it will + # silently ignore it instead and default to a release installation. + specifies_binaryPath = matches( + args, + "--firefox.binaryPath", + "--firefox.release", + "--firefox.nightly", + "--firefox.beta", + "--firefox.developer", + ) + + if not specifies_binaryPath: + specifies_binaryPath = extract_browser_name(args) == "chrome" + + if not specifies_binaryPath: + try: + extra_args.extend( + ("--firefox.binaryPath", command_context.get_binary_path()) + ) + except BinaryNotFoundException as e: + command_context.log( + logging.ERROR, + "browsertime", + {"error": str(e)}, + "ERROR: {error}", + ) + command_context.log( + logging.INFO, + "browsertime", + {}, + "Please run |./mach build| " + "or specify a Firefox binary with --firefox.binaryPath.", + ) + return 1 + + if extra_args: + command_context.log( + logging.DEBUG, + "browsertime", + {"extra_args": extra_args}, + "Running browsertime with extra default arguments: {extra_args}", + ) + + return extra_args + + +def _verify_node_install(command_context): + # check if Node is installed + sys.path.append(mozpath.join(command_context.topsrcdir, "tools", "lint", "eslint")) + import setup_helper + + with silence(): + node_valid = setup_helper.check_node_executables_valid() + if not node_valid: + print("Can't find Node. did you run ./mach bootstrap ?") + return False + + # check if the browsertime package has been deployed correctly + # for this we just check for the browsertime directory presence + if not os.path.exists(browsertime_path()): + print("Could not find browsertime.js, try ./mach browsertime --setup") + print("If that still fails, try ./mach browsertime --setup --clobber") + return False + + return True + + +@Command( + "browsertime", + category="testing", + description="Run [browsertime](https://github.com/sitespeedio/browsertime) " + "performance tests.", +) +@CommandArgument( + "--verbose", + action="store_true", + help="Verbose output for what commands the build is running.", +) +@CommandArgument("--update-upstream-url", default="") +@CommandArgument("--setup", default=False, action="store_true") +@CommandArgument("--clobber", default=False, action="store_true") +@CommandArgument( + "--skip-cache", + default=False, + action="store_true", + help="Skip all local caches to force re-fetching remote artifacts.", +) +@CommandArgument("--check-browsertime", default=False, action="store_true") +@CommandArgument( + "--install-vismet-reqs", + default=False, + action="store_true", + help="Add this flag to get the visual metrics requirements installed.", +) +@CommandArgument( + "--browsertime-help", + default=False, + action="store_true", + help="Show the browsertime help message.", +) +@CommandArgument("args", nargs=argparse.REMAINDER) +def browsertime( + command_context, + args, + verbose=False, + update_upstream_url="", + setup=False, + clobber=False, + skip_cache=False, + check_browsertime=False, + browsertime_help=False, + install_vismet_reqs=False, +): + command_context._set_log_level(verbose) + + # Output a message before going further to make sure the + # user knows that this tool is unsupported by the perftest + # team and point them to our supported tools. Pause a bit to + # make sure the user sees this message. + command_context.log( + logging.INFO, + "browsertime", + {}, + "[INFO] This command should be used for browsertime setup only.\n" + "If you are looking to run performance tests on your patch, use " + "`./mach raptor --browsertime` instead.\n\nYou can get visual-metrics " + "by using the --browsertime-video and --browsertime-visualmetrics. " + "Here is a sample command for raptor-browsertime: \n\t`./mach raptor " + "--browsertime -t amazon --browsertime-video --browsertime-visualmetrics`\n\n" + "See this wiki page for more information if needed: " + "https://wiki.mozilla.org/TestEngineering/Performance/Raptor/Browsertime\n\n", + ) + time.sleep(5) + + if update_upstream_url: + return setup_browsertime( + command_context, + new_upstream_url=update_upstream_url, + install_vismet_reqs=install_vismet_reqs, + ) + elif setup: + return setup_browsertime( + command_context, + should_clobber=clobber, + install_vismet_reqs=install_vismet_reqs, + ) + else: + if not _verify_node_install(command_context): + return 1 + + if check_browsertime: + return check(command_context) + + if browsertime_help: + args.append("--help") + + activate_browsertime_virtualenv(command_context) + default_args = extra_default_args(command_context, args) + if default_args == 1: + return 1 + return node(command_context, [browsertime_path()] + default_args + args) diff --git a/tools/browsertime/moz.build b/tools/browsertime/moz.build new file mode 100644 index 0000000000..d590ad00cf --- /dev/null +++ b/tools/browsertime/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Testing", "Raptor") diff --git a/tools/browsertime/package-lock.json b/tools/browsertime/package-lock.json new file mode 100644 index 0000000000..9beb6ae165 --- /dev/null +++ b/tools/browsertime/package-lock.json @@ -0,0 +1,5715 @@ +{ + "name": "mozilla-central-tools-browsertime", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "mozilla-central-tools-browsertime", + "license": "MPL-2.0", + "dependencies": { + "package.json": "^2.0.1" + }, + "devDependencies": { + "browsertime": "https://github.com/sitespeedio/browsertime/tarball/6f45b9e5a6e0419fe6408f05909f2ab5a1f6bacc" + } + }, + "node_modules/@cypress/xvfb": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@cypress/xvfb/-/xvfb-1.2.4.tgz", + "integrity": "sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q==", + "dev": true, + "dependencies": { + "debug": "^3.1.0", + "lodash.once": "^4.1.1" + } + }, + "node_modules/@devicefarmer/adbkit": { + "version": "2.11.3", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit/-/adbkit-2.11.3.tgz", + "integrity": "sha512-rsgWREAvSRQjdP9/3GoAV6Tq+o97haywgbTfCgt5yUqiDpaaq3hlH9FTo9XsdG8x+Jd0VQ9nTC2IXsDu8JGRSA==", + "dev": true, + "dependencies": { + "@devicefarmer/adbkit-logcat": "^1.1.0", + "@devicefarmer/adbkit-monkey": "~1.0.1", + "bluebird": "~2.9.24", + "commander": "^2.3.0", + "debug": "~2.6.3", + "node-forge": "^0.10.0", + "split": "~0.3.3" + }, + "bin": { + "adbkit": "bin/adbkit" + }, + "engines": { + "node": ">= 0.10.4" + } + }, + "node_modules/@devicefarmer/adbkit-logcat": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit-logcat/-/adbkit-logcat-1.1.0.tgz", + "integrity": "sha512-K90P5gUXM/w+yzLvJIRQ+tJooNU6ipUPPQkljtPJ0laR66TGtpt4Gqsjm0n9dPHK1W5KGgU1R5wnCd6RTSlPNA==", + "dev": true, + "engines": { + "node": ">= 0.10.4" + } + }, + "node_modules/@devicefarmer/adbkit-monkey": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit-monkey/-/adbkit-monkey-1.0.1.tgz", + "integrity": "sha512-HilPrVrCosYWqSyjfpDtaaN1kJwdlBpS+IAflP3z+e7nsEgk3JGJf1Vg0NgHJooTf5HDfXSyZqMVg+5jvXCK0g==", + "dev": true, + "dependencies": { + "async": "~0.2.9" + }, + "engines": { + "node": ">= 0.10.4" + } + }, + "node_modules/@devicefarmer/adbkit/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/@devicefarmer/adbkit/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/@jimp/bmp": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.22.7.tgz", + "integrity": "sha512-0cfBPvugURS7G+60vRBL+penDRst8x40alS5Rhn2nlGsgsBHljFDw7+H4o5r6gldw9nv9PR9JA90Wloy7KMZdQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7", + "bmp-js": "^0.1.0" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/core": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.22.7.tgz", + "integrity": "sha512-lg4z+pw23v2Gp9LWQur0NqYtnmoNWnyN/Or96elhJgeEJskrDGwROdajortHCCOI1xDnUZSirg8sFvStC8BIlg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7", + "any-base": "^1.1.0", + "buffer": "^5.2.0", + "exif-parser": "^0.1.12", + "file-type": "^16.5.4", + "isomorphic-fetch": "^3.0.0", + "mkdirp": "^2.1.3", + "pixelmatch": "^4.0.2", + "tinycolor2": "^1.6.0" + } + }, + "node_modules/@jimp/custom": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.22.7.tgz", + "integrity": "sha512-n+1+ZVDNumB1E+sL7KdGKAJ6MbgniX1/v/xOEFEQ46WDZ4cRTqP4+tXjHTuHSlOXiANH+K9zD6qgzqmgO6mCVw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/core": "^0.22.7" + } + }, + "node_modules/@jimp/gif": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.22.7.tgz", + "integrity": "sha512-PGZMS8sYFnDcqg+t8IT3RaSJLrqB+3GzhI0hU5D4mmSuJ5UO/6Bdgu8nrwh3uFPxw0ZH6h9ozYk88cz0pKEhLQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7", + "gifwrap": "^0.9.2", + "omggif": "^1.0.9" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/jpeg": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.22.7.tgz", + "integrity": "sha512-ptwWyX/7RPcREy8SpPN/8IlywbwyPXiuXmoHwM6m4iKcyaCmmnfCdZwLNXYliJzFAFLLOWDuOrwO3cZSkH6Czg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7", + "jpeg-js": "^0.4.4" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-blit": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.22.7.tgz", + "integrity": "sha512-8oXcBTSd/sBmTQATrCxQ1ZBER31Lge8vXzWqNCbC3b1ZvRggCcqnDzRRH1+JiI4i+jPRo3Fi6/sdvEUyQ5LY3g==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-blur": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.22.7.tgz", + "integrity": "sha512-M+0I5CKFIpnIQE27j8o8NECBsOFBd4z7C95ydy2UohYopugFq+hSVtMs1D4pQgb0RW1DJPiXD/4PHqb+lzV5mA==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-circle": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.22.7.tgz", + "integrity": "sha512-zfZKKpOhlyiDeFjGW5JB9K4h/kvbdaAJWUEwmKrvvGar67G3j8dKu46AX0MeWRNZ1yk/lfz+JIa7TzKfxEBf6w==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-color": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.22.7.tgz", + "integrity": "sha512-OXro9pdB0twQjV4LgW0bTEXaX1VgBsTBcFoDAs8q9mtQzD5p3UQmJ+ykCiQ5rTPxNN1Buc44tcCIfp8haB1ZVQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7", + "tinycolor2": "^1.6.0" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-contain": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.22.7.tgz", + "integrity": "sha512-GwUxZp4jMA0O0qbknUPDONJAfHFaTRs8kK+jgRtUfgb1Xi96l5RN/PMMDv4owZCUiPVAON80X1BMj7nSQWNVUw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blit": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5", + "@jimp/plugin-scale": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-cover": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.22.7.tgz", + "integrity": "sha512-PVXeQyofGepMoJaQ5XapLwCcZfsOF1IoAotHosh8AOP8niCP/Erm8T6ZWf5tf0sMJiLHQMPUyns186H5isqEMQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-crop": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5", + "@jimp/plugin-scale": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-crop": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.22.7.tgz", + "integrity": "sha512-XXvUU+hPdodtTBSgyUJUnzh7JgKMVlS1GxjcQsjYU8iGr1dbpuazKMTQxc76ChVmy8ue4goi8bGstacWUHpl/Q==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-displace": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.22.7.tgz", + "integrity": "sha512-CCNAkmm2OS4QQtNRfQvXqoAMxNE0maSlVEV5DNdioHOUKycy02EJ5hNYR3l0FG+NraQHOuqv9XV37sGRl6QzMA==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-dither": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.22.7.tgz", + "integrity": "sha512-ndCW5MIGMdh3aBvvgRCO7el9cIPG29kU7xQYlOs5+3JsDk3Vf7X30QGPjzxABOY95qLUNUjf5Qe/p/tqv/vbcw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-fisheye": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.22.7.tgz", + "integrity": "sha512-boI1QowhZRfb6OF+ZPWtiSJP1GATsTHjd5Oy/lJ+n0L4rp439ZOTB1Elzcgc44O2C1mgZDdybRPQQvYdPF8slA==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-flip": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.22.7.tgz", + "integrity": "sha512-/jkbgtvQPcKadAEV5ZXyoEpSdd7GEvGs/Ya/f48+LNszc+S24u4UXtuP3QPRJ5FHm0Re1t4uztM7xa6IPklAOA==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-rotate": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-gaussian": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.22.7.tgz", + "integrity": "sha512-OB1sdnjzq2rfUHmx9Rvi3SJIDbQAgWFgYEw6KhN3TSVOdrJHvwrQkEnwR9PoUzQg992VIpGcVc9Y1s/SOU2oCA==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-invert": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.22.7.tgz", + "integrity": "sha512-dX/TqACJ/M5uXDIEJlVPPwietMD6EWUeA/CV4uvhLz9EMjTgHociJ3TWqGCY/70phhIBLbhLcHUVBL/q65ynfQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-mask": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.22.7.tgz", + "integrity": "sha512-rfKHKJLAtJG7qbB4zYAMcQ9ue3CIFRuAJ3xX0lzCxC0fGvCVuXlcxiAEauBxqaTWqiKMnahqpR3/Ah679K2FKQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-normalize": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.22.7.tgz", + "integrity": "sha512-t8x2jjKDmvUAZB4Wbeagr4D0BvoVCIWquy94mpglvSZ8ujKLt0aQBl3CBEIbXFAoVqNif+G36NtxPHNsjxIXOg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-print": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.22.7.tgz", + "integrity": "sha512-kx0+cPeinki1IFg9cJy7LC4uVuOEOa8TIrcERioB6PVgJ7EDzCAfatTKULZ+t4uSs2K/lQF97wPYlbiyxs/Hzg==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7", + "load-bmfont": "^1.4.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blit": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-resize": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.22.7.tgz", + "integrity": "sha512-pg7i0JIYt7x7ag+CoD/yG70Xvwm1sKRfcFjQh954yestiin14uppPgXchAmTBmctecBjLNdsVlqSXbPvU4Jvxw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-rotate": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.22.7.tgz", + "integrity": "sha512-Uh3Gb18IY8uXWk6E1bzMopum2GP+xwohbnMIDE0MSWmLaz7LXrfnvgXFba1uRGgn73CJz8UDS4fC1KIJMuxQZA==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blit": ">=0.3.5", + "@jimp/plugin-crop": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-scale": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.22.7.tgz", + "integrity": "sha512-3uHUrk5Rl6MCxuoJtHTSeJjSHIxHWqOOgmD2caKIvyxds0Zmofu/Fva+N4V/m80E4q4G2RXNsUplFpFGhUM7hw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-shadow": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.22.7.tgz", + "integrity": "sha512-NKEq5VR8U/d0OKf0hxFtrrbMCuNv7by31V+Kwgxb1oTP+j+zZEaww+m3YgEwIwRe7E8/yeDSHa5bJ+CmuyFZjw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-blur": ">=0.3.5", + "@jimp/plugin-resize": ">=0.3.5" + } + }, + "node_modules/@jimp/plugin-threshold": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.22.7.tgz", + "integrity": "sha512-BH4aLwfmnqjRVhdzMIqUns4ycZ6QoHHFR6Qz+X2iSpH5a33xFA4DRbd3Ehtrs4Gk7XiCjWkUyM6wjmH7l/1hNQ==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5", + "@jimp/plugin-color": ">=0.8.0", + "@jimp/plugin-resize": ">=0.8.0" + } + }, + "node_modules/@jimp/plugins": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.22.7.tgz", + "integrity": "sha512-AJmzTG/sa+CDpvle/UE89hjHR85gnRGSwLuQqPbhlY6GFCmC3uqHRJz9O5I8A4zdi9+e8LsBphuTlKV7RbuXOw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/plugin-blit": "^0.22.7", + "@jimp/plugin-blur": "^0.22.7", + "@jimp/plugin-circle": "^0.22.7", + "@jimp/plugin-color": "^0.22.7", + "@jimp/plugin-contain": "^0.22.7", + "@jimp/plugin-cover": "^0.22.7", + "@jimp/plugin-crop": "^0.22.7", + "@jimp/plugin-displace": "^0.22.7", + "@jimp/plugin-dither": "^0.22.7", + "@jimp/plugin-fisheye": "^0.22.7", + "@jimp/plugin-flip": "^0.22.7", + "@jimp/plugin-gaussian": "^0.22.7", + "@jimp/plugin-invert": "^0.22.7", + "@jimp/plugin-mask": "^0.22.7", + "@jimp/plugin-normalize": "^0.22.7", + "@jimp/plugin-print": "^0.22.7", + "@jimp/plugin-resize": "^0.22.7", + "@jimp/plugin-rotate": "^0.22.7", + "@jimp/plugin-scale": "^0.22.7", + "@jimp/plugin-shadow": "^0.22.7", + "@jimp/plugin-threshold": "^0.22.7", + "timm": "^1.6.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/png": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.22.7.tgz", + "integrity": "sha512-LxD3O9FKEwVv+j+HcUV7ez72Miy+823EjhtFZbBYXNp9qjHtHFBpgcSJBftUOCei8OlmmVgULYn9XjyfPsDgGw==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/utils": "^0.22.7", + "pngjs": "^6.0.0" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/tiff": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.22.7.tgz", + "integrity": "sha512-/oE8kLumzBfU1Z6h4TrDXYCGQNc4CjbZQvPssjImEqNLr5vbefpIpoy1fVMpsyuHZHsGovsBhBHxTJaRLO4+Og==", + "dev": true, + "optional": true, + "dependencies": { + "utif2": "^4.0.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/types": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.22.7.tgz", + "integrity": "sha512-1T8BxwDh5HJvBh3tt6HUd8r7ir5Ge3JWATXC8O3Y9QYwOaERjA2+FVhGSjtoo5xCeJvLRjSzEtfZ8heowMBL4w==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/bmp": "^0.22.7", + "@jimp/gif": "^0.22.7", + "@jimp/jpeg": "^0.22.7", + "@jimp/png": "^0.22.7", + "@jimp/tiff": "^0.22.7", + "timm": "^1.6.1" + }, + "peerDependencies": { + "@jimp/custom": ">=0.3.5" + } + }, + "node_modules/@jimp/utils": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.22.7.tgz", + "integrity": "sha512-4ax4IOWLIERx4yz9y3fNXKvQaPOY23yJF5h4sizxVkQUObkZHWE0kL0TVHodBt3rS8ksdbCL8Jkz4GeNP/Katg==", + "dev": true, + "optional": true, + "dependencies": { + "regenerator-runtime": "^0.13.3" + } + }, + "node_modules/@sitespeed.io/chromedriver": { + "version": "112.0.5615-28", + "resolved": "https://registry.npmjs.org/@sitespeed.io/chromedriver/-/chromedriver-112.0.5615-28.tgz", + "integrity": "sha512-Um/D7PQ5NlfCT2Khq8iEzescNy2L4xR+cK/vC/tRRCBAOZXMCXb1oY0OE7Bw2CIhtPf4a2Ldi3cRU6oSEAUb+w==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "node-downloader-helper": "2.1.5", + "node-stream-zip": "1.15.0" + } + }, + "node_modules/@sitespeed.io/edgedriver": { + "version": "112.0.1722-34", + "resolved": "https://registry.npmjs.org/@sitespeed.io/edgedriver/-/edgedriver-112.0.1722-34.tgz", + "integrity": "sha512-px7L9eZPGQ3iWti3vBurFs7eoUg7x9yA7QpQmy9AdkgUm3kzuGjxt/sszvy+s/DLgOtDhJGzwMAb9JuGslzj/w==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "node-downloader-helper": "2.1.6", + "node-stream-zip": "1.15.0" + } + }, + "node_modules/@sitespeed.io/edgedriver/node_modules/node-downloader-helper": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.6.tgz", + "integrity": "sha512-VkOvAXIopI3xMuM/MC5UL7NqqnizQ/9QXZt28jR8FPZ6fHLQm4xe4+YXJ9FqsWwLho5BLXrF51nfOQ0QcohRkQ==", + "dev": true, + "bin": { + "ndh": "bin/ndh" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sitespeed.io/geckodriver": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/@sitespeed.io/geckodriver/-/geckodriver-0.33.0.tgz", + "integrity": "sha512-w6w+x9/Q44JekTPi8NlRsfh5Uz4TfquJcUEs0tA/oEcxLVxRS7VtaiaJEE0GzzN6cUmFS6Twas7E4bCA4k/Yxg==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "node-downloader-helper": "2.1.5", + "node-stream-zip": "1.15.0", + "tar": "6.1.13" + } + }, + "node_modules/@sitespeed.io/throttle": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@sitespeed.io/throttle/-/throttle-5.0.0.tgz", + "integrity": "sha512-eul4I7IllA6l3+GGX1aW/D75XYux0ODuZDzstKD0kAuvIkpQ4BVLkFBoLXQN50gLMFGqZ3QWMobhQ5L2/6sFgg==", + "dev": true, + "dependencies": { + "minimist": "1.2.6" + }, + "bin": { + "throttle": "bin/index.js" + }, + "engines": { + "node": ">=14.16" + } + }, + "node_modules/@sitespeed.io/tracium": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@sitespeed.io/tracium/-/tracium-0.3.3.tgz", + "integrity": "sha512-dNZafjM93Y+F+sfwTO5gTpsGXlnc/0Q+c2+62ViqP3gkMWvHEMSKkaEHgVJLcLg3i/g19GSIPziiKpgyne07Bw==", + "dev": true, + "dependencies": { + "debug": "^4.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@sitespeed.io/tracium/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@sitespeed.io/tracium/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "dev": true, + "optional": true + }, + "node_modules/@types/node": { + "version": "16.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", + "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==", + "dev": true, + "optional": true + }, + "node_modules/abs": { + "version": "1.3.14", + "resolved": "https://registry.npmjs.org/abs/-/abs-1.3.14.tgz", + "integrity": "sha512-PrS26IzwKLWwuURpiKl8wRmJ2KdR/azaVrLEBWG/TALwT20Y7qjtYp1qcMLHA4206hBHY5phv3w4pjf9NPv4Vw==", + "dependencies": { + "ul": "^5.0.0" + } + }, + "node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/any-base": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz", + "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==", + "dev": true, + "optional": true + }, + "node_modules/async": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==", + "dev": true + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, + "node_modules/bluebird": { + "version": "2.9.34", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.34.tgz", + "integrity": "sha512-ZDzCb87X7/IP1uzQ5eJZB+WoQRGTnKL5DHWvPw6kkMbQseouiQIrEi3P1UGE0D1k0N5/+aP/5GMCyHZ1xYJyHQ==", + "dev": true + }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "dev": true, + "optional": true + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/browsertime": { + "version": "17.8.1", + "resolved": "https://github.com/sitespeedio/browsertime/tarball/6f45b9e5a6e0419fe6408f05909f2ab5a1f6bacc", + "integrity": "sha512-bAUNDY0aTl7OvS5OpU2vVopykhhNWmwk71qXi7ttAzEyr+ZD3rVjbsfmJ+kE02vSjxAiTGgfGR7Sr5KM5mMjIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cypress/xvfb": "1.2.4", + "@devicefarmer/adbkit": "2.11.3", + "@sitespeed.io/chromedriver": "112.0.5615-28", + "@sitespeed.io/edgedriver": "112.0.1722-34", + "@sitespeed.io/geckodriver": "0.33.0", + "@sitespeed.io/throttle": "5.0.0", + "@sitespeed.io/tracium": "0.3.3", + "btoa": "1.2.1", + "chrome-har": "0.13.1", + "chrome-remote-interface": "0.32.1", + "dayjs": "1.11.7", + "execa": "7.0.0", + "fast-stats": "0.0.6", + "ff-test-bidi-har-export": "0.0.10", + "find-up": "6.3.0", + "get-port": "6.1.2", + "hasbin": "1.2.3", + "intel": "1.2.0", + "lodash.get": "4.4.2", + "lodash.groupby": "4.6.0", + "lodash.isempty": "4.4.0", + "lodash.merge": "4.6.2", + "lodash.pick": "4.4.0", + "lodash.set": "4.3.2", + "selenium-webdriver": "4.8.2", + "yargs": "17.7.1" + }, + "bin": { + "browsertime": "bin/browsertime.js" + }, + "engines": { + "node": ">=14.19.1" + }, + "optionalDependencies": { + "jimp": "0.22.7" + } + }, + "node_modules/btoa": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", + "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==", + "dev": true, + "bin": { + "btoa": "bin/btoa.js" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true, + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-equal": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz", + "integrity": "sha512-RgSV6InVQ9ODPdLWJ5UAqBqJBOg370Nz6ZQtRzpt6nUjc8v0St97uJ4PYC6NztqIScrAXafKM3mZPMygSe1ggA==", + "dev": true, + "optional": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/capture-stack-trace": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.2.tgz", + "integrity": "sha512-X/WM2UQs6VMHUtjUDnZTRI+i1crWteJySFzr9UpGoQa4WQffXVTTXuekjl7TjZRlcF2XfjgITT0HxZ9RnxeT0w==", + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dev": true, + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/chrome-har": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/chrome-har/-/chrome-har-0.13.1.tgz", + "integrity": "sha512-R9OXxVeCOqJYmqO8Oco+nouJ4frwVFQ6rGUgj5dJOfVNxBMmwCannp+DH2IAfqY/K8WJ5VYCQyq7SLTNZtcCXA==", + "dev": true, + "dependencies": { + "dayjs": "1.11.7", + "debug": "4.3.4", + "tough-cookie": "4.1.2", + "uuid": "9.0.0" + } + }, + "node_modules/chrome-har/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/chrome-har/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/chrome-remote-interface": { + "version": "0.32.1", + "resolved": "https://registry.npmjs.org/chrome-remote-interface/-/chrome-remote-interface-0.32.1.tgz", + "integrity": "sha512-CU3/K/8YlU2H0DjsLRbxPsG4piiSGUcIy6GkGXF11SqOYoIeuUBivOsGXScaZnTyC1p4wFSR+GNmAM434/ALWw==", + "dev": true, + "dependencies": { + "commander": "2.11.x", + "ws": "^7.2.0" + }, + "bin": { + "chrome-remote-interface": "bin/client.js" + } + }, + "node_modules/chrome-remote-interface/node_modules/commander": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", + "dev": true + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "node_modules/create-error-class": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", + "integrity": "sha512-gYTKKexFO3kh200H1Nit76sRwRtOY32vQd3jpAQKpLtZqyNsSQNfI4N7o3eP2wUjV35pTWKRYqFUDBvUha/Pkw==", + "dependencies": { + "capture-stack-trace": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/dayjs": { + "version": "1.11.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz", + "integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==", + "dev": true + }, + "node_modules/dbug": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/dbug/-/dbug-0.4.2.tgz", + "integrity": "sha512-nrmsMK1msY0WXwfA2czrKVDgpIYJR2JJaq5cX4DwW7Rxm11nXHqouh9wmubEs44bHYxk8CqeP/Jx4URqSB961w==", + "dev": true + }, + "node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deffy": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/deffy/-/deffy-2.2.4.tgz", + "integrity": "sha512-pLc9lsbsWjr6RxmJ2OLyvm+9l4j1yK69h+TML/gUit/t3vTijpkNGh8LioaJYTGO7F25m6HZndADcUOo2PsiUg==", + "dependencies": { + "typpy": "^2.0.0" + } + }, + "node_modules/dom-walk": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.2.tgz", + "integrity": "sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w==", + "dev": true, + "optional": true + }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/err": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/err/-/err-1.1.1.tgz", + "integrity": "sha512-N97Ybd2jJHVQ+Ft3Q5+C2gM3kgygkdeQmEqbN2z15UTVyyEsIwLA1VK39O1DHEJhXbwIFcJLqm6iARNhFANcQA==", + "dependencies": { + "typpy": "^2.2.0" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/exec-limiter": { + "version": "3.2.13", + "resolved": "https://registry.npmjs.org/exec-limiter/-/exec-limiter-3.2.13.tgz", + "integrity": "sha512-86Ri699bwiHZVBzTzNj8gspqAhCPchg70zPVWIh3qzUOA1pUMcb272Em3LPk8AE0mS95B9yMJhtqF8vFJAn0dA==", + "dependencies": { + "limit-it": "^3.0.0", + "typpy": "^2.1.0" + } + }, + "node_modules/execa": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-7.0.0.tgz", + "integrity": "sha512-tQbH0pH/8LHTnwTrsKWideqi6rFB/QNUawEwrn+WHyz7PX1Tuz2u7wfTvbaNBdP5JD5LVWxNo8/A8CHNZ3bV6g==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.1", + "human-signals": "^4.3.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^3.0.7", + "strip-final-newline": "^3.0.0" + }, + "engines": { + "node": "^14.18.0 || ^16.14.0 || >=18.0.0" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exif-parser": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz", + "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==", + "dev": true, + "optional": true + }, + "node_modules/fast-stats": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/fast-stats/-/fast-stats-0.0.6.tgz", + "integrity": "sha512-m0zkwa7Z07Wc4xm1YtcrCHmhzNxiYRrrfUyhkdhSZPzaAH/Ewbocdaq7EPVBFz19GWfIyyPcLfRHjHJYe83jlg==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/ff-test-bidi-har-export": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/ff-test-bidi-har-export/-/ff-test-bidi-har-export-0.0.10.tgz", + "integrity": "sha512-0azA5fUrA3Fn4KFAg2LcFqM2Xav7+H41gj4GHgLMhZxSyHfPV0yhbs1y+6sv+skgak/IbBb75KZmkj0VFsdyEg==", + "dev": true + }, + "node_modules/file-type": { + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "dev": true, + "optional": true, + "dependencies": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "node_modules/find-up": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", + "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==", + "dev": true, + "dependencies": { + "locate-path": "^7.1.0", + "path-exists": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "node_modules/function.name": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/function.name/-/function.name-1.0.13.tgz", + "integrity": "sha512-mVrqdoy5npWZyoXl4DxCeuVF6delDcQjVS9aPdvLYlBxtMTZDR2B5GVEQEoM1jJyspCqg3C0v4ABkLE7tp9xFA==", + "dependencies": { + "noop6": "^1.0.1" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-port": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", + "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gifwrap": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/gifwrap/-/gifwrap-0.9.4.tgz", + "integrity": "sha512-MDMwbhASQuVeD4JKd1fKgNgCRL3fGqMM4WaqpNhWO0JiMOAjbQdumbs4BbBZEy9/M00EHEjKN3HieVhCUlwjeQ==", + "dev": true, + "optional": true, + "dependencies": { + "image-q": "^4.0.0", + "omggif": "^1.0.10" + } + }, + "node_modules/git-package-json": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/git-package-json/-/git-package-json-1.4.10.tgz", + "integrity": "sha512-DRAcvbzd2SxGK7w8OgYfvKqhFliT5keX0lmSmVdgScgf1kkl5tbbo7Pam6uYoCa1liOiipKxQZG8quCtGWl/fA==", + "dependencies": { + "deffy": "^2.2.1", + "err": "^1.1.1", + "gry": "^5.0.0", + "normalize-package-data": "^2.3.5", + "oargv": "^3.4.1", + "one-by-one": "^3.1.0", + "r-json": "^1.2.1", + "r-package-json": "^1.0.0", + "tmp": "0.0.28" + } + }, + "node_modules/git-source": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/git-source/-/git-source-1.1.10.tgz", + "integrity": "sha512-XZZ7ZgnLL35oLgM/xjnLYgtlKlxJG0FohC1kWDvGkU7s1VKGXK0pFF/g1itQEwQ3D+uTQzBnzPi8XbqOv7Wc1Q==", + "dependencies": { + "git-url-parse": "^5.0.1" + } + }, + "node_modules/git-up": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/git-up/-/git-up-1.2.1.tgz", + "integrity": "sha512-SRVN3rOLACva8imc7BFrB6ts5iISWKH1/h/1Z+JZYoUI7UVQM7gQqk4M2yxUENbq2jUUT09NEND5xwP1i7Ktlw==", + "dependencies": { + "is-ssh": "^1.0.0", + "parse-url": "^1.0.0" + } + }, + "node_modules/git-url-parse": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/git-url-parse/-/git-url-parse-5.0.1.tgz", + "integrity": "sha512-4uSiOgrryNEMBX+gTWogenYRUh2j1D+95STTSEF2RCTgLkfJikl8c7BGr0Bn274hwuxTsbS2/FQ5pVS9FoXegQ==", + "dependencies": { + "git-up": "^1.0.0" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/global": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", + "integrity": "sha512-wv/LAoHdRE3BeTGz53FAamhGlPLhlssK45usmGFThIi4XqnBmjKQ16u+RNbP7WvigRZDxUsM0J3gcQ5yicaL0w==", + "dev": true, + "optional": true, + "dependencies": { + "min-document": "^2.19.0", + "process": "^0.11.10" + } + }, + "node_modules/got": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-5.6.0.tgz", + "integrity": "sha512-MnypzkaW8dldA8AbJFjMs7y14+ykd2V8JCLKSvX1Gmzx1alH3Y+3LArywHDoAF2wS3pnZp4gacoYtvqBeF6drQ==", + "dependencies": { + "create-error-class": "^3.0.1", + "duplexer2": "^0.1.4", + "is-plain-obj": "^1.0.0", + "is-redirect": "^1.0.0", + "is-retry-allowed": "^1.0.0", + "is-stream": "^1.0.0", + "lowercase-keys": "^1.0.0", + "node-status-codes": "^1.0.0", + "object-assign": "^4.0.1", + "parse-json": "^2.1.0", + "pinkie-promise": "^2.0.0", + "read-all-stream": "^3.0.0", + "readable-stream": "^2.0.5", + "timed-out": "^2.0.0", + "unzip-response": "^1.0.0", + "url-parse-lax": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/got/node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gry": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/gry/-/gry-5.0.8.tgz", + "integrity": "sha512-meq9ZjYVpLzZh3ojhTg7IMad9grGsx6rUUKHLqPnhLXzJkRQvEL2U3tQpS5/WentYTtHtxkT3Ew/mb10D6F6/g==", + "dependencies": { + "abs": "^1.2.1", + "exec-limiter": "^3.0.0", + "one-by-one": "^3.0.0", + "ul": "^5.0.0" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/hasbin": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/hasbin/-/hasbin-1.2.3.tgz", + "integrity": "sha512-CCd8e/w2w28G8DyZvKgiHnQJ/5XXDz6qiUHnthvtag/6T5acUeN5lqq+HMoBqcmgWueWDhiCplrw0Kb1zDACRg==", + "dev": true, + "dependencies": { + "async": "~1.5" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/hasbin/node_modules/async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha512-nSVgobk4rv61R9PUSDtYt7mPVB2olxNR5RWJcAsH676/ef11bUZwvu7+RGYrYauVdDPcO519v68wRhXQtxsV9w==", + "dev": true + }, + "node_modules/hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==" + }, + "node_modules/human-signals": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-4.3.1.tgz", + "integrity": "sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ==", + "dev": true, + "engines": { + "node": ">=14.18.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, + "node_modules/image-q": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/image-q/-/image-q-4.0.0.tgz", + "integrity": "sha512-PfJGVgIfKQJuq3s0tTDOKtztksibuUEbJQIYT3by6wctQo+Rdlh7ef4evJ5NCdxY4CfMbvFkocEwbl4BF8RlJw==", + "dev": true, + "optional": true, + "dependencies": { + "@types/node": "16.9.1" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "dev": true + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "node_modules/intel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/intel/-/intel-1.2.0.tgz", + "integrity": "sha512-CUDyAtEeEeDo5YtwANOuDhxuFEOgInHvbMrBbhXCD4tAaHuzHM2llevtTeq2bmP8Jf7NkpN305pwDncRmhc1Wg==", + "dev": true, + "dependencies": { + "chalk": "^1.1.0", + "dbug": "~0.4.2", + "stack-trace": "~0.0.9", + "strftime": "~0.10.0", + "symbol": "~0.3.1", + "utcstring": "~0.1.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" + }, + "node_modules/is-core-module": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz", + "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==", + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-function": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.2.tgz", + "integrity": "sha512-lw7DUp0aWXYg+CBCN+JKkcE0Q2RayZnSvnZBlwgxHBQhqt5pZNVy4Ri7H9GmmXkdu7LUthszM+Tor1u/2iBcpQ==", + "dev": true, + "optional": true + }, + "node_modules/is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha512-cr/SlUEe5zOGmzvj9bUyC4LVvkNVAXu4GytXLNMr1pny+a65MpQ9IJzFHD5vi7FyJgb4qt27+eS3TuQnqB+RQw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-retry-allowed": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", + "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-ssh": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/is-ssh/-/is-ssh-1.4.0.tgz", + "integrity": "sha512-x7+VxdxOdlV3CYpjvRLBv5Lo9OJerlYanjwFrPR9fuGPjCiNiCzFgAWpiLAohSbsnH4ZAys3SBh+hq5rJosxUQ==", + "dependencies": { + "protocols": "^2.0.1" + } + }, + "node_modules/is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/isomorphic-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", + "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", + "dev": true, + "optional": true, + "dependencies": { + "node-fetch": "^2.6.1", + "whatwg-fetch": "^3.4.1" + } + }, + "node_modules/iterate-object": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/iterate-object/-/iterate-object-1.3.4.tgz", + "integrity": "sha512-4dG1D1x/7g8PwHS9aK6QV5V94+ZvyP4+d19qDv43EzImmrndysIl4prmJ1hWWIGCqrZHyaHBm6BSEWHOLnpoNw==" + }, + "node_modules/jimp": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.22.7.tgz", + "integrity": "sha512-TJCTJ4ZcFUw6W8XZnR6ajdEu8vSyPi3AuoChs+zLHalXnhAPZgwkzwcXnxey4LNjh1p9dfIUkg8YSQ+q8pBW0A==", + "dev": true, + "optional": true, + "dependencies": { + "@jimp/custom": "^0.22.7", + "@jimp/plugins": "^0.22.7", + "@jimp/types": "^0.22.7", + "regenerator-runtime": "^0.13.3" + } + }, + "node_modules/jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "dev": true, + "optional": true + }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "dev": true, + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/limit-it": { + "version": "3.2.10", + "resolved": "https://registry.npmjs.org/limit-it/-/limit-it-3.2.10.tgz", + "integrity": "sha512-T0NK99pHnkimldr1WUqvbGV1oWDku/xC9J/OqzJFsV1jeOS6Bwl8W7vkeQIBqwiON9dTALws+rX/XPMQqWerDQ==", + "dependencies": { + "typpy": "^2.0.0" + } + }, + "node_modules/load-bmfont": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", + "integrity": "sha512-8UyQoYmdRDy81Brz6aLAUhfZLwr5zV0L3taTQ4hju7m6biuwiWiJXjPhBJxbUQJA8PrkvJ/7Enqmwk2sM14soA==", + "dev": true, + "optional": true, + "dependencies": { + "buffer-equal": "0.0.1", + "mime": "^1.3.4", + "parse-bmfont-ascii": "^1.0.3", + "parse-bmfont-binary": "^1.0.5", + "parse-bmfont-xml": "^1.1.4", + "phin": "^2.9.1", + "xhr": "^2.0.1", + "xtend": "^4.0.0" + } + }, + "node_modules/locate-path": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", + "dev": true, + "dependencies": { + "p-locate": "^6.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "dev": true + }, + "node_modules/lodash.isempty": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz", + "integrity": "sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg==", + "dev": true + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "dev": true + }, + "node_modules/lodash.pick": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", + "integrity": "sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==", + "dev": true + }, + "node_modules/lodash.set": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", + "integrity": "sha512-4hNPN5jlm/N/HLMCO43v8BXKq9Z7QdAGc/VGrRD61w8gN9g/6jF9A4L1pbUgBLCffi0w9VsXfTOij5x8iTyFvg==", + "dev": true + }, + "node_modules/lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "optional": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/min-document": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", + "integrity": "sha512-9Wy1B3m3f66bPPmU5hdA4DR4PB2OfDU/+GS3yAB7IQozE3tqXaVv2zOjgla7MEGSRv95+ILmOuvhLkOK6wJtCQ==", + "dev": true, + "optional": true, + "dependencies": { + "dom-walk": "^0.1.0" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + }, + "node_modules/minipass": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.8.tgz", + "integrity": "sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mkdirp": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-2.1.6.tgz", + "integrity": "sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A==", + "dev": true, + "optional": true, + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/node-downloader-helper": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.5.tgz", + "integrity": "sha512-sLedzfv8C4VMAvTdDQcjLFAl3gydNeBXh2bLcCzvZRmd4EK0rkoTxJ8tkxnriUSJO/n13skJzH7l6CzCdBwYGg==", + "dev": true, + "bin": { + "ndh": "bin/ndh" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/node-fetch": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", + "dev": true, + "optional": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-forge": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==", + "dev": true, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/node-status-codes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz", + "integrity": "sha512-1cBMgRxdMWE8KeWCqk2RIOrvUb0XCwYfEsY5/y2NlXyq4Y/RumnOZvTj4Nbr77+Vb2C+kyBoRTdkNOS8L3d/aQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/node-stream-zip": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.15.0.tgz", + "integrity": "sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw==", + "dev": true, + "engines": { + "node": ">=0.12.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/antelle" + } + }, + "node_modules/noop6": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/noop6/-/noop6-1.0.9.tgz", + "integrity": "sha512-DB3Hwyd89dPr5HqEPg3YHjzvwh/mCqizC1zZ8vyofqc+TQRyPDnT4wgXXbLGF4z9YAzwwTLi8pNLhGqcbSjgkA==" + }, + "node_modules/normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dependencies": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "node_modules/npm-run-path": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.1.0.tgz", + "integrity": "sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==", + "dev": true, + "dependencies": { + "path-key": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/oargv": { + "version": "3.4.10", + "resolved": "https://registry.npmjs.org/oargv/-/oargv-3.4.10.tgz", + "integrity": "sha512-SXaMANv9sr7S/dP0vj0+Ybipa47UE1ntTWQ2rpPRhC6Bsvfl+Jg03Xif7jfL0sWKOYWK8oPjcZ5eJ82t8AP/8g==", + "dependencies": { + "iterate-object": "^1.1.0", + "ul": "^5.0.0" + } + }, + "node_modules/obj-def": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/obj-def/-/obj-def-1.0.9.tgz", + "integrity": "sha512-bQ4ya3VYD6FAA1+s6mEhaURRHSmw4+sKaXE6UyXZ1XDYc5D+c7look25dFdydmLd18epUegh398gdDkMUZI9xg==", + "dependencies": { + "deffy": "^2.2.2" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/omggif": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", + "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==", + "dev": true, + "optional": true + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/one-by-one": { + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/one-by-one/-/one-by-one-3.2.8.tgz", + "integrity": "sha512-HR/pSzZdm46Xqj58K+Bu64kMbSTw8/u77AwWvV+rprO/OsuR++pPlkUJn+SmwqBGRgHKwSKQ974V3uls7crIeQ==", + "dependencies": { + "obj-def": "^1.0.0", + "sliced": "^1.0.1" + } + }, + "node_modules/onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "dev": true, + "dependencies": { + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/p-limit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^1.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz", + "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==", + "dev": true, + "dependencies": { + "p-limit": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-2.4.0.tgz", + "integrity": "sha512-PRg65iXMTt/uK8Rfh5zvzkUbfAPitF17YaCY+IbHsYgksiLvtzWWTUildHth3mVaZ7871OJ7gtP4LBRBlmAdXg==", + "dependencies": { + "got": "^5.0.0", + "registry-auth-token": "^3.0.1", + "registry-url": "^3.0.3", + "semver": "^5.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/package-json-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/package-json-path/-/package-json-path-1.0.9.tgz", + "integrity": "sha512-uNu7f6Ef7tQHZRnkyVnCtzdSYVN9uBtge/sG7wzcUaawFWkPYUq67iXxRGrQSg/q0tzxIB8jSyIYUKjG2Jn//A==", + "dependencies": { + "abs": "^1.2.1" + } + }, + "node_modules/package.json": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/package.json/-/package.json-2.0.1.tgz", + "integrity": "sha512-pSxZ6XR5yEawRN2ekxx9IKgPN5uNAYco7MCPxtBEWMKO3UKWa1X2CtQMzMgloeGj2g2o6cue3Sb5iPkByIJqlw==", + "deprecated": "Use pkg.json instead.", + "dependencies": { + "git-package-json": "^1.4.0", + "git-source": "^1.1.0", + "package-json": "^2.3.1" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true + }, + "node_modules/parse-bmfont-ascii": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz", + "integrity": "sha512-U4RrVsUFCleIOBsIGYOMKjn9PavsGOXxbvYGtMOEfnId0SVNsgehXh1DxUdVPLoxd5mvcEtvmKs2Mmf0Mpa1ZA==", + "dev": true, + "optional": true + }, + "node_modules/parse-bmfont-binary": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-binary/-/parse-bmfont-binary-1.0.6.tgz", + "integrity": "sha512-GxmsRea0wdGdYthjuUeWTMWPqm2+FAd4GI8vCvhgJsFnoGhTrLhXDDupwTo7rXVAgaLIGoVHDZS9p/5XbSqeWA==", + "dev": true, + "optional": true + }, + "node_modules/parse-bmfont-xml": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/parse-bmfont-xml/-/parse-bmfont-xml-1.1.4.tgz", + "integrity": "sha512-bjnliEOmGv3y1aMEfREMBJ9tfL3WR0i0CKPj61DnSLaoxWR3nLrsQrEbCId/8rF4NyRF0cCqisSVXyQYWM+mCQ==", + "dev": true, + "optional": true, + "dependencies": { + "xml-parse-from-string": "^1.0.0", + "xml2js": "^0.4.5" + } + }, + "node_modules/parse-headers": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.5.tgz", + "integrity": "sha512-ft3iAoLOB/MlwbNXgzy43SWGP6sQki2jQvAyBg/zDFAgr9bfNWZIUj42Kw2eJIl8kEi4PbgE6U1Zau/HwI75HA==", + "dev": true, + "optional": true + }, + "node_modules/parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==", + "dependencies": { + "error-ex": "^1.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parse-url": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/parse-url/-/parse-url-1.3.11.tgz", + "integrity": "sha512-1wj9nkgH/5EboDxLwaTMGJh3oH3f+Gue+aGdh631oCqoSBpokzmMmOldvOeBPtB8GJBYJbaF93KPzlkU+Y1ksg==", + "dependencies": { + "is-ssh": "^1.3.0", + "protocols": "^1.4.0" + } + }, + "node_modules/parse-url/node_modules/protocols": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/protocols/-/protocols-1.4.8.tgz", + "integrity": "sha512-IgjKyaUSjsROSO8/D49Ab7hP8mJgTYcqApOqdPhLoPxAplXmkp+zRvsrSQjFn5by0rhm4VH0GAUELIPpx7B1yg==" + }, + "node_modules/path-exists": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", + "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, + "node_modules/peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/phin": { + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/phin/-/phin-2.9.3.tgz", + "integrity": "sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==", + "dev": true, + "optional": true + }, + "node_modules/pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "dependencies": { + "pinkie": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pixelmatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-4.0.2.tgz", + "integrity": "sha512-J8B6xqiO37sU/gkcMglv6h5Jbd9xNER7aHzpfRdNmV4IbQBzBpe4l9XmbG+xPF/znacgu2jfEw+wHffaq/YkXA==", + "dev": true, + "optional": true, + "dependencies": { + "pngjs": "^3.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, + "node_modules/pixelmatch/node_modules/pngjs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==", + "dev": true, + "optional": true, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=12.13.0" + } + }, + "node_modules/prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha512-PhmXi5XmoyKw1Un4E+opM2KcsJInDvKyuOumcjjw3waw86ZNjHwVUOOWLc4bCzLdcKNaWBH9e99sbWzDQsVaYg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "optional": true, + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "node_modules/protocols": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.1.tgz", + "integrity": "sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==" + }, + "node_modules/psl": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", + "dev": true + }, + "node_modules/punycode": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true + }, + "node_modules/r-json": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/r-json/-/r-json-1.2.10.tgz", + "integrity": "sha512-hu9vyLjSlHXT62NAS7DjI9WazDlvjN0lgp3n431dCVnirVcLkZIpzSwA3orhZEKzdDD2jqNYI+w0yG0aFf4kpA==" + }, + "node_modules/r-package-json": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/r-package-json/-/r-package-json-1.0.9.tgz", + "integrity": "sha512-G4Vpf1KImWmmPFGdtWQTU0L9zk0SjqEC4qs/jE7AQ+Ylmr5kizMzGeC4wnHp5+ijPqNN+2ZPpvyjVNdN1CDVcg==", + "dependencies": { + "package-json-path": "^1.0.0", + "r-json": "^1.2.1" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/read-all-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/read-all-stream/-/read-all-stream-3.1.0.tgz", + "integrity": "sha512-DI1drPHbmBcUDWrJ7ull/F2Qb8HkwBncVx8/RpKYFSIACYaVRQReISYPdZz/mt1y1+qMCOrfReTopERmaxtP6w==", + "dependencies": { + "pinkie-promise": "^2.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "dev": true, + "optional": true, + "dependencies": { + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/readable-web-to-node-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "optional": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "dev": true, + "optional": true + }, + "node_modules/registry-auth-token": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", + "integrity": "sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==", + "dependencies": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha512-ZbgR5aZEdf4UKZVBPYIgaglBmSF2Hi94s2PcIHhRGFjKYu+chjJdYfHn4rt3hB6eCKLJ8giVIIfgMa1ehDfZKA==", + "dependencies": { + "rc": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, + "node_modules/resolve": { + "version": "1.22.2", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", + "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", + "dependencies": { + "is-core-module": "^2.11.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", + "dev": true, + "optional": true + }, + "node_modules/selenium-webdriver": { + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.8.2.tgz", + "integrity": "sha512-d2dcpDLPcXlBy5qcPtB1B8RYTtj1N+JiHQLViFx3OP+i5hqkAuqTfJEYUh4qNX11S4NvbxjteiwN3OPwK3vPVw==", + "dev": true, + "dependencies": { + "jszip": "^3.10.0", + "tmp": "^0.2.1", + "ws": ">=8.11.0" + }, + "engines": { + "node": ">= 14.20.0" + } + }, + "node_modules/selenium-webdriver/node_modules/tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "dependencies": { + "rimraf": "^3.0.0" + }, + "engines": { + "node": ">=8.17.0" + } + }, + "node_modules/selenium-webdriver/node_modules/ws": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "dev": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "dev": true + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "node_modules/sliced": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", + "integrity": "sha512-VZBmZP8WU3sMOZm1bdgTadsQbcscK0UM8oKxKVBs4XAhUo2Xxzm/OFMGBkPusxw9xL3Uy8LrzEqGqJhclsr0yA==" + }, + "node_modules/spdx-correct": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", + "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", + "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==" + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.13", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.13.tgz", + "integrity": "sha512-XkD+zwiqXHikFZm4AX/7JSCXA98U5Db4AFd5XUg/+9UNtnH75+Z9KxtpYiJZx36mUDVOwH83pl7yvCer6ewM3w==" + }, + "node_modules/split": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==", + "dev": true, + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/strftime": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.1.tgz", + "integrity": "sha512-nVvH6JG8KlXFPC0f8lojLgEsPA18lRpLZ+RrJh/NkQV2tqOgZfbas8gcU8SFgnnqR3rWzZPYu6N2A3xzs/8rQg==", + "dev": true, + "engines": { + "node": ">=0.2.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "dev": true, + "optional": true, + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/symbol": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/symbol/-/symbol-0.3.1.tgz", + "integrity": "sha512-SxMrE6uv9zhnBmTCpZna1u0TcZix1k2QASZ/DpF13rAo+0Ts40faFYsMTuAirgvbbjHw1byhJ949/fP20XzVZA==", + "dev": true + }, + "node_modules/tar": { + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==", + "dev": true, + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^4.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, + "node_modules/timed-out": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-2.0.0.tgz", + "integrity": "sha512-pqqJOi1rF5zNs/ps4vmbE4SFCrM4iR7LW+GHAsHqO/EumqbIWceioevYLM5xZRgQSH6gFgL9J/uB7EcJhQ9niQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/timm": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/timm/-/timm-1.7.1.tgz", + "integrity": "sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw==", + "dev": true, + "optional": true + }, + "node_modules/tinycolor2": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", + "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", + "dev": true, + "optional": true + }, + "node_modules/tmp": { + "version": "0.0.28", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz", + "integrity": "sha512-c2mmfiBmND6SOVxzogm1oda0OJ1HZVIk/5n26N59dDTh80MUeavpiCls4PGAdkX1PFkKokLpcf7prSjCeXLsJg==", + "dependencies": { + "os-tmpdir": "~1.0.1" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "dev": true, + "optional": true, + "dependencies": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/tough-cookie": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", + "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", + "dev": true, + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "optional": true + }, + "node_modules/typpy": { + "version": "2.3.13", + "resolved": "https://registry.npmjs.org/typpy/-/typpy-2.3.13.tgz", + "integrity": "sha512-vOxIcQz9sxHi+rT09SJ5aDgVgrPppQjwnnayTrMye1ODaU8gIZTDM19t9TxmEElbMihx2Nq/0/b/MtyKfayRqA==", + "dependencies": { + "function.name": "^1.0.3" + } + }, + "node_modules/ul": { + "version": "5.2.15", + "resolved": "https://registry.npmjs.org/ul/-/ul-5.2.15.tgz", + "integrity": "sha512-svLEUy8xSCip5IWnsRa0UOg+2zP0Wsj4qlbjTmX6GJSmvKMHADBuHOm1dpNkWqWPIGuVSqzUkV3Cris5JrlTRQ==", + "dependencies": { + "deffy": "^2.2.2", + "typpy": "^2.3.4" + } + }, + "node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/unzip-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-1.0.2.tgz", + "integrity": "sha512-pwCcjjhEcpW45JZIySExBHYv5Y9EeL2OIGEfrSKp2dMUFGFv4CpvZkwJbVge8OvGH2BNNtJBx67DuKuJhf+N5Q==", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/url-parse-lax": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", + "integrity": "sha512-BVA4lR5PIviy2PMseNd2jbFQ+jwSwQGdJejf5ctd1rEXt0Ypd7yanUK9+lYechVlN5VaTJGsu2U/3MDDu6KgBA==", + "dependencies": { + "prepend-http": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/utcstring": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/utcstring/-/utcstring-0.1.0.tgz", + "integrity": "sha512-1EpWQ6CECkoys7aX3LImrFo4nYIigY2RQHJTvgzZQCB4/oA6jJvTLTcgilTxX57GrSHDIVMtGwYd+SujGJvvyw==", + "dev": true + }, + "node_modules/utif2": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/utif2/-/utif2-4.0.1.tgz", + "integrity": "sha512-KMaD76dbzK1VjbwsckHJiqDjhP3pbpwyV+FdqkY6XFQenc2o/HS6pjPSYdu4+NQMHf2NLTW+nVP/eFP1CvOYQQ==", + "dev": true, + "optional": true, + "dependencies": { + "pako": "^1.0.11" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "optional": true + }, + "node_modules/whatwg-fetch": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.2.tgz", + "integrity": "sha512-bJlen0FcuU/0EMLrdbJ7zOnW6ITZLrZMIarMUVmdKtsGvZna8vxKYaexICWPfZ8qwf9fzNq+UEIZrnSaApt6RA==", + "dev": true, + "optional": true + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "optional": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "dev": true, + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xhr": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/xhr/-/xhr-2.6.0.tgz", + "integrity": "sha512-/eCGLb5rxjx5e3mF1A7s+pLlR6CGyqWN91fv1JgER5mVWg1MZmlhBvy9kjcsOdRk8RrIujotWyJamfyrp+WIcA==", + "dev": true, + "optional": true, + "dependencies": { + "global": "~4.4.0", + "is-function": "^1.0.1", + "parse-headers": "^2.0.0", + "xtend": "^4.0.0" + } + }, + "node_modules/xml-parse-from-string": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz", + "integrity": "sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==", + "dev": true, + "optional": true + }, + "node_modules/xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "dev": true, + "optional": true, + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "dev": true, + "optional": true, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, + "optional": true, + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "node_modules/yargs": { + "version": "17.7.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.1.tgz", + "integrity": "sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "dev": true, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + }, + "dependencies": { + "@cypress/xvfb": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@cypress/xvfb/-/xvfb-1.2.4.tgz", + "integrity": "sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q==", + "dev": true, + "requires": { + "debug": "^3.1.0", + "lodash.once": "^4.1.1" + } + }, + "@devicefarmer/adbkit": { + "version": "2.11.3", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit/-/adbkit-2.11.3.tgz", + "integrity": "sha512-rsgWREAvSRQjdP9/3GoAV6Tq+o97haywgbTfCgt5yUqiDpaaq3hlH9FTo9XsdG8x+Jd0VQ9nTC2IXsDu8JGRSA==", + "dev": true, + "requires": { + "@devicefarmer/adbkit-logcat": "^1.1.0", + "@devicefarmer/adbkit-monkey": "~1.0.1", + "bluebird": "~2.9.24", + "commander": "^2.3.0", + "debug": "~2.6.3", + "node-forge": "^0.10.0", + "split": "~0.3.3" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + } + } + }, + "@devicefarmer/adbkit-logcat": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit-logcat/-/adbkit-logcat-1.1.0.tgz", + "integrity": "sha512-K90P5gUXM/w+yzLvJIRQ+tJooNU6ipUPPQkljtPJ0laR66TGtpt4Gqsjm0n9dPHK1W5KGgU1R5wnCd6RTSlPNA==", + "dev": true + }, + "@devicefarmer/adbkit-monkey": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@devicefarmer/adbkit-monkey/-/adbkit-monkey-1.0.1.tgz", + "integrity": "sha512-HilPrVrCosYWqSyjfpDtaaN1kJwdlBpS+IAflP3z+e7nsEgk3JGJf1Vg0NgHJooTf5HDfXSyZqMVg+5jvXCK0g==", + "dev": true, + "requires": { + "async": "~0.2.9" + } + }, + "@jimp/bmp": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/bmp/-/bmp-0.22.7.tgz", + "integrity": "sha512-0cfBPvugURS7G+60vRBL+penDRst8x40alS5Rhn2nlGsgsBHljFDw7+H4o5r6gldw9nv9PR9JA90Wloy7KMZdQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7", + "bmp-js": "^0.1.0" + } + }, + "@jimp/core": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/core/-/core-0.22.7.tgz", + "integrity": "sha512-lg4z+pw23v2Gp9LWQur0NqYtnmoNWnyN/Or96elhJgeEJskrDGwROdajortHCCOI1xDnUZSirg8sFvStC8BIlg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7", + "any-base": "^1.1.0", + "buffer": "^5.2.0", + "exif-parser": "^0.1.12", + "file-type": "^16.5.4", + "isomorphic-fetch": "^3.0.0", + "mkdirp": "^2.1.3", + "pixelmatch": "^4.0.2", + "tinycolor2": "^1.6.0" + } + }, + "@jimp/custom": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.22.7.tgz", + "integrity": "sha512-n+1+ZVDNumB1E+sL7KdGKAJ6MbgniX1/v/xOEFEQ46WDZ4cRTqP4+tXjHTuHSlOXiANH+K9zD6qgzqmgO6mCVw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/core": "^0.22.7" + } + }, + "@jimp/gif": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/gif/-/gif-0.22.7.tgz", + "integrity": "sha512-PGZMS8sYFnDcqg+t8IT3RaSJLrqB+3GzhI0hU5D4mmSuJ5UO/6Bdgu8nrwh3uFPxw0ZH6h9ozYk88cz0pKEhLQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7", + "gifwrap": "^0.9.2", + "omggif": "^1.0.9" + } + }, + "@jimp/jpeg": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/jpeg/-/jpeg-0.22.7.tgz", + "integrity": "sha512-ptwWyX/7RPcREy8SpPN/8IlywbwyPXiuXmoHwM6m4iKcyaCmmnfCdZwLNXYliJzFAFLLOWDuOrwO3cZSkH6Czg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7", + "jpeg-js": "^0.4.4" + } + }, + "@jimp/plugin-blit": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.22.7.tgz", + "integrity": "sha512-8oXcBTSd/sBmTQATrCxQ1ZBER31Lge8vXzWqNCbC3b1ZvRggCcqnDzRRH1+JiI4i+jPRo3Fi6/sdvEUyQ5LY3g==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-blur": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.22.7.tgz", + "integrity": "sha512-M+0I5CKFIpnIQE27j8o8NECBsOFBd4z7C95ydy2UohYopugFq+hSVtMs1D4pQgb0RW1DJPiXD/4PHqb+lzV5mA==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-circle": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-circle/-/plugin-circle-0.22.7.tgz", + "integrity": "sha512-zfZKKpOhlyiDeFjGW5JB9K4h/kvbdaAJWUEwmKrvvGar67G3j8dKu46AX0MeWRNZ1yk/lfz+JIa7TzKfxEBf6w==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-color": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.22.7.tgz", + "integrity": "sha512-OXro9pdB0twQjV4LgW0bTEXaX1VgBsTBcFoDAs8q9mtQzD5p3UQmJ+ykCiQ5rTPxNN1Buc44tcCIfp8haB1ZVQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7", + "tinycolor2": "^1.6.0" + } + }, + "@jimp/plugin-contain": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-contain/-/plugin-contain-0.22.7.tgz", + "integrity": "sha512-GwUxZp4jMA0O0qbknUPDONJAfHFaTRs8kK+jgRtUfgb1Xi96l5RN/PMMDv4owZCUiPVAON80X1BMj7nSQWNVUw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-cover": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-cover/-/plugin-cover-0.22.7.tgz", + "integrity": "sha512-PVXeQyofGepMoJaQ5XapLwCcZfsOF1IoAotHosh8AOP8niCP/Erm8T6ZWf5tf0sMJiLHQMPUyns186H5isqEMQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-crop": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.22.7.tgz", + "integrity": "sha512-XXvUU+hPdodtTBSgyUJUnzh7JgKMVlS1GxjcQsjYU8iGr1dbpuazKMTQxc76ChVmy8ue4goi8bGstacWUHpl/Q==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-displace": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-displace/-/plugin-displace-0.22.7.tgz", + "integrity": "sha512-CCNAkmm2OS4QQtNRfQvXqoAMxNE0maSlVEV5DNdioHOUKycy02EJ5hNYR3l0FG+NraQHOuqv9XV37sGRl6QzMA==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-dither": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-dither/-/plugin-dither-0.22.7.tgz", + "integrity": "sha512-ndCW5MIGMdh3aBvvgRCO7el9cIPG29kU7xQYlOs5+3JsDk3Vf7X30QGPjzxABOY95qLUNUjf5Qe/p/tqv/vbcw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-fisheye": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-fisheye/-/plugin-fisheye-0.22.7.tgz", + "integrity": "sha512-boI1QowhZRfb6OF+ZPWtiSJP1GATsTHjd5Oy/lJ+n0L4rp439ZOTB1Elzcgc44O2C1mgZDdybRPQQvYdPF8slA==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-flip": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-flip/-/plugin-flip-0.22.7.tgz", + "integrity": "sha512-/jkbgtvQPcKadAEV5ZXyoEpSdd7GEvGs/Ya/f48+LNszc+S24u4UXtuP3QPRJ5FHm0Re1t4uztM7xa6IPklAOA==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-gaussian": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-gaussian/-/plugin-gaussian-0.22.7.tgz", + "integrity": "sha512-OB1sdnjzq2rfUHmx9Rvi3SJIDbQAgWFgYEw6KhN3TSVOdrJHvwrQkEnwR9PoUzQg992VIpGcVc9Y1s/SOU2oCA==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-invert": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-invert/-/plugin-invert-0.22.7.tgz", + "integrity": "sha512-dX/TqACJ/M5uXDIEJlVPPwietMD6EWUeA/CV4uvhLz9EMjTgHociJ3TWqGCY/70phhIBLbhLcHUVBL/q65ynfQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-mask": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-mask/-/plugin-mask-0.22.7.tgz", + "integrity": "sha512-rfKHKJLAtJG7qbB4zYAMcQ9ue3CIFRuAJ3xX0lzCxC0fGvCVuXlcxiAEauBxqaTWqiKMnahqpR3/Ah679K2FKQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-normalize": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-normalize/-/plugin-normalize-0.22.7.tgz", + "integrity": "sha512-t8x2jjKDmvUAZB4Wbeagr4D0BvoVCIWquy94mpglvSZ8ujKLt0aQBl3CBEIbXFAoVqNif+G36NtxPHNsjxIXOg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-print": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-print/-/plugin-print-0.22.7.tgz", + "integrity": "sha512-kx0+cPeinki1IFg9cJy7LC4uVuOEOa8TIrcERioB6PVgJ7EDzCAfatTKULZ+t4uSs2K/lQF97wPYlbiyxs/Hzg==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7", + "load-bmfont": "^1.4.1" + } + }, + "@jimp/plugin-resize": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.22.7.tgz", + "integrity": "sha512-pg7i0JIYt7x7ag+CoD/yG70Xvwm1sKRfcFjQh954yestiin14uppPgXchAmTBmctecBjLNdsVlqSXbPvU4Jvxw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-rotate": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.22.7.tgz", + "integrity": "sha512-Uh3Gb18IY8uXWk6E1bzMopum2GP+xwohbnMIDE0MSWmLaz7LXrfnvgXFba1uRGgn73CJz8UDS4fC1KIJMuxQZA==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-scale": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.22.7.tgz", + "integrity": "sha512-3uHUrk5Rl6MCxuoJtHTSeJjSHIxHWqOOgmD2caKIvyxds0Zmofu/Fva+N4V/m80E4q4G2RXNsUplFpFGhUM7hw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-shadow": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-shadow/-/plugin-shadow-0.22.7.tgz", + "integrity": "sha512-NKEq5VR8U/d0OKf0hxFtrrbMCuNv7by31V+Kwgxb1oTP+j+zZEaww+m3YgEwIwRe7E8/yeDSHa5bJ+CmuyFZjw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugin-threshold": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugin-threshold/-/plugin-threshold-0.22.7.tgz", + "integrity": "sha512-BH4aLwfmnqjRVhdzMIqUns4ycZ6QoHHFR6Qz+X2iSpH5a33xFA4DRbd3Ehtrs4Gk7XiCjWkUyM6wjmH7l/1hNQ==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7" + } + }, + "@jimp/plugins": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/plugins/-/plugins-0.22.7.tgz", + "integrity": "sha512-AJmzTG/sa+CDpvle/UE89hjHR85gnRGSwLuQqPbhlY6GFCmC3uqHRJz9O5I8A4zdi9+e8LsBphuTlKV7RbuXOw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/plugin-blit": "^0.22.7", + "@jimp/plugin-blur": "^0.22.7", + "@jimp/plugin-circle": "^0.22.7", + "@jimp/plugin-color": "^0.22.7", + "@jimp/plugin-contain": "^0.22.7", + "@jimp/plugin-cover": "^0.22.7", + "@jimp/plugin-crop": "^0.22.7", + "@jimp/plugin-displace": "^0.22.7", + "@jimp/plugin-dither": "^0.22.7", + "@jimp/plugin-fisheye": "^0.22.7", + "@jimp/plugin-flip": "^0.22.7", + "@jimp/plugin-gaussian": "^0.22.7", + "@jimp/plugin-invert": "^0.22.7", + "@jimp/plugin-mask": "^0.22.7", + "@jimp/plugin-normalize": "^0.22.7", + "@jimp/plugin-print": "^0.22.7", + "@jimp/plugin-resize": "^0.22.7", + "@jimp/plugin-rotate": "^0.22.7", + "@jimp/plugin-scale": "^0.22.7", + "@jimp/plugin-shadow": "^0.22.7", + "@jimp/plugin-threshold": "^0.22.7", + "timm": "^1.6.1" + } + }, + "@jimp/png": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/png/-/png-0.22.7.tgz", + "integrity": "sha512-LxD3O9FKEwVv+j+HcUV7ez72Miy+823EjhtFZbBYXNp9qjHtHFBpgcSJBftUOCei8OlmmVgULYn9XjyfPsDgGw==", + "dev": true, + "optional": true, + "requires": { + "@jimp/utils": "^0.22.7", + "pngjs": "^6.0.0" + } + }, + "@jimp/tiff": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/tiff/-/tiff-0.22.7.tgz", + "integrity": "sha512-/oE8kLumzBfU1Z6h4TrDXYCGQNc4CjbZQvPssjImEqNLr5vbefpIpoy1fVMpsyuHZHsGovsBhBHxTJaRLO4+Og==", + "dev": true, + "optional": true, + "requires": { + "utif2": "^4.0.1" + } + }, + "@jimp/types": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/types/-/types-0.22.7.tgz", + "integrity": "sha512-1T8BxwDh5HJvBh3tt6HUd8r7ir5Ge3JWATXC8O3Y9QYwOaERjA2+FVhGSjtoo5xCeJvLRjSzEtfZ8heowMBL4w==", + "dev": true, + "optional": true, + "requires": { + "@jimp/bmp": "^0.22.7", + "@jimp/gif": "^0.22.7", + "@jimp/jpeg": "^0.22.7", + "@jimp/png": "^0.22.7", + "@jimp/tiff": "^0.22.7", + "timm": "^1.6.1" + } + }, + "@jimp/utils": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/@jimp/utils/-/utils-0.22.7.tgz", + "integrity": "sha512-4ax4IOWLIERx4yz9y3fNXKvQaPOY23yJF5h4sizxVkQUObkZHWE0kL0TVHodBt3rS8ksdbCL8Jkz4GeNP/Katg==", + "dev": true, + "optional": true, + "requires": { + "regenerator-runtime": "^0.13.3" + } + }, + "@sitespeed.io/chromedriver": { + "version": "112.0.5615-28", + "resolved": "https://registry.npmjs.org/@sitespeed.io/chromedriver/-/chromedriver-112.0.5615-28.tgz", + "integrity": "sha512-Um/D7PQ5NlfCT2Khq8iEzescNy2L4xR+cK/vC/tRRCBAOZXMCXb1oY0OE7Bw2CIhtPf4a2Ldi3cRU6oSEAUb+w==", + "dev": true, + "requires": { + "node-downloader-helper": "2.1.5", + "node-stream-zip": "1.15.0" + } + }, + "@sitespeed.io/edgedriver": { + "version": "112.0.1722-34", + "resolved": "https://registry.npmjs.org/@sitespeed.io/edgedriver/-/edgedriver-112.0.1722-34.tgz", + "integrity": "sha512-px7L9eZPGQ3iWti3vBurFs7eoUg7x9yA7QpQmy9AdkgUm3kzuGjxt/sszvy+s/DLgOtDhJGzwMAb9JuGslzj/w==", + "dev": true, + "requires": { + "node-downloader-helper": "2.1.6", + "node-stream-zip": "1.15.0" + }, + "dependencies": { + "node-downloader-helper": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.6.tgz", + "integrity": "sha512-VkOvAXIopI3xMuM/MC5UL7NqqnizQ/9QXZt28jR8FPZ6fHLQm4xe4+YXJ9FqsWwLho5BLXrF51nfOQ0QcohRkQ==", + "dev": true + } + } + }, + "@sitespeed.io/geckodriver": { + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/@sitespeed.io/geckodriver/-/geckodriver-0.33.0.tgz", + "integrity": "sha512-w6w+x9/Q44JekTPi8NlRsfh5Uz4TfquJcUEs0tA/oEcxLVxRS7VtaiaJEE0GzzN6cUmFS6Twas7E4bCA4k/Yxg==", + "dev": true, + "requires": { + "node-downloader-helper": "2.1.5", + "node-stream-zip": "1.15.0", + "tar": "6.1.13" + } + }, + "@sitespeed.io/throttle": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@sitespeed.io/throttle/-/throttle-5.0.0.tgz", + "integrity": "sha512-eul4I7IllA6l3+GGX1aW/D75XYux0ODuZDzstKD0kAuvIkpQ4BVLkFBoLXQN50gLMFGqZ3QWMobhQ5L2/6sFgg==", + "dev": true, + "requires": { + "minimist": "1.2.6" + } + }, + "@sitespeed.io/tracium": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@sitespeed.io/tracium/-/tracium-0.3.3.tgz", + "integrity": "sha512-dNZafjM93Y+F+sfwTO5gTpsGXlnc/0Q+c2+62ViqP3gkMWvHEMSKkaEHgVJLcLg3i/g19GSIPziiKpgyne07Bw==", + "dev": true, + "requires": { + "debug": "^4.1.1" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "dev": true, + "optional": true + }, + "@types/node": { + "version": "16.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-16.9.1.tgz", + "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==", + "dev": true, + "optional": true + }, + "abs": { + "version": "1.3.14", + "resolved": "https://registry.npmjs.org/abs/-/abs-1.3.14.tgz", + "integrity": "sha512-PrS26IzwKLWwuURpiKl8wRmJ2KdR/azaVrLEBWG/TALwT20Y7qjtYp1qcMLHA4206hBHY5phv3w4pjf9NPv4Vw==", + "requires": { + "ul": "^5.0.0" + } + }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "dev": true + }, + "any-base": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz", + "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==", + "dev": true, + "optional": true + }, + "async": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==", + "dev": true + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "optional": true + }, + "bluebird": { + "version": "2.9.34", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.34.tgz", + "integrity": "sha512-ZDzCb87X7/IP1uzQ5eJZB+WoQRGTnKL5DHWvPw6kkMbQseouiQIrEi3P1UGE0D1k0N5/+aP/5GMCyHZ1xYJyHQ==", + "dev": true + }, + "bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "dev": true, + "optional": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "browsertime": { + "version": "https://github.com/sitespeedio/browsertime/tarball/6f45b9e5a6e0419fe6408f05909f2ab5a1f6bacc", + "integrity": "sha512-bAUNDY0aTl7OvS5OpU2vVopykhhNWmwk71qXi7ttAzEyr+ZD3rVjbsfmJ+kE02vSjxAiTGgfGR7Sr5KM5mMjIg==", + "dev": true, + "requires": { + "@cypress/xvfb": "1.2.4", + "@devicefarmer/adbkit": "2.11.3", + "@sitespeed.io/chromedriver": "112.0.5615-28", + "@sitespeed.io/edgedriver": "112.0.1722-34", + "@sitespeed.io/geckodriver": "0.33.0", + "@sitespeed.io/throttle": "5.0.0", + "@sitespeed.io/tracium": "0.3.3", + "btoa": "1.2.1", + "chrome-har": "0.13.1", + "chrome-remote-interface": "0.32.1", + "dayjs": "1.11.7", + "execa": "7.0.0", + "fast-stats": "0.0.6", + "ff-test-bidi-har-export": "0.0.10", + "find-up": "6.3.0", + "get-port": "6.1.2", + "hasbin": "1.2.3", + "intel": "1.2.0", + "jimp": "0.22.7", + "lodash.get": "4.4.2", + "lodash.groupby": "4.6.0", + "lodash.isempty": "4.4.0", + "lodash.merge": "4.6.2", + "lodash.pick": "4.4.0", + "lodash.set": "4.3.2", + "selenium-webdriver": "4.8.2", + "yargs": "17.7.1" + } + }, + "btoa": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", + "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==", + "dev": true + }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "optional": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "buffer-equal": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-0.0.1.tgz", + "integrity": "sha512-RgSV6InVQ9ODPdLWJ5UAqBqJBOg370Nz6ZQtRzpt6nUjc8v0St97uJ4PYC6NztqIScrAXafKM3mZPMygSe1ggA==", + "dev": true, + "optional": true + }, + "capture-stack-trace": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.2.tgz", + "integrity": "sha512-X/WM2UQs6VMHUtjUDnZTRI+i1crWteJySFzr9UpGoQa4WQffXVTTXuekjl7TjZRlcF2XfjgITT0HxZ9RnxeT0w==" + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "dev": true + }, + "chrome-har": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/chrome-har/-/chrome-har-0.13.1.tgz", + "integrity": "sha512-R9OXxVeCOqJYmqO8Oco+nouJ4frwVFQ6rGUgj5dJOfVNxBMmwCannp+DH2IAfqY/K8WJ5VYCQyq7SLTNZtcCXA==", + "dev": true, + "requires": { + "dayjs": "1.11.7", + "debug": "4.3.4", + "tough-cookie": "4.1.2", + "uuid": "9.0.0" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "chrome-remote-interface": { + "version": "0.32.1", + "resolved": "https://registry.npmjs.org/chrome-remote-interface/-/chrome-remote-interface-0.32.1.tgz", + "integrity": "sha512-CU3/K/8YlU2H0DjsLRbxPsG4piiSGUcIy6GkGXF11SqOYoIeuUBivOsGXScaZnTyC1p4wFSR+GNmAM434/ALWw==", + "dev": true, + "requires": { + "commander": "2.11.x", + "ws": "^7.2.0" + }, + "dependencies": { + "commander": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.11.0.tgz", + "integrity": "sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==", + "dev": true + } + } + }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "create-error-class": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", + "integrity": "sha512-gYTKKexFO3kh200H1Nit76sRwRtOY32vQd3jpAQKpLtZqyNsSQNfI4N7o3eP2wUjV35pTWKRYqFUDBvUha/Pkw==", + "requires": { + "capture-stack-trace": "^1.0.0" + } + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "dayjs": { + "version": "1.11.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.7.tgz", + "integrity": "sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==", + "dev": true + }, + "dbug": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/dbug/-/dbug-0.4.2.tgz", + "integrity": "sha512-nrmsMK1msY0WXwfA2czrKVDgpIYJR2JJaq5cX4DwW7Rxm11nXHqouh9wmubEs44bHYxk8CqeP/Jx4URqSB961w==", + "dev": true + }, + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, + "deffy": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/deffy/-/deffy-2.2.4.tgz", + "integrity": "sha512-pLc9lsbsWjr6RxmJ2OLyvm+9l4j1yK69h+TML/gUit/t3vTijpkNGh8LioaJYTGO7F25m6HZndADcUOo2PsiUg==", + "requires": { + "typpy": "^2.0.0" + } + }, + "dom-walk": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.2.tgz", + "integrity": "sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w==", + "dev": true, + "optional": true + }, + "duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "requires": { + "readable-stream": "^2.0.2" + } + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "err": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/err/-/err-1.1.1.tgz", + "integrity": "sha512-N97Ybd2jJHVQ+Ft3Q5+C2gM3kgygkdeQmEqbN2z15UTVyyEsIwLA1VK39O1DHEJhXbwIFcJLqm6iARNhFANcQA==", + "requires": { + "typpy": "^2.2.0" + } + }, + "error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true + }, + "exec-limiter": { + "version": "3.2.13", + "resolved": "https://registry.npmjs.org/exec-limiter/-/exec-limiter-3.2.13.tgz", + "integrity": "sha512-86Ri699bwiHZVBzTzNj8gspqAhCPchg70zPVWIh3qzUOA1pUMcb272Em3LPk8AE0mS95B9yMJhtqF8vFJAn0dA==", + "requires": { + "limit-it": "^3.0.0", + "typpy": "^2.1.0" + } + }, + "execa": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-7.0.0.tgz", + "integrity": "sha512-tQbH0pH/8LHTnwTrsKWideqi6rFB/QNUawEwrn+WHyz7PX1Tuz2u7wfTvbaNBdP5JD5LVWxNo8/A8CHNZ3bV6g==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.1", + "human-signals": "^4.3.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^3.0.7", + "strip-final-newline": "^3.0.0" + } + }, + "exif-parser": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz", + "integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw==", + "dev": true, + "optional": true + }, + "fast-stats": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/fast-stats/-/fast-stats-0.0.6.tgz", + "integrity": "sha512-m0zkwa7Z07Wc4xm1YtcrCHmhzNxiYRrrfUyhkdhSZPzaAH/Ewbocdaq7EPVBFz19GWfIyyPcLfRHjHJYe83jlg==", + "dev": true + }, + "ff-test-bidi-har-export": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/ff-test-bidi-har-export/-/ff-test-bidi-har-export-0.0.10.tgz", + "integrity": "sha512-0azA5fUrA3Fn4KFAg2LcFqM2Xav7+H41gj4GHgLMhZxSyHfPV0yhbs1y+6sv+skgak/IbBb75KZmkj0VFsdyEg==", + "dev": true + }, + "file-type": { + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "dev": true, + "optional": true, + "requires": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + } + }, + "find-up": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-6.3.0.tgz", + "integrity": "sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==", + "dev": true, + "requires": { + "locate-path": "^7.1.0", + "path-exists": "^5.0.0" + } + }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dev": true, + "requires": { + "minipass": "^3.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + } + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "function.name": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/function.name/-/function.name-1.0.13.tgz", + "integrity": "sha512-mVrqdoy5npWZyoXl4DxCeuVF6delDcQjVS9aPdvLYlBxtMTZDR2B5GVEQEoM1jJyspCqg3C0v4ABkLE7tp9xFA==", + "requires": { + "noop6": "^1.0.1" + } + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "get-port": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", + "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", + "dev": true + }, + "get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true + }, + "gifwrap": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/gifwrap/-/gifwrap-0.9.4.tgz", + "integrity": "sha512-MDMwbhASQuVeD4JKd1fKgNgCRL3fGqMM4WaqpNhWO0JiMOAjbQdumbs4BbBZEy9/M00EHEjKN3HieVhCUlwjeQ==", + "dev": true, + "optional": true, + "requires": { + "image-q": "^4.0.0", + "omggif": "^1.0.10" + } + }, + "git-package-json": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/git-package-json/-/git-package-json-1.4.10.tgz", + "integrity": "sha512-DRAcvbzd2SxGK7w8OgYfvKqhFliT5keX0lmSmVdgScgf1kkl5tbbo7Pam6uYoCa1liOiipKxQZG8quCtGWl/fA==", + "requires": { + "deffy": "^2.2.1", + "err": "^1.1.1", + "gry": "^5.0.0", + "normalize-package-data": "^2.3.5", + "oargv": "^3.4.1", + "one-by-one": "^3.1.0", + "r-json": "^1.2.1", + "r-package-json": "^1.0.0", + "tmp": "0.0.28" + } + }, + "git-source": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/git-source/-/git-source-1.1.10.tgz", + "integrity": "sha512-XZZ7ZgnLL35oLgM/xjnLYgtlKlxJG0FohC1kWDvGkU7s1VKGXK0pFF/g1itQEwQ3D+uTQzBnzPi8XbqOv7Wc1Q==", + "requires": { + "git-url-parse": "^5.0.1" + } + }, + "git-up": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/git-up/-/git-up-1.2.1.tgz", + "integrity": "sha512-SRVN3rOLACva8imc7BFrB6ts5iISWKH1/h/1Z+JZYoUI7UVQM7gQqk4M2yxUENbq2jUUT09NEND5xwP1i7Ktlw==", + "requires": { + "is-ssh": "^1.0.0", + "parse-url": "^1.0.0" + } + }, + "git-url-parse": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/git-url-parse/-/git-url-parse-5.0.1.tgz", + "integrity": "sha512-4uSiOgrryNEMBX+gTWogenYRUh2j1D+95STTSEF2RCTgLkfJikl8c7BGr0Bn274hwuxTsbS2/FQ5pVS9FoXegQ==", + "requires": { + "git-up": "^1.0.0" + } + }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "global": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/global/-/global-4.4.0.tgz", + "integrity": "sha512-wv/LAoHdRE3BeTGz53FAamhGlPLhlssK45usmGFThIi4XqnBmjKQ16u+RNbP7WvigRZDxUsM0J3gcQ5yicaL0w==", + "dev": true, + "optional": true, + "requires": { + "min-document": "^2.19.0", + "process": "^0.11.10" + } + }, + "got": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-5.6.0.tgz", + "integrity": "sha512-MnypzkaW8dldA8AbJFjMs7y14+ykd2V8JCLKSvX1Gmzx1alH3Y+3LArywHDoAF2wS3pnZp4gacoYtvqBeF6drQ==", + "requires": { + "create-error-class": "^3.0.1", + "duplexer2": "^0.1.4", + "is-plain-obj": "^1.0.0", + "is-redirect": "^1.0.0", + "is-retry-allowed": "^1.0.0", + "is-stream": "^1.0.0", + "lowercase-keys": "^1.0.0", + "node-status-codes": "^1.0.0", + "object-assign": "^4.0.1", + "parse-json": "^2.1.0", + "pinkie-promise": "^2.0.0", + "read-all-stream": "^3.0.0", + "readable-stream": "^2.0.5", + "timed-out": "^2.0.0", + "unzip-response": "^1.0.0", + "url-parse-lax": "^1.0.0" + }, + "dependencies": { + "is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==" + } + } + }, + "gry": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/gry/-/gry-5.0.8.tgz", + "integrity": "sha512-meq9ZjYVpLzZh3ojhTg7IMad9grGsx6rUUKHLqPnhLXzJkRQvEL2U3tQpS5/WentYTtHtxkT3Ew/mb10D6F6/g==", + "requires": { + "abs": "^1.2.1", + "exec-limiter": "^3.0.0", + "one-by-one": "^3.0.0", + "ul": "^5.0.0" + } + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "hasbin": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/hasbin/-/hasbin-1.2.3.tgz", + "integrity": "sha512-CCd8e/w2w28G8DyZvKgiHnQJ/5XXDz6qiUHnthvtag/6T5acUeN5lqq+HMoBqcmgWueWDhiCplrw0Kb1zDACRg==", + "dev": true, + "requires": { + "async": "~1.5" + }, + "dependencies": { + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha512-nSVgobk4rv61R9PUSDtYt7mPVB2olxNR5RWJcAsH676/ef11bUZwvu7+RGYrYauVdDPcO519v68wRhXQtxsV9w==", + "dev": true + } + } + }, + "hosted-git-info": { + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", + "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==" + }, + "human-signals": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-4.3.1.tgz", + "integrity": "sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ==", + "dev": true + }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "optional": true + }, + "image-q": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/image-q/-/image-q-4.0.0.tgz", + "integrity": "sha512-PfJGVgIfKQJuq3s0tTDOKtztksibuUEbJQIYT3by6wctQo+Rdlh7ef4evJ5NCdxY4CfMbvFkocEwbl4BF8RlJw==", + "dev": true, + "optional": true, + "requires": { + "@types/node": "16.9.1" + } + }, + "immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "intel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/intel/-/intel-1.2.0.tgz", + "integrity": "sha512-CUDyAtEeEeDo5YtwANOuDhxuFEOgInHvbMrBbhXCD4tAaHuzHM2llevtTeq2bmP8Jf7NkpN305pwDncRmhc1Wg==", + "dev": true, + "requires": { + "chalk": "^1.1.0", + "dbug": "~0.4.2", + "stack-trace": "~0.0.9", + "strftime": "~0.10.0", + "symbol": "~0.3.1", + "utcstring": "~0.1.0" + } + }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" + }, + "is-core-module": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz", + "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==", + "requires": { + "has": "^1.0.3" + } + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "is-function": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-function/-/is-function-1.0.2.tgz", + "integrity": "sha512-lw7DUp0aWXYg+CBCN+JKkcE0Q2RayZnSvnZBlwgxHBQhqt5pZNVy4Ri7H9GmmXkdu7LUthszM+Tor1u/2iBcpQ==", + "dev": true, + "optional": true + }, + "is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==" + }, + "is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha512-cr/SlUEe5zOGmzvj9bUyC4LVvkNVAXu4GytXLNMr1pny+a65MpQ9IJzFHD5vi7FyJgb4qt27+eS3TuQnqB+RQw==" + }, + "is-retry-allowed": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", + "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==" + }, + "is-ssh": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/is-ssh/-/is-ssh-1.4.0.tgz", + "integrity": "sha512-x7+VxdxOdlV3CYpjvRLBv5Lo9OJerlYanjwFrPR9fuGPjCiNiCzFgAWpiLAohSbsnH4ZAys3SBh+hq5rJosxUQ==", + "requires": { + "protocols": "^2.0.1" + } + }, + "is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "isomorphic-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", + "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", + "dev": true, + "optional": true, + "requires": { + "node-fetch": "^2.6.1", + "whatwg-fetch": "^3.4.1" + } + }, + "iterate-object": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/iterate-object/-/iterate-object-1.3.4.tgz", + "integrity": "sha512-4dG1D1x/7g8PwHS9aK6QV5V94+ZvyP4+d19qDv43EzImmrndysIl4prmJ1hWWIGCqrZHyaHBm6BSEWHOLnpoNw==" + }, + "jimp": { + "version": "0.22.7", + "resolved": "https://registry.npmjs.org/jimp/-/jimp-0.22.7.tgz", + "integrity": "sha512-TJCTJ4ZcFUw6W8XZnR6ajdEu8vSyPi3AuoChs+zLHalXnhAPZgwkzwcXnxey4LNjh1p9dfIUkg8YSQ+q8pBW0A==", + "dev": true, + "optional": true, + "requires": { + "@jimp/custom": "^0.22.7", + "@jimp/plugins": "^0.22.7", + "@jimp/types": "^0.22.7", + "regenerator-runtime": "^0.13.3" + } + }, + "jpeg-js": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz", + "integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg==", + "dev": true, + "optional": true + }, + "jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "dev": true, + "requires": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "requires": { + "immediate": "~3.0.5" + } + }, + "limit-it": { + "version": "3.2.10", + "resolved": "https://registry.npmjs.org/limit-it/-/limit-it-3.2.10.tgz", + "integrity": "sha512-T0NK99pHnkimldr1WUqvbGV1oWDku/xC9J/OqzJFsV1jeOS6Bwl8W7vkeQIBqwiON9dTALws+rX/XPMQqWerDQ==", + "requires": { + "typpy": "^2.0.0" + } + }, + "load-bmfont": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.1.tgz", + "integrity": "sha512-8UyQoYmdRDy81Brz6aLAUhfZLwr5zV0L3taTQ4hju7m6biuwiWiJXjPhBJxbUQJA8PrkvJ/7Enqmwk2sM14soA==", + "dev": true, + "optional": true, + "requires": { + "buffer-equal": "0.0.1", + "mime": "^1.3.4", + "parse-bmfont-ascii": "^1.0.3", + "parse-bmfont-binary": "^1.0.5", + "parse-bmfont-xml": "^1.1.4", + "phin": "^2.9.1", + "xhr": "^2.0.1", + "xtend": "^4.0.0" + } + }, + "locate-path": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-7.2.0.tgz", + "integrity": "sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==", + "dev": true, + "requires": { + "p-locate": "^6.0.0" + } + }, + "lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, + "lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "dev": true + }, + "lodash.isempty": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz", + "integrity": "sha512-oKMuF3xEeqDltrGMfDxAPGIVMSSRv8tbRSODbrs4KGsRRLEhrW8N8Rd4DRgB2+621hY8A8XwwrTVhXWpxFvMzg==", + "dev": true + }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, + "lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "dev": true + }, + "lodash.pick": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", + "integrity": "sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==", + "dev": true + }, + "lodash.set": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", + "integrity": "sha512-4hNPN5jlm/N/HLMCO43v8BXKq9Z7QdAGc/VGrRD61w8gN9g/6jF9A4L1pbUgBLCffi0w9VsXfTOij5x8iTyFvg==", + "dev": true + }, + "lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" + }, + "merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true + }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "optional": true + }, + "mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true + }, + "min-document": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", + "integrity": "sha512-9Wy1B3m3f66bPPmU5hdA4DR4PB2OfDU/+GS3yAB7IQozE3tqXaVv2zOjgla7MEGSRv95+ILmOuvhLkOK6wJtCQ==", + "dev": true, + "optional": true, + "requires": { + "dom-walk": "^0.1.0" + } + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" + }, + "minipass": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.8.tgz", + "integrity": "sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==", + "dev": true + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dev": true, + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "dependencies": { + "minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + } + } + }, + "mkdirp": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-2.1.6.tgz", + "integrity": "sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A==", + "dev": true, + "optional": true + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node-downloader-helper": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/node-downloader-helper/-/node-downloader-helper-2.1.5.tgz", + "integrity": "sha512-sLedzfv8C4VMAvTdDQcjLFAl3gydNeBXh2bLcCzvZRmd4EK0rkoTxJ8tkxnriUSJO/n13skJzH7l6CzCdBwYGg==", + "dev": true + }, + "node-fetch": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", + "dev": true, + "optional": true, + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "node-forge": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==", + "dev": true + }, + "node-status-codes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz", + "integrity": "sha512-1cBMgRxdMWE8KeWCqk2RIOrvUb0XCwYfEsY5/y2NlXyq4Y/RumnOZvTj4Nbr77+Vb2C+kyBoRTdkNOS8L3d/aQ==" + }, + "node-stream-zip": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.15.0.tgz", + "integrity": "sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw==", + "dev": true + }, + "noop6": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/noop6/-/noop6-1.0.9.tgz", + "integrity": "sha512-DB3Hwyd89dPr5HqEPg3YHjzvwh/mCqizC1zZ8vyofqc+TQRyPDnT4wgXXbLGF4z9YAzwwTLi8pNLhGqcbSjgkA==" + }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "requires": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "npm-run-path": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.1.0.tgz", + "integrity": "sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==", + "dev": true, + "requires": { + "path-key": "^4.0.0" + }, + "dependencies": { + "path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true + } + } + }, + "oargv": { + "version": "3.4.10", + "resolved": "https://registry.npmjs.org/oargv/-/oargv-3.4.10.tgz", + "integrity": "sha512-SXaMANv9sr7S/dP0vj0+Ybipa47UE1ntTWQ2rpPRhC6Bsvfl+Jg03Xif7jfL0sWKOYWK8oPjcZ5eJ82t8AP/8g==", + "requires": { + "iterate-object": "^1.1.0", + "ul": "^5.0.0" + } + }, + "obj-def": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/obj-def/-/obj-def-1.0.9.tgz", + "integrity": "sha512-bQ4ya3VYD6FAA1+s6mEhaURRHSmw4+sKaXE6UyXZ1XDYc5D+c7look25dFdydmLd18epUegh398gdDkMUZI9xg==", + "requires": { + "deffy": "^2.2.2" + } + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" + }, + "omggif": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", + "integrity": "sha512-LMJTtvgc/nugXj0Vcrrs68Mn2D1r0zf630VNtqtpI1FEO7e+O9FP4gqs9AcnBaSEeoHIPm28u6qgPR0oyEpGSw==", + "dev": true, + "optional": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "one-by-one": { + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/one-by-one/-/one-by-one-3.2.8.tgz", + "integrity": "sha512-HR/pSzZdm46Xqj58K+Bu64kMbSTw8/u77AwWvV+rprO/OsuR++pPlkUJn+SmwqBGRgHKwSKQ974V3uls7crIeQ==", + "requires": { + "obj-def": "^1.0.0", + "sliced": "^1.0.1" + } + }, + "onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "dev": true, + "requires": { + "mimic-fn": "^4.0.0" + } + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==" + }, + "p-limit": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", + "dev": true, + "requires": { + "yocto-queue": "^1.0.0" + } + }, + "p-locate": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-6.0.0.tgz", + "integrity": "sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==", + "dev": true, + "requires": { + "p-limit": "^4.0.0" + } + }, + "package-json": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-2.4.0.tgz", + "integrity": "sha512-PRg65iXMTt/uK8Rfh5zvzkUbfAPitF17YaCY+IbHsYgksiLvtzWWTUildHth3mVaZ7871OJ7gtP4LBRBlmAdXg==", + "requires": { + "got": "^5.0.0", + "registry-auth-token": "^3.0.1", + "registry-url": "^3.0.3", + "semver": "^5.1.0" + } + }, + "package-json-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/package-json-path/-/package-json-path-1.0.9.tgz", + "integrity": "sha512-uNu7f6Ef7tQHZRnkyVnCtzdSYVN9uBtge/sG7wzcUaawFWkPYUq67iXxRGrQSg/q0tzxIB8jSyIYUKjG2Jn//A==", + "requires": { + "abs": "^1.2.1" + } + }, + "package.json": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/package.json/-/package.json-2.0.1.tgz", + "integrity": "sha512-pSxZ6XR5yEawRN2ekxx9IKgPN5uNAYco7MCPxtBEWMKO3UKWa1X2CtQMzMgloeGj2g2o6cue3Sb5iPkByIJqlw==", + "requires": { + "git-package-json": "^1.4.0", + "git-source": "^1.1.0", + "package-json": "^2.3.1" + } + }, + "pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true + }, + "parse-bmfont-ascii": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-ascii/-/parse-bmfont-ascii-1.0.6.tgz", + "integrity": "sha512-U4RrVsUFCleIOBsIGYOMKjn9PavsGOXxbvYGtMOEfnId0SVNsgehXh1DxUdVPLoxd5mvcEtvmKs2Mmf0Mpa1ZA==", + "dev": true, + "optional": true + }, + "parse-bmfont-binary": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/parse-bmfont-binary/-/parse-bmfont-binary-1.0.6.tgz", + "integrity": "sha512-GxmsRea0wdGdYthjuUeWTMWPqm2+FAd4GI8vCvhgJsFnoGhTrLhXDDupwTo7rXVAgaLIGoVHDZS9p/5XbSqeWA==", + "dev": true, + "optional": true + }, + "parse-bmfont-xml": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/parse-bmfont-xml/-/parse-bmfont-xml-1.1.4.tgz", + "integrity": "sha512-bjnliEOmGv3y1aMEfREMBJ9tfL3WR0i0CKPj61DnSLaoxWR3nLrsQrEbCId/8rF4NyRF0cCqisSVXyQYWM+mCQ==", + "dev": true, + "optional": true, + "requires": { + "xml-parse-from-string": "^1.0.0", + "xml2js": "^0.4.5" + } + }, + "parse-headers": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.5.tgz", + "integrity": "sha512-ft3iAoLOB/MlwbNXgzy43SWGP6sQki2jQvAyBg/zDFAgr9bfNWZIUj42Kw2eJIl8kEi4PbgE6U1Zau/HwI75HA==", + "dev": true, + "optional": true + }, + "parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==", + "requires": { + "error-ex": "^1.2.0" + } + }, + "parse-url": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/parse-url/-/parse-url-1.3.11.tgz", + "integrity": "sha512-1wj9nkgH/5EboDxLwaTMGJh3oH3f+Gue+aGdh631oCqoSBpokzmMmOldvOeBPtB8GJBYJbaF93KPzlkU+Y1ksg==", + "requires": { + "is-ssh": "^1.3.0", + "protocols": "^1.4.0" + }, + "dependencies": { + "protocols": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/protocols/-/protocols-1.4.8.tgz", + "integrity": "sha512-IgjKyaUSjsROSO8/D49Ab7hP8mJgTYcqApOqdPhLoPxAplXmkp+zRvsrSQjFn5by0rhm4VH0GAUELIPpx7B1yg==" + } + } + }, + "path-exists": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-5.0.0.tgz", + "integrity": "sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==", + "dev": true + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, + "peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", + "dev": true, + "optional": true + }, + "phin": { + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/phin/-/phin-2.9.3.tgz", + "integrity": "sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA==", + "dev": true, + "optional": true + }, + "pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==" + }, + "pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "requires": { + "pinkie": "^2.0.0" + } + }, + "pixelmatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-4.0.2.tgz", + "integrity": "sha512-J8B6xqiO37sU/gkcMglv6h5Jbd9xNER7aHzpfRdNmV4IbQBzBpe4l9XmbG+xPF/znacgu2jfEw+wHffaq/YkXA==", + "dev": true, + "optional": true, + "requires": { + "pngjs": "^3.0.0" + }, + "dependencies": { + "pngjs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==", + "dev": true, + "optional": true + } + } + }, + "pngjs": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-6.0.0.tgz", + "integrity": "sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==", + "dev": true, + "optional": true + }, + "prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha512-PhmXi5XmoyKw1Un4E+opM2KcsJInDvKyuOumcjjw3waw86ZNjHwVUOOWLc4bCzLdcKNaWBH9e99sbWzDQsVaYg==" + }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "optional": true + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "protocols": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/protocols/-/protocols-2.0.1.tgz", + "integrity": "sha512-/XJ368cyBJ7fzLMwLKv1e4vLxOju2MNAIokcr7meSaNcVbWz/CPcW22cP04mwxOErdA5mwjA8Q6w/cdAQxVn7Q==" + }, + "psl": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", + "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==", + "dev": true + }, + "punycode": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dev": true + }, + "querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "dev": true + }, + "r-json": { + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/r-json/-/r-json-1.2.10.tgz", + "integrity": "sha512-hu9vyLjSlHXT62NAS7DjI9WazDlvjN0lgp3n431dCVnirVcLkZIpzSwA3orhZEKzdDD2jqNYI+w0yG0aFf4kpA==" + }, + "r-package-json": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/r-package-json/-/r-package-json-1.0.9.tgz", + "integrity": "sha512-G4Vpf1KImWmmPFGdtWQTU0L9zk0SjqEC4qs/jE7AQ+Ylmr5kizMzGeC4wnHp5+ijPqNN+2ZPpvyjVNdN1CDVcg==", + "requires": { + "package-json-path": "^1.0.0", + "r-json": "^1.2.1" + } + }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + } + }, + "read-all-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/read-all-stream/-/read-all-stream-3.1.0.tgz", + "integrity": "sha512-DI1drPHbmBcUDWrJ7ull/F2Qb8HkwBncVx8/RpKYFSIACYaVRQReISYPdZz/mt1y1+qMCOrfReTopERmaxtP6w==", + "requires": { + "pinkie-promise": "^2.0.0", + "readable-stream": "^2.0.0" + } + }, + "readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "dev": true, + "optional": true, + "requires": { + "readable-stream": "^3.6.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "optional": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "dev": true, + "optional": true + }, + "registry-auth-token": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", + "integrity": "sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==", + "requires": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha512-ZbgR5aZEdf4UKZVBPYIgaglBmSF2Hi94s2PcIHhRGFjKYu+chjJdYfHn4rt3hB6eCKLJ8giVIIfgMa1ehDfZKA==", + "requires": { + "rc": "^1.0.1" + } + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true + }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true + }, + "resolve": { + "version": "1.22.2", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", + "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", + "requires": { + "is-core-module": "^2.11.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", + "dev": true, + "optional": true + }, + "selenium-webdriver": { + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.8.2.tgz", + "integrity": "sha512-d2dcpDLPcXlBy5qcPtB1B8RYTtj1N+JiHQLViFx3OP+i5hqkAuqTfJEYUh4qNX11S4NvbxjteiwN3OPwK3vPVw==", + "dev": true, + "requires": { + "jszip": "^3.10.0", + "tmp": "^0.2.1", + "ws": ">=8.11.0" + }, + "dependencies": { + "tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "requires": { + "rimraf": "^3.0.0" + } + }, + "ws": { + "version": "8.13.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "dev": true, + "requires": {} + } + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "dev": true + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "sliced": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", + "integrity": "sha512-VZBmZP8WU3sMOZm1bdgTadsQbcscK0UM8oKxKVBs4XAhUo2Xxzm/OFMGBkPusxw9xL3Uy8LrzEqGqJhclsr0yA==" + }, + "spdx-correct": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", + "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", + "requires": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-exceptions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", + "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==" + }, + "spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "requires": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-license-ids": { + "version": "3.0.13", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.13.tgz", + "integrity": "sha512-XkD+zwiqXHikFZm4AX/7JSCXA98U5Db4AFd5XUg/+9UNtnH75+Z9KxtpYiJZx36mUDVOwH83pl7yvCer6ewM3w==" + }, + "split": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==", + "dev": true, + "requires": { + "through": "2" + } + }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "dev": true + }, + "strftime": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/strftime/-/strftime-0.10.1.tgz", + "integrity": "sha512-nVvH6JG8KlXFPC0f8lojLgEsPA18lRpLZ+RrJh/NkQV2tqOgZfbas8gcU8SFgnnqR3rWzZPYu6N2A3xzs/8rQg==", + "dev": true + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "dev": true + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==" + }, + "strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "dev": true, + "optional": true, + "requires": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "dev": true + }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" + }, + "symbol": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/symbol/-/symbol-0.3.1.tgz", + "integrity": "sha512-SxMrE6uv9zhnBmTCpZna1u0TcZix1k2QASZ/DpF13rAo+0Ts40faFYsMTuAirgvbbjHw1byhJ949/fP20XzVZA==", + "dev": true + }, + "tar": { + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==", + "dev": true, + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^4.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "dependencies": { + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + } + } + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, + "timed-out": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-2.0.0.tgz", + "integrity": "sha512-pqqJOi1rF5zNs/ps4vmbE4SFCrM4iR7LW+GHAsHqO/EumqbIWceioevYLM5xZRgQSH6gFgL9J/uB7EcJhQ9niQ==" + }, + "timm": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/timm/-/timm-1.7.1.tgz", + "integrity": "sha512-IjZc9KIotudix8bMaBW6QvMuq64BrJWFs1+4V0lXwWGQZwH+LnX87doAYhem4caOEusRP9/g6jVDQmZ8XOk1nw==", + "dev": true, + "optional": true + }, + "tinycolor2": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/tinycolor2/-/tinycolor2-1.6.0.tgz", + "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", + "dev": true, + "optional": true + }, + "tmp": { + "version": "0.0.28", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.28.tgz", + "integrity": "sha512-c2mmfiBmND6SOVxzogm1oda0OJ1HZVIk/5n26N59dDTh80MUeavpiCls4PGAdkX1PFkKokLpcf7prSjCeXLsJg==", + "requires": { + "os-tmpdir": "~1.0.1" + } + }, + "token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "dev": true, + "optional": true, + "requires": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + } + }, + "tough-cookie": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.2.tgz", + "integrity": "sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==", + "dev": true, + "requires": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + } + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "optional": true + }, + "typpy": { + "version": "2.3.13", + "resolved": "https://registry.npmjs.org/typpy/-/typpy-2.3.13.tgz", + "integrity": "sha512-vOxIcQz9sxHi+rT09SJ5aDgVgrPppQjwnnayTrMye1ODaU8gIZTDM19t9TxmEElbMihx2Nq/0/b/MtyKfayRqA==", + "requires": { + "function.name": "^1.0.3" + } + }, + "ul": { + "version": "5.2.15", + "resolved": "https://registry.npmjs.org/ul/-/ul-5.2.15.tgz", + "integrity": "sha512-svLEUy8xSCip5IWnsRa0UOg+2zP0Wsj4qlbjTmX6GJSmvKMHADBuHOm1dpNkWqWPIGuVSqzUkV3Cris5JrlTRQ==", + "requires": { + "deffy": "^2.2.2", + "typpy": "^2.3.4" + } + }, + "universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "dev": true + }, + "unzip-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-1.0.2.tgz", + "integrity": "sha512-pwCcjjhEcpW45JZIySExBHYv5Y9EeL2OIGEfrSKp2dMUFGFv4CpvZkwJbVge8OvGH2BNNtJBx67DuKuJhf+N5Q==" + }, + "url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dev": true, + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "url-parse-lax": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", + "integrity": "sha512-BVA4lR5PIviy2PMseNd2jbFQ+jwSwQGdJejf5ctd1rEXt0Ypd7yanUK9+lYechVlN5VaTJGsu2U/3MDDu6KgBA==", + "requires": { + "prepend-http": "^1.0.1" + } + }, + "utcstring": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/utcstring/-/utcstring-0.1.0.tgz", + "integrity": "sha512-1EpWQ6CECkoys7aX3LImrFo4nYIigY2RQHJTvgzZQCB4/oA6jJvTLTcgilTxX57GrSHDIVMtGwYd+SujGJvvyw==", + "dev": true + }, + "utif2": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/utif2/-/utif2-4.0.1.tgz", + "integrity": "sha512-KMaD76dbzK1VjbwsckHJiqDjhP3pbpwyV+FdqkY6XFQenc2o/HS6pjPSYdu4+NQMHf2NLTW+nVP/eFP1CvOYQQ==", + "dev": true, + "optional": true, + "requires": { + "pako": "^1.0.11" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "dev": true + }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "requires": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "optional": true + }, + "whatwg-fetch": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.2.tgz", + "integrity": "sha512-bJlen0FcuU/0EMLrdbJ7zOnW6ITZLrZMIarMUVmdKtsGvZna8vxKYaexICWPfZ8qwf9fzNq+UEIZrnSaApt6RA==", + "dev": true, + "optional": true + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "optional": true, + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + } + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "dev": true, + "requires": {} + }, + "xhr": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/xhr/-/xhr-2.6.0.tgz", + "integrity": "sha512-/eCGLb5rxjx5e3mF1A7s+pLlR6CGyqWN91fv1JgER5mVWg1MZmlhBvy9kjcsOdRk8RrIujotWyJamfyrp+WIcA==", + "dev": true, + "optional": true, + "requires": { + "global": "~4.4.0", + "is-function": "^1.0.1", + "parse-headers": "^2.0.0", + "xtend": "^4.0.0" + } + }, + "xml-parse-from-string": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz", + "integrity": "sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==", + "dev": true, + "optional": true + }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "dev": true, + "optional": true, + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "dev": true, + "optional": true + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, + "optional": true + }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, + "yargs": { + "version": "17.7.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.1.tgz", + "integrity": "sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw==", + "dev": true, + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true + }, + "yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "dev": true + } + } +} diff --git a/tools/browsertime/package.json b/tools/browsertime/package.json new file mode 100644 index 0000000000..9a110e494c --- /dev/null +++ b/tools/browsertime/package.json @@ -0,0 +1 @@ +{"name":"mozilla-central-tools-browsertime","description":"This package file is for node modules used in mozilla-central/tools/browsertime","repository":{},"license":"MPL-2.0","dependencies":{"package.json":"^2.0.1"},"devDependencies":{"browsertime":"https://github.com/sitespeedio/browsertime/tarball/6f45b9e5a6e0419fe6408f05909f2ab5a1f6bacc"},"notes(private)":"We don't want to publish to npm, so this is marked as private","private":true} \ No newline at end of file diff --git a/tools/clang-tidy/config.yaml b/tools/clang-tidy/config.yaml new file mode 100644 index 0000000000..36053d7870 --- /dev/null +++ b/tools/clang-tidy/config.yaml @@ -0,0 +1,327 @@ +--- +target: obj-x86_64-pc-linux-gnu +# It is used by 'mach static-analysis' and 'phabricator static-analysis bot' +# in order to have consistency across the used checkers. +# All the clang checks used by the static-analysis tools. +# +# To add a new checker: +# 1. Add it in this file +# 2. Create a C/C++ test case in tools/clang-tidy/test/ reproducing the +# warning/error that the checker will detect +# 3. Run './mach static-analysis autotest -d' to create the reference +# 4. Check the json file in tools/clang-tidy/test/ +# 5. Commit this file + the .cpp test case + the json result +platforms: + - linux64 + - macosx64 + - win32 + - win64 +# Minimum clang-tidy version that is required for all the following checkers +# to work properly. +# This is also used by 'mach clang-format' +package_version: "16.0.4" +clang_checkers: + - name: -* + publish: !!bool no + - name: bugprone-argument-comment + reliability: high + - name: bugprone-assert-side-effect + reliability: high + - name: bugprone-bool-pointer-implicit-conversion + reliability: low + - name: bugprone-forward-declaration-namespace + reliability: high + - name: bugprone-incorrect-roundings + reliability: high + - name: bugprone-integer-division + reliability: high + - name: bugprone-macro-parentheses + reliability: medium + - name: bugprone-macro-repeated-side-effects + reliability: high + - name: bugprone-misplaced-widening-cast + reliability: high + - name: bugprone-move-forwarding-reference + reliability: high + - name: bugprone-multiple-statement-macro + # Incompatible with our code base, see bug 1496379. + publish: !!bool no + reliability: high + - name: bugprone-sizeof-expression + reliability: high + - name: bugprone-string-constructor + reliability: high + - name: bugprone-string-integer-assignment + reliability: high + - name: bugprone-suspicious-memset-usage + reliability: high + - name: bugprone-suspicious-missing-comma + reliability: high + - name: bugprone-suspicious-semicolon + reliability: high + - name: bugprone-suspicious-string-compare + reliability: high + - name: bugprone-swapped-arguments + reliability: high + - name: bugprone-too-small-loop-variable + reliability: high + - name: bugprone-unused-raii + reliability: high + - name: bugprone-use-after-move + reliability: high + - name: clang-analyzer-core.CallAndMessage + reliability: medium + - name: clang-analyzer-core.DivideZero + reliability: high + - name: clang-analyzer-core.NonNullParamChecker + reliability: high + - name: clang-analyzer-core.NullDereference + reliability: medium + - name: clang-analyzer-core.UndefinedBinaryOperatorResult + reliability: medium + - name: clang-analyzer-core.uninitialized.Assign + reliability: medium + - name: clang-analyzer-core.uninitialized.Branch + reliability: medium + - name: clang-analyzer-cplusplus.Move + reliability: high + - name: clang-analyzer-cplusplus.NewDelete + reliability: medium + - name: clang-analyzer-cplusplus.NewDeleteLeaks + reliability: medium + - name: clang-analyzer-deadcode.DeadStores + reliability: high + - name: clang-analyzer-optin.performance.Padding + reliability: high + config: + - key: AllowedPad + value: 2 + - name: clang-analyzer-security.FloatLoopCounter + reliability: high + - name: clang-analyzer-security.insecureAPI.bcmp + reliability: high + - name: clang-analyzer-security.insecureAPI.bcopy + reliability: high + - name: clang-analyzer-security.insecureAPI.bzero + reliability: high + - name: clang-analyzer-security.insecureAPI.getpw + reliability: high + # We don't add clang-analyzer-security.insecureAPI.gets here; it's deprecated. + - name: clang-analyzer-security.insecureAPI.mkstemp + reliability: high + - name: clang-analyzer-security.insecureAPI.mktemp + reliability: high + - name: clang-analyzer-security.insecureAPI.rand + reliability: low + # C checker, that is outdated and doesn't check for the new std::rand calls. + publish: !!bool no + - name: clang-analyzer-security.insecureAPI.strcpy + reliability: low + # The functions that should be used differ on POSIX and Windows, and there + # isn't a consensus on how we should approach this. + publish: !!bool no + - name: clang-analyzer-security.insecureAPI.UncheckedReturn + reliability: low + - name: clang-analyzer-security.insecureAPI.vfork + reliability: medium + - name: clang-analyzer-unix.Malloc + reliability: high + - name: clang-analyzer-unix.cstring.BadSizeArg + reliability: high + - name: clang-analyzer-unix.cstring.NullArg + reliability: high + - name: cppcoreguidelines-narrowing-conversions + reliability: high + - name: cppcoreguidelines-pro-type-member-init + reliability: medium + - name: misc-non-copyable-objects + reliability: high + - name: misc-redundant-expression + reliability: medium + - name: misc-unused-alias-decls + reliability: high + - name: misc-unused-using-decls + reliability: high + - name: modernize-avoid-bind + restricted-platforms: + - win32 + - win64 + reliability: medium + - name: modernize-concat-nested-namespaces + reliability: high + - name: modernize-deprecated-ios-base-aliases + reliability: high + - name: modernize-loop-convert + reliability: high + - name: modernize-raw-string-literal + reliability: high + - name: modernize-redundant-void-arg + reliability: high + # We still have some old C code that is built with a C compiler, so this + # might break the build. + publish: !!bool no + - name: modernize-shrink-to-fit + reliability: high + - name: modernize-use-auto + reliability: high + # Controversial, see bug 1371052. + publish: !!bool no + - name: modernize-use-bool-literals + reliability: high + - name: modernize-use-equals-default + reliability: high + - name: modernize-use-equals-delete + reliability: high + - name: modernize-use-nullptr + reliability: high + - name: modernize-use-override + reliability: low + # Too noisy because of the way how we implement NS_IMETHOD. See Bug 1420366. + publish: !!bool no + - name: modernize-use-using + reliability: high + - name: mozilla-* + reliability: high + - name: performance-faster-string-find + reliability: high + - name: performance-for-range-copy + reliability: high + - name: performance-implicit-conversion-in-loop + reliability: high + - name: performance-inefficient-algorithm + restricted-platforms: + - linux64 + - macosx64 + reliability: high + # Disable as the test does not support C++17 yet + publish: !!bool no + - name: performance-inefficient-string-concatenation + reliability: high + - name: performance-inefficient-vector-operation + reliability: high + - name: performance-move-const-arg + reliability: high + config: + - key: CheckTriviallyCopyableMove + # As per Bug 1558359 - disable detection of trivially copyable types + # that do not have a move constructor. + value: 0 + - name: performance-move-constructor-init + reliability: high + - name: performance-noexcept-move-constructor + reliability: high + - name: performance-type-promotion-in-math-fn + reliability: high + - name: performance-unnecessary-copy-initialization + reliability: high + - name: performance-unnecessary-value-param + reliability: high + - name: readability-braces-around-statements + reliability: high + config: + - key: ShortStatementLines + # Allow `if (foo) return;` without braces + # Still warns on `if (foo)\n return;` + value: 1 + - name: readability-const-return-type + reliability: high + # Note: this can be loosened up by using the ShortStatementLines option + - name: readability-container-size-empty + reliability: high + - name: readability-delete-null-pointer + reliability: high + - name: readability-else-after-return + reliability: high + config: + - key: WarnOnConditionVariables + # Disable as we don't mind this kind of behavior + value: 0 + - name: readability-implicit-bool-conversion + reliability: low + # On automation the config flags act strange. Please see Bug 1500241. + publish: !!bool no + config: + - key: AllowIntegerConditions + # The check will allow conditional integer conversions. + value: 1 + - key: AllowPointerConditions + # The check will allow conditional pointer conversions. + value: 1 + - name: readability-inconsistent-declaration-parameter-name + reliability: high + - name: readability-isolate-declaration + # As per bug 1558987 - we don't want to have this enabled + publish: !!bool no + reliability: high + - name: readability-magic-numbers + # Bug 1553495 - we must see first its impact on our code. + publish: !!bool no + reliability: high + - name: readability-misleading-indentation + reliability: high + - name: readability-non-const-parameter + reliability: high + - name: readability-qualified-auto + reliability: high + - name: readability-redundant-control-flow + reliability: high + - name: readability-redundant-preprocessor + reliability: high + - name: readability-redundant-smartptr-get + reliability: high + - name: readability-redundant-string-cstr + reliability: high + - name: readability-redundant-string-init + reliability: high + - name: readability-static-accessed-through-instance + reliability: high + - name: readability-simplify-boolean-expr + reliability: high + config: + - key: SimplifyDeMorgan + # Don't want to enable DeMorgan expressions because of MOZ_ASSERT() + # See Bug 1804160 + value: 0 + - name: readability-uniqueptr-delete-release + reliability: high + # We don't publish the google checkers since we are interested in only having + # a general idea how our code complies with the rules added by these checkers. + - name: google-build-explicit-make-pair + reliability: low + publish: !!bool no + - name: google-build-namespaces + reliability: low + publish: !!bool no + - name: google-build-using-namespace + reliability: low + publish: !!bool no + - name: google-default-arguments + reliability: low + publish: !!bool no + - name: google-explicit-constructor + reliability: low + publish: !!bool no + - name: google-global-names-in-headers + reliability: low + publish: !!bool no + - name: google-readability-casting + reliability: low + publish: !!bool no + - name: google-readability-function-size + reliability: low + publish: !!bool no + - name: google-readability-namespace-comments + reliability: low + publish: !!bool no + - name: google-readability-todo + reliability: low + publish: !!bool no + - name: google-runtime-int + reliability: low + publish: !!bool no + - name: google-runtime-operator + reliability: low + publish: !!bool no + - name: google-runtime-references + reliability: low + publish: !!bool no diff --git a/tools/clang-tidy/test/bugprone-argument-comment.cpp b/tools/clang-tidy/test/bugprone-argument-comment.cpp new file mode 100644 index 0000000000..3099575a35 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-argument-comment.cpp @@ -0,0 +1,6 @@ +// bugprone-argument-comment + +void f(int x, int y); +void g() { + f(/*y=*/0, /*z=*/0); +} diff --git a/tools/clang-tidy/test/bugprone-argument-comment.json b/tools/clang-tidy/test/bugprone-argument-comment.json new file mode 100644 index 0000000000..034f6dbf87 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-argument-comment.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "argument name 'y' in comment does not match parameter name 'x'", + "bugprone-argument-comment" + ], + [ + "warning", + "argument name 'z' in comment does not match parameter name 'y'", + "bugprone-argument-comment" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-assert-side-effect.cpp b/tools/clang-tidy/test/bugprone-assert-side-effect.cpp new file mode 100644 index 0000000000..7cc0a79cf1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-assert-side-effect.cpp @@ -0,0 +1,8 @@ +#include "structures.h" + +// bugprone-assert-side-effect +void misc_assert_side_effect() { + int X = 0; + assert(X == 1); + assert(X = 1); +} diff --git a/tools/clang-tidy/test/bugprone-assert-side-effect.json b/tools/clang-tidy/test/bugprone-assert-side-effect.json new file mode 100644 index 0000000000..b17a456064 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-assert-side-effect.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "side effect in assert() condition discarded in release builds", + "bugprone-assert-side-effect" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.cpp b/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.cpp new file mode 100644 index 0000000000..602fa9a578 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.cpp @@ -0,0 +1,20 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-bool-pointer-implicit-conversion.html + +bool test(bool* pointer_to_bool, int* pointer_to_int) +{ + if (pointer_to_bool) { // warning for pointer to bool + } + + if (pointer_to_int) { // no warning for pointer to int + } + + if (!pointer_to_bool) { // no warning, but why not?? + } + + if (pointer_to_bool != nullptr) { // no warning for nullptr comparison + } + + // no warning on return, but why not?? + // clang-tidy bug: https://bugs.llvm.org/show_bug.cgi?id=38060 + return pointer_to_bool; +} diff --git a/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.json b/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.json new file mode 100644 index 0000000000..e4d64ee683 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-bool-pointer-implicit-conversion.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "dubious check of 'bool *' against 'nullptr', did you mean to dereference it?", + "bugprone-bool-pointer-implicit-conversion" + ], + { "reliability": "low" } +] diff --git a/tools/clang-tidy/test/bugprone-forward-declaration-namespace.cpp b/tools/clang-tidy/test/bugprone-forward-declaration-namespace.cpp new file mode 100644 index 0000000000..f93d54b0a1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-forward-declaration-namespace.cpp @@ -0,0 +1,3 @@ +namespace na { struct A; } +namespace nb { struct A {}; } +nb::A a; diff --git a/tools/clang-tidy/test/bugprone-forward-declaration-namespace.json b/tools/clang-tidy/test/bugprone-forward-declaration-namespace.json new file mode 100644 index 0000000000..aee1ec88e5 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-forward-declaration-namespace.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "no definition found for 'A', but a definition with the same name 'A' found in another namespace 'nb'", + "bugprone-forward-declaration-namespace" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-incorrect-roundings.cpp b/tools/clang-tidy/test/bugprone-incorrect-roundings.cpp new file mode 100644 index 0000000000..ee37b56ae0 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-incorrect-roundings.cpp @@ -0,0 +1,7 @@ +void f1() +{ + double d; + int x; + + x = (d + 0.5); +} diff --git a/tools/clang-tidy/test/bugprone-incorrect-roundings.json b/tools/clang-tidy/test/bugprone-incorrect-roundings.json new file mode 100644 index 0000000000..f8da228b16 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-incorrect-roundings.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "casting (double + 0.5) to integer leads to incorrect rounding; consider using lround (#include ) instead", + "bugprone-incorrect-roundings" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-integer-division.cpp b/tools/clang-tidy/test/bugprone-integer-division.cpp new file mode 100644 index 0000000000..058edffe39 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-integer-division.cpp @@ -0,0 +1,5 @@ +float f() { + int a = 2; + int b = 10; + return a/b; +} diff --git a/tools/clang-tidy/test/bugprone-integer-division.json b/tools/clang-tidy/test/bugprone-integer-division.json new file mode 100644 index 0000000000..95c43fc8e0 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-integer-division.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "result of integer division used in a floating point context; possible loss of precision", + "bugprone-integer-division" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-macro-parentheses.cpp b/tools/clang-tidy/test/bugprone-macro-parentheses.cpp new file mode 100644 index 0000000000..53c22090a1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-macro-parentheses.cpp @@ -0,0 +1,5 @@ +#define BAD1 -1 +#define BAD2 1+2 +#define BAD3(A) (A+1) +#define BAD4(x) ((unsigned char)(x & 0xff)) +#define BAD5(X) A*B=(C*)X+2 diff --git a/tools/clang-tidy/test/bugprone-macro-parentheses.json b/tools/clang-tidy/test/bugprone-macro-parentheses.json new file mode 100644 index 0000000000..c50bf76cbd --- /dev/null +++ b/tools/clang-tidy/test/bugprone-macro-parentheses.json @@ -0,0 +1,28 @@ +[ + [ + "warning", + "macro replacement list should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + [ + "warning", + "macro replacement list should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + [ + "warning", + "macro argument should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + [ + "warning", + "macro argument should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + [ + "warning", + "macro argument should be enclosed in parentheses", + "bugprone-macro-parentheses" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.cpp b/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.cpp new file mode 100644 index 0000000000..2dedb3aca9 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.cpp @@ -0,0 +1,28 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-macro-repeated-side-effects.html + +#define MACRO_WITHOUT_REPEATED_ARG(x) (x) +#define MACRO_WITH_REPEATED_ARG(x) ((x) + (x)) + +static int g; + +int function_with_side_effects(int i) +{ + g += i; + return g; +} + +void test() +{ + int i; + i = MACRO_WITHOUT_REPEATED_ARG(1); // OK + i = MACRO_WITH_REPEATED_ARG(1); // OK + + i = MACRO_WITHOUT_REPEATED_ARG(i); // OK + i = MACRO_WITH_REPEATED_ARG(i); // OK + + i = MACRO_WITHOUT_REPEATED_ARG(function_with_side_effects(i)); // OK + i = MACRO_WITH_REPEATED_ARG(function_with_side_effects(i)); // NO WARNING + + i = MACRO_WITHOUT_REPEATED_ARG(i++); // OK + i = MACRO_WITH_REPEATED_ARG(i++); // WARNING +} diff --git a/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.json b/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.json new file mode 100644 index 0000000000..236120ae05 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-macro-repeated-side-effects.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "side effects in the 1st macro argument 'x' are repeated in macro expansion", + "bugprone-macro-repeated-side-effects" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-misplaced-widening-cast.cpp b/tools/clang-tidy/test/bugprone-misplaced-widening-cast.cpp new file mode 100644 index 0000000000..e75541bf6e --- /dev/null +++ b/tools/clang-tidy/test/bugprone-misplaced-widening-cast.cpp @@ -0,0 +1,3 @@ +long f(int x) { + return (long)(x * 1000); +} diff --git a/tools/clang-tidy/test/bugprone-misplaced-widening-cast.json b/tools/clang-tidy/test/bugprone-misplaced-widening-cast.json new file mode 100644 index 0000000000..bd0d9f2c00 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-misplaced-widening-cast.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "either cast from 'int' to 'long' is ineffective, or there is loss of precision before the conversion", + "bugprone-misplaced-widening-cast" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-move-forwarding-reference.cpp b/tools/clang-tidy/test/bugprone-move-forwarding-reference.cpp new file mode 100644 index 0000000000..45be15ee65 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-move-forwarding-reference.cpp @@ -0,0 +1,25 @@ + +namespace std { +template struct remove_reference; + +template struct remove_reference { typedef _Tp type; }; + +template struct remove_reference<_Tp &> { typedef _Tp type; }; + +template struct remove_reference<_Tp &&> { typedef _Tp type; }; + +template +constexpr typename std::remove_reference<_Tp>::type &&move(_Tp &&__t); + +} // namespace std + +// Standard case. +template void f1(U &&SomeU) { + T SomeT(std::move(SomeU)); + // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: forwarding reference passed to + // CHECK-FIXES: T SomeT(std::forward(SomeU)); +} + +void foo() { + f1(2); +} diff --git a/tools/clang-tidy/test/bugprone-move-forwarding-reference.json b/tools/clang-tidy/test/bugprone-move-forwarding-reference.json new file mode 100644 index 0000000000..43166e4af1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-move-forwarding-reference.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "forwarding reference passed to std::move(), which may unexpectedly cause lvalues to be moved; use std::forward() instead", + "bugprone-move-forwarding-reference" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-multiple-statement-macro.cpp b/tools/clang-tidy/test/bugprone-multiple-statement-macro.cpp new file mode 100644 index 0000000000..7ad2a2df64 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-multiple-statement-macro.cpp @@ -0,0 +1,10 @@ +void F(); + +#define BAD_MACRO(x) \ + F(); \ + F() + +void positives() { + if (1) + BAD_MACRO(1); +} diff --git a/tools/clang-tidy/test/bugprone-multiple-statement-macro.json b/tools/clang-tidy/test/bugprone-multiple-statement-macro.json new file mode 100644 index 0000000000..81c3dc89e5 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-multiple-statement-macro.json @@ -0,0 +1,7 @@ +[ + [ + "warning", + "multiple statement macro used without braces; some statements will be unconditionally executed", + "bugprone-multiple-statement-macro" + ] +] diff --git a/tools/clang-tidy/test/bugprone-sizeof-expression.cpp b/tools/clang-tidy/test/bugprone-sizeof-expression.cpp new file mode 100644 index 0000000000..f5f1f469ce --- /dev/null +++ b/tools/clang-tidy/test/bugprone-sizeof-expression.cpp @@ -0,0 +1,3 @@ +class C { + int size() { return sizeof(this); } +}; diff --git a/tools/clang-tidy/test/bugprone-sizeof-expression.json b/tools/clang-tidy/test/bugprone-sizeof-expression.json new file mode 100644 index 0000000000..438194296b --- /dev/null +++ b/tools/clang-tidy/test/bugprone-sizeof-expression.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "suspicious usage of 'sizeof(this)'; did you mean 'sizeof(*this)'", + "bugprone-sizeof-expression" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-string-constructor.cpp b/tools/clang-tidy/test/bugprone-string-constructor.cpp new file mode 100644 index 0000000000..8b6a4980a3 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-string-constructor.cpp @@ -0,0 +1,17 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-string-constructor.html + +#include "structures.h" + +void test() +{ + // A common mistake is to swap parameters to the ‘fill’ string-constructor. + std::string str('x', 50); // should be str(50, 'x') + + // Calling the string-literal constructor with a length bigger than the + // literal is suspicious and adds extra random characters to the string. + std::string("test", 200); // Will include random characters after "test". + + // Creating an empty string from constructors with parameters is considered + // suspicious. The programmer should use the empty constructor instead. + std::string("test", 0); // Creation of an empty string. +} diff --git a/tools/clang-tidy/test/bugprone-string-constructor.json b/tools/clang-tidy/test/bugprone-string-constructor.json new file mode 100644 index 0000000000..7e2ba1764f --- /dev/null +++ b/tools/clang-tidy/test/bugprone-string-constructor.json @@ -0,0 +1,18 @@ +[ + [ + "warning", + "string constructor parameters are probably swapped; expecting string(count, character)", + "bugprone-string-constructor" + ], + [ + "warning", + "length is bigger than string literal size", + "bugprone-string-constructor" + ], + [ + "warning", + "constructor creating an empty string", + "bugprone-string-constructor" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-string-integer-assignment.cpp b/tools/clang-tidy/test/bugprone-string-integer-assignment.cpp new file mode 100644 index 0000000000..30ef46b922 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-string-integer-assignment.cpp @@ -0,0 +1,28 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-string-integer-assignment.html + +#include "structures.h" + +void test_int() +{ + // Numeric types can be implicitly casted to character types. + std::string s; + int x = 5965; + s = 6; // warning + s = x; // warning +} + +void test_conversion() +{ + // Use the appropriate conversion functions or character literals. + std::string s; + int x = 5965; + s = '6'; // OK + s = std::to_string(x); // OK +} + +void test_cast() +{ + // In order to suppress false positives, use an explicit cast. + std::string s; + s = static_cast(6); // OK +} diff --git a/tools/clang-tidy/test/bugprone-string-integer-assignment.json b/tools/clang-tidy/test/bugprone-string-integer-assignment.json new file mode 100644 index 0000000000..37912fd950 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-string-integer-assignment.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "an integer is interpreted as a character code when assigning it to a string; if this is intended, cast the integer to the appropriate character type; if you want a string representation, use the appropriate conversion facility", + "bugprone-string-integer-assignment" + ], + [ + "warning", + "an integer is interpreted as a character code when assigning it to a string; if this is intended, cast the integer to the appropriate character type; if you want a string representation, use the appropriate conversion facility", + "bugprone-string-integer-assignment" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-suspicious-memset-usage.cpp b/tools/clang-tidy/test/bugprone-suspicious-memset-usage.cpp new file mode 100644 index 0000000000..71fe7239a1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-memset-usage.cpp @@ -0,0 +1,22 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-suspicious-memset-usage.html + +#include "structures.h" + +void test(int* ip, char* cp) +{ + // Case 1: Fill value is a character '0' instead of NUL '\0'. + memset(ip, '0', 1); // WARNING: suspicious for non-char pointers + memset(cp, '0', 1); // OK for char pointers + + // Case 2: Fill value is truncated. + memset(ip, 0xabcd, 1); // WARNING: fill value gets truncated + memset(ip, 0x00cd, 1); // OK because value 0xcd is not truncated. + memset(ip, 0x00, 1); // OK because value is not truncated. + + // Case 3: Byte count is zero. + memset(ip, sizeof(int), 0); // WARNING: zero length, potentially swapped + memset(ip, sizeof(int), 1); // OK with non-zero length + + // See clang bug https://bugs.llvm.org/show_bug.cgi?id=38098 + memset(ip, 8, 0); // OK with zero length without sizeof +} diff --git a/tools/clang-tidy/test/bugprone-suspicious-memset-usage.json b/tools/clang-tidy/test/bugprone-suspicious-memset-usage.json new file mode 100644 index 0000000000..73f26cf7c1 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-memset-usage.json @@ -0,0 +1,18 @@ +[ + [ + "warning", + "memset fill value is char '0', potentially mistaken for int 0", + "bugprone-suspicious-memset-usage" + ], + [ + "warning", + "memset fill value is out of unsigned character range, gets truncated", + "bugprone-suspicious-memset-usage" + ], + [ + "warning", + "memset of size zero, potentially swapped arguments", + "bugprone-suspicious-memset-usage" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-suspicious-missing-comma.cpp b/tools/clang-tidy/test/bugprone-suspicious-missing-comma.cpp new file mode 100644 index 0000000000..de1634e11f --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-missing-comma.cpp @@ -0,0 +1,9 @@ +const char* Cartoons[] = { + "Bugs Bunny", + "Homer Simpson", + "Mickey Mouse", + "Bart Simpson", + "Charlie Brown" // There is a missing comma here. + "Fred Flintstone", + "Popeye", +}; diff --git a/tools/clang-tidy/test/bugprone-suspicious-missing-comma.json b/tools/clang-tidy/test/bugprone-suspicious-missing-comma.json new file mode 100644 index 0000000000..f4adb6b33a --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-missing-comma.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "suspicious string literal, probably missing a comma", + "bugprone-suspicious-missing-comma" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-suspicious-semicolon.cpp b/tools/clang-tidy/test/bugprone-suspicious-semicolon.cpp new file mode 100644 index 0000000000..7a90a87cd3 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-semicolon.cpp @@ -0,0 +1,8 @@ + +// bugprone-suspicious-semicolon +void nop(); +void fail1() +{ + int x = 0; + if(x > 5); nop(); +} diff --git a/tools/clang-tidy/test/bugprone-suspicious-semicolon.json b/tools/clang-tidy/test/bugprone-suspicious-semicolon.json new file mode 100644 index 0000000000..c94a1d5abb --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-semicolon.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "potentially unintended semicolon", + "bugprone-suspicious-semicolon" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-suspicious-string-compare.cpp b/tools/clang-tidy/test/bugprone-suspicious-string-compare.cpp new file mode 100644 index 0000000000..505a9d282c --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-string-compare.cpp @@ -0,0 +1,8 @@ +static const char A[] = "abc"; + +int strcmp(const char *, const char *); + +int test_warning_patterns() { + if (strcmp(A, "a")) + return 0; +} diff --git a/tools/clang-tidy/test/bugprone-suspicious-string-compare.json b/tools/clang-tidy/test/bugprone-suspicious-string-compare.json new file mode 100644 index 0000000000..eda430ef01 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-suspicious-string-compare.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "function 'strcmp' is called without explicitly comparing result", + "bugprone-suspicious-string-compare" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-swapped-arguments.cpp b/tools/clang-tidy/test/bugprone-swapped-arguments.cpp new file mode 100644 index 0000000000..074df52050 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-swapped-arguments.cpp @@ -0,0 +1,26 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-swapped-arguments.html + +void test_d_i(double d, int i); +void test_d_i_i(double d, int i, int ii); +void test_i_d(int i, double d); +void test_i_i_d(int i, int ii, double d); + +void test() +{ + double d = 1; + int i = 1; + + test_d_i(d, i); // OK + test_d_i(i, d); // WARNING + + test_i_d(i, d); // OK + test_i_d(d, i); // WARNING + + test_i_i_d(i, i, d); // OK + test_i_i_d(i, d, i); // WARNING + test_i_i_d(d, i, i); // NO WARNING after second parameter + + test_d_i_i(d, i, i); // OK + test_d_i_i(i, d, i); // WARNING + test_d_i_i(i, i, d); // NO WARNING after second parameter +} diff --git a/tools/clang-tidy/test/bugprone-swapped-arguments.json b/tools/clang-tidy/test/bugprone-swapped-arguments.json new file mode 100644 index 0000000000..44c8093e52 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-swapped-arguments.json @@ -0,0 +1,23 @@ +[ + [ + "warning", + "argument with implicit conversion from 'double' to 'int' followed by argument converted from 'int' to 'double', potentially swapped arguments.", + "bugprone-swapped-arguments" + ], + [ + "warning", + "argument with implicit conversion from 'int' to 'double' followed by argument converted from 'double' to 'int', potentially swapped arguments.", + "bugprone-swapped-arguments" + ], + [ + "warning", + "argument with implicit conversion from 'int' to 'double' followed by argument converted from 'double' to 'int', potentially swapped arguments.", + "bugprone-swapped-arguments" + ], + [ + "warning", + "argument with implicit conversion from 'double' to 'int' followed by argument converted from 'int' to 'double', potentially swapped arguments.", + "bugprone-swapped-arguments" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-too-small-loop-variable.cpp b/tools/clang-tidy/test/bugprone-too-small-loop-variable.cpp new file mode 100644 index 0000000000..4dc8ed7a22 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-too-small-loop-variable.cpp @@ -0,0 +1,4 @@ +int main() { + long size = 294967296l; + for (short i = 0; i < size; ++i) {} +} diff --git a/tools/clang-tidy/test/bugprone-too-small-loop-variable.json b/tools/clang-tidy/test/bugprone-too-small-loop-variable.json new file mode 100644 index 0000000000..23f781936c --- /dev/null +++ b/tools/clang-tidy/test/bugprone-too-small-loop-variable.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "loop variable has narrower type 'short' than iteration's upper bound 'long'", + "bugprone-too-small-loop-variable" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-unused-raii.cpp b/tools/clang-tidy/test/bugprone-unused-raii.cpp new file mode 100644 index 0000000000..6a4a0a15ab --- /dev/null +++ b/tools/clang-tidy/test/bugprone-unused-raii.cpp @@ -0,0 +1,25 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/bugprone-unused-raii.html + +struct scoped_lock +{ + scoped_lock() {} + ~scoped_lock() {} +}; + +#define SCOPED_LOCK_MACRO(m) scoped_lock() + +struct trivial_scoped_lock +{ + trivial_scoped_lock() {} +}; + +scoped_lock test() +{ + scoped_lock(); // misc-unused-raii warning! + + SCOPED_LOCK_MACRO(); // no warning for macros + + trivial_scoped_lock(); // no warning for trivial objects without destructors + + return scoped_lock(); // no warning for return values +} diff --git a/tools/clang-tidy/test/bugprone-unused-raii.json b/tools/clang-tidy/test/bugprone-unused-raii.json new file mode 100644 index 0000000000..99a9936485 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-unused-raii.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "object destroyed immediately after creation; did you mean to name the object?", + "bugprone-unused-raii" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/bugprone-use-after-move.cpp b/tools/clang-tidy/test/bugprone-use-after-move.cpp new file mode 100644 index 0000000000..f90a8700d6 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-use-after-move.cpp @@ -0,0 +1,62 @@ +namespace std { +typedef unsigned size_t; + +template +struct unique_ptr { + unique_ptr(); + T *get() const; + explicit operator bool() const; + void reset(T *ptr); + T &operator*() const; + T *operator->() const; + T& operator[](size_t i) const; +}; + +template +struct remove_reference; + +template +struct remove_reference { + typedef _Tp type; +}; + +template +struct remove_reference<_Tp &> { + typedef _Tp type; +}; + +template +struct remove_reference<_Tp &&> { + typedef _Tp type; +}; + +template +constexpr typename std::remove_reference<_Tp>::type &&move(_Tp &&__t) noexcept { + return static_cast::type &&>(__t); +} +} + +class A { +public: + A(); + A(const A &); + A(A &&); + + A &operator=(const A &); + A &operator=(A &&); + + void foo() const; + int getInt() const; + + operator bool() const; + + int i; +}; + +void func() { + std::unique_ptr ptr; + std::move(ptr); + ptr.get(); + static_cast(ptr); + *ptr; +} diff --git a/tools/clang-tidy/test/bugprone-use-after-move.json b/tools/clang-tidy/test/bugprone-use-after-move.json new file mode 100644 index 0000000000..50d7210781 --- /dev/null +++ b/tools/clang-tidy/test/bugprone-use-after-move.json @@ -0,0 +1,4 @@ +[ + ["warning", "'ptr' used after it was moved", "bugprone-use-after-move"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.cpp b/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.cpp new file mode 100644 index 0000000000..286189e25b --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.cpp @@ -0,0 +1,10 @@ +struct S { + int x; +}; + +void f(struct S s); + +void test() { + struct S s; + f(s); +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.json b/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.json new file mode 100644 index 0000000000..ecf097e287 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.CallAndMessage.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Passed-by-value struct argument contains uninitialized data (e.g., field: 'x')", + "clang-analyzer-core.CallAndMessage" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.DivideZero.cpp b/tools/clang-tidy/test/clang-analyzer-core.DivideZero.cpp new file mode 100644 index 0000000000..aac9f7c9f6 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.DivideZero.cpp @@ -0,0 +1,4 @@ +void test(int z) { + if (z == 0) + int x = 1 / z; +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.DivideZero.json b/tools/clang-tidy/test/clang-analyzer-core.DivideZero.json new file mode 100644 index 0000000000..78a64ad35b --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.DivideZero.json @@ -0,0 +1,4 @@ +[ + ["warning", "Division by zero", "clang-analyzer-core.DivideZero"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.cpp b/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.cpp new file mode 100644 index 0000000000..bc071f5453 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.cpp @@ -0,0 +1,6 @@ +int f(int *p) __attribute__((nonnull)); + +void test(int *p) { + if (!p) + f(p); // warn +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.json b/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.json new file mode 100644 index 0000000000..deaae128ff --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.NonNullParamChecker.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Null pointer passed to 1st parameter expecting 'nonnull'", + "clang-analyzer-core.NonNullParamChecker" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.NullDereference.cpp b/tools/clang-tidy/test/clang-analyzer-core.NullDereference.cpp new file mode 100644 index 0000000000..6c9c555532 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.NullDereference.cpp @@ -0,0 +1,9 @@ +class C { +public: + int x; +}; + +void test() { + C *pc = 0; + int k = pc->x; +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.NullDereference.json b/tools/clang-tidy/test/clang-analyzer-core.NullDereference.json new file mode 100644 index 0000000000..4c9258b2b7 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.NullDereference.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Access to field 'x' results in a dereference of a null pointer (loaded from variable 'pc')", + "clang-analyzer-core.NullDereference" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.cpp b/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.cpp new file mode 100644 index 0000000000..1351c35f44 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.cpp @@ -0,0 +1,4 @@ +void test() { + int x; + int y = x + 1; +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.json b/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.json new file mode 100644 index 0000000000..9ec6c8a809 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.UndefinedBinaryOperatorResult.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The left operand of '+' is a garbage value", + "clang-analyzer-core.UndefinedBinaryOperatorResult" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.cpp b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.cpp new file mode 100644 index 0000000000..e9685d48a4 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.cpp @@ -0,0 +1,4 @@ +void test() { + int x; + x |= 1; +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.json b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.json new file mode 100644 index 0000000000..3691af741c --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Assign.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The left expression of the compound assignment is an uninitialized value. The computed value will also be garbage", + "clang-analyzer-core.uninitialized.Assign" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.cpp b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.cpp new file mode 100644 index 0000000000..7a985eb080 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.cpp @@ -0,0 +1,6 @@ +void test() { + int x; + if (x) { + return; + } +} diff --git a/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.json b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.json new file mode 100644 index 0000000000..00c6b3ad1e --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-core.uninitialized.Branch.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Branch condition evaluates to a garbage value", + "clang-analyzer-core.uninitialized.Branch" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.cpp b/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.cpp new file mode 100644 index 0000000000..65f2e36561 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.cpp @@ -0,0 +1,10 @@ +class P {}; + +void bar(P) {} + +void foo(int n) { + P x; + for (int i = n; i >= 0; --i) { + bar(static_cast(x)); + } +} diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.json b/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.json new file mode 100644 index 0000000000..e60f052e86 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.Move.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Moved-from object 'x' is moved", + "clang-analyzer-cplusplus.Move" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.cpp b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.cpp new file mode 100644 index 0000000000..d886d74989 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.cpp @@ -0,0 +1,50 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +void use(int *p); + +void test_use_parameter_after_delete(int *p) +{ + delete p; + use(p); // warning: use after free +} + +class SomeClass { +public: + void f(); +}; + +void test_use_local_after_delete() +{ + SomeClass *c = new SomeClass; + delete c; + c->f(); // warning: use after free +} + +// XXX clang documentation says this should cause a warning but it doesn't! +void test_delete_alloca() +{ + int *p = (int *)__builtin_alloca(sizeof(int)); + delete p; // NO warning: deleting memory allocated by alloca +} + +void test_double_free() +{ + int *p = new int; + delete p; + delete p; // warning: attempt to free released +} + +void test_delete_local() +{ + int i; + delete &i; // warning: delete address of local +} + +// XXX clang documentation says this should cause a warning but it doesn't! +void test_delete_offset() +{ + int *p = new int[1]; + delete[] (++p); + // NO warning: argument to 'delete[]' is offset by 4 bytes + // from the start of memory allocated by 'new[]' +} diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.json b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.json new file mode 100644 index 0000000000..99a9cd0dd2 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDelete.json @@ -0,0 +1,23 @@ +[ + [ + "warning", + "Use of memory after it is freed", + "clang-analyzer-cplusplus.NewDelete" + ], + [ + "warning", + "Use of memory after it is freed", + "clang-analyzer-cplusplus.NewDelete" + ], + [ + "warning", + "Attempt to free released memory", + "clang-analyzer-cplusplus.NewDelete" + ], + [ + "warning", + "Argument to 'delete' is the address of the local variable 'i', which is not memory allocated by 'new'", + "clang-analyzer-cplusplus.NewDelete" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.cpp b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.cpp new file mode 100644 index 0000000000..60772a30db --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.cpp @@ -0,0 +1,6 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +void test() +{ + int *p = new int; +} // warning diff --git a/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.json b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.json new file mode 100644 index 0000000000..37490a67f8 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-cplusplus.NewDeleteLeaks.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Potential leak of memory pointed to by 'p'", + "clang-analyzer-cplusplus.NewDeleteLeaks" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.cpp b/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.cpp new file mode 100644 index 0000000000..4e1c2c851d --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.cpp @@ -0,0 +1,6 @@ + +// clang-analyzer-deadcode.DeadStores +void test() { + int x; + x = 1; // warn +} diff --git a/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.json b/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.json new file mode 100644 index 0000000000..c46d5f2c2a --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-deadcode.DeadStores.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Value stored to 'x' is never read", + "clang-analyzer-deadcode.DeadStores" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.cpp b/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.cpp new file mode 100644 index 0000000000..0b851a68d3 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.cpp @@ -0,0 +1,6 @@ +struct OverlyAlignedChar { + char c1; + int x; + char c2; + char c __attribute__((aligned(4096))); +}; diff --git a/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.json b/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.json new file mode 100644 index 0000000000..a7ce9df9a5 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-optin.performance.Padding.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Excessive padding in 'struct OverlyAlignedChar' (8185 padding bytes, where 4089 is optimal). Optimal fields order: c, c1, c2, x, consider reordering the fields or adding explicit padding members", + "clang-analyzer-optin.performance.Padding" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.cpp b/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.cpp new file mode 100644 index 0000000000..60dcdad746 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.cpp @@ -0,0 +1,3 @@ +void test() { + for (float x = 0.1f; x <= 1.0f; x += 0.1f) {} +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.json b/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.json new file mode 100644 index 0000000000..792f5b13dc --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.FloatLoopCounter.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Variable 'x' with floating point type 'float' should not be used as a loop counter", + "clang-analyzer-security.FloatLoopCounter" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.cpp new file mode 100644 index 0000000000..f1dede2d51 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test() { + setuid(1); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.json new file mode 100644 index 0000000000..a4f89bc0d9 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.UncheckedReturn.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The return value from the call to 'setuid' is not checked. If an error occurs in 'setuid', the following code may execute with unexpected privileges", + "clang-analyzer-security.insecureAPI.UncheckedReturn" + ], + { "reliability": "low" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.cpp new file mode 100644 index 0000000000..8821807518 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +int test_bcmp(void *a, void *b, size_t n) { + return bcmp(a, b, n); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.json new file mode 100644 index 0000000000..ea7d6267ee --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcmp.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The bcmp() function is obsoleted by memcmp()", + "clang-analyzer-security.insecureAPI.bcmp" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.cpp new file mode 100644 index 0000000000..67df7d1638 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test_bcopy(void *a, void *b, size_t n) { + bcopy(a, b, n); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.json new file mode 100644 index 0000000000..d752f6c7a8 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bcopy.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The bcopy() function is obsoleted by memcpy() or memmove()", + "clang-analyzer-security.insecureAPI.bcopy" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.cpp new file mode 100644 index 0000000000..d3b5aa685f --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test_bzero(void *a, size_t n) { + bzero(a, n); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.json new file mode 100644 index 0000000000..cdc654a176 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.bzero.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The bzero() function is obsoleted by memset()", + "clang-analyzer-security.insecureAPI.bzero" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.cpp new file mode 100644 index 0000000000..c3da0b1970 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.cpp @@ -0,0 +1,6 @@ +#include "structures.h" + +void test() { + char buff[1024]; + getpw(2, buff); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.json new file mode 100644 index 0000000000..2f80393d54 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.getpw.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "The getpw() function is dangerous as it may overflow the provided buffer. It is obsoleted by getpwuid()", + "clang-analyzer-security.insecureAPI.getpw" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.cpp new file mode 100644 index 0000000000..f096c29de3 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.cpp @@ -0,0 +1,6 @@ +#include + +void test() { + char buff[1024]; + gets(buff); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.json new file mode 100644 index 0000000000..1d2212ee27 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.gets.json @@ -0,0 +1 @@ +"[[\"error\", \"use of undeclared identifier 'gets'\", \"clang-diagnostic-error\"]]" diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.cpp new file mode 100644 index 0000000000..904fc92ce6 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test() { + mkstemp("XX"); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.json new file mode 100644 index 0000000000..cca843ce93 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mkstemp.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Call to 'mkstemp' should have at least 6 'X's in the format string to be secure (2 'X's seen)", + "clang-analyzer-security.insecureAPI.mkstemp" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.cpp new file mode 100644 index 0000000000..8bb511a7d7 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test() { + char *x = mktemp("/tmp/zxcv"); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.json new file mode 100644 index 0000000000..ce58bfdddf --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.mktemp.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Call to function 'mktemp' is insecure as it always creates or uses insecure temporary file. Use 'mkstemp' instead", + "clang-analyzer-security.insecureAPI.mktemp" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.cpp new file mode 100644 index 0000000000..2274127c50 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.cpp @@ -0,0 +1,4 @@ +#include +void test() { + random(); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.json new file mode 100644 index 0000000000..7669f38d3e --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.rand.json @@ -0,0 +1 @@ +"[[\"warning\", \"The 'random' function produces a sequence of values that an adversary may be able to predict. Use 'arc4random' instead\", \"clang-analyzer-security.insecureAPI.rand\"]]" diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.cpp new file mode 100644 index 0000000000..41713adb4e --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.cpp @@ -0,0 +1,7 @@ +#include +void test() { + char x[4]; + char *y = "abcd"; + + strcpy(x, y); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.json new file mode 100644 index 0000000000..874de88ded --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.strcpy.json @@ -0,0 +1 @@ +"[[\"warning\", \"Call to function 'strcpy' is insecure as it does not provide bounding of the memory buffer. Replace unbounded copy functions with analogous functions that support length arguments such as 'strlcpy'. CWE-119\", \"clang-analyzer-security.insecureAPI.strcpy\"], [\"note\", \"Call to function 'strcpy' is insecure as it does not provide bounding of the memory buffer. Replace unbounded copy functions with analogous functions that support length arguments such as 'strlcpy'. CWE-119\", null]]" diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.cpp b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.cpp new file mode 100644 index 0000000000..619d986cf7 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +void test() { + vfork(); +} diff --git a/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.json b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.json new file mode 100644 index 0000000000..dd681fd537 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-security.insecureAPI.vfork.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Call to function 'vfork' is insecure as it can lead to denial of service situations in the parent process. Replace calls to vfork with calls to the safer 'posix_spawn' function", + "clang-analyzer-security.insecureAPI.vfork" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-unix.Malloc.cpp b/tools/clang-tidy/test/clang-analyzer-unix.Malloc.cpp new file mode 100644 index 0000000000..a08422b336 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.Malloc.cpp @@ -0,0 +1,37 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +#include "structures.h" + +void test_malloc() +{ + int *p = (int*) malloc(1); + free(p); + free(p); // warning: attempt to free released memory +} + +void test_use_after_free() +{ + int *p = (int*) malloc(sizeof(int)); + free(p); + *p = 1; // warning: use after free +} + +void test_leak() +{ + int *p = (int*) malloc(1); + if (p) + return; // warning: memory is never released +} + +void test_free_local() +{ + int a[] = { 1 }; + free(a); // warning: argument is not allocated by malloc +} + +void test_free_offset() +{ + int *p = (int*) malloc(sizeof(char)); + p = p - 1; + free(p); // warning: argument to free() is offset by -4 bytes +} diff --git a/tools/clang-tidy/test/clang-analyzer-unix.Malloc.json b/tools/clang-tidy/test/clang-analyzer-unix.Malloc.json new file mode 100644 index 0000000000..701cbba680 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.Malloc.json @@ -0,0 +1,20 @@ +[ + ["warning", "Attempt to free released memory", "clang-analyzer-unix.Malloc"], + ["warning", "Use of memory after it is freed", "clang-analyzer-unix.Malloc"], + [ + "warning", + "Potential leak of memory pointed to by 'p'", + "clang-analyzer-unix.Malloc" + ], + [ + "warning", + "Argument to free() is the address of the local variable 'a', which is not memory allocated by malloc()", + "clang-analyzer-unix.Malloc" + ], + [ + "warning", + "Argument to free() is offset by -4 bytes from the start of memory allocated by malloc()", + "clang-analyzer-unix.Malloc" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.cpp b/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.cpp new file mode 100644 index 0000000000..124737c3f4 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.cpp @@ -0,0 +1,9 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +#include "structures.h" + +void test() +{ + char dest[3]; + strncat(dest, "***", sizeof(dest)); // warning : potential buffer overflow +} diff --git a/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.json b/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.json new file mode 100644 index 0000000000..dc6b9facf1 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.cstring.BadSizeArg.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Potential buffer overflow. Replace with 'sizeof(dest) - strlen(dest) - 1' or use a safer 'strlcat' API", + "clang-analyzer-unix.cstring.BadSizeArg" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.cpp b/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.cpp new file mode 100644 index 0000000000..30cdaf6ce9 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.cpp @@ -0,0 +1,14 @@ +// https://clang-analyzer.llvm.org/available_checks.html + +#include "structures.h" + +int my_strlen(const char* s) +{ + return strlen(s); // warning +} + +int bad_caller() +{ + const char* s = nullptr; + return my_strlen(s); +} diff --git a/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.json b/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.json new file mode 100644 index 0000000000..a1270cafd8 --- /dev/null +++ b/tools/clang-tidy/test/clang-analyzer-unix.cstring.NullArg.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "Null pointer passed as 1st argument to string length function", + "clang-analyzer-unix.cstring.NullArg" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.cpp b/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.cpp new file mode 100644 index 0000000000..b10b0fa1c3 --- /dev/null +++ b/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.cpp @@ -0,0 +1,4 @@ +class Foo { + int f; + void a_f(double val) { f = val;} +}; diff --git a/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.json b/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.json new file mode 100644 index 0000000000..9dca259044 --- /dev/null +++ b/tools/clang-tidy/test/cppcoreguidelines-narrowing-conversions.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "narrowing conversion from 'double' to 'int'", + "cppcoreguidelines-narrowing-conversions" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.cpp b/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.cpp new file mode 100644 index 0000000000..be331829f3 --- /dev/null +++ b/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.cpp @@ -0,0 +1,7 @@ +struct Foo final { + int x; +}; + +void foo() { + Foo y; +} diff --git a/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.json b/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.json new file mode 100644 index 0000000000..11e79f1579 --- /dev/null +++ b/tools/clang-tidy/test/cppcoreguidelines-pro-type-member-init.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "uninitialized record type: 'y'", + "cppcoreguidelines-pro-type-member-init" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/misc-non-copyable-objects.cpp b/tools/clang-tidy/test/misc-non-copyable-objects.cpp new file mode 100644 index 0000000000..2f9060a818 --- /dev/null +++ b/tools/clang-tidy/test/misc-non-copyable-objects.cpp @@ -0,0 +1,5 @@ +namespace std { +typedef struct FILE {} FILE; +} + +void g(std::FILE f); diff --git a/tools/clang-tidy/test/misc-non-copyable-objects.json b/tools/clang-tidy/test/misc-non-copyable-objects.json new file mode 100644 index 0000000000..30826f3700 --- /dev/null +++ b/tools/clang-tidy/test/misc-non-copyable-objects.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'f' declared as type 'FILE', which is unsafe to copy; did you mean 'FILE *'?", + "misc-non-copyable-objects" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/misc-redundant-expression.cpp b/tools/clang-tidy/test/misc-redundant-expression.cpp new file mode 100644 index 0000000000..0ccc9c55f7 --- /dev/null +++ b/tools/clang-tidy/test/misc-redundant-expression.cpp @@ -0,0 +1,3 @@ +int TestSimpleEquivalent(int X, int Y) { + if (X - X) return 1; +} diff --git a/tools/clang-tidy/test/misc-redundant-expression.json b/tools/clang-tidy/test/misc-redundant-expression.json new file mode 100644 index 0000000000..e4bffef44c --- /dev/null +++ b/tools/clang-tidy/test/misc-redundant-expression.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "both sides of operator are equivalent", + "misc-redundant-expression" + ], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/misc-unused-alias-decls.cpp b/tools/clang-tidy/test/misc-unused-alias-decls.cpp new file mode 100644 index 0000000000..1bbaa2db0a --- /dev/null +++ b/tools/clang-tidy/test/misc-unused-alias-decls.cpp @@ -0,0 +1,18 @@ +// https://clang.llvm.org/extra/clang-tidy/checks/misc-unused-alias-decls.html + +namespace n1 { + namespace n2 { + namespace n3 { + int qux = 42; + } + } +} + +namespace n1_unused = ::n1; // WARNING +namespace n12_unused = n1::n2; // WARNING +namespace n123 = n1::n2::n3; // OK + +int test() +{ + return n123::qux; +} diff --git a/tools/clang-tidy/test/misc-unused-alias-decls.json b/tools/clang-tidy/test/misc-unused-alias-decls.json new file mode 100644 index 0000000000..1144566e4e --- /dev/null +++ b/tools/clang-tidy/test/misc-unused-alias-decls.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "namespace alias decl 'n1_unused' is unused", + "misc-unused-alias-decls" + ], + [ + "warning", + "namespace alias decl 'n12_unused' is unused", + "misc-unused-alias-decls" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/misc-unused-using-decls.cpp b/tools/clang-tidy/test/misc-unused-using-decls.cpp new file mode 100644 index 0000000000..6564711aba --- /dev/null +++ b/tools/clang-tidy/test/misc-unused-using-decls.cpp @@ -0,0 +1,4 @@ + +// misc-unused-using-decls +namespace n { class C; } +using n::C; diff --git a/tools/clang-tidy/test/misc-unused-using-decls.json b/tools/clang-tidy/test/misc-unused-using-decls.json new file mode 100644 index 0000000000..fc4156adca --- /dev/null +++ b/tools/clang-tidy/test/misc-unused-using-decls.json @@ -0,0 +1,4 @@ +[ + ["warning", "using decl 'C' is unused", "misc-unused-using-decls"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-avoid-bind.cpp b/tools/clang-tidy/test/modernize-avoid-bind.cpp new file mode 100644 index 0000000000..c46fb31cd5 --- /dev/null +++ b/tools/clang-tidy/test/modernize-avoid-bind.cpp @@ -0,0 +1,6 @@ +#include "structures.h" + +int add(int x, int y) { return x + y; } +void f_bind() { + auto clj = std::bind(add, 2, 2); +} diff --git a/tools/clang-tidy/test/modernize-avoid-bind.json b/tools/clang-tidy/test/modernize-avoid-bind.json new file mode 100644 index 0000000000..915f62f042 --- /dev/null +++ b/tools/clang-tidy/test/modernize-avoid-bind.json @@ -0,0 +1,4 @@ +[ + ["warning", "prefer a lambda to std::bind", "modernize-avoid-bind"], + { "reliability": "medium" } +] diff --git a/tools/clang-tidy/test/modernize-concat-nested-namespaces.cpp b/tools/clang-tidy/test/modernize-concat-nested-namespaces.cpp new file mode 100644 index 0000000000..0ff35d0e07 --- /dev/null +++ b/tools/clang-tidy/test/modernize-concat-nested-namespaces.cpp @@ -0,0 +1,5 @@ +namespace mozilla { +namespace dom { +void foo(); +} +} diff --git a/tools/clang-tidy/test/modernize-concat-nested-namespaces.json b/tools/clang-tidy/test/modernize-concat-nested-namespaces.json new file mode 100644 index 0000000000..2c7bc6f52b --- /dev/null +++ b/tools/clang-tidy/test/modernize-concat-nested-namespaces.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "nested namespaces can be concatenated", + "modernize-concat-nested-namespaces" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.cpp b/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.cpp new file mode 100644 index 0000000000..288fac8483 --- /dev/null +++ b/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.cpp @@ -0,0 +1,18 @@ +namespace std { +class ios_base { +public: + typedef int io_state; + typedef int open_mode; + typedef int seek_dir; + + typedef int streampos; + typedef int streamoff; +}; + +template +class basic_ios : public ios_base { +}; +} // namespace std + +// Test function return values (declaration) +std::ios_base::io_state f_5(); diff --git a/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.json b/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.json new file mode 100644 index 0000000000..7d6d0d055f --- /dev/null +++ b/tools/clang-tidy/test/modernize-deprecated-ios-base-aliases.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'std::ios_base::io_state' is deprecated; use 'std::ios_base::iostate' instead", + "modernize-deprecated-ios-base-aliases" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-loop-convert.cpp b/tools/clang-tidy/test/modernize-loop-convert.cpp new file mode 100644 index 0000000000..2205846c7e --- /dev/null +++ b/tools/clang-tidy/test/modernize-loop-convert.cpp @@ -0,0 +1,7 @@ +int arr[6] = {1, 2, 3, 4, 5, 6}; + +void bar(void) { + for (int i = 0; i < 6; ++i) { + (void)arr[i]; + } +} diff --git a/tools/clang-tidy/test/modernize-loop-convert.json b/tools/clang-tidy/test/modernize-loop-convert.json new file mode 100644 index 0000000000..105e44bc2b --- /dev/null +++ b/tools/clang-tidy/test/modernize-loop-convert.json @@ -0,0 +1,4 @@ +[ + ["warning", "use range-based for loop instead", "modernize-loop-convert"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-raw-string-literal.cpp b/tools/clang-tidy/test/modernize-raw-string-literal.cpp new file mode 100644 index 0000000000..d120aad9bf --- /dev/null +++ b/tools/clang-tidy/test/modernize-raw-string-literal.cpp @@ -0,0 +1 @@ +char const *const ManyQuotes("quotes:\'\'\'\'"); diff --git a/tools/clang-tidy/test/modernize-raw-string-literal.json b/tools/clang-tidy/test/modernize-raw-string-literal.json new file mode 100644 index 0000000000..595deaa7db --- /dev/null +++ b/tools/clang-tidy/test/modernize-raw-string-literal.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "escaped string literal can be written as a raw string literal", + "modernize-raw-string-literal" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-redundant-void-arg.cpp b/tools/clang-tidy/test/modernize-redundant-void-arg.cpp new file mode 100644 index 0000000000..078e0e13ca --- /dev/null +++ b/tools/clang-tidy/test/modernize-redundant-void-arg.cpp @@ -0,0 +1,5 @@ +// modernize-redundant-void-arg + +int foo(void) { + return 0; +} diff --git a/tools/clang-tidy/test/modernize-redundant-void-arg.json b/tools/clang-tidy/test/modernize-redundant-void-arg.json new file mode 100644 index 0000000000..a34b4069b6 --- /dev/null +++ b/tools/clang-tidy/test/modernize-redundant-void-arg.json @@ -0,0 +1 @@ +"[[\"warning\", \"redundant void argument list in function definition\", \"modernize-redundant-void-arg\"]]" diff --git a/tools/clang-tidy/test/modernize-shrink-to-fit.cpp b/tools/clang-tidy/test/modernize-shrink-to-fit.cpp new file mode 100644 index 0000000000..c545a517a3 --- /dev/null +++ b/tools/clang-tidy/test/modernize-shrink-to-fit.cpp @@ -0,0 +1,10 @@ +#include "structures.h" + +void f() { + std::vector v; + + std::vector(v).swap(v); + + std::vector &vref = v; + std::vector(vref).swap(vref); +} diff --git a/tools/clang-tidy/test/modernize-shrink-to-fit.json b/tools/clang-tidy/test/modernize-shrink-to-fit.json new file mode 100644 index 0000000000..f8e4eda19f --- /dev/null +++ b/tools/clang-tidy/test/modernize-shrink-to-fit.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "the shrink_to_fit method should be used to reduce the capacity of a shrinkable container", + "modernize-shrink-to-fit" + ], + [ + "warning", + "the shrink_to_fit method should be used to reduce the capacity of a shrinkable container", + "modernize-shrink-to-fit" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-use-auto.cpp b/tools/clang-tidy/test/modernize-use-auto.cpp new file mode 100644 index 0000000000..ba54d0cb8a --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-auto.cpp @@ -0,0 +1,11 @@ +#include + +void func() { + int val = 42; + std::vector my_container; + for (std::vector::iterator I = my_container.begin(), + E = my_container.end(); + I != E; + ++I) { + } +} diff --git a/tools/clang-tidy/test/modernize-use-auto.json b/tools/clang-tidy/test/modernize-use-auto.json new file mode 100644 index 0000000000..5885e8b6d5 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-auto.json @@ -0,0 +1 @@ +"[[\"warning\", \"use auto when declaring iterators\", \"modernize-use-auto\"]]" diff --git a/tools/clang-tidy/test/modernize-use-bool-literals.cpp b/tools/clang-tidy/test/modernize-use-bool-literals.cpp new file mode 100644 index 0000000000..58b7ec8f29 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-bool-literals.cpp @@ -0,0 +1,5 @@ +void foo() { + bool p = 1; + bool f = static_cast(1); + bool x = p ? 1 : 0; +} diff --git a/tools/clang-tidy/test/modernize-use-bool-literals.json b/tools/clang-tidy/test/modernize-use-bool-literals.json new file mode 100644 index 0000000000..4d5effc1b9 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-bool-literals.json @@ -0,0 +1,23 @@ +[ + [ + "warning", + "converting integer literal to bool, use bool literal instead", + "modernize-use-bool-literals" + ], + [ + "warning", + "converting integer literal to bool, use bool literal instead", + "modernize-use-bool-literals" + ], + [ + "warning", + "converting integer literal to bool, use bool literal instead", + "modernize-use-bool-literals" + ], + [ + "warning", + "converting integer literal to bool, use bool literal instead", + "modernize-use-bool-literals" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-use-equals-default.cpp b/tools/clang-tidy/test/modernize-use-equals-default.cpp new file mode 100644 index 0000000000..ea5f71d93a --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-equals-default.cpp @@ -0,0 +1,6 @@ + +class IL { +public: + IL() {} + ~IL() {} +}; diff --git a/tools/clang-tidy/test/modernize-use-equals-default.json b/tools/clang-tidy/test/modernize-use-equals-default.json new file mode 100644 index 0000000000..5119e361b3 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-equals-default.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "use '= default' to define a trivial default constructor", + "modernize-use-equals-default" + ], + [ + "warning", + "use '= default' to define a trivial destructor", + "modernize-use-equals-default" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-use-equals-delete.cpp b/tools/clang-tidy/test/modernize-use-equals-delete.cpp new file mode 100644 index 0000000000..f08848c0a6 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-equals-delete.cpp @@ -0,0 +1,7 @@ +struct PositivePrivate { +private: + PositivePrivate(); + PositivePrivate(const PositivePrivate &); + PositivePrivate &operator=(PositivePrivate &&); + ~PositivePrivate(); +}; diff --git a/tools/clang-tidy/test/modernize-use-equals-delete.json b/tools/clang-tidy/test/modernize-use-equals-delete.json new file mode 100644 index 0000000000..bcb9354149 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-equals-delete.json @@ -0,0 +1,23 @@ +[ + [ + "warning", + "use '= delete' to prohibit calling of a special member function", + "modernize-use-equals-delete" + ], + [ + "warning", + "use '= delete' to prohibit calling of a special member function", + "modernize-use-equals-delete" + ], + [ + "warning", + "use '= delete' to prohibit calling of a special member function", + "modernize-use-equals-delete" + ], + [ + "warning", + "use '= delete' to prohibit calling of a special member function", + "modernize-use-equals-delete" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/modernize-use-nullptr.cpp b/tools/clang-tidy/test/modernize-use-nullptr.cpp new file mode 100644 index 0000000000..4b8b3ee3c0 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-nullptr.cpp @@ -0,0 +1,5 @@ +#define NULL 0 +void f(void) { + char *str = NULL; // ok + (void)str; +} diff --git a/tools/clang-tidy/test/modernize-use-nullptr.json b/tools/clang-tidy/test/modernize-use-nullptr.json new file mode 100644 index 0000000000..44bd18a10c --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-nullptr.json @@ -0,0 +1 @@ +[["warning", "use nullptr", "modernize-use-nullptr"], { "reliability": "high" }] diff --git a/tools/clang-tidy/test/modernize-use-override.cpp b/tools/clang-tidy/test/modernize-use-override.cpp new file mode 100644 index 0000000000..1cbec3868c --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-override.cpp @@ -0,0 +1,8 @@ +class Base { +public: + virtual void foo() = 0; +}; + +class Deriv : public Base { + void foo(); +}; diff --git a/tools/clang-tidy/test/modernize-use-override.json b/tools/clang-tidy/test/modernize-use-override.json new file mode 100644 index 0000000000..64f8b6e870 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-override.json @@ -0,0 +1 @@ +"[[\"warning\", \"annotate this function with 'override' or (rarely) 'final'\", \"modernize-use-override\"]]" diff --git a/tools/clang-tidy/test/modernize-use-using.cpp b/tools/clang-tidy/test/modernize-use-using.cpp new file mode 100644 index 0000000000..0e7b73222c --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-using.cpp @@ -0,0 +1,5 @@ +template +class Test { + typedef typename T::iterator Iter; +}; +typedef int Type; diff --git a/tools/clang-tidy/test/modernize-use-using.json b/tools/clang-tidy/test/modernize-use-using.json new file mode 100644 index 0000000000..50ee7060a9 --- /dev/null +++ b/tools/clang-tidy/test/modernize-use-using.json @@ -0,0 +1,5 @@ +[ + ["warning", "use 'using' instead of 'typedef'", "modernize-use-using"], + ["warning", "use 'using' instead of 'typedef'", "modernize-use-using"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-faster-string-find.cpp b/tools/clang-tidy/test/performance-faster-string-find.cpp new file mode 100644 index 0000000000..d7ac3d0c3c --- /dev/null +++ b/tools/clang-tidy/test/performance-faster-string-find.cpp @@ -0,0 +1,6 @@ +#include "structures.h" + +void foo() { + std::string str; + str.find("A"); +} diff --git a/tools/clang-tidy/test/performance-faster-string-find.json b/tools/clang-tidy/test/performance-faster-string-find.json new file mode 100644 index 0000000000..1ab2d7ba08 --- /dev/null +++ b/tools/clang-tidy/test/performance-faster-string-find.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'find' called with a string literal consisting of a single character; consider using the more effective overload accepting a character", + "performance-faster-string-find" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-for-range-copy.cpp b/tools/clang-tidy/test/performance-for-range-copy.cpp new file mode 100644 index 0000000000..264dd42896 --- /dev/null +++ b/tools/clang-tidy/test/performance-for-range-copy.cpp @@ -0,0 +1,30 @@ +template +struct Iterator { + void operator++() {} + const T& operator*() { + static T* TT = new T(); + return *TT; + } + bool operator!=(const Iterator &) { return false; } + typedef const T& const_reference; +}; +template +struct View { + T begin() { return T(); } + T begin() const { return T(); } + T end() { return T(); } + T end() const { return T(); } + typedef typename T::const_reference const_reference; +}; + +struct S { + S(); + S(const S &); + ~S(); + S &operator=(const S &); +}; + +void negativeConstReference() { + for (const S S1 : View>()) { + } +} diff --git a/tools/clang-tidy/test/performance-for-range-copy.json b/tools/clang-tidy/test/performance-for-range-copy.json new file mode 100644 index 0000000000..2082041aad --- /dev/null +++ b/tools/clang-tidy/test/performance-for-range-copy.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the loop variable's type is not a reference type; this creates a copy in each iteration; consider making this a reference", + "performance-for-range-copy" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-implicit-conversion-in-loop.cpp b/tools/clang-tidy/test/performance-implicit-conversion-in-loop.cpp new file mode 100644 index 0000000000..75ed510f3f --- /dev/null +++ b/tools/clang-tidy/test/performance-implicit-conversion-in-loop.cpp @@ -0,0 +1,43 @@ +// Iterator returning by value. +template +struct Iterator { + void operator++(); + T operator*(); + bool operator!=(const Iterator& other); +}; + +// The template argument is an iterator type, and a view is an object you can +// run a for loop on. +template +struct View { + T begin(); + T end(); +}; + +// With this class, the implicit conversion is a call to the (implicit) +// constructor of the class. +template +class ImplicitWrapper { + public: + // Implicit! + ImplicitWrapper(const T& t); +}; + +template +class OperatorWrapper { + public: + OperatorWrapper() = delete; +}; + +struct SimpleClass { + int foo; + operator OperatorWrapper(); +}; + +typedef View> SimpleView; + +void ImplicitSimpleClassIterator() { + for (const ImplicitWrapper& foo : SimpleView()) {} + for (const ImplicitWrapper foo : SimpleView()) {} + for (ImplicitWrapper foo : SimpleView()) {} +} diff --git a/tools/clang-tidy/test/performance-implicit-conversion-in-loop.json b/tools/clang-tidy/test/performance-implicit-conversion-in-loop.json new file mode 100644 index 0000000000..d9bcb1e00a --- /dev/null +++ b/tools/clang-tidy/test/performance-implicit-conversion-in-loop.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the type of the loop variable 'foo' is different from the one returned by the iterator and generates an implicit conversion; you can either change the type to the matching one ('const SimpleClass &' but 'const auto&' is always a valid option) or remove the reference to make it explicit that you are creating a new value", + "performance-implicit-conversion-in-loop" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-inefficient-algorithm.cpp b/tools/clang-tidy/test/performance-inefficient-algorithm.cpp new file mode 100644 index 0000000000..b508260dbd --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-algorithm.cpp @@ -0,0 +1,30 @@ +namespace std { +template struct less { + bool operator()(const T &lhs, const T &rhs) { return lhs < rhs; } +}; + +template struct greater { + bool operator()(const T &lhs, const T &rhs) { return lhs > rhs; } +}; + +struct iterator_type {}; + +template > struct set { + typedef iterator_type iterator; + iterator find(const K &k); + unsigned count(const K &k); + + iterator begin(); + iterator end(); + iterator begin() const; + iterator end() const; +}; + +template +FwIt find(FwIt, FwIt end, const K &) { return end; } +} + +template void f(const T &t) { + std::set s; + find(s.begin(), s.end(), 46); +} diff --git a/tools/clang-tidy/test/performance-inefficient-algorithm.json b/tools/clang-tidy/test/performance-inefficient-algorithm.json new file mode 100644 index 0000000000..e3c575bd85 --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-algorithm.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "this STL algorithm call should be replaced with a container method", + "performance-inefficient-algorithm" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-inefficient-string-concatenation.cpp b/tools/clang-tidy/test/performance-inefficient-string-concatenation.cpp new file mode 100644 index 0000000000..5a5860f215 --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-string-concatenation.cpp @@ -0,0 +1,13 @@ +#include "structures.h" + +extern void fstring(std::string); + +void foo() { + std::string mystr1, mystr2; + auto myautostr1 = mystr1; + auto myautostr2 = mystr2; + + for (int i = 0; i < 10; ++i) { + fstring(mystr1 + mystr2 + mystr1); + } +} diff --git a/tools/clang-tidy/test/performance-inefficient-string-concatenation.json b/tools/clang-tidy/test/performance-inefficient-string-concatenation.json new file mode 100644 index 0000000000..8c7223db6d --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-string-concatenation.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "string concatenation results in allocation of unnecessary temporary strings; consider using 'operator+=' or 'string::append()' instead", + "performance-inefficient-string-concatenation" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-inefficient-vector-operation.cpp b/tools/clang-tidy/test/performance-inefficient-vector-operation.cpp new file mode 100644 index 0000000000..4f0e143aa2 --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-vector-operation.cpp @@ -0,0 +1,10 @@ +#include "structures.h" + +void foo() +{ + std::vector v; + int n = 100; + for (int i = 0; i < n; ++i) { + v.push_back(n); + } +} diff --git a/tools/clang-tidy/test/performance-inefficient-vector-operation.json b/tools/clang-tidy/test/performance-inefficient-vector-operation.json new file mode 100644 index 0000000000..d3e9e769d2 --- /dev/null +++ b/tools/clang-tidy/test/performance-inefficient-vector-operation.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'push_back' is called inside a loop; consider pre-allocating the container capacity before the loop", + "performance-inefficient-vector-operation" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-move-const-arg.cpp b/tools/clang-tidy/test/performance-move-const-arg.cpp new file mode 100644 index 0000000000..39f1ce06ec --- /dev/null +++ b/tools/clang-tidy/test/performance-move-const-arg.cpp @@ -0,0 +1,33 @@ +namespace std { +template +struct remove_reference { + typedef _Tp type; +}; + +template +constexpr typename std::remove_reference<_Tp>::type &&move(_Tp &&__t) { + return static_cast::type &&>(__t); +} +} // namespace std + +struct TriviallyCopyable { + int i; +}; + +class A { +public: + A() {} + A(const A &rhs) {} + A(A &&rhs) {} +}; + +void f(TriviallyCopyable) {} + +void g() { + TriviallyCopyable obj; + f(std::move(obj)); +} + +A f5(const A x5) { + return std::move(x5); +} diff --git a/tools/clang-tidy/test/performance-move-const-arg.json b/tools/clang-tidy/test/performance-move-const-arg.json new file mode 100644 index 0000000000..d56fe6bf37 --- /dev/null +++ b/tools/clang-tidy/test/performance-move-const-arg.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "std::move of the const variable 'x5' has no effect; remove std::move() or make the variable non-const", + "performance-move-const-arg" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-move-constructor-init.cpp b/tools/clang-tidy/test/performance-move-constructor-init.cpp new file mode 100644 index 0000000000..243b399e95 --- /dev/null +++ b/tools/clang-tidy/test/performance-move-constructor-init.cpp @@ -0,0 +1,11 @@ +struct B { + B() {} + B(const B&) {} + B(B &&) {} +}; + +struct D : B { + D() : B() {} + D(const D &RHS) : B(RHS) {} + D(D &&RHS) : B(RHS) {} +}; diff --git a/tools/clang-tidy/test/performance-move-constructor-init.json b/tools/clang-tidy/test/performance-move-constructor-init.json new file mode 100644 index 0000000000..17582a86e7 --- /dev/null +++ b/tools/clang-tidy/test/performance-move-constructor-init.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "move constructor initializes base class by calling a copy constructor", + "performance-move-constructor-init" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-noexcept-move-constructor.cpp b/tools/clang-tidy/test/performance-noexcept-move-constructor.cpp new file mode 100644 index 0000000000..8b4900b00d --- /dev/null +++ b/tools/clang-tidy/test/performance-noexcept-move-constructor.cpp @@ -0,0 +1,4 @@ +class A { + A(A &&); + A &operator=(A &&); +}; diff --git a/tools/clang-tidy/test/performance-noexcept-move-constructor.json b/tools/clang-tidy/test/performance-noexcept-move-constructor.json new file mode 100644 index 0000000000..94823b9ed5 --- /dev/null +++ b/tools/clang-tidy/test/performance-noexcept-move-constructor.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "move constructors should be marked noexcept", + "performance-noexcept-move-constructor" + ], + [ + "warning", + "move assignment operators should be marked noexcept", + "performance-noexcept-move-constructor" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-type-promotion-in-math-fn.cpp b/tools/clang-tidy/test/performance-type-promotion-in-math-fn.cpp new file mode 100644 index 0000000000..9a6fcf9848 --- /dev/null +++ b/tools/clang-tidy/test/performance-type-promotion-in-math-fn.cpp @@ -0,0 +1,7 @@ +double acos(double); + +void check_all_fns() +{ + float a; + acos(a); +} diff --git a/tools/clang-tidy/test/performance-type-promotion-in-math-fn.json b/tools/clang-tidy/test/performance-type-promotion-in-math-fn.json new file mode 100644 index 0000000000..577d2ddc91 --- /dev/null +++ b/tools/clang-tidy/test/performance-type-promotion-in-math-fn.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "call to 'acos' promotes float to double", + "performance-type-promotion-in-math-fn" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-unnecessary-copy-initialization.cpp b/tools/clang-tidy/test/performance-unnecessary-copy-initialization.cpp new file mode 100644 index 0000000000..ca0f591a3e --- /dev/null +++ b/tools/clang-tidy/test/performance-unnecessary-copy-initialization.cpp @@ -0,0 +1,7 @@ +#include "structures.h" + +extern const std::string& constReference(); + +void foo() { + const std::string UnnecessaryCopy = constReference(); +} diff --git a/tools/clang-tidy/test/performance-unnecessary-copy-initialization.json b/tools/clang-tidy/test/performance-unnecessary-copy-initialization.json new file mode 100644 index 0000000000..fcb16746ed --- /dev/null +++ b/tools/clang-tidy/test/performance-unnecessary-copy-initialization.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the const qualified variable 'UnnecessaryCopy' is copy-constructed from a const reference but is never used; consider removing the statement", + "performance-unnecessary-copy-initialization" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/performance-unnecessary-value-param.cpp b/tools/clang-tidy/test/performance-unnecessary-value-param.cpp new file mode 100644 index 0000000000..ed5f36f7fa --- /dev/null +++ b/tools/clang-tidy/test/performance-unnecessary-value-param.cpp @@ -0,0 +1,4 @@ +#include "structures.h" + +void f(const std::string Value) { +} diff --git a/tools/clang-tidy/test/performance-unnecessary-value-param.json b/tools/clang-tidy/test/performance-unnecessary-value-param.json new file mode 100644 index 0000000000..35ed09e4be --- /dev/null +++ b/tools/clang-tidy/test/performance-unnecessary-value-param.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the const qualified parameter 'Value' is copied for each invocation; consider making it a reference", + "performance-unnecessary-value-param" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-braces-around-statements.cpp b/tools/clang-tidy/test/readability-braces-around-statements.cpp new file mode 100644 index 0000000000..ce456ec2e3 --- /dev/null +++ b/tools/clang-tidy/test/readability-braces-around-statements.cpp @@ -0,0 +1,38 @@ + +void do_something(const char *) {} + +bool cond(const char *) { + return false; +} + +void test() { +if (cond("if0") /*comment*/) do_something("same-line"); + +if (cond("if1") /*comment*/) + do_something("next-line"); + + if (!1) return; + if (!2) { return; } + + if (!3) + return; + + if (!4) { + return; + } +} + +void foo() { +if (1) while (2) if (3) for (;;) do ; while(false) /**/;/**/ +} + +void f() {} + +void foo2() { + constexpr bool a = true; + if constexpr (a) { + f(); + } else { + f(); + } +} diff --git a/tools/clang-tidy/test/readability-braces-around-statements.json b/tools/clang-tidy/test/readability-braces-around-statements.json new file mode 100644 index 0000000000..7cc8ae280a --- /dev/null +++ b/tools/clang-tidy/test/readability-braces-around-statements.json @@ -0,0 +1,13 @@ +[ + [ + "warning", + "statement should be inside braces", + "readability-braces-around-statements" + ], + [ + "warning", + "statement should be inside braces", + "readability-braces-around-statements" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-const-return-type.cpp b/tools/clang-tidy/test/readability-const-return-type.cpp new file mode 100644 index 0000000000..695ed1d83f --- /dev/null +++ b/tools/clang-tidy/test/readability-const-return-type.cpp @@ -0,0 +1,5 @@ +const int p1() { +// CHECK-MESSAGES: [[@LINE-1]]:1: warning: return type 'const int' is 'const'-qualified at the top level, which may reduce code readability without improving const correctness +// CHECK-FIXES: int p1() { + return 1; +} diff --git a/tools/clang-tidy/test/readability-const-return-type.json b/tools/clang-tidy/test/readability-const-return-type.json new file mode 100644 index 0000000000..f6c634c0b1 --- /dev/null +++ b/tools/clang-tidy/test/readability-const-return-type.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "return type 'const int' is 'const'-qualified at the top level, which may reduce code readability without improving const correctness", + "readability-const-return-type" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-container-size-empty.cpp b/tools/clang-tidy/test/readability-container-size-empty.cpp new file mode 100644 index 0000000000..1dcfe68bd7 --- /dev/null +++ b/tools/clang-tidy/test/readability-container-size-empty.cpp @@ -0,0 +1,7 @@ +#include "structures.h" + +void foo() { + std::string a; + if (a.size()) + return; +} diff --git a/tools/clang-tidy/test/readability-container-size-empty.json b/tools/clang-tidy/test/readability-container-size-empty.json new file mode 100644 index 0000000000..8c51b1c039 --- /dev/null +++ b/tools/clang-tidy/test/readability-container-size-empty.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "the 'empty' method should be used to check for emptiness instead of 'size'", + "readability-container-size-empty" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-delete-null-pointer.cpp b/tools/clang-tidy/test/readability-delete-null-pointer.cpp new file mode 100644 index 0000000000..e083404043 --- /dev/null +++ b/tools/clang-tidy/test/readability-delete-null-pointer.cpp @@ -0,0 +1,6 @@ +void func() { + int* f = 0; + if (f) { + delete f; + } +} diff --git a/tools/clang-tidy/test/readability-delete-null-pointer.json b/tools/clang-tidy/test/readability-delete-null-pointer.json new file mode 100644 index 0000000000..2a7184cdb1 --- /dev/null +++ b/tools/clang-tidy/test/readability-delete-null-pointer.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'if' statement is unnecessary; deleting null pointer has no effect", + "readability-delete-null-pointer" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-else-after-return.cpp b/tools/clang-tidy/test/readability-else-after-return.cpp new file mode 100644 index 0000000000..2c72b68a92 --- /dev/null +++ b/tools/clang-tidy/test/readability-else-after-return.cpp @@ -0,0 +1,10 @@ +void f() { + +} + +void foo() { + if (true) + return; + else + f(); +} diff --git a/tools/clang-tidy/test/readability-else-after-return.json b/tools/clang-tidy/test/readability-else-after-return.json new file mode 100644 index 0000000000..7d2f8b0adf --- /dev/null +++ b/tools/clang-tidy/test/readability-else-after-return.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "do not use 'else' after 'return'", + "readability-else-after-return" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-implicit-bool-conversion.cpp b/tools/clang-tidy/test/readability-implicit-bool-conversion.cpp new file mode 100644 index 0000000000..c30089a126 --- /dev/null +++ b/tools/clang-tidy/test/readability-implicit-bool-conversion.cpp @@ -0,0 +1,55 @@ + +#define MOZ_IMPLICIT __attribute__((annotate("moz_implicit"))) + +void takesChar(char); +void takesShort(short); +void takesInt(int); +void takesLong(long); + +void takesUChar(unsigned char); +void takesUShort(unsigned short); +void takesUInt(unsigned int); +void takesULong(unsigned long); + +struct InitializedWithInt { + MOZ_IMPLICIT InitializedWithInt(int); +}; + +void f() { + bool b = true; + char s0 = b; + short s1 = b; + int s2 = b; + long s3 = b; + + unsigned char u0 = b; + unsigned short u1 = b; + unsigned u2 = b; + unsigned long u3 = b; + + takesChar(b); + takesShort(b); + takesInt(b); + takesLong(b); + takesUChar(b); + takesUShort(b); + takesUInt(b); + takesULong(b); + + InitializedWithInt i = b; + (InitializedWithInt(b)); + + bool x = b; + + int exp = (int)true; + + if (x == b) {} + if (x != b) {} + + if (b == exp) {} + if (exp == b) {} + + char* ptr; + // Shouldn't trigger a checker warning since we are using AllowPointerConditions + if (ptr) {} +} diff --git a/tools/clang-tidy/test/readability-implicit-bool-conversion.json b/tools/clang-tidy/test/readability-implicit-bool-conversion.json new file mode 100644 index 0000000000..b10044f03f --- /dev/null +++ b/tools/clang-tidy/test/readability-implicit-bool-conversion.json @@ -0,0 +1,102 @@ +[ + [ + "warning", + "implicit conversion bool -> 'char'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'short'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'long'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned char'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned short'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned long'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'char'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'short'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'long'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned char'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned short'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'unsigned long'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ], + [ + "warning", + "implicit conversion bool -> 'int'", + "readability-implicit-bool-conversion" + ] +] diff --git a/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.cpp b/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.cpp new file mode 100644 index 0000000000..acf13b240b --- /dev/null +++ b/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.cpp @@ -0,0 +1,6 @@ +struct S { + void f(int x); +}; + +void S::f(int y) { +} diff --git a/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.json b/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.json new file mode 100644 index 0000000000..f5e04a1041 --- /dev/null +++ b/tools/clang-tidy/test/readability-inconsistent-declaration-parameter-name.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "function 'S::f' has a definition with different parameter names", + "readability-inconsistent-declaration-parameter-name" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-isolate-declaration.cpp b/tools/clang-tidy/test/readability-isolate-declaration.cpp new file mode 100644 index 0000000000..5d64e0171f --- /dev/null +++ b/tools/clang-tidy/test/readability-isolate-declaration.cpp @@ -0,0 +1,7 @@ +void f() { + int * pointer = nullptr, value = 42, * const const_ptr = &value; + // This declaration will be diagnosed and transformed into: + // int * pointer = nullptr; + // int value = 42; + // int * const const_ptr = &value; +} diff --git a/tools/clang-tidy/test/readability-isolate-declaration.json b/tools/clang-tidy/test/readability-isolate-declaration.json new file mode 100644 index 0000000000..3f8ccf52fa --- /dev/null +++ b/tools/clang-tidy/test/readability-isolate-declaration.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "multiple declarations in a single statement reduces readability", + "readability-isolate-declaration" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-magic-numbers.cpp b/tools/clang-tidy/test/readability-magic-numbers.cpp new file mode 100644 index 0000000000..430fa17056 --- /dev/null +++ b/tools/clang-tidy/test/readability-magic-numbers.cpp @@ -0,0 +1,4 @@ +void func() { + int radius = 2; + double circleArea = 3.1415926535 * radius * radius; +} diff --git a/tools/clang-tidy/test/readability-magic-numbers.json b/tools/clang-tidy/test/readability-magic-numbers.json new file mode 100644 index 0000000000..ee5d0f204c --- /dev/null +++ b/tools/clang-tidy/test/readability-magic-numbers.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "3.1415926535 is a magic number; consider replacing it with a named constant", + "readability-magic-numbers" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-misleading-indentation.cpp b/tools/clang-tidy/test/readability-misleading-indentation.cpp new file mode 100644 index 0000000000..8b23010170 --- /dev/null +++ b/tools/clang-tidy/test/readability-misleading-indentation.cpp @@ -0,0 +1,20 @@ +void f() +{ +} + +void foo() { + if (1) + if (0) + f(); + else + f(); +} + +void foo2() { + constexpr bool a = true; + if constexpr (a) { + f(); + } else { + f(); + } +} diff --git a/tools/clang-tidy/test/readability-misleading-indentation.json b/tools/clang-tidy/test/readability-misleading-indentation.json new file mode 100644 index 0000000000..9ef6d30c45 --- /dev/null +++ b/tools/clang-tidy/test/readability-misleading-indentation.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "different indentation for 'if' and corresponding 'else'", + "readability-misleading-indentation" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-non-const-parameter.cpp b/tools/clang-tidy/test/readability-non-const-parameter.cpp new file mode 100644 index 0000000000..5a0ffd0e30 --- /dev/null +++ b/tools/clang-tidy/test/readability-non-const-parameter.cpp @@ -0,0 +1,5 @@ +void warn1(int *first, int *last) { + *first = 0; + if (first < last) { + } +} diff --git a/tools/clang-tidy/test/readability-non-const-parameter.json b/tools/clang-tidy/test/readability-non-const-parameter.json new file mode 100644 index 0000000000..fb83ec8572 --- /dev/null +++ b/tools/clang-tidy/test/readability-non-const-parameter.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "pointer parameter 'last' can be pointer to const", + "readability-non-const-parameter" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-qualified-auto.cpp b/tools/clang-tidy/test/readability-qualified-auto.cpp new file mode 100644 index 0000000000..03c5164e2c --- /dev/null +++ b/tools/clang-tidy/test/readability-qualified-auto.cpp @@ -0,0 +1,6 @@ +typedef int *MyPtr; +MyPtr getPtr(); + +void foo() { + auto TdNakedPtr = getPtr(); +} diff --git a/tools/clang-tidy/test/readability-qualified-auto.json b/tools/clang-tidy/test/readability-qualified-auto.json new file mode 100644 index 0000000000..eaf0a176cf --- /dev/null +++ b/tools/clang-tidy/test/readability-qualified-auto.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "'auto TdNakedPtr' can be declared as 'auto *TdNakedPtr'", + "readability-qualified-auto" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-control-flow.cpp b/tools/clang-tidy/test/readability-redundant-control-flow.cpp new file mode 100644 index 0000000000..d221715542 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-control-flow.cpp @@ -0,0 +1,5 @@ +extern void g(); +void f() { + g(); + return; +} diff --git a/tools/clang-tidy/test/readability-redundant-control-flow.json b/tools/clang-tidy/test/readability-redundant-control-flow.json new file mode 100644 index 0000000000..efedaf1009 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-control-flow.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "redundant return statement at the end of a function with a void return type", + "readability-redundant-control-flow" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-preprocessor.cpp b/tools/clang-tidy/test/readability-redundant-preprocessor.cpp new file mode 100644 index 0000000000..317905dab8 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-preprocessor.cpp @@ -0,0 +1,5 @@ +#ifndef FOO +#ifdef FOO // inner ifdef is considered redundant +void f(); +#endif +#endif diff --git a/tools/clang-tidy/test/readability-redundant-preprocessor.json b/tools/clang-tidy/test/readability-redundant-preprocessor.json new file mode 100644 index 0000000000..e3b7fb4ce1 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-preprocessor.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "nested redundant #ifdef; consider removing it", + "readability-redundant-preprocessor" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-smartptr-get.cpp b/tools/clang-tidy/test/readability-redundant-smartptr-get.cpp new file mode 100644 index 0000000000..66eb8d6c37 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-smartptr-get.cpp @@ -0,0 +1,19 @@ +#define NULL __null + +namespace std { + +template +struct unique_ptr { + T& operator*() const; + T* operator->() const; + T* get() const; + explicit operator bool() const noexcept; +}; +} + +struct A { +}; + +void foo() { + A& b2 = *std::unique_ptr().get(); +} diff --git a/tools/clang-tidy/test/readability-redundant-smartptr-get.json b/tools/clang-tidy/test/readability-redundant-smartptr-get.json new file mode 100644 index 0000000000..c5f06cbc21 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-smartptr-get.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "redundant get() call on smart pointer", + "readability-redundant-smartptr-get" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-string-cstr.cpp b/tools/clang-tidy/test/readability-redundant-string-cstr.cpp new file mode 100644 index 0000000000..d35a2998e5 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-string-cstr.cpp @@ -0,0 +1,7 @@ +#include "structures.h" + +void foo() { + std::string a = "Mozilla"; + std::string tmp; + tmp.assign(a.c_str()); +} diff --git a/tools/clang-tidy/test/readability-redundant-string-cstr.json b/tools/clang-tidy/test/readability-redundant-string-cstr.json new file mode 100644 index 0000000000..cce79a38a2 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-string-cstr.json @@ -0,0 +1,4 @@ +[ + ["warning", "redundant call to 'c_str'", "readability-redundant-string-cstr"], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-redundant-string-init.cpp b/tools/clang-tidy/test/readability-redundant-string-init.cpp new file mode 100644 index 0000000000..39bae4bd56 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-string-init.cpp @@ -0,0 +1,5 @@ +#include "structures.h" + +int foo() { + std::string a = ""; +} diff --git a/tools/clang-tidy/test/readability-redundant-string-init.json b/tools/clang-tidy/test/readability-redundant-string-init.json new file mode 100644 index 0000000000..3092808205 --- /dev/null +++ b/tools/clang-tidy/test/readability-redundant-string-init.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "redundant string initialization", + "readability-redundant-string-init" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-simplify-boolean-expr.cpp b/tools/clang-tidy/test/readability-simplify-boolean-expr.cpp new file mode 100644 index 0000000000..277a469880 --- /dev/null +++ b/tools/clang-tidy/test/readability-simplify-boolean-expr.cpp @@ -0,0 +1,2 @@ +bool a1 = false; +bool aa = false == a1; diff --git a/tools/clang-tidy/test/readability-simplify-boolean-expr.json b/tools/clang-tidy/test/readability-simplify-boolean-expr.json new file mode 100644 index 0000000000..45eb9e1a66 --- /dev/null +++ b/tools/clang-tidy/test/readability-simplify-boolean-expr.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "redundant boolean literal supplied to boolean operator", + "readability-simplify-boolean-expr" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-static-accessed-through-instance.cpp b/tools/clang-tidy/test/readability-static-accessed-through-instance.cpp new file mode 100644 index 0000000000..95f13b994c --- /dev/null +++ b/tools/clang-tidy/test/readability-static-accessed-through-instance.cpp @@ -0,0 +1,11 @@ +struct C { + static int x; +}; + +int C::x = 0; + +// Expressions with side effects +C &f(int, int, int, int); +void g() { + f(1, 2, 3, 4).x; +} diff --git a/tools/clang-tidy/test/readability-static-accessed-through-instance.json b/tools/clang-tidy/test/readability-static-accessed-through-instance.json new file mode 100644 index 0000000000..67c84c188b --- /dev/null +++ b/tools/clang-tidy/test/readability-static-accessed-through-instance.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "static member accessed through instance", + "readability-static-accessed-through-instance" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/readability-uniqueptr-delete-release.cpp b/tools/clang-tidy/test/readability-uniqueptr-delete-release.cpp new file mode 100644 index 0000000000..4fce9cde4e --- /dev/null +++ b/tools/clang-tidy/test/readability-uniqueptr-delete-release.cpp @@ -0,0 +1,6 @@ +#include "structures.h" + +int foo() { + std::unique_ptr P; + delete P.release(); +} diff --git a/tools/clang-tidy/test/readability-uniqueptr-delete-release.json b/tools/clang-tidy/test/readability-uniqueptr-delete-release.json new file mode 100644 index 0000000000..79330346e1 --- /dev/null +++ b/tools/clang-tidy/test/readability-uniqueptr-delete-release.json @@ -0,0 +1,8 @@ +[ + [ + "warning", + "prefer '= nullptr' to reset 'unique_ptr<>' objects", + "readability-uniqueptr-delete-release" + ], + { "reliability": "high" } +] diff --git a/tools/clang-tidy/test/structures.h b/tools/clang-tidy/test/structures.h new file mode 100644 index 0000000000..0966109635 --- /dev/null +++ b/tools/clang-tidy/test/structures.h @@ -0,0 +1,108 @@ +// Proxy file in order to define generic data types, to avoid binding with system headers + +typedef __SIZE_TYPE__ size_t; + +namespace std { + +typedef size_t size_t; + +template +class vector { + public: + typedef T* iterator; + typedef const T* const_iterator; + typedef T& reference; + typedef const T& const_reference; + typedef size_t size_type; + + explicit vector(); + explicit vector(size_type n); + + void swap(vector &other); + void push_back(const T& val); + + template void emplace_back(Args &&... args); + + void reserve(size_t n); + void resize(size_t n); + + size_t size(); + const_reference operator[] (size_type) const; + reference operator[] (size_type); + + const_iterator begin() const; + const_iterator end() const; +}; + +template +class basic_string { +public: + typedef basic_string _Type; + basic_string() {} + basic_string(const T *p); + basic_string(const T *p, size_t count); + basic_string(size_t count, char ch); + ~basic_string() {} + size_t size() const; + bool empty() const; + size_t find (const char* s, size_t pos = 0) const; + const T *c_str() const; + _Type& assign(const T *s); + basic_string &operator=(T ch); + basic_string *operator+=(const basic_string &) {} + friend basic_string operator+(const basic_string &, const basic_string &) {} +}; +typedef basic_string string; +typedef basic_string wstring; + +string to_string(int value); + +template +struct default_delete {}; + +template > +class unique_ptr { + public: + unique_ptr(); + ~unique_ptr(); + explicit unique_ptr(T*); + template + unique_ptr(unique_ptr&&); + T* release(); +}; + +template +class bind_rt {}; + +template +bind_rt bind(Fp &&, Arguments &&...); +} + +typedef unsigned int uid_t; +typedef unsigned int pid_t; + +int bcmp(void *, void *, size_t); +void bcopy(void *, void *, size_t); +void bzero(void *, size_t); + +int getpw(uid_t uid, char *buf); +int setuid(uid_t uid); + +int mkstemp(char *tmpl); +char *mktemp(char *tmpl); + +pid_t vfork(void); + +int abort() { return 0; } + +#define assert(x) \ + if (!(x)) \ + (void)abort() + +size_t strlen(const char *s); +char *strncat(char *s1, const char *s2, size_t); + +void free(void *ptr); +void *malloc(size_t size); + +void *memset(void *b, int c, size_t len); diff --git a/tools/code-coverage/CodeCoverageHandler.cpp b/tools/code-coverage/CodeCoverageHandler.cpp new file mode 100644 index 0000000000..fbe7494c6b --- /dev/null +++ b/tools/code-coverage/CodeCoverageHandler.cpp @@ -0,0 +1,158 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include +#ifdef XP_WIN +# include +# define getpid _getpid +#else +# include +# include +#endif +#include "js/experimental/CodeCoverage.h" +#include "mozilla/Atomics.h" +#include "mozilla/dom/ScriptSettings.h" // for AutoJSAPI +#include "mozilla/CodeCoverageHandler.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/DebugOnly.h" +#include "nsAppRunner.h" +#include "nsIFile.h" +#include "nsIOutputStream.h" +#include "nsNetUtil.h" +#include "nsPrintfCString.h" +#include "prtime.h" + +using namespace mozilla; + +// The __gcov_flush function writes the coverage counters to gcda files and then +// resets them to zero. It is defined at +// https://github.com/gcc-mirror/gcc/blob/aad93da1a579b9ae23ede6b9cf8523360f0a08b4/libgcc/libgcov-interface.c. +// __gcov_flush is protected by a mutex in GCC, but not in LLVM, so we are using +// a CrossProcessMutex to protect it. + +extern "C" void __gcov_flush(); +extern "C" void __gcov_dump(); +extern "C" void __gcov_reset(); + +StaticAutoPtr CodeCoverageHandler::instance; + +void CodeCoverageHandler::FlushCounters(const bool initialized) { + static Atomic hasBeenInitialized(false); + if (!hasBeenInitialized) { + hasBeenInitialized = initialized; + return; + } + + printf_stderr("[CodeCoverage] Requested flush for %d.\n", getpid()); + + CrossProcessMutexAutoLock lock(*CodeCoverageHandler::Get()->GetMutex()); + +#if defined(__clang__) && __clang_major__ >= 12 + __gcov_dump(); + __gcov_reset(); +#else + __gcov_flush(); +#endif + + printf_stderr("[CodeCoverage] flush completed.\n"); + + const char* outDir = getenv("JS_CODE_COVERAGE_OUTPUT_DIR"); + if (!outDir || *outDir == 0) { + return; + } + + dom::AutoJSAPI jsapi; + jsapi.Init(); + size_t length; + JS::UniqueChars result = js::GetCodeCoverageSummaryAll(jsapi.cx(), &length); + if (!result) { + return; + } + + nsCOMPtr file; + + nsresult rv = NS_NewNativeLocalFile(nsDependentCString(outDir), false, + getter_AddRefs(file)); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + rv = file->AppendNative( + nsPrintfCString("%lu-%d.info", PR_Now() / PR_USEC_PER_MSEC, getpid())); + + rv = file->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 0666); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + nsCOMPtr outputStream; + rv = NS_NewLocalFileOutputStream(getter_AddRefs(outputStream), file); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + char* data = result.get(); + while (length) { + uint32_t n = 0; + rv = outputStream->Write(data, length, &n); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + data += n; + length -= n; + } + + rv = outputStream->Close(); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + printf_stderr("[CodeCoverage] JS flush completed.\n"); +} + +void CodeCoverageHandler::FlushCountersSignalHandler(int) { FlushCounters(); } + +void CodeCoverageHandler::SetSignalHandlers() { +#ifndef XP_WIN + printf_stderr("[CodeCoverage] Setting handlers for process %d.\n", getpid()); + + struct sigaction dump_sa; + dump_sa.sa_handler = CodeCoverageHandler::FlushCountersSignalHandler; + dump_sa.sa_flags = SA_RESTART; + sigemptyset(&dump_sa.sa_mask); + DebugOnly r1 = sigaction(SIGUSR1, &dump_sa, nullptr); + MOZ_ASSERT(r1 == 0, "Failed to install GCOV SIGUSR1 handler"); +#endif +} + +CodeCoverageHandler::CodeCoverageHandler() : mGcovLock("GcovLock") { + SetSignalHandlers(); +} + +CodeCoverageHandler::CodeCoverageHandler(CrossProcessMutexHandle aHandle) + : mGcovLock(std::move(aHandle)) { + SetSignalHandlers(); +} + +void CodeCoverageHandler::Init() { + MOZ_ASSERT(!instance); + MOZ_ASSERT(XRE_IsParentProcess()); + instance = new CodeCoverageHandler(); + ClearOnShutdown(&instance); + + // Don't really flush but just make FlushCounters usable. + FlushCounters(true); +} + +void CodeCoverageHandler::Init(CrossProcessMutexHandle aHandle) { + MOZ_ASSERT(!instance); + MOZ_ASSERT(!XRE_IsParentProcess()); + instance = new CodeCoverageHandler(std::move(aHandle)); + ClearOnShutdown(&instance); + + // Don't really flush but just make FlushCounters usable. + FlushCounters(true); +} + +CodeCoverageHandler* CodeCoverageHandler::Get() { + MOZ_ASSERT(instance); + return instance; +} + +CrossProcessMutex* CodeCoverageHandler::GetMutex() { return &mGcovLock; } + +CrossProcessMutexHandle CodeCoverageHandler::GetMutexHandle() { + return mGcovLock.CloneHandle(); +} diff --git a/tools/code-coverage/CodeCoverageHandler.h b/tools/code-coverage/CodeCoverageHandler.h new file mode 100644 index 0000000000..7c1b33ec8a --- /dev/null +++ b/tools/code-coverage/CodeCoverageHandler.h @@ -0,0 +1,38 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_codecoveragehandler_h +#define mozilla_codecoveragehandler_h + +#include "mozilla/StaticPtr.h" +#include "mozilla/ipc/CrossProcessMutex.h" + +namespace mozilla { + +class CodeCoverageHandler { + public: + static void Init(); + static void Init(CrossProcessMutexHandle aHandle); + static CodeCoverageHandler* Get(); + CrossProcessMutex* GetMutex(); + CrossProcessMutexHandle GetMutexHandle(); + static void FlushCounters(const bool initialized = false); + static void FlushCountersSignalHandler(int); + + private: + CodeCoverageHandler(); + explicit CodeCoverageHandler(CrossProcessMutexHandle aHandle); + + static StaticAutoPtr instance; + CrossProcessMutex mGcovLock; + + DISALLOW_COPY_AND_ASSIGN(CodeCoverageHandler); + + void SetSignalHandlers(); +}; + +} // namespace mozilla + +#endif // mozilla_codecoveragehandler_h diff --git a/tools/code-coverage/PerTestCoverageUtils.sys.mjs b/tools/code-coverage/PerTestCoverageUtils.sys.mjs new file mode 100644 index 0000000000..1ae0887a07 --- /dev/null +++ b/tools/code-coverage/PerTestCoverageUtils.sys.mjs @@ -0,0 +1,90 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* exported PerTestCoverageUtils */ + +// This is the directory where gcov is emitting the gcda files. +const gcovPrefixPath = Services.env.get("GCOV_PREFIX"); +// This is the directory where codecoverage.py is expecting to see the gcda files. +const gcovResultsPath = Services.env.get("GCOV_RESULTS_DIR"); +// This is the directory where the JS engine is emitting the lcov files. +const jsvmPrefixPath = Services.env.get("JS_CODE_COVERAGE_OUTPUT_DIR"); +// This is the directory where codecoverage.py is expecting to see the lcov files. +const jsvmResultsPath = Services.env.get("JSVM_RESULTS_DIR"); + +function awaitPromise(promise) { + let ret; + let complete = false; + let error = null; + promise + .catch(e => (error = e)) + .then(v => { + ret = v; + complete = true; + }); + Services.tm.spinEventLoopUntil( + "PerTestCoverageUtils.sys.mjs:awaitPromise", + () => complete + ); + if (error) { + throw new Error(error); + } + return ret; +} + +export var PerTestCoverageUtils = class PerTestCoverageUtilsClass { + // Resets the counters to 0. + static async beforeTest() { + if (!PerTestCoverageUtils.enabled) { + return; + } + + // Flush the counters. + let codeCoverageService = Cc[ + "@mozilla.org/tools/code-coverage;1" + ].getService(Ci.nsICodeCoverage); + await codeCoverageService.flushCounters(); + + // Remove coverage files created by the flush, and those that might have been created between the end of a previous test and the beginning of the next one (e.g. some tests can create a new content process for every sub-test). + await IOUtils.remove(gcovPrefixPath, { + recursive: true, + ignoreAbsent: true, + }); + await IOUtils.remove(jsvmPrefixPath, { + recursive: true, + ignoreAbsent: true, + }); + + // Move coverage files from the GCOV_RESULTS_DIR and JSVM_RESULTS_DIR directories, so we can accumulate the counters. + await IOUtils.move(gcovResultsPath, gcovPrefixPath); + await IOUtils.move(jsvmResultsPath, jsvmPrefixPath); + } + + static beforeTestSync() { + awaitPromise(this.beforeTest()); + } + + // Dumps counters and moves the gcda files in the directory expected by codecoverage.py. + static async afterTest() { + if (!PerTestCoverageUtils.enabled) { + return; + } + + // Flush the counters. + let codeCoverageService = Cc[ + "@mozilla.org/tools/code-coverage;1" + ].getService(Ci.nsICodeCoverage); + await codeCoverageService.flushCounters(); + + // Move the coverage files in GCOV_RESULTS_DIR and JSVM_RESULTS_DIR, so that the execution from now to shutdown (or next test) is not counted. + await IOUtils.move(gcovPrefixPath, gcovResultsPath); + await IOUtils.move(jsvmPrefixPath, jsvmResultsPath); + } + + static afterTestSync() { + awaitPromise(this.afterTest()); + } +}; + +PerTestCoverageUtils.enabled = !!gcovResultsPath; diff --git a/tools/code-coverage/components.conf b/tools/code-coverage/components.conf new file mode 100644 index 0000000000..a30600525c --- /dev/null +++ b/tools/code-coverage/components.conf @@ -0,0 +1,14 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + 'cid': '{93576af0-a62f-4c88-bc12-f1855d4e0173}', + 'contract_ids': ['@mozilla.org/tools/code-coverage;1'], + 'type': 'nsCodeCoverage', + 'headers': ['/tools/code-coverage/nsCodeCoverage.h'] + }, +] diff --git a/tools/code-coverage/docs/index.rst b/tools/code-coverage/docs/index.rst new file mode 100644 index 0000000000..36333b4829 --- /dev/null +++ b/tools/code-coverage/docs/index.rst @@ -0,0 +1,209 @@ +Code coverage +============= + +What is Code Coverage? +---------------------- + +**Code coverage** essentially measures how often certain lines are hit, +branches taken or conditions met in a program, given some test that you +run on it. + +There are two very important things to keep in mind when talking about +code coverage: + +- If a certain branch of code is not hit at all while running tests, + then those tests will never be able to find a bug in this particular + piece of the code. +- If a certain branch of code is executed (even very often), this still + is not a clear indication of the *quality of a test*. It could be + that a test exercises the code but does not actually check that the + code performs *correctly*. + +As a conclusion, we can use code coverage to find areas that need (more) +tests, but we cannot use it to confirm that certain areas are well +tested. + + +Firefox Code Coverage reports +----------------------------- + +We automatically run code coverage builds and tests on all +mozilla-central runs, for Linux and Windows. C/C++, Rust and JavaScript +are supported. + +The generated reports can be found at https://coverage.moz.tools/. The +reports can be filtered by platform and/or test suite. + +We also generate a report of all totally uncovered files, which can be +found at https://coverage.moz.tools/#view=zero. You can use this to find +areas of code that should be tested, or code that is no longer used +(dead code, which could be removed). + + +C/C++ Code Coverage on Firefox +------------------------------ + +There are several ways to get C/C++ coverage information for +mozilla-central, including creating your own coverage builds. The next +sections describe the available options. + + +Generate Code Coverage report from a try build (or any other CI build) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To spin a code coverage build, you need to select the linux64-ccov +platform (use --full when using the fuzzy selector to get the ccov +builds to show up). + +E.g. for a try build: + +.. code:: shell + + ./mach try fuzzy -q 'linux64-ccov' + +There are two options now, you can either generate the report locally or +use a one-click loaner. + + +Generate report using a one-click loaner +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Select the B job on Treeherder and get a one-click loaner. + +In the loaner, download and execute the script +https://github.com/mozilla/code-coverage/blob/master/report/firefox_code_coverage/codecoverage.py: + +.. code:: shell + + wget https://raw.githubusercontent.com/mozilla/code-coverage/master/report/firefox_code_coverage/codecoverage.py + python codecoverage.py + +This command will automatically generate a HTML report of the code +coverage information in the **report** subdirectory in your current +working directory. + + +Generate report locally +^^^^^^^^^^^^^^^^^^^^^^^ + +Prerequisites: + +- Create and activate a new `virtualenv`_, then run: + +.. code:: shell + + pip install firefox-code-coverage + +Given a treeherder linux64-ccov build (with its branch, e.g. +\`mozilla-central\` or \`try`, and revision, the tip commit hash of your +push), run the following command: + +.. code:: shell + + firefox-code-coverage PATH/TO/MOZILLA/SRC/DIR/ BRANCH REVISION + +This command will automatically download code coverage artifacts from +the treeherder build and generate an HTML report of the code coverage +information. The report will be stored in the **report** subdirectory in +your current working directory. + +.. _virtualenv: https://docs.python.org/3/tutorial/venv.html + +Creating your own Coverage Build +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Linux, Windows and Mac OS X it is straightforward to generate an +instrumented build using GCC or Clang. Adding the following lines to +your ``.mozconfig`` file should be sufficient: + +.. code:: shell + + # Enable code coverage + ac_add_options --enable-coverage + + # Needed for e10s: + # With the sandbox, content processes can't write updated coverage counters in the gcda files. + ac_add_options --disable-sandbox + +Some additional options might be needed, check the code-coverage +mozconfigs used on CI to be sure: +browser/config/mozconfigs/linux64/code-coverage, +browser/config/mozconfigs/win64/code-coverage, +browser/config/mozconfigs/macosx64/code-coverage. + +Make sure you are not running with :ref:`artifact build ` +enabled, as it can prevent coverage artifacts from being created. + +You can then create your build as usual. Once the build is complete, you +can run any tests/tools you would like to run and the coverage data gets +automatically written to special files. In order to view/process this +data, we recommend using the +`grcov `__ tool, a tool to manage and +visualize gcov results. You can also use the same process explained +earlier for CI builds. + + +Debugging Failing Tests on the Try Server +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When code coverage is run through a push to try, all the data that is +created is ingested by ActiveData and processed into a different data +format for analysis. Anytime a code coverage run generates \*.gcda and +\*.gcno files, ActiveData starts working. Now, sometimes, a test will +permanently fail when it is running on a build that is instrumented with +GCOV. To debug these issues without overloading ActiveData with garbage +coverage data, open the file +`taskcluster/gecko_taskgraph/transforms/test/__init__.py `__ +and add the following line, + +.. code:: python + + test['mozharness'].setdefault('extra-options', []).append('--disable-ccov-upload') + +right after this line of code: + +.. code:: python + + test['mozharness'].setdefault('extra-options', []).append('--code-coverage') + +Now when you push to try to debug some failing tests, or anything else, +there will not be any code coverage artifacts uploaded from the build +machines or from the test machines. + + +JS Debugger Per Test Code Coverage on Firefox +--------------------------------------------- + +There are two ways to get javascript per test code coverage information +for mozilla-central. The next sections describe these options. + + +Generate Per Test Code Coverage from a try build (or any other treeherder build) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To spin a code coverage build, you need to select the linux64-jsdcov +platform. E.g. for a try build: + +.. code:: shell + + ./mach try fuzzy -q 'linux64-jsdcov' + +This produces JavaScript Object Notation (JSON) files that can be +downloaded from the treeherder testing machines and processed or +analyzed locally. + + +Generate Per Test Code Coverage Locally +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To generate the JSON files containing coverage information locally, simply +add an extra argument called ``--jscov-dir-prefix`` which accepts a +directory as it's input and stores the resulting data in that directory. +For example, to collect code coverage for the entire Mochitest suite: + +.. code:: shell + + ./mach mochitest --jscov-dir-prefix /PATH/TO/COVERAGE/DIR/ + +Currently, only the Mochitest and Xpcshell test suites have this +capability. diff --git a/tools/code-coverage/moz.build b/tools/code-coverage/moz.build new file mode 100644 index 0000000000..e0ad22eba4 --- /dev/null +++ b/tools/code-coverage/moz.build @@ -0,0 +1,39 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +TESTING_JS_MODULES += ["PerTestCoverageUtils.sys.mjs"] + +if CONFIG["MOZ_CODE_COVERAGE"]: + XPIDL_MODULE = "code-coverage" + + XPIDL_SOURCES += [ + "nsICodeCoverage.idl", + ] + + SOURCES += [ + "CodeCoverageHandler.cpp", + "nsCodeCoverage.cpp", + ] + + XPCOM_MANIFESTS += [ + "components.conf", + ] + + EXPORTS.mozilla += [ + "CodeCoverageHandler.h", + ] + + LOCAL_INCLUDES += [ + "/ipc/chromium/src", + "/xpcom/base", + ] + + include("/ipc/chromium/chromium-config.mozbuild") + + XPCSHELL_TESTS_MANIFESTS += ["tests/xpcshell/xpcshell.ini"] + MOCHITEST_MANIFESTS += ["tests/mochitest/mochitest.ini"] + + FINAL_LIBRARY = "xul" diff --git a/tools/code-coverage/nsCodeCoverage.cpp b/tools/code-coverage/nsCodeCoverage.cpp new file mode 100644 index 0000000000..5d7ed1927c --- /dev/null +++ b/tools/code-coverage/nsCodeCoverage.cpp @@ -0,0 +1,96 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCodeCoverage.h" +#include "mozilla/CodeCoverageHandler.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/dom/Promise.h" + +using mozilla::dom::ContentParent; +using mozilla::dom::Promise; + +NS_IMPL_ISUPPORTS(nsCodeCoverage, nsICodeCoverage) + +nsCodeCoverage::nsCodeCoverage() {} + +nsCodeCoverage::~nsCodeCoverage() {} + +enum RequestType { Flush }; + +class ProcessCount final { + NS_INLINE_DECL_REFCOUNTING(ProcessCount); + + public: + explicit ProcessCount(uint32_t c) : mCount(c) {} + operator uint32_t() const { return mCount; } + ProcessCount& operator--() { + mCount--; + return *this; + } + + private: + ~ProcessCount() {} + uint32_t mCount; +}; + +namespace { + +nsresult Request(JSContext* cx, Promise** aPromise, RequestType requestType) { + MOZ_ASSERT(XRE_IsParentProcess()); + MOZ_ASSERT(NS_IsMainThread()); + + nsIGlobalObject* global = xpc::CurrentNativeGlobal(cx); + if (NS_WARN_IF(!global)) { + return NS_ERROR_FAILURE; + } + + mozilla::ErrorResult result; + RefPtr promise = Promise::Create(global, result); + if (NS_WARN_IF(result.Failed())) { + return result.StealNSResult(); + } + + uint32_t processCount = 0; + for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) { + mozilla::Unused << cp; + ++processCount; + } + + if (requestType == RequestType::Flush) { + mozilla::CodeCoverageHandler::FlushCounters(); + } + + if (processCount == 0) { + promise->MaybeResolveWithUndefined(); + } else { + RefPtr processCountHolder(new ProcessCount(processCount)); + + auto resolve = [processCountHolder, promise](bool unused) { + if (--(*processCountHolder) == 0) { + promise->MaybeResolveWithUndefined(); + } + }; + + auto reject = [promise](mozilla::ipc::ResponseRejectReason&& aReason) { + promise->MaybeReject(NS_ERROR_FAILURE); + }; + + for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) { + if (requestType == RequestType::Flush) { + cp->SendFlushCodeCoverageCounters(resolve, reject); + } + } + } + + promise.forget(aPromise); + return NS_OK; +} + +} // anonymous namespace + +NS_IMETHODIMP nsCodeCoverage::FlushCounters(JSContext* cx, Promise** aPromise) { + return Request(cx, aPromise, RequestType::Flush); +} diff --git a/tools/code-coverage/nsCodeCoverage.h b/tools/code-coverage/nsCodeCoverage.h new file mode 100644 index 0000000000..936566ac02 --- /dev/null +++ b/tools/code-coverage/nsCodeCoverage.h @@ -0,0 +1,22 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef tools_codecoverage_nscodecoverage_h +#define tools_codecoverage_nscodecoverage_h + +#include "nsICodeCoverage.h" + +class nsCodeCoverage final : nsICodeCoverage { + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICODECOVERAGE + + nsCodeCoverage(); + + private: + ~nsCodeCoverage(); +}; + +#endif // tools_codecoverage_nscodecoverage_h diff --git a/tools/code-coverage/nsICodeCoverage.idl b/tools/code-coverage/nsICodeCoverage.idl new file mode 100644 index 0000000000..ec4ad40ae1 --- /dev/null +++ b/tools/code-coverage/nsICodeCoverage.idl @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +/** + * The nsICodeCoverage component allows controlling the code coverage counters + * collected by Firefox during execution. + * By flushing the counters, one can analyze the coverage information + * for a subset of the program execution (e.g. startup code coverage). + * + */ + +[scriptable, uuid(57d92056-37b4-4d0a-a52f-deb8f6dac8bc)] +interface nsICodeCoverage : nsISupports +{ + /** + * Write the coverage counters to disk, and reset them in memory to 0. + */ + [implicit_jscontext] + Promise flushCounters(); +}; diff --git a/tools/code-coverage/tests/mochitest/mochitest.ini b/tools/code-coverage/tests/mochitest/mochitest.ini new file mode 100644 index 0000000000..f2c8782e76 --- /dev/null +++ b/tools/code-coverage/tests/mochitest/mochitest.ini @@ -0,0 +1,3 @@ +[DEFAULT] + +[test_coverage_specialpowers.html] diff --git a/tools/code-coverage/tests/mochitest/test_coverage_specialpowers.html b/tools/code-coverage/tests/mochitest/test_coverage_specialpowers.html new file mode 100644 index 0000000000..301206ac48 --- /dev/null +++ b/tools/code-coverage/tests/mochitest/test_coverage_specialpowers.html @@ -0,0 +1,38 @@ + + + + + + Test for Bug 123456 + + + + + +Mozilla Bug 1380659 +

+ +
+
+ + diff --git a/tools/code-coverage/tests/xpcshell/head.js b/tools/code-coverage/tests/xpcshell/head.js new file mode 100644 index 0000000000..3642c5794c --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/head.js @@ -0,0 +1,100 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); + +function getFiles() { + // This is the directory where gcov is emitting the gcda files. + const jsCoveragePath = Services.env.get("JS_CODE_COVERAGE_OUTPUT_DIR"); + + const jsCoverageDir = Cc["@mozilla.org/file/local;1"].createInstance( + Ci.nsIFile + ); + jsCoverageDir.initWithPath(jsCoveragePath); + + let files = []; + + let entries = jsCoverageDir.directoryEntries; + while (entries.hasMoreElements()) { + files.push(entries.nextFile); + } + + return files; +} + +function diffFiles(files_after, files_before) { + let files_before_set = new Set(files_before.map(file => file.leafName)); + return files_after.filter(file => !files_before_set.has(file.leafName)); +} + +const BASENAME_RE = new RegExp("([^/\\\\]+)$"); + +function parseRecords(files) { + let records = new Map(); + + for (let file of files) { + const lines = Cu.readUTF8File(file).split("\n"); + let currentSF = null; + + for (let line of lines) { + let [recordType, ...recordContent] = line.split(":"); + recordContent = recordContent.join(":"); + + switch (recordType) { + case "FNDA": { + if (currentSF == null) { + throw new Error("SF missing"); + } + + let [hits, name] = recordContent.split(","); + currentSF.push({ + type: "FNDA", + hits, + name, + }); + break; + } + + case "FN": { + if (currentSF == null) { + throw new Error("SF missing"); + } + + let name = recordContent.split(",")[1]; + currentSF.push({ + type: "FN", + name, + }); + break; + } + + case "SF": { + if ( + recordContent.startsWith("resource:") || + recordContent.startsWith("chrome:") + ) { + recordContent = recordContent.split("/").at(-1); + } else { + if (AppConstants.platform == "win") { + recordContent = recordContent.replace(/\//g, "\\"); + } + const match = BASENAME_RE.exec(recordContent); + if (match.length) { + recordContent = match[0]; + } + } + + currentSF = []; + + records.set(recordContent, currentSF); + break; + } + } + } + } + + return records; +} diff --git a/tools/code-coverage/tests/xpcshell/support.js b/tools/code-coverage/tests/xpcshell/support.js new file mode 100644 index 0000000000..9189427111 --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/support.js @@ -0,0 +1,5 @@ +function test_code_coverage_func2() { + return 22; +} + +test_code_coverage_func2(); diff --git a/tools/code-coverage/tests/xpcshell/test_basic.js b/tools/code-coverage/tests/xpcshell/test_basic.js new file mode 100644 index 0000000000..9523a37ca2 --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/test_basic.js @@ -0,0 +1,115 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function test_code_coverage_func1() { + return 22; +} + +function test_code_coverage_func2() { + return 22; +} + +async function run_test() { + do_test_pending(); + + Assert.ok("@mozilla.org/tools/code-coverage;1" in Cc); + + const codeCoverageCc = Cc["@mozilla.org/tools/code-coverage;1"]; + Assert.ok(!!codeCoverageCc); + + const codeCoverage = codeCoverageCc.getService(Ci.nsICodeCoverage); + Assert.ok(!!codeCoverage); + + const files_orig = getFiles(); + + test_code_coverage_func1(); + + // Flush counters for the first time, we should see this function executed, but test_code_coverage_func not executed. + await codeCoverage.flushCounters(); + + const first_flush_files = getFiles(); + const first_flush_records = parseRecords( + diffFiles(first_flush_files, files_orig) + ); + + Assert.ok(first_flush_records.has("test_basic.js")); + let fnRecords = first_flush_records + .get("test_basic.js") + .filter(record => record.type == "FN"); + let fndaRecords = first_flush_records + .get("test_basic.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok(fnRecords.some(record => record.name == "run_test")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func1") + ); + Assert.ok( + fndaRecords.some(record => record.name == "run_test" && record.hits == 1) + ); + Assert.ok( + !fndaRecords.some(record => record.name == "run_test" && record.hits != 1) + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits == 1 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits != 1 + ) + ); + Assert.ok( + !fndaRecords.some(record => record.name == "test_code_coverage_func2") + ); + + test_code_coverage_func2(); + + // Flush counters for the second time, we should see this function not executed, but test_code_coverage_func executed. + await codeCoverage.flushCounters(); + + const second_flush_files = getFiles(); + const second_flush_records = parseRecords( + diffFiles(second_flush_files, first_flush_files) + ); + + Assert.ok(second_flush_records.has("test_basic.js")); + fnRecords = second_flush_records + .get("test_basic.js") + .filter(record => record.type == "FN"); + fndaRecords = second_flush_records + .get("test_basic.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok(fnRecords.some(record => record.name == "run_test")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func1") + ); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func2") + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits == 0 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits != 0 + ) + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func2" && record.hits == 1 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func2" && record.hits != 1 + ) + ); + + do_test_finished(); +} diff --git a/tools/code-coverage/tests/xpcshell/test_basic_child_and_parent.js b/tools/code-coverage/tests/xpcshell/test_basic_child_and_parent.js new file mode 100644 index 0000000000..f074c20776 --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/test_basic_child_and_parent.js @@ -0,0 +1,120 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function test_code_coverage_func1() { + return 22; +} + +async function run_test() { + do_load_child_test_harness(); + do_test_pending(); + + const codeCoverage = Cc["@mozilla.org/tools/code-coverage;1"].getService( + Ci.nsICodeCoverage + ); + + const files_orig = getFiles(); + + test_code_coverage_func1(); + + await codeCoverage.flushCounters(); + + const first_flush_files = getFiles(); + const first_flush_records = parseRecords( + diffFiles(first_flush_files, files_orig) + ); + + Assert.ok(first_flush_records.has("test_basic_child_and_parent.js")); + Assert.ok(!first_flush_records.has("support.js")); + let fnRecords = first_flush_records + .get("test_basic_child_and_parent.js") + .filter(record => record.type == "FN"); + let fndaRecords = first_flush_records + .get("test_basic_child_and_parent.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok(fnRecords.some(record => record.name == "run_test")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func1") + ); + Assert.ok( + fndaRecords.some(record => record.name == "run_test" && record.hits == 1) + ); + Assert.ok( + !fndaRecords.some(record => record.name == "run_test" && record.hits != 1) + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits == 1 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits != 1 + ) + ); + + sendCommand("load('support.js');", async function () { + await codeCoverage.flushCounters(); + + const second_flush_files = getFiles(); + const second_flush_records = parseRecords( + diffFiles(second_flush_files, first_flush_files) + ); + + Assert.ok(second_flush_records.has("test_basic_child_and_parent.js")); + fnRecords = second_flush_records + .get("test_basic_child_and_parent.js") + .filter(record => record.type == "FN"); + fndaRecords = second_flush_records + .get("test_basic_child_and_parent.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok(fnRecords.some(record => record.name == "run_test")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func1") + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits == 0 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func1" && record.hits != 0 + ) + ); + Assert.ok(second_flush_records.has("support.js")); + fnRecords = second_flush_records + .get("support.js") + .filter(record => record.type == "FN"); + fndaRecords = second_flush_records + .get("support.js") + .filter(record => record.type == "FNDA"); + Assert.ok(fnRecords.some(record => record.name == "top-level")); + Assert.ok( + fnRecords.some(record => record.name == "test_code_coverage_func2") + ); + Assert.ok( + fndaRecords.some(record => record.name == "top-level" && record.hits == 1) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "top-level" && record.hits != 1 + ) + ); + Assert.ok( + fndaRecords.some( + record => record.name == "test_code_coverage_func2" && record.hits == 1 + ) + ); + Assert.ok( + !fndaRecords.some( + record => record.name == "test_code_coverage_func2" && record.hits != 1 + ) + ); + + do_test_finished(); + }); +} diff --git a/tools/code-coverage/tests/xpcshell/xpcshell.ini b/tools/code-coverage/tests/xpcshell/xpcshell.ini new file mode 100644 index 0000000000..93f756be8b --- /dev/null +++ b/tools/code-coverage/tests/xpcshell/xpcshell.ini @@ -0,0 +1,7 @@ +[DEFAULT] +head = head.js +support-files = + support.js + +[test_basic.js] +[test_basic_child_and_parent.js] diff --git a/tools/compare-locales/mach_commands.py b/tools/compare-locales/mach_commands.py new file mode 100644 index 0000000000..56d101467b --- /dev/null +++ b/tools/compare-locales/mach_commands.py @@ -0,0 +1,409 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import logging +import os +import tempfile +from pathlib import Path + +from appdirs import user_config_dir +from hglib.error import CommandError +from mach.base import FailedCommandError +from mach.decorators import Command, CommandArgument +from mozrelease.scriptworker_canary import get_secret +from redo import retry + + +@Command( + "compare-locales", + category="build", + description="Run source checks on a localization.", +) +@CommandArgument( + "config_paths", + metavar="l10n.toml", + nargs="+", + help="TOML or INI file for the project", +) +@CommandArgument( + "l10n_base_dir", + metavar="l10n-base-dir", + help="Parent directory of localizations", +) +@CommandArgument( + "locales", + nargs="*", + metavar="locale-code", + help="Locale code and top-level directory of each localization", +) +@CommandArgument( + "-q", + "--quiet", + action="count", + default=0, + help="""Show less data. +Specified once, don't show obsolete entities. Specified twice, also hide +missing entities. Specify thrice to exclude warnings and four times to +just show stats""", +) +@CommandArgument("-m", "--merge", help="""Use this directory to stage merged files""") +@CommandArgument( + "--validate", action="store_true", help="Run compare-locales against reference" +) +@CommandArgument( + "--json", + help="""Serialize to JSON. Value is the name of +the output file, pass "-" to serialize to stdout and hide the default output. +""", +) +@CommandArgument( + "-D", + action="append", + metavar="var=value", + default=[], + dest="defines", + help="Overwrite variables in TOML files", +) +@CommandArgument( + "--full", action="store_true", help="Compare projects that are disabled" +) +@CommandArgument( + "--return-zero", action="store_true", help="Return 0 regardless of l10n status" +) +def compare(command_context, **kwargs): + """Run compare-locales.""" + from compare_locales.commands import CompareLocales + + class ErrorHelper(object): + """Dummy ArgumentParser to marshall compare-locales + commandline errors to mach exceptions. + """ + + def error(self, msg): + raise FailedCommandError(msg) + + def exit(self, message=None, status=0): + raise FailedCommandError(message, exit_code=status) + + cmd = CompareLocales() + cmd.parser = ErrorHelper() + return cmd.handle(**kwargs) + + +# https://stackoverflow.com/a/14117511 +def _positive_int(value): + value = int(value) + if value <= 0: + raise argparse.ArgumentTypeError(f"{value} must be a positive integer.") + return value + + +class RetryError(Exception): + ... + + +VCT_PATH = Path(".").resolve() / "vct" +VCT_URL = "https://hg.mozilla.org/hgcustom/version-control-tools/" +FXTREE_PATH = VCT_PATH / "hgext" / "firefoxtree" +HGRC_PATH = Path(user_config_dir("hg")).joinpath("hgrc") + + +@Command( + "l10n-cross-channel", + category="misc", + description="Create cross-channel content.", +) +@CommandArgument( + "--strings-path", + "-s", + metavar="en-US", + type=Path, + default=Path("en-US"), + help="Path to mercurial repository for gecko-strings-quarantine", +) +@CommandArgument( + "--outgoing-path", + "-o", + type=Path, + help="create an outgoing() patch if there are changes", +) +@CommandArgument( + "--attempts", + type=_positive_int, + default=1, + help="Number of times to try (for automation)", +) +@CommandArgument( + "--ssh-secret", + action="store", + help="Taskcluster secret to use to push (for automation)", +) +@CommandArgument( + "actions", + choices=("prep", "create", "push", "clean"), + nargs="+", + # This help block will be poorly formatted until we fix bug 1714239 + help=""" + "prep": clone repos and pull heads. + "create": create the en-US strings commit an optionally create an + outgoing() patch. + "push": push the en-US strings to the quarantine repo. + "clean": clean up any sub-repos. + """, +) +def cross_channel( + command_context, + strings_path, + outgoing_path, + actions, + attempts, + ssh_secret, + **kwargs, +): + """Run l10n cross-channel content generation.""" + # This can be any path, as long as the name of the directory is en-US. + # Not entirely sure where this is a requirement; perhaps in l10n + # string manipulation logic? + if strings_path.name != "en-US": + raise FailedCommandError("strings_path needs to be named `en-US`") + command_context.activate_virtualenv() + # XXX pin python requirements + command_context.virtualenv_manager.install_pip_requirements( + Path(os.path.dirname(__file__)) / "requirements.in" + ) + strings_path = strings_path.resolve() # abspath + if outgoing_path: + outgoing_path = outgoing_path.resolve() # abspath + get_config = kwargs.get("get_config", None) + try: + with tempfile.TemporaryDirectory() as ssh_key_dir: + retry( + _do_create_content, + attempts=attempts, + retry_exceptions=(RetryError,), + args=( + command_context, + strings_path, + outgoing_path, + ssh_secret, + Path(ssh_key_dir), + actions, + get_config, + ), + ) + except RetryError as exc: + raise FailedCommandError(exc) from exc + + +def _do_create_content( + command_context, + strings_path, + outgoing_path, + ssh_secret, + ssh_key_dir, + actions, + get_config, +): + from mozxchannel import CrossChannelCreator, get_default_config + + get_config = get_config or get_default_config + + config = get_config(Path(command_context.topsrcdir), strings_path) + ccc = CrossChannelCreator(config) + status = 0 + changes = False + ssh_key_secret = None + ssh_key_file = None + + if "prep" in actions: + if ssh_secret: + if not os.environ.get("MOZ_AUTOMATION"): + raise CommandError( + "I don't know how to fetch the ssh secret outside of automation!" + ) + ssh_key_secret = get_secret(ssh_secret) + ssh_key_file = ssh_key_dir.joinpath("id_rsa") + ssh_key_file.write_text(ssh_key_secret["ssh_privkey"]) + ssh_key_file.chmod(0o600) + # Set up firefoxtree for comm per bug 1659691 comment 22 + if os.environ.get("MOZ_AUTOMATION") and not HGRC_PATH.exists(): + _clone_hg_repo(command_context, VCT_URL, VCT_PATH) + hgrc_content = [ + "[extensions]", + f"firefoxtree = {FXTREE_PATH}", + "", + "[ui]", + "username = trybld", + ] + if ssh_key_file: + hgrc_content.extend( + [ + f"ssh = ssh -i {ssh_key_file} -l {ssh_key_secret['user']}", + ] + ) + HGRC_PATH.write_text("\n".join(hgrc_content)) + if strings_path.exists() and _check_outgoing(command_context, strings_path): + _strip_outgoing(command_context, strings_path) + # Clone strings + source repos, pull heads + for repo_config in (config["strings"], *config["source"].values()): + if not repo_config["path"].exists(): + _clone_hg_repo( + command_context, repo_config["url"], str(repo_config["path"]) + ) + for head in repo_config["heads"].keys(): + command = ["hg", "--cwd", str(repo_config["path"]), "pull"] + command.append(head) + status = _retry_run_process( + command_context, command, ensure_exit_code=False + ) + if status not in (0, 255): # 255 on pull with no changes + raise RetryError(f"Failure on pull: status {status}!") + if repo_config.get("update_on_pull"): + command = [ + "hg", + "--cwd", + str(repo_config["path"]), + "up", + "-C", + "-r", + head, + ] + status = _retry_run_process( + command_context, command, ensure_exit_code=False + ) + if status not in (0, 255): # 255 on pull with no changes + raise RetryError(f"Failure on update: status {status}!") + _check_hg_repo( + command_context, + repo_config["path"], + heads=repo_config.get("heads", {}).keys(), + ) + else: + _check_hg_repo(command_context, strings_path) + for repo_config in config.get("source", {}).values(): + _check_hg_repo( + command_context, + repo_config["path"], + heads=repo_config.get("heads", {}).keys(), + ) + if _check_outgoing(command_context, strings_path): + raise RetryError(f"check: Outgoing changes in {strings_path}!") + + if "create" in actions: + try: + status = ccc.create_content() + changes = True + _create_outgoing_patch(command_context, outgoing_path, strings_path) + except CommandError as exc: + if exc.ret != 1: + raise RetryError(exc) from exc + command_context.log(logging.INFO, "create", {}, "No new strings.") + + if "push" in actions: + if changes: + _retry_run_process( + command_context, + [ + "hg", + "--cwd", + str(strings_path), + "push", + "-r", + ".", + config["strings"]["push_url"], + ], + line_handler=print, + ) + else: + command_context.log(logging.INFO, "push", {}, "Skipping empty push.") + + if "clean" in actions: + for repo_config in config.get("source", {}).values(): + if repo_config.get("post-clobber", False): + _nuke_hg_repo(command_context, str(repo_config["path"])) + + return status + + +def _check_outgoing(command_context, strings_path): + status = _retry_run_process( + command_context, + ["hg", "--cwd", str(strings_path), "out", "-r", "."], + ensure_exit_code=False, + ) + if status == 0: + return True + if status == 1: + return False + raise RetryError(f"Outgoing check in {strings_path} returned unexpected {status}!") + + +def _strip_outgoing(command_context, strings_path): + _retry_run_process( + command_context, + [ + "hg", + "--config", + "extensions.strip=", + "--cwd", + str(strings_path), + "strip", + "--no-backup", + "outgoing()", + ], + ) + + +def _create_outgoing_patch(command_context, path, strings_path): + if not path: + return + if not path.parent.exists(): + os.makedirs(path.parent) + with open(path, "w") as fh: + + def writeln(line): + fh.write(f"{line}\n") + + _retry_run_process( + command_context, + [ + "hg", + "--cwd", + str(strings_path), + "log", + "--patch", + "--verbose", + "-r", + "outgoing()", + ], + line_handler=writeln, + ) + + +def _retry_run_process(command_context, *args, error_msg=None, **kwargs): + try: + return command_context.run_process(*args, **kwargs) + except Exception as exc: + raise RetryError(error_msg or str(exc)) from exc + + +def _check_hg_repo(command_context, path, heads=None): + if not (path.is_dir() and (path / ".hg").is_dir()): + raise RetryError(f"{path} is not a Mercurial repository") + if heads: + for head in heads: + _retry_run_process( + command_context, + ["hg", "--cwd", str(path), "log", "-r", head], + error_msg=f"check: {path} has no head {head}!", + ) + + +def _clone_hg_repo(command_context, url, path): + _retry_run_process(command_context, ["hg", "clone", url, str(path)]) + + +def _nuke_hg_repo(command_context, path): + _retry_run_process(command_context, ["rm", "-rf", str(path)]) diff --git a/tools/compare-locales/requirements.in b/tools/compare-locales/requirements.in new file mode 100644 index 0000000000..d6fbb9d25b --- /dev/null +++ b/tools/compare-locales/requirements.in @@ -0,0 +1,2 @@ +redo +dataclasses; python_version < '3.7' diff --git a/tools/crashreporter/injector/app.mozbuild b/tools/crashreporter/injector/app.mozbuild new file mode 100644 index 0000000000..3e7a6f9cf1 --- /dev/null +++ b/tools/crashreporter/injector/app.mozbuild @@ -0,0 +1,14 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + '/toolkit/crashreporter/google-breakpad/src/common', + '/toolkit/crashreporter/google-breakpad/src/processor', + '/tools/crashreporter/injector', +] + +if CONFIG['OS_TARGET'] in ('Linux', 'Android'): + DIRS += [ + '/toolkit/crashreporter/google-breakpad/src/common/linux', + ] diff --git a/tools/crashreporter/injector/injector.cc b/tools/crashreporter/injector/injector.cc new file mode 100644 index 0000000000..eeb74038c8 --- /dev/null +++ b/tools/crashreporter/injector/injector.cc @@ -0,0 +1,40 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "linux/handler/exception_handler.h" + +using google_breakpad::ExceptionHandler; + +static ExceptionHandler* gExceptionHandler = nullptr; + +// Flag saying whether to generate a minidump. Can be (probably temporarily) set +// to false when a crash is expected. +bool __attribute__((visibility("default"))) gBreakpadInjectorEnabled = true; + +bool TestEnabled(void* /* context */) { return gBreakpadInjectorEnabled; } + +bool SetGlobalExceptionHandler( + ExceptionHandler::FilterCallback filterCallback, + ExceptionHandler::MinidumpCallback minidumpCallback) { + const char* tempPath = getenv("TMPDIR"); + if (!tempPath) tempPath = "/tmp"; + + google_breakpad::MinidumpDescriptor descriptor(tempPath); + + gExceptionHandler = new ExceptionHandler(descriptor, filterCallback, + minidumpCallback, nullptr, true, -1); + if (!gExceptionHandler) return false; + + return true; +} + +// Called when loading the DLL (eg via LD_PRELOAD, or the JS shell --dll +// option). +void __attribute__((constructor)) SetBreakpadExceptionHandler() { + if (gExceptionHandler) abort(); + + if (!SetGlobalExceptionHandler(TestEnabled, nullptr)) abort(); + + if (!gExceptionHandler) abort(); +} diff --git a/tools/crashreporter/injector/moz.build b/tools/crashreporter/injector/moz.build new file mode 100644 index 0000000000..d16300fbfb --- /dev/null +++ b/tools/crashreporter/injector/moz.build @@ -0,0 +1,36 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SharedLibrary("breakpadinjector") + +UNIFIED_SOURCES += [ + "/toolkit/crashreporter/breakpad-client/linux/crash_generation/crash_generation_client.cc", + "/toolkit/crashreporter/breakpad-client/linux/dump_writer_common/thread_info.cc", + "/toolkit/crashreporter/breakpad-client/linux/dump_writer_common/ucontext_reader.cc", + "/toolkit/crashreporter/breakpad-client/linux/handler/exception_handler.cc", + "/toolkit/crashreporter/breakpad-client/linux/handler/guid_generator.cc", + "/toolkit/crashreporter/breakpad-client/linux/handler/minidump_descriptor.cc", + "/toolkit/crashreporter/breakpad-client/linux/log/log.cc", + "/toolkit/crashreporter/breakpad-client/linux/microdump_writer/microdump_writer.cc", + "/toolkit/crashreporter/breakpad-client/linux/minidump_writer/linux_dumper.cc", + "/toolkit/crashreporter/breakpad-client/linux/minidump_writer/linux_ptrace_dumper.cc", + "/toolkit/crashreporter/breakpad-client/linux/minidump_writer/minidump_writer.cc", + "/toolkit/crashreporter/breakpad-client/minidump_file_writer.cc", + "injector.cc", +] + +USE_LIBS += [ + "breakpad_common_s", + "breakpad_linux_common_s", +] + +DisableStlWrapping() + +# On Linux we override the guid_creator.h header and use our own instead +if CONFIG["OS_TARGET"] in ("Linux", "Android"): + DEFINES["COMMON_LINUX_GUID_CREATOR_H__"] = 1 + +include("/toolkit/crashreporter/crashreporter.mozbuild") diff --git a/tools/crashreporter/injector/moz.configure b/tools/crashreporter/injector/moz.configure new file mode 100644 index 0000000000..6fbe8159b2 --- /dev/null +++ b/tools/crashreporter/injector/moz.configure @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py b/tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py new file mode 100755 index 0000000000..64b55b5007 --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/PackageSymbolDumper.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python + +# Copyright 2015 Michael R. Miller. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +PackageSymbolDumper.py + +Dumps Breakpad symbols for the contents of an Apple update installer. Given a +path to an Apple update installer as a .dmg or a path to a specific package +within the disk image, PackageSymbolDumper mounts, traverses, and dumps symbols +for all applicable frameworks and dylibs found within. + +Required tools for Linux: + pax + gzip + tar + xpwn's dmg (https://github.com/planetbeing/xpwn) + +Created on Apr 11, 2012 + +@author: mrmiller +""" +import argparse +import concurrent.futures +import errno +import logging +import os +import shutil +import stat +import subprocess +import tempfile +import traceback + +from mozpack.macpkg import Pbzx, uncpio, unxar +from scrapesymbols.gathersymbols import process_paths + + +def expand_pkg(pkg_path, out_path): + """ + Expands the contents of an installer package to some directory. + + @param pkg_path: a path to an installer package (.pkg) + @param out_path: a path to hold the package contents + """ + for name, content in unxar(open(pkg_path, "rb")): + with open(os.path.join(out_path, name), "wb") as fh: + shutil.copyfileobj(content, fh) + + +def expand_dmg(dmg_path, out_path): + """ + Expands the contents of a DMG file to some directory. + + @param dmg_path: a path to a disk image file (.dmg) + @param out_path: a path to hold the image contents + """ + + with tempfile.NamedTemporaryFile() as f: + subprocess.check_call( + ["dmg", "extract", dmg_path, f.name], stdout=subprocess.DEVNULL + ) + subprocess.check_call( + ["hfsplus", f.name, "extractall"], stdout=subprocess.DEVNULL, cwd=out_path + ) + + +def expand_zip(zip_path, out_path): + """ + Expands the contents of a ZIP archive to some directory. + + @param dmg_path: a path to a ZIP archive (.zip) + @param out_path: a path to hold the archive contents + """ + subprocess.check_call( + ["unzip", "-d", out_path, zip_path], stdout=subprocess.DEVNULL + ) + + +def filter_files(function, path): + """ + Yield file paths matching a filter function by walking the + hierarchy rooted at path. + + @param function: a function taking in a filename that returns true to + include the path + @param path: the root path of the hierarchy to traverse + """ + for root, _dirs, files in os.walk(path): + for filename in files: + if function(filename): + yield os.path.join(root, filename) + + +def find_packages(path): + """ + Returns a list of installer packages (as determined by the .pkg extension), + disk images (as determined by the .dmg extension) or ZIP archives found + within path. + + @param path: root path to search for .pkg, .dmg and .zip files + """ + return filter_files( + lambda filename: os.path.splitext(filename)[1] in (".pkg", ".dmg", ".zip") + and not filename.startswith("._"), + path, + ) + + +def find_all_packages(paths): + """ + Yield installer package files, disk images and ZIP archives found in all + of `paths`. + + @param path: list of root paths to search for .pkg & .dmg files + """ + for path in paths: + logging.info("find_all_packages: {}".format(path)) + for pkg in find_packages(path): + yield pkg + + +def find_payloads(path): + """ + Returns a list of possible installer package payload paths. + + @param path: root path for an installer package + """ + return filter_files( + lambda filename: "Payload" in filename or ".pax.gz" in filename, path + ) + + +def extract_payload(payload_path, output_path): + """ + Extracts the contents of an installer package payload to a given directory. + + @param payload_path: path to an installer package's payload + @param output_path: output path for the payload's contents + @return True for success, False for failure. + """ + header = open(payload_path, "rb").read(2) + try: + if header == b"BZ": + logging.info("Extracting bzip2 payload") + extract = "bzip2" + subprocess.check_call( + 'cd {dest} && {extract} -dc {payload} | pax -r -k -s ":^/::"'.format( + extract=extract, payload=payload_path, dest=output_path + ), + shell=True, + ) + return True + elif header == b"\x1f\x8b": + logging.info("Extracting gzip payload") + extract = "gzip" + subprocess.check_call( + 'cd {dest} && {extract} -dc {payload} | pax -r -k -s ":^/::"'.format( + extract=extract, payload=payload_path, dest=output_path + ), + shell=True, + ) + return True + elif header == b"pb": + logging.info("Extracting pbzx payload") + + for path, st, content in uncpio(Pbzx(open(payload_path, "rb"))): + if not path or not stat.S_ISREG(st.mode): + continue + out = os.path.join(output_path, path.decode()) + os.makedirs(os.path.dirname(out), exist_ok=True) + with open(out, "wb") as fh: + shutil.copyfileobj(content, fh) + + return True + else: + # Unsupported format + logging.error( + "Unknown payload format: 0x{0:x}{1:x}".format(header[0], header[1]) + ) + return False + + except Exception: + return False + + +def shutil_error_handler(caller, path, excinfo): + logging.error('Could not remove "{path}": {info}'.format(path=path, info=excinfo)) + + +def write_symbol_file(dest, filename, contents): + full_path = os.path.join(dest, filename) + try: + os.makedirs(os.path.dirname(full_path)) + with open(full_path, "wb") as sym_file: + sym_file.write(contents) + except os.error as e: + if e.errno != errno.EEXIST: + raise + + +def dump_symbols(executor, dump_syms, path, dest): + system_library = os.path.join("System", "Library") + subdirectories = [ + os.path.join(system_library, "Frameworks"), + os.path.join(system_library, "PrivateFrameworks"), + os.path.join(system_library, "Extensions"), + os.path.join("usr", "lib"), + ] + + paths_to_dump = [os.path.join(path, d) for d in subdirectories] + existing_paths = [path for path in paths_to_dump if os.path.exists(path)] + + for filename, contents in process_paths( + paths=existing_paths, + executor=executor, + dump_syms=dump_syms, + verbose=True, + write_all=True, + platform="darwin", + ): + if filename and contents: + logging.info("Added symbol file " + str(filename, "utf-8")) + write_symbol_file(dest, str(filename, "utf-8"), contents) + + +def dump_symbols_from_payload(executor, dump_syms, payload_path, dest): + """ + Dumps all the symbols found inside the payload of an installer package. + + @param dump_syms: path to the dump_syms executable + @param payload_path: path to an installer package's payload + @param dest: output path for symbols + """ + temp_dir = None + logging.info("Dumping symbols from payload: " + payload_path) + try: + temp_dir = tempfile.mkdtemp() + logging.info("Extracting payload to {path}.".format(path=temp_dir)) + if not extract_payload(payload_path, temp_dir): + logging.error("Could not extract payload: " + payload_path) + return False + + dump_symbols(executor, dump_syms, temp_dir, dest) + + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir, onerror=shutil_error_handler) + + return True + + +def dump_symbols_from_package(executor, dump_syms, pkg, dest): + """ + Dumps all the symbols found inside an installer package. + + @param dump_syms: path to the dump_syms executable + @param pkg: path to an installer package + @param dest: output path for symbols + """ + successful = True + temp_dir = None + logging.info("Dumping symbols from package: " + pkg) + try: + temp_dir = tempfile.mkdtemp() + if os.path.splitext(pkg)[1] == ".pkg": + expand_pkg(pkg, temp_dir) + elif os.path.splitext(pkg)[1] == ".zip": + expand_zip(pkg, temp_dir) + else: + expand_dmg(pkg, temp_dir) + + # check for any subpackages + for subpackage in find_packages(temp_dir): + logging.info("Found subpackage at: " + subpackage) + res = dump_symbols_from_package(executor, dump_syms, subpackage, dest) + if not res: + logging.error("Error while dumping subpackage: " + subpackage) + + # dump symbols from any payloads (only expecting one) in the package + for payload in find_payloads(temp_dir): + res = dump_symbols_from_payload(executor, dump_syms, payload, dest) + if not res: + successful = False + + # dump symbols directly extracted from the package + dump_symbols(executor, dump_syms, temp_dir, dest) + + except Exception as e: + traceback.print_exc() + logging.error("Exception while dumping symbols from package: {}".format(e)) + successful = False + + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir, onerror=shutil_error_handler) + + return successful + + +def read_processed_packages(tracking_file): + if tracking_file is None or not os.path.exists(tracking_file): + return set() + logging.info("Reading processed packages from {}".format(tracking_file)) + return set(open(tracking_file, "r").read().splitlines()) + + +def write_processed_packages(tracking_file, processed_packages): + if tracking_file is None: + return + logging.info( + "Writing {} processed packages to {}".format( + len(processed_packages), tracking_file + ) + ) + open(tracking_file, "w").write("\n".join(processed_packages)) + + +def process_packages(package_finder, to, tracking_file, dump_syms): + processed_packages = read_processed_packages(tracking_file) + with concurrent.futures.ProcessPoolExecutor() as executor: + for pkg in package_finder(): + if pkg in processed_packages: + logging.info("Skipping already-processed package: {}".format(pkg)) + else: + dump_symbols_from_package(executor, dump_syms, pkg, to) + processed_packages.add(pkg) + write_processed_packages(tracking_file, processed_packages) + + +def main(): + parser = argparse.ArgumentParser( + description="Extracts Breakpad symbols from a Mac OS X support update." + ) + parser.add_argument( + "--dump_syms", + default="dump_syms", + type=str, + help="path to the Breakpad dump_syms executable", + ) + parser.add_argument( + "--tracking-file", + type=str, + help="Path to a file in which to store information " + + "about already-processed packages", + ) + parser.add_argument( + "search", nargs="+", help="Paths to search recursively for packages" + ) + parser.add_argument("to", type=str, help="destination path for the symbols") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + for p in ("requests.packages.urllib3.connectionpool", "urllib3"): + urllib3_logger = logging.getLogger(p) + urllib3_logger.setLevel(logging.ERROR) + + if not args.search or not all(os.path.exists(p) for p in args.search): + logging.error("Invalid search path") + return + if not os.path.exists(args.to): + logging.error("Invalid path to destination") + return + + def finder(): + return find_all_packages(args.search) + + process_packages(finder, args.to, args.tracking_file, args.dump_syms) + + +if __name__ == "__main__": + main() diff --git a/tools/crashreporter/system-symbols/mac/get_update_packages.py b/tools/crashreporter/system-symbols/mac/get_update_packages.py new file mode 100644 index 0000000000..3192fa3ef0 --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/get_update_packages.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python + +# Copyright (c) 2015 Ted Mielczarek +# and Michael R. Miller +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import argparse +import concurrent.futures +import logging +import os +import re +import shutil +import subprocess +import tempfile + +import requests +import urlparse +from PackageSymbolDumper import find_packages, process_packages + +OSX_RE = re.compile(r"10\.[0-9]+\.[0-9]+") + + +def extract_dmg(dmg_path, dest): + logging.info("extract_dmg({}, {})".format(dmg_path, dest)) + with tempfile.NamedTemporaryFile() as f: + subprocess.check_call( + ["dmg", "extract", dmg_path, f.name], stdout=subprocess.DEVNULL + ) + subprocess.check_call(["hfsplus", f.name, "extractall"], cwd=dest) + + +def get_update_packages(): + for i in range(16): + logging.info("get_update_packages: page " + str(i)) + url = ( + "https://km.support.apple.com/kb/index?page=downloads_browse&sort=recency" + "&facet=all&category=PF6&locale=en_US&offset=%d" % i + ) + res = requests.get(url) + if res.status_code != 200: + break + data = res.json() + downloads = data.get("downloads", []) + if not downloads: + break + for d in downloads: + title = d.get("title", "") + if OSX_RE.search(title) and "Combo" not in title: + logging.info("Title: " + title) + if "fileurl" in d: + yield d["fileurl"] + else: + logging.warn("No fileurl in download!") + + +def fetch_url_to_file(url, download_dir): + filename = os.path.basename(urlparse.urlsplit(url).path) + local_filename = os.path.join(download_dir, filename) + if os.path.isfile(local_filename): + logging.info("{} already exists, skipping".format(local_filename)) + return None + r = requests.get(url, stream=True) + res_len = int(r.headers.get("content-length", "0")) + logging.info("Downloading {} -> {} ({} bytes)".format(url, local_filename, res_len)) + with open(local_filename, "wb") as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + return local_filename + + +def fetch_and_extract_dmg(url, tmpdir): + logging.info("fetch_and_extract_dmg: " + url) + filename = fetch_url_to_file(url, tmpdir) + if not filename: + return [] + # Extract dmg contents to a subdir + subdir = tempfile.mkdtemp(dir=tmpdir) + extract_dmg(filename, subdir) + packages = list(find_packages(subdir)) + logging.info( + "fetch_and_extract_dmg({}): found packages: {}".format(url, str(packages)) + ) + return packages + + +def find_update_packages(tmpdir): + logging.info("find_update_packages") + # Only download 2 packages at a time. + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + jobs = dict( + (executor.submit(fetch_and_extract_dmg, url, tmpdir), url) + for url in get_update_packages() + ) + for future in concurrent.futures.as_completed(jobs): + url = jobs[future] + if future.exception() is not None: + logging.error( + "exception downloading {}: {}".format(url, future.exception()) + ) + else: + for pkg in future.result(): + yield pkg + + +def main(): + parser = argparse.ArgumentParser( + description="Download OS X update packages and dump symbols from them" + ) + parser.add_argument( + "--dump_syms", + default="dump_syms", + type=str, + help="path to the Breakpad dump_syms executable", + ) + parser.add_argument("to", type=str, help="destination path for the symbols") + args = parser.parse_args() + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + for p in ("requests.packages.urllib3.connectionpool", "urllib3"): + urllib3_logger = logging.getLogger(p) + urllib3_logger.setLevel(logging.ERROR) + try: + tmpdir = tempfile.mkdtemp(suffix=".osxupdates") + + def finder(): + return find_update_packages(tmpdir) + + process_packages(finder, args.to, None, args.dump_syms) + finally: + shutil.rmtree(tmpdir) + + +if __name__ == "__main__": + main() diff --git a/tools/crashreporter/system-symbols/mac/list-packages.py b/tools/crashreporter/system-symbols/mac/list-packages.py new file mode 100755 index 0000000000..444c27be9d --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/list-packages.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# Copyright 2015 Ted Mielczarek. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import sys + +from reposadolib import reposadocommon + +reposadocommon.get_main_dir = lambda: "/usr/local/bin/" + +products = reposadocommon.get_product_info() +args = [] +for product_id, product in products.items(): + try: + title = product["title"] + except KeyError: + print("Missing title in {}, skipping".format(product), file=sys.stderr) + continue + + try: + major_version = int(product["version"].split(".")[0]) + except Exception: + print( + "Cannot extract the major version number in {}, skipping".format(product), + file=sys.stderr, + ) + continue + + if ( + title.startswith("OS X") + or title.startswith("Mac OS X") + or title.startswith("macOS") + ) and major_version <= 10: + args.append(product_id) + else: + print("Skipping %r for repo_sync" % title, file=sys.stderr) +if "JUST_ONE_PACKAGE" in os.environ: + args = args[:1] + +print("\n".join(args)) diff --git a/tools/crashreporter/system-symbols/mac/run.sh b/tools/crashreporter/system-symbols/mac/run.sh new file mode 100755 index 0000000000..8dec95dffe --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/run.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +set -v -e -x + +base="$(realpath "$(dirname "$0")")" +export PATH="$PATH:/builds/worker/bin:$base:${MOZ_FETCHES_DIR}/dmg" + +cd /builds/worker + +if test "$PROCESSED_PACKAGES_INDEX" && test "$PROCESSED_PACKAGES_PATH" && test "$TASKCLUSTER_ROOT_URL"; then + PROCESSED_PACKAGES="$TASKCLUSTER_ROOT_URL/api/index/v1/task/$PROCESSED_PACKAGES_INDEX/artifacts/$PROCESSED_PACKAGES_PATH" +fi + +if test "$PROCESSED_PACKAGES"; then + rm -f processed-packages + if test `curl --output /dev/null --silent --head --location "$PROCESSED_PACKAGES" -w "%{http_code}"` = 200; then + curl -L "$PROCESSED_PACKAGES" | gzip -dc > processed-packages + elif test -f "$PROCESSED_PACKAGES"; then + gzip -dc "$PROCESSED_PACKAGES" > processed-packages + fi + if test -f processed-packages; then + # Prevent reposado from downloading packages that have previously been + # dumped. + for f in $(cat processed-packages); do + mkdir -p "$(dirname "$f")" + touch "$f" + done + fi +fi + +mkdir -p /opt/data-reposado/html /opt/data-reposado/metadata artifacts + +# First, just fetch all the update info. +python3 /usr/local/bin/repo_sync --no-download + +# Next, fetch just the update packages we're interested in. +packages=$(python3 "${base}/list-packages.py") + +for package in ${packages}; do + # repo_sync is super-chatty, let's pipe stderr to separate files + python3 /usr/local/bin/repo_sync "--product-id=${package}" 2> "artifacts/repo_sync-product-id-${package}.stderr" + # Stop downloading packages if we have more than 10 GiB of them to process + download_size=$(du -B1073741824 -s /opt/data-reposado | cut -f1) + if [ ${download_size} -gt 10 ]; then + break + fi +done + +du -sh /opt/data-reposado + +# Now scrape symbols out of anything that was downloaded. +mkdir -p symbols tmp +env TMP=tmp python3 "${base}/PackageSymbolDumper.py" --tracking-file=/builds/worker/processed-packages --dump_syms=$MOZ_FETCHES_DIR/dump_syms/dump_syms /opt/data-reposado/html/content/downloads /builds/worker/symbols + +# Hand out artifacts +gzip -c processed-packages > artifacts/processed-packages.gz + +cd symbols +zip -r9 /builds/worker/artifacts/target.crashreporter-symbols.zip ./* || echo "No symbols dumped" diff --git a/tools/crashreporter/system-symbols/mac/scrapesymbols/__init__.py b/tools/crashreporter/system-symbols/mac/scrapesymbols/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/crashreporter/system-symbols/mac/scrapesymbols/gathersymbols.py b/tools/crashreporter/system-symbols/mac/scrapesymbols/gathersymbols.py new file mode 100644 index 0000000000..9998dcac27 --- /dev/null +++ b/tools/crashreporter/system-symbols/mac/scrapesymbols/gathersymbols.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import argparse +import concurrent.futures +import datetime +import os +import subprocess +import sys +import traceback +import urllib +import zipfile + +import requests + +if sys.platform == "darwin": + SYSTEM_DIRS = [ + "/usr/lib", + "/System/Library/Frameworks", + "/System/Library/PrivateFrameworks", + "/System/Library/Extensions", + ] +else: + SYSTEM_DIRS = ["/lib", "/usr/lib"] +SYMBOL_SERVER_URL = "https://symbols.mozilla.org/" + + +def should_process(f, platform=sys.platform): + """Determine if a file is a platform binary""" + if platform == "darwin": + """ + The 'file' command can error out. One example is "illegal byte + sequence" on a Japanese language UTF8 text file. So we must wrap the + command in a try/except block to prevent the script from terminating + prematurely when this happens. + """ + try: + filetype = subprocess.check_output(["file", "-Lb", f], text=True) + except subprocess.CalledProcessError: + return False + """Skip kernel extensions""" + if "kext bundle" in filetype: + return False + return filetype.startswith("Mach-O") + else: + return subprocess.check_output(["file", "-Lb", f], text=True).startswith("ELF") + return False + + +def get_archs(filename, platform=sys.platform): + """ + Find the list of architectures present in a Mach-O file, or a single-element + list on non-OS X. + """ + architectures = [] + output = subprocess.check_output(["file", "-Lb", filename], text=True) + for string in output.split(" "): + if string == "arm64e": + architectures.append("arm64e") + elif string == "x86_64_haswell": + architectures.append("x86_64h") + elif string == "x86_64": + architectures.append("x86_64") + elif string == "i386": + architectures.append("i386") + + return architectures + + +def server_has_file(filename): + """ + Send the symbol server a HEAD request to see if it has this symbol file. + """ + try: + r = requests.head( + urllib.parse.urljoin(SYMBOL_SERVER_URL, urllib.parse.quote(filename)) + ) + return r.status_code == 200 + except requests.exceptions.RequestException: + return False + + +def process_file(dump_syms, path, arch, verbose, write_all): + arch_arg = ["-a", arch] + try: + stderr = None if verbose else subprocess.DEVNULL + stdout = subprocess.check_output([dump_syms] + arch_arg + [path], stderr=stderr) + except subprocess.CalledProcessError: + if verbose: + print("Processing %s%s...failed." % (path, " [%s]" % arch if arch else "")) + return None, None + module = stdout.splitlines()[0] + bits = module.split(b" ", 4) + if len(bits) != 5: + return None, None + _, platform, cpu_arch, debug_id, debug_file = bits + if verbose: + sys.stdout.write("Processing %s [%s]..." % (path, arch)) + filename = os.path.join(debug_file, debug_id, debug_file + b".sym") + # see if the server already has this symbol file + if not write_all: + if server_has_file(filename): + if verbose: + print("already on server.") + return None, None + # Collect for uploading + if verbose: + print("done.") + return filename, stdout + + +def get_files(paths, platform=sys.platform): + """ + For each entry passed in paths if the path is a file that can + be processed, yield it, otherwise if it is a directory yield files + under it that can be processed. + """ + for path in paths: + if os.path.isdir(path): + for root, subdirs, files in os.walk(path): + for f in files: + fullpath = os.path.join(root, f) + if should_process(fullpath, platform=platform): + yield fullpath + elif should_process(path, platform=platform): + yield path + + +def process_paths( + paths, executor, dump_syms, verbose, write_all=False, platform=sys.platform +): + jobs = set() + for fullpath in get_files(paths, platform=platform): + while os.path.islink(fullpath): + fullpath = os.path.join(os.path.dirname(fullpath), os.readlink(fullpath)) + if platform == "linux": + # See if there's a -dbg package installed and dump that instead. + dbgpath = "/usr/lib/debug" + fullpath + if os.path.isfile(dbgpath): + fullpath = dbgpath + for arch in get_archs(fullpath, platform=platform): + jobs.add( + executor.submit( + process_file, dump_syms, fullpath, arch, verbose, write_all + ) + ) + for job in concurrent.futures.as_completed(jobs): + try: + yield job.result() + except Exception as e: + traceback.print_exc(file=sys.stderr) + print("Error: %s" % str(e), file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-v", "--verbose", action="store_true", help="Produce verbose output" + ) + parser.add_argument( + "--all", + action="store_true", + help="Gather all system symbols, not just missing ones.", + ) + parser.add_argument("dump_syms", help="Path to dump_syms binary") + parser.add_argument( + "files", nargs="*", help="Specific files from which to gather symbols." + ) + args = parser.parse_args() + args.dump_syms = os.path.abspath(args.dump_syms) + # check for the dump_syms binary + if ( + not os.path.isabs(args.dump_syms) + or not os.path.exists(args.dump_syms) + or not os.access(args.dump_syms, os.X_OK) + ): + print( + "Error: can't find dump_syms binary at %s!" % args.dump_syms, + file=sys.stderr, + ) + return 1 + file_list = set() + executor = concurrent.futures.ProcessPoolExecutor() + zip_path = os.path.abspath("symbols.zip") + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + for filename, contents in process_paths( + args.files if args.files else SYSTEM_DIRS, + executor, + args.dump_syms, + args.verbose, + args.all, + ): + if filename and contents and filename not in file_list: + file_list.add(filename) + zf.writestr(filename, contents) + zf.writestr( + "ossyms-1.0-{platform}-{date}-symbols.txt".format( + platform=sys.platform.title(), + date=datetime.datetime.now().strftime("%Y%m%d%H%M%S"), + ), + "\n".join(file_list), + ) + if file_list: + if args.verbose: + print("Generated %s with %d symbols" % (zip_path, len(file_list))) + else: + os.unlink("symbols.zip") + + +if __name__ == "__main__": + main() diff --git a/tools/crashreporter/system-symbols/win/LICENSE b/tools/crashreporter/system-symbols/win/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/tools/crashreporter/system-symbols/win/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/crashreporter/system-symbols/win/known-microsoft-symbols.txt b/tools/crashreporter/system-symbols/win/known-microsoft-symbols.txt new file mode 100644 index 0000000000..d63dc716e9 --- /dev/null +++ b/tools/crashreporter/system-symbols/win/known-microsoft-symbols.txt @@ -0,0 +1,17 @@ +d2d1.pdb +d3d10level9.pdb +d3d10warp.pdb +d3d11.pdb +d3d9.pdb +d3dcompiler_47.pdb +d3dim700.pdb +kernel32.pdb +kernelbase.pdb +ntdll.pdb +user32.pdb +wkernel32.pdb +wkernelbase.pdb +wntdll.pdb +ws2_32.pdb +wuser32.pdb +zipwriter.pdb diff --git a/tools/crashreporter/system-symbols/win/run.sh b/tools/crashreporter/system-symbols/win/run.sh new file mode 100755 index 0000000000..f95b2b160a --- /dev/null +++ b/tools/crashreporter/system-symbols/win/run.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -v -e -x + +base="$(realpath "$(dirname "$0")")" + +export DUMP_SYMS_PATH="${MOZ_FETCHES_DIR}/dump_syms/dump_syms" + +mkdir -p artifacts && \ +ulimit -n 16384 && \ +python3 "${base}/symsrv-fetch.py" artifacts/target.crashreporter-symbols.zip diff --git a/tools/crashreporter/system-symbols/win/scrape-report.py b/tools/crashreporter/system-symbols/win/scrape-report.py new file mode 100644 index 0000000000..9bc21801c3 --- /dev/null +++ b/tools/crashreporter/system-symbols/win/scrape-report.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# +# Copyright 2016 Mozilla +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import json +import logging +import os +import sys + +import requests +import urlparse + +log = logging.getLogger() + + +def fetch_missing_symbols_from_crash(file_or_crash): + if os.path.isfile(file_or_crash): + log.info("Fetching missing symbols from JSON file: %s" % file_or_crash) + j = {"json_dump": json.load(open(file_or_crash, "rb"))} + else: + if "report/index/" in file_or_crash: + crash_id = urlparse.urlparse(file_or_crash).path.split("/")[-1] + else: + crash_id = file_or_crash + url = ( + "https://crash-stats.mozilla.org/api/ProcessedCrash/" + "?crash_id={crash_id}&datatype=processed".format(crash_id=crash_id) + ) + log.info("Fetching missing symbols from crash: %s" % url) + r = requests.get(url) + if r.status_code != 200: + log.error("Failed to fetch crash %s" % url) + return set() + j = r.json() + return set( + [ + (m["debug_file"], m["debug_id"], m["filename"], m["code_id"]) + for m in j["json_dump"]["modules"] + if "missing_symbols" in m + ] + ) + + +def main(): + logging.basicConfig() + log.setLevel(logging.DEBUG) + urllib3_logger = logging.getLogger("urllib3") + urllib3_logger.setLevel(logging.ERROR) + + if len(sys.argv) < 2: + log.error("Specify a crash URL or ID") + sys.exit(1) + symbols = fetch_missing_symbols_from_crash(sys.argv[1]) + log.info("Found %d missing symbols" % len(symbols)) + c = csv.writer(sys.stdout) + c.writerow(["debug_file", "debug_id", "code_file", "code_id"]) + for row in symbols: + c.writerow(row) + + +if __name__ == "__main__": + main() diff --git a/tools/crashreporter/system-symbols/win/skiplist.txt b/tools/crashreporter/system-symbols/win/skiplist.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/crashreporter/system-symbols/win/symsrv-fetch.py b/tools/crashreporter/system-symbols/win/symsrv-fetch.py new file mode 100644 index 0000000000..bc3b421cd0 --- /dev/null +++ b/tools/crashreporter/system-symbols/win/symsrv-fetch.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python +# +# Copyright 2016 Mozilla +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This script will read a CSV of modules from Socorro, and try to retrieve +# missing symbols from Microsoft's symbol server. It honors a list +# (ignorelist.txt) of symbols that are known to be from our applications, +# and it maintains its own list of symbols that the MS symbol server +# doesn't have (skiplist.txt). +# +# The script also depends on having write access to the directory it is +# installed in, to write the skiplist text file. + +import argparse +import asyncio +import logging +import os +import shutil +import zipfile +from collections import defaultdict +from tempfile import mkdtemp +from urllib.parse import quote, urljoin + +from aiofile import AIOFile, LineReader +from aiohttp import ClientSession, ClientTimeout +from aiohttp.connector import TCPConnector + +# Just hardcoded here +MICROSOFT_SYMBOL_SERVER = "https://msdl.microsoft.com/download/symbols/" +USER_AGENT = "Microsoft-Symbol-Server/6.3.0.0" +MOZILLA_SYMBOL_SERVER = "https://symbols.mozilla.org/" +MISSING_SYMBOLS_URL = "https://symbols.mozilla.org/missingsymbols.csv?microsoft=only" +HEADERS = {"User-Agent": USER_AGENT} +SYM_SRV = "SRV*{0}*https://msdl.microsoft.com/download/symbols;SRV*{0}*https://software.intel.com/sites/downloads/symbols;SRV*{0}*https://download.amd.com/dir/bin;SRV*{0}*https://driver-symbols.nvidia.com" # noqa +TIMEOUT = 7200 +RETRIES = 5 + + +log = logging.getLogger() + + +def get_type(data): + # PDB v7 + if data.startswith(b"Microsoft C/C++ MSF 7.00"): + return "pdb-v7" + # PDB v2 + if data.startswith(b"Microsoft C/C++ program database 2.00"): + return "pdb-v2" + # DLL + if data.startswith(b"MZ"): + return "dll" + # CAB + if data.startswith(b"MSCF"): + return "cab" + + return "unknown" + + +async def exp_backoff(retry_num): + await asyncio.sleep(2 ** retry_num) + + +async def server_has_file(client, server, filename): + """ + Send the symbol server a HEAD request to see if it has this symbol file. + """ + url = urljoin(server, quote(filename)) + for i in range(RETRIES): + try: + async with client.head(url, headers=HEADERS, allow_redirects=True) as resp: + if resp.status == 200 and ( + ( + "microsoft" in server + and resp.headers["Content-Type"] == "application/octet-stream" + ) + or "mozilla" in server + ): + log.debug(f"File exists: {url}") + return True + else: + return False + except Exception as e: + # Sometimes we've SSL errors or disconnections... so in such a situation just retry + log.warning(f"Error with {url}: retry") + log.exception(e) + await exp_backoff(i) + + log.debug(f"Too many retries (HEAD) for {url}: give up.") + return False + + +async def fetch_file(client, server, filename): + """ + Fetch the file from the server + """ + url = urljoin(server, quote(filename)) + log.debug(f"Fetch url: {url}") + for i in range(RETRIES): + try: + async with client.get(url, headers=HEADERS, allow_redirects=True) as resp: + if resp.status == 200: + data = await resp.read() + typ = get_type(data) + if typ == "unknown": + # try again + await exp_backoff(i) + elif typ == "pdb-v2": + # too old: skip it + log.debug(f"PDB v2 (skipped because too old): {url}") + return None + else: + return data + else: + log.error(f"Cannot get data (status {resp.status}) for {url}: ") + except Exception as e: + log.warning(f"Error with {url}") + log.exception(e) + await asyncio.sleep(0.5) + + log.debug(f"Too many retries (GET) for {url}: give up.") + return None + + +def write_skiplist(skiplist): + with open("skiplist.txt", "w") as sf: + sf.writelines( + f"{debug_id} {debug_file}\n" for debug_id, debug_file in skiplist.items() + ) + + +async def fetch_missing_symbols(u): + log.info("Trying missing symbols from %s" % u) + async with ClientSession() as client: + async with client.get(u, headers=HEADERS) as resp: + # The server currently does not set an encoding so force it to UTF-8 + data = await resp.text("UTF-8") + # just skip the first line since it contains column headers + return data.splitlines()[1:] + + +async def get_list(filename): + alist = set() + try: + async with AIOFile(filename, "r") as In: + async for line in LineReader(In): + line = line.rstrip() + alist.add(line) + except FileNotFoundError: + pass + + log.debug(f"{filename} contains {len(alist)} items") + + return alist + + +async def get_skiplist(): + skiplist = {} + path = "skiplist.txt" + try: + async with AIOFile(path, "r") as In: + async for line in LineReader(In): + line = line.strip() + if line == "": + continue + s = line.split(" ", maxsplit=1) + if len(s) != 2: + continue + debug_id, debug_file = s + skiplist[debug_id] = debug_file.lower() + except FileNotFoundError: + pass + + log.debug(f"{path} contains {len(skiplist)} items") + + return skiplist + + +def get_missing_symbols(missing_symbols, skiplist, ignorelist): + modules = defaultdict(set) + stats = {"ignorelist": 0, "skiplist": 0} + for line in missing_symbols: + line = line.rstrip() + bits = line.split(",") + if len(bits) < 2: + continue + pdb, debug_id = bits[:2] + code_file, code_id = None, None + if len(bits) >= 4: + code_file, code_id = bits[2:4] + if pdb and debug_id and pdb.endswith(".pdb"): + if pdb.lower() in ignorelist: + stats["ignorelist"] += 1 + continue + + if skiplist.get(debug_id) != pdb.lower(): + modules[pdb].add((debug_id, code_file, code_id)) + else: + stats["skiplist"] += 1 + # We've asked the symbol server previously about this, + # so skip it. + log.debug("%s/%s already in skiplist", pdb, debug_id) + + return modules, stats + + +async def collect_info(client, filename, debug_id, code_file, code_id): + pdb_path = os.path.join(filename, debug_id, filename) + sym_path = os.path.join(filename, debug_id, filename.replace(".pdb", "") + ".sym") + + has_pdb = await server_has_file(client, MICROSOFT_SYMBOL_SERVER, pdb_path) + has_code = is_there = False + if has_pdb: + if not await server_has_file(client, MOZILLA_SYMBOL_SERVER, sym_path): + has_code = ( + code_file + and code_id + and await server_has_file( + client, + MICROSOFT_SYMBOL_SERVER, + f"{code_file}/{code_id}/{code_file}", + ) + ) + else: + # if the file is on moz sym server no need to do anything + is_there = True + has_pdb = False + + return (filename, debug_id, code_file, code_id, has_pdb, has_code, is_there) + + +async def check_x86_file(path): + async with AIOFile(path, "rb") as In: + head = b"MODULE windows x86 " + chunk = await In.read(len(head)) + if chunk == head: + return True + return False + + +async def run_command(cmd): + proc = await asyncio.create_subprocess_shell( + cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + _, err = await proc.communicate() + err = err.decode().strip() + + return err + + +async def dump_module( + output, symcache, filename, debug_id, code_file, code_id, has_code, dump_syms +): + sym_path = os.path.join(filename, debug_id, filename.replace(".pdb", ".sym")) + output_path = os.path.join(output, sym_path) + sym_srv = SYM_SRV.format(symcache) + res = {"path": sym_path, "error": "ok"} + + if has_code: + cmd = ( + f"{dump_syms} {code_file} --code-id {code_id} --check-cfi --inlines " + f"--store {output} --symbol-server '{sym_srv}' --verbose error" + ) + else: + cmd = ( + f"{dump_syms} {filename} --debug-id {debug_id} --check-cfi --inlines " + f"--store {output} --symbol-server '{sym_srv}' --verbose error" + ) + + err = await run_command(cmd) + + if err: + log.error(f"Error with {cmd}") + log.error(err) + res["error"] = "dump error" + return res + + if not os.path.exists(output_path): + log.error(f"Could not find file {output_path} after running {cmd}") + res["error"] = "dump error" + return res + + if not has_code and not await check_x86_file(output_path): + # PDB for 32 bits contains everything we need (symbols + stack unwind info) + # But PDB for 64 bits don't contain stack unwind info + # (they're in the binary (.dll/.exe) itself). + # So here we're logging because we've got a PDB (64 bits) without its DLL/EXE. + if code_file and code_id: + log.debug(f"x86_64 binary {code_file}/{code_id} required") + else: + log.debug(f"x86_64 binary for {filename}/{debug_id} required") + res["error"] = "no binary" + return res + + log.info(f"Successfully dumped: {filename}/{debug_id}") + return res + + +async def dump(output, symcache, modules, dump_syms): + tasks = [] + for filename, debug_id, code_file, code_id, has_code in modules: + tasks.append( + dump_module( + output, + symcache, + filename, + debug_id, + code_file, + code_id, + has_code, + dump_syms, + ) + ) + + res = await asyncio.gather(*tasks) + + # Even if we haven't CFI the generated file is useful to get symbols + # from addresses so keep error == 2. + file_index = {x["path"] for x in res if x["error"] in ["ok", "no binary"]} + stats = { + "dump_error": sum(1 for x in res if x["error"] == "dump error"), + "no_bin": sum(1 for x in res if x["error"] == "no binary"), + } + + return file_index, stats + + +async def collect(modules): + loop = asyncio.get_event_loop() + tasks = [] + + # In case of errors (Too many open files), just change limit_per_host + connector = TCPConnector(limit=100, limit_per_host=4) + + async with ClientSession( + loop=loop, timeout=ClientTimeout(total=TIMEOUT), connector=connector + ) as client: + for filename, ids in modules.items(): + for debug_id, code_file, code_id in ids: + tasks.append( + collect_info(client, filename, debug_id, code_file, code_id) + ) + + res = await asyncio.gather(*tasks) + to_dump = [] + stats = {"no_pdb": 0, "is_there": 0} + for filename, debug_id, code_file, code_id, has_pdb, has_code, is_there in res: + if not has_pdb: + if is_there: + stats["is_there"] += 1 + else: + stats["no_pdb"] += 1 + log.info(f"No pdb for {filename}/{debug_id}") + continue + + log.info( + f"To dump: {filename}/{debug_id}, {code_file}/{code_id} and has_code = {has_code}" + ) + to_dump.append((filename, debug_id, code_file, code_id, has_code)) + + log.info(f"Collected {len(to_dump)} files to dump") + + return to_dump, stats + + +async def make_dirs(path): + loop = asyncio.get_event_loop() + + def helper(path): + os.makedirs(path, exist_ok=True) + + await loop.run_in_executor(None, helper, path) + + +async def fetch_and_write(output, client, filename, file_id): + path = os.path.join(filename, file_id, filename) + data = await fetch_file(client, MICROSOFT_SYMBOL_SERVER, path) + + if not data: + return False + + output_dir = os.path.join(output, filename, file_id) + await make_dirs(output_dir) + + output_path = os.path.join(output_dir, filename) + async with AIOFile(output_path, "wb") as Out: + await Out.write(data) + + return True + + +async def fetch_all(output, modules): + loop = asyncio.get_event_loop() + tasks = [] + fetched_modules = [] + + # In case of errors (Too many open files), just change limit_per_host + connector = TCPConnector(limit=100, limit_per_host=0) + + async with ClientSession( + loop=loop, timeout=ClientTimeout(total=TIMEOUT), connector=connector + ) as client: + for filename, debug_id, code_file, code_id, has_code in modules: + tasks.append(fetch_and_write(output, client, filename, debug_id)) + if has_code: + tasks.append(fetch_and_write(output, client, code_file, code_id)) + + res = await asyncio.gather(*tasks) + res = iter(res) + for filename, debug_id, code_file, code_id, has_code in modules: + fetched_pdb = next(res) + if has_code: + has_code = next(res) + if fetched_pdb: + fetched_modules.append( + (filename, debug_id, code_file, code_id, has_code) + ) + + return fetched_modules + + +def get_base_data(url): + async def helper(url): + return await asyncio.gather( + fetch_missing_symbols(url), + # Symbols that we know belong to us, so don't ask Microsoft for them. + get_list("ignorelist.txt"), + # Symbols that we know belong to Microsoft, so don't skiplist them. + get_list("known-microsoft-symbols.txt"), + # Symbols that we've asked for in the past unsuccessfully + get_skiplist(), + ) + + return asyncio.run(helper(url)) + + +def gen_zip(output, output_dir, file_index): + if not file_index: + return + + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as z: + for f in file_index: + z.write(os.path.join(output_dir, f), f) + log.info(f"Wrote zip as {output}") + + +def main(): + parser = argparse.ArgumentParser( + description="Fetch missing symbols from Microsoft symbol server" + ) + parser.add_argument( + "--missing-symbols", + type=str, + help="missing symbols URL", + default=MISSING_SYMBOLS_URL, + ) + parser.add_argument("zip", type=str, help="output zip file") + parser.add_argument( + "--dump-syms", + type=str, + help="dump_syms path", + default=os.environ.get("DUMP_SYMS_PATH"), + ) + + args = parser.parse_args() + + assert args.dump_syms, "dump_syms path is empty" + + logging.basicConfig(level=logging.DEBUG) + aiohttp_logger = logging.getLogger("aiohttp.client") + aiohttp_logger.setLevel(logging.INFO) + log.info("Started") + + missing_symbols, ignorelist, known_ms_symbols, skiplist = get_base_data( + args.missing_symbols + ) + + modules, stats_skipped = get_missing_symbols(missing_symbols, skiplist, ignorelist) + + symbol_path = mkdtemp("symsrvfetch") + temp_path = mkdtemp(prefix="symcache") + + modules, stats_collect = asyncio.run(collect(modules)) + modules = asyncio.run(fetch_all(temp_path, modules)) + + file_index, stats_dump = asyncio.run( + dump(symbol_path, temp_path, modules, args.dump_syms) + ) + + gen_zip(args.zip, symbol_path, file_index) + + shutil.rmtree(symbol_path, True) + shutil.rmtree(temp_path, True) + + write_skiplist(skiplist) + + if not file_index: + log.info(f"No symbols downloaded: {len(missing_symbols)} considered") + else: + log.info( + f"Total files: {len(missing_symbols)}, Stored {len(file_index)} symbol files" + ) + + log.info( + f"{stats_collect['is_there']} already present, {stats_skipped['ignorelist']} in ignored list, " # noqa + f"{stats_skipped['skiplist']} skipped, {stats_collect['no_pdb']} not found, " + f"{stats_dump['dump_error']} processed with errors, " + f"{stats_dump['no_bin']} processed but with no binaries (x86_64)" + ) + log.info("Finished, exiting") + + +if __name__ == "__main__": + main() diff --git a/tools/esmify/import-to-import_esmodule.js b/tools/esmify/import-to-import_esmodule.js new file mode 100644 index 0000000000..2d7ded70a3 --- /dev/null +++ b/tools/esmify/import-to-import_esmodule.js @@ -0,0 +1,476 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// jscodeshift rule to replace import calls for JSM with import calls for ESM +// or static import for ESM. + +/* eslint-env node */ + +const _path = require("path"); +const { isESMified } = require(_path.resolve(__dirname, "./is-esmified.js")); +const { + jsmExtPattern, + esmifyExtension, + isIdentifier, + isString, + warnForPath, + getPrevStatement, + getNextStatement, + isMemberExpressionWithIdentifiers, + rewriteMemberExpressionWithIdentifiers, + createMemberExpressionWithIdentifiers, +} = require(_path.resolve(__dirname, "./utils.js")); +const { + isImportESModuleCall, + replaceImportESModuleCall, + tryReplacingWithStaticImport, +} = require(_path.resolve(__dirname, "./static-import.js")); + +module.exports = function (fileInfo, api) { + const { jscodeshift } = api; + const root = jscodeshift(fileInfo.source); + doTranslate(fileInfo.path, jscodeshift, root); + return root.toSource({ lineTerminator: "\n" }); +}; + +module.exports.doTranslate = doTranslate; + +function isESMifiedAndTarget(resourceURI) { + const files = []; + if (!isESMified(resourceURI, files)) { + return false; + } + + if ("ESMIFY_TARGET_PREFIX" in process.env) { + const targetPrefix = process.env.ESMIFY_TARGET_PREFIX; + for (const esm of files) { + if (esm.startsWith(targetPrefix)) { + return true; + } + } + + return false; + } + + return true; +} + +const importCalls = [ + { + from: ["Cu", "import"], + to: ["ChromeUtils", "importESModule"], + }, + { + from: ["ChromeUtils", "import"], + to: ["ChromeUtils", "importESModule"], + }, + { + from: ["SpecialPowers", "ChromeUtils", "import"], + to: ["SpecialPowers", "ChromeUtils", "importESModule"], + }, +]; + +const singleLazyGetterCalls = [ + { + from: ["XPCOMUtils", "defineLazyModuleGetter"], + to: ["ChromeUtils", "defineESModuleGetters"], + }, + { + from: ["ChromeUtils", "defineModuleGetter"], + to: ["ChromeUtils", "defineESModuleGetters"], + }, + { + from: ["SpecialPowers", "ChromeUtils", "defineModuleGetter"], + to: ["SpecialPowers", "ChromeUtils", "defineESModuleGetters"], + }, +]; + +const multiLazyGettersCalls = [ + { + from: ["XPCOMUtils", "defineLazyModuleGetters"], + to: ["ChromeUtils", "defineESModuleGetters"], + }, +]; + +function isMemberExpressionMatchingPatterns(node, patterns) { + for (const item of patterns) { + if (isMemberExpressionWithIdentifiers(node, item.from)) { + return item; + } + } + + return null; +} + +function replaceImportCall(inputFile, jscodeshift, path, rewriteItem) { + if (path.node.arguments.length !== 1) { + warnForPath(inputFile, path, `import call should have only one argument`); + return; + } + + const resourceURINode = path.node.arguments[0]; + if (!isString(resourceURINode)) { + warnForPath(inputFile, path, `resource URI should be a string`); + return; + } + + const resourceURI = resourceURINode.value; + if (!resourceURI.match(jsmExtPattern)) { + warnForPath(inputFile, path, `Non-jsm: ${resourceURI}`); + return; + } + + if (!isESMifiedAndTarget(resourceURI)) { + return; + } + + if ( + !tryReplacingWithStaticImport( + jscodeshift, + inputFile, + path, + resourceURINode, + false + ) + ) { + rewriteMemberExpressionWithIdentifiers(path.node.callee, rewriteItem.to); + resourceURINode.value = esmifyExtension(resourceURI); + } +} + +// Find `ChromeUtils.defineESModuleGetters` or variant statement specified by +// expectedIDs, adjacent to `path` which uses the same target object. +function findDefineESModuleGettersStmt(path, expectedIDs) { + // `path` must be top-level. + if (path.parent.node.type !== "ExpressionStatement") { + return null; + } + + if (path.parent.parent.node.type !== "Program") { + return null; + } + + // Get previous or next statement with ChromeUtils.defineESModuleGetters. + let callStmt; + const prev = getPrevStatement(path.parent); + if ( + prev && + prev.type === "ExpressionStatement" && + prev.expression.type === "CallExpression" && + isMemberExpressionWithIdentifiers(prev.expression.callee, expectedIDs) + ) { + callStmt = prev; + } else { + const next = getNextStatement(path.parent); + if ( + next && + next.type === "ExpressionStatement" && + next.expression.type === "CallExpression" && + isMemberExpressionWithIdentifiers(next.expression.callee, expectedIDs) + ) { + callStmt = next; + } else { + return null; + } + } + + const call = callStmt.expression; + + if (call.arguments.length !== 2) { + return null; + } + + const modulesNode = call.arguments[1]; + if (modulesNode.type !== "ObjectExpression") { + return null; + } + + // Check if the target object is same. + if ( + path.node.arguments[0].type === "ThisExpression" && + call.arguments[0].type === "ThisExpression" + ) { + return callStmt; + } + + if ( + path.node.arguments[0].type === "Identifier" && + call.arguments[0].type === "Identifier" && + path.node.arguments[0].name === call.arguments[0].name + ) { + return callStmt; + } + + return null; +} + +function getPropKeyString(prop) { + if (prop.key.type === "Identifier") { + return prop.key.name; + } + + if (prop.key.type === "Literal") { + return prop.key.value.toString(); + } + + return ""; +} + +function sortProps(obj) { + obj.properties.sort((a, b) => { + return getPropKeyString(a) < getPropKeyString(b) ? -1 : 1; + }); +} + +// Move comments above `nodeFrom` before `nodeTo`. +function moveComments(nodeTo, nodeFrom) { + if (!nodeFrom.comments) { + return; + } + if (nodeTo.comments) { + nodeTo.comments = [...nodeTo.comments, ...nodeFrom.comments]; + } else { + nodeTo.comments = nodeFrom.comments; + } + nodeFrom.comments = []; +} + +function replaceLazyGetterCall(inputFile, jscodeshift, path, rewriteItem) { + if (path.node.arguments.length !== 3) { + warnForPath(inputFile, path, `lazy getter call should have 3 arguments`); + return; + } + + const nameNode = path.node.arguments[1]; + if (!isString(nameNode)) { + warnForPath(inputFile, path, `name should be a string`); + return; + } + + const resourceURINode = path.node.arguments[2]; + if (!isString(resourceURINode)) { + warnForPath(inputFile, path, `resource URI should be a string`); + return; + } + + const resourceURI = resourceURINode.value; + if (!resourceURI.match(jsmExtPattern)) { + warnForPath(inputFile, path, `Non-js/jsm: ${resourceURI}`); + return; + } + + if (!isESMifiedAndTarget(resourceURI)) { + return; + } + + resourceURINode.value = esmifyExtension(resourceURI); + const prop = jscodeshift.property( + "init", + jscodeshift.identifier(nameNode.value), + resourceURINode + ); + + const callStmt = findDefineESModuleGettersStmt(path, rewriteItem.to); + if (callStmt) { + // Move a property to existing ChromeUtils.defineESModuleGetters call. + + moveComments(callStmt, path.parent.node); + path.parent.prune(); + + callStmt.expression.arguments[1].properties.push(prop); + sortProps(callStmt.expression.arguments[1]); + } else { + // Convert this call into ChromeUtils.defineESModuleGetters. + + rewriteMemberExpressionWithIdentifiers(path.node.callee, rewriteItem.to); + path.node.arguments = [ + path.node.arguments[0], + jscodeshift.objectExpression([prop]), + ]; + } +} + +function replaceLazyGettersCall(inputFile, jscodeshift, path, rewriteItem) { + if (path.node.arguments.length !== 2) { + warnForPath(inputFile, path, `lazy getters call should have 2 arguments`); + return; + } + + const modulesNode = path.node.arguments[1]; + if (modulesNode.type !== "ObjectExpression") { + warnForPath(inputFile, path, `modules parameter should be an object`); + return; + } + + const esmProps = []; + const jsmProps = []; + + for (const prop of modulesNode.properties) { + const resourceURINode = prop.value; + if (!isString(resourceURINode)) { + warnForPath(inputFile, path, `resource URI should be a string`); + jsmProps.push(prop); + continue; + } + + const resourceURI = resourceURINode.value; + if (!resourceURI.match(jsmExtPattern)) { + warnForPath(inputFile, path, `Non-js/jsm: ${resourceURI}`); + jsmProps.push(prop); + continue; + } + + if (!isESMifiedAndTarget(resourceURI)) { + jsmProps.push(prop); + continue; + } + + esmProps.push(prop); + } + + if (esmProps.length === 0) { + return; + } + + let callStmt = findDefineESModuleGettersStmt(path, rewriteItem.to); + if (jsmProps.length === 0) { + if (callStmt) { + // Move all properties to existing ChromeUtils.defineESModuleGetters call. + + moveComments(callStmt, path.parent.node); + path.parent.prune(); + + for (const prop of esmProps) { + const resourceURINode = prop.value; + resourceURINode.value = esmifyExtension(resourceURINode.value); + callStmt.expression.arguments[1].properties.push(prop); + } + sortProps(callStmt.expression.arguments[1]); + } else { + // Convert this call into ChromeUtils.defineESModuleGetters. + + rewriteMemberExpressionWithIdentifiers(path.node.callee, rewriteItem.to); + for (const prop of esmProps) { + const resourceURINode = prop.value; + resourceURINode.value = esmifyExtension(resourceURINode.value); + } + } + } else { + // Move some properties to ChromeUtils.defineESModuleGetters. + + if (path.parent.node.type !== "ExpressionStatement") { + warnForPath(inputFile, path, `lazy getters call in unexpected context`); + return; + } + + if (!callStmt) { + callStmt = jscodeshift.expressionStatement( + jscodeshift.callExpression( + createMemberExpressionWithIdentifiers(jscodeshift, rewriteItem.to), + [path.node.arguments[0], jscodeshift.objectExpression([])] + ) + ); + path.parent.insertBefore(callStmt); + } + + moveComments(callStmt, path.parent.node); + + for (const prop of esmProps) { + const resourceURINode = prop.value; + resourceURINode.value = esmifyExtension(resourceURINode.value); + callStmt.expression.arguments[1].properties.push(prop); + } + sortProps(callStmt.expression.arguments[1]); + + path.node.arguments[1].properties = jsmProps; + } +} + +function getProp(obj, key) { + if (obj.type !== "ObjectExpression") { + return null; + } + + for (const prop of obj.properties) { + if (prop.computed) { + continue; + } + + if (!prop.key) { + continue; + } + + if (isIdentifier(prop.key, key)) { + return prop; + } + } + + return null; +} + +function tryReplaceActorDefinition(inputFile, path, name) { + const obj = path.node; + + const prop = getProp(obj, name); + if (!prop) { + return; + } + + const moduleURIProp = getProp(prop.value, "moduleURI"); + if (!moduleURIProp) { + return; + } + + if (!isString(moduleURIProp.value)) { + warnForPath(inputFile, path, `${name} moduleURI should be a string`); + return; + } + + const moduleURI = moduleURIProp.value.value; + if (!moduleURI.match(jsmExtPattern)) { + warnForPath(inputFile, path, `${name} Non-js/jsm: ${moduleURI}`); + return; + } + + if (!isESMifiedAndTarget(moduleURI)) { + return; + } + + moduleURIProp.key.name = "esModuleURI"; + moduleURIProp.value.value = esmifyExtension(moduleURI); +} + +function doTranslate(inputFile, jscodeshift, root) { + root.find(jscodeshift.CallExpression).forEach(path => { + if (isImportESModuleCall(path.node)) { + replaceImportESModuleCall(inputFile, jscodeshift, path, false); + return; + } + + const callee = path.node.callee; + + let item; + item = isMemberExpressionMatchingPatterns(callee, importCalls); + if (item) { + replaceImportCall(inputFile, jscodeshift, path, item); + return; + } + + item = isMemberExpressionMatchingPatterns(callee, singleLazyGetterCalls); + if (item) { + replaceLazyGetterCall(inputFile, jscodeshift, path, item); + return; + } + + item = isMemberExpressionMatchingPatterns(callee, multiLazyGettersCalls); + if (item) { + replaceLazyGettersCall(inputFile, jscodeshift, path, item); + } + }); + + root.find(jscodeshift.ObjectExpression).forEach(path => { + tryReplaceActorDefinition(inputFile, path, "parent"); + tryReplaceActorDefinition(inputFile, path, "child"); + }); +} diff --git a/tools/esmify/is-esmified.js b/tools/esmify/is-esmified.js new file mode 100644 index 0000000000..872b4cd877 --- /dev/null +++ b/tools/esmify/is-esmified.js @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// A utility to check if given JSM is already ESM-ified. + +/* eslint-env node */ + +const fs = require("fs"); +const _path = require("path"); +const { esmifyExtension } = require(_path.resolve(__dirname, "./utils.js")); + +let json_map; +if (process.env.ESMIFY_MAP_JSON) { + json_map = _path.resolve(process.env.ESMIFY_MAP_JSON); +} else { + json_map = _path.resolve(__dirname, "./map.json"); +} +const uri_map = JSON.parse(fs.readFileSync(json_map)); +const esm_uri_map = generateESMURIMap(uri_map); + +function generateESMURIMap(jsm_map) { + const esm_map = {}; + + for (let [uri, jsms] of Object.entries(jsm_map)) { + if (typeof jsms === "string") { + jsms = [jsms]; + } + esm_map[esmifyExtension(uri)] = jsms.map(esmifyExtension); + } + + return esm_map; +} + +function isESMifiedSlow(resourceURI) { + if (!(resourceURI in uri_map)) { + console.log(`WARNING: Unknown module: ${resourceURI}`); + return { result: false, jsms: [] }; + } + + let jsms = uri_map[resourceURI]; + if (typeof jsms === "string") { + jsms = [jsms]; + } + + const prefix = "../../"; + for (const jsm of jsms) { + if (fs.existsSync(prefix + jsm)) { + return { result: false, jsms }; + } + const esm = esmifyExtension(jsm); + if (!fs.existsSync(prefix + esm)) { + return { result: false, jsms }; + } + } + + return { result: true, jsms }; +} + +const isESMified_memo = {}; +function isESMified(resourceURI, files) { + if (!(resourceURI in isESMified_memo)) { + isESMified_memo[resourceURI] = isESMifiedSlow(resourceURI); + } + + for (const jsm of isESMified_memo[resourceURI].jsms) { + files.push(esmifyExtension(jsm)); + } + + return isESMified_memo[resourceURI].result; +} + +function getESMFiles(resourceURI) { + if (resourceURI in esm_uri_map) { + return esm_uri_map[resourceURI]; + } + return []; +} + +exports.isESMified = isESMified; +exports.getESMFiles = getESMFiles; diff --git a/tools/esmify/mach_commands.py b/tools/esmify/mach_commands.py new file mode 100644 index 0000000000..787de48424 --- /dev/null +++ b/tools/esmify/mach_commands.py @@ -0,0 +1,910 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os +import pathlib +import re +import subprocess +import sys + +from mach.decorators import Command, CommandArgument + + +def path_sep_to_native(path_str): + """Make separators in the path OS native.""" + return pathlib.os.sep.join(path_str.split("/")) + + +def path_sep_from_native(path): + """Make separators in the path OS native.""" + return "/".join(str(path).split(pathlib.os.sep)) + + +excluded_from_convert_prefix = list( + map( + path_sep_to_native, + [ + # Testcases for actors. + "toolkit/actors/TestProcessActorChild.jsm", + "toolkit/actors/TestProcessActorParent.jsm", + "toolkit/actors/TestWindowChild.jsm", + "toolkit/actors/TestWindowParent.jsm", + "js/xpconnect/tests/unit/", + # Testcase for build system. + "python/mozbuild/mozbuild/test/", + ], + ) +) + + +def is_excluded_from_convert(path): + """Returns true if the JSM file shouldn't be converted to ESM.""" + path_str = str(path) + for prefix in excluded_from_convert_prefix: + if path_str.startswith(prefix): + return True + + return False + + +excluded_from_imports_prefix = list( + map( + path_sep_to_native, + [ + # Vendored or auto-generated files. + "browser/components/pocket/content/panels/js/vendor.bundle.js", + "devtools/client/debugger/dist/parser-worker.js", + "devtools/client/debugger/test/mochitest/examples/react/build/main.js", + "devtools/client/debugger/test/mochitest/examples/sourcemapped/polyfill-bundle.js", + "devtools/client/inspector/markup/test/shadowdom_open_debugger.min.js", + "devtools/client/shared/source-map-loader/test/browser/fixtures/bundle.js", + "layout/style/test/property_database.js", + "services/fxaccounts/FxAccountsPairingChannel.js", + "testing/talos/talos/tests/devtools/addon/content/pages/custom/debugger/static/js/main.js", # noqa E501 + "testing/web-platform/", + # Unrelated testcases that has edge case syntax. + "browser/components/sessionstore/test/unit/data/", + "devtools/client/debugger/src/workers/parser/tests/fixtures/", + "devtools/client/debugger/test/mochitest/examples/sourcemapped/fixtures/", + "devtools/client/webconsole/test/browser/test-syntaxerror-worklet.js", + "devtools/server/tests/xpcshell/test_framebindings-03.js", + "devtools/server/tests/xpcshell/test_framebindings-04.js", + "devtools/shared/tests/xpcshell/test_eventemitter_basic.js", + "devtools/shared/tests/xpcshell/test_eventemitter_static.js", + "dom/base/crashtests/module-with-syntax-error.js", + "dom/base/test/file_bug687859-16.js", + "dom/base/test/file_bug687859-16.js", + "dom/base/test/file_js_cache_syntax_error.js", + "dom/base/test/jsmodules/module_badSyntax.js", + "dom/canvas/test/reftest/webgl-utils.js", + "dom/encoding/test/file_utf16_be_bom.js", + "dom/encoding/test/file_utf16_le_bom.js", + "dom/html/test/bug649134/file_bug649134-1.sjs", + "dom/html/test/bug649134/file_bug649134-2.sjs", + "dom/media/webrtc/tests/mochitests/identity/idp-bad.js", + "dom/serviceworkers/test/file_js_cache_syntax_error.js", + "dom/serviceworkers/test/parse_error_worker.js", + "dom/workers/test/importScripts_worker_imported3.js", + "dom/workers/test/invalid.js", + "dom/workers/test/threadErrors_worker1.js", + "dom/xhr/tests/browser_blobFromFile.js", + "image/test/browser/browser_image.js", + "js/xpconnect/tests/chrome/test_bug732665_meta.js", + "js/xpconnect/tests/mochitest/class_static_worker.js", + "js/xpconnect/tests/unit/bug451678_subscript.js", + "js/xpconnect/tests/unit/error_other.sys.mjs", + "js/xpconnect/tests/unit/es6module_parse_error.js", + "js/xpconnect/tests/unit/recursive_importA.jsm", + "js/xpconnect/tests/unit/recursive_importB.jsm", + "js/xpconnect/tests/unit/syntax_error.jsm", + "js/xpconnect/tests/unit/test_defineModuleGetter.js", + "js/xpconnect/tests/unit/test_import.js", + "js/xpconnect/tests/unit/test_import_shim.js", + "js/xpconnect/tests/unit/test_recursive_import.js", + "js/xpconnect/tests/unit/test_unload.js", + "modules/libpref/test/unit/data/testParser.js", + "python/mozbuild/mozbuild/test/", + "remote/shared/messagehandler/test/browser/resources/modules/root/invalid.sys.mjs", + "testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js", + "testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js", + "toolkit/components/reader/Readerable.sys.mjs", + "toolkit/components/workerloader/tests/moduleF-syntax-error.js", + "tools/lint/test/", + "tools/update-packaging/test/", + # SpiderMonkey internals. + "js/examples/", + "js/src/", + # Files has macro. + "browser/app/profile/firefox.js", + "browser/branding/official/pref/firefox-branding.js", + "browser/components/enterprisepolicies/schemas/schema.sys.mjs", + "browser/locales/en-US/firefox-l10n.js", + "mobile/android/app/geckoview-prefs.js", + "mobile/android/app/mobile.js", + "mobile/android/locales/en-US/mobile-l10n.js", + "modules/libpref/greprefs.js", + "modules/libpref/init/all.js", + "testing/condprofile/condprof/tests/profile/user.js", + "testing/mozbase/mozprofile/tests/files/prefs_with_comments.js", + "toolkit/modules/AppConstants.sys.mjs", + "toolkit/mozapps/update/tests/data/xpcshellConstantsPP.js", + # Uniffi templates + "toolkit/components/uniffi-bindgen-gecko-js/src/templates/js/", + ], + ) +) + +EXCLUSION_FILES = [ + os.path.join("tools", "rewriting", "Generated.txt"), + os.path.join("tools", "rewriting", "ThirdPartyPaths.txt"), +] + + +def load_exclusion_files(): + for path in EXCLUSION_FILES: + with open(path, "r") as f: + for line in f: + p = path_sep_to_native(re.sub("\*$", "", line.strip())) + excluded_from_imports_prefix.append(p) + + +def is_excluded_from_imports(path): + """Returns true if the JS file content shouldn't be handled by + jscodeshift. + + This filter is necessary because jscodeshift cannot handle some + syntax edge cases and results in unexpected rewrite.""" + path_str = str(path) + for prefix in excluded_from_imports_prefix: + if path_str.startswith(prefix): + return True + + return False + + +# Wrapper for hg/git operations +class VCSUtils: + def run(self, cmd): + # Do not pass check=True because the pattern can match no file. + lines = subprocess.run(cmd, stdout=subprocess.PIPE).stdout.decode() + return filter(lambda x: x != "", lines.split("\n")) + + +class HgUtils(VCSUtils): + def is_available(): + return pathlib.Path(".hg").exists() + + def rename(self, before, after): + cmd = ["hg", "rename", before, after] + subprocess.run(cmd, check=True) + + def find_jsms(self, path): + jsms = [] + + # NOTE: `set:glob:` syntax does not accept backslash on windows. + path = path_sep_from_native(path) + + cmd = ["hg", "files", f'set:glob:"{path}/**/*.jsm"'] + for line in self.run(cmd): + jsm = pathlib.Path(line) + if is_excluded_from_convert(jsm): + continue + jsms.append(jsm) + + cmd = [ + "hg", + "files", + f"set:grep('EXPORTED_SYMBOLS = \[') and glob:\"{path}/**/*.js\"", + ] + for line in self.run(cmd): + jsm = pathlib.Path(line) + if is_excluded_from_convert(jsm): + continue + jsms.append(jsm) + + return jsms + + def find_all_jss(self, path): + jss = [] + + # NOTE: `set:glob:` syntax does not accept backslash on windows. + path = path_sep_from_native(path) + + cmd = [ + "hg", + "files", + f'set:glob:"{path}/**/*.jsm" or glob:"{path}/**/*.js" or ' + + f'glob:"{path}/**/*.mjs" or glob:"{path}/**/*.sjs"', + ] + for line in self.run(cmd): + js = pathlib.Path(line) + if is_excluded_from_imports(js): + continue + jss.append(js) + + return jss + + +class GitUtils(VCSUtils): + def is_available(): + return pathlib.Path(".git").exists() + + def rename(self, before, after): + cmd = ["git", "mv", before, after] + subprocess.run(cmd, check=True) + + def find_jsms(self, path): + jsms = [] + + cmd = ["git", "ls-files", f"{path}/*.jsm"] + for line in self.run(cmd): + jsm = pathlib.Path(line) + if is_excluded_from_convert(jsm): + continue + jsms.append(jsm) + + handled = {} + cmd = ["git", "grep", "EXPORTED_SYMBOLS = \[", f"{path}/*.js"] + for line in self.run(cmd): + m = re.search("^([^:]+):", line) + if not m: + continue + filename = m.group(1) + if filename in handled: + continue + handled[filename] = True + jsm = pathlib.Path(filename) + if is_excluded_from_convert(jsm): + continue + jsms.append(jsm) + + return jsms + + def find_all_jss(self, path): + jss = [] + + cmd = [ + "git", + "ls-files", + f"{path}/*.jsm", + f"{path}/*.js", + f"{path}/*.mjs", + f"{path}/*.sjs", + ] + for line in self.run(cmd): + js = pathlib.Path(line) + if is_excluded_from_imports(js): + continue + jss.append(js) + + return jss + + +class Summary: + def __init__(self): + self.convert_errors = [] + self.import_errors = [] + self.rename_errors = [] + self.no_refs = [] + + +@Command( + "esmify", + category="misc", + description="ESMify JSM files.", +) +@CommandArgument( + "path", + nargs=1, + help="Path to the JSM file to ESMify, or the directory that contains " + "JSM files and/or JS files that imports ESM-ified JSM.", +) +@CommandArgument( + "--convert", + action="store_true", + help="Only perform the step 1 = convert part", +) +@CommandArgument( + "--imports", + action="store_true", + help="Only perform the step 2 = import calls part", +) +@CommandArgument( + "--prefix", + default="", + help="Restrict the target of import in the step 2 to ESM-ified JSM, by the " + "prefix match for the JSM file's path. e.g. 'browser/'.", +) +def esmify(command_context, path=None, convert=False, imports=False, prefix=""): + """ + This command does the following 2 steps: + 1. Convert the JSM file specified by `path` to ESM file, or the JSM files + inside the directory specified by `path` to ESM files, and also + fix references in build files and test definitions + 2. Convert import calls inside file(s) specified by `path` for ESM-ified + files to use new APIs + + Example 1: + # Convert all JSM files inside `browser/components/pagedata` directory, + # and replace all references for ESM-ified files in the entire tree to use + # new APIs + + $ ./mach esmify --convert browser/components/pagedata + $ ./mach esmify --imports . --prefix=browser/components/pagedata + + Example 2: + # Convert all JSM files inside `browser` directory, and replace all + # references for the JSM files inside `browser` directory to use + # new APIs + + $ ./mach esmify browser + """ + + def error(text): + command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}") + + def warn(text): + command_context.log(logging.WARN, "esmify", {}, f"[WARN] {text}") + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + # If no options is specified, perform both. + if not convert and not imports: + convert = True + imports = True + + path = pathlib.Path(path[0]) + + if not verify_path(command_context, path): + return 1 + + if HgUtils.is_available(): + vcs_utils = HgUtils() + elif GitUtils.is_available(): + vcs_utils = GitUtils() + else: + error( + "This script needs to be run inside mozilla-central " + "checkout of either mercurial or git." + ) + return 1 + + load_exclusion_files() + + info("Setting up jscodeshift...") + setup_jscodeshift() + + is_single_file = path.is_file() + + modified_files = [] + summary = Summary() + + if convert: + info("Searching files to convert to ESM...") + if is_single_file: + jsms = [path] + else: + jsms = vcs_utils.find_jsms(path) + + info(f"Found {len(jsms)} file(s) to convert to ESM.") + + info("Converting to ESM...") + jsms = convert_module(jsms, summary) + if jsms is None: + error("Failed to rewrite exports.") + return 1 + + info("Renaming...") + esms = rename_jsms(command_context, vcs_utils, jsms, summary) + + modified_files += esms + + if imports: + info("Searching files to rewrite imports...") + + if is_single_file: + if convert: + # Already converted above + jss = esms + else: + jss = [path] + else: + jss = vcs_utils.find_all_jss(path) + + info(f"Checking {len(jss)} JS file(s). Rewriting any matching imports...") + + result = rewrite_imports(jss, prefix, summary) + if result is None: + return 1 + + info(f"Rewritten {len(result)} file(s).") + + # Only modified files needs eslint fix + modified_files += result + + modified_files = list(set(modified_files)) + + info(f"Applying eslint --fix for {len(modified_files)} file(s)...") + eslint_fix(command_context, modified_files) + + def print_files(f, errors): + for [path, message] in errors: + f(f" * {path}") + if message: + f(f" {message}") + + if len(summary.convert_errors): + error("========") + error("Following files are not converted into ESM due to error:") + print_files(error, summary.convert_errors) + + if len(summary.import_errors): + warn("========") + warn("Following files are not rewritten to import ESMs due to error:") + warn( + "(NOTE: Errors related to 'private names' are mostly due to " + " preprocessor macros in the file):" + ) + print_files(warn, summary.import_errors) + + if len(summary.rename_errors): + error("========") + error("Following files are not renamed due to error:") + print_files(error, summary.rename_errors) + + if len(summary.no_refs): + warn("========") + warn("Following files are not found in any build files.") + warn("Please update references to those files manually:") + print_files(warn, summary.rename_errors) + + return 0 + + +def verify_path(command_context, path): + """Check if the path passed to the command is valid relative path.""" + + def error(text): + command_context.log(logging.ERROR, "esmify", {}, f"[ERROR] {text}") + + if not path.exists(): + error(f"{path} does not exist.") + return False + + if path.is_absolute(): + error("Path must be a relative path from mozilla-central checkout.") + return False + + return True + + +def find_file(path, target): + """Find `target` file in ancestor of path.""" + target_path = path.parent / target + if not target_path.exists(): + if path.parent == path: + return None + + return find_file(path.parent, target) + + return target_path + + +def try_rename_in(command_context, path, target, jsm_name, esm_name, jsm_path): + """Replace the occurrences of `jsm_name` with `esm_name` in `target` + file.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + if type(target) is str: + # Target is specified by filename, that may exist somewhere in + # the jsm's directory or ancestor directories. + target_path = find_file(path, target) + if not target_path: + return False + + # JSM should be specified with relative path in the file. + # + # Single moz.build or jar.mn can contain multiple files with same name. + # Search for relative path. + jsm_relative_path = jsm_path.relative_to(target_path.parent) + jsm_path_str = path_sep_from_native(str(jsm_relative_path)) + else: + # Target is specified by full path. + target_path = target + + # JSM should be specified with full path in the file. + jsm_path_str = path_sep_from_native(str(jsm_path)) + + jsm_path_re = re.compile(r"\b" + jsm_path_str.replace(".", r"\.") + r"\b") + jsm_name_re = re.compile(r"\b" + jsm_name.replace(".", r"\.") + r"\b") + + modified = False + content = "" + with open(target_path, "r") as f: + for line in f: + if jsm_path_re.search(line): + modified = True + line = jsm_name_re.sub(esm_name, line) + + content += line + + if modified: + info(f" {str(target_path)}") + info(f" {jsm_name} => {esm_name}") + with open(target_path, "w", newline="\n") as f: + f.write(content) + + return True + + +def try_rename_uri_in(command_context, target, jsm_name, esm_name, jsm_uri, esm_uri): + """Replace the occurrences of `jsm_uri` with `esm_uri` in `target` file.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + modified = False + content = "" + with open(target, "r") as f: + for line in f: + if jsm_uri in line: + modified = True + line = line.replace(jsm_uri, esm_uri) + + content += line + + if modified: + info(f" {str(target)}") + info(f" {jsm_name} => {esm_name}") + with open(target, "w", newline="\n") as f: + f.write(content) + + return True + + +def try_rename_components_conf(command_context, path, jsm_name, esm_name): + """Replace the occurrences of `jsm_name` with `esm_name` in components.conf + file.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + target_path = find_file(path, "components.conf") + if not target_path: + return False + + # Unlike try_rename_in, components.conf contains the URL instead of + # relative path, and also there are no known files with same name. + # Simply replace the filename. + + with open(target_path, "r") as f: + content = f.read() + + prop_re = re.compile( + "[\"']jsm[\"']:(.*)" + r"\b" + jsm_name.replace(".", r"\.") + r"\b" + ) + + if not prop_re.search(content): + return False + + info(f" {str(target_path)}") + info(f" {jsm_name} => {esm_name}") + + content = prop_re.sub(r"'esModule':\1" + esm_name, content) + with open(target_path, "w", newline="\n") as f: + f.write(content) + + return True + + +def esmify_name(name): + return re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", name) + + +def esmify_path(jsm_path): + jsm_name = jsm_path.name + esm_name = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_name) + esm_path = jsm_path.parent / esm_name + return esm_path + + +path_to_uri_map = None + + +def load_path_to_uri_map(): + global path_to_uri_map + + if path_to_uri_map: + return + + if "ESMIFY_MAP_JSON" in os.environ: + json_map = pathlib.Path(os.environ["ESMIFY_MAP_JSON"]) + else: + json_map = pathlib.Path(__file__).parent / "map.json" + + with open(json_map, "r") as f: + uri_to_path_map = json.loads(f.read()) + + path_to_uri_map = dict() + + for uri, paths in uri_to_path_map.items(): + if type(paths) is str: + paths = [paths] + + for path in paths: + path_to_uri_map[path] = uri + + +def find_jsm_uri(jsm_path): + load_path_to_uri_map() + + path = path_sep_from_native(jsm_path) + + if path in path_to_uri_map: + return path_to_uri_map[path] + + return None + + +def rename_single_file(command_context, vcs_utils, jsm_path, summary): + """Rename `jsm_path` to .sys.mjs, and fix references to the file in build + and test definitions.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + esm_path = esmify_path(jsm_path) + + jsm_name = jsm_path.name + esm_name = esm_path.name + + target_files = [ + ".eslintignore", + "moz.build", + "jar.mn", + "browser.ini", + "browser-common.ini", + "chrome.ini", + "mochitest.ini", + "xpcshell.ini", + "xpcshell-child-process.ini", + "xpcshell-common.ini", + "xpcshell-parent-process.ini", + pathlib.Path("tools", "lint", "eslint.yml"), + pathlib.Path("tools", "lint", "rejected-words.yml"), + ] + + info(f"{jsm_path} => {esm_path}") + + renamed = False + for target in target_files: + if try_rename_in( + command_context, jsm_path, target, jsm_name, esm_name, jsm_path + ): + renamed = True + + if try_rename_components_conf(command_context, jsm_path, jsm_name, esm_name): + renamed = True + + uri_target_files = [ + pathlib.Path( + "browser", "base", "content", "test", "performance", "browser_startup.js" + ), + pathlib.Path( + "browser", + "base", + "content", + "test", + "performance", + "browser_startup_content.js", + ), + pathlib.Path( + "browser", + "base", + "content", + "test", + "performance", + "browser_startup_content_subframe.js", + ), + pathlib.Path( + "toolkit", + "components", + "backgroundtasks", + "tests", + "browser", + "browser_xpcom_graph_wait.js", + ), + ] + + jsm_uri = find_jsm_uri(jsm_path) + if jsm_uri: + esm_uri = re.sub(r"\.(jsm|js|jsm\.js)$", ".sys.mjs", jsm_uri) + + for target in uri_target_files: + if try_rename_uri_in( + command_context, target, jsm_uri, esm_uri, jsm_name, esm_name + ): + renamed = True + + if not renamed: + summary.no_refs.append([jsm_path, None]) + + if not esm_path.exists(): + vcs_utils.rename(jsm_path, esm_path) + else: + summary.rename_errors.append([jsm_path, f"{esm_path} already exists"]) + + return esm_path + + +def rename_jsms(command_context, vcs_utils, jsms, summary): + esms = [] + for jsm in jsms: + esm = rename_single_file(command_context, vcs_utils, jsm, summary) + esms.append(esm) + + return esms + + +npm_prefix = pathlib.Path("tools") / "esmify" +path_from_npm_prefix = pathlib.Path("..") / ".." + + +def setup_jscodeshift(): + """Install jscodeshift.""" + cmd = [ + sys.executable, + "./mach", + "npm", + "install", + "jscodeshift", + "--save-dev", + "--prefix", + str(npm_prefix), + ] + subprocess.run(cmd, check=True) + + +def run_npm_command(args, env, stdin): + cmd = [ + sys.executable, + "./mach", + "npm", + "run", + ] + args + p = subprocess.Popen(cmd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p.stdin.write(stdin) + p.stdin.close() + + ok_files = [] + errors = [] + while True: + line = p.stdout.readline() + if not line: + break + line = line.rstrip().decode() + + if line.startswith(" NOC "): + continue + + print(line) + + m = re.search(r"^ (OKK|ERR) ([^ ]+)(?: (.+))?", line) + if not m: + continue + + result = m.group(1) + # NOTE: path is written from `tools/esmify`. + path = pathlib.Path(m.group(2)).relative_to(path_from_npm_prefix) + error = m.group(3) + + if result == "OKK": + ok_files.append(path) + + if result == "ERR": + errors.append([path, error]) + + if p.wait() != 0: + return [None, None] + + return ok_files, errors + + +def convert_module(jsms, summary): + """Replace EXPORTED_SYMBOLS with export declarations, and replace + ChromeUtils.importESModule with static import as much as possible, + and return the list of successfully rewritten files.""" + + if len(jsms) == 0: + return [] + + env = os.environ.copy() + + stdin = "\n".join(map(str, paths_from_npm_prefix(jsms))).encode() + + ok_files, errors = run_npm_command( + [ + "convert_module", + "--prefix", + str(npm_prefix), + ], + env=env, + stdin=stdin, + ) + + if ok_files is None and errors is None: + return None + + summary.convert_errors.extend(errors) + + return ok_files + + +def rewrite_imports(jss, prefix, summary): + """Replace import calls for JSM with import calls for ESM or static import + for ESM.""" + + if len(jss) == 0: + return [] + + env = os.environ.copy() + env["ESMIFY_TARGET_PREFIX"] = prefix + + stdin = "\n".join(map(str, paths_from_npm_prefix(jss))).encode() + + ok_files, errors = run_npm_command( + [ + "rewrite_imports", + "--prefix", + str(npm_prefix), + ], + env=env, + stdin=stdin, + ) + + if ok_files is None and errors is None: + return None + + summary.import_errors.extend(errors) + + return ok_files + + +def paths_from_npm_prefix(paths): + """Convert relative path from mozilla-central to relative path from + tools/esmify.""" + return list(map(lambda path: path_from_npm_prefix / path, paths)) + + +def eslint_fix(command_context, files): + """Auto format files.""" + + def info(text): + command_context.log(logging.INFO, "esmify", {}, f"[INFO] {text}") + + if len(files) == 0: + return + + remaining = files[0:] + + # There can be too many files for single command line, perform by chunk. + max_files = 16 + while len(remaining) > max_files: + info(f"{len(remaining)} files remaining") + + chunk = remaining[0:max_files] + remaining = remaining[max_files:] + + cmd = [sys.executable, "./mach", "eslint", "--fix"] + chunk + subprocess.run(cmd, check=True) + + info(f"{len(remaining)} files remaining") + chunk = remaining + cmd = [sys.executable, "./mach", "eslint", "--fix"] + chunk + subprocess.run(cmd, check=True) diff --git a/tools/esmify/map.json b/tools/esmify/map.json new file mode 100644 index 0000000000..1a8aaad7bb --- /dev/null +++ b/tools/esmify/map.json @@ -0,0 +1,1154 @@ +{ + "chrome://devtools-startup/content/DevToolsShim.jsm": "devtools/startup/DevToolsShim.jsm", + "chrome://global/content/tabprompts.jsm": "toolkit/components/prompts/content/tabprompts.jsm", + "chrome://mochikit/content/ShutdownLeaksCollector.jsm": "testing/mochitest/ShutdownLeaksCollector.jsm", + "chrome://mochikit/content/tests/SimpleTest/StructuredLog.jsm": "testing/modules/StructuredLog.jsm", + "chrome://mochitests/content/browser/accessible/tests/browser/Common.jsm": "accessible/tests/browser/Common.jsm", + "chrome://mochitests/content/browser/accessible/tests/browser/Layout.jsm": "accessible/tests/browser/Layout.jsm", + "chrome://mochitests/content/browser/devtools/client/shared/sourceeditor/test/CodeMirrorTestActors.jsm": "devtools/client/shared/sourceeditor/test/CodeMirrorTestActors.jsm", + "chrome://mochitests/content/chrome/docshell/test/chrome/DocShellHelpers.jsm": "docshell/test/chrome/DocShellHelpers.jsm", + "chrome://mochitests/content/chrome/dom/console/tests/console.jsm": "dom/console/tests/console.jsm", + "chrome://mochitests/content/chrome/dom/network/tests/tcpsocket_test.jsm": "dom/network/tests/tcpsocket_test.jsm", + "chrome://mochitests/content/chrome/dom/url/tests/file_worker_url.jsm": "dom/url/tests/file_worker_url.jsm", + "chrome://mochitests/content/chrome/dom/url/tests/test_bug883784.jsm": "dom/url/tests/test_bug883784.jsm", + "chrome://mochitests/content/chrome/dom/workers/test/WorkerTest.jsm": "dom/workers/test/WorkerTest.jsm", + "chrome://pocket/content/Pocket.jsm": "browser/components/pocket/content/Pocket.jsm", + "chrome://pocket/content/SaveToPocket.jsm": "browser/components/pocket/content/SaveToPocket.jsm", + "chrome://pocket/content/pktApi.jsm": "browser/components/pocket/content/pktApi.jsm", + "chrome://pocket/content/pktTelemetry.jsm": "browser/components/pocket/content/pktTelemetry.jsm", + "chrome://remote/content/server/HTTPD.jsm": "netwerk/test/httpserver/httpd.js", + "resource:///actors/ASRouterChild.jsm": "browser/components/newtab/actors/ASRouterChild.jsm", + "resource:///actors/ASRouterParent.jsm": "browser/components/newtab/actors/ASRouterParent.jsm", + "resource:///actors/AboutLoginsChild.jsm": "browser/components/aboutlogins/AboutLoginsChild.jsm", + "resource:///actors/AboutLoginsParent.jsm": "browser/components/aboutlogins/AboutLoginsParent.jsm", + "resource:///actors/AboutNewTabChild.jsm": "browser/actors/AboutNewTabChild.jsm", + "resource:///actors/AboutNewTabParent.jsm": "browser/actors/AboutNewTabParent.jsm", + "resource:///actors/AboutPluginsChild.jsm": "browser/actors/AboutPluginsChild.jsm", + "resource:///actors/AboutPluginsParent.jsm": "browser/actors/AboutPluginsParent.jsm", + "resource:///actors/AboutPocketChild.jsm": "browser/actors/AboutPocketChild.jsm", + "resource:///actors/AboutPocketParent.jsm": "browser/actors/AboutPocketParent.jsm", + "resource:///actors/AboutPrivateBrowsingChild.jsm": "browser/actors/AboutPrivateBrowsingChild.jsm", + "resource:///actors/AboutPrivateBrowsingParent.jsm": "browser/actors/AboutPrivateBrowsingParent.jsm", + "resource:///actors/AboutProtectionsChild.jsm": "browser/actors/AboutProtectionsChild.jsm", + "resource:///actors/AboutProtectionsParent.jsm": "browser/actors/AboutProtectionsParent.jsm", + "resource:///actors/AboutReaderChild.jsm": "browser/actors/AboutReaderChild.jsm", + "resource:///actors/AboutReaderParent.jsm": "browser/actors/AboutReaderParent.jsm", + "resource:///actors/AboutTabCrashedChild.jsm": "browser/actors/AboutTabCrashedChild.jsm", + "resource:///actors/AboutTabCrashedParent.jsm": "browser/actors/AboutTabCrashedParent.jsm", + "resource:///actors/AboutWelcomeChild.jsm": "browser/components/newtab/aboutwelcome/AboutWelcomeChild.jsm", + "resource:///actors/AboutWelcomeParent.jsm": "browser/components/newtab/aboutwelcome/AboutWelcomeParent.jsm", + "resource:///actors/BlockedSiteChild.jsm": "browser/actors/BlockedSiteChild.jsm", + "resource:///actors/BlockedSiteParent.jsm": "browser/actors/BlockedSiteParent.jsm", + "resource:///actors/BrowserProcessChild.jsm": "browser/actors/BrowserProcessChild.jsm", + "resource:///actors/BrowserTabChild.jsm": "browser/actors/BrowserTabChild.jsm", + "resource:///actors/BrowserTabParent.jsm": "browser/actors/BrowserTabParent.jsm", + "resource:///actors/ClickHandlerChild.jsm": "browser/actors/ClickHandlerChild.jsm", + "resource:///actors/ClickHandlerParent.jsm": "browser/actors/ClickHandlerParent.jsm", + "resource:///actors/ContentDelegateChild.jsm": "mobile/android/actors/ContentDelegateChild.jsm", + "resource:///actors/ContentDelegateParent.jsm": "mobile/android/actors/ContentDelegateParent.jsm", + "resource:///actors/ContentSearchChild.jsm": "browser/actors/ContentSearchChild.jsm", + "resource:///actors/ContentSearchParent.jsm": "browser/actors/ContentSearchParent.jsm", + "resource:///actors/ContextMenuChild.jsm": "browser/actors/ContextMenuChild.jsm", + "resource:///actors/ContextMenuParent.jsm": "browser/actors/ContextMenuParent.jsm", + "resource:///actors/DOMFullscreenChild.jsm": "browser/actors/DOMFullscreenChild.jsm", + "resource:///actors/DOMFullscreenParent.jsm": "browser/actors/DOMFullscreenParent.jsm", + "resource:///actors/DecoderDoctorChild.jsm": "browser/actors/DecoderDoctorChild.jsm", + "resource:///actors/DecoderDoctorParent.jsm": "browser/actors/DecoderDoctorParent.jsm", + "resource:///actors/EncryptedMediaChild.jsm": "browser/actors/EncryptedMediaChild.jsm", + "resource:///actors/EncryptedMediaParent.jsm": "browser/actors/EncryptedMediaParent.jsm", + "resource:///actors/FormValidationChild.jsm": "browser/actors/FormValidationChild.jsm", + "resource:///actors/FormValidationParent.jsm": "browser/actors/FormValidationParent.jsm", + "resource:///actors/GeckoViewAutoFillChild.jsm": "mobile/android/actors/GeckoViewAutoFillChild.jsm", + "resource:///actors/GeckoViewAutoFillParent.jsm": "mobile/android/actors/GeckoViewAutoFillParent.jsm", + "resource:///actors/GeckoViewContentChild.jsm": "mobile/android/actors/GeckoViewContentChild.jsm", + "resource:///actors/GeckoViewContentParent.jsm": "mobile/android/actors/GeckoViewContentParent.jsm", + "resource:///actors/GeckoViewFormValidationChild.jsm": "mobile/android/actors/GeckoViewFormValidationChild.jsm", + "resource:///actors/GeckoViewPermissionChild.jsm": "mobile/android/actors/GeckoViewPermissionChild.jsm", + "resource:///actors/GeckoViewPermissionParent.jsm": "mobile/android/actors/GeckoViewPermissionParent.jsm", + "resource:///actors/GeckoViewPermissionProcessChild.jsm": "mobile/android/actors/GeckoViewPermissionProcessChild.jsm", + "resource:///actors/GeckoViewPermissionProcessParent.jsm": "mobile/android/actors/GeckoViewPermissionProcessParent.jsm", + "resource:///actors/GeckoViewPromptChild.jsm": "mobile/android/actors/GeckoViewPromptChild.jsm", + "resource:///actors/GeckoViewPrompterChild.jsm": "mobile/android/actors/GeckoViewPrompterChild.jsm", + "resource:///actors/GeckoViewPrompterParent.jsm": "mobile/android/actors/GeckoViewPrompterParent.jsm", + "resource:///actors/GeckoViewSettingsChild.jsm": "mobile/android/actors/GeckoViewSettingsChild.jsm", + "resource:///actors/InteractionsChild.jsm": "browser/components/places/InteractionsChild.jsm", + "resource:///actors/InteractionsParent.jsm": "browser/components/places/InteractionsParent.jsm", + "resource:///actors/LightweightThemeChild.jsm": "browser/actors/LightweightThemeChild.jsm", + "resource:///actors/LinkHandlerChild.jsm": "browser/actors/LinkHandlerChild.jsm", + "resource:///actors/LinkHandlerParent.jsm": "browser/actors/LinkHandlerParent.jsm", + "resource:///actors/LoadURIDelegateChild.jsm": "mobile/android/actors/LoadURIDelegateChild.jsm", + "resource:///actors/LoadURIDelegateParent.jsm": "mobile/android/actors/LoadURIDelegateParent.jsm", + "resource:///actors/MediaControlDelegateChild.jsm": "mobile/android/actors/MediaControlDelegateChild.jsm", + "resource:///actors/MediaControlDelegateParent.jsm": "mobile/android/actors/MediaControlDelegateParent.jsm", + "resource:///actors/PageDataChild.jsm": "browser/components/pagedata/PageDataChild.jsm", + "resource:///actors/PageDataParent.jsm": "browser/components/pagedata/PageDataParent.jsm", + "resource:///actors/PageInfoChild.jsm": "browser/actors/PageInfoChild.jsm", + "resource:///actors/PageStyleChild.jsm": "browser/actors/PageStyleChild.jsm", + "resource:///actors/PageStyleParent.jsm": "browser/actors/PageStyleParent.jsm", + "resource:///actors/PluginChild.jsm": "browser/actors/PluginChild.jsm", + "resource:///actors/PluginParent.jsm": "browser/actors/PluginParent.jsm", + "resource:///actors/PointerLockChild.jsm": "browser/actors/PointerLockChild.jsm", + "resource:///actors/PointerLockParent.jsm": "browser/actors/PointerLockParent.jsm", + "resource:///actors/ProgressDelegateChild.jsm": "mobile/android/actors/ProgressDelegateChild.jsm", + "resource:///actors/ProgressDelegateParent.jsm": "mobile/android/actors/ProgressDelegateParent.jsm", + "resource:///actors/PromptParent.jsm": "browser/actors/PromptParent.jsm", + "resource:///actors/RFPHelperChild.jsm": "browser/actors/RFPHelperChild.jsm", + "resource:///actors/RFPHelperParent.jsm": "browser/actors/RFPHelperParent.jsm", + "resource:///actors/RefreshBlockerChild.jsm": "browser/actors/RefreshBlockerChild.jsm", + "resource:///actors/RefreshBlockerParent.jsm": "browser/actors/RefreshBlockerParent.jsm", + "resource:///actors/ScreenshotsComponentChild.jsm": "browser/actors/ScreenshotsComponentChild.jsm", + "resource:///actors/ScrollDelegateChild.jsm": "mobile/android/actors/ScrollDelegateChild.jsm", + "resource:///actors/ScrollDelegateParent.jsm": "mobile/android/actors/ScrollDelegateParent.jsm", + "resource:///actors/SearchSERPTelemetryChild.jsm": "browser/actors/SearchSERPTelemetryChild.jsm", + "resource:///actors/SearchSERPTelemetryParent.jsm": "browser/actors/SearchSERPTelemetryParent.jsm", + "resource:///actors/SelectionActionDelegateChild.jsm": "mobile/android/actors/SelectionActionDelegateChild.jsm", + "resource:///actors/SelectionActionDelegateParent.jsm": "mobile/android/actors/SelectionActionDelegateParent.jsm", + "resource:///actors/SwitchDocumentDirectionChild.jsm": "browser/actors/SwitchDocumentDirectionChild.jsm", + "resource:///actors/WebRTCChild.jsm": "browser/actors/WebRTCChild.jsm", + "resource:///actors/WebRTCParent.jsm": "browser/actors/WebRTCParent.jsm", + "resource:///modules/360seMigrationUtils.jsm": "browser/components/migration/360seMigrationUtils.jsm", + "resource:///modules/AboutDebuggingRegistration.jsm": "devtools/startup/AboutDebuggingRegistration.jsm", + "resource:///modules/AboutDevToolsToolboxRegistration.jsm": "devtools/startup/AboutDevToolsToolboxRegistration.jsm", + "resource:///modules/AboutNewTab.jsm": "browser/modules/AboutNewTab.jsm", + "resource:///modules/AboutNewTabService.jsm": "browser/components/newtab/AboutNewTabService.jsm", + "resource://gre/modules/AppUpdater.jsm": "toolkit/mozapps/update/AppUpdater.jsm", + "resource:///modules/AsyncTabSwitcher.jsm": "browser/modules/AsyncTabSwitcher.jsm", + "resource:///modules/AttributionCode.jsm": "browser/components/attribution/AttributionCode.jsm", + "resource:///modules/BrowserContentHandler.jsm": "browser/components/BrowserContentHandler.jsm", + "resource:///modules/BrowserGlue.jsm": "browser/components/BrowserGlue.jsm", + "resource:///modules/BrowserSearchTelemetry.jsm": "browser/components/search/BrowserSearchTelemetry.jsm", + "resource:///modules/BrowserUIUtils.jsm": "browser/modules/BrowserUIUtils.jsm", + "resource:///modules/BrowserWindowTracker.jsm": "browser/modules/BrowserWindowTracker.jsm", + "resource:///modules/BuiltInThemeConfig.jsm": "browser/themes/BuiltInThemeConfig.jsm", + "resource:///modules/BuiltInThemes.jsm": "browser/themes/BuiltInThemes.jsm", + "resource:///modules/CaptiveDetect.jsm": "toolkit/components/captivedetect/CaptiveDetect.jsm", + "resource:///modules/ChromeMacOSLoginCrypto.jsm": "browser/components/migration/ChromeMacOSLoginCrypto.jsm", + "resource:///modules/ChromeMigrationUtils.jsm": "browser/components/migration/ChromeMigrationUtils.jsm", + "resource:///modules/ChromeProfileMigrator.jsm": "browser/components/migration/ChromeProfileMigrator.jsm", + "resource:///modules/ChromeWindowsLoginCrypto.jsm": "browser/components/migration/ChromeWindowsLoginCrypto.jsm", + "resource:///modules/CommonNames.jsm": "browser/components/places/CommonNames.jsm", + "resource:///modules/ContentCrashHandlers.jsm": "browser/modules/ContentCrashHandlers.jsm", + "resource:///modules/CustomizableUI.jsm": "browser/components/customizableui/CustomizableUI.jsm", + "resource:///modules/CustomizableWidgets.jsm": "browser/components/customizableui/CustomizableWidgets.jsm", + "resource:///modules/CustomizeMode.jsm": "browser/components/customizableui/CustomizeMode.jsm", + "resource:///modules/DevToolsStartup.jsm": "devtools/startup/DevToolsStartup.jsm", + "resource:///modules/Discovery.jsm": "browser/modules/Discovery.jsm", + "resource:///modules/DoHConfig.jsm": "browser/components/doh/DoHConfig.jsm", + "resource:///modules/DoHController.jsm": "browser/components/doh/DoHController.jsm", + "resource:///modules/DoHHeuristics.jsm": "browser/components/doh/DoHHeuristics.jsm", + "resource:///modules/DomainGroupBuilder.jsm": "browser/components/places/DomainGroupBuilder.jsm", + "resource:///modules/DownloadSpamProtection.jsm": "browser/components/downloads/DownloadSpamProtection.jsm", + "resource:///modules/DownloadsCommon.jsm": "browser/components/downloads/DownloadsCommon.jsm", + "resource:///modules/DownloadsMacFinderProgress.jsm": "browser/components/downloads/DownloadsMacFinderProgress.jsm", + "resource:///modules/DownloadsTaskbar.jsm": "browser/components/downloads/DownloadsTaskbar.jsm", + "resource:///modules/DownloadsViewUI.jsm": "browser/components/downloads/DownloadsViewUI.jsm", + "resource:///modules/DownloadsViewableInternally.jsm": "browser/components/downloads/DownloadsViewableInternally.jsm", + "resource:///modules/DragPositionManager.jsm": "browser/components/customizableui/DragPositionManager.jsm", + "resource:///modules/ESEDBReader.jsm": "browser/components/migration/ESEDBReader.jsm", + "resource:///modules/EdgeProfileMigrator.jsm": "browser/components/migration/EdgeProfileMigrator.jsm", + "resource:///modules/EveryWindow.jsm": "browser/modules/EveryWindow.jsm", + "resource:///modules/ExtensionControlledPopup.jsm": "browser/components/extensions/ExtensionControlledPopup.jsm", + "resource:///modules/ExtensionPopups.jsm": "browser/components/extensions/ExtensionPopups.jsm", + "resource:///modules/ExtensionsUI.jsm": "browser/modules/ExtensionsUI.jsm", + "resource:///modules/FaviconLoader.jsm": "browser/modules/FaviconLoader.jsm", + "resource:///modules/FirefoxProfileMigrator.jsm": "browser/components/migration/FirefoxProfileMigrator.jsm", + "resource:///modules/HeadlessShell.jsm": "browser/components/shell/HeadlessShell.jsm", + "resource:///modules/HomePage.jsm": "browser/modules/HomePage.jsm", + "resource:///modules/IEProfileMigrator.jsm": "browser/components/migration/IEProfileMigrator.jsm", + "resource:///modules/InstallerPrefs.jsm": "browser/components/installerprefs/InstallerPrefs.jsm", + "resource:///modules/Interactions.jsm": "browser/components/places/Interactions.jsm", + "resource:///modules/InteractionsBlocklist.jsm": "browser/components/places/InteractionsBlocklist.jsm", + "resource:///modules/LaterRun.jsm": "browser/modules/LaterRun.jsm", + "resource:///modules/LoginBreaches.jsm": "browser/components/aboutlogins/LoginBreaches.jsm", + "resource:///modules/MSMigrationUtils.jsm": "browser/components/migration/MSMigrationUtils.jsm", + "resource:///modules/MacAttribution.jsm": "browser/components/attribution/MacAttribution.jsm", + "resource:///modules/MacTouchBar.jsm": "browser/components/touchbar/MacTouchBar.jsm", + "resource:///modules/MigrationUtils.jsm": "browser/components/migration/MigrationUtils.jsm", + "resource:///modules/NewTabPagePreloading.jsm": "browser/modules/NewTabPagePreloading.jsm", + "resource:///modules/OpenInTabsUtils.jsm": "browser/modules/OpenInTabsUtils.jsm", + "resource:///modules/PageActions.jsm": "browser/modules/PageActions.jsm", + "resource:///modules/PanelMultiView.jsm": "browser/components/customizableui/PanelMultiView.jsm", + "resource:///modules/PartnerLinkAttribution.jsm": "browser/modules/PartnerLinkAttribution.jsm", + "resource:///modules/PingCentre.jsm": "browser/modules/PingCentre.jsm", + "resource:///modules/PinnedGroupBuilder.jsm": "browser/components/places/PinnedGroupBuilder.jsm", + "resource:///modules/PlacesUIUtils.jsm": "browser/components/places/PlacesUIUtils.jsm", + "resource:///modules/ProcessHangMonitor.jsm": "browser/modules/ProcessHangMonitor.jsm", + "resource:///modules/ProfileMigrator.jsm": "browser/components/migration/ProfileMigrator.jsm", + "resource:///modules/PromptCollection.jsm": "browser/components/prompts/PromptCollection.jsm", + "resource:///modules/SafariProfileMigrator.jsm": "browser/components/migration/SafariProfileMigrator.jsm", + "resource:///modules/Sanitizer.jsm": "browser/modules/Sanitizer.jsm", + "resource:///modules/ScreenshotChild.jsm": "browser/components/shell/ScreenshotChild.jsm", + "resource:///modules/ScreenshotsOverlayChild.jsm": "browser/components/screenshots/ScreenshotsOverlayChild.jsm", + "resource:///modules/ScreenshotsUtils.jsm": "browser/components/screenshots/ScreenshotsUtils.jsm", + "resource:///modules/SearchOneOffs.jsm": "browser/components/search/SearchOneOffs.jsm", + "resource:///modules/SearchSERPTelemetry.jsm": "browser/components/search/SearchSERPTelemetry.jsm", + "resource:///modules/SearchUIUtils.jsm": "browser/components/search/SearchUIUtils.jsm", + "resource:///modules/SearchWidgetTracker.jsm": "browser/components/customizableui/SearchWidgetTracker.jsm", + "resource:///modules/SelectionChangedMenulist.jsm": "browser/modules/SelectionChangedMenulist.jsm", + "resource:///modules/ShellService.jsm": "browser/components/shell/ShellService.jsm", + "resource:///modules/SiteDataManager.jsm": "browser/modules/SiteDataManager.jsm", + "resource:///modules/SitePermissions.jsm": "browser/modules/SitePermissions.jsm", + "resource:///modules/SnapshotGroups.jsm": "browser/components/places/SnapshotGroups.jsm", + "resource:///modules/SnapshotMonitor.jsm": "browser/components/places/SnapshotMonitor.jsm", + "resource:///modules/SnapshotScorer.jsm": "browser/components/places/SnapshotScorer.jsm", + "resource:///modules/SnapshotSelector.jsm": "browser/components/places/SnapshotSelector.jsm", + "resource:///modules/Snapshots.jsm": "browser/components/places/Snapshots.jsm", + "resource:///modules/StartupRecorder.jsm": "browser/components/StartupRecorder.jsm", + "resource:///modules/TRRPerformance.jsm": "browser/components/doh/TRRPerformance.jsm", + "resource:///modules/TabUnloader.jsm": "browser/modules/TabUnloader.jsm", + "resource:///modules/TabsList.jsm": "browser/modules/TabsList.jsm", + "resource:///modules/ThemeVariableMap.jsm": "browser/themes/ThemeVariableMap.jsm", + "resource:///modules/TransientPrefs.jsm": "browser/modules/TransientPrefs.jsm", + "resource:///modules/UITour.jsm": "browser/components/uitour/UITour.jsm", + "resource:///modules/UITourChild.jsm": "browser/components/uitour/UITourChild.jsm", + "resource:///modules/UITourParent.jsm": "browser/components/uitour/UITourParent.jsm", + "resource:///modules/UnitConverterSimple.jsm": "browser/components/urlbar/unitconverters/UnitConverterSimple.jsm", + "resource:///modules/UnitConverterTemperature.jsm": "browser/components/urlbar/unitconverters/UnitConverterTemperature.jsm", + "resource:///modules/UnitConverterTimezone.jsm": "browser/components/urlbar/unitconverters/UnitConverterTimezone.jsm", + "resource:///modules/UrlbarController.jsm": "browser/components/urlbar/UrlbarController.jsm", + "resource:///modules/UrlbarEventBufferer.jsm": "browser/components/urlbar/UrlbarEventBufferer.jsm", + "resource:///modules/UrlbarInput.jsm": "browser/components/urlbar/UrlbarInput.jsm", + "resource:///modules/UrlbarMuxerUnifiedComplete.jsm": "browser/components/urlbar/UrlbarMuxerUnifiedComplete.jsm", + "resource:///modules/UrlbarPrefs.jsm": "browser/components/urlbar/UrlbarPrefs.jsm", + "resource:///modules/UrlbarProviderAboutPages.jsm": "browser/components/urlbar/UrlbarProviderAboutPages.jsm", + "resource:///modules/UrlbarProviderAliasEngines.jsm": "browser/components/urlbar/UrlbarProviderAliasEngines.jsm", + "resource:///modules/UrlbarProviderAutofill.jsm": "browser/components/urlbar/UrlbarProviderAutofill.jsm", + "resource:///modules/UrlbarProviderBookmarkKeywords.jsm": "browser/components/urlbar/UrlbarProviderBookmarkKeywords.jsm", + "resource:///modules/UrlbarProviderCalculator.jsm": "browser/components/urlbar/UrlbarProviderCalculator.jsm", + "resource:///modules/UrlbarProviderExtension.jsm": "browser/components/urlbar/UrlbarProviderExtension.jsm", + "resource:///modules/UrlbarProviderHeuristicFallback.jsm": "browser/components/urlbar/UrlbarProviderHeuristicFallback.jsm", + "resource:///modules/UrlbarProviderInputHistory.jsm": "browser/components/urlbar/UrlbarProviderInputHistory.jsm", + "resource:///modules/UrlbarProviderInterventions.jsm": "browser/components/urlbar/UrlbarProviderInterventions.jsm", + "resource:///modules/UrlbarProviderOmnibox.jsm": "browser/components/urlbar/UrlbarProviderOmnibox.jsm", + "resource:///modules/UrlbarProviderOpenTabs.jsm": "browser/components/urlbar/UrlbarProviderOpenTabs.jsm", + "resource:///modules/UrlbarProviderPlaces.jsm": "browser/components/urlbar/UrlbarProviderPlaces.jsm", + "resource:///modules/UrlbarProviderPreloadedSites.jsm": "browser/components/urlbar/UrlbarProviderPreloadedSites.jsm", + "resource:///modules/UrlbarProviderPrivateSearch.jsm": "browser/components/urlbar/UrlbarProviderPrivateSearch.jsm", + "resource:///modules/UrlbarProviderQuickSuggest.jsm": "browser/components/urlbar/UrlbarProviderQuickSuggest.jsm", + "resource:///modules/UrlbarProviderRemoteTabs.jsm": "browser/components/urlbar/UrlbarProviderRemoteTabs.jsm", + "resource:///modules/UrlbarProviderSearchSuggestions.jsm": "browser/components/urlbar/UrlbarProviderSearchSuggestions.jsm", + "resource:///modules/UrlbarProviderSearchTips.jsm": "browser/components/urlbar/UrlbarProviderSearchTips.jsm", + "resource:///modules/UrlbarProviderTabToSearch.jsm": "browser/components/urlbar/UrlbarProviderTabToSearch.jsm", + "resource:///modules/UrlbarProviderTokenAliasEngines.jsm": "browser/components/urlbar/UrlbarProviderTokenAliasEngines.jsm", + "resource:///modules/UrlbarProviderTopSites.jsm": "browser/components/urlbar/UrlbarProviderTopSites.jsm", + "resource:///modules/UrlbarProviderUnitConversion.jsm": "browser/components/urlbar/UrlbarProviderUnitConversion.jsm", + "resource:///modules/UrlbarProvidersManager.jsm": "browser/components/urlbar/UrlbarProvidersManager.jsm", + "resource:///modules/UrlbarResult.jsm": "browser/components/urlbar/UrlbarResult.jsm", + "resource:///modules/UrlbarSearchOneOffs.jsm": "browser/components/urlbar/UrlbarSearchOneOffs.jsm", + "resource:///modules/UrlbarSearchUtils.jsm": "browser/components/urlbar/UrlbarSearchUtils.jsm", + "resource:///modules/UrlbarTokenizer.jsm": "browser/components/urlbar/UrlbarTokenizer.jsm", + "resource:///modules/UrlbarUtils.jsm": "browser/components/urlbar/UrlbarUtils.jsm", + "resource:///modules/UrlbarValueFormatter.jsm": "browser/components/urlbar/UrlbarValueFormatter.jsm", + "resource:///modules/UrlbarView.jsm": "browser/components/urlbar/UrlbarView.jsm", + "resource:///modules/WebProtocolHandlerRegistrar.jsm": "browser/components/protocolhandler/WebProtocolHandlerRegistrar.jsm", + "resource:///modules/Windows8WindowFrameColor.jsm": "browser/themes/Windows8WindowFrameColor.jsm", + "resource:///modules/WindowsJumpLists.jsm": "browser/modules/WindowsJumpLists.jsm", + "resource:///modules/WindowsPreviewPerTab.jsm": "browser/modules/WindowsPreviewPerTab.jsm", + "resource:///modules/ZoomUI.jsm": "browser/modules/ZoomUI.jsm", + "resource:///modules/distribution.js": "browser/components/distribution.js", + "resource:///modules/pagedata/OpenGraphPageData.jsm": "browser/components/pagedata/OpenGraphPageData.jsm", + "resource:///modules/pagedata/PageDataSchema.jsm": "browser/components/pagedata/PageDataSchema.jsm", + "resource:///modules/pagedata/PageDataService.jsm": "browser/components/pagedata/PageDataService.jsm", + "resource:///modules/pagedata/SchemaOrgPageData.jsm": "browser/components/pagedata/SchemaOrgPageData.jsm", + "resource:///modules/pagedata/TwitterPageData.jsm": "browser/components/pagedata/TwitterPageData.jsm", + "resource:///modules/policies/BookmarksPolicies.jsm": "browser/components/enterprisepolicies/helpers/BookmarksPolicies.jsm", + "resource:///modules/policies/Policies.jsm": "browser/components/enterprisepolicies/Policies.jsm", + "resource:///modules/policies/ProxyPolicies.jsm": "browser/components/enterprisepolicies/helpers/ProxyPolicies.jsm", + "resource:///modules/policies/WebsiteFilter.jsm": "browser/components/enterprisepolicies/helpers/WebsiteFilter.jsm", + "resource:///modules/policies/schema.jsm": "browser/components/enterprisepolicies/schemas/schema.jsm", + "resource:///modules/sessionstore/ContentRestore.jsm": "browser/components/sessionstore/ContentRestore.jsm", + "resource:///modules/sessionstore/ContentSessionStore.jsm": "browser/components/sessionstore/ContentSessionStore.jsm", + "resource:///modules/sessionstore/GlobalState.jsm": "browser/components/sessionstore/GlobalState.jsm", + "resource:///modules/sessionstore/RecentlyClosedTabsAndWindowsMenuUtils.jsm": "browser/components/sessionstore/RecentlyClosedTabsAndWindowsMenuUtils.jsm", + "resource:///modules/sessionstore/RunState.jsm": "browser/components/sessionstore/RunState.jsm", + "resource:///modules/sessionstore/SessionCookies.jsm": "browser/components/sessionstore/SessionCookies.jsm", + "resource:///modules/sessionstore/SessionFile.jsm": "browser/components/sessionstore/SessionFile.jsm", + "resource:///modules/sessionstore/SessionMigration.jsm": "browser/components/sessionstore/SessionMigration.jsm", + "resource:///modules/sessionstore/SessionSaver.jsm": "browser/components/sessionstore/SessionSaver.jsm", + "resource:///modules/sessionstore/SessionStartup.jsm": "browser/components/sessionstore/SessionStartup.jsm", + "resource:///modules/sessionstore/SessionStore.jsm": "browser/components/sessionstore/SessionStore.jsm", + "resource:///modules/sessionstore/SessionWriter.jsm": "browser/components/sessionstore/SessionWriter.jsm", + "resource:///modules/sessionstore/StartupPerformance.jsm": "browser/components/sessionstore/StartupPerformance.jsm", + "resource:///modules/sessionstore/TabAttributes.jsm": "browser/components/sessionstore/TabAttributes.jsm", + "resource:///modules/sessionstore/TabState.jsm": "browser/components/sessionstore/TabState.jsm", + "resource:///modules/sessionstore/TabStateCache.jsm": "browser/components/sessionstore/TabStateCache.jsm", + "resource:///modules/sessionstore/TabStateFlusher.jsm": "browser/components/sessionstore/TabStateFlusher.jsm", + "resource:///modules/syncedtabs/EventEmitter.jsm": "browser/components/syncedtabs/EventEmitter.jsm", + "resource:///modules/syncedtabs/SyncedTabsDeckComponent.js": "browser/components/syncedtabs/SyncedTabsDeckComponent.js", + "resource:///modules/syncedtabs/SyncedTabsDeckStore.js": "browser/components/syncedtabs/SyncedTabsDeckStore.js", + "resource:///modules/syncedtabs/SyncedTabsDeckView.js": "browser/components/syncedtabs/SyncedTabsDeckView.js", + "resource:///modules/syncedtabs/SyncedTabsListStore.js": "browser/components/syncedtabs/SyncedTabsListStore.js", + "resource:///modules/syncedtabs/TabListComponent.js": "browser/components/syncedtabs/TabListComponent.js", + "resource:///modules/syncedtabs/TabListView.js": "browser/components/syncedtabs/TabListView.js", + "resource:///modules/syncedtabs/util.js": "browser/components/syncedtabs/util.js", + "resource:///modules/translation/BingTranslator.jsm": "browser/components/translation/BingTranslator.jsm", + "resource:///modules/translation/TranslationChild.jsm": "browser/components/translation/TranslationChild.jsm", + "resource:///modules/translation/TranslationDocument.jsm": "browser/components/translation/TranslationDocument.jsm", + "resource:///modules/translation/TranslationParent.jsm": "browser/components/translation/TranslationParent.jsm", + "resource:///modules/translation/YandexTranslator.jsm": "browser/components/translation/YandexTranslator.jsm", + "resource:///modules/webrtcUI.jsm": "browser/modules/webrtcUI.jsm", + "resource://activity-stream/aboutwelcome/lib/AboutWelcomeDefaults.jsm": "browser/components/newtab/aboutwelcome/lib/AboutWelcomeDefaults.jsm", + "resource://activity-stream/aboutwelcome/lib/AboutWelcomeTelemetry.jsm": "browser/components/newtab/aboutwelcome/lib/AboutWelcomeTelemetry.jsm", + "resource://activity-stream/common/Actions.jsm": "browser/components/newtab/common/Actions.jsm", + "resource://activity-stream/common/ActorConstants.jsm": "browser/components/newtab/common/ActorConstants.jsm", + "resource://activity-stream/common/Dedupe.jsm": "browser/components/newtab/common/Dedupe.jsm", + "resource://activity-stream/common/Reducers.jsm": "browser/components/newtab/common/Reducers.jsm", + "resource://activity-stream/lib/ASRouter.jsm": "browser/components/newtab/lib/ASRouter.jsm", + "resource://activity-stream/lib/ASRouterDefaultConfig.jsm": "browser/components/newtab/lib/ASRouterDefaultConfig.jsm", + "resource://activity-stream/lib/ASRouterNewTabHook.jsm": "browser/components/newtab/lib/ASRouterNewTabHook.jsm", + "resource://activity-stream/lib/ASRouterParentProcessMessageHandler.jsm": "browser/components/newtab/lib/ASRouterParentProcessMessageHandler.jsm", + "resource://activity-stream/lib/ASRouterPreferences.jsm": "browser/components/newtab/lib/ASRouterPreferences.jsm", + "resource://activity-stream/lib/ASRouterTargeting.jsm": "browser/components/newtab/lib/ASRouterTargeting.jsm", + "resource://activity-stream/lib/ASRouterTriggerListeners.jsm": "browser/components/newtab/lib/ASRouterTriggerListeners.jsm", + "resource://activity-stream/lib/AboutPreferences.jsm": "browser/components/newtab/lib/AboutPreferences.jsm", + "resource://activity-stream/lib/ActivityStream.jsm": "browser/components/newtab/lib/ActivityStream.jsm", + "resource://activity-stream/lib/ActivityStreamMessageChannel.jsm": "browser/components/newtab/lib/ActivityStreamMessageChannel.jsm", + "resource://activity-stream/lib/ActivityStreamPrefs.jsm": "browser/components/newtab/lib/ActivityStreamPrefs.jsm", + "resource://activity-stream/lib/ActivityStreamStorage.jsm": "browser/components/newtab/lib/ActivityStreamStorage.jsm", + "resource://activity-stream/lib/CFRMessageProvider.jsm": "browser/components/newtab/lib/CFRMessageProvider.jsm", + "resource://activity-stream/lib/CFRPageActions.jsm": "browser/components/newtab/lib/CFRPageActions.jsm", + "resource://activity-stream/lib/DefaultSites.jsm": "browser/components/newtab/lib/DefaultSites.jsm", + "resource://activity-stream/lib/DiscoveryStreamFeed.jsm": "browser/components/newtab/lib/DiscoveryStreamFeed.jsm", + "resource://activity-stream/lib/DownloadsManager.jsm": "browser/components/newtab/lib/DownloadsManager.jsm", + "resource://activity-stream/lib/FaviconFeed.jsm": "browser/components/newtab/lib/FaviconFeed.jsm", + "resource://activity-stream/lib/FeatureCalloutMessages.jsm": "browser/components/newtab/lib/FeatureCalloutMessages.jsm", + "resource://activity-stream/lib/FilterAdult.jsm": "browser/components/newtab/lib/FilterAdult.jsm", + "resource://activity-stream/lib/HighlightsFeed.jsm": "browser/components/newtab/lib/HighlightsFeed.jsm", + "resource://activity-stream/lib/InfoBar.jsm": "browser/components/newtab/lib/InfoBar.jsm", + "resource://activity-stream/lib/LinksCache.jsm": "browser/components/newtab/lib/LinksCache.jsm", + "resource://activity-stream/lib/MomentsPageHub.jsm": "browser/components/newtab/lib/MomentsPageHub.jsm", + "resource://activity-stream/lib/NewTabInit.jsm": "browser/components/newtab/lib/NewTabInit.jsm", + "resource://activity-stream/lib/OnboardingMessageProvider.jsm": "browser/components/newtab/lib/OnboardingMessageProvider.jsm", + "resource://activity-stream/lib/PanelTestProvider.jsm": "browser/components/newtab/lib/PanelTestProvider.jsm", + "resource://activity-stream/lib/PersistentCache.jsm": "browser/components/newtab/lib/PersistentCache.jsm", + "resource://activity-stream/lib/PersonalityProvider/NaiveBayesTextTagger.jsm": "browser/components/newtab/lib/PersonalityProvider/NaiveBayesTextTagger.jsm", + "resource://activity-stream/lib/PersonalityProvider/NmfTextTagger.jsm": "browser/components/newtab/lib/PersonalityProvider/NmfTextTagger.jsm", + "resource://activity-stream/lib/PersonalityProvider/PersonalityProvider.jsm": "browser/components/newtab/lib/PersonalityProvider/PersonalityProvider.jsm", + "resource://activity-stream/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm": "browser/components/newtab/lib/PersonalityProvider/PersonalityProviderWorkerClass.jsm", + "resource://activity-stream/lib/PersonalityProvider/RecipeExecutor.jsm": "browser/components/newtab/lib/PersonalityProvider/RecipeExecutor.jsm", + "resource://activity-stream/lib/PersonalityProvider/Tokenize.jsm": "browser/components/newtab/lib/PersonalityProvider/Tokenize.jsm", + "resource://activity-stream/lib/PlacesFeed.jsm": "browser/components/newtab/lib/PlacesFeed.jsm", + "resource://activity-stream/lib/PrefsFeed.jsm": "browser/components/newtab/lib/PrefsFeed.jsm", + "resource://activity-stream/lib/RecommendationProvider.jsm": "browser/components/newtab/lib/RecommendationProvider.jsm", + "resource://activity-stream/lib/RemoteL10n.jsm": "browser/components/newtab/lib/RemoteL10n.jsm", + "resource://activity-stream/lib/Screenshots.jsm": "browser/components/newtab/lib/Screenshots.jsm", + "resource://activity-stream/lib/SearchShortcuts.jsm": "browser/components/newtab/lib/SearchShortcuts.jsm", + "resource://activity-stream/lib/SectionsManager.jsm": "browser/components/newtab/lib/SectionsManager.jsm", + "resource://activity-stream/lib/ShortURL.jsm": "browser/components/newtab/lib/ShortURL.jsm", + "resource://activity-stream/lib/SiteClassifier.jsm": "browser/components/newtab/lib/SiteClassifier.jsm", + "resource://activity-stream/lib/SnippetsTestMessageProvider.jsm": "browser/components/newtab/lib/SnippetsTestMessageProvider.jsm", + "resource://activity-stream/lib/Spotlight.jsm": "browser/components/newtab/lib/Spotlight.jsm", + "resource://activity-stream/lib/Store.jsm": "browser/components/newtab/lib/Store.jsm", + "resource://activity-stream/lib/SystemTickFeed.jsm": "browser/components/newtab/lib/SystemTickFeed.jsm", + "resource://activity-stream/lib/TelemetryFeed.jsm": "browser/components/newtab/lib/TelemetryFeed.jsm", + "resource://activity-stream/lib/TippyTopProvider.jsm": "browser/components/newtab/lib/TippyTopProvider.jsm", + "resource://activity-stream/lib/ToastNotification.jsm": "browser/components/newtab/lib/ToastNotification.jsm", + "resource://activity-stream/lib/ToolbarBadgeHub.jsm": "browser/components/newtab/lib/ToolbarBadgeHub.jsm", + "resource://activity-stream/lib/ToolbarPanelHub.jsm": "browser/components/newtab/lib/ToolbarPanelHub.jsm", + "resource://activity-stream/lib/TopSitesFeed.jsm": "browser/components/newtab/lib/TopSitesFeed.jsm", + "resource://activity-stream/lib/TopStoriesFeed.jsm": "browser/components/newtab/lib/TopStoriesFeed.jsm", + "resource://activity-stream/lib/UTEventReporting.jsm": "browser/components/newtab/lib/UTEventReporting.jsm", + "resource://activity-stream/vendor/Redux.jsm": "browser/components/newtab/vendor/Redux.jsm", + "resource://android/assets/web_extensions/test-support/TestSupportChild.jsm": "mobile/android/geckoview/src/androidTest/assets/web_extensions/test-support/TestSupportChild.jsm", + "resource://android/assets/web_extensions/test-support/TestSupportProcessChild.jsm": "mobile/android/geckoview/src/androidTest/assets/web_extensions/test-support/TestSupportProcessChild.jsm", + "resource://app/modules/SnapshotSelector.jsm": "browser/components/places/SnapshotSelector.jsm", + "resource://app/modules/translation/BingTranslator.jsm": "browser/components/translation/BingTranslator.jsm", + "resource://app/modules/translation/GoogleTranslator.jsm": "browser/components/translation/GoogleTranslator.jsm", + "resource://app/modules/translation/YandexTranslator.jsm": "browser/components/translation/YandexTranslator.jsm", + "resource://autofill/CreditCardRuleset.jsm": "toolkit/components/formautofill/CreditCardRuleset.jsm", + "resource://autofill/FormAutofill.jsm": "toolkit/components/formautofill/FormAutofill.jsm", + "resource://autofill/FormAutofillChild.jsm": "toolkit/components/formautofill/FormAutofillChild.jsm", + "resource://autofill/FormAutofillContent.jsm": "toolkit/components/formautofill/FormAutofillContent.jsm", + "resource://autofill/FormAutofillHandler.jsm": "toolkit/components/formautofill/FormAutofillHandler.jsm", + "resource://autofill/FormAutofillHeuristics.jsm": "toolkit/components/formautofill/FormAutofillHeuristics.jsm", + "resource://autofill/FormAutofillNameUtils.jsm": "toolkit/components/formautofill/FormAutofillNameUtils.jsm", + "resource://autofill/FormAutofillParent.jsm": "toolkit/components/formautofill/FormAutofillParent.jsm", + "resource://autofill/FormAutofillPreferences.jsm": "toolkit/components/formautofill/FormAutofillPreferences.jsm", + "resource://autofill/FormAutofillStorageBase.jsm": "toolkit/components/formautofill/FormAutofillStorageBase.jsm", + "resource://autofill/FormAutofillSync.jsm": "toolkit/components/formautofill/FormAutofillSync.jsm", + "resource://autofill/Autofilltelemetry.jsm": "toolkit/components/formautofill/Autofilltelemetry.jsm", + "resource://autofill/FormAutofillUtils.jsm": "toolkit/components/formautofill/FormAutofillUtils.jsm", + "resource://autofill/ProfileAutoCompleteResult.jsm": "toolkit/components/formautofill/ProfileAutoCompleteResult.jsm", + "resource://autofill/phonenumberutils/PhoneNumber.jsm": "toolkit/components/formautofill/phonenumberutils/PhoneNumber.jsm", + "resource://autofill/phonenumberutils/PhoneNumberMetaData.jsm": "toolkit/components/formautofill/phonenumberutils/PhoneNumberMetaData.jsm", + "resource://autofill/phonenumberutils/PhoneNumberNormalizer.jsm": "toolkit/components/formautofill/phonenumberutils/PhoneNumberNormalizer.jsm", + "resource://damp-test/content/actors/DampLoadChild.jsm": "testing/talos/talos/tests/devtools/addon/content/actors/DampLoadChild.jsm", + "resource://damp-test/content/actors/DampLoadParent.jsm": "testing/talos/talos/tests/devtools/addon/content/actors/DampLoadParent.jsm", + "resource://devtools/client/framework/browser-toolbox/Launcher.jsm": "devtools/client/framework/browser-toolbox/Launcher.jsm", + "resource://devtools/client/jsonview/Converter.jsm": "devtools/client/jsonview/Converter.jsm", + "resource://devtools/client/jsonview/Sniffer.jsm": "devtools/client/jsonview/Sniffer.jsm", + "resource://devtools/client/performance-new/shared/background.jsm.js": "devtools/client/performance-new/shared/background.jsm.js", + "resource://devtools/client/performance-new/popup/menu-button.jsm.js": "devtools/client/performance-new/popup/menu-button.jsm.js", + "resource://devtools/client/performance-new/popup/logic.jsm.js": "devtools/client/performance-new/popup/logic.jsm.js", + "resource://devtools/client/performance-new/shared/symbolication.jsm.js": "devtools/client/performance-new/shared/symbolication.jsm.js", + "resource://devtools/client/performance-new/shared/typescript-lazy-load.jsm.js": "devtools/client/performance-new/shared/typescript-lazy-load.jsm.js", + "resource://devtools/client/storage/VariablesView.jsm": "devtools/client/storage/VariablesView.jsm", + "resource://devtools/client/styleeditor/StyleEditorUI.jsm": "devtools/client/styleeditor/StyleEditorUI.jsm", + "resource://devtools/client/styleeditor/StyleEditorUtil.jsm": "devtools/client/styleeditor/StyleEditorUtil.jsm", + "resource://devtools/client/styleeditor/StyleSheetEditor.jsm": "devtools/client/styleeditor/StyleSheetEditor.jsm", + "resource://devtools/server/actors/targets/target-actor-registry.jsm": "devtools/server/actors/targets/target-actor-registry.jsm", + "resource://devtools/server/actors/watcher/SessionDataHelpers.jsm": "devtools/server/actors/watcher/SessionDataHelpers.jsm", + "resource://devtools/server/actors/watcher/WatcherRegistry.jsm": "devtools/server/actors/watcher/WatcherRegistry.jsm", + "resource://devtools/server/actors/watcher/browsing-context-helpers.jsm": "devtools/server/actors/watcher/browsing-context-helpers.jsm", + "resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm": "devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm", + "resource://devtools/server/connectors/js-window-actor/DevToolsFrameParent.jsm": "devtools/server/connectors/js-window-actor/DevToolsFrameParent.jsm", + "resource://devtools/server/connectors/js-window-actor/DevToolsWorkerChild.jsm": "devtools/server/connectors/js-window-actor/DevToolsWorkerChild.jsm", + "resource://devtools/server/connectors/js-window-actor/DevToolsWorkerParent.jsm": "devtools/server/connectors/js-window-actor/DevToolsWorkerParent.jsm", + "resource://devtools/server/connectors/js-window-actor/WindowGlobalLogger.jsm": "devtools/server/connectors/js-window-actor/WindowGlobalLogger.jsm", + "resource://devtools/server/startup/content-process.jsm": "devtools/server/startup/content-process.jsm", + "resource://devtools/shared/loader/Loader.jsm": "devtools/shared/loader/Loader.jsm", + "resource://devtools/shared/loader/base-loader.js": "devtools/shared/loader/base-loader.js", + "resource://devtools/shared/loader/browser-loader.js": "devtools/shared/loader/browser-loader.js", + "resource://devtools/shared/loader/loader-plugin-raw.jsm": "devtools/shared/loader/loader-plugin-raw.jsm", + "resource://devtools/shared/loader/worker-loader.js": "devtools/shared/loader/worker-loader.js", + "resource://devtools/shared/security/DevToolsSocketStatus.jsm": "devtools/shared/security/DevToolsSocketStatus.jsm", + "resource://devtools/shared/test-helpers/tracked-objects.jsm": "devtools/shared/test-helpers/tracked-objects.jsm", + "resource://devtools/shared/validate-breakpoint.jsm": "devtools/shared/validate-breakpoint.jsm", + "resource://devtools/shared/worker/worker.js": "devtools/shared/worker/worker.js", + "resource://featuregates/FeatureGate.jsm": "toolkit/components/featuregates/FeatureGate.jsm", + "resource://featuregates/FeatureGateImplementation.jsm": "toolkit/components/featuregates/FeatureGateImplementation.jsm", + "resource://gre/actors/AboutHttpsOnlyErrorChild.jsm": "toolkit/actors/AboutHttpsOnlyErrorChild.jsm", + "resource://gre/actors/AboutHttpsOnlyErrorParent.jsm": "toolkit/actors/AboutHttpsOnlyErrorParent.jsm", + "resource://gre/actors/AudioPlaybackChild.jsm": "toolkit/actors/AudioPlaybackChild.jsm", + "resource://gre/actors/AudioPlaybackParent.jsm": "toolkit/actors/AudioPlaybackParent.jsm", + "resource://gre/actors/AutoCompleteChild.jsm": "toolkit/actors/AutoCompleteChild.jsm", + "resource://gre/actors/AutoCompleteParent.jsm": "toolkit/actors/AutoCompleteParent.jsm", + "resource://gre/actors/AutoScrollChild.jsm": "toolkit/actors/AutoScrollChild.jsm", + "resource://gre/actors/AutoScrollParent.jsm": "toolkit/actors/AutoScrollParent.jsm", + "resource://gre/actors/AutoplayChild.jsm": "toolkit/actors/AutoplayChild.jsm", + "resource://gre/actors/AutoplayParent.jsm": "toolkit/actors/AutoplayParent.jsm", + "resource://gre/actors/BackgroundThumbnailsChild.jsm": "toolkit/actors/BackgroundThumbnailsChild.jsm", + "resource://gre/actors/BrowserElementChild.jsm": "toolkit/actors/BrowserElementChild.jsm", + "resource://gre/actors/BrowserElementParent.jsm": "toolkit/actors/BrowserElementParent.jsm", + "resource://gre/actors/ClipboardReadPasteChild.jsm": "toolkit/actors/ClipboardReadPasteChild.jsm", + "resource://gre/actors/ClipboardReadPasteParent.jsm": "toolkit/actors/ClipboardReadPasteParent.jsm", + "resource://gre/actors/ContentMetaChild.jsm": "toolkit/actors/ContentMetaChild.jsm", + "resource://gre/actors/ContentMetaParent.jsm": "toolkit/actors/ContentMetaParent.jsm", + "resource://gre/actors/ControllersChild.jsm": "toolkit/actors/ControllersChild.jsm", + "resource://gre/actors/ControllersParent.jsm": "toolkit/actors/ControllersParent.jsm", + "resource://gre/actors/CookieBannerChild.jsm.jsm": "toolkit/components/cookiebanners/CookieBannerChild.jsm.jsm", + "resource://gre/actors/CookieBannerParent.jsm": "toolkit/components/cookiebanners/CookieBannerParent.jsm", + "resource://gre/actors/DateTimePickerChild.jsm": "toolkit/actors/DateTimePickerChild.jsm", + "resource://gre/actors/DateTimePickerParent.jsm": "toolkit/actors/DateTimePickerParent.jsm", + "resource://gre/actors/ExtFindChild.jsm": "toolkit/actors/ExtFindChild.jsm", + "resource://gre/actors/FindBarChild.jsm": "toolkit/actors/FindBarChild.jsm", + "resource://gre/actors/FindBarParent.jsm": "toolkit/actors/FindBarParent.jsm", + "resource://gre/actors/FinderChild.jsm": "toolkit/actors/FinderChild.jsm", + "resource://gre/actors/FormHistoryChild.jsm": "toolkit/components/satchel/FormHistoryChild.jsm", + "resource://gre/actors/FormHistoryParent.jsm": "toolkit/components/satchel/FormHistoryParent.jsm", + "resource://gre/actors/InlineSpellCheckerChild.jsm": "toolkit/actors/InlineSpellCheckerChild.jsm", + "resource://gre/actors/InlineSpellCheckerParent.jsm": "toolkit/actors/InlineSpellCheckerParent.jsm", + "resource://gre/actors/KeyPressEventModelCheckerChild.jsm": "toolkit/actors/KeyPressEventModelCheckerChild.jsm", + "resource://gre/actors/LayoutDebugChild.jsm": "layout/tools/layout-debug/LayoutDebugChild.jsm", + "resource://gre/actors/NetErrorChild.jsm": "toolkit/actors/NetErrorChild.jsm", + "resource://gre/actors/NetErrorParent.jsm": "toolkit/actors/NetErrorParent.jsm", + "resource://gre/actors/PictureInPictureChild.jsm": "toolkit/actors/PictureInPictureChild.jsm", + "resource://gre/actors/PopupBlockingChild.jsm": "toolkit/actors/PopupBlockingChild.jsm", + "resource://gre/actors/PopupBlockingParent.jsm": "toolkit/actors/PopupBlockingParent.jsm", + "resource://gre/actors/PrintingChild.jsm": "toolkit/actors/PrintingChild.jsm", + "resource://gre/actors/PrintingParent.jsm": "toolkit/actors/PrintingParent.jsm", + "resource://gre/actors/PrintingSelectionChild.jsm": "toolkit/actors/PrintingSelectionChild.jsm", + "resource://gre/actors/PurgeSessionHistoryChild.jsm": "toolkit/actors/PurgeSessionHistoryChild.jsm", + "resource://gre/actors/RemotePageChild.jsm": "toolkit/actors/RemotePageChild.jsm", + "resource://gre/actors/SelectChild.jsm": "toolkit/actors/SelectChild.jsm", + "resource://gre/actors/SelectParent.jsm": "toolkit/actors/SelectParent.jsm", + "resource://gre/actors/ThumbnailsChild.jsm": "toolkit/actors/ThumbnailsChild.jsm", + "resource://gre/actors/UAWidgetsChild.jsm": "toolkit/actors/UAWidgetsChild.jsm", + "resource://gre/actors/UnselectedTabHoverChild.jsm": "toolkit/actors/UnselectedTabHoverChild.jsm", + "resource://gre/actors/UnselectedTabHoverParent.jsm": "toolkit/actors/UnselectedTabHoverParent.jsm", + "resource://gre/actors/ViewSourceChild.jsm": "toolkit/actors/ViewSourceChild.jsm", + "resource://gre/actors/ViewSourcePageChild.jsm": "toolkit/actors/ViewSourcePageChild.jsm", + "resource://gre/actors/ViewSourcePageParent.jsm": "toolkit/actors/ViewSourcePageParent.jsm", + "resource://gre/actors/WebChannelChild.jsm": "toolkit/actors/WebChannelChild.jsm", + "resource://gre/actors/WebChannelParent.jsm": "toolkit/actors/WebChannelParent.jsm", + "resource://gre/modules/AboutCertViewerChild.jsm": "toolkit/components/certviewer/AboutCertViewerChild.jsm", + "resource://gre/modules/AboutCertViewerParent.jsm": "toolkit/components/certviewer/AboutCertViewerParent.jsm", + "resource://gre/modules/AboutPagesUtils.jsm": "toolkit/modules/AboutPagesUtils.jsm", + "resource://gre/modules/AboutReader.jsm": "toolkit/components/reader/AboutReader.jsm", + "resource://gre/modules/AbuseReporter.jsm": "toolkit/mozapps/extensions/AbuseReporter.jsm", + "resource://gre/modules/ActorManagerParent.jsm": "toolkit/modules/ActorManagerParent.jsm", + "resource://gre/modules/AddonManager.jsm": "toolkit/mozapps/extensions/AddonManager.jsm", + "resource://gre/modules/AddonSearchEngine.jsm": "toolkit/components/search/AddonSearchEngine.jsm", + "resource://gre/modules/AndroidLog.jsm": "mobile/android/modules/geckoview/AndroidLog.jsm", + "resource://gre/modules/AppConstants.jsm": "toolkit/modules/AppConstants.jsm", + "resource://gre/modules/AppMenuNotifications.jsm": "toolkit/modules/AppMenuNotifications.jsm", + "resource://gre/modules/AsanReporter.jsm": "toolkit/modules/AsanReporter.jsm", + "resource://gre/modules/AsyncPrefs.jsm": "toolkit/modules/AsyncPrefs.jsm", + "resource://gre/modules/AsyncShutdown.jsm": "toolkit/components/asyncshutdown/AsyncShutdown.jsm", + "resource://gre/modules/AutoCompleteSimpleSearch.jsm": "toolkit/components/autocomplete/AutoCompleteSimpleSearch.jsm", + "resource://gre/modules/BHRTelemetryService.jsm": "toolkit/components/backgroundhangmonitor/BHRTelemetryService.jsm", + "resource://gre/modules/BackgroundPageThumbs.jsm": "toolkit/components/thumbnails/BackgroundPageThumbs.jsm", + "resource://gre/modules/BackgroundTasksManager.jsm": "toolkit/components/backgroundtasks/BackgroundTasksManager.jsm", + "resource://gre/modules/BackgroundTasksUtils.jsm": "toolkit/components/backgroundtasks/BackgroundTasksUtils.jsm", + "resource://gre/modules/BackgroundUpdate.jsm": "toolkit/mozapps/update/BackgroundUpdate.jsm", + "resource://gre/modules/BinarySearch.jsm": "toolkit/modules/BinarySearch.jsm", + "resource://gre/modules/Bits.jsm": "toolkit/components/bitsdownload/Bits.jsm", + "resource://gre/modules/Blocklist.jsm": "toolkit/mozapps/extensions/Blocklist.jsm", + "resource://gre/modules/BookmarkHTMLUtils.jsm": "toolkit/components/places/BookmarkHTMLUtils.jsm", + "resource://gre/modules/BookmarkJSONUtils.jsm": "toolkit/components/places/BookmarkJSONUtils.jsm", + "resource://gre/modules/Bookmarks.jsm": "toolkit/components/places/Bookmarks.jsm", + "resource://gre/modules/BrowserElementParent.jsm": "dom/browser-element/BrowserElementParent.jsm", + "resource://gre/modules/BrowserElementPromptService.jsm": "dom/browser-element/BrowserElementPromptService.jsm", + "resource://gre/modules/BrowserTelemetryUtils.jsm": "toolkit/modules/BrowserTelemetryUtils.jsm", + "resource://gre/modules/BrowserUtils.jsm": "toolkit/modules/BrowserUtils.jsm", + "resource://gre/modules/CSV.js": "toolkit/components/passwordmgr/CSV.js", + "resource://gre/modules/CanonicalJSON.jsm": "toolkit/modules/CanonicalJSON.jsm", + "resource://gre/modules/CaptiveDetect.jsm": "toolkit/components/captivedetect/CaptiveDetect.jsm", + "resource://gre/modules/CertUtils.jsm": "toolkit/modules/CertUtils.jsm", + "resource://gre/modules/ChildCrashHandler.jsm": "mobile/android/modules/geckoview/ChildCrashHandler.jsm", + "resource://gre/modules/ClearDataService.jsm": "toolkit/components/cleardata/ClearDataService.jsm", + "resource://gre/modules/ClientID.jsm": "toolkit/components/telemetry/app/ClientID.jsm", + "resource://gre/modules/Color.jsm": "toolkit/modules/Color.jsm", + "resource://gre/modules/ColorPickerDelegate.jsm": "mobile/android/components/geckoview/ColorPickerDelegate.jsm", + "resource://gre/modules/CommonDialog.jsm": "toolkit/components/prompts/src/CommonDialog.jsm", + "resource://gre/modules/ComponentUtils.jsm": "js/xpconnect/loader/ComponentUtils.jsm", + "resource://gre/modules/ConduitsChild.jsm": "toolkit/components/extensions/ConduitsChild.jsm", + "resource://gre/modules/ConduitsParent.jsm": "toolkit/components/extensions/ConduitsParent.jsm", + "resource://gre/modules/Console.jsm": "toolkit/modules/Console.jsm", + "resource://gre/modules/ConsoleAPIStorage.jsm": "dom/console/ConsoleAPIStorage.jsm", + "resource://gre/modules/ContentAreaDropListener.jsm": "dom/base/ContentAreaDropListener.jsm", + "resource://gre/modules/ContentBlockingAllowList.jsm": "toolkit/components/antitracking/ContentBlockingAllowList.jsm", + "resource://gre/modules/ContentDOMReference.jsm": "toolkit/modules/ContentDOMReference.jsm", + "resource://gre/modules/ContentDispatchChooser.jsm": "toolkit/mozapps/handling/ContentDispatchChooser.jsm", + "resource://gre/modules/ContentPrefService2.jsm": "toolkit/components/contentprefs/ContentPrefService2.jsm", + "resource://gre/modules/ContentPrefServiceChild.jsm": "toolkit/components/contentprefs/ContentPrefServiceChild.jsm", + "resource://gre/modules/ContentPrefServiceParent.jsm": "toolkit/components/contentprefs/ContentPrefServiceParent.jsm", + "resource://gre/modules/ContentPrefStore.jsm": "toolkit/components/contentprefs/ContentPrefStore.jsm", + "resource://gre/modules/ContentPrefUtils.jsm": "toolkit/components/contentprefs/ContentPrefUtils.jsm", + "resource://gre/modules/ContextualIdentityService.jsm": "toolkit/components/contextualidentity/ContextualIdentityService.jsm", + "resource://gre/modules/Corroborate.jsm": "toolkit/components/corroborator/Corroborate.jsm", + "resource://gre/modules/CoveragePing.jsm": "toolkit/components/telemetry/pings/CoveragePing.jsm", + "resource://gre/modules/CrashManager.jsm": "toolkit/components/crashes/CrashManager.in.jsm", + "resource://gre/modules/CrashMonitor.jsm": "toolkit/components/crashmonitor/CrashMonitor.jsm", + "resource://gre/modules/CrashReports.jsm": "toolkit/crashreporter/CrashReports.jsm", + "resource://gre/modules/CrashService.jsm": "toolkit/components/crashes/CrashService.jsm", + "resource://gre/modules/CrashSubmit.jsm": "toolkit/crashreporter/CrashSubmit.jsm", + "resource://gre/modules/Credentials.jsm": "services/fxaccounts/Credentials.jsm", + "resource://gre/modules/CreditCard.jsm": "toolkit/modules/CreditCard.jsm", + "resource://gre/modules/CustomElementsListener.jsm": "toolkit/components/processsingleton/CustomElementsListener.jsm", + "resource://gre/modules/DOMRequestHelper.jsm": "dom/base/DOMRequestHelper.jsm", + "resource://gre/modules/DateTimePickerPanel.jsm": "toolkit/modules/DateTimePickerPanel.jsm", + "resource://gre/modules/DefaultCLH.jsm": "toolkit/components/DefaultCLH.jsm", + "resource://gre/modules/DeferredTask.jsm": "toolkit/modules/DeferredTask.jsm", + "resource://gre/modules/DelayedInit.jsm": "mobile/android/modules/geckoview/DelayedInit.jsm", + "resource://gre/modules/Deprecated.jsm": "toolkit/modules/Deprecated.jsm", + "resource://gre/modules/DownloadCore.jsm": "toolkit/components/downloads/DownloadCore.jsm", + "resource://gre/modules/DownloadHistory.jsm": "toolkit/components/downloads/DownloadHistory.jsm", + "resource://gre/modules/DownloadIntegration.jsm": "toolkit/components/downloads/DownloadIntegration.jsm", + "resource://gre/modules/DownloadLastDir.jsm": "toolkit/mozapps/downloads/DownloadLastDir.jsm", + "resource://gre/modules/DownloadLegacy.jsm": "toolkit/components/downloads/DownloadLegacy.jsm", + "resource://gre/modules/DownloadList.jsm": "toolkit/components/downloads/DownloadList.jsm", + "resource://gre/modules/DownloadPaths.jsm": "toolkit/components/downloads/DownloadPaths.jsm", + "resource://gre/modules/DownloadStore.jsm": "toolkit/components/downloads/DownloadStore.jsm", + "resource://gre/modules/DownloadUIHelper.jsm": "toolkit/components/downloads/DownloadUIHelper.jsm", + "resource://gre/modules/DownloadUtils.jsm": "toolkit/mozapps/downloads/DownloadUtils.jsm", + "resource://gre/modules/Downloads.jsm": "toolkit/components/downloads/Downloads.jsm", + "resource://gre/modules/E10SUtils.jsm": "toolkit/modules/E10SUtils.jsm", + "resource://gre/modules/EnterprisePolicies.jsm": "toolkit/components/enterprisepolicies/EnterprisePolicies.jsm", + "resource://gre/modules/EnterprisePoliciesContent.jsm": "toolkit/components/enterprisepolicies/EnterprisePoliciesContent.jsm", + "resource://gre/modules/EnterprisePoliciesParent.jsm": "toolkit/components/enterprisepolicies/EnterprisePoliciesParent.jsm", + "resource://gre/modules/EventEmitter.jsm": "toolkit/modules/EventEmitter.jsm", + "resource://gre/modules/EventPing.jsm": "toolkit/components/telemetry/pings/EventPing.jsm", + "resource://gre/modules/ExtHandlerService.jsm": "uriloader/exthandler/ExtHandlerService.jsm", + "resource://gre/modules/Extension.jsm": "toolkit/components/extensions/Extension.jsm", + "resource://gre/modules/ExtensionActions.jsm": "toolkit/components/extensions/ExtensionActions.jsm", + "resource://gre/modules/ExtensionActivityLog.jsm": "toolkit/components/extensions/ExtensionActivityLog.jsm", + "resource://gre/modules/ExtensionChild.jsm": "toolkit/components/extensions/ExtensionChild.jsm", + "resource://gre/modules/ExtensionChildDevToolsUtils.jsm": "toolkit/components/extensions/ExtensionChildDevToolsUtils.jsm", + "resource://gre/modules/ExtensionCommon.jsm": "toolkit/components/extensions/ExtensionCommon.jsm", + "resource://gre/modules/ExtensionContent.jsm": "toolkit/components/extensions/ExtensionContent.jsm", + "resource://gre/modules/ExtensionPageChild.jsm": "toolkit/components/extensions/ExtensionPageChild.jsm", + "resource://gre/modules/ExtensionParent.jsm": "toolkit/components/extensions/ExtensionParent.jsm", + "resource://gre/modules/ExtensionPermissions.jsm": "toolkit/components/extensions/ExtensionPermissions.jsm", + "resource://gre/modules/ExtensionPreferencesManager.jsm": "toolkit/components/extensions/ExtensionPreferencesManager.jsm", + "resource://gre/modules/ExtensionProcessScript.jsm": "toolkit/components/extensions/ExtensionProcessScript.jsm", + "resource://gre/modules/ExtensionScriptingStore.jsm": "toolkit/components/extensions/ExtensionScriptingStore.jsm", + "resource://gre/modules/ExtensionSearchHandler.jsm": "toolkit/components/places/ExtensionSearchHandler.jsm", + "resource://gre/modules/ExtensionSettingsStore.jsm": "toolkit/components/extensions/ExtensionSettingsStore.jsm", + "resource://gre/modules/ExtensionShortcuts.jsm": "toolkit/components/extensions/ExtensionShortcuts.jsm", + "resource://gre/modules/ExtensionStorage.jsm": "toolkit/components/extensions/ExtensionStorage.jsm", + "resource://gre/modules/ExtensionStorageComponents.jsm": "toolkit/components/extensions/storage/ExtensionStorageComponents.jsm", + "resource://gre/modules/ExtensionStorageIDB.jsm": "toolkit/components/extensions/ExtensionStorageIDB.jsm", + "resource://gre/modules/ExtensionStorageSync.jsm": "toolkit/components/extensions/ExtensionStorageSync.jsm", + "resource://gre/modules/ExtensionStorageSyncKinto.jsm": "toolkit/components/extensions/ExtensionStorageSyncKinto.jsm", + "resource://gre/modules/ExtensionTelemetry.jsm": "toolkit/components/extensions/ExtensionTelemetry.jsm", + "resource://gre/modules/ExtensionUtils.jsm": "toolkit/components/extensions/ExtensionUtils.jsm", + "resource://gre/modules/ExtensionWorkerChild.jsm": "toolkit/components/extensions/ExtensionWorkerChild.jsm", + "resource://gre/modules/FilePickerDelegate.jsm": "mobile/android/components/geckoview/FilePickerDelegate.jsm", + "resource://gre/modules/FileUtils.jsm": "toolkit/modules/FileUtils.jsm", + "resource://gre/modules/FindBarContent.jsm": "toolkit/modules/FindBarContent.jsm", + "resource://gre/modules/FindContent.jsm": "toolkit/components/extensions/FindContent.jsm", + "resource://gre/modules/Finder.jsm": "toolkit/modules/Finder.jsm", + "resource://gre/modules/FinderHighlighter.jsm": "toolkit/modules/FinderHighlighter.jsm", + "resource://gre/modules/FinderIterator.jsm": "toolkit/modules/FinderIterator.jsm", + "resource://gre/modules/FinderParent.jsm": "toolkit/modules/FinderParent.jsm", + "resource://gre/modules/FirefoxRelay.jsm": "toolkit/components/passwordmgr/FirefoxRelay.jsm", + "resource://gre/modules/FirstStartup.jsm": "toolkit/modules/FirstStartup.jsm", + "resource://gre/modules/ForgetAboutSite.jsm": "toolkit/components/forgetaboutsite/ForgetAboutSite.jsm", + "resource://gre/modules/FormAutoComplete.jsm": "toolkit/components/satchel/FormAutoComplete.jsm", + "resource://gre/modules/FormHistory.jsm": "toolkit/components/satchel/FormHistory.jsm", + "resource://gre/modules/FormHistoryStartup.jsm": "toolkit/components/satchel/FormHistoryStartup.jsm", + "resource://gre/modules/FormLikeFactory.jsm": "toolkit/modules/FormLikeFactory.jsm", + "resource://gre/modules/FxAccounts.jsm": "services/fxaccounts/FxAccounts.jsm", + "resource://gre/modules/FxAccountsClient.jsm": "services/fxaccounts/FxAccountsClient.jsm", + "resource://gre/modules/FxAccountsCommands.js": "services/fxaccounts/FxAccountsCommands.js", + "resource://gre/modules/FxAccountsCommon.js": "services/fxaccounts/FxAccountsCommon.js", + "resource://gre/modules/FxAccountsConfig.jsm": "services/fxaccounts/FxAccountsConfig.jsm", + "resource://gre/modules/FxAccountsDevice.jsm": "services/fxaccounts/FxAccountsDevice.jsm", + "resource://gre/modules/FxAccountsKeys.jsm": "services/fxaccounts/FxAccountsKeys.jsm", + "resource://gre/modules/FxAccountsPairing.jsm": "services/fxaccounts/FxAccountsPairing.jsm", + "resource://gre/modules/FxAccountsPairingChannel.js": "services/fxaccounts/FxAccountsPairingChannel.js", + "resource://gre/modules/FxAccountsProfile.jsm": "services/fxaccounts/FxAccountsProfile.jsm", + "resource://gre/modules/FxAccountsProfileClient.jsm": "services/fxaccounts/FxAccountsProfileClient.jsm", + "resource://gre/modules/FxAccountsPush.jsm": "services/fxaccounts/FxAccountsPush.jsm", + "resource://gre/modules/FxAccountsStorage.jsm": "services/fxaccounts/FxAccountsStorage.jsm", + "resource://gre/modules/FxAccountsTelemetry.jsm": "services/fxaccounts/FxAccountsTelemetry.jsm", + "resource://gre/modules/FxAccountsWebChannel.jsm": "services/fxaccounts/FxAccountsWebChannel.jsm", + "resource://gre/modules/GMPInstallManager.jsm": "toolkit/modules/GMPInstallManager.jsm", + "resource://gre/modules/GMPUtils.jsm": "toolkit/modules/GMPUtils.jsm", + "resource://gre/modules/GeckoViewAutocomplete.jsm": "mobile/android/modules/geckoview/GeckoViewAutocomplete.jsm", + "resource://gre/modules/GeckoViewAutofill.jsm": "mobile/android/modules/geckoview/GeckoViewAutofill.jsm", + "resource://gre/modules/GeckoViewChildModule.jsm": "mobile/android/modules/geckoview/GeckoViewChildModule.jsm", + "resource://gre/modules/GeckoViewConsole.jsm": "mobile/android/modules/geckoview/GeckoViewConsole.jsm", + "resource://gre/modules/GeckoViewContent.jsm": "mobile/android/modules/geckoview/GeckoViewContent.jsm", + "resource://gre/modules/GeckoViewContentBlocking.jsm": "mobile/android/modules/geckoview/GeckoViewContentBlocking.jsm", + "resource://gre/modules/GeckoViewMediaControl.jsm": "mobile/android/modules/geckoview/GeckoViewMediaControl.jsm", + "resource://gre/modules/GeckoViewModule.jsm": "mobile/android/modules/geckoview/GeckoViewModule.jsm", + "resource://gre/modules/GeckoViewNavigation.jsm": "mobile/android/modules/geckoview/GeckoViewNavigation.jsm", + "resource://gre/modules/GeckoViewPermission.jsm": "mobile/android/components/geckoview/GeckoViewPermission.jsm", + "resource://gre/modules/GeckoViewProcessHangMonitor.jsm": "mobile/android/modules/geckoview/GeckoViewProcessHangMonitor.jsm", + "resource://gre/modules/GeckoViewProgress.jsm": "mobile/android/modules/geckoview/GeckoViewProgress.jsm", + "resource://gre/modules/GeckoViewPrompt.jsm": "mobile/android/components/geckoview/GeckoViewPrompt.jsm", + "resource://gre/modules/GeckoViewPush.jsm": "mobile/android/components/geckoview/GeckoViewPush.jsm", + "resource://gre/modules/GeckoViewPushController.jsm": "mobile/android/modules/geckoview/GeckoViewPushController.jsm", + "resource://gre/modules/GeckoViewRemoteDebugger.jsm": "mobile/android/modules/geckoview/GeckoViewRemoteDebugger.jsm", + "resource://gre/modules/GeckoViewSelectionAction.jsm": "mobile/android/modules/geckoview/GeckoViewSelectionAction.jsm", + "resource://gre/modules/GeckoViewSettings.jsm": "mobile/android/modules/geckoview/GeckoViewSettings.jsm", + "resource://gre/modules/GeckoViewStartup.jsm": "mobile/android/components/geckoview/GeckoViewStartup.jsm", + "resource://gre/modules/GeckoViewStorageController.jsm": "mobile/android/modules/geckoview/GeckoViewStorageController.jsm", + "resource://gre/modules/GeckoViewTab.jsm": "mobile/android/modules/geckoview/GeckoViewTab.jsm", + "resource://gre/modules/GeckoViewTelemetry.jsm": "mobile/android/modules/geckoview/GeckoViewTelemetry.jsm", + "resource://gre/modules/GeckoViewTestUtils.jsm": "mobile/android/modules/geckoview/GeckoViewTestUtils.jsm", + "resource://gre/modules/GeckoViewWebExtension.jsm": "mobile/android/modules/geckoview/GeckoViewWebExtension.jsm", + "resource://gre/modules/Geometry.jsm": "toolkit/modules/Geometry.jsm", + "resource://gre/modules/HealthPing.jsm": "toolkit/components/telemetry/pings/HealthPing.jsm", + "resource://gre/modules/HelperAppDlg.jsm": "toolkit/mozapps/downloads/HelperAppDlg.jsm", + "resource://gre/modules/HiddenFrame.jsm": "toolkit/modules/HiddenFrame.jsm", + "resource://gre/modules/History.jsm": "toolkit/components/places/History.jsm", + "resource://gre/modules/Http.jsm": "toolkit/modules/Http.jsm", + "resource://gre/modules/IgnoreLists.jsm": "toolkit/modules/IgnoreLists.jsm", + "resource://gre/modules/ImageObjectProcessor.jsm": "dom/manifest/ImageObjectProcessor.jsm", + "resource://gre/modules/IndexedDB.jsm": "toolkit/modules/IndexedDB.jsm", + "resource://gre/modules/InlineSpellChecker.jsm": "toolkit/modules/InlineSpellChecker.jsm", + "resource://gre/modules/InlineSpellCheckerContent.jsm": "toolkit/modules/InlineSpellCheckerContent.jsm", + "resource://gre/modules/InputListAutoComplete.jsm": "toolkit/components/satchel/InputListAutoComplete.jsm", + "resource://gre/modules/InsecurePasswordUtils.jsm": "toolkit/components/passwordmgr/InsecurePasswordUtils.jsm", + "resource://gre/modules/Integration.jsm": "toolkit/modules/Integration.jsm", + "resource://gre/modules/JSONFile.jsm": "toolkit/modules/JSONFile.jsm", + "resource://gre/modules/JsonSchema.jsm": "toolkit/modules/JsonSchema.jsm", + "resource://gre/modules/KeywordUtils.jsm": "toolkit/modules/KeywordUtils.jsm", + "resource://gre/modules/LangPackMatcher.jsm": "intl/locale/LangPackMatcher.jsm", + "resource://gre/modules/LayoutUtils.jsm": "toolkit/modules/LayoutUtils.jsm", + "resource://gre/modules/LightweightThemeConsumer.jsm": "toolkit/modules/LightweightThemeConsumer.jsm", + "resource://gre/modules/LightweightThemeManager.jsm": "toolkit/mozapps/extensions/LightweightThemeManager.jsm", + "resource://gre/modules/LoadURIDelegate.jsm": "mobile/android/modules/geckoview/LoadURIDelegate.jsm", + "resource://gre/modules/LocationHelper.jsm": "dom/base/LocationHelper.jsm", + "resource://gre/modules/Log.jsm": "toolkit/modules/Log.jsm", + "resource://gre/modules/LoginAutoComplete.jsm": "toolkit/components/passwordmgr/LoginAutoComplete.jsm", + "resource://gre/modules/LoginCSVImport.jsm": "toolkit/components/passwordmgr/LoginCSVImport.jsm", + "resource://gre/modules/LoginExport.jsm": "toolkit/components/passwordmgr/LoginExport.jsm", + "resource://gre/modules/LoginFormFactory.jsm": "toolkit/components/passwordmgr/LoginFormFactory.jsm", + "resource://gre/modules/LoginHelper.jsm": "toolkit/components/passwordmgr/LoginHelper.jsm", + "resource://gre/modules/LoginInfo.jsm": "toolkit/components/passwordmgr/LoginInfo.jsm", + "resource://gre/modules/LoginManager.jsm": "toolkit/components/passwordmgr/LoginManager.jsm", + "resource://gre/modules/LoginManagerAuthPrompter.jsm": "toolkit/components/passwordmgr/LoginManagerAuthPrompter.jsm", + "resource://gre/modules/LoginManagerChild.jsm": "toolkit/components/passwordmgr/LoginManagerChild.jsm", + "resource://gre/modules/LoginManagerContextMenu.jsm": "toolkit/components/passwordmgr/LoginManagerContextMenu.jsm", + "resource://gre/modules/LoginManagerParent.jsm": "toolkit/components/passwordmgr/LoginManagerParent.jsm", + "resource://gre/modules/LoginManagerPrompter.jsm": "toolkit/components/passwordmgr/LoginManagerPrompter.jsm", + "resource://gre/modules/LoginRecipes.jsm": "toolkit/components/passwordmgr/LoginRecipes.jsm", + "resource://gre/modules/LoginRelatedRealms.jsm": "toolkit/components/passwordmgr/LoginRelatedRealms.jsm", + "resource://gre/modules/LoginStorageDelegate.jsm": "mobile/android/components/geckoview/LoginStorageDelegate.jsm", + "resource://gre/modules/LoginStore.jsm": "toolkit/components/passwordmgr/LoginStore.jsm", + "resource://gre/modules/MainProcessSingleton.jsm": "toolkit/components/processsingleton/MainProcessSingleton.jsm", + "resource://gre/modules/Manifest.jsm": "dom/manifest/Manifest.jsm", + "resource://gre/modules/ManifestFinder.jsm": "dom/manifest/ManifestFinder.jsm", + "resource://gre/modules/ManifestIcons.jsm": "dom/manifest/ManifestIcons.jsm", + "resource://gre/modules/ManifestMessagesChild.jsm": "dom/ipc/ManifestMessagesChild.jsm", + "resource://gre/modules/ManifestObtainer.jsm": "dom/manifest/ManifestObtainer.jsm", + "resource://gre/modules/ManifestProcessor.jsm": "dom/manifest/ManifestProcessor.jsm", + "resource://gre/modules/MatchURLFilters.jsm": "toolkit/components/extensions/MatchURLFilters.jsm", + "resource://gre/modules/MediaUtils.jsm": "mobile/android/modules/geckoview/MediaUtils.jsm", + "resource://gre/modules/MessageManagerProxy.jsm": "toolkit/components/extensions/MessageManagerProxy.jsm", + "resource://gre/modules/ModulesPing.jsm": "toolkit/components/telemetry/pings/ModulesPing.jsm", + "resource://gre/modules/MozProtocolHandler.jsm": "toolkit/components/mozprotocol/MozProtocolHandler.jsm", + "resource://gre/modules/NLP.jsm": "toolkit/modules/NLP.jsm", + "resource://gre/modules/NativeManifests.jsm": "toolkit/components/extensions/NativeManifests.jsm", + "resource://gre/modules/NativeMessaging.jsm": "toolkit/components/extensions/NativeMessaging.jsm", + "resource://gre/modules/NetUtil.jsm": "netwerk/base/NetUtil.jsm", + "resource://gre/modules/NetworkGeolocationProvider.jsm": "dom/system/NetworkGeolocationProvider.jsm", + "resource://gre/modules/NewPasswordModel.jsm": "toolkit/components/passwordmgr/NewPasswordModel.jsm", + "resource://gre/modules/NewTabUtils.jsm": "toolkit/modules/NewTabUtils.jsm", + "resource://gre/modules/NotificationStorage.jsm": "dom/notification/NotificationStorage.jsm", + "resource://gre/modules/OSCrypto.jsm": "toolkit/components/passwordmgr/OSCrypto.jsm", + "resource://gre/modules/OSCrypto_win.jsm": "toolkit/components/passwordmgr/OSCrypto_win.jsm", + "resource://gre/modules/OSKeyStore.jsm": "toolkit/modules/OSKeyStore.jsm", + "resource://gre/modules/ObjectUtils.jsm": "toolkit/modules/ObjectUtils.jsm", + "resource://gre/modules/OpenSearchEngine.jsm": "toolkit/components/search/OpenSearchEngine.jsm", + "resource://gre/modules/OsEnvironment.jsm": "toolkit/modules/OsEnvironment.jsm", + "resource://gre/modules/PageThumbUtils.jsm": "toolkit/components/thumbnails/PageThumbUtils.jsm", + "resource://gre/modules/PageThumbs.jsm": "toolkit/components/thumbnails/PageThumbs.jsm", + "resource://gre/modules/PageThumbsStorageService.jsm": "toolkit/components/thumbnails/PageThumbsStorageService.jsm", + "resource://gre/modules/PartitioningExceptionListService.jsm": "toolkit/components/antitracking/PartitioningExceptionListService.jsm", + "resource://gre/modules/PasswordGenerator.jsm": "toolkit/components/passwordmgr/PasswordGenerator.jsm", + "resource://gre/modules/PasswordRulesManager.jsm": "toolkit/components/passwordmgr/PasswordRulesManager.jsm", + "resource://gre/modules/PasswordRulesParser.jsm": "toolkit/components/passwordmgr/PasswordRulesParser.jsm", + "resource://gre/modules/PerformanceCounters.jsm": "toolkit/components/extensions/PerformanceCounters.jsm", + "resource://gre/modules/PermissionsUtils.jsm": "toolkit/modules/PermissionsUtils.jsm", + "resource://gre/modules/PictureInPicture.jsm": "toolkit/components/pictureinpicture/PictureInPicture.jsm", + "resource://gre/modules/PictureInPictureControls.jsm": "toolkit/components/pictureinpicture/PictureInPictureControls.jsm", + "resource://gre/modules/PlacesBackups.jsm": "toolkit/components/places/PlacesBackups.jsm", + "resource://gre/modules/PlacesDBUtils.jsm": "toolkit/components/places/PlacesDBUtils.jsm", + "resource://gre/modules/PlacesExpiration.jsm": "toolkit/components/places/PlacesExpiration.jsm", + "resource://gre/modules/PlacesPreviews.jsm": "toolkit/components/places/PlacesPreviews.jsm", + "resource://gre/modules/PlacesSyncUtils.jsm": "toolkit/components/places/PlacesSyncUtils.jsm", + "resource://gre/modules/PlacesTransactions.jsm": "toolkit/components/places/PlacesTransactions.jsm", + "resource://gre/modules/PlacesUtils.jsm": "toolkit/components/places/PlacesUtils.jsm", + "resource://gre/modules/PluralForm.jsm": "intl/locale/PluralForm.jsm", + "resource://gre/modules/PolicySearchEngine.jsm": "toolkit/components/search/PolicySearchEngine.jsm", + "resource://gre/modules/PopupNotifications.jsm": "toolkit/modules/PopupNotifications.jsm", + "resource://gre/modules/Preferences.jsm": "toolkit/modules/Preferences.jsm", + "resource://gre/modules/PrincipalsCollector.jsm": "toolkit/components/cleardata/PrincipalsCollector.jsm", + "resource://gre/modules/PrivateBrowsingUtils.jsm": "toolkit/modules/PrivateBrowsingUtils.jsm", + "resource://gre/modules/ProcessSelector.jsm": "dom/base/ProcessSelector.jsm", + "resource://gre/modules/ProcessType.jsm": "toolkit/modules/ProcessType.jsm", + "resource://gre/modules/ProfileAge.jsm": "toolkit/modules/ProfileAge.jsm", + "resource://gre/modules/PromiseUtils.jsm": "toolkit/modules/PromiseUtils.jsm", + "resource://gre/modules/PromiseWorker.jsm": "toolkit/components/promiseworker/PromiseWorker.jsm", + "resource://gre/modules/PromptCollection.jsm": "mobile/android/components/geckoview/PromptCollection.jsm", + "resource://gre/modules/Prompter.jsm": "toolkit/components/prompts/src/Prompter.jsm", + "resource://gre/modules/PropertyListUtils.jsm": "toolkit/modules/PropertyListUtils.jsm", + "resource://gre/modules/ProxyChannelFilter.jsm": "toolkit/components/extensions/ProxyChannelFilter.jsm", + "resource://gre/modules/PurgeTrackerService.jsm": "toolkit/components/antitracking/PurgeTrackerService.jsm", + "resource://gre/modules/RFPHelper.jsm": "toolkit/components/resistfingerprinting/RFPHelper.jsm", + "resource://gre/modules/ReaderMode.jsm": "toolkit/components/reader/ReaderMode.jsm", + "resource://gre/modules/Readerable.jsm": "toolkit/components/reader/Readerable.jsm", + "resource://gre/modules/Region.jsm": "toolkit/modules/Region.jsm", + "resource://gre/modules/RemotePageAccessManager.jsm": "toolkit/modules/RemotePageAccessManager.jsm", + "resource://gre/modules/RemoteWebNavigation.jsm": "toolkit/components/remotebrowserutils/RemoteWebNavigation.jsm", + "resource://gre/modules/ResetProfile.jsm": "toolkit/modules/ResetProfile.jsm", + "resource://gre/modules/ResponsivenessMonitor.jsm": "toolkit/modules/ResponsivenessMonitor.jsm", + "resource://gre/modules/SafeBrowsing.jsm": "toolkit/components/url-classifier/SafeBrowsing.jsm", + "resource://gre/modules/SanityTest.jsm": "toolkit/components/gfx/SanityTest.jsm", + "resource://gre/modules/Schemas.jsm": "toolkit/components/extensions/Schemas.jsm", + "resource://gre/modules/SearchEngine.jsm": "toolkit/components/search/SearchEngine.jsm", + "resource://gre/modules/SearchEngineSelector.jsm": "toolkit/components/search/SearchEngineSelector.jsm", + "resource://gre/modules/SearchService.jsm": "toolkit/components/search/SearchService.jsm", + "resource://gre/modules/SearchSettings.jsm": "toolkit/components/search/SearchSettings.jsm", + "resource://gre/modules/SearchStaticData.jsm": "toolkit/components/search/SearchStaticData.jsm", + "resource://gre/modules/SearchSuggestionController.jsm": "toolkit/components/search/SearchSuggestionController.jsm", + "resource://gre/modules/SearchSuggestions.jsm": "toolkit/components/search/SearchSuggestions.jsm", + "resource://gre/modules/SearchUtils.jsm": "toolkit/components/search/SearchUtils.jsm", + "resource://gre/modules/SecurityInfo.jsm": "toolkit/components/extensions/webrequest/SecurityInfo.jsm", + "resource://gre/modules/SelectionUtils.jsm": "toolkit/modules/SelectionUtils.jsm", + "resource://gre/modules/ServiceRequest.jsm": "toolkit/modules/ServiceRequest.jsm", + "resource://gre/modules/ServiceWorkerCleanUp.jsm": "toolkit/components/cleardata/ServiceWorkerCleanUp.jsm", + "resource://gre/modules/Services.jsm": "toolkit/modules/Services.jsm", + "resource://gre/modules/SessionStoreFunctions.jsm": "toolkit/components/sessionstore/SessionStoreFunctions.jsm", + "resource://gre/modules/ShareDelegate.jsm": "mobile/android/components/geckoview/ShareDelegate.jsm", + "resource://gre/modules/SharedPromptUtils.jsm": "toolkit/components/prompts/src/PromptUtils.jsm", + "resource://gre/modules/ShieldContentProcess.jsm": "toolkit/components/normandy/ShieldContentProcess.jsm", + "resource://gre/modules/ShortcutUtils.jsm": "toolkit/modules/ShortcutUtils.jsm", + "resource://gre/modules/SimpleServices.jsm": "toolkit/components/utils/SimpleServices.jsm", + "resource://gre/modules/SignUpFormRuleset.jsm": "toolkit/components/passwordmgr/SignUpFormRuleset.jsm", + "resource://gre/modules/SlowScriptDebug.jsm": "dom/base/SlowScriptDebug.jsm", + "resource://gre/modules/Sqlite.jsm": "toolkit/modules/Sqlite.jsm", + "resource://gre/modules/SubDialog.jsm": "toolkit/modules/SubDialog.jsm", + "resource://gre/modules/Subprocess.jsm": "toolkit/modules/subprocess/Subprocess.jsm", + "resource://gre/modules/SyncedBookmarksMirror.jsm": "toolkit/components/places/SyncedBookmarksMirror.jsm", + "resource://gre/modules/TaggingService.jsm": "toolkit/components/places/TaggingService.jsm", + "resource://gre/modules/TaskScheduler.jsm": "toolkit/components/taskscheduler/TaskScheduler.jsm", + "resource://gre/modules/TaskSchedulerMacOSImpl.jsm": "toolkit/components/taskscheduler/TaskSchedulerMacOSImpl.jsm", + "resource://gre/modules/TaskSchedulerWinImpl.jsm": "toolkit/components/taskscheduler/TaskSchedulerWinImpl.jsm", + "resource://gre/modules/TelemetryArchive.jsm": "toolkit/components/telemetry/app/TelemetryArchive.jsm", + "resource://gre/modules/TelemetryController.jsm": "toolkit/components/telemetry/app/TelemetryController.jsm", + "resource://gre/modules/TelemetryControllerBase.jsm": "toolkit/components/telemetry/app/TelemetryControllerBase.jsm", + "resource://gre/modules/TelemetryControllerContent.jsm": "toolkit/components/telemetry/app/TelemetryControllerContent.jsm", + "resource://gre/modules/TelemetryControllerParent.jsm": "toolkit/components/telemetry/app/TelemetryControllerParent.jsm", + "resource://gre/modules/TelemetryEnvironment.jsm": "toolkit/components/telemetry/app/TelemetryEnvironment.jsm", + "resource://gre/modules/TelemetryReportingPolicy.jsm": "toolkit/components/telemetry/app/TelemetryReportingPolicy.jsm", + "resource://gre/modules/TelemetryScheduler.jsm": "toolkit/components/telemetry/app/TelemetryScheduler.jsm", + "resource://gre/modules/TelemetrySend.jsm": "toolkit/components/telemetry/app/TelemetrySend.jsm", + "resource://gre/modules/TelemetrySession.jsm": "toolkit/components/telemetry/pings/TelemetrySession.jsm", + "resource://gre/modules/TelemetryStartup.jsm": "toolkit/components/telemetry/TelemetryStartup.jsm", + "resource://gre/modules/TelemetryStorage.jsm": "toolkit/components/telemetry/app/TelemetryStorage.jsm", + "resource://gre/modules/TelemetryTimestamps.jsm": "toolkit/components/telemetry/app/TelemetryTimestamps.jsm", + "resource://gre/modules/TelemetryUtils.jsm": "toolkit/components/telemetry/app/TelemetryUtils.jsm", + "resource://gre/modules/TerminatorTelemetry.jsm": "toolkit/components/terminator/TerminatorTelemetry.jsm", + "resource://gre/modules/Timer.jsm": "toolkit/modules/Timer.jsm", + "resource://gre/modules/TooltipTextProvider.jsm": "toolkit/components/tooltiptext/TooltipTextProvider.jsm", + "resource://gre/modules/TrackingDBService.jsm": "toolkit/components/antitracking/TrackingDBService.jsm", + "resource://gre/modules/Troubleshoot.jsm": "toolkit/modules/Troubleshoot.jsm", + "resource://gre/modules/URIFixup.jsm": "docshell/base/URIFixup.jsm", + "resource://gre/modules/URLDecorationAnnotationsService.jsm": "toolkit/components/antitracking/URLDecorationAnnotationsService.jsm", + "resource://gre/modules/URLFormatter.jsm": "toolkit/components/urlformatter/URLFormatter.jsm", + "resource://gre/modules/URLQueryStrippingListService.jsm": "toolkit/components/antitracking/URLQueryStrippingListService.jsm", + "resource://gre/modules/UninstallPing.jsm": "toolkit/components/telemetry/pings/UninstallPing.jsm", + "resource://gre/modules/UntrustedModulesPing.jsm": "toolkit/components/telemetry/pings/UntrustedModulesPing.jsm", + "resource://gre/modules/UpdateListener.jsm": "toolkit/mozapps/update/UpdateListener.jsm", + "resource://gre/modules/UpdatePing.jsm": "toolkit/components/telemetry/pings/UpdatePing.jsm", + "resource://gre/modules/UpdateService.jsm": "toolkit/mozapps/update/UpdateService.jsm", + "resource://gre/modules/UpdateServiceStub.jsm": "toolkit/mozapps/update/UpdateServiceStub.jsm", + "resource://gre/modules/UpdateTelemetry.jsm": "toolkit/mozapps/update/UpdateTelemetry.jsm", + "resource://gre/modules/UpdateTimerManager.jsm": "toolkit/components/timermanager/UpdateTimerManager.jsm", + "resource://gre/modules/UpdateUtils.jsm": "toolkit/modules/UpdateUtils.jsm", + "resource://gre/modules/UrlClassifierExceptionListService.jsm": "netwerk/url-classifier/UrlClassifierExceptionListService.jsm", + "resource://gre/modules/UrlClassifierHashCompleter.jsm": "toolkit/components/url-classifier/UrlClassifierHashCompleter.jsm", + "resource://gre/modules/UrlClassifierLib.jsm": "toolkit/components/url-classifier/UrlClassifierLib.jsm", + "resource://gre/modules/UrlClassifierListManager.jsm": "toolkit/components/url-classifier/UrlClassifierListManager.jsm", + "resource://gre/modules/UserSearchEngine.jsm": "toolkit/components/search/UserSearchEngine.jsm", + "resource://gre/modules/ValueExtractor.jsm": "dom/manifest/ValueExtractor.jsm", + "resource://gre/modules/WebChannel.jsm": "toolkit/modules/WebChannel.jsm", + "resource://gre/modules/WebHandlerApp.jsm": "uriloader/exthandler/WebHandlerApp.jsm", + "resource://gre/modules/WebNavigation.jsm": "toolkit/components/extensions/WebNavigation.jsm", + "resource://gre/modules/WebNavigationFrames.jsm": "toolkit/components/extensions/WebNavigationFrames.jsm", + "resource://gre/modules/WebRequest.jsm": "toolkit/components/extensions/webrequest/WebRequest.jsm", + "resource://gre/modules/WebRequestUpload.jsm": "toolkit/components/extensions/webrequest/WebRequestUpload.jsm", + "resource://gre/modules/WebVTTParserWrapper.jsm": "dom/media/webvtt/WebVTTParserWrapper.jsm", + "resource://gre/modules/WellKnownOpportunisticUtils.jsm": "netwerk/protocol/http/WellKnownOpportunisticUtils.jsm", + "resource://gre/modules/WindowsRegistry.jsm": "toolkit/modules/WindowsRegistry.jsm", + "resource://gre/modules/XPCOMUtils.jsm": "js/xpconnect/loader/XPCOMUtils.jsm", + "resource://gre/modules/addonManager.js": "toolkit/mozapps/extensions/addonManager.js", + "resource://gre/modules/addons/AddonRepository.jsm": "toolkit/mozapps/extensions/internal/AddonRepository.jsm", + "resource://gre/modules/addons/AddonSettings.jsm": "toolkit/mozapps/extensions/internal/AddonSettings.jsm", + "resource://gre/modules/addons/AddonUpdateChecker.jsm": "toolkit/mozapps/extensions/internal/AddonUpdateChecker.jsm", + "resource://gre/modules/addons/GMPProvider.jsm": "toolkit/mozapps/extensions/internal/GMPProvider.jsm", + "resource://gre/modules/addons/ProductAddonChecker.jsm": "toolkit/mozapps/extensions/internal/ProductAddonChecker.jsm", + "resource://gre/modules/addons/XPIDatabase.jsm": "toolkit/mozapps/extensions/internal/XPIDatabase.jsm", + "resource://gre/modules/addons/XPIInstall.jsm": "toolkit/mozapps/extensions/internal/XPIInstall.jsm", + "resource://gre/modules/addons/XPIProvider.jsm": "toolkit/mozapps/extensions/internal/XPIProvider.jsm", + "resource://gre/modules/amContentHandler.jsm": "toolkit/mozapps/extensions/amContentHandler.jsm", + "resource://gre/modules/amInstallTrigger.jsm": "toolkit/mozapps/extensions/amInstallTrigger.jsm", + "resource://gre/modules/amWebAPI.jsm": "toolkit/mozapps/extensions/amWebAPI.jsm", + "resource://gre/modules/backgroundtasks/BackgroundTask_backgroundupdate.jsm": "toolkit/mozapps/update/BackgroundTask_backgroundupdate.jsm", + "resource://gre/modules/components-utils/ClientEnvironment.jsm": "toolkit/components/utils/ClientEnvironment.jsm", + "resource://gre/modules/components-utils/FilterExpressions.jsm": "toolkit/components/utils/FilterExpressions.jsm", + "resource://gre/modules/components-utils/JsonSchemaValidator.jsm": "toolkit/components/utils/JsonSchemaValidator.jsm", + "resource://gre/modules/components-utils/PreferenceFilters.jsm": "toolkit/components/utils/PreferenceFilters.jsm", + "resource://gre/modules/components-utils/Sampling.jsm": "toolkit/components/utils/Sampling.jsm", + "resource://gre/modules/components-utils/WindowsInstallsInfo.jsm": "toolkit/components/utils/WindowsInstallsInfo.jsm", + "resource://gre/modules/components-utils/WindowsVersionInfo.jsm": "toolkit/components/utils/WindowsVersionInfo.jsm", + "resource://gre/modules/components-utils/mozjexl.js": "toolkit/components/utils/mozjexl.js", + "resource://gre/modules/crypto-SDR.js": "toolkit/components/passwordmgr/crypto-SDR.js", + "resource://gre/modules/ctypes.jsm": "toolkit/components/ctypes/ctypes.jsm", + "resource://gre/modules/handlers/HandlerList.jsm": "uriloader/exthandler/HandlerList.jsm", + "resource://gre/modules/jsdebugger.jsm": "devtools/platform/jsdebugger.jsm", + "resource://gre/modules/kvstore.jsm": "toolkit/components/kvstore/kvstore.jsm", + "resource://gre/modules/lz4.js": "toolkit/components/lz4/lz4.js", + "resource://gre/modules/lz4_internal.js": "toolkit/components/lz4/lz4_internal.js", + "resource://gre/modules/media/IdpSandbox.jsm": "dom/media/IdpSandbox.jsm", + "resource://gre/modules/media/PeerConnection.jsm": "dom/media/PeerConnection.jsm", + "resource://gre/modules/media/PeerConnectionIdp.jsm": "dom/media/PeerConnectionIdp.jsm", + "resource://gre/modules/mozIntl.jsm": "toolkit/components/mozintl/mozIntl.jsm", + "resource://gre/modules/narrate/NarrateControls.jsm": "toolkit/components/narrate/NarrateControls.jsm", + "resource://gre/modules/narrate/Narrator.jsm": "toolkit/components/narrate/Narrator.jsm", + "resource://gre/modules/narrate/VoiceSelect.jsm": "toolkit/components/narrate/VoiceSelect.jsm", + "resource://gre/modules/netwerk-dns/PublicSuffixList.jsm": "netwerk/dns/PublicSuffixList.jsm", + "resource://gre/modules/nsAsyncShutdown.jsm": "toolkit/components/asyncshutdown/nsAsyncShutdown.jsm", + "resource://gre/modules/nsCrashMonitor.jsm": "toolkit/components/crashmonitor/nsCrashMonitor.jsm", + "resource://gre/modules/nsFormAutoCompleteResult.jsm": "toolkit/components/satchel/nsFormAutoCompleteResult.jsm", + "resource://gre/modules/pdfjs.js": "toolkit/components/pdfjs/pdfjs.js", + "resource://gre/modules/policies/WindowsGPOParser.jsm": "toolkit/components/enterprisepolicies/WindowsGPOParser.jsm", + "resource://gre/modules/policies/macOSPoliciesParser.jsm": "toolkit/components/enterprisepolicies/macOSPoliciesParser.jsm", + "resource://gre/modules/psm/DER.jsm": "security/manager/ssl/DER.jsm", + "resource://gre/modules/psm/RemoteSecuritySettings.jsm": "security/manager/ssl/RemoteSecuritySettings.jsm", + "resource://gre/modules/psm/X509.jsm": "security/manager/ssl/X509.jsm", + "resource://gre/modules/reader/ReaderWorker.jsm": "toolkit/components/reader/ReaderWorker.jsm", + "resource://gre/modules/reflect.jsm": "toolkit/components/reflect/reflect.jsm", + "resource://gre/modules/remotepagemanager/MessagePort.jsm": "toolkit/components/remotepagemanager/MessagePort.jsm", + "resource://gre/modules/remotepagemanager/RemotePageManagerChild.jsm": "toolkit/components/remotepagemanager/RemotePageManagerChild.jsm", + "resource://gre/modules/remotepagemanager/RemotePageManagerParent.jsm": "toolkit/components/remotepagemanager/RemotePageManagerParent.jsm", + "resource://gre/modules/services-automation/ServicesAutomation.jsm": "services/automation/ServicesAutomation.jsm", + "resource://gre/modules/sessionstore/PrivacyFilter.jsm": "toolkit/modules/sessionstore/PrivacyFilter.jsm", + "resource://gre/modules/sessionstore/PrivacyLevel.jsm": "toolkit/modules/sessionstore/PrivacyLevel.jsm", + "resource://gre/modules/sessionstore/SessionHistory.jsm": "toolkit/modules/sessionstore/SessionHistory.jsm", + "resource://gre/modules/sessionstore/Utils.jsm": "toolkit/modules/sessionstore/Utils.jsm", + "resource://gre/modules/storage-geckoview.js": "toolkit/components/passwordmgr/storage-geckoview.js", + "resource://gre/modules/storage-json.js": "toolkit/components/passwordmgr/storage-json.js", + "resource://gre/modules/subprocess/subprocess_common.jsm": "toolkit/modules/subprocess/subprocess_common.jsm", + "resource://gre/modules/subprocess/subprocess_unix.jsm": "toolkit/modules/subprocess/subprocess_unix.jsm", + "resource://gre/modules/subprocess/subprocess_win.jsm": "toolkit/modules/subprocess/subprocess_win.jsm", + "resource://gre/modules/third_party/fathom/fathom.jsm": "toolkit/modules/third_party/fathom/fathom.jsm", + "resource://gre/modules/third_party/jsesc/jsesc.js": "toolkit/modules/third_party/jsesc/jsesc.js", + "resource://gre/modules/translation/LanguageDetector.jsm": "toolkit/components/translation/LanguageDetector.jsm", + "resource://gre/modules/txEXSLTRegExFunctions.jsm": "dom/xslt/xslt/txEXSLTRegExFunctions.jsm", + "resource://gre/modules/vtt.jsm": "dom/media/webvtt/vtt.jsm", + "resource://messaging-system/lib/Logger.jsm": "toolkit/components/messaging-system/lib/Logger.jsm", + "resource://messaging-system/lib/SpecialMessageActions.jsm": "toolkit/components/messaging-system/lib/SpecialMessageActions.jsm", + "resource://messaging-system/targeting/Targeting.jsm": "toolkit/components/messaging-system/targeting/Targeting.jsm", + "resource://mozscreenshots/Screenshot.jsm": "browser/tools/mozscreenshots/mozscreenshots/extension/Screenshot.jsm", + "resource://mozscreenshots/TestRunner.jsm": "browser/tools/mozscreenshots/mozscreenshots/extension/TestRunner.jsm", + "resource://nimbus/ExperimentAPI.jsm": "toolkit/components/nimbus/ExperimentAPI.jsm", + "resource://nimbus/lib/ExperimentManager.jsm": "toolkit/components/nimbus/lib/ExperimentManager.jsm", + "resource://nimbus/lib/ExperimentStore.jsm": "toolkit/components/nimbus/lib/ExperimentStore.jsm", + "resource://nimbus/lib/RemoteSettingsExperimentLoader.jsm": "toolkit/components/nimbus/lib/RemoteSettingsExperimentLoader.jsm", + "resource://nimbus/lib/SharedDataMap.jsm": "toolkit/components/nimbus/lib/SharedDataMap.jsm", + "resource://normandy-content/AboutPages.jsm": "toolkit/components/normandy/content/AboutPages.jsm", + "resource://normandy-content/ShieldFrameChild.jsm": "toolkit/components/normandy/content/ShieldFrameChild.jsm", + "resource://normandy-content/ShieldFrameParent.jsm": "toolkit/components/normandy/content/ShieldFrameParent.jsm", + "resource://normandy-vendor/PropTypes.js": "toolkit/components/normandy/vendor/PropTypes.js", + "resource://normandy-vendor/React.js": "toolkit/components/normandy/vendor/React.js", + "resource://normandy-vendor/ReactDOM.js": "toolkit/components/normandy/vendor/ReactDOM.js", + "resource://normandy-vendor/classnames.js": "toolkit/components/normandy/vendor/classnames.js", + "resource://normandy/Normandy.jsm": "toolkit/components/normandy/Normandy.jsm", + "resource://normandy/NormandyMigrations.jsm": "toolkit/components/normandy/NormandyMigrations.jsm", + "resource://normandy/actions/AddonRollbackAction.jsm": "toolkit/components/normandy/actions/AddonRollbackAction.jsm", + "resource://normandy/actions/AddonRolloutAction.jsm": "toolkit/components/normandy/actions/AddonRolloutAction.jsm", + "resource://normandy/actions/BaseAction.jsm": "toolkit/components/normandy/actions/BaseAction.jsm", + "resource://normandy/actions/BaseStudyAction.jsm": "toolkit/components/normandy/actions/BaseStudyAction.jsm", + "resource://normandy/actions/BranchedAddonStudyAction.jsm": "toolkit/components/normandy/actions/BranchedAddonStudyAction.jsm", + "resource://normandy/actions/ConsoleLogAction.jsm": "toolkit/components/normandy/actions/ConsoleLogAction.jsm", + "resource://normandy/actions/MessagingExperimentAction.jsm": "toolkit/components/normandy/actions/MessagingExperimentAction.jsm", + "resource://normandy/actions/PreferenceExperimentAction.jsm": "toolkit/components/normandy/actions/PreferenceExperimentAction.jsm", + "resource://normandy/actions/PreferenceRollbackAction.jsm": "toolkit/components/normandy/actions/PreferenceRollbackAction.jsm", + "resource://normandy/actions/PreferenceRolloutAction.jsm": "toolkit/components/normandy/actions/PreferenceRolloutAction.jsm", + "resource://normandy/actions/ShowHeartbeatAction.jsm": "toolkit/components/normandy/actions/ShowHeartbeatAction.jsm", + "resource://normandy/actions/schemas/index.js": "toolkit/components/normandy/actions/schemas/index.js", + "resource://normandy/lib/ActionsManager.jsm": "toolkit/components/normandy/lib/ActionsManager.jsm", + "resource://normandy/lib/AddonRollouts.jsm": "toolkit/components/normandy/lib/AddonRollouts.jsm", + "resource://normandy/lib/AddonStudies.jsm": "toolkit/components/normandy/lib/AddonStudies.jsm", + "resource://normandy/lib/CleanupManager.jsm": "toolkit/components/normandy/lib/CleanupManager.jsm", + "resource://normandy/lib/ClientEnvironment.jsm": "toolkit/components/normandy/lib/ClientEnvironment.jsm", + "resource://normandy/lib/EventEmitter.jsm": "toolkit/components/normandy/lib/EventEmitter.jsm", + "resource://normandy/lib/Heartbeat.jsm": "toolkit/components/normandy/lib/Heartbeat.jsm", + "resource://normandy/lib/LogManager.jsm": "toolkit/components/normandy/lib/LogManager.jsm", + "resource://normandy/lib/NormandyAddonManager.jsm": "toolkit/components/normandy/lib/NormandyAddonManager.jsm", + "resource://normandy/lib/NormandyApi.jsm": "toolkit/components/normandy/lib/NormandyApi.jsm", + "resource://normandy/lib/NormandyUtils.jsm": "toolkit/components/normandy/lib/NormandyUtils.jsm", + "resource://normandy/lib/PrefUtils.jsm": "toolkit/components/normandy/lib/PrefUtils.jsm", + "resource://normandy/lib/PreferenceExperiments.jsm": "toolkit/components/normandy/lib/PreferenceExperiments.jsm", + "resource://normandy/lib/PreferenceRollouts.jsm": "toolkit/components/normandy/lib/PreferenceRollouts.jsm", + "resource://normandy/lib/RecipeRunner.jsm": "toolkit/components/normandy/lib/RecipeRunner.jsm", + "resource://normandy/lib/ShieldPreferences.jsm": "toolkit/components/normandy/lib/ShieldPreferences.jsm", + "resource://normandy/lib/Storage.jsm": "toolkit/components/normandy/lib/Storage.jsm", + "resource://normandy/lib/TelemetryEvents.jsm": "toolkit/components/normandy/lib/TelemetryEvents.jsm", + "resource://normandy/lib/Uptake.jsm": "toolkit/components/normandy/lib/Uptake.jsm", + "resource://pdf.js/PdfJs.jsm": "toolkit/components/pdfjs/content/PdfJs.jsm", + "resource://pdf.js/PdfJsDefaultPreferences.jsm": "toolkit/components/pdfjs/content/PdfJsDefaultPreferences.jsm", + "resource://pdf.js/PdfJsNetwork.jsm": "toolkit/components/pdfjs/content/PdfJsNetwork.jsm", + "resource://pdf.js/PdfJsTelemetry.jsm": "toolkit/components/pdfjs/content/PdfJsTelemetry.jsm", + "resource://pdf.js/PdfSandbox.jsm": "toolkit/components/pdfjs/content/PdfSandbox.jsm", + "resource://pdf.js/PdfStreamConverter.jsm": "toolkit/components/pdfjs/content/PdfStreamConverter.jsm", + "resource://pdf.js/PdfjsChild.jsm": "toolkit/components/pdfjs/content/PdfjsChild.jsm", + "resource://pdf.js/PdfjsParent.jsm": "toolkit/components/pdfjs/content/PdfjsParent.jsm", + "resource://pdf.js/build/pdf.sandbox.external.js": "toolkit/components/pdfjs/content/build/pdf.sandbox.external.js", + "resource://reftest/AsyncSpellCheckTestHelper.jsm": "editor/AsyncSpellCheckTestHelper.jsm", + "resource://testing-common/AsyncSpellCheckTestHelper.jsm": "editor/AsyncSpellCheckTestHelper.jsm", + "resource://reftest/PerTestCoverageUtils.jsm": "tools/code-coverage/PerTestCoverageUtils.jsm", + "resource://reftest/ReftestFissionChild.jsm": "layout/tools/reftest/ReftestFissionChild.jsm", + "resource://reftest/ReftestFissionParent.jsm": "layout/tools/reftest/ReftestFissionParent.jsm", + "resource://reftest/StructuredLog.jsm": "testing/modules/StructuredLog.jsm", + "resource://reftest/globals.jsm": "layout/tools/reftest/globals.jsm", + "resource://reftest/manifest.jsm": "layout/tools/reftest/manifest.jsm", + "resource://reftest/reftest.jsm": "layout/tools/reftest/reftest.jsm", + "resource://report-site-issue/tabExtrasActor.jsm": "browser/extensions/report-site-issue/experimentalAPIs/actors/tabExtrasActor.jsm", + "resource://services-automation/ServicesAutomation.jsm": "services/automation/ServicesAutomation.jsm", + "resource://services-common/async.js": "services/common/async.js", + "resource://services-common/hawkclient.js": "services/common/hawkclient.js", + "resource://services-common/hawkrequest.js": "services/common/hawkrequest.js", + "resource://services-common/kinto-http-client.js": "services/common/kinto-http-client.js", + "resource://services-common/kinto-offline-client.js": "services/common/kinto-offline-client.js", + "resource://services-common/kinto-storage-adapter.js": "services/common/kinto-storage-adapter.js", + "resource://services-common/logmanager.js": "services/common/logmanager.js", + "resource://services-common/observers.js": "services/common/observers.js", + "resource://services-common/rest.js": "services/common/rest.js", + "resource://services-common/tokenserverclient.js": "services/common/tokenserverclient.js", + "resource://services-common/uptake-telemetry.js": "services/common/uptake-telemetry.js", + "resource://services-common/utils.js": "services/common/utils.js", + "resource://services-crypto/WeaveCrypto.js": "services/crypto/modules/WeaveCrypto.js", + "resource://services-crypto/jwcrypto.jsm": "services/crypto/modules/jwcrypto.jsm", + "resource://services-crypto/utils.js": "services/crypto/modules/utils.js", + "resource://services-settings/Attachments.jsm": "services/settings/Attachments.jsm", + "resource://services-settings/Database.jsm": "services/settings/Database.jsm", + "resource://services-settings/IDBHelpers.jsm": "services/settings/IDBHelpers.jsm", + "resource://services-settings/RemoteSettingsClient.jsm": "services/settings/RemoteSettingsClient.jsm", + "resource://services-settings/RemoteSettingsComponents.jsm": "services/settings/RemoteSettingsComponents.jsm", + "resource://services-settings/RemoteSettingsWorker.jsm": "services/settings/RemoteSettingsWorker.jsm", + "resource://services-settings/SharedUtils.jsm": "services/settings/SharedUtils.jsm", + "resource://services-settings/SyncHistory.jsm": "services/settings/SyncHistory.jsm", + "resource://services-settings/Utils.jsm": "services/settings/Utils.jsm", + "resource://services-settings/remote-settings.js": "services/settings/remote-settings.js", + "resource://services-sync/SyncDisconnect.jsm": "services/sync/modules/SyncDisconnect.jsm", + "resource://services-sync/SyncedTabs.jsm": "services/sync/modules/SyncedTabs.jsm", + "resource://services-sync/UIState.jsm": "services/sync/modules/UIState.jsm", + "resource://services-sync/Weave.jsm": "services/sync/Weave.jsm", + "resource://services-sync/addonsreconciler.js": "services/sync/modules/addonsreconciler.js", + "resource://services-sync/addonutils.js": "services/sync/modules/addonutils.js", + "resource://services-sync/bridged_engine.js": "services/sync/modules/bridged_engine.js", + "resource://services-sync/collection_validator.js": "services/sync/modules/collection_validator.js", + "resource://services-sync/constants.js": "services/sync/modules/constants.js", + "resource://services-sync/doctor.js": "services/sync/modules/doctor.js", + "resource://services-sync/engines.js": "services/sync/modules/engines.js", + "resource://services-sync/engines/addons.js": "services/sync/modules/engines/addons.js", + "resource://services-sync/engines/bookmarks.js": "services/sync/modules/engines/bookmarks.js", + "resource://services-sync/engines/clients.js": "services/sync/modules/engines/clients.js", + "resource://services-sync/engines/extension-storage.js": "services/sync/modules/engines/extension-storage.js", + "resource://services-sync/engines/forms.js": "services/sync/modules/engines/forms.js", + "resource://services-sync/engines/history.js": "services/sync/modules/engines/history.js", + "resource://services-sync/engines/passwords.js": "services/sync/modules/engines/passwords.js", + "resource://services-sync/engines/prefs.js": "services/sync/modules/engines/prefs.js", + "resource://services-sync/engines/tabs.js": "services/sync/modules/engines/tabs.js", + "resource://services-sync/keys.js": "services/sync/modules/keys.js", + "resource://services-sync/main.js": "services/sync/modules/main.js", + "resource://services-sync/policies.js": "services/sync/modules/policies.js", + "resource://services-sync/record.js": "services/sync/modules/record.js", + "resource://services-sync/resource.js": "services/sync/modules/resource.js", + "resource://services-sync/service.js": "services/sync/modules/service.js", + "resource://services-sync/stages/declined.js": "services/sync/modules/stages/declined.js", + "resource://services-sync/stages/enginesync.js": "services/sync/modules/stages/enginesync.js", + "resource://services-sync/status.js": "services/sync/modules/status.js", + "resource://services-sync/sync_auth.js": "services/sync/modules/sync_auth.js", + "resource://services-sync/telemetry.js": "services/sync/modules/telemetry.js", + "resource://services-sync/util.js": "services/sync/modules/util.js", + "resource://specialpowers/AppTestDelegate.jsm": "testing/specialpowers/content/AppTestDelegate.jsm", + "resource://specialpowers/AppTestDelegateChild.jsm": "testing/specialpowers/content/AppTestDelegateChild.jsm", + "resource://specialpowers/AppTestDelegateParent.jsm": "testing/specialpowers/content/AppTestDelegateParent.jsm", + "resource://specialpowers/MockColorPicker.jsm": "testing/specialpowers/content/MockColorPicker.jsm", + "resource://specialpowers/MockFilePicker.jsm": "testing/specialpowers/content/MockFilePicker.jsm", + "resource://specialpowers/MockPermissionPrompt.jsm": "testing/specialpowers/content/MockPermissionPrompt.jsm", + "resource://specialpowers/SpecialPowersChild.jsm": "testing/specialpowers/content/SpecialPowersChild.jsm", + "resource://specialpowers/SpecialPowersEventUtils.jsm": "testing/specialpowers/content/SpecialPowersEventUtils.jsm", + "resource://specialpowers/SpecialPowersParent.jsm": "testing/specialpowers/content/SpecialPowersParent.jsm", + "resource://specialpowers/SpecialPowersSandbox.jsm": "testing/specialpowers/content/SpecialPowersSandbox.jsm", + "resource://specialpowers/WrapPrivileged.jsm": "testing/specialpowers/content/WrapPrivileged.jsm", + "resource://talos-powers/TalosParentProfiler.jsm": "testing/talos/talos/talos-powers/content/TalosParentProfiler.jsm", + "resource://test/AllowJavascriptChild.jsm": "docshell/test/unit/AllowJavascriptChild.jsm", + "resource://test/AllowJavascriptParent.jsm": "docshell/test/unit/AllowJavascriptParent.jsm", + "resource://test/Census.jsm": "devtools/shared/heapsnapshot/tests/xpcshell/Census.jsm", + "resource://test/CrashTestUtils.jsm": "toolkit/crashreporter/test/CrashTestUtils.jsm", + "resource://test/GlobalObjectsModule.jsm": "dom/indexedDB/test/unit/GlobalObjectsModule.jsm", + "resource://test/Match.jsm": "devtools/shared/heapsnapshot/tests/xpcshell/Match.jsm", + "resource://test/TestRunner.jsm": "browser/tools/mozscreenshots/mozscreenshots/extension/TestRunner.jsm", + "resource://test/broadcast_handler.jsm": "dom/push/test/xpcshell/broadcast_handler.jsm", + "resource://testing-common/AddonTestUtils.jsm": "toolkit/mozapps/extensions/internal/AddonTestUtils.jsm", + "resource://testing-common/AppData.jsm": "testing/modules/AppData.jsm", + "resource://testing-common/AppInfo.jsm": "testing/modules/AppInfo.jsm", + "resource://testing-common/Assert.jsm": "testing/modules/Assert.jsm", + "resource://testing-common/AsyncSpellCheckTestHelper.jsm": "editor/AsyncSpellCheckTestHelper.jsm", + "resource://testing-common/BackgroundTasksTestUtils.jsm": "toolkit/components/backgroundtasks/BackgroundTasksTestUtils.jsm", + "resource://testing-common/BrowserTestUtils.jsm": "testing/mochitest/BrowserTestUtils/BrowserTestUtils.jsm", + "resource://testing-common/BrowserTestUtilsChild.jsm": "testing/mochitest/BrowserTestUtils/BrowserTestUtilsChild.jsm", + "resource://testing-common/BrowserTestUtilsParent.jsm": "testing/mochitest/BrowserTestUtils/BrowserTestUtilsParent.jsm", + "resource://testing-common/ContentEventListenerChild.jsm": "testing/mochitest/BrowserTestUtils/ContentEventListenerChild.jsm", + "resource://testing-common/ContentEventListenerParent.jsm": "testing/mochitest/BrowserTestUtils/ContentEventListenerParent.jsm", + "resource://testing-common/ContentTask.jsm": "testing/mochitest/BrowserTestUtils/ContentTask.jsm", + "resource://testing-common/ContentTaskUtils.jsm": "testing/mochitest/BrowserTestUtils/ContentTaskUtils.jsm", + "resource://testing-common/CookieXPCShellUtils.jsm": "netwerk/cookie/CookieXPCShellUtils.jsm", + "resource://testing-common/CoverageUtils.jsm": "testing/modules/CoverageUtils.jsm", + "resource://testing-common/CrashManagerTest.jsm": "toolkit/components/crashes/CrashManagerTest.jsm", + "resource://testing-common/CustomizableUITestUtils.jsm": "browser/components/customizableui/test/CustomizableUITestUtils.jsm", + "resource://testing-common/DoHTestUtils.jsm": "browser/components/doh/DoHTestUtils.jsm", + "resource://testing-common/EnterprisePolicyTesting.jsm": "toolkit/components/enterprisepolicies/tests/EnterprisePolicyTesting.jsm", + "resource://testing-common/ExtensionTestCommon.jsm": "toolkit/components/extensions/ExtensionTestCommon.jsm", + "resource://testing-common/ExtensionXPCShellUtils.jsm": "toolkit/components/extensions/ExtensionXPCShellUtils.jsm", + "resource://testing-common/FileTestUtils.jsm": "testing/modules/FileTestUtils.jsm", + "resource://testing-common/FluentSyntax.jsm": "intl/l10n/FluentSyntax.jsm", + "resource://testing-common/FormHistoryTestUtils.jsm": "toolkit/components/satchel/test/FormHistoryTestUtils.jsm", + "resource://testing-common/HandlerServiceTestUtils.jsm": "uriloader/exthandler/tests/HandlerServiceTestUtils.jsm", + "resource://testing-common/ImportTesting.jsm": "testing/mochitest/tests/Harness_sanity/ImportTesting.jsm", + "resource://testing-common/LangPackMatcherTestUtils.jsm": "intl/locale/tests/LangPackMatcherTestUtils.jsm", + "resource://testing-common/LoginTestUtils.jsm": "toolkit/components/passwordmgr/test/LoginTestUtils.jsm", + "resource://testing-common/MessageChannel.jsm": "toolkit/components/extensions/MessageChannel.jsm", + "resource://testing-common/MockDocument.jsm": "toolkit/modules/tests/modules/MockDocument.jsm", + "resource://testing-common/MockFilePicker.jsm": "testing/specialpowers/content/MockFilePicker.jsm", + "resource://testing-common/MockRegistrar.jsm": "testing/modules/MockRegistrar.jsm", + "resource://testing-common/MockRegistry.jsm": "testing/modules/MockRegistry.jsm", + "resource://testing-common/NimbusTestUtils.jsm": "toolkit/components/nimbus/test/NimbusTestUtils.jsm", + "resource://testing-common/NormandyTestUtils.jsm": "toolkit/components/normandy/test/NormandyTestUtils.jsm", + "resource://testing-common/OSKeyStoreTestUtils.jsm": "toolkit/modules/tests/modules/OSKeyStoreTestUtils.jsm", + "resource://testing-common/PerTestCoverageUtils.jsm": "tools/code-coverage/PerTestCoverageUtils.jsm", + "resource://testing-common/PermissionTestUtils.jsm": "extensions/permissions/test/PermissionTestUtils.jsm", + "resource://testing-common/PlacesTestUtils.jsm": "toolkit/components/places/tests/PlacesTestUtils.jsm", + "resource://testing-common/PromiseTestUtils.jsm": "toolkit/modules/tests/modules/PromiseTestUtils.jsm", + "resource://testing-common/PromptTestUtils.jsm": "toolkit/components/prompts/test/PromptTestUtils.jsm", + "resource://testing-common/QuickSuggestTestUtils.jsm": "browser/components/urlbar/tests/quicksuggest/QuickSuggestTestUtils.jsm", + "resource://testing-common/RegionTestUtils.jsm": "toolkit/modules/tests/xpcshell/RegionTestUtils.jsm", + "resource://testing-common/SearchTestUtils.jsm": "toolkit/components/search/tests/SearchTestUtils.jsm", + "resource://testing-common/Sinon.jsm": "testing/modules/Sinon.jsm", + "resource://testing-common/SiteDataTestUtils.jsm": "toolkit/components/cleardata/SiteDataTestUtils.jsm", + "resource://testing-common/StructuredLog.jsm": "testing/modules/StructuredLog.jsm", + "resource://testing-common/TelemetryArchiveTesting.jsm": "toolkit/components/telemetry/tests/unit/TelemetryArchiveTesting.jsm", + "resource://testing-common/TelemetryEnvironmentTesting.jsm": "toolkit/components/telemetry/tests/unit/TelemetryEnvironmentTesting.jsm", + "resource://testing-common/TelemetryTestUtils.jsm": "toolkit/components/telemetry/tests/utils/TelemetryTestUtils.jsm", + "resource://testing-common/TestIntegration.jsm": "toolkit/modules/tests/xpcshell/TestIntegration.jsm", + "resource://testing-common/TestInterfaceJS.jsm": "dom/bindings/test/TestInterfaceJS.jsm", + "resource://testing-common/TestProcessActorChild.jsm": "toolkit/actors/TestProcessActorChild.jsm", + "resource://testing-common/TestProcessActorParent.jsm": "toolkit/actors/TestProcessActorParent.jsm", + "resource://testing-common/TestUtils.jsm": "testing/modules/TestUtils.jsm", + "resource://testing-common/TestWindowChild.jsm": "toolkit/actors/TestWindowChild.jsm", + "resource://testing-common/TestWindowParent.jsm": "toolkit/actors/TestWindowParent.jsm", + "resource://testing-common/UrlClassifierTestUtils.jsm": "toolkit/components/url-classifier/tests/UrlClassifierTestUtils.jsm", + "resource://testing-common/UrlbarTestUtils.jsm": "browser/components/urlbar/tests/UrlbarTestUtils.jsm", + "resource://testing-common/XPCShellContentUtils.jsm": "testing/modules/XPCShellContentUtils.jsm", + "resource://testing-common/backgroundtasks/BackgroundTask_shouldprocessupdates.jsm": "toolkit/components/backgroundtasks/tests/BackgroundTask_shouldprocessupdates.jsm", + "resource://testing-common/backgroundtasks/BackgroundTask_wait.jsm": "toolkit/components/backgroundtasks/tests/BackgroundTask_wait.jsm", + "resource://testing-common/cookie_filtering_helper.jsm": "netwerk/test/browser/cookie_filtering_helper.jsm", + "resource://testing-common/dom/quota/test/modules/ModuleLoader.jsm": "dom/quota/test/modules/system/ModuleLoader.jsm", + "resource://testing-common/dom/quota/test/modules/StorageUtils.jsm": "dom/quota/test/modules/system/StorageUtils.jsm", + "resource://testing-common/dom/quota/test/modules/WorkerDriver.jsm": "dom/quota/test/modules/system/WorkerDriver.jsm", + "resource://testing-common/early_hint_preload_test_helper.jsm": "netwerk/test/browser/early_hint_preload_test_helper.jsm", + "resource://testing-common/httpd.js": "netwerk/test/httpserver/httpd.js", + "resource://testing-common/services/common/logging.js": "services/common/modules-testing/logging.js", + "resource://testing-common/services/sync/fakeservices.js": "services/sync/modules-testing/fakeservices.js", + "resource://testing-common/services/sync/fxa_utils.js": "services/sync/modules-testing/fxa_utils.js", + "resource://testing-common/services/sync/rotaryengine.js": "services/sync/modules-testing/rotaryengine.js", + "resource://testing-common/services/sync/utils.js": "services/sync/modules-testing/utils.js", + "resource://tps/auth/fxaccounts.jsm": "services/sync/tps/extensions/tps/resource/auth/fxaccounts.jsm", + "resource://tps/logger.jsm": "services/sync/tps/extensions/tps/resource/logger.jsm", + "resource://tps/modules/addons.jsm": "services/sync/tps/extensions/tps/resource/modules/addons.jsm", + "resource://tps/modules/bookmarkValidator.jsm": "services/sync/tps/extensions/tps/resource/modules/bookmarkValidator.jsm", + "resource://tps/modules/bookmarks.jsm": "services/sync/tps/extensions/tps/resource/modules/bookmarks.jsm", + "resource://tps/modules/formautofill.jsm": "services/sync/tps/extensions/tps/resource/modules/formautofill.jsm", + "resource://tps/modules/forms.jsm": "services/sync/tps/extensions/tps/resource/modules/forms.jsm", + "resource://tps/modules/history.jsm": "services/sync/tps/extensions/tps/resource/modules/history.jsm", + "resource://tps/modules/passwords.jsm": "services/sync/tps/extensions/tps/resource/modules/passwords.jsm", + "resource://tps/modules/prefs.jsm": "services/sync/tps/extensions/tps/resource/modules/prefs.jsm", + "resource://tps/modules/tabs.jsm": "services/sync/tps/extensions/tps/resource/modules/tabs.jsm", + "resource://tps/modules/windows.jsm": "services/sync/tps/extensions/tps/resource/modules/windows.jsm", + "resource://tps/quit.js": "services/sync/tps/extensions/tps/resource/quit.js", + "resource://tps/tps.jsm": "services/sync/tps/extensions/tps/resource/tps.jsm", + "resource://webcompat/AboutCompat.jsm": "browser/extensions/webcompat/about-compat/AboutCompat.jsm", + "resource://testing-common/PerfTestHelpers.jsm": "browser/base/content/test/performance/PerfTestHelpers.jsm", + "resource:///modules/QuickActionsLoaderDefault.jsm": "browser/components/urlbar/QuickActionsLoaderDefault.jsm", + "resource:///modules/UrlbarProviderQuickActions.jsm": "browser/components/urlbar/UrlbarProviderQuickActions.jsm", + "resource://gre/modules/UrlClassifierRemoteSettingsService.jsm": "toolkit/components/url-classifier/UrlClassifierRemoteSettingsService.jsm", + + "resource:///modules/BrowserUsageTelemetry.jsm": [ + "browser/modules/BrowserUsageTelemetry.jsm", + "mobile/android/modules/geckoview/BrowserUsageTelemetry.jsm" + ], + "resource:///modules/ExtensionBrowsingData.jsm": [ + "browser/components/extensions/ExtensionBrowsingData.jsm", + "mobile/android/components/extensions/ExtensionBrowsingData.jsm" + ], + "resource://autofill/FormAutofillPrompter.jsm": [ + "toolkit/components/formautofill/android/FormAutofillPrompter.jsm", + "toolkit/components/formautofill/default/FormAutofillPrompter.jsm" + ], + "resource://autofill/FormAutofillStorage.jsm": [ + "toolkit/components/formautofill/android/FormAutofillStorage.jsm", + "toolkit/components/formautofill/default/FormAutofillStorage.jsm" + ], + "resource://gre/modules/NotificationDB.jsm": [ + "dom/notification/new/NotificationDB.jsm", + "dom/notification/old/NotificationDB.jsm" + ], + "resource://gre/modules/XULStore.jsm": [ + "toolkit/components/xulstore/new/XULStore.jsm", + "toolkit/components/xulstore/old/XULStore.jsm" + ], + "resource://testing-common/AppUiTestDelegate.jsm": [ + "browser/components/extensions/test/AppUiTestDelegate.jsm", + "mobile/android/modules/test/AppUiTestDelegate.jsm" + ] +} diff --git a/tools/esmify/package-lock.json b/tools/esmify/package-lock.json new file mode 100644 index 0000000000..dea539947b --- /dev/null +++ b/tools/esmify/package-lock.json @@ -0,0 +1,7895 @@ +{ + "name": "esmify", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "devDependencies": { + "jscodeshift": "^0.13.1" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", + "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.1.0", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", + "integrity": "sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==", + "dev": true, + "dependencies": { + "@babel/highlight": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.18.6.tgz", + "integrity": "sha512-tzulrgDT0QD6U7BJ4TKVk2SDDg7wlP39P9yAx1RfLy7vP/7rsDRlWVfbWxElslu56+r7QOhB2NSDsabYYruoZQ==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.18.6.tgz", + "integrity": "sha512-cQbWBpxcbbs/IUredIPkHiAGULLV8iwgNRMFzvbhEXISp4f3rUUXE5+TIw6KwUWUR3DwyI6gmBRnmAtYaWehwQ==", + "dev": true, + "dependencies": { + "@ampproject/remapping": "^2.1.0", + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helpers": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.1", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.18.6.tgz", + "integrity": "sha512-AIwwoOS8axIC5MZbhNHRLKi3D+DMpvDf9XUcu3pIVAfOHFT45f4AoDAltRbHIQomCipkCZxrNkfpOEHhJz/VKw==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6", + "@jridgewell/gen-mapping": "^0.3.0", + "jsesc": "^2.5.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/generator/node_modules/@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/helper-annotate-as-pure": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz", + "integrity": "sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-builder-binary-assignment-operator-visitor": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.18.6.tgz", + "integrity": "sha512-KT10c1oWEpmrIRYnthbzHgoOf6B+Xd6a5yhdbNtdhtG7aO1or5HViuf1TQR36xY/QprXA5nvxO6nAjhJ4y38jw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-explode-assignable-expression": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.18.6.tgz", + "integrity": "sha512-vFjbfhNCzqdeAtZflUFrG5YIFqGTqsctrtkZ1D/NB0mDW9TwW3GmmUepYY4G9wCET5rY5ugz4OGTcLd614IzQg==", + "dev": true, + "dependencies": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "browserslist": "^4.20.2", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-class-features-plugin": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.18.6.tgz", + "integrity": "sha512-YfDzdnoxHGV8CzqHGyCbFvXg5QESPFkXlHtvdCkesLjjVMT2Adxe4FGUR5ChIb3DxSaXO12iIOCWoXdsUVwnqw==", + "dev": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-member-expression-to-functions": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-regexp-features-plugin": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.18.6.tgz", + "integrity": "sha512-7LcpH1wnQLGrI+4v+nPp+zUvIkF9x0ddv1Hkdue10tg3gmRnLy97DXh4STiOf1qeIInyD69Qv5kKSZzKD8B/7A==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "regexpu-core": "^5.1.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-define-polyfill-provider": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.1.tgz", + "integrity": "sha512-J9hGMpJQmtWmj46B3kBHmL38UhJGhYX7eqkcq+2gsstyYt341HmPeWspihX43yVRA0mS+8GGk2Gckc7bY/HCmA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-compilation-targets": "^7.13.0", + "@babel/helper-module-imports": "^7.12.13", + "@babel/helper-plugin-utils": "^7.13.0", + "@babel/traverse": "^7.13.0", + "debug": "^4.1.1", + "lodash.debounce": "^4.0.8", + "resolve": "^1.14.2", + "semver": "^6.1.2" + }, + "peerDependencies": { + "@babel/core": "^7.4.0-0" + } + }, + "node_modules/@babel/helper-environment-visitor": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.6.tgz", + "integrity": "sha512-8n6gSfn2baOY+qlp+VSzsosjCVGFqWKmDF0cCWOybh52Dw3SEyoWR1KrhMJASjLwIEkkAufZ0xvr+SxLHSpy2Q==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-explode-assignable-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz", + "integrity": "sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-function-name": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.18.6.tgz", + "integrity": "sha512-0mWMxV1aC97dhjCah5U5Ua7668r5ZmSC2DLfH2EZnf9c3/dHZKiFa5pRLMH5tjSl471tY6496ZWk/kjNONBxhw==", + "dev": true, + "dependencies": { + "@babel/template": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-hoist-variables": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz", + "integrity": "sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-member-expression-to-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.18.6.tgz", + "integrity": "sha512-CeHxqwwipekotzPDUuJOfIMtcIHBuc7WAzLmTYWctVigqS5RktNMQ5bEwQSuGewzYnCtTWa3BARXeiLxDTv+Ng==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz", + "integrity": "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.18.6.tgz", + "integrity": "sha512-L//phhB4al5uucwzlimruukHB3jRd5JGClwRMD/ROrVjXfLqovYnvQrK/JK36WYyVwGGO7OD3kMyVTjx+WVPhw==", + "dev": true, + "dependencies": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/helper-validator-identifier": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-optimise-call-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.18.6.tgz", + "integrity": "sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.18.6.tgz", + "integrity": "sha512-gvZnm1YAAxh13eJdkb9EWHBnF3eAub3XTLCZEehHT2kWxiKVRL64+ae5Y6Ivne0mVHmMYKT+xWgZO+gQhuLUBg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-remap-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.18.6.tgz", + "integrity": "sha512-z5wbmV55TveUPZlCLZvxWHtrjuJd+8inFhk7DG0WW87/oJuGDcjDiu7HIvGcpf5464L6xKCg3vNkmlVVz9hwyQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-wrap-function": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-replace-supers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.18.6.tgz", + "integrity": "sha512-fTf7zoXnUGl9gF25fXCWE26t7Tvtyn6H4hkLSYhATwJvw2uYxd3aoXplMSe0g9XbwK7bmxNes7+FGO0rB/xC0g==", + "dev": true, + "dependencies": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-member-expression-to-functions": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-simple-access": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.18.6.tgz", + "integrity": "sha512-iNpIgTgyAvDQpDj76POqg+YEt8fPxx3yaNBg3S30dxNKm2SWfYhD0TGrK/Eu9wHpUW63VQU894TsTg+GLbUa1g==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.18.6.tgz", + "integrity": "sha512-4KoLhwGS9vGethZpAhYnMejWkX64wsnHPDwvOsKWU6Fg4+AlK2Jz3TyjQLMEPvz+1zemi/WBdkYxCD0bAfIkiw==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-split-export-declaration": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz", + "integrity": "sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==", + "dev": true, + "dependencies": { + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.18.6.tgz", + "integrity": "sha512-MmetCkz9ej86nJQV+sFCxoGGrUbU3q02kgLciwkrt9QqEB7cP39oKEY0PakknEO0Gu20SskMRi+AYZ3b1TpN9g==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz", + "integrity": "sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-wrap-function": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.18.6.tgz", + "integrity": "sha512-I5/LZfozwMNbwr/b1vhhuYD+J/mU+gfGAj5td7l5Rv9WYmH6i3Om69WGKNmlIpsVW/mF6O5bvTKbvDQZVgjqOw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-function-name": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.18.6.tgz", + "integrity": "sha512-vzSiiqbQOghPngUYt/zWGvK3LAsPhz55vc9XNN0xAl2gV4ieShI2OQli5duxWHD+72PZPTKAcfcZDE1Cwc5zsQ==", + "dev": true, + "dependencies": { + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz", + "integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.18.6", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.18.6.tgz", + "integrity": "sha512-uQVSa9jJUe/G/304lXspfWVpKpK4euFLgGiMQFOCpM/bgcAdeoHwi/OQz23O9GK2osz26ZiXRRV9aV+Yl1O8tw==", + "dev": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz", + "integrity": "sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.18.6.tgz", + "integrity": "sha512-Udgu8ZRgrBrttVz6A0EVL0SJ1z+RLbIeqsu632SA1hf0awEppD6TvdznoH+orIF8wtFFAV/Enmw9Y+9oV8TQcw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.13.0" + } + }, + "node_modules/@babel/plugin-proposal-async-generator-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.18.6.tgz", + "integrity": "sha512-WAz4R9bvozx4qwf74M+sfqPMKfSqwM0phxPTR6iJIi8robgzXwkEgmeJG1gEKhm6sDqT/U9aV3lfcqybIpev8w==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-class-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz", + "integrity": "sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==", + "dev": true, + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-class-static-block": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.18.6.tgz", + "integrity": "sha512-+I3oIiNxrCpup3Gi8n5IGMwj0gOCAjcJUSQEcotNnCCPMEnixawOQ+KeJPlgfjzx+FKQ1QSyZOWe7wmoJp7vhw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-class-static-block": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.12.0" + } + }, + "node_modules/@babel/plugin-proposal-dynamic-import": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.18.6.tgz", + "integrity": "sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-dynamic-import": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-export-namespace-from": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.6.tgz", + "integrity": "sha512-zr/QcUlUo7GPo6+X1wC98NJADqmy5QTFWWhqeQWiki4XHafJtLl/YMGkmRB2szDD2IYJCCdBTd4ElwhId9T7Xw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-json-strings": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.18.6.tgz", + "integrity": "sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-logical-assignment-operators": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.18.6.tgz", + "integrity": "sha512-zMo66azZth/0tVd7gmkxOkOjs2rpHyhpcFo565PUP37hSp6hSd9uUKIfTDFMz58BwqgQKhJ9YxtM5XddjXVn+Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz", + "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-numeric-separator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz", + "integrity": "sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-numeric-separator": "^7.10.4" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-object-rest-spread": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.18.6.tgz", + "integrity": "sha512-9yuM6wr4rIsKa1wlUAbZEazkCrgw2sMPEXCr4Rnwetu7cEW1NydkCWytLuYletbf8vFxdJxFhwEZqMpOx2eZyw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-transform-parameters": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-catch-binding": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.18.6.tgz", + "integrity": "sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-chaining": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.18.6.tgz", + "integrity": "sha512-PatI6elL5eMzoypFAiYDpYQyMtXTn+iMhuxxQt5mAXD4fEmKorpSI3PHd+i3JXBJN3xyA6MvJv7at23HffFHwA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6", + "@babel/plugin-syntax-optional-chaining": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-private-methods": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz", + "integrity": "sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-private-property-in-object": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.18.6.tgz", + "integrity": "sha512-9Rysx7FOctvT5ouj5JODjAFAkgGoudQuLPamZb0v1TGLpapdNaftzifU8NTWQm0IRjqoYypdrSmyWgkocDQ8Dw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-unicode-property-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.18.6.tgz", + "integrity": "sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-export-namespace-from": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz", + "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.3" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-flow": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz", + "integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-assertions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.18.6.tgz", + "integrity": "sha512-/DU3RXad9+bZwrgWJQKbr39gYbJpLJHezqEzRzi/BHRlJ9zsQb4CK2CA/5apllXNomwA1qHwzvHl+AdEmC5krQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.18.6.tgz", + "integrity": "sha512-mAWAuq4rvOepWCBid55JuRNvpTNf2UGVgoz4JV0fXEKolsVZDzsa4NqCef758WZJj/GDu0gVGItjKFiClTAmZA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-arrow-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.18.6.tgz", + "integrity": "sha512-9S9X9RUefzrsHZmKMbDXxweEH+YlE8JJEuat9FdvW9Qh1cw7W64jELCtWNkPBPX5En45uy28KGvA/AySqUh8CQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.18.6.tgz", + "integrity": "sha512-ARE5wZLKnTgPW7/1ftQmSi1CmkqqHo2DNmtztFhvgtOWSDfq0Cq9/9L+KnZNYSNrydBekhW3rwShduf59RoXag==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoped-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.18.6.tgz", + "integrity": "sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoping": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.18.6.tgz", + "integrity": "sha512-pRqwb91C42vs1ahSAWJkxOxU1RHWDn16XAa6ggQ72wjLlWyYeAcLvTtE0aM8ph3KNydy9CQF2nLYcjq1WysgxQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-classes": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.18.6.tgz", + "integrity": "sha512-XTg8XW/mKpzAF3actL554Jl/dOYoJtv3l8fxaEczpgz84IeeVf+T1u2CSvPHuZbt0w3JkIx4rdn/MRQI7mo0HQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-computed-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.18.6.tgz", + "integrity": "sha512-9repI4BhNrR0KenoR9vm3/cIc1tSBIo+u1WVjKCAynahj25O8zfbiE6JtAtHPGQSs4yZ+bA8mRasRP+qc+2R5A==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-destructuring": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.18.6.tgz", + "integrity": "sha512-tgy3u6lRp17ilY8r1kP4i2+HDUwxlVqq3RTc943eAWSzGgpU1qhiKpqZ5CMyHReIYPHdo3Kg8v8edKtDqSVEyQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-dotall-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.18.6.tgz", + "integrity": "sha512-6S3jpun1eEbAxq7TdjLotAsl4WpQI9DxfkycRcKrjhQYzU87qpXdknpBg/e+TdcMehqGnLFi7tnFUBR02Vq6wg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-duplicate-keys": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.18.6.tgz", + "integrity": "sha512-NJU26U/208+sxYszf82nmGYqVF9QN8py2HFTblPT9hbawi8+1C5a9JubODLTGFuT0qlkqVinmkwOD13s0sZktg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-exponentiation-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.18.6.tgz", + "integrity": "sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-builder-binary-assignment-operator-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-flow-strip-types": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.18.6.tgz", + "integrity": "sha512-wE0xtA7csz+hw4fKPwxmu5jnzAsXPIO57XnRwzXP3T19jWh1BODnPGoG9xKYwvAwusP7iUktHayRFbMPGtODaQ==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-flow": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-for-of": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.18.6.tgz", + "integrity": "sha512-WAjoMf4wIiSsy88KmG7tgj2nFdEK7E46tArVtcgED7Bkj6Fg/tG5SbvNIOKxbFS2VFgNh6+iaPswBeQZm4ox8w==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-function-name": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.18.6.tgz", + "integrity": "sha512-kJha/Gbs5RjzIu0CxZwf5e3aTTSlhZnHMT8zPWnJMjNpLOUgqevg+PN5oMH68nMCXnfiMo4Bhgxqj59KHTlAnA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.18.6.tgz", + "integrity": "sha512-x3HEw0cJZVDoENXOp20HlypIHfl0zMIhMVZEBVTfmqbObIpsMxMbmU5nOEO8R7LYT+z5RORKPlTI5Hj4OsO9/Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-member-expression-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.18.6.tgz", + "integrity": "sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-amd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.18.6.tgz", + "integrity": "sha512-Pra5aXsmTsOnjM3IajS8rTaLCy++nGM4v3YR4esk5PCsyg9z8NA5oQLwxzMUtDBd8F+UmVza3VxoAaWCbzH1rg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-commonjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.18.6.tgz", + "integrity": "sha512-Qfv2ZOWikpvmedXQJDSbxNqy7Xr/j2Y8/KfijM0iJyKkBTmWuvCA1yeH1yDM7NJhBW/2aXxeucLj6i80/LAJ/Q==", + "dev": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-systemjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.18.6.tgz", + "integrity": "sha512-UbPYpXxLjTw6w6yXX2BYNxF3p6QY225wcTkfQCy3OMnSlS/C3xGtwUjEzGkldb/sy6PWLiCQ3NbYfjWUTI3t4g==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-identifier": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-umd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.18.6.tgz", + "integrity": "sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.18.6.tgz", + "integrity": "sha512-UmEOGF8XgaIqD74bC8g7iV3RYj8lMf0Bw7NJzvnS9qQhM4mg+1WHKotUIdjxgD2RGrgFLZZPCFPFj3P/kVDYhg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-new-target": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.18.6.tgz", + "integrity": "sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-object-super": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.18.6.tgz", + "integrity": "sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-parameters": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.18.6.tgz", + "integrity": "sha512-FjdqgMv37yVl/gwvzkcB+wfjRI8HQmc5EgOG9iGNvUY1ok+TjsoaMP7IqCDZBhkFcM5f3OPVMs6Dmp03C5k4/A==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-property-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.18.6.tgz", + "integrity": "sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-regenerator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.18.6.tgz", + "integrity": "sha512-poqRI2+qiSdeldcz4wTSTXBRryoq3Gc70ye7m7UD5Ww0nE29IXqMl6r7Nd15WBgRd74vloEMlShtH6CKxVzfmQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "regenerator-transform": "^0.15.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-reserved-words": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.18.6.tgz", + "integrity": "sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-shorthand-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.18.6.tgz", + "integrity": "sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-spread": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.18.6.tgz", + "integrity": "sha512-ayT53rT/ENF8WWexIRg9AiV9h0aIteyWn5ptfZTZQrjk/+f3WdrJGCY4c9wcgl2+MKkKPhzbYp97FTsquZpDCw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-sticky-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.18.6.tgz", + "integrity": "sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-template-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.18.6.tgz", + "integrity": "sha512-UuqlRrQmT2SWRvahW46cGSany0uTlcj8NYOS5sRGYi8FxPYPoLd5DDmMd32ZXEj2Jq+06uGVQKHxa/hJx2EzKw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typeof-symbol": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.18.6.tgz", + "integrity": "sha512-7m71iS/QhsPk85xSjFPovHPcH3H9qeyzsujhTc+vcdnsXavoWYJ74zx0lP5RhpC5+iDnVLO+PPMHzC11qels1g==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.18.6.tgz", + "integrity": "sha512-ijHNhzIrLj5lQCnI6aaNVRtGVuUZhOXFLRVFs7lLrkXTHip4FKty5oAuQdk4tywG0/WjXmjTfQCWmuzrvFer1w==", + "dev": true, + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-typescript": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-escapes": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.18.6.tgz", + "integrity": "sha512-XNRwQUXYMP7VLuy54cr/KS/WeL3AZeORhrmeZ7iewgu+X2eBqmpaLI/hzqr9ZxCeUoq0ASK4GUzSM0BDhZkLFw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.18.6.tgz", + "integrity": "sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-env": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.18.6.tgz", + "integrity": "sha512-WrthhuIIYKrEFAwttYzgRNQ5hULGmwTj+D6l7Zdfsv5M7IWV/OZbUfbeL++Qrzx1nVJwWROIFhCHRYQV4xbPNw==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.18.6", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.18.6", + "@babel/plugin-proposal-async-generator-functions": "^7.18.6", + "@babel/plugin-proposal-class-properties": "^7.18.6", + "@babel/plugin-proposal-class-static-block": "^7.18.6", + "@babel/plugin-proposal-dynamic-import": "^7.18.6", + "@babel/plugin-proposal-export-namespace-from": "^7.18.6", + "@babel/plugin-proposal-json-strings": "^7.18.6", + "@babel/plugin-proposal-logical-assignment-operators": "^7.18.6", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", + "@babel/plugin-proposal-numeric-separator": "^7.18.6", + "@babel/plugin-proposal-object-rest-spread": "^7.18.6", + "@babel/plugin-proposal-optional-catch-binding": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.6", + "@babel/plugin-proposal-private-methods": "^7.18.6", + "@babel/plugin-proposal-private-property-in-object": "^7.18.6", + "@babel/plugin-proposal-unicode-property-regex": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3", + "@babel/plugin-syntax-import-assertions": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5", + "@babel/plugin-transform-arrow-functions": "^7.18.6", + "@babel/plugin-transform-async-to-generator": "^7.18.6", + "@babel/plugin-transform-block-scoped-functions": "^7.18.6", + "@babel/plugin-transform-block-scoping": "^7.18.6", + "@babel/plugin-transform-classes": "^7.18.6", + "@babel/plugin-transform-computed-properties": "^7.18.6", + "@babel/plugin-transform-destructuring": "^7.18.6", + "@babel/plugin-transform-dotall-regex": "^7.18.6", + "@babel/plugin-transform-duplicate-keys": "^7.18.6", + "@babel/plugin-transform-exponentiation-operator": "^7.18.6", + "@babel/plugin-transform-for-of": "^7.18.6", + "@babel/plugin-transform-function-name": "^7.18.6", + "@babel/plugin-transform-literals": "^7.18.6", + "@babel/plugin-transform-member-expression-literals": "^7.18.6", + "@babel/plugin-transform-modules-amd": "^7.18.6", + "@babel/plugin-transform-modules-commonjs": "^7.18.6", + "@babel/plugin-transform-modules-systemjs": "^7.18.6", + "@babel/plugin-transform-modules-umd": "^7.18.6", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.18.6", + "@babel/plugin-transform-new-target": "^7.18.6", + "@babel/plugin-transform-object-super": "^7.18.6", + "@babel/plugin-transform-parameters": "^7.18.6", + "@babel/plugin-transform-property-literals": "^7.18.6", + "@babel/plugin-transform-regenerator": "^7.18.6", + "@babel/plugin-transform-reserved-words": "^7.18.6", + "@babel/plugin-transform-shorthand-properties": "^7.18.6", + "@babel/plugin-transform-spread": "^7.18.6", + "@babel/plugin-transform-sticky-regex": "^7.18.6", + "@babel/plugin-transform-template-literals": "^7.18.6", + "@babel/plugin-transform-typeof-symbol": "^7.18.6", + "@babel/plugin-transform-unicode-escapes": "^7.18.6", + "@babel/plugin-transform-unicode-regex": "^7.18.6", + "@babel/preset-modules": "^0.1.5", + "@babel/types": "^7.18.6", + "babel-plugin-polyfill-corejs2": "^0.3.1", + "babel-plugin-polyfill-corejs3": "^0.5.2", + "babel-plugin-polyfill-regenerator": "^0.3.1", + "core-js-compat": "^3.22.1", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-flow": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-flow/-/preset-flow-7.18.6.tgz", + "integrity": "sha512-E7BDhL64W6OUqpuyHnSroLnqyRTcG6ZdOBl1OKI/QK/HJfplqK/S3sq1Cckx7oTodJ5yOXyfw7rEADJ6UjoQDQ==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-transform-flow-strip-types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-modules": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.5.tgz", + "integrity": "sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.4.4", + "@babel/plugin-transform-dotall-regex": "^7.4.4", + "@babel/types": "^7.4.4", + "esutils": "^2.0.2" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.18.6.tgz", + "integrity": "sha512-s9ik86kXBAnD760aybBucdpnLsAt0jK1xqJn2juOn9lkOvSHV60os5hxoVJsPzMQxvnUJFAlkont2DvvaYEBtQ==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-transform-typescript": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/register": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.18.6.tgz", + "integrity": "sha512-tkYtONzaO8rQubZzpBnvZPFcHgh8D9F55IjOsYton4X2IBoyRn2ZSWQqySTZnUn2guZbxbQiAB27hJEbvXamhQ==", + "dev": true, + "dependencies": { + "clone-deep": "^4.0.1", + "find-cache-dir": "^2.0.0", + "make-dir": "^2.1.0", + "pirates": "^4.0.5", + "source-map-support": "^0.5.16" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.6.tgz", + "integrity": "sha512-t9wi7/AW6XtKahAe20Yw0/mMljKq0B1r2fPdvaAdV/KPDZewFXdaaa6K7lxmZBZ8FBNpCiAT6iHPmd6QO9bKfQ==", + "dev": true, + "peer": true, + "dependencies": { + "regenerator-runtime": "^0.13.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.6.tgz", + "integrity": "sha512-JoDWzPe+wgBsTTgdnIma3iHNFC7YVJoPssVBDjiHfNlyt4YcunDtcDOUmfVDfCK5MfdsaIoX9PkijPhjH3nYUw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/types": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.18.6.tgz", + "integrity": "sha512-zS/OKyqmD7lslOtFqbscH6gMLFYOfG1YPqCKfAW5KrTeolKqvB8UelR49Fpr6y93kYkW2Ik00mT1LOGiAGvizw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/types": "^7.18.6", + "debug": "^4.1.0", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.18.6.tgz", + "integrity": "sha512-NdBNzPDwed30fZdDQtVR7ZgaO4UKjuaQFH9VArS+HMnurlOY0JWN+4ROlu/iapMFwjRQU4pOG4StZfDmulEwGA==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.18.6", + "to-fast-properties": "^2.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", + "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.0.0", + "@jridgewell/sourcemap-codec": "^1.4.10" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.8.tgz", + "integrity": "sha512-YK5G9LaddzGbcucK4c8h5tWFmMPBvRZ/uyWmN1/SbBdIvqGUdWGkJ5BAaccgs6XbzVLsqbPJrBSFwKv3kT9i7w==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.14", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.14.tgz", + "integrity": "sha512-bJWEfQ9lPTvm3SneWwRFVLzrh6nhjwqw7TUFFBEMzwvg7t7PCDenf2lDwqo4NQXzdpgBXyFgDWnQA+2vkruksQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/arr-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", + "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/arr-flatten": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-unique": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", + "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/assign-symbols": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", + "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ast-types": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.14.2.tgz", + "integrity": "sha512-O0yuUDnZeQDL+ncNGlJ78BiO4jnYI3bvMsD5prT0/nsgijG/LpNBIr63gTjVTNsiGkgQhiyCShTgxt8oXOrklA==", + "dev": true, + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/atob": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "dev": true, + "bin": { + "atob": "bin/atob.js" + }, + "engines": { + "node": ">= 4.5.0" + } + }, + "node_modules/babel-core": { + "version": "7.0.0-bridge.0", + "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz", + "integrity": "sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==", + "dev": true, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/babel-plugin-dynamic-import-node": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", + "dev": true, + "dependencies": { + "object.assign": "^4.1.0" + } + }, + "node_modules/babel-plugin-polyfill-corejs2": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.1.tgz", + "integrity": "sha512-v7/T6EQcNfVLfcN2X8Lulb7DjprieyLWJK/zOWH5DUYcAgex9sP3h25Q+DLsX9TloXe3y1O8l2q2Jv9q8UVB9w==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/compat-data": "^7.13.11", + "@babel/helper-define-polyfill-provider": "^0.3.1", + "semver": "^6.1.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/babel-plugin-polyfill-corejs3": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.5.2.tgz", + "integrity": "sha512-G3uJih0XWiID451fpeFaYGVuxHEjzKTHtc9uGFEjR6hHrvNzeS/PX+LLLcetJcytsB5m4j+K3o/EpXJNb/5IEQ==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.3.1", + "core-js-compat": "^3.21.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/babel-plugin-polyfill-regenerator": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.3.1.tgz", + "integrity": "sha512-Y2B06tvgHYt1x0yz17jGkGeeMr5FeKUu+ASJ+N6nB5lQ8Dapfg42i0OVrf8PNGJ3zKL4A23snMi1IRwrqqND7A==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.3.1" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/base": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "dev": true, + "dependencies": { + "cache-base": "^1.0.1", + "class-utils": "^0.3.5", + "component-emitter": "^1.2.1", + "define-property": "^1.0.0", + "isobject": "^3.0.1", + "mixin-deep": "^1.2.0", + "pascalcase": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, + "dependencies": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/braces/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/braces/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/browserslist": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.0.tgz", + "integrity": "sha512-UQxE0DIhRB5z/zDz9iA03BOfxaN2+GQdBYH/2WrSIWEUrnpzTPJbhqt+umq6r3acaPRTW1FNTkrcp0PXgtFkvA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001358", + "electron-to-chromium": "^1.4.164", + "node-releases": "^2.0.5", + "update-browserslist-db": "^1.0.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, + "node_modules/cache-base": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "dev": true, + "dependencies": { + "collection-visit": "^1.0.0", + "component-emitter": "^1.2.1", + "get-value": "^2.0.6", + "has-value": "^1.0.0", + "isobject": "^3.0.1", + "set-value": "^2.0.0", + "to-object-path": "^0.3.0", + "union-value": "^1.0.0", + "unset-value": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001359", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001359.tgz", + "integrity": "sha512-Xln/BAsPzEuiVLgJ2/45IaqD9jShtk3Y33anKb4+yLwQzws3+v6odKfpgES/cDEaZMLzSChpIGdbOYtH9MyuHw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + } + ] + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/class-utils": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "dev": true, + "dependencies": { + "arr-union": "^3.1.0", + "define-property": "^0.2.5", + "isobject": "^3.0.0", + "static-extend": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/collection-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", + "integrity": "sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==", + "dev": true, + "dependencies": { + "map-visit": "^1.0.0", + "object-visit": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true + }, + "node_modules/component-emitter": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/convert-source-map": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz", + "integrity": "sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.1" + } + }, + "node_modules/copy-descriptor": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", + "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/core-js-compat": { + "version": "3.23.3", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.23.3.tgz", + "integrity": "sha512-WSzUs2h2vvmKsacLHNTdpyOC9k43AEhcGoFlVgCY4L7aw98oSBKtPL6vD0/TqZjRWRQYdDSLkzZIni4Crbbiqw==", + "dev": true, + "peer": true, + "dependencies": { + "browserslist": "^4.21.0", + "semver": "7.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-js-compat/node_modules/semver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz", + "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==", + "dev": true, + "peer": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "dev": true, + "dependencies": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-property": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.2", + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.4.172", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.172.tgz", + "integrity": "sha512-yDoFfTJnqBAB6hSiPvzmsBJSrjOXJtHSJoqJdI/zSIh7DYupYnIOHt/bbPw/WE31BJjNTybDdNAs21gCMnTh0Q==", + "dev": true + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", + "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==", + "dev": true, + "dependencies": { + "debug": "^2.3.3", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "posix-character-classes": "^0.1.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/expand-brackets/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/extend-shallow": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", + "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==", + "dev": true, + "dependencies": { + "assign-symbols": "^1.0.0", + "is-extendable": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", + "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dev": true, + "dependencies": { + "array-unique": "^0.3.2", + "define-property": "^1.0.0", + "expand-brackets": "^2.1.4", + "extend-shallow": "^2.0.1", + "fragment-cache": "^0.2.1", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fill-range/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fill-range/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/flow-parser": { + "version": "0.181.1", + "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.181.1.tgz", + "integrity": "sha512-+Mx87/GkmF5+FHk8IXc5WppD/oC4wB+05MuIv7qmIMgThND3RhOBGl7Npyc2L7NLVenme00ZlwEKVieiMz4bqA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fragment-cache": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", + "integrity": "sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==", + "dev": true, + "dependencies": { + "map-cache": "^0.2.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-value": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", + "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "dev": true + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", + "integrity": "sha512-IBXk4GTsLYdQ7Rvt+GRBrFSVEkmuOUy4re0Xjd9kJSUQpnTrWR4/y9RpfexN9vkAPMFuQoeWKwqzPozRTlasGw==", + "dev": true, + "dependencies": { + "get-value": "^2.0.6", + "has-values": "^1.0.0", + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", + "integrity": "sha512-ODYZC64uqzmtfGMEAX/FvZiRyWLpAC3vYnNunURUnkGVTS+mI0smVsWaPydRBsE3g+ok7h960jChO8mFcWlHaQ==", + "dev": true, + "dependencies": { + "is-number": "^3.0.0", + "kind-of": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values/node_modules/kind-of": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", + "integrity": "sha512-24XsCxmEbRwEDbz/qz3stgin8TTzZ1ESR56OMCN0ujYg+vRutNSiOj9bHH9u85DKgXguraugV5sFuvbD4FW/hw==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true + }, + "node_modules/is-core-module": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", + "integrity": "sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A==", + "dev": true, + "peer": true, + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "node_modules/jscodeshift": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/jscodeshift/-/jscodeshift-0.13.1.tgz", + "integrity": "sha512-lGyiEbGOvmMRKgWk4vf+lUrCWO/8YR8sUR3FKF1Cq5fovjZDlIcw3Hu5ppLHAnEXshVffvaM0eyuY/AbOeYpnQ==", + "dev": true, + "dependencies": { + "@babel/core": "^7.13.16", + "@babel/parser": "^7.13.16", + "@babel/plugin-proposal-class-properties": "^7.13.0", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", + "@babel/plugin-proposal-optional-chaining": "^7.13.12", + "@babel/plugin-transform-modules-commonjs": "^7.13.8", + "@babel/preset-flow": "^7.13.13", + "@babel/preset-typescript": "^7.13.0", + "@babel/register": "^7.13.16", + "babel-core": "^7.0.0-bridge.0", + "chalk": "^4.1.2", + "flow-parser": "0.*", + "graceful-fs": "^4.2.4", + "micromatch": "^3.1.10", + "neo-async": "^2.5.0", + "node-dir": "^0.1.17", + "recast": "^0.20.4", + "temp": "^0.8.4", + "write-file-atomic": "^2.3.0" + }, + "bin": { + "jscodeshift": "bin/jscodeshift.js" + }, + "peerDependencies": { + "@babel/preset-env": "^7.1.6" + } + }, + "node_modules/jscodeshift/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/jscodeshift/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/jscodeshift/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/jscodeshift/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/jscodeshift/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/jscodeshift/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true, + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "dev": true, + "peer": true + }, + "node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/map-cache": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", + "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/map-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", + "integrity": "sha512-4y7uGv8bd2WdM9vpQsiQNo41Ln1NvhvDRuVt0k2JZQ+ezN2uaQes7lZeZ+QQUHOLQAtDaBJ+7wCbi+ab/KFs+w==", + "dev": true, + "dependencies": { + "object-visit": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mixin-deep": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "dev": true, + "dependencies": { + "for-in": "^1.0.2", + "is-extendable": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/nanomatch": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "dev": true, + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "fragment-cache": "^0.2.1", + "is-windows": "^1.0.2", + "kind-of": "^6.0.2", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true + }, + "node_modules/node-dir": { + "version": "0.1.17", + "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", + "integrity": "sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==", + "dev": true, + "dependencies": { + "minimatch": "^3.0.2" + }, + "engines": { + "node": ">= 0.10.5" + } + }, + "node_modules/node-releases": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.5.tgz", + "integrity": "sha512-U9h1NLROZTq9uE1SNffn6WuPDg8icmi3ns4rEl/oTfIle4iLjTliCzgTsbaIFMq/Xn078/lfY/BL0GWZ+psK4Q==", + "dev": true + }, + "node_modules/object-copy": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", + "integrity": "sha512-79LYn6VAb63zgtmAteVOWo9Vdj71ZVBy3Pbse+VqxDpEP83XuujMrGqHIwAXJ5I/aM0zU7dIyIAhifVTPrNItQ==", + "dev": true, + "dependencies": { + "copy-descriptor": "^0.1.0", + "define-property": "^0.2.5", + "kind-of": "^3.0.3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/is-descriptor/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object-visit": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", + "integrity": "sha512-GBaMwwAVK9qbQN3Scdo0OyvgPW7l3lnaVMj84uTOZlswkX0KpF6fyDBJhtTthf7pymztoN36/KEr1DyhF96zEA==", + "dev": true, + "dependencies": { + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/object.pick": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", + "integrity": "sha512-tqa/UMy/CCoYmj+H5qc07qvSL9dqcs/WZENZ1JbtWBlATP+iVOe778gE6MSijnyCnORzDuX6hU+LA4SZ09YjFQ==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pascalcase": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", + "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "peer": true + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pirates": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz", + "integrity": "sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/posix-character-classes": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", + "integrity": "sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/recast": { + "version": "0.20.5", + "resolved": "https://registry.npmjs.org/recast/-/recast-0.20.5.tgz", + "integrity": "sha512-E5qICoPoNL4yU0H0NoBDntNB0Q5oMSNh9usFctYniLBluTthi3RsQVBXIJNbApOlvSwW/RGxIuokPcAc59J5fQ==", + "dev": true, + "dependencies": { + "ast-types": "0.14.2", + "esprima": "~4.0.0", + "source-map": "~0.6.1", + "tslib": "^2.0.1" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==", + "dev": true, + "peer": true + }, + "node_modules/regenerate-unicode-properties": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.0.1.tgz", + "integrity": "sha512-vn5DU6yg6h8hP/2OkQo3K7uVILvY4iu0oI4t3HFa81UPkhGJwkRwM10JEc3upjdhHjs/k8GJY1sRBhk5sr69Bw==", + "dev": true, + "peer": true, + "dependencies": { + "regenerate": "^1.4.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==", + "dev": true, + "peer": true + }, + "node_modules/regenerator-transform": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.0.tgz", + "integrity": "sha512-LsrGtPmbYg19bcPHwdtmXwbW+TqNvtY4riE3P83foeHRroMbH6/2ddFBfab3t7kbzc7v7p4wbkIecHImqt0QNg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/runtime": "^7.8.4" + } + }, + "node_modules/regex-not": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "dev": true, + "dependencies": { + "extend-shallow": "^3.0.2", + "safe-regex": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/regexpu-core": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-5.1.0.tgz", + "integrity": "sha512-bb6hk+xWd2PEOkj5It46A16zFMs2mv86Iwpdu94la4S3sJ7C973h2dHpYKwIBGaWSO7cIRJ+UX0IeMaWcO4qwA==", + "dev": true, + "peer": true, + "dependencies": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.0.1", + "regjsgen": "^0.6.0", + "regjsparser": "^0.8.2", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regjsgen": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.6.0.tgz", + "integrity": "sha512-ozE883Uigtqj3bx7OhL1KNbCzGyW2NQZPl6Hs09WTvCuZD5sTI4JY58bkbQWa/Y9hxIsvJ3M8Nbf7j54IqeZbA==", + "dev": true, + "peer": true + }, + "node_modules/regjsparser": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.8.4.tgz", + "integrity": "sha512-J3LABycON/VNEu3abOviqGHuB/LOtOQj8SKmfP9anY5GfAVw/SPjwzSjxGjbZXIxbGfqTHtJw58C2Li/WkStmA==", + "dev": true, + "peer": true, + "dependencies": { + "jsesc": "~0.5.0" + }, + "bin": { + "regjsparser": "bin/parser" + } + }, + "node_modules/regjsparser/node_modules/jsesc": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "integrity": "sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==", + "dev": true, + "peer": true, + "bin": { + "jsesc": "bin/jsesc" + } + }, + "node_modules/repeat-element": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.4.tgz", + "integrity": "sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/resolve": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "dev": true, + "peer": true, + "dependencies": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==", + "deprecated": "https://github.com/lydell/resolve-url#deprecated", + "dev": true + }, + "node_modules/ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "dev": true, + "engines": { + "node": ">=0.12" + } + }, + "node_modules/rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/safe-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha512-aJXcif4xnaNUzvUuC5gcb46oTS7zvg4jpMTnuqtrEPlR3vFr4pxtdTwaF1Qs3Enjn9HK+ZlwQui+a7z0SywIzg==", + "dev": true, + "dependencies": { + "ret": "~0.1.10" + } + }, + "node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/set-value": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "is-extendable": "^0.1.1", + "is-plain-object": "^2.0.3", + "split-string": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/set-value/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/set-value/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "node_modules/snapdragon": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "dev": true, + "dependencies": { + "base": "^0.11.1", + "debug": "^2.2.0", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "map-cache": "^0.2.2", + "source-map": "^0.5.6", + "source-map-resolve": "^0.5.0", + "use": "^3.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "dev": true, + "dependencies": { + "define-property": "^1.0.0", + "isobject": "^3.0.0", + "snapdragon-util": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-util": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "dev": true, + "dependencies": { + "kind-of": "^3.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-util/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/snapdragon/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/snapdragon/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-resolve": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", + "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "deprecated": "See https://github.com/lydell/source-map-resolve#deprecated", + "dev": true, + "dependencies": { + "atob": "^2.1.2", + "decode-uri-component": "^0.2.0", + "resolve-url": "^0.2.1", + "source-map-url": "^0.4.0", + "urix": "^0.1.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/source-map-url": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.1.tgz", + "integrity": "sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw==", + "deprecated": "See https://github.com/lydell/source-map-url#deprecated", + "dev": true + }, + "node_modules/split-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "dev": true, + "dependencies": { + "extend-shallow": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", + "integrity": "sha512-72E9+uLc27Mt718pMHt9VMNiAL4LMsmDbBva8mxWUCkT07fSzEGMYUCk0XWY6lp0j6RBAG4cJ3mWuZv2OE3s0g==", + "dev": true, + "dependencies": { + "define-property": "^0.2.5", + "object-copy": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "peer": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/temp": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/temp/-/temp-0.8.4.tgz", + "integrity": "sha512-s0ZZzd0BzYv5tLSptZooSjK8oj6C+c19p7Vqta9+6NPOf7r+fxq0cJe6/oN4LTC79sy5NY8ucOJNgwsKCSbfqg==", + "dev": true, + "dependencies": { + "rimraf": "~2.6.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/to-object-path": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", + "integrity": "sha512-9mWHdnGRuh3onocaHzukyvCZhzvr6tiflAy/JRFXcJX0TjgfWA9pk9t8CMbzmBE4Jfw58pXbkngtBtqYxzNEyg==", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-object-path/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-regex": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "dev": true, + "dependencies": { + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "regex-not": "^1.0.2", + "safe-regex": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==", + "dev": true, + "dependencies": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, + "node_modules/unicode-canonical-property-names-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz", + "integrity": "sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "dev": true, + "peer": true, + "dependencies": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-value-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.0.0.tgz", + "integrity": "sha512-7Yhkc0Ye+t4PNYzOGKedDhXbYIBe1XEQYQxOPyhcXNMJ0WCABqqj6ckydd6pWRZTHV4GuCPKdBAUiMc60tsKVw==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-property-aliases-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.0.0.tgz", + "integrity": "sha512-5Zfuy9q/DFr4tfO7ZPeVXb1aPoeQSdeFMLpYuFebehDAhbuevLs5yxSZmIFN1tP5F9Wl4IpJrYojg85/zgyZHQ==", + "dev": true, + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/union-value": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "dev": true, + "dependencies": { + "arr-union": "^3.1.0", + "get-value": "^2.0.6", + "is-extendable": "^0.1.1", + "set-value": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/union-value/node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", + "integrity": "sha512-PcA2tsuGSF9cnySLHTLSh2qrQiJ70mn+r+Glzxv2TWZblxsxCC52BDlZoPCsz7STd9pN7EZetkWZBAvk4cgZdQ==", + "dev": true, + "dependencies": { + "has-value": "^0.3.1", + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-value": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", + "integrity": "sha512-gpG936j8/MzaeID5Yif+577c17TxaDmhuyVgSwtnL/q8UUTySg8Mecb+8Cf1otgLoD7DDH75axp86ER7LFsf3Q==", + "dev": true, + "dependencies": { + "get-value": "^2.0.3", + "has-values": "^0.1.4", + "isobject": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-value/node_modules/isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "dev": true, + "dependencies": { + "isarray": "1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-values": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", + "integrity": "sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.4.tgz", + "integrity": "sha512-jnmO2BEGUjsMOe/Fg9u0oczOe/ppIDZPebzccl1yDWGLFP16Pa1/RM5wEoKYPG2zstNcDuAStejyxsOuKINdGA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + }, + "bin": { + "browserslist-lint": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/urix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", + "integrity": "sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==", + "deprecated": "Please see https://github.com/lydell/urix#deprecated", + "dev": true + }, + "node_modules/use": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + } + }, + "dependencies": { + "@ampproject/remapping": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", + "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", + "dev": true, + "requires": { + "@jridgewell/gen-mapping": "^0.1.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@babel/code-frame": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", + "integrity": "sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==", + "dev": true, + "requires": { + "@babel/highlight": "^7.18.6" + } + }, + "@babel/compat-data": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.18.6.tgz", + "integrity": "sha512-tzulrgDT0QD6U7BJ4TKVk2SDDg7wlP39P9yAx1RfLy7vP/7rsDRlWVfbWxElslu56+r7QOhB2NSDsabYYruoZQ==", + "dev": true + }, + "@babel/core": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.18.6.tgz", + "integrity": "sha512-cQbWBpxcbbs/IUredIPkHiAGULLV8iwgNRMFzvbhEXISp4f3rUUXE5+TIw6KwUWUR3DwyI6gmBRnmAtYaWehwQ==", + "dev": true, + "requires": { + "@ampproject/remapping": "^2.1.0", + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helpers": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.1", + "semver": "^6.3.0" + } + }, + "@babel/generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.18.6.tgz", + "integrity": "sha512-AIwwoOS8axIC5MZbhNHRLKi3D+DMpvDf9XUcu3pIVAfOHFT45f4AoDAltRbHIQomCipkCZxrNkfpOEHhJz/VKw==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6", + "@jridgewell/gen-mapping": "^0.3.0", + "jsesc": "^2.5.1" + }, + "dependencies": { + "@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dev": true, + "requires": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + } + } + } + }, + "@babel/helper-annotate-as-pure": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.18.6.tgz", + "integrity": "sha512-duORpUiYrEpzKIop6iNbjnwKLAKnJ47csTyRACyEmWj0QdUrm5aqNJGHSSEQSUAvNW0ojX0dOmK9dZduvkfeXA==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-builder-binary-assignment-operator-visitor": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-builder-binary-assignment-operator-visitor/-/helper-builder-binary-assignment-operator-visitor-7.18.6.tgz", + "integrity": "sha512-KT10c1oWEpmrIRYnthbzHgoOf6B+Xd6a5yhdbNtdhtG7aO1or5HViuf1TQR36xY/QprXA5nvxO6nAjhJ4y38jw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-explode-assignable-expression": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-compilation-targets": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.18.6.tgz", + "integrity": "sha512-vFjbfhNCzqdeAtZflUFrG5YIFqGTqsctrtkZ1D/NB0mDW9TwW3GmmUepYY4G9wCET5rY5ugz4OGTcLd614IzQg==", + "dev": true, + "requires": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "browserslist": "^4.20.2", + "semver": "^6.3.0" + } + }, + "@babel/helper-create-class-features-plugin": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.18.6.tgz", + "integrity": "sha512-YfDzdnoxHGV8CzqHGyCbFvXg5QESPFkXlHtvdCkesLjjVMT2Adxe4FGUR5ChIb3DxSaXO12iIOCWoXdsUVwnqw==", + "dev": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-member-expression-to-functions": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6" + } + }, + "@babel/helper-create-regexp-features-plugin": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.18.6.tgz", + "integrity": "sha512-7LcpH1wnQLGrI+4v+nPp+zUvIkF9x0ddv1Hkdue10tg3gmRnLy97DXh4STiOf1qeIInyD69Qv5kKSZzKD8B/7A==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "regexpu-core": "^5.1.0" + } + }, + "@babel/helper-define-polyfill-provider": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.3.1.tgz", + "integrity": "sha512-J9hGMpJQmtWmj46B3kBHmL38UhJGhYX7eqkcq+2gsstyYt341HmPeWspihX43yVRA0mS+8GGk2Gckc7bY/HCmA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-compilation-targets": "^7.13.0", + "@babel/helper-module-imports": "^7.12.13", + "@babel/helper-plugin-utils": "^7.13.0", + "@babel/traverse": "^7.13.0", + "debug": "^4.1.1", + "lodash.debounce": "^4.0.8", + "resolve": "^1.14.2", + "semver": "^6.1.2" + } + }, + "@babel/helper-environment-visitor": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.18.6.tgz", + "integrity": "sha512-8n6gSfn2baOY+qlp+VSzsosjCVGFqWKmDF0cCWOybh52Dw3SEyoWR1KrhMJASjLwIEkkAufZ0xvr+SxLHSpy2Q==", + "dev": true + }, + "@babel/helper-explode-assignable-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.18.6.tgz", + "integrity": "sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==", + "dev": true, + "peer": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-function-name": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.18.6.tgz", + "integrity": "sha512-0mWMxV1aC97dhjCah5U5Ua7668r5ZmSC2DLfH2EZnf9c3/dHZKiFa5pRLMH5tjSl471tY6496ZWk/kjNONBxhw==", + "dev": true, + "requires": { + "@babel/template": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-hoist-variables": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.18.6.tgz", + "integrity": "sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-member-expression-to-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.18.6.tgz", + "integrity": "sha512-CeHxqwwipekotzPDUuJOfIMtcIHBuc7WAzLmTYWctVigqS5RktNMQ5bEwQSuGewzYnCtTWa3BARXeiLxDTv+Ng==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-module-imports": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.18.6.tgz", + "integrity": "sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-module-transforms": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.18.6.tgz", + "integrity": "sha512-L//phhB4al5uucwzlimruukHB3jRd5JGClwRMD/ROrVjXfLqovYnvQrK/JK36WYyVwGGO7OD3kMyVTjx+WVPhw==", + "dev": true, + "requires": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/helper-validator-identifier": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-optimise-call-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.18.6.tgz", + "integrity": "sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-plugin-utils": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.18.6.tgz", + "integrity": "sha512-gvZnm1YAAxh13eJdkb9EWHBnF3eAub3XTLCZEehHT2kWxiKVRL64+ae5Y6Ivne0mVHmMYKT+xWgZO+gQhuLUBg==", + "dev": true + }, + "@babel/helper-remap-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.18.6.tgz", + "integrity": "sha512-z5wbmV55TveUPZlCLZvxWHtrjuJd+8inFhk7DG0WW87/oJuGDcjDiu7HIvGcpf5464L6xKCg3vNkmlVVz9hwyQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-wrap-function": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-replace-supers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.18.6.tgz", + "integrity": "sha512-fTf7zoXnUGl9gF25fXCWE26t7Tvtyn6H4hkLSYhATwJvw2uYxd3aoXplMSe0g9XbwK7bmxNes7+FGO0rB/xC0g==", + "dev": true, + "requires": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-member-expression-to-functions": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-simple-access": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.18.6.tgz", + "integrity": "sha512-iNpIgTgyAvDQpDj76POqg+YEt8fPxx3yaNBg3S30dxNKm2SWfYhD0TGrK/Eu9wHpUW63VQU894TsTg+GLbUa1g==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.18.6.tgz", + "integrity": "sha512-4KoLhwGS9vGethZpAhYnMejWkX64wsnHPDwvOsKWU6Fg4+AlK2Jz3TyjQLMEPvz+1zemi/WBdkYxCD0bAfIkiw==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.18.6.tgz", + "integrity": "sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==", + "dev": true, + "requires": { + "@babel/types": "^7.18.6" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.18.6.tgz", + "integrity": "sha512-MmetCkz9ej86nJQV+sFCxoGGrUbU3q02kgLciwkrt9QqEB7cP39oKEY0PakknEO0Gu20SskMRi+AYZ3b1TpN9g==", + "dev": true + }, + "@babel/helper-validator-option": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.18.6.tgz", + "integrity": "sha512-XO7gESt5ouv/LRJdrVjkShckw6STTaB7l9BrpBaAHDeF5YZT+01PCwmR0SJHnkW6i8OwW/EVWRShfi4j2x+KQw==", + "dev": true + }, + "@babel/helper-wrap-function": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.18.6.tgz", + "integrity": "sha512-I5/LZfozwMNbwr/b1vhhuYD+J/mU+gfGAj5td7l5Rv9WYmH6i3Om69WGKNmlIpsVW/mF6O5bvTKbvDQZVgjqOw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-function-name": "^7.18.6", + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/helpers": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.18.6.tgz", + "integrity": "sha512-vzSiiqbQOghPngUYt/zWGvK3LAsPhz55vc9XNN0xAl2gV4ieShI2OQli5duxWHD+72PZPTKAcfcZDE1Cwc5zsQ==", + "dev": true, + "requires": { + "@babel/template": "^7.18.6", + "@babel/traverse": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/highlight": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz", + "integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.18.6", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, + "@babel/parser": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.18.6.tgz", + "integrity": "sha512-uQVSa9jJUe/G/304lXspfWVpKpK4euFLgGiMQFOCpM/bgcAdeoHwi/OQz23O9GK2osz26ZiXRRV9aV+Yl1O8tw==", + "dev": true + }, + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.18.6.tgz", + "integrity": "sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.18.6.tgz", + "integrity": "sha512-Udgu8ZRgrBrttVz6A0EVL0SJ1z+RLbIeqsu632SA1hf0awEppD6TvdznoH+orIF8wtFFAV/Enmw9Y+9oV8TQcw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.6" + } + }, + "@babel/plugin-proposal-async-generator-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.18.6.tgz", + "integrity": "sha512-WAz4R9bvozx4qwf74M+sfqPMKfSqwM0phxPTR6iJIi8robgzXwkEgmeJG1gEKhm6sDqT/U9aV3lfcqybIpev8w==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4" + } + }, + "@babel/plugin-proposal-class-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz", + "integrity": "sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==", + "dev": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-proposal-class-static-block": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-static-block/-/plugin-proposal-class-static-block-7.18.6.tgz", + "integrity": "sha512-+I3oIiNxrCpup3Gi8n5IGMwj0gOCAjcJUSQEcotNnCCPMEnixawOQ+KeJPlgfjzx+FKQ1QSyZOWe7wmoJp7vhw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-class-static-block": "^7.14.5" + } + }, + "@babel/plugin-proposal-dynamic-import": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-dynamic-import/-/plugin-proposal-dynamic-import-7.18.6.tgz", + "integrity": "sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-dynamic-import": "^7.8.3" + } + }, + "@babel/plugin-proposal-export-namespace-from": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-namespace-from/-/plugin-proposal-export-namespace-from-7.18.6.tgz", + "integrity": "sha512-zr/QcUlUo7GPo6+X1wC98NJADqmy5QTFWWhqeQWiki4XHafJtLl/YMGkmRB2szDD2IYJCCdBTd4ElwhId9T7Xw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3" + } + }, + "@babel/plugin-proposal-json-strings": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-json-strings/-/plugin-proposal-json-strings-7.18.6.tgz", + "integrity": "sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3" + } + }, + "@babel/plugin-proposal-logical-assignment-operators": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-logical-assignment-operators/-/plugin-proposal-logical-assignment-operators-7.18.6.tgz", + "integrity": "sha512-zMo66azZth/0tVd7gmkxOkOjs2rpHyhpcFo565PUP37hSp6hSd9uUKIfTDFMz58BwqgQKhJ9YxtM5XddjXVn+Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4" + } + }, + "@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz", + "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3" + } + }, + "@babel/plugin-proposal-numeric-separator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-numeric-separator/-/plugin-proposal-numeric-separator-7.18.6.tgz", + "integrity": "sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-numeric-separator": "^7.10.4" + } + }, + "@babel/plugin-proposal-object-rest-spread": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.18.6.tgz", + "integrity": "sha512-9yuM6wr4rIsKa1wlUAbZEazkCrgw2sMPEXCr4Rnwetu7cEW1NydkCWytLuYletbf8vFxdJxFhwEZqMpOx2eZyw==", + "dev": true, + "peer": true, + "requires": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-transform-parameters": "^7.18.6" + } + }, + "@babel/plugin-proposal-optional-catch-binding": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-catch-binding/-/plugin-proposal-optional-catch-binding-7.18.6.tgz", + "integrity": "sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3" + } + }, + "@babel/plugin-proposal-optional-chaining": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.18.6.tgz", + "integrity": "sha512-PatI6elL5eMzoypFAiYDpYQyMtXTn+iMhuxxQt5mAXD4fEmKorpSI3PHd+i3JXBJN3xyA6MvJv7at23HffFHwA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6", + "@babel/plugin-syntax-optional-chaining": "^7.8.3" + } + }, + "@babel/plugin-proposal-private-methods": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-methods/-/plugin-proposal-private-methods-7.18.6.tgz", + "integrity": "sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-proposal-private-property-in-object": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.18.6.tgz", + "integrity": "sha512-9Rysx7FOctvT5ouj5JODjAFAkgGoudQuLPamZb0v1TGLpapdNaftzifU8NTWQm0IRjqoYypdrSmyWgkocDQ8Dw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5" + } + }, + "@babel/plugin-proposal-unicode-property-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-unicode-property-regex/-/plugin-proposal-unicode-property-regex-7.18.6.tgz", + "integrity": "sha512-2BShG/d5yoZyXZfVePH91urL5wTG6ASZU9M4o03lKK8u8UW1y08OMttBSOADTcJrnPMpvDXRG3G8fyLh4ovs8w==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.12.13" + } + }, + "@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.14.5" + } + }, + "@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-export-namespace-from": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz", + "integrity": "sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.3" + } + }, + "@babel/plugin-syntax-flow": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.18.6.tgz", + "integrity": "sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-syntax-import-assertions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.18.6.tgz", + "integrity": "sha512-/DU3RXad9+bZwrgWJQKbr39gYbJpLJHezqEzRzi/BHRlJ9zsQb4CK2CA/5apllXNomwA1qHwzvHl+AdEmC5krQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + } + }, + "@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.14.5" + } + }, + "@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.14.5" + } + }, + "@babel/plugin-syntax-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.18.6.tgz", + "integrity": "sha512-mAWAuq4rvOepWCBid55JuRNvpTNf2UGVgoz4JV0fXEKolsVZDzsa4NqCef758WZJj/GDu0gVGItjKFiClTAmZA==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-arrow-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.18.6.tgz", + "integrity": "sha512-9S9X9RUefzrsHZmKMbDXxweEH+YlE8JJEuat9FdvW9Qh1cw7W64jELCtWNkPBPX5En45uy28KGvA/AySqUh8CQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-async-to-generator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.18.6.tgz", + "integrity": "sha512-ARE5wZLKnTgPW7/1ftQmSi1CmkqqHo2DNmtztFhvgtOWSDfq0Cq9/9L+KnZNYSNrydBekhW3rwShduf59RoXag==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-module-imports": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-remap-async-to-generator": "^7.18.6" + } + }, + "@babel/plugin-transform-block-scoped-functions": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.18.6.tgz", + "integrity": "sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-block-scoping": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.18.6.tgz", + "integrity": "sha512-pRqwb91C42vs1ahSAWJkxOxU1RHWDn16XAa6ggQ72wjLlWyYeAcLvTtE0aM8ph3KNydy9CQF2nLYcjq1WysgxQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-classes": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.18.6.tgz", + "integrity": "sha512-XTg8XW/mKpzAF3actL554Jl/dOYoJtv3l8fxaEczpgz84IeeVf+T1u2CSvPHuZbt0w3JkIx4rdn/MRQI7mo0HQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-annotate-as-pure": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-optimise-call-expression": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "globals": "^11.1.0" + } + }, + "@babel/plugin-transform-computed-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.18.6.tgz", + "integrity": "sha512-9repI4BhNrR0KenoR9vm3/cIc1tSBIo+u1WVjKCAynahj25O8zfbiE6JtAtHPGQSs4yZ+bA8mRasRP+qc+2R5A==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-destructuring": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.18.6.tgz", + "integrity": "sha512-tgy3u6lRp17ilY8r1kP4i2+HDUwxlVqq3RTc943eAWSzGgpU1qhiKpqZ5CMyHReIYPHdo3Kg8v8edKtDqSVEyQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-dotall-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.18.6.tgz", + "integrity": "sha512-6S3jpun1eEbAxq7TdjLotAsl4WpQI9DxfkycRcKrjhQYzU87qpXdknpBg/e+TdcMehqGnLFi7tnFUBR02Vq6wg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-duplicate-keys": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.18.6.tgz", + "integrity": "sha512-NJU26U/208+sxYszf82nmGYqVF9QN8py2HFTblPT9hbawi8+1C5a9JubODLTGFuT0qlkqVinmkwOD13s0sZktg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-exponentiation-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.18.6.tgz", + "integrity": "sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-builder-binary-assignment-operator-visitor": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-flow-strip-types": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.18.6.tgz", + "integrity": "sha512-wE0xtA7csz+hw4fKPwxmu5jnzAsXPIO57XnRwzXP3T19jWh1BODnPGoG9xKYwvAwusP7iUktHayRFbMPGtODaQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-flow": "^7.18.6" + } + }, + "@babel/plugin-transform-for-of": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.18.6.tgz", + "integrity": "sha512-WAjoMf4wIiSsy88KmG7tgj2nFdEK7E46tArVtcgED7Bkj6Fg/tG5SbvNIOKxbFS2VFgNh6+iaPswBeQZm4ox8w==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-function-name": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.18.6.tgz", + "integrity": "sha512-kJha/Gbs5RjzIu0CxZwf5e3aTTSlhZnHMT8zPWnJMjNpLOUgqevg+PN5oMH68nMCXnfiMo4Bhgxqj59KHTlAnA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.18.6.tgz", + "integrity": "sha512-x3HEw0cJZVDoENXOp20HlypIHfl0zMIhMVZEBVTfmqbObIpsMxMbmU5nOEO8R7LYT+z5RORKPlTI5Hj4OsO9/Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-member-expression-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.18.6.tgz", + "integrity": "sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-modules-amd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.18.6.tgz", + "integrity": "sha512-Pra5aXsmTsOnjM3IajS8rTaLCy++nGM4v3YR4esk5PCsyg9z8NA5oQLwxzMUtDBd8F+UmVza3VxoAaWCbzH1rg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, + "@babel/plugin-transform-modules-commonjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.18.6.tgz", + "integrity": "sha512-Qfv2ZOWikpvmedXQJDSbxNqy7Xr/j2Y8/KfijM0iJyKkBTmWuvCA1yeH1yDM7NJhBW/2aXxeucLj6i80/LAJ/Q==", + "dev": true, + "requires": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, + "@babel/plugin-transform-modules-systemjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.18.6.tgz", + "integrity": "sha512-UbPYpXxLjTw6w6yXX2BYNxF3p6QY225wcTkfQCy3OMnSlS/C3xGtwUjEzGkldb/sy6PWLiCQ3NbYfjWUTI3t4g==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-identifier": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, + "@babel/plugin-transform-modules-umd": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.18.6.tgz", + "integrity": "sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.18.6.tgz", + "integrity": "sha512-UmEOGF8XgaIqD74bC8g7iV3RYj8lMf0Bw7NJzvnS9qQhM4mg+1WHKotUIdjxgD2RGrgFLZZPCFPFj3P/kVDYhg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-new-target": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.18.6.tgz", + "integrity": "sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-object-super": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.18.6.tgz", + "integrity": "sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-replace-supers": "^7.18.6" + } + }, + "@babel/plugin-transform-parameters": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.18.6.tgz", + "integrity": "sha512-FjdqgMv37yVl/gwvzkcB+wfjRI8HQmc5EgOG9iGNvUY1ok+TjsoaMP7IqCDZBhkFcM5f3OPVMs6Dmp03C5k4/A==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-property-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.18.6.tgz", + "integrity": "sha512-cYcs6qlgafTud3PAzrrRNbQtfpQ8+y/+M5tKmksS9+M1ckbH6kzY8MrexEM9mcA6JDsukE19iIRvAyYl463sMg==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-regenerator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.18.6.tgz", + "integrity": "sha512-poqRI2+qiSdeldcz4wTSTXBRryoq3Gc70ye7m7UD5Ww0nE29IXqMl6r7Nd15WBgRd74vloEMlShtH6CKxVzfmQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "regenerator-transform": "^0.15.0" + } + }, + "@babel/plugin-transform-reserved-words": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.18.6.tgz", + "integrity": "sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-shorthand-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.18.6.tgz", + "integrity": "sha512-eCLXXJqv8okzg86ywZJbRn19YJHU4XUa55oz2wbHhaQVn/MM+XhukiT7SYqp/7o00dg52Rj51Ny+Ecw4oyoygw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-spread": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.18.6.tgz", + "integrity": "sha512-ayT53rT/ENF8WWexIRg9AiV9h0aIteyWn5ptfZTZQrjk/+f3WdrJGCY4c9wcgl2+MKkKPhzbYp97FTsquZpDCw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-skip-transparent-expression-wrappers": "^7.18.6" + } + }, + "@babel/plugin-transform-sticky-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.18.6.tgz", + "integrity": "sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-template-literals": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.18.6.tgz", + "integrity": "sha512-UuqlRrQmT2SWRvahW46cGSany0uTlcj8NYOS5sRGYi8FxPYPoLd5DDmMd32ZXEj2Jq+06uGVQKHxa/hJx2EzKw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-typeof-symbol": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.18.6.tgz", + "integrity": "sha512-7m71iS/QhsPk85xSjFPovHPcH3H9qeyzsujhTc+vcdnsXavoWYJ74zx0lP5RhpC5+iDnVLO+PPMHzC11qels1g==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.18.6.tgz", + "integrity": "sha512-ijHNhzIrLj5lQCnI6aaNVRtGVuUZhOXFLRVFs7lLrkXTHip4FKty5oAuQdk4tywG0/WjXmjTfQCWmuzrvFer1w==", + "dev": true, + "requires": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-typescript": "^7.18.6" + } + }, + "@babel/plugin-transform-unicode-escapes": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.18.6.tgz", + "integrity": "sha512-XNRwQUXYMP7VLuy54cr/KS/WeL3AZeORhrmeZ7iewgu+X2eBqmpaLI/hzqr9ZxCeUoq0ASK4GUzSM0BDhZkLFw==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/plugin-transform-unicode-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.18.6.tgz", + "integrity": "sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + } + }, + "@babel/preset-env": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.18.6.tgz", + "integrity": "sha512-WrthhuIIYKrEFAwttYzgRNQ5hULGmwTj+D6l7Zdfsv5M7IWV/OZbUfbeL++Qrzx1nVJwWROIFhCHRYQV4xbPNw==", + "dev": true, + "peer": true, + "requires": { + "@babel/compat-data": "^7.18.6", + "@babel/helper-compilation-targets": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.18.6", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.18.6", + "@babel/plugin-proposal-async-generator-functions": "^7.18.6", + "@babel/plugin-proposal-class-properties": "^7.18.6", + "@babel/plugin-proposal-class-static-block": "^7.18.6", + "@babel/plugin-proposal-dynamic-import": "^7.18.6", + "@babel/plugin-proposal-export-namespace-from": "^7.18.6", + "@babel/plugin-proposal-json-strings": "^7.18.6", + "@babel/plugin-proposal-logical-assignment-operators": "^7.18.6", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", + "@babel/plugin-proposal-numeric-separator": "^7.18.6", + "@babel/plugin-proposal-object-rest-spread": "^7.18.6", + "@babel/plugin-proposal-optional-catch-binding": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.18.6", + "@babel/plugin-proposal-private-methods": "^7.18.6", + "@babel/plugin-proposal-private-property-in-object": "^7.18.6", + "@babel/plugin-proposal-unicode-property-regex": "^7.18.6", + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-syntax-export-namespace-from": "^7.8.3", + "@babel/plugin-syntax-import-assertions": "^7.18.6", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5", + "@babel/plugin-transform-arrow-functions": "^7.18.6", + "@babel/plugin-transform-async-to-generator": "^7.18.6", + "@babel/plugin-transform-block-scoped-functions": "^7.18.6", + "@babel/plugin-transform-block-scoping": "^7.18.6", + "@babel/plugin-transform-classes": "^7.18.6", + "@babel/plugin-transform-computed-properties": "^7.18.6", + "@babel/plugin-transform-destructuring": "^7.18.6", + "@babel/plugin-transform-dotall-regex": "^7.18.6", + "@babel/plugin-transform-duplicate-keys": "^7.18.6", + "@babel/plugin-transform-exponentiation-operator": "^7.18.6", + "@babel/plugin-transform-for-of": "^7.18.6", + "@babel/plugin-transform-function-name": "^7.18.6", + "@babel/plugin-transform-literals": "^7.18.6", + "@babel/plugin-transform-member-expression-literals": "^7.18.6", + "@babel/plugin-transform-modules-amd": "^7.18.6", + "@babel/plugin-transform-modules-commonjs": "^7.18.6", + "@babel/plugin-transform-modules-systemjs": "^7.18.6", + "@babel/plugin-transform-modules-umd": "^7.18.6", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.18.6", + "@babel/plugin-transform-new-target": "^7.18.6", + "@babel/plugin-transform-object-super": "^7.18.6", + "@babel/plugin-transform-parameters": "^7.18.6", + "@babel/plugin-transform-property-literals": "^7.18.6", + "@babel/plugin-transform-regenerator": "^7.18.6", + "@babel/plugin-transform-reserved-words": "^7.18.6", + "@babel/plugin-transform-shorthand-properties": "^7.18.6", + "@babel/plugin-transform-spread": "^7.18.6", + "@babel/plugin-transform-sticky-regex": "^7.18.6", + "@babel/plugin-transform-template-literals": "^7.18.6", + "@babel/plugin-transform-typeof-symbol": "^7.18.6", + "@babel/plugin-transform-unicode-escapes": "^7.18.6", + "@babel/plugin-transform-unicode-regex": "^7.18.6", + "@babel/preset-modules": "^0.1.5", + "@babel/types": "^7.18.6", + "babel-plugin-polyfill-corejs2": "^0.3.1", + "babel-plugin-polyfill-corejs3": "^0.5.2", + "babel-plugin-polyfill-regenerator": "^0.3.1", + "core-js-compat": "^3.22.1", + "semver": "^6.3.0" + } + }, + "@babel/preset-flow": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-flow/-/preset-flow-7.18.6.tgz", + "integrity": "sha512-E7BDhL64W6OUqpuyHnSroLnqyRTcG6ZdOBl1OKI/QK/HJfplqK/S3sq1Cckx7oTodJ5yOXyfw7rEADJ6UjoQDQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-transform-flow-strip-types": "^7.18.6" + } + }, + "@babel/preset-modules": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.5.tgz", + "integrity": "sha512-A57th6YRG7oR3cq/yt/Y84MvGgE0eJG2F1JLhKuyG+jFxEgrd/HAMJatiFtmOiZurz+0DkrvbheCLaV5f2JfjA==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/plugin-proposal-unicode-property-regex": "^7.4.4", + "@babel/plugin-transform-dotall-regex": "^7.4.4", + "@babel/types": "^7.4.4", + "esutils": "^2.0.2" + } + }, + "@babel/preset-typescript": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.18.6.tgz", + "integrity": "sha512-s9ik86kXBAnD760aybBucdpnLsAt0jK1xqJn2juOn9lkOvSHV60os5hxoVJsPzMQxvnUJFAlkont2DvvaYEBtQ==", + "dev": true, + "requires": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-validator-option": "^7.18.6", + "@babel/plugin-transform-typescript": "^7.18.6" + } + }, + "@babel/register": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.18.6.tgz", + "integrity": "sha512-tkYtONzaO8rQubZzpBnvZPFcHgh8D9F55IjOsYton4X2IBoyRn2ZSWQqySTZnUn2guZbxbQiAB27hJEbvXamhQ==", + "dev": true, + "requires": { + "clone-deep": "^4.0.1", + "find-cache-dir": "^2.0.0", + "make-dir": "^2.1.0", + "pirates": "^4.0.5", + "source-map-support": "^0.5.16" + } + }, + "@babel/runtime": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.6.tgz", + "integrity": "sha512-t9wi7/AW6XtKahAe20Yw0/mMljKq0B1r2fPdvaAdV/KPDZewFXdaaa6K7lxmZBZ8FBNpCiAT6iHPmd6QO9bKfQ==", + "dev": true, + "peer": true, + "requires": { + "regenerator-runtime": "^0.13.4" + } + }, + "@babel/template": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.6.tgz", + "integrity": "sha512-JoDWzPe+wgBsTTgdnIma3iHNFC7YVJoPssVBDjiHfNlyt4YcunDtcDOUmfVDfCK5MfdsaIoX9PkijPhjH3nYUw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/types": "^7.18.6" + } + }, + "@babel/traverse": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.18.6.tgz", + "integrity": "sha512-zS/OKyqmD7lslOtFqbscH6gMLFYOfG1YPqCKfAW5KrTeolKqvB8UelR49Fpr6y93kYkW2Ik00mT1LOGiAGvizw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.18.6", + "@babel/generator": "^7.18.6", + "@babel/helper-environment-visitor": "^7.18.6", + "@babel/helper-function-name": "^7.18.6", + "@babel/helper-hoist-variables": "^7.18.6", + "@babel/helper-split-export-declaration": "^7.18.6", + "@babel/parser": "^7.18.6", + "@babel/types": "^7.18.6", + "debug": "^4.1.0", + "globals": "^11.1.0" + } + }, + "@babel/types": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.18.6.tgz", + "integrity": "sha512-NdBNzPDwed30fZdDQtVR7ZgaO4UKjuaQFH9VArS+HMnurlOY0JWN+4ROlu/iapMFwjRQU4pOG4StZfDmulEwGA==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.18.6", + "to-fast-properties": "^2.0.0" + } + }, + "@jridgewell/gen-mapping": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", + "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", + "dev": true, + "requires": { + "@jridgewell/set-array": "^1.0.0", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.8.tgz", + "integrity": "sha512-YK5G9LaddzGbcucK4c8h5tWFmMPBvRZ/uyWmN1/SbBdIvqGUdWGkJ5BAaccgs6XbzVLsqbPJrBSFwKv3kT9i7w==", + "dev": true + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.14", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.14.tgz", + "integrity": "sha512-bJWEfQ9lPTvm3SneWwRFVLzrh6nhjwqw7TUFFBEMzwvg7t7PCDenf2lDwqo4NQXzdpgBXyFgDWnQA+2vkruksQ==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "arr-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", + "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==", + "dev": true + }, + "arr-flatten": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "dev": true + }, + "arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "dev": true + }, + "array-unique": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", + "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==", + "dev": true + }, + "assign-symbols": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", + "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==", + "dev": true + }, + "ast-types": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.14.2.tgz", + "integrity": "sha512-O0yuUDnZeQDL+ncNGlJ78BiO4jnYI3bvMsD5prT0/nsgijG/LpNBIr63gTjVTNsiGkgQhiyCShTgxt8oXOrklA==", + "dev": true, + "requires": { + "tslib": "^2.0.1" + } + }, + "atob": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "dev": true + }, + "babel-core": { + "version": "7.0.0-bridge.0", + "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz", + "integrity": "sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==", + "dev": true, + "requires": {} + }, + "babel-plugin-dynamic-import-node": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", + "dev": true, + "requires": { + "object.assign": "^4.1.0" + } + }, + "babel-plugin-polyfill-corejs2": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.1.tgz", + "integrity": "sha512-v7/T6EQcNfVLfcN2X8Lulb7DjprieyLWJK/zOWH5DUYcAgex9sP3h25Q+DLsX9TloXe3y1O8l2q2Jv9q8UVB9w==", + "dev": true, + "peer": true, + "requires": { + "@babel/compat-data": "^7.13.11", + "@babel/helper-define-polyfill-provider": "^0.3.1", + "semver": "^6.1.1" + } + }, + "babel-plugin-polyfill-corejs3": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.5.2.tgz", + "integrity": "sha512-G3uJih0XWiID451fpeFaYGVuxHEjzKTHtc9uGFEjR6hHrvNzeS/PX+LLLcetJcytsB5m4j+K3o/EpXJNb/5IEQ==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-define-polyfill-provider": "^0.3.1", + "core-js-compat": "^3.21.0" + } + }, + "babel-plugin-polyfill-regenerator": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.3.1.tgz", + "integrity": "sha512-Y2B06tvgHYt1x0yz17jGkGeeMr5FeKUu+ASJ+N6nB5lQ8Dapfg42i0OVrf8PNGJ3zKL4A23snMi1IRwrqqND7A==", + "dev": true, + "peer": true, + "requires": { + "@babel/helper-define-polyfill-provider": "^0.3.1" + } + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "base": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "dev": true, + "requires": { + "cache-base": "^1.0.1", + "class-utils": "^0.3.5", + "component-emitter": "^1.2.1", + "define-property": "^1.0.0", + "isobject": "^3.0.1", + "mixin-deep": "^1.2.0", + "pascalcase": "^0.1.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.0" + } + } + } + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, + "requires": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "browserslist": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.0.tgz", + "integrity": "sha512-UQxE0DIhRB5z/zDz9iA03BOfxaN2+GQdBYH/2WrSIWEUrnpzTPJbhqt+umq6r3acaPRTW1FNTkrcp0PXgtFkvA==", + "dev": true, + "requires": { + "caniuse-lite": "^1.0.30001358", + "electron-to-chromium": "^1.4.164", + "node-releases": "^2.0.5", + "update-browserslist-db": "^1.0.0" + } + }, + "buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true + }, + "cache-base": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "dev": true, + "requires": { + "collection-visit": "^1.0.0", + "component-emitter": "^1.2.1", + "get-value": "^2.0.6", + "has-value": "^1.0.0", + "isobject": "^3.0.1", + "set-value": "^2.0.0", + "to-object-path": "^0.3.0", + "union-value": "^1.0.0", + "unset-value": "^1.0.0" + } + }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "caniuse-lite": { + "version": "1.0.30001359", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001359.tgz", + "integrity": "sha512-Xln/BAsPzEuiVLgJ2/45IaqD9jShtk3Y33anKb4+yLwQzws3+v6odKfpgES/cDEaZMLzSChpIGdbOYtH9MyuHw==", + "dev": true + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "class-utils": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "dev": true, + "requires": { + "arr-union": "^3.1.0", + "define-property": "^0.2.5", + "isobject": "^3.0.0", + "static-extend": "^0.1.1" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + } + } + }, + "clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "requires": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + } + }, + "collection-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", + "integrity": "sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==", + "dev": true, + "requires": { + "map-visit": "^1.0.0", + "object-visit": "^1.0.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true + }, + "component-emitter": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "convert-source-map": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.8.0.tgz", + "integrity": "sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.1" + } + }, + "copy-descriptor": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", + "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==", + "dev": true + }, + "core-js-compat": { + "version": "3.23.3", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.23.3.tgz", + "integrity": "sha512-WSzUs2h2vvmKsacLHNTdpyOC9k43AEhcGoFlVgCY4L7aw98oSBKtPL6vD0/TqZjRWRQYdDSLkzZIni4Crbbiqw==", + "dev": true, + "peer": true, + "requires": { + "browserslist": "^4.21.0", + "semver": "7.0.0" + }, + "dependencies": { + "semver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.0.0.tgz", + "integrity": "sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==", + "dev": true, + "peer": true + } + } + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==", + "dev": true + }, + "define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "dev": true, + "requires": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + } + }, + "define-property": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.2", + "isobject": "^3.0.1" + } + }, + "electron-to-chromium": { + "version": "1.4.172", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.172.tgz", + "integrity": "sha512-yDoFfTJnqBAB6hSiPvzmsBJSrjOXJtHSJoqJdI/zSIh7DYupYnIOHt/bbPw/WE31BJjNTybDdNAs21gCMnTh0Q==", + "dev": true + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "peer": true + }, + "expand-brackets": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", + "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==", + "dev": true, + "requires": { + "debug": "^2.3.3", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "posix-character-classes": "^0.1.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + } + } + }, + "extend-shallow": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", + "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==", + "dev": true, + "requires": { + "assign-symbols": "^1.0.0", + "is-extendable": "^1.0.1" + } + }, + "extglob": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", + "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dev": true, + "requires": { + "array-unique": "^0.3.2", + "define-property": "^1.0.0", + "expand-brackets": "^2.1.4", + "extend-shallow": "^2.0.1", + "fragment-cache": "^0.2.1", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==", + "dev": true, + "requires": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + } + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "flow-parser": { + "version": "0.181.1", + "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.181.1.tgz", + "integrity": "sha512-+Mx87/GkmF5+FHk8IXc5WppD/oC4wB+05MuIv7qmIMgThND3RhOBGl7Npyc2L7NLVenme00ZlwEKVieiMz4bqA==", + "dev": true + }, + "for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "dev": true + }, + "fragment-cache": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", + "integrity": "sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==", + "dev": true, + "requires": { + "map-cache": "^0.2.2" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true + }, + "get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, + "get-value": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", + "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==", + "dev": true + }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "dev": true + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true + }, + "has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "dev": true, + "requires": { + "get-intrinsic": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true + }, + "has-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", + "integrity": "sha512-IBXk4GTsLYdQ7Rvt+GRBrFSVEkmuOUy4re0Xjd9kJSUQpnTrWR4/y9RpfexN9vkAPMFuQoeWKwqzPozRTlasGw==", + "dev": true, + "requires": { + "get-value": "^2.0.6", + "has-values": "^1.0.0", + "isobject": "^3.0.0" + } + }, + "has-values": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", + "integrity": "sha512-ODYZC64uqzmtfGMEAX/FvZiRyWLpAC3vYnNunURUnkGVTS+mI0smVsWaPydRBsE3g+ok7h960jChO8mFcWlHaQ==", + "dev": true, + "requires": { + "is-number": "^3.0.0", + "kind-of": "^4.0.0" + }, + "dependencies": { + "kind-of": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", + "integrity": "sha512-24XsCxmEbRwEDbz/qz3stgin8TTzZ1ESR56OMCN0ujYg+vRutNSiOj9bHH9u85DKgXguraugV5sFuvbD4FW/hw==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true + }, + "is-core-module": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", + "integrity": "sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A==", + "dev": true, + "peer": true, + "requires": { + "has": "^1.0.3" + } + }, + "is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "requires": { + "kind-of": "^6.0.0" + } + }, + "is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + }, + "is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, + "requires": { + "is-plain-object": "^2.0.4" + } + }, + "is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "requires": { + "isobject": "^3.0.1" + } + }, + "is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "jscodeshift": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/jscodeshift/-/jscodeshift-0.13.1.tgz", + "integrity": "sha512-lGyiEbGOvmMRKgWk4vf+lUrCWO/8YR8sUR3FKF1Cq5fovjZDlIcw3Hu5ppLHAnEXshVffvaM0eyuY/AbOeYpnQ==", + "dev": true, + "requires": { + "@babel/core": "^7.13.16", + "@babel/parser": "^7.13.16", + "@babel/plugin-proposal-class-properties": "^7.13.0", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", + "@babel/plugin-proposal-optional-chaining": "^7.13.12", + "@babel/plugin-transform-modules-commonjs": "^7.13.8", + "@babel/preset-flow": "^7.13.13", + "@babel/preset-typescript": "^7.13.0", + "@babel/register": "^7.13.16", + "babel-core": "^7.0.0-bridge.0", + "chalk": "^4.1.2", + "flow-parser": "0.*", + "graceful-fs": "^4.2.4", + "micromatch": "^3.1.10", + "neo-async": "^2.5.0", + "node-dir": "^0.1.17", + "recast": "^0.20.4", + "temp": "^0.8.4", + "write-file-atomic": "^2.3.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true + }, + "json5": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.1.tgz", + "integrity": "sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA==", + "dev": true + }, + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "dev": true, + "peer": true + }, + "make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "requires": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + } + } + }, + "map-cache": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", + "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==", + "dev": true + }, + "map-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", + "integrity": "sha512-4y7uGv8bd2WdM9vpQsiQNo41Ln1NvhvDRuVt0k2JZQ+ezN2uaQes7lZeZ+QQUHOLQAtDaBJ+7wCbi+ab/KFs+w==", + "dev": true, + "requires": { + "object-visit": "^1.0.0" + } + }, + "micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, + "requires": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + } + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "mixin-deep": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "dev": true, + "requires": { + "for-in": "^1.0.2", + "is-extendable": "^1.0.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "nanomatch": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "dev": true, + "requires": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "fragment-cache": "^0.2.1", + "is-windows": "^1.0.2", + "kind-of": "^6.0.2", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + } + }, + "neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true + }, + "node-dir": { + "version": "0.1.17", + "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", + "integrity": "sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==", + "dev": true, + "requires": { + "minimatch": "^3.0.2" + } + }, + "node-releases": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.5.tgz", + "integrity": "sha512-U9h1NLROZTq9uE1SNffn6WuPDg8icmi3ns4rEl/oTfIle4iLjTliCzgTsbaIFMq/Xn078/lfY/BL0GWZ+psK4Q==", + "dev": true + }, + "object-copy": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", + "integrity": "sha512-79LYn6VAb63zgtmAteVOWo9Vdj71ZVBy3Pbse+VqxDpEP83XuujMrGqHIwAXJ5I/aM0zU7dIyIAhifVTPrNItQ==", + "dev": true, + "requires": { + "copy-descriptor": "^0.1.0", + "define-property": "^0.2.5", + "kind-of": "^3.0.3" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "dependencies": { + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + } + } + }, + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "object-visit": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", + "integrity": "sha512-GBaMwwAVK9qbQN3Scdo0OyvgPW7l3lnaVMj84uTOZlswkX0KpF6fyDBJhtTthf7pymztoN36/KEr1DyhF96zEA==", + "dev": true, + "requires": { + "isobject": "^3.0.0" + } + }, + "object.assign": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", + "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "dev": true, + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + } + }, + "object.pick": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", + "integrity": "sha512-tqa/UMy/CCoYmj+H5qc07qvSL9dqcs/WZENZ1JbtWBlATP+iVOe778gE6MSijnyCnORzDuX6hU+LA4SZ09YjFQ==", + "dev": true, + "requires": { + "isobject": "^3.0.1" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "pascalcase": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", + "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==", + "dev": true + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "dev": true + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "peer": true + }, + "picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true + }, + "pirates": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz", + "integrity": "sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==", + "dev": true + }, + "pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "requires": { + "find-up": "^3.0.0" + } + }, + "posix-character-classes": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", + "integrity": "sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==", + "dev": true + }, + "recast": { + "version": "0.20.5", + "resolved": "https://registry.npmjs.org/recast/-/recast-0.20.5.tgz", + "integrity": "sha512-E5qICoPoNL4yU0H0NoBDntNB0Q5oMSNh9usFctYniLBluTthi3RsQVBXIJNbApOlvSwW/RGxIuokPcAc59J5fQ==", + "dev": true, + "requires": { + "ast-types": "0.14.2", + "esprima": "~4.0.0", + "source-map": "~0.6.1", + "tslib": "^2.0.1" + } + }, + "regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==", + "dev": true, + "peer": true + }, + "regenerate-unicode-properties": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.0.1.tgz", + "integrity": "sha512-vn5DU6yg6h8hP/2OkQo3K7uVILvY4iu0oI4t3HFa81UPkhGJwkRwM10JEc3upjdhHjs/k8GJY1sRBhk5sr69Bw==", + "dev": true, + "peer": true, + "requires": { + "regenerate": "^1.4.2" + } + }, + "regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==", + "dev": true, + "peer": true + }, + "regenerator-transform": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.0.tgz", + "integrity": "sha512-LsrGtPmbYg19bcPHwdtmXwbW+TqNvtY4riE3P83foeHRroMbH6/2ddFBfab3t7kbzc7v7p4wbkIecHImqt0QNg==", + "dev": true, + "peer": true, + "requires": { + "@babel/runtime": "^7.8.4" + } + }, + "regex-not": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "dev": true, + "requires": { + "extend-shallow": "^3.0.2", + "safe-regex": "^1.1.0" + } + }, + "regexpu-core": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-5.1.0.tgz", + "integrity": "sha512-bb6hk+xWd2PEOkj5It46A16zFMs2mv86Iwpdu94la4S3sJ7C973h2dHpYKwIBGaWSO7cIRJ+UX0IeMaWcO4qwA==", + "dev": true, + "peer": true, + "requires": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.0.1", + "regjsgen": "^0.6.0", + "regjsparser": "^0.8.2", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.0.0" + } + }, + "regjsgen": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.6.0.tgz", + "integrity": "sha512-ozE883Uigtqj3bx7OhL1KNbCzGyW2NQZPl6Hs09WTvCuZD5sTI4JY58bkbQWa/Y9hxIsvJ3M8Nbf7j54IqeZbA==", + "dev": true, + "peer": true + }, + "regjsparser": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.8.4.tgz", + "integrity": "sha512-J3LABycON/VNEu3abOviqGHuB/LOtOQj8SKmfP9anY5GfAVw/SPjwzSjxGjbZXIxbGfqTHtJw58C2Li/WkStmA==", + "dev": true, + "peer": true, + "requires": { + "jsesc": "~0.5.0" + }, + "dependencies": { + "jsesc": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "integrity": "sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==", + "dev": true, + "peer": true + } + } + }, + "repeat-element": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.4.tgz", + "integrity": "sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ==", + "dev": true + }, + "repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "dev": true + }, + "resolve": { + "version": "1.22.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", + "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "dev": true, + "peer": true, + "requires": { + "is-core-module": "^2.9.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, + "resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==", + "dev": true + }, + "ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "dev": true + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "safe-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha512-aJXcif4xnaNUzvUuC5gcb46oTS7zvg4jpMTnuqtrEPlR3vFr4pxtdTwaF1Qs3Enjn9HK+ZlwQui+a7z0SywIzg==", + "dev": true, + "requires": { + "ret": "~0.1.10" + } + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "set-value": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "dev": true, + "requires": { + "extend-shallow": "^2.0.1", + "is-extendable": "^0.1.1", + "is-plain-object": "^2.0.3", + "split-string": "^3.0.1" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "requires": { + "kind-of": "^6.0.2" + } + }, + "signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true + }, + "snapdragon": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "dev": true, + "requires": { + "base": "^0.11.1", + "debug": "^2.2.0", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "map-cache": "^0.2.2", + "source-map": "^0.5.6", + "source-map-resolve": "^0.5.0", + "use": "^3.1.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true + } + } + }, + "snapdragon-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "dev": true, + "requires": { + "define-property": "^1.0.0", + "isobject": "^3.0.0", + "snapdragon-util": "^3.0.1" + }, + "dependencies": { + "define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, + "requires": { + "is-descriptor": "^1.0.0" + } + } + } + }, + "snapdragon-util": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "dev": true, + "requires": { + "kind-of": "^3.2.0" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "source-map-resolve": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", + "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "dev": true, + "requires": { + "atob": "^2.1.2", + "decode-uri-component": "^0.2.0", + "resolve-url": "^0.2.1", + "source-map-url": "^0.4.0", + "urix": "^0.1.0" + } + }, + "source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "source-map-url": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.1.tgz", + "integrity": "sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw==", + "dev": true + }, + "split-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "dev": true, + "requires": { + "extend-shallow": "^3.0.0" + } + }, + "static-extend": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", + "integrity": "sha512-72E9+uLc27Mt718pMHt9VMNiAL4LMsmDbBva8mxWUCkT07fSzEGMYUCk0XWY6lp0j6RBAG4cJ3mWuZv2OE3s0g==", + "dev": true, + "requires": { + "define-property": "^0.2.5", + "object-copy": "^0.1.0" + }, + "dependencies": { + "define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, + "requires": { + "is-descriptor": "^0.1.0" + } + }, + "is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "requires": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + } + } + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "peer": true + }, + "temp": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/temp/-/temp-0.8.4.tgz", + "integrity": "sha512-s0ZZzd0BzYv5tLSptZooSjK8oj6C+c19p7Vqta9+6NPOf7r+fxq0cJe6/oN4LTC79sy5NY8ucOJNgwsKCSbfqg==", + "dev": true, + "requires": { + "rimraf": "~2.6.2" + } + }, + "to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", + "dev": true + }, + "to-object-path": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", + "integrity": "sha512-9mWHdnGRuh3onocaHzukyvCZhzvr6tiflAy/JRFXcJX0TjgfWA9pk9t8CMbzmBE4Jfw58pXbkngtBtqYxzNEyg==", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "to-regex": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "dev": true, + "requires": { + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "regex-not": "^1.0.2", + "safe-regex": "^1.1.0" + } + }, + "to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==", + "dev": true, + "requires": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + } + }, + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, + "unicode-canonical-property-names-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz", + "integrity": "sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==", + "dev": true, + "peer": true + }, + "unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "dev": true, + "peer": true, + "requires": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + } + }, + "unicode-match-property-value-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.0.0.tgz", + "integrity": "sha512-7Yhkc0Ye+t4PNYzOGKedDhXbYIBe1XEQYQxOPyhcXNMJ0WCABqqj6ckydd6pWRZTHV4GuCPKdBAUiMc60tsKVw==", + "dev": true, + "peer": true + }, + "unicode-property-aliases-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.0.0.tgz", + "integrity": "sha512-5Zfuy9q/DFr4tfO7ZPeVXb1aPoeQSdeFMLpYuFebehDAhbuevLs5yxSZmIFN1tP5F9Wl4IpJrYojg85/zgyZHQ==", + "dev": true, + "peer": true + }, + "union-value": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "dev": true, + "requires": { + "arr-union": "^3.1.0", + "get-value": "^2.0.6", + "is-extendable": "^0.1.1", + "set-value": "^2.0.1" + }, + "dependencies": { + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true + } + } + }, + "unset-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", + "integrity": "sha512-PcA2tsuGSF9cnySLHTLSh2qrQiJ70mn+r+Glzxv2TWZblxsxCC52BDlZoPCsz7STd9pN7EZetkWZBAvk4cgZdQ==", + "dev": true, + "requires": { + "has-value": "^0.3.1", + "isobject": "^3.0.0" + }, + "dependencies": { + "has-value": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", + "integrity": "sha512-gpG936j8/MzaeID5Yif+577c17TxaDmhuyVgSwtnL/q8UUTySg8Mecb+8Cf1otgLoD7DDH75axp86ER7LFsf3Q==", + "dev": true, + "requires": { + "get-value": "^2.0.3", + "has-values": "^0.1.4", + "isobject": "^2.0.0" + }, + "dependencies": { + "isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "dev": true, + "requires": { + "isarray": "1.0.0" + } + } + } + }, + "has-values": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", + "integrity": "sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==", + "dev": true + } + } + }, + "update-browserslist-db": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.4.tgz", + "integrity": "sha512-jnmO2BEGUjsMOe/Fg9u0oczOe/ppIDZPebzccl1yDWGLFP16Pa1/RM5wEoKYPG2zstNcDuAStejyxsOuKINdGA==", + "dev": true, + "requires": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + } + }, + "urix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", + "integrity": "sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==", + "dev": true + }, + "use": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "dev": true + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + } + } +} diff --git a/tools/esmify/package.json b/tools/esmify/package.json new file mode 100644 index 0000000000..152aaae5da --- /dev/null +++ b/tools/esmify/package.json @@ -0,0 +1,9 @@ +{ + "devDependencies": { + "jscodeshift": "^0.13.1" + }, + "scripts": { + "convert_module": "jscodeshift -t use-import-export-declarations.js --stdin --verbose=2", + "rewrite_imports": "jscodeshift -t import-to-import_esmodule.js --stdin --verbose=2" + } +} diff --git a/tools/esmify/static-import.js b/tools/esmify/static-import.js new file mode 100644 index 0000000000..e99bfb3380 --- /dev/null +++ b/tools/esmify/static-import.js @@ -0,0 +1,147 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* eslint-env node */ + +const _path = require("path"); +const { getESMFiles } = require(_path.resolve(__dirname, "./is-esmified.js")); +const { + esmifyExtension, + isString, + warnForPath, + isMemberExpressionWithIdentifiers, +} = require(_path.resolve(__dirname, "./utils.js")); + +function isTargetESM(resourceURI) { + if ("ESMIFY_TARGET_PREFIX" in process.env) { + const files = getESMFiles(resourceURI); + const targetPrefix = process.env.ESMIFY_TARGET_PREFIX; + for (const esm of files) { + if (esm.startsWith(targetPrefix)) { + return true; + } + } + + return false; + } + + return true; +} + +function isImportESModuleCall(node) { + return isMemberExpressionWithIdentifiers(node.callee, [ + "ChromeUtils", + "importESModule", + ]); +} + +// Replace `ChromeUtils.import`, `Cu.import`, and `ChromeUtils.importESModule` +// with static import if it's at the top-level of system ESM file. +function tryReplacingWithStaticImport( + jscodeshift, + inputFile, + path, + resourceURINode, + alwaysReplace +) { + if (!alwaysReplace && !inputFile.endsWith(".sys.mjs")) { + // Static import is available only in system ESM. + return false; + } + + // Check if it's at the top-level. + if (path.parent.node.type !== "VariableDeclarator") { + return false; + } + + if (path.parent.parent.node.type !== "VariableDeclaration") { + return false; + } + + const decls = path.parent.parent.node; + if (decls.declarations.length !== 1) { + return false; + } + + if (path.parent.parent.parent.node.type !== "Program") { + return false; + } + + if (path.node.arguments.length !== 1) { + return false; + } + + const resourceURI = resourceURINode.value; + + // Collect imported symbols. + const specs = []; + if (path.parent.node.id.type === "Identifier") { + specs.push(jscodeshift.importNamespaceSpecifier(path.parent.node.id)); + } else if (path.parent.node.id.type === "ObjectPattern") { + for (const prop of path.parent.node.id.properties) { + if (prop.shorthand) { + specs.push(jscodeshift.importSpecifier(prop.key)); + } else if (prop.value.type === "Identifier") { + specs.push(jscodeshift.importSpecifier(prop.key, prop.value)); + } else { + return false; + } + } + } else { + return false; + } + + // If this is `ChromeUtils.import` or `Cu.import`, replace the extension. + // no-op for `ChromeUtils.importESModule`. + resourceURINode.value = esmifyExtension(resourceURI); + + const e = jscodeshift.importDeclaration(specs, resourceURINode); + e.comments = path.parent.parent.node.comments; + path.parent.parent.node.comments = []; + path.parent.parent.replace(e); + + return true; +} + +function replaceImportESModuleCall( + inputFile, + jscodeshift, + path, + alwaysReplace +) { + if (path.node.arguments.length !== 1) { + warnForPath( + inputFile, + path, + `importESModule call should have only one argument` + ); + return; + } + + const resourceURINode = path.node.arguments[0]; + if (!isString(resourceURINode)) { + warnForPath(inputFile, path, `resource URI should be a string`); + return; + } + + if (!alwaysReplace) { + const resourceURI = resourceURINode.value; + if (!isTargetESM(resourceURI)) { + return; + } + } + + // If this cannot be replaced with static import, do nothing. + tryReplacingWithStaticImport( + jscodeshift, + inputFile, + path, + resourceURINode, + alwaysReplace + ); +} + +exports.isImportESModuleCall = isImportESModuleCall; +exports.tryReplacingWithStaticImport = tryReplacingWithStaticImport; +exports.replaceImportESModuleCall = replaceImportESModuleCall; diff --git a/tools/esmify/use-import-export-declarations.js b/tools/esmify/use-import-export-declarations.js new file mode 100644 index 0000000000..fe8c4dd286 --- /dev/null +++ b/tools/esmify/use-import-export-declarations.js @@ -0,0 +1,208 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// jscodeshift rule to replace EXPORTED_SYMBOLS with export declarations, +// and also convert existing ChromeUtils.importESModule to static import. + +/* eslint-env node */ + +const _path = require("path"); +const { + warnForPath, + getPrevStatement, + getNextStatement, +} = require(_path.resolve(__dirname, "./utils.js")); +const { + isImportESModuleCall, + replaceImportESModuleCall, +} = require(_path.resolve(__dirname, "./static-import.js")); + +module.exports = function (fileInfo, api) { + const { jscodeshift } = api; + const root = jscodeshift(fileInfo.source); + doTranslate(fileInfo.path, jscodeshift, root); + return root.toSource({ lineTerminator: "\n" }); +}; + +module.exports.doTranslate = doTranslate; + +// Move the comment for `path.node` to adjacent statement, keeping the position +// as much as possible. +function moveComments(inputFile, path) { + const next = getNextStatement(path); + if (next) { + if (next.comments) { + next.comments = [...path.node.comments, ...next.comments]; + } else { + next.comments = path.node.comments; + } + path.node.comments = []; + + return; + } + + const prev = getPrevStatement(path); + if (prev) { + path.node.comments.forEach(c => { + c.leading = false; + c.trailing = true; + }); + + if (prev.comments) { + prev.comments = [...prev.comments, ...path.node.comments]; + } else { + prev.comments = path.node.comments; + } + path.node.comments = []; + + return; + } + + warnForPath( + inputFile, + path, + `EXPORTED_SYMBOLS has comments and it cannot be preserved` + ); +} + +function collectAndRemoveExportedSymbols(inputFile, root) { + const nodes = root.findVariableDeclarators("EXPORTED_SYMBOLS"); + if (!nodes.length) { + throw Error(`EXPORTED_SYMBOLS not found`); + } + + let path = nodes.get(0); + const obj = nodes.get(0).node.init; + if (!obj) { + throw Error(`EXPORTED_SYMBOLS is not statically known`); + } + + if (path.parent.node.declarations.length !== 1) { + throw Error(`EXPORTED_SYMBOLS shouldn't be declared with other variables`); + } + + if (path.parent.node.comments && path.parent.node.comments.length) { + moveComments(inputFile, path.parent); + } + + path.parent.prune(); + + const EXPORTED_SYMBOLS = new Set(); + if (obj.type !== "ArrayExpression") { + throw Error(`EXPORTED_SYMBOLS is not statically known`); + } + + for (const elem of obj.elements) { + if (elem.type !== "Literal") { + throw Error(`EXPORTED_SYMBOLS is not statically known`); + } + var name = elem.value; + if (typeof name !== "string") { + throw Error(`EXPORTED_SYMBOLS item must be a string`); + } + EXPORTED_SYMBOLS.add(name); + } + + return EXPORTED_SYMBOLS; +} + +function isTopLevel(path) { + return path.parent.node.type === "Program"; +} + +function convertToExport(jscodeshift, path, name) { + const e = jscodeshift.exportNamedDeclaration(path.node); + e.comments = []; + e.comments = path.node.comments; + path.node.comments = []; + + path.replace(e); +} + +function doTranslate(inputFile, jscodeshift, root) { + const EXPORTED_SYMBOLS = collectAndRemoveExportedSymbols(inputFile, root); + + root.find(jscodeshift.FunctionDeclaration).forEach(path => { + if (!isTopLevel(path)) { + return; + } + const name = path.node.id.name; + if (!EXPORTED_SYMBOLS.has(name)) { + return; + } + EXPORTED_SYMBOLS.delete(name); + convertToExport(jscodeshift, path, name); + }); + + root.find(jscodeshift.ClassDeclaration).forEach(path => { + if (!isTopLevel(path)) { + return; + } + const name = path.node.id.name; + if (!EXPORTED_SYMBOLS.has(name)) { + return; + } + EXPORTED_SYMBOLS.delete(name); + convertToExport(jscodeshift, path, name); + }); + + root.find(jscodeshift.VariableDeclaration).forEach(path => { + if (!isTopLevel(path)) { + return; + } + + let exists = false; + let name; + for (const decl of path.node.declarations) { + if (decl.id.type === "Identifier") { + name = decl.id.name; + if (EXPORTED_SYMBOLS.has(name)) { + exists = true; + break; + } + } + + if (decl.id.type === "ObjectPattern") { + if (decl.id.properties.length === 1) { + const prop = decl.id.properties[0]; + if (prop.shorthand) { + if (prop.key.type === "Identifier") { + name = prop.key.name; + if (EXPORTED_SYMBOLS.has(name)) { + exists = true; + break; + } + } + } + } + } + } + if (!exists) { + return; + } + + if (path.node.declarations.length !== 1) { + throw Error( + `exported variable shouldn't be declared with other variables` + ); + } + + EXPORTED_SYMBOLS.delete(name); + convertToExport(jscodeshift, path, name); + }); + + if (EXPORTED_SYMBOLS.size !== 0) { + throw Error( + `exported symbols ${[...EXPORTED_SYMBOLS].join(", ")} not found` + ); + } + + root.find(jscodeshift.CallExpression).forEach(path => { + if (isImportESModuleCall(path.node)) { + // This file is not yet renamed. Skip the extension check. + // Also skip the isTargetESM. + replaceImportESModuleCall(inputFile, jscodeshift, path, true); + } + }); +} diff --git a/tools/esmify/utils.js b/tools/esmify/utils.js new file mode 100644 index 0000000000..801aab62af --- /dev/null +++ b/tools/esmify/utils.js @@ -0,0 +1,165 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Shared utility functions. + +/* eslint-env node */ + +function warnForPath(inputFile, path, message) { + const loc = path.node.loc; + console.log( + `WARNING: ${inputFile}:${loc.start.line}:${loc.start.column} : ${message}` + ); +} + +// Get the previous statement of `path.node` in `Program`. +function getPrevStatement(path) { + const parent = path.parent; + if (parent.node.type !== "Program") { + return null; + } + + const index = parent.node.body.findIndex(n => n == path.node); + if (index === -1) { + return null; + } + + if (index === 0) { + return null; + } + + return parent.node.body[index - 1]; +} + +// Get the next statement of `path.node` in `Program`. +function getNextStatement(path) { + const parent = path.parent; + if (parent.node.type !== "Program") { + return null; + } + + const index = parent.node.body.findIndex(n => n == path.node); + if (index === -1) { + return null; + } + + if (index + 1 == parent.node.body.length) { + return null; + } + + return parent.node.body[index + 1]; +} + +function isIdentifier(node, name) { + if (node.type !== "Identifier") { + return false; + } + if (node.name !== name) { + return false; + } + return true; +} + +function isString(node) { + return node.type === "Literal" && typeof node.value === "string"; +} + +const jsmExtPattern = /\.(jsm|js|jsm\.js)$/; + +function esmifyExtension(path) { + return path.replace(jsmExtPattern, ".sys.mjs"); +} + +// Given possible member expression, return the list of Identifier nodes in +// the source order. +// +// Returns an empty array if: +// * not a simple MemberExpression tree with Identifiers +// * there's computed property +function memberExpressionsToIdentifiers(memberExpr) { + let ids = []; + + function f(node) { + if (node.type !== "MemberExpression" || node.computed) { + return false; + } + + if (node.object.type === "Identifier") { + ids.push(node.object); + ids.push(node.property); + return true; + } + + if (!f(node.object)) { + return false; + } + ids.push(node.property); + return true; + } + + if (!f(memberExpr)) { + return []; + } + + return ids; +} + +// Returns true if the node is a simple MemberExpression tree with Identifiers +// matches expectedIDs. +function isMemberExpressionWithIdentifiers(node, expectedIDs) { + const actualIDs = memberExpressionsToIdentifiers(node); + if (actualIDs.length !== expectedIDs.length) { + return false; + } + + for (let i = 0; i < expectedIDs.length; i++) { + if (actualIDs[i].name !== expectedIDs[i]) { + return false; + } + } + + return true; +} + +// Rewrite the Identifiers of MemberExpression tree to toIDs. +// `node` must be a simple MemberExpression tree with Identifiers, and +// the length of Identifiers should match. +function rewriteMemberExpressionWithIdentifiers(node, toIDs) { + const actualIDs = memberExpressionsToIdentifiers(node); + for (let i = 0; i < toIDs.length; i++) { + actualIDs[i].name = toIDs[i]; + } +} + +// Create a simple MemberExpression tree with given Identifiers. +function createMemberExpressionWithIdentifiers(jscodeshift, ids) { + if (ids.length < 2) { + throw new Error("Unexpected length of ids for member expression"); + } + + if (ids.length > 2) { + return jscodeshift.memberExpression( + createMemberExpressionWithIdentifiers(jscodeshift, ids.slice(0, -1)), + jscodeshift.identifier(ids[ids.length - 1]) + ); + } + + return jscodeshift.memberExpression( + jscodeshift.identifier(ids[0]), + jscodeshift.identifier(ids[1]) + ); +} + +exports.warnForPath = warnForPath; +exports.getPrevStatement = getPrevStatement; +exports.getNextStatement = getNextStatement; +exports.isIdentifier = isIdentifier; +exports.isString = isString; +exports.jsmExtPattern = jsmExtPattern; +exports.esmifyExtension = esmifyExtension; +exports.isMemberExpressionWithIdentifiers = isMemberExpressionWithIdentifiers; +exports.rewriteMemberExpressionWithIdentifiers = + rewriteMemberExpressionWithIdentifiers; +exports.createMemberExpressionWithIdentifiers = + createMemberExpressionWithIdentifiers; diff --git a/tools/fuzzing/common/FuzzingMutate.cpp b/tools/fuzzing/common/FuzzingMutate.cpp new file mode 100644 index 0000000000..bb5e930125 --- /dev/null +++ b/tools/fuzzing/common/FuzzingMutate.cpp @@ -0,0 +1,32 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzingMutate.h" +#include "FuzzingTraits.h" + +namespace mozilla { +namespace fuzzing { + +/** + * Randomly mutates a byte inside |aData| by using bit manipulation. + */ +/* static */ +void FuzzingMutate::ChangeBit(uint8_t* aData, size_t aLength) { + size_t offset = RandomIntegerRange(0, aLength); + aData[offset] ^= (1 << FuzzingTraits::Random(9)); +} + +/** + * Randomly replaces a byte inside |aData| with one in the range of [0, 255]. + */ +/* static */ +void FuzzingMutate::ChangeByte(uint8_t* aData, size_t aLength) { + size_t offset = RandomIntegerRange(0, aLength); + aData[offset] = RandomInteger(); +} + +} // namespace fuzzing +} // namespace mozilla diff --git a/tools/fuzzing/common/FuzzingMutate.h b/tools/fuzzing/common/FuzzingMutate.h new file mode 100644 index 0000000000..f24f557669 --- /dev/null +++ b/tools/fuzzing/common/FuzzingMutate.h @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_fuzzing_FuzzingMutate_h +#define mozilla_fuzzing_FuzzingMutate_h + +#include + +namespace mozilla { +namespace fuzzing { + +class FuzzingMutate { + public: + static void ChangeBit(uint8_t* aData, size_t aLength); + static void ChangeByte(uint8_t* aData, size_t aLength); +}; + +} // namespace fuzzing +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/common/FuzzingTraits.cpp b/tools/fuzzing/common/FuzzingTraits.cpp new file mode 100644 index 0000000000..9e6ba3ac1d --- /dev/null +++ b/tools/fuzzing/common/FuzzingTraits.cpp @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include +#include +#include +#include "FuzzingTraits.h" + +namespace mozilla { +namespace fuzzing { + +/* static */ +unsigned int FuzzingTraits::Random(unsigned int aMax) { + MOZ_ASSERT(aMax > 0, "aMax needs to be bigger than 0"); + std::uniform_int_distribution d(0, aMax); + return d(Rng()); +} + +/* static */ +bool FuzzingTraits::Sometimes(unsigned int aProbability) { + return FuzzingTraits::Random(aProbability) == 0; +} + +/* static */ +size_t FuzzingTraits::Frequency(const size_t aSize, const uint64_t aFactor) { + return RandomIntegerRange(0, ceil(float(aSize) / aFactor)) + 1; +} + +/* static */ +std::mt19937_64& FuzzingTraits::Rng() { + static std::mt19937_64 rng; + static std::once_flag flag; + std::call_once(flag, [&] { rng.seed(PR_IntervalNow()); }); + return rng; +} + +} // namespace fuzzing +} // namespace mozilla diff --git a/tools/fuzzing/common/FuzzingTraits.h b/tools/fuzzing/common/FuzzingTraits.h new file mode 100644 index 0000000000..b8c9d76ab7 --- /dev/null +++ b/tools/fuzzing/common/FuzzingTraits.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_fuzzing_FuzzingTraits_h +#define mozilla_fuzzing_FuzzingTraits_h + +#include "mozilla/Assertions.h" +#include +#include +#include + +namespace mozilla { +namespace fuzzing { + +class FuzzingTraits { + public: + static unsigned int Random(unsigned int aMax); + static bool Sometimes(unsigned int aProbability); + /** + * Frequency() defines how many mutations of a kind shall be applied to a + * target buffer by using a user definable factor. The higher the factor, + * the less mutations are being made. + */ + static size_t Frequency(const size_t aSize, const uint64_t aFactor); + + static std::mt19937_64& Rng(); +}; + +/** + * RandomNumericLimit returns either the min or max limit of an arithmetic + * data type. + */ +template +T RandomNumericLimit() { + static_assert(std::is_arithmetic_v == true, + "T must be an arithmetic type"); + return FuzzingTraits::Sometimes(2) ? std::numeric_limits::min() + : std::numeric_limits::max(); +} + +/** + * RandomInteger generates negative and positive integers in 2**n increments. + */ +template +T RandomInteger() { + static_assert(std::is_integral_v == true, "T must be an integral type"); + double r = + static_cast(FuzzingTraits::Random((sizeof(T) * CHAR_BIT) + 1)); + T x = static_cast(pow(2.0, r)) - 1; + if (std::numeric_limits::is_signed && FuzzingTraits::Sometimes(2)) { + return (x * -1) - 1; + } + return x; +} + +/** + * RandomIntegerRange returns a random integral within a [min, max] range. + */ +template +T RandomIntegerRange(T min, T max) { + static_assert(std::is_integral_v == true, "T must be an integral type"); + MOZ_ASSERT(min < max); + std::uniform_int_distribution d(min, max); + return d(FuzzingTraits::Rng()); +} +/** + * uniform_int_distribution is undefined for char/uchar. Need to handle them + * separately. + */ +template <> +inline unsigned char RandomIntegerRange(unsigned char min, unsigned char max) { + MOZ_ASSERT(min < max); + std::uniform_int_distribution d(min, max); + return static_cast(d(FuzzingTraits::Rng())); +} +template <> +inline char RandomIntegerRange(char min, char max) { + MOZ_ASSERT(min < max); + std::uniform_int_distribution d(min, max); + return static_cast(d(FuzzingTraits::Rng())); +} + +/** + * RandomFloatingPointRange returns a random floating-point number within a + * [min, max] range. + */ +template +T RandomFloatingPointRange(T min, T max) { + static_assert(std::is_floating_point_v == true, + "T must be a floating point type"); + MOZ_ASSERT(min < max); + std::uniform_real_distribution d( + min, std::nextafter(max, std::numeric_limits::max())); + return d(FuzzingTraits::Rng()); +} + +/** + * RandomFloatingPoint returns a random floating-point number in 2**n + * increments. + */ +template +T RandomFloatingPoint() { + static_assert(std::is_floating_point_v == true, + "T must be a floating point type"); + int radix = RandomIntegerRange(std::numeric_limits::min_exponent, + std::numeric_limits::max_exponent); + T x = static_cast(pow(2.0, static_cast(radix))); + return x * RandomFloatingPointRange(-1.0, 1.0); +} + +} // namespace fuzzing +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/common/moz.build b/tools/fuzzing/common/moz.build new file mode 100644 index 0000000000..afa19eddb4 --- /dev/null +++ b/tools/fuzzing/common/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += ["FuzzingMutate.cpp", "FuzzingTraits.cpp"] + +EXPORTS += ["FuzzingMutate.h", "FuzzingTraits.h"] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/docs/fuzzing_interface.rst b/tools/fuzzing/docs/fuzzing_interface.rst new file mode 100644 index 0000000000..561727d540 --- /dev/null +++ b/tools/fuzzing/docs/fuzzing_interface.rst @@ -0,0 +1,496 @@ +Fuzzing Interface +================= + +The fuzzing interface is glue code living in mozilla-central in order to +make it easier for developers and security researchers to test C/C++ +code with either `libFuzzer `__ or +`afl-fuzz `__. + +These fuzzing tools, are based on *compile-time instrumentation* to measure +things like branch coverage and more advanced heuristics per fuzzing test. +Doing so allows these tools to progress through code with little to no custom +logic/knowledge implemented in the fuzzer itself. Usually, the only thing +these tools need is a code "shim" that provides the entry point for the fuzzer +to the code to be tested. We call this additional code a *fuzzing target* and +the rest of this manual describes how to implement and work with these targets. + +As for the tools used with these targets, we currently recommend the use of +libFuzzer over afl-fuzz, as the latter is no longer maintained while libFuzzer +is being actively developed. Furthermore, libFuzzer has some advanced +instrumentation features (e.g. value profiling to deal with complicated +comparisons in code), making it overall more effective. + +What can be tested? +~~~~~~~~~~~~~~~~~~~ + +The interface can be used to test all C/C++ code that either ends up in +``libxul`` (more precisely, the gtest version of ``libxul``) **or** is +part of the JS engine. + +Note that this is not the right testing approach for testing the full +browser as a whole. It is rather meant for component-based testing +(especially as some components cannot be easily separated out of the +full build). + +.. note:: + + **Note:** If you are working on the JS engine (trying to reproduce a + bug or seeking to develop a new fuzzing target), then please also read + the :ref:`JS Engine Specifics Section ` at the end + of this documentation, as the JS engine offers additional options for + implementing and running fuzzing targets. + + +Reproducing bugs for existing fuzzing targets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are working on a bug that involves an existing fuzzing interface target, +you have two options for reproducing the issue: + + +Using existing builds +^^^^^^^^^^^^^^^^^^^^^ + +We have several fuzzing builds in CI that you can simply download. We recommend +using ``fuzzfetch`` for this purpose, as it makes downloading and unpacking +these builds much easier. + +You can install ``fuzzfetch`` from +`Github `__ or +`via pip `__. + +Afterwards, you can run + +:: + + $ python -m fuzzfetch -a --fuzzing --gtest -n firefox-fuzzing + +to fetch the latest optimized build. Alternatively, we offer non-ASan debug builds +which you can download using + +:: + + $ python -m fuzzfetch -d --fuzzing --gtest -n firefox-fuzzing + +In both commands, ``firefox-fuzzing`` indicates the name of the directory that +will be created for the download. + +Afterwards, you can reproduce the bug using + +:: + + $ FUZZER=TargetName firefox-fuzzing/firefox test.bin + +assuming that ``TargetName`` is the name of the fuzzing target specified in the +bug you are working on and ``test.bin`` is the attached testcase. + +.. note:: + + **Note:** You should not export the ``FUZZER`` variable permanently + in your shell, especially if you plan to do local builds. If the ``FUZZER`` + variable is exported, it will affect the build process. + +If the CI builds don't meet your requirements and you need a local build instead, +you can follow the steps below to create one: + +.. _Local build requirements and flags: + +Local build requirements and flags +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You will need a Linux environment with a recent Clang. Using the Clang downloaded +by ``./mach bootstrap`` or a newer version is recommended. + +The only build flag required to enable the fuzzing targets is ``--enable-fuzzing``, +so adding + +:: + + ac_add_options --enable-fuzzing + +to your ``.mozconfig`` is already sufficient for producing a fuzzing build. +However, for improved crash handling capabilities and to detect additional errors, +it is strongly recommended to combine libFuzzer with :ref:`AddressSanitizer
` +at least for optimized builds and bugs requiring ASan to reproduce at all +(e.g. you are working on a bug where ASan reports a memory safety violation +of some sort). + +Once your build is complete, if you want to run gtests, you **must** additionally run + +:: + + $ ./mach gtest dontruntests + +to force the gtest libxul to be built. + +.. note:: + + **Note:** If you modify any code, please ensure that you run **both** build + commands to ensure that the gtest libxul is also rebuilt. It is a common mistake + to only run ``./mach build`` and miss the second command. + +Once these steps are complete, you can reproduce the bug locally using the same +steps as described above for the downloaded builds. + + +Developing new fuzzing targets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Developing a new fuzzing target using the fuzzing interface only requires a few steps. + + +Determine if the fuzzing interface is the right tool +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The fuzzing interface is not suitable for every kind of testing. In particular +if your testing requires the full browser to be running, then you might want to +look into other testing methods. + +The interface uses the ``ScopedXPCOM`` implementation to provide an environment +in which XPCOM is available and initialized. You can initialize further subsystems +that you might require, but you are responsible yourself for any kind of +initialization steps. + +There is (in theory) no limit as to how far you can take browser initialization. +However, the more subsystems are involved, the more problems might occur due to +non-determinism and loss of performance. + +If you are unsure if the fuzzing interface is the right approach for you or you +require help in evaluating what could be done for your particular task, please +don't hestitate to :ref:`contact us `. + + +Develop the fuzzing code +^^^^^^^^^^^^^^^^^^^^^^^^ + +Where to put your fuzzing code +'''''''''''''''''''''''''''''' + +The code using the fuzzing interface usually lives in a separate directory +called ``fuzztest`` that is on the same level as gtests. If your component +has no gtests, then a subdirectory either in tests or in your main directory +will work. If such a directory does not exist yet in your component, then you +need to create one with a suitable ``moz.build``. See `the transport target +for an example `__ + +In order to include the new subdirectory into the build process, you will +also have to modify the toplevel ``moz.build`` file accordingly. For this +purpose, you should add your directory to ``TEST_DIRS`` only if ``FUZZING_INTERFACES`` +is set. See again `the transport target for an example +`__. + +How your code should look like +'''''''''''''''''''''''''''''' + +In order to define your fuzzing target ``MyTarget``, you only need to implement 2 functions: + +1. A one-time initialization function. + + At startup, the fuzzing interface calls this function **once**, so this can + be used to perform one-time operations like initializing subsystems or parsing + extra fuzzing options. + + This function is the equivalent of the `LLVMFuzzerInitialize `__ + function and has the same signature. However, with our fuzzing interface, + it won't be resolved by its name, so it can be defined ``static`` and called + whatever you prefer. Note that the function should always ``return 0`` and + can (except for the return), remain empty. + + For the sake of this documentation, we assume that you have ``static int FuzzingInitMyTarget(int* argc, char*** argv);`` + +2. The fuzzing iteration function. + + This is where the actual fuzzing happens, and this function is the equivalent + of `LLVMFuzzerTestOneInput `__. + Again, the difference to the fuzzing interface is that the function won't be + resolved by its name. In addition, we offer two different possible signatures + for this function, either + + ``static int FuzzingRunMyTarget(const uint8_t* data, size_t size);`` + + or + + ``static int FuzzingRunMyTarget(nsCOMPtr inputStream);`` + + The latter is just a wrapper around the first one for implementations that + usually work with streams. No matter which of the two signatures you choose + to work with, the only thing you need to implement inside the function + is the use of the provided data with your target implementation. This can + mean to simply feed the data to your target, using the data to drive operations + on the target API, or a mix of both. + + While doing so, you should avoid altering global state in a permanent way, + using additional sources of data/randomness or having code run beyond the + lifetime of the iteration function (e.g. on another thread), for one simple + reason: Coverage-guided fuzzing tools depend on the **deterministic** nature + of the iteration function. If the same input to this function does not lead + to the same execution when run twice (e.g. because the resulting state depends + on multiple successive calls or because of additional external influences), + then the tool will not be able to reproduce its fuzzing progress and perform + badly. Dealing with this restriction can be challenging e.g. when dealing + with asynchronous targets that run multi-threaded, but can usually be managed + by synchronizing execution on all threads at the end of the iteration function. + For implementations accumulating global state, it might be necessary to + (re)initialize this global state in each iteration, rather than doing it once + in the initialization function, even if this costs additional performance. + + Note that unlike the vanilla libFuzzer approach, you are allowed to ``return 1`` + in this function to indicate that an input is "bad". Doing so will cause + libFuzzer to discard the input, no matter if it generated new coverage or not. + This is particularly useful if you have means to internally detect and catch + bad testcase behavior such as timeouts/excessive resource usage etc. to avoid + these tests to end up in your corpus. + + +Once you have implemented the two functions, the only thing remaining is to +register them with the fuzzing interface. For this purpose, we offer two +macros, depending on which iteration function signature you used. If you +sticked to the classic signature using buffer and size, you can simply use + +:: + + #include "FuzzingInterface.h" + + // Your includes and code + + MOZ_FUZZING_INTERFACE_RAW(FuzzingInitMyTarget, FuzzingRunMyTarget, MyTarget); + +where ``MyTarget`` is the name of the target and will be used later to decide +at runtime which target should be used. + +If instead you went for the streaming interface, you need a different include, +but the macro invocation is quite similar: + +:: + + #include "FuzzingInterfaceStream.h" + + // Your includes and code + + MOZ_FUZZING_INTERFACE_STREAM(FuzzingInitMyTarget, FuzzingRunMyTarget, MyTarget); + +For a live example, see also the `implementation of the STUN fuzzing target +`__. + +Add instrumentation to the code being tested +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +libFuzzer requires that the code you are trying to test is instrumented +with special compiler flags. Fortunately, adding these on a per-directory basis +can be done just by including the following directive in each ``moz.build`` +file that builds code under test: + +:: + + # Add libFuzzer configuration directives + include('/tools/fuzzing/libfuzzer-config.mozbuild') + + +The include already does the appropriate configuration checks to be only +active in fuzzing builds, so you don't have to guard this in any way. + +.. note:: + + **Note:** This include modifies `CFLAGS` and `CXXFLAGS` accordingly + but this only works for source files defined in this particular + directory. The flags are **not** propagated to subdirectories automatically + and you have to ensure that each directory that builds source files + for your target has the include added to its ``moz.build`` file. + +By keeping the instrumentation limited to the parts that are actually being +tested using this tool, you not only increase the performance but also potentially +reduce the amount of noise that libFuzzer sees. + + +Build your code +^^^^^^^^^^^^^^^ + +See the :ref:`Build instructions above ` for instructions +how to modify your ``.mozconfig`` to create the appropriate build. + + +Running your code and building a corpus +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You need to set the following environment variable to enable running the +fuzzing code inside Firefox instead of the regular browser. + +- ``FUZZER=name`` + +Where ``name`` is the name of your fuzzing module that you specified +when calling the ``MOZ_FUZZING_INTERFACE_RAW`` macro. For the example +above, this would be ``MyTarget`` or ``StunParser`` for the live example. + +Now when you invoke the firefox binary in your build directory with the +``-help=1`` parameter, you should see the regular libFuzzer help. On +Linux for example: + +:: + + $ FUZZER=StunParser obj-asan/dist/bin/firefox -help=1 + +You should see an output similar to this: + +:: + + Running Fuzzer tests... + Usage: + + To run fuzzing pass 0 or more directories. + obj-asan/dist/bin/firefox [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ] + + To run individual tests without fuzzing pass 1 or more files: + obj-asan/dist/bin/firefox [-flag1=val1 [-flag2=val2 ...] ] file1 [file2 ...] + + Flags: (strictly in form -flag=value) + verbosity 1 Verbosity level. + seed 0 Random seed. If 0, seed is generated. + runs -1 Number of individual test runs (-1 for infinite runs). + max_len 0 Maximum length of the test input. If 0, libFuzzer tries to guess a good value based on the corpus and reports it. + ... + + +Reproducing a Crash +''''''''''''''''''' + +In order to reproduce a crash from a given test file, simply put the +file as the only argument on the command line, e.g. + +:: + + $ FUZZER=StunParser obj-asan/dist/bin/firefox test.bin + +This should reproduce the given problem. + + +FuzzManager and libFuzzer +''''''''''''''''''''''''' + +Our FuzzManager project comes with a harness for running libFuzzer with +an optional connection to a FuzzManager server instance. Note that this +connection is not mandatory, even without a server you can make use of +the local harness. + +You can find the harness +`here `__. + +An example invocation for the harness to use with StunParser could look +like this: + +:: + + FUZZER=StunParser python /path/to/afl-libfuzzer-daemon.py --fuzzmanager \ + --stats libfuzzer-stunparser.stats --libfuzzer-auto-reduce-min 500 --libfuzzer-auto-reduce 30 \ + --tool libfuzzer-stunparser --libfuzzer --libfuzzer-instances 6 obj-asan/dist/bin/firefox \ + -max_len=256 -use_value_profile=1 -rss_limit_mb=3000 corpus-stunparser + +What this does is + +- run libFuzzer on the ``StunParser`` target with 6 parallel instances + using the corpus in the ``corpus-stunparser`` directory (with the + specified libFuzzer options such as ``-max_len`` and + ``-use_value_profile``) +- automatically reduce the corpus and restart if it grew by 30% (and + has at least 500 files) +- use FuzzManager (need a local ``.fuzzmanagerconf`` and a + ``firefox.fuzzmanagerconf`` binary configuration as described in the + FuzzManager manual) and submit crashes as ``libfuzzer-stunparser`` + tool +- write statistics to the ``libfuzzer-stunparser.stats`` file + +.. _JS Engine Specifics: + +JS Engine Specifics +~~~~~~~~~~~~~~~~~~~ + +The fuzzing interface can also be used for testing the JS engine, in fact there +are two separate options to implement and run fuzzing targets: + +Implementing in C++ +^^^^^^^^^^^^^^^^^^^ + +Similar to the fuzzing interface in Firefox, you can implement your target in +entirely C++ with very similar interfaces compared to what was described before. + +There are a few minor differences though: + +1. All of the fuzzing targets live in `js/src/fuzz-tests`. + +2. All of the code is linked into a separate binary called `fuzz-tests`, + similar to how all JSAPI tests end up in `jsapi-tests`. In order for this + binary to be built, you must build a JS shell with ``--enable-fuzzing`` + **and** ``--enable-tests``. Again, this can and should be combined with + AddressSanitizer for maximum effectiveness. This also means that there is no + need to (re)build gtests when dealing with a JS fuzzing target and using + a shell as part of a full browser build. + +3. The harness around the JS implementation already provides you with an + initialized ``JSContext`` and global object. You can access these in + your target by declaring + + ``extern JS::PersistentRootedObject gGlobal;`` + + and + + ``extern JSContext* gCx;`` + + but there is no obligation for you to use these. + +For a live example, see also the `implementation of the StructuredCloneReader target +`__. + + +Implementing in JS +^^^^^^^^^^^^^^^^^^ + +In addition to the C++ targets, you can also implement targets in JavaScript +using the JavaScript Runtime (JSRT) fuzzing approach. Using this approach is +not only much simpler (since you don't need to know anything about the +JSAPI or engine internals), but it also gives you full access to everything +defined in the JS shell, including handy functions such as ``timeout()``. + +Of course, this approach also comes with disadvantages: Calling into JS and +performing the fuzzing operations there costs performance. Also, there is more +chance for causing global side-effects or non-determinism compared to a +fairly isolated C++ target. + +As a rule of thumb, you should implement the target in JS if + +* you don't know C++ and/or how to use the JSAPI (after all, a JS fuzzing target is better than none), +* your target is expected to have lots of hangs/timeouts (you can catch these internally), +* or your target is not isolated enough for a C++ target and/or you need specific JS shell functions. + + +There is an `example target `__ +in-tree that shows roughly how to implement such a fuzzing target. + +To run such a target, you must run the ``js`` (shell) binary instead of the +``fuzz-tests`` binary and point the ``FUZZER`` variable to the file containing +your fuzzing target, e.g. + +:: + + $ FUZZER=/path/to/jsrtfuzzing-example.js obj-asan/dist/bin/js --fuzzing-safe --no-threads -- + +More elaborate targets can be found in `js/src/fuzz-tests/ `__. + +Troubleshooting +~~~~~~~~~~~~~~~ + + +Fuzzing Interface: Error: No testing callback found +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This error means that the fuzzing callback with the name you specified +using the ``FUZZER`` environment variable could not be found. Reasons +for are typically either a misspelled name or that your code wasn't +built (check your ``moz.build`` file and build log). + + +``mach build`` doesn't seem to update my fuzzing code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Keep in mind you always need to run both the ``mach build`` and +``mach gtest dontruntests`` commands in order to update your fuzzing +code. The latter rebuilds the gtest version of ``libxul``, containing +your code. diff --git a/tools/fuzzing/docs/index.rst b/tools/fuzzing/docs/index.rst new file mode 100644 index 0000000000..9a4e2d01c4 --- /dev/null +++ b/tools/fuzzing/docs/index.rst @@ -0,0 +1,438 @@ +Fuzzing +======= + +.. toctree:: + :maxdepth: 1 + :hidden: + :glob: + :reversed: + + * + +This section focuses on explaining the software testing technique called +“Fuzzing” or “Fuzz Testing” and its application to the Mozilla codebase. +The overall goal is to educate developers about the capabilities and +usefulness of fuzzing and also allow them to write their own fuzzing +targets. Note that not all fuzzing tools used at Mozilla are open +source. Some tools are for internal use only because they can easily +find critical security vulnerabilities. + +What is Fuzzing? +---------------- + +Fuzzing (or Fuzz Testing) is a technique to randomly use a program or +parts of it with the goal to uncover bugs. Random usage can have a wide +variety of forms, a few common ones are + +- random input data (e.g. file formats, network data, source code, etc.) + +- random API usage + +- random UI interaction + +with the first two being the most practical methods used in the field. +Of course, these methods are not entirely separate, combinations are +possible. Fuzzing is a great way to find quality issues, some of them +being also security issues. + +Random input data +~~~~~~~~~~~~~~~~~ + +This is probably the most obvious fuzzing method: You have code that +processes data and you provide it with random or mutated data, hoping +that it will uncover bugs in your implementation. Examples are media +formats like JPEG or H.264, but basically anything that involves +processing a “blob” of data can be a valuable target. Countless security +vulnerabilities in a variety of libraries and programs have been found +using this method (the AFLFuzz +`bug-o-rama `__ gives a good +impression). + +Common tools for this task are e.g. +`libFuzzer `__ and +`AFLFuzz `__, but also specialized +tools with custom logic like +`LangFuzz `__ +and `Avalanche `__. + +Random API Usage +~~~~~~~~~~~~~~~~ + +Randomly testing APIs is especially helpful with parts of software that +expose a well-defined interface (see also :ref:`Well-defined +behavior and Safety `). If this interface is additionally exposed to +untrusted parties/content, then this is a strong sign that random API +testing would be worthwhile here, also for security reasons. APIs can be +anything from C++ layer code to APIs offered in the browser. + +A good example for a fuzzing target here is the DOM (Document Object +Model) and various other browser APIs. The browser exposes a variety of +different APIs for working with documents, media, communication, +storage, etc. with a growing complexity. Each of these APIs has +potential bugs that can be uncovered with fuzzing. At Mozilla, we +currently use domino (internal tool) for this purpose. + +Random UI Interaction +~~~~~~~~~~~~~~~~~~~~~ + +A third way to test programs and in particular user interfaces is by +directly interacting with the UI in a random way, typically in +combination with other actions the program has to perform. Imagine for +example an automated browser that surfs through the web and randomly +performs actions such as scrolling, zooming and clicking links. The nice +thing about this approach is that you likely find many issues that the +end-user also experiences. However, this approach typically suffers from +bad reproducibility (see also :ref:`Reproducibility `) and is therefore +often of limited use. + +An example for a fuzzing tool using this technique is `Android +Monkey `__. At +Mozilla however, we currently don’t make much use of this approach. + +Why Fuzzing Helps You +--------------------- + +Understanding the value of fuzzing for you as a developer and software +quality in general is important to justify the support this testing +method might need from you. When your component is fuzzed for the first +time there are two common things you will be confronted with: + +**Bug reports that don’t seem real bugs or not important:** Fuzzers +find all sorts of bugs in various corners of your component, even +obscure ones. This automatically leads to a larger number of bugs that +either don’t seem to be bugs (see also the :ref:`Well-defined behavior and +safety ` section below) or that don’t seem to be important bugs. + +Fixing these bugs is still important for the fuzzers because ignoring them +in fuzzing costs resources (performance, human resources) and might even +prevent the fuzzer from hitting other bugs. For example certain fuzzing tools +like libFuzzer run in-process and have to restart on every crash, involving a +costly re-read of the fuzzing samples. + +Also, as some of our code evolves quickly, a corner case might become a +hot code path in a few months. + +**New steps to reproduce:** Fuzzing tools are very likely to exercise +your component using different methods than an average end-user. A +common technique is modify existing parts of a program or write entirely +new code to yield a fuzzing "target". This target is specifically +designed to work with the fuzzing tools in use. Reproducing the reported +bugs might require you to learn these new steps to reproduce, including +building/acquiring that target and having the right environment. + +Both of these issues might seem like a waste of time in some cases, +however, realizing that both steps are a one-time investment for a +constant stream of valuable bug reports is paramount here. Helping your +security engineers to overcome these issues will ensure that future +regressions in your code can be detected at an earlier stage and in a +form that is more easily actionable. Especially if you are dealing with +regressions in your code already, fuzzing has the potential to make your +job as a developer easier. + +One of the best examples at Mozilla is the JavaScript engine. The JS +team has put great quite some effort into getting fuzzing started and +supporting our work. Here’s what Jan de Mooij, a senior platform +engineer for the JavaScript engine, has to say about it: + +*“Bugs in the engine can cause mysterious browser crashes and bugs that +are incredibly hard to track down. Fortunately, we don't have to deal +with these time consuming browser issues very often: usually the fuzzers +find a reliable shell test long before the bug makes it into a release. +Fuzzing is invaluable to us and I cannot imagine working on this project +without it.”* + +Levels of Fuzzing in Firefox/Gecko +---------------------------------- + +Applying fuzzing to e.g. Firefox happens at different "levels", similar +to the different types of automated tests we have: + +Full Browser Fuzzing +~~~~~~~~~~~~~~~~~~~~ + +The most obvious method of testing would be to test the full browser and +doing so is required for certain features like the DOM and other APIs. +The advantage here is that we have all the features of the browser +available and testing happens closely to what we actually ship. The +downside here though is that browser testing is by far the slowest of +all testing methods. In addition, it has the most amount of +non-determinism involved (resulting e.g. in intermittent testcases). +Browser fuzzing at Mozilla is largely done with the `Grizzly +framework `__ +(`meta bug `__) +and one of the most successful fuzzers is the Domino tool (`meta +bug `__). + +Summarizing, full browser fuzzing is the right technique to investigate +if your feature really requires it. Consider using other methods (see +below) if your code can be exercised in this way. + +The Fuzzing Interface +~~~~~~~~~~~~~~~~~~~~~ + +**Fuzzing Interface** + +The fuzzing interface is glue code living in mozilla-central in order to make it +easier for developers and security researchers to test C/C++ code with either libFuzzer or afl-fuzz. + +This interface offers a gtest (C++ unit test) level component based +fuzzing approach and is suitable for anything that could also be +tested/exercised using a gtest. This method is by far the fastest, but +usually limited to testing isolated components that can be instantiated +on this level. Utilizing this method requires you to write a fuzzing +target similar to writing a gtest. This target will automatically be +usable with libFuzzer and AFLFuzz. We offer a :ref:`comprehensive manual ` +that describes how to write and utilize your own target. + +A simple example here is the `SDP parser +target `__, +which tests the SipccSdpParser in our codebase. + +Shell-based Fuzzing +~~~~~~~~~~~~~~~~~~~ + +Some of our fuzzing, e.g. JS Engine testing, happens in a separate shell +program. For JS, this is the JS shell also used for most of the JS tests +and development. In theory, xpcshell could also be used for testing but +so far, there has not been a use case for this (most things that can be +reached through xpcshell can also be tested on the gtest level). + +Identifying the right level of fuzzing is the first step towards +continuous fuzz testing of your code. + +Code/Process Requirements for Fuzzing +------------------------------------- + +In this section, we are going to discuss how code should be written in +order to yield optimal results with fuzzing. + +Defect Oracles +~~~~~~~~~~~~~~ + +Fuzzing is only effective if you are able to know when a problem has +been found. Crashes are typically problems if the unit being tested is +safe for fuzzing (see Well-defined behavior and Safety). But there are +many more problems that you would want to find, correctness issues, +corruptions that don’t necessarily crash etc. For this, you need an +*oracle* that tells you something is wrong. + +The simplest defect oracle is the assertion (ex: ``MOZ_ASSERT``). +Assertions are a very powerful instrument because they can be used to +determine if your program is performing correctly, even if the bug would +not lead to any sort of crash. They can encode arbitrarily complex +information about what is considered correct, information that might +otherwise only exist in the developers’ minds. + +External tools like the sanitizers (AddressSanitizer aka ASan, +ThreadSanitizer aka TSan, MemorySanitizer aka MSan and +UndefinedBehaviorSanitizer - UBSan) can also serve as oracles for +sometimes severe issues that would not necessarily crash. Making sure +that these tools can be used on your code is highly useful. + +Examples for bugs found with sanitizers are `bug +1419608 `__, +`bug 1580288 `__ +and `bug 922603 `__, +but since we started using sanitizers, we have found over 1000 bugs with +these tools. + +Another defect oracle can be a reference implementation. Comparing +program behavior (typically output) between two programs or two modes of +the same program that should produce the same outputs can find complex +correctness issues. This method is often called differential testing. + +One example where this is regularly used to find issues is the Mozilla +JavaScript engine: Running random programs with and without JIT +compilation enabled finds lots of problems with the JIT implementation. +One example for such a bug is `Bug +1404636 `__. + +Component Decoupling +~~~~~~~~~~~~~~~~~~~~ + +Being able to test components in isolation can be an advantage for +fuzzing (both for performance and reproducibility). Clear boundaries +between different components and documentation that explains the +contracts usually help with this goal. Sometimes it might be useful to +mock a certain component that the target component is interacting with +and that is much harder if the components are tightly coupled and their +contracts unclear. Of course, this does not mean that one should only +test components in isolation. Sometimes, testing the interaction between +them is even desirable and does not hurt performance at all. + +Avoiding external I/O +~~~~~~~~~~~~~~~~~~~~~ + +External I/O like network or file interactions are bad for performance +and can introduce additional non-determinism. Providing interfaces to +process data directly from memory instead is usually much more helpful. + +.. _Well defined behaviour and safety: + +Well-defined Behavior and Safety +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This requirement mostly ties in where defect oracles ended and is one of +the most important problems seen in the wild nowadays with fuzzing. If a +part of your program’s behavior is unspecified, then this potentially +leads to bad times if the behavior is considered a defect by fuzzing. +For example, if your code has crashes that are not considered bugs, then +your code might be unsuitable for fuzzing. Your component should be +fuzzing safe, meaning that any defect oracle (e.g. assertion or crash) +triggered by the fuzzer is considered a bug. This important aspect is +often neglected. Be aware that any false positives cause both +performance degradation and additional manual work for your fuzzing +team. The Mozilla JS developers for example have implemented this +concept in a “--fuzzing-safe” switch which disables harmful functions. +Sometimes, crashes cannot be avoided for handling certain error +conditions. In such situations, it is important to mark these crashes in +a way the fuzzer can recognize and distinguish them from undesired +crashes. However, keep in mind that crashes in general can be disruptive +to the fuzzing process. Performance is an important aspect of fuzzing +and frequent crashes can severely degrade performance. + +.. _Reproducibility: + +Reproducibility +~~~~~~~~~~~~~~~ + +Being able to reproduce issues found with fuzzing is necessary for +several reasons: First, you as the developer probably want a test that +reproduces the issue so you can debug it better. Our feedback from most +developers is that traces without a reproducible test can help to find a +problem, but it makes the whole process very complicated. Some of these +non-reproducible bugs never get fixed. Second, having a reproducible +test also helps the triage process by allowing an automated bisection to +find the responsible developer. Last but not least, the test can be +added to a test suite, used for automated verification of fixes and even +serve as a basis for more fuzzing. + +Adding functionality to the program that improve reproducibility is +therefore a good idea in case non-reproducible issues are found. Some +examples are shown in the next section. + +While many problems with reproducibility are specific for the project +you are working on, there is one source of these problems that many +programs have in common: Threading. While some bugs only occur in the +first place due to concurrency, some other bugs would be perfectly +reproducible without threads, but are intermittent and hard to with +threading enabled. If the bug is indeed caused by a data race, then +tools like ThreadSanitizer will help and we are currently working on +making ThreadSanitizer usable on Firefox. For bugs that are not caused +by threading, it sometimes makes sense to be able to disable threading +or limit the amount of worker threads involved. + +Supporting Code +~~~~~~~~~~~~~~~ + +Some possibilities of what support implementations for fuzzing can do +have already been named in the previous sections: Additional defect +oracles and functionality to improve reproducibility and safety. In +fact, many features added specifically for fuzzing fit into one of these +categories. However, there’s room for more: Often, there are ways to +make it easier for fuzzers to exercise complex and hard to reach parts +of your code. For example, if a certain optimization feature is only +turned on under very specific conditions (that are not a requirement for +the optimization), then it makes sense to add a functionality to force +it on. Then, a fuzzer can hit the optimization code much more +frequently, increasing the chance to find issues. Some examples from +Firefox and SpiderMonkey: + +- The `FuzzingFunctions `__ + interface in the browser allows fuzzing tools to perform GC/CC, tune various + settings related to garbage collection or enable features like accessibility + mode. Being able to force a garbage collection at a specific time helped + identifying lots of problems in the past. + +- The --ion-eager and --baseline-eager flags for the JS shell force JIT + compilation at various stages, rather than using the builtin + heuristic to enable it only for hot functions. + +- The --no-threads flag disables all threading (if possible) in the JS shell. + This makes some bugs reproduce deterministically that would otherwise be + intermittent and harder to find. However, some bugs that only occur with + threading can’t be found with this option enabled. + +Another important feature that must be turned off for fuzzing is +checksums. Many file formats use checksums to validate a file before +processing it. If a checksum feature is still enabled, fuzzers are +likely never going to produce valid files. The same often holds for +cryptographic signatures. Being able to turn off the validation of these +features as part of a fuzzing switch is extremely helpful. + +An example for such a checksum can be found in the +`FlacDemuxer `__. + +Test Samples +~~~~~~~~~~~~ + +Some fuzzing strategies make use of existing data that is mutated to +produce the new random data. In fact, mutation-based strategies are +typically superior to others if the original samples are of good quality +because the originals carry a lot of semantics that the fuzzer does not +have to know about or implement. However, success here really stands and +falls with the quality of the samples. If the originals don’t cover +certain parts of the implementation, then the fuzzer will also have to +do more work to get there. + + +Fuzz Blockers +~~~~~~~~~~~~~ + +Fuzz blockers are issues that prevent fuzzers from being as +effective as possible. Depending on the fuzzer and its scope a fuzz blocker +in one area (or component) can impede performance in other areas and in +some cases block the fuzzer all together. Some examples are: + +- Frequent crashes - These can block code paths and waste compute + resources due to the need to relaunch the fuzzing target and handle + the results (regardless of whether it is ignored or reported). This can also + include assertions that are mostly benign in many cases are but easily + triggered by fuzzers. + +- Frequent hangs / timeouts - This includes any issue that slows down + or blocks execution of the fuzzer or the target. + +- Hard to bucket - This includes crashes such as stack overflows or any issue + that crashes in an inconsistent location. This also includes issues that + corrupt logs/debugger output or provide a broken/invalid crash report. + +- Broken builds - This is fairly straightforward, without up-to-date builds + fuzzers are unable to run or verify fixes. + +- Missing instrumentation - In some cases tools such as ASan are used as + defect oracles and are required by the fuzzing tools to allow for proper + automation. In other cases incomplete instrumentation can give a false sense + of stability or make investigating issues much more time consuming. Although + this is not necessarily blocking the fuzzers it should be prioritized + appropriately. + +Since these types of crashes harm the overall fuzzing progress, it is important +for them to be addressed in a timely manner. Even if the bug itself might seem +trivial and low priority for the product, it can still have devastating effects +on fuzzing and hence prevent finding other critical issues. + +Issues in Bugzilla are marked as fuzz blockers by adding “[fuzzblocker]” +to the “Whiteboard” field. A list of open issues marked as fuzz blockers +can be found on `Bugzilla `__. + + +Documentation +~~~~~~~~~~~~~ + +It is important for the fuzzing team to know how your software, tests +and designs work. Even obvious tasks, like how a test program is +supposed to be invoked, which options are safe, etc. might be hard to +figure out for the person doing the testing, just as you are reading +this manual right now to find out what is important in fuzzing. + +Contact Us +~~~~~~~~~~ + +The fuzzing team can be reached at +`fuzzing@mozilla.com `__ or +`on Matrix `__ +and will be happy to help you with any questions about fuzzing +you might have. We can help you find the right method of fuzzing for +your feature, collaborate on the implementation and provide the +infrastructure to run it and process the results accordingly. diff --git a/tools/fuzzing/interface/FuzzingInterface.cpp b/tools/fuzzing/interface/FuzzingInterface.cpp new file mode 100644 index 0000000000..f06ca68656 --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterface.cpp @@ -0,0 +1,30 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Common code for the unified fuzzing interface + */ + +#include +#include +#include "FuzzingInterface.h" + +namespace mozilla { + +#ifdef JS_STANDALONE +static bool fuzzing_verbose = !!getenv("MOZ_FUZZ_LOG"); +void fuzzing_log(const char* aFmt, ...) { + if (fuzzing_verbose) { + va_list ap; + va_start(ap, aFmt); + vfprintf(stderr, aFmt, ap); + va_end(ap); + } +} +#else +LazyLogModule gFuzzingLog("nsFuzzing"); +#endif + +} // namespace mozilla diff --git a/tools/fuzzing/interface/FuzzingInterface.h b/tools/fuzzing/interface/FuzzingInterface.h new file mode 100644 index 0000000000..792f0809ec --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterface.h @@ -0,0 +1,115 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface definitions for the unified fuzzing interface + */ + +#ifndef FuzzingInterface_h__ +#define FuzzingInterface_h__ + +#include + +#ifdef LIBFUZZER +# include "FuzzerExtFunctions.h" +#endif + +#include "FuzzerRegistry.h" +#include "mozilla/Assertions.h" + +#ifndef JS_STANDALONE +# include "mozilla/Logging.h" +#endif + +namespace mozilla { + +#ifdef JS_STANDALONE +void fuzzing_log(const char* aFmt, ...); +# define MOZ_LOG_EXPAND_ARGS(...) __VA_ARGS__ + +# define FUZZING_LOG(args) fuzzing_log(MOZ_LOG_EXPAND_ARGS args); +#else +extern LazyLogModule gFuzzingLog; + +# define FUZZING_LOG(args) \ + MOZ_LOG(mozilla::gFuzzingLog, mozilla::LogLevel::Verbose, args) +#endif // JS_STANDALONE + +typedef int (*FuzzingTestFuncRaw)(const uint8_t*, size_t); + +#ifdef AFLFUZZ + +static int afl_interface_raw(const char* testFile, + FuzzingTestFuncRaw testFunc) { + char* buf = NULL; + + while (__AFL_LOOP(1000)) { + std::ifstream is; + is.open(testFile, std::ios::binary); + is.seekg(0, std::ios::end); + int len = is.tellg(); + is.seekg(0, std::ios::beg); + MOZ_RELEASE_ASSERT(len >= 0); + if (!len) { + is.close(); + continue; + } + buf = (char*)realloc(buf, len); + MOZ_RELEASE_ASSERT(buf); + is.read(buf, len); + is.close(); + testFunc((uint8_t*)buf, (size_t)len); + } + + free(buf); + + return 0; +} + +# define MOZ_AFL_INTERFACE_COMMON() \ + char* testFilePtr = getenv("MOZ_FUZZ_TESTFILE"); \ + if (!testFilePtr) { \ + fprintf(stderr, \ + "Must specify testfile in MOZ_FUZZ_TESTFILE environment " \ + "variable.\n"); \ + return 1; \ + } \ + /* Make a copy of testFilePtr so the testing function can safely call \ + * getenv \ + */ \ + std::string testFile(testFilePtr); + +# define MOZ_AFL_INTERFACE_RAW(initFunc, testFunc, moduleName) \ + static int afl_fuzz_##moduleName(const uint8_t* data, size_t size) { \ + MOZ_RELEASE_ASSERT(data == NULL && size == 0); \ + MOZ_AFL_INTERFACE_COMMON(); \ + return ::mozilla::afl_interface_raw(testFile.c_str(), testFunc); \ + } \ + static void __attribute__((constructor)) AFLRegister##moduleName() { \ + ::mozilla::FuzzerRegistry::getInstance().registerModule( \ + #moduleName, initFunc, afl_fuzz_##moduleName); \ + } +#else +# define MOZ_AFL_INTERFACE_RAW(initFunc, testFunc, moduleName) /* Nothing */ +#endif // AFLFUZZ + +#ifdef LIBFUZZER +# define MOZ_LIBFUZZER_INTERFACE_RAW(initFunc, testFunc, moduleName) \ + static void __attribute__((constructor)) LibFuzzerRegister##moduleName() { \ + ::mozilla::FuzzerRegistry::getInstance().registerModule( \ + #moduleName, initFunc, testFunc); \ + } +#else +# define MOZ_LIBFUZZER_INTERFACE_RAW(initFunc, testFunc, \ + moduleName) /* Nothing */ +#endif + +#define MOZ_FUZZING_INTERFACE_RAW(initFunc, testFunc, moduleName) \ + MOZ_LIBFUZZER_INTERFACE_RAW(initFunc, testFunc, moduleName); \ + MOZ_AFL_INTERFACE_RAW(initFunc, testFunc, moduleName); + +} // namespace mozilla + +#endif // FuzzingInterface_h__ diff --git a/tools/fuzzing/interface/FuzzingInterfaceStream.cpp b/tools/fuzzing/interface/FuzzingInterfaceStream.cpp new file mode 100644 index 0000000000..f2c5c891e9 --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterfaceStream.cpp @@ -0,0 +1,54 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface implementation for the unified fuzzing interface + */ + +#include "nsIFile.h" +#include "nsIPrefService.h" +#include "nsIProperties.h" + +#include "FuzzingInterfaceStream.h" + +#include "mozilla/Assertions.h" + +#ifndef JS_STANDALONE +# include "nsNetUtil.h" +#endif + +namespace mozilla { + +#ifdef AFLFUZZ + +void afl_interface_stream(const char* testFile, + FuzzingTestFuncStream testFunc) { + nsresult rv; + nsCOMPtr dirService = + do_GetService(NS_DIRECTORY_SERVICE_CONTRACTID); + MOZ_RELEASE_ASSERT(dirService != nullptr); + nsCOMPtr file; + rv = dirService->Get(NS_OS_CURRENT_WORKING_DIR, NS_GET_IID(nsIFile), + getter_AddRefs(file)); + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + file->AppendNative(nsDependentCString(testFile)); + while (__AFL_LOOP(1000)) { + nsCOMPtr inputStream; + rv = NS_NewLocalFileInputStream(getter_AddRefs(inputStream), file); + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + if (!NS_InputStreamIsBuffered(inputStream)) { + nsCOMPtr bufStream; + rv = NS_NewBufferedInputStream(getter_AddRefs(bufStream), + inputStream.forget(), 1024); + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + inputStream = bufStream; + } + testFunc(inputStream.forget()); + } +} + +#endif + +} // namespace mozilla diff --git a/tools/fuzzing/interface/FuzzingInterfaceStream.h b/tools/fuzzing/interface/FuzzingInterfaceStream.h new file mode 100644 index 0000000000..1542020794 --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterfaceStream.h @@ -0,0 +1,90 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface definitions for the unified fuzzing interface with streaming + * support + */ + +#ifndef FuzzingInterfaceStream_h__ +#define FuzzingInterfaceStream_h__ + +#ifdef JS_STANDALONE +# error "FuzzingInterfaceStream.h cannot be used in JS standalone builds." +#endif + +#include "gtest/gtest.h" +#include "nsComponentManagerUtils.h" +#include "nsCOMPtr.h" +#include "nsIInputStream.h" + +#include "nsDirectoryServiceDefs.h" +#include "nsStreamUtils.h" +#include "nsStringStream.h" + +#include + +#include "FuzzingInterface.h" + +namespace mozilla { + +typedef int (*FuzzingTestFuncStream)(nsCOMPtr); + +#ifdef AFLFUZZ +void afl_interface_stream(const char* testFile, FuzzingTestFuncStream testFunc); + +# define MOZ_AFL_INTERFACE_COMMON(initFunc) \ + if (initFunc) initFunc(NULL, NULL); \ + char* testFilePtr = getenv("MOZ_FUZZ_TESTFILE"); \ + if (!testFilePtr) { \ + fprintf(stderr, \ + "Must specify testfile in MOZ_FUZZ_TESTFILE environment " \ + "variable.\n"); \ + return; \ + } \ + /* Make a copy of testFilePtr so the testing function can safely call \ + * getenv \ + */ \ + std::string testFile(testFilePtr); + +# define MOZ_AFL_INTERFACE_STREAM(initFunc, testFunc, moduleName) \ + TEST(AFL, moduleName) \ + { \ + MOZ_AFL_INTERFACE_COMMON(initFunc); \ + ::mozilla::afl_interface_stream(testFile.c_str(), testFunc); \ + } +#else +# define MOZ_AFL_INTERFACE_STREAM(initFunc, testFunc, moduleName) /* Nothing \ + */ +#endif + +#ifdef LIBFUZZER +# define MOZ_LIBFUZZER_INTERFACE_STREAM(initFunc, testFunc, moduleName) \ + static int LibFuzzerTest##moduleName(const uint8_t* data, size_t size) { \ + if (size > INT32_MAX) return 0; \ + nsCOMPtr stream; \ + nsresult rv = NS_NewByteInputStream(getter_AddRefs(stream), \ + Span((const char*)data, size), \ + NS_ASSIGNMENT_DEPEND); \ + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); \ + testFunc(stream.forget()); \ + return 0; \ + } \ + static void __attribute__((constructor)) LibFuzzerRegister##moduleName() { \ + ::mozilla::FuzzerRegistry::getInstance().registerModule( \ + #moduleName, initFunc, LibFuzzerTest##moduleName); \ + } +#else +# define MOZ_LIBFUZZER_INTERFACE_STREAM(initFunc, testFunc, \ + moduleName) /* Nothing */ +#endif + +#define MOZ_FUZZING_INTERFACE_STREAM(initFunc, testFunc, moduleName) \ + MOZ_LIBFUZZER_INTERFACE_STREAM(initFunc, testFunc, moduleName); \ + MOZ_AFL_INTERFACE_STREAM(initFunc, testFunc, moduleName); + +} // namespace mozilla + +#endif // FuzzingInterfaceStream_h__ diff --git a/tools/fuzzing/interface/harness/FuzzerRunner.cpp b/tools/fuzzing/interface/harness/FuzzerRunner.cpp new file mode 100644 index 0000000000..fc2094aa59 --- /dev/null +++ b/tools/fuzzing/interface/harness/FuzzerRunner.cpp @@ -0,0 +1,91 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include + +#include "FuzzerRunner.h" +#include "mozilla/Attributes.h" +#include "prenv.h" + +#include "FuzzerTestHarness.h" + +namespace mozilla { + +// We use a static var 'fuzzerRunner' defined in nsAppRunner.cpp. +// fuzzerRunner is initialized to nullptr but if this file is linked in, +// then fuzzerRunner will be set here indicating that +// we want to call into either LibFuzzer's main or the AFL entrypoint. +class _InitFuzzer { + public: + _InitFuzzer() { fuzzerRunner = new FuzzerRunner(); } + void InitXPCOM() { mScopedXPCOM = new ScopedXPCOM("Fuzzer"); } + void DeinitXPCOM() { + if (mScopedXPCOM) delete mScopedXPCOM; + mScopedXPCOM = nullptr; + } + + private: + ScopedXPCOM* mScopedXPCOM; +} InitLibFuzzer; + +static void DeinitXPCOM() { InitLibFuzzer.DeinitXPCOM(); } + +int FuzzerRunner::Run(int* argc, char*** argv) { + /* + * libFuzzer uses exit() calls in several places instead of returning, + * so the destructor of ScopedXPCOM is not called in some cases. + * For fuzzing, this does not make a difference, but in debug builds + * when running a single testcase, this causes an assertion when destroying + * global linked lists. For this reason, we allocate ScopedXPCOM on the heap + * using the global InitLibFuzzer class, combined with an atexit call to + * destroy the ScopedXPCOM instance again. + */ + InitLibFuzzer.InitXPCOM(); + std::atexit(DeinitXPCOM); + + const char* fuzzerEnv = getenv("FUZZER"); + + if (!fuzzerEnv) { + fprintf(stderr, + "Must specify fuzzing target in FUZZER environment variable\n"); + exit(1); + } + + std::string moduleNameStr(fuzzerEnv); + FuzzerFunctions funcs = + FuzzerRegistry::getInstance().getModuleFunctions(moduleNameStr); + FuzzerInitFunc initFunc = funcs.first; + FuzzerTestingFunc testingFunc = funcs.second; + if (initFunc) { + int ret = initFunc(argc, argv); + if (ret) { + fprintf(stderr, "Fuzzing Interface: Error: Initialize callback failed\n"); + exit(1); + } + } + + if (!testingFunc) { + fprintf(stderr, "Fuzzing Interface: Error: No testing callback found\n"); + exit(1); + } + +#ifdef LIBFUZZER + int ret = mFuzzerDriver(argc, argv, testingFunc); +#else + // For AFL, testingFunc points to the entry function we need. + int ret = testingFunc(NULL, 0); +#endif + + InitLibFuzzer.DeinitXPCOM(); + return ret; +} + +#ifdef LIBFUZZER +void FuzzerRunner::setParams(LibFuzzerDriver aDriver) { + mFuzzerDriver = aDriver; +} +#endif + +} // namespace mozilla diff --git a/tools/fuzzing/interface/harness/FuzzerRunner.h b/tools/fuzzing/interface/harness/FuzzerRunner.h new file mode 100644 index 0000000000..6b19e751cd --- /dev/null +++ b/tools/fuzzing/interface/harness/FuzzerRunner.h @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzerRegistry.h" + +namespace mozilla { + +class FuzzerRunner { + public: + int Run(int* argc, char*** argv); + +#ifdef LIBFUZZER + void setParams(LibFuzzerDriver aDriver); + + private: + LibFuzzerDriver mFuzzerDriver; +#endif +}; + +extern FuzzerRunner* fuzzerRunner; + +} // namespace mozilla diff --git a/tools/fuzzing/interface/harness/FuzzerTestHarness.h b/tools/fuzzing/interface/harness/FuzzerTestHarness.h new file mode 100644 index 0000000000..d7bb1064cf --- /dev/null +++ b/tools/fuzzing/interface/harness/FuzzerTestHarness.h @@ -0,0 +1,256 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Test harness for XPCOM objects, providing a scoped XPCOM initializer, + * nsCOMPtr, nsRefPtr, do_CreateInstance, do_GetService, ns(Auto|C|)String, + * and stdio.h/stdlib.h. + */ + +#ifndef FuzzerTestHarness_h__ +#define FuzzerTestHarness_h__ + +#include "mozilla/ArrayUtils.h" +#include "mozilla/Attributes.h" + +#include "prenv.h" +#include "nsComponentManagerUtils.h" +#include "nsServiceManagerUtils.h" +#include "nsCOMPtr.h" +#include "nsString.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsDirectoryServiceDefs.h" +#include "nsDirectoryServiceUtils.h" +#include "nsIDirectoryService.h" +#include "nsIFile.h" +#include "nsIObserverService.h" +#include "nsIServiceManager.h" +#include "nsXULAppAPI.h" +#include "mozilla/AppShutdown.h" +#include +#include +#include + +namespace { + +static uint32_t gFailCount = 0; + +/** + * Prints the given failure message and arguments using printf, prepending + * "TEST-UNEXPECTED-FAIL " for the benefit of the test harness and + * appending "\n" to eliminate having to type it at each call site. + */ +MOZ_FORMAT_PRINTF(1, 2) void fail(const char* msg, ...) { + va_list ap; + + printf("TEST-UNEXPECTED-FAIL | "); + + va_start(ap, msg); + vprintf(msg, ap); + va_end(ap); + + putchar('\n'); + ++gFailCount; +} + +//----------------------------------------------------------------------------- + +class ScopedXPCOM final : public nsIDirectoryServiceProvider2 { + public: + NS_DECL_ISUPPORTS + + explicit ScopedXPCOM(const char* testName, + nsIDirectoryServiceProvider* dirSvcProvider = nullptr) + : mDirSvcProvider(dirSvcProvider) { + mTestName = testName; + printf("Running %s tests...\n", mTestName); + + nsresult rv = NS_InitXPCOM(&mServMgr, nullptr, this); + if (NS_FAILED(rv)) { + fail("NS_InitXPCOM returned failure code 0x%" PRIx32, + static_cast(rv)); + mServMgr = nullptr; + return; + } + } + + ~ScopedXPCOM() { + // If we created a profile directory, we need to remove it. + if (mProfD) { + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdownNetTeardown); + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdownTeardown); + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdown); + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdownQM); + mozilla::AppShutdown::AdvanceShutdownPhase( + mozilla::ShutdownPhase::AppShutdownTelemetry); + + if (NS_FAILED(mProfD->Remove(true))) { + NS_WARNING("Problem removing profile directory"); + } + + mProfD = nullptr; + } + + if (mServMgr) { + NS_RELEASE(mServMgr); + nsresult rv = NS_ShutdownXPCOM(nullptr); + if (NS_FAILED(rv)) { + fail("XPCOM shutdown failed with code 0x%" PRIx32, + static_cast(rv)); + exit(1); + } + } + + printf("Finished running %s tests.\n", mTestName); + } + + already_AddRefed GetProfileDirectory() { + if (mProfD) { + nsCOMPtr copy = mProfD; + return copy.forget(); + } + + // Create a unique temporary folder to use for this test. + // Note that runcppunittests.py will run tests with a temp + // directory as the cwd, so just put something under that. + nsCOMPtr profD; + nsresult rv = NS_GetSpecialDirectory(NS_OS_CURRENT_PROCESS_DIR, + getter_AddRefs(profD)); + NS_ENSURE_SUCCESS(rv, nullptr); + + rv = profD->Append(u"cpp-unit-profd"_ns); + NS_ENSURE_SUCCESS(rv, nullptr); + + rv = profD->CreateUnique(nsIFile::DIRECTORY_TYPE, 0755); + NS_ENSURE_SUCCESS(rv, nullptr); + + mProfD = profD; + return profD.forget(); + } + + already_AddRefed GetGREDirectory() { + if (mGRED) { + nsCOMPtr copy = mGRED; + return copy.forget(); + } + + char* env = PR_GetEnv("MOZ_XRE_DIR"); + nsCOMPtr greD; + if (env) { + NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, getter_AddRefs(greD)); + } + + mGRED = greD; + return greD.forget(); + } + + already_AddRefed GetGREBinDirectory() { + if (mGREBinD) { + nsCOMPtr copy = mGREBinD; + return copy.forget(); + } + + nsCOMPtr greD = GetGREDirectory(); + if (!greD) { + return greD.forget(); + } + greD->Clone(getter_AddRefs(mGREBinD)); + +#ifdef XP_MACOSX + nsAutoCString leafName; + mGREBinD->GetNativeLeafName(leafName); + if (leafName.EqualsLiteral("Resources")) { + mGREBinD->SetNativeLeafName("MacOS"_ns); + } +#endif + + nsCOMPtr copy = mGREBinD; + return copy.forget(); + } + + //////////////////////////////////////////////////////////////////////////// + //// nsIDirectoryServiceProvider + + NS_IMETHODIMP GetFile(const char* aProperty, bool* _persistent, + nsIFile** _result) override { + // If we were supplied a directory service provider, ask it first. + if (mDirSvcProvider && NS_SUCCEEDED(mDirSvcProvider->GetFile( + aProperty, _persistent, _result))) { + return NS_OK; + } + + // Otherwise, the test harness provides some directories automatically. + if (0 == strcmp(aProperty, NS_APP_USER_PROFILE_50_DIR) || + 0 == strcmp(aProperty, NS_APP_USER_PROFILE_LOCAL_50_DIR) || + 0 == strcmp(aProperty, NS_APP_PROFILE_LOCAL_DIR_STARTUP)) { + nsCOMPtr profD = GetProfileDirectory(); + NS_ENSURE_TRUE(profD, NS_ERROR_FAILURE); + + nsCOMPtr clone; + nsresult rv = profD->Clone(getter_AddRefs(clone)); + NS_ENSURE_SUCCESS(rv, rv); + + *_persistent = true; + clone.forget(_result); + return NS_OK; + } else if (0 == strcmp(aProperty, NS_GRE_DIR)) { + nsCOMPtr greD = GetGREDirectory(); + NS_ENSURE_TRUE(greD, NS_ERROR_FAILURE); + + *_persistent = true; + greD.forget(_result); + return NS_OK; + } else if (0 == strcmp(aProperty, NS_GRE_BIN_DIR)) { + nsCOMPtr greBinD = GetGREBinDirectory(); + NS_ENSURE_TRUE(greBinD, NS_ERROR_FAILURE); + + *_persistent = true; + greBinD.forget(_result); + return NS_OK; + } + + return NS_ERROR_FAILURE; + } + + //////////////////////////////////////////////////////////////////////////// + //// nsIDirectoryServiceProvider2 + + NS_IMETHODIMP GetFiles(const char* aProperty, + nsISimpleEnumerator** _enum) override { + // If we were supplied a directory service provider, ask it first. + nsCOMPtr provider = + do_QueryInterface(mDirSvcProvider); + if (provider && NS_SUCCEEDED(provider->GetFiles(aProperty, _enum))) { + return NS_OK; + } + + return NS_ERROR_FAILURE; + } + + private: + const char* mTestName; + nsIServiceManager* mServMgr; + nsCOMPtr mDirSvcProvider; + nsCOMPtr mProfD; + nsCOMPtr mGRED; + nsCOMPtr mGREBinD; +}; + +NS_IMPL_QUERY_INTERFACE(ScopedXPCOM, nsIDirectoryServiceProvider, + nsIDirectoryServiceProvider2) + +NS_IMETHODIMP_(MozExternalRefCountType) +ScopedXPCOM::AddRef() { return 2; } + +NS_IMETHODIMP_(MozExternalRefCountType) +ScopedXPCOM::Release() { return 1; } + +} // namespace + +#endif // FuzzerTestHarness_h__ diff --git a/tools/fuzzing/interface/harness/moz.build b/tools/fuzzing/interface/harness/moz.build new file mode 100644 index 0000000000..0eff84c8aa --- /dev/null +++ b/tools/fuzzing/interface/harness/moz.build @@ -0,0 +1,16 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-runner") + +SOURCES += [ + "FuzzerRunner.cpp", +] +EXPORTS += [ + "FuzzerRunner.h", +] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/interface/moz.build b/tools/fuzzing/interface/moz.build new file mode 100644 index 0000000000..8a51007174 --- /dev/null +++ b/tools/fuzzing/interface/moz.build @@ -0,0 +1,32 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-interface") + +EXPORTS += [ + "FuzzingInterface.h", +] + +SOURCES += [ + "FuzzingInterface.cpp", +] + +if CONFIG["JS_STANDALONE"]: + FINAL_LIBRARY = "js" +else: + EXPORTS += [ + "FuzzingInterfaceStream.h", + ] + + SOURCES += [ + "FuzzingInterfaceStream.cpp", + ] + + DIRS += [ + "harness", + ] + + FINAL_LIBRARY = "xul-gtest" diff --git a/tools/fuzzing/ipc/IPCFuzzController.cpp b/tools/fuzzing/ipc/IPCFuzzController.cpp new file mode 100644 index 0000000000..649b7f780f --- /dev/null +++ b/tools/fuzzing/ipc/IPCFuzzController.cpp @@ -0,0 +1,848 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "IPCFuzzController.h" +#include "mozilla/Fuzzing.h" +#include "mozilla/SpinEventLoopUntil.h" +#include "mozilla/SyncRunnable.h" + +#include "nsIThread.h" +#include "nsThreadUtils.h" + +#include "mozilla/ipc/MessageChannel.h" +#include "mozilla/ipc/MessageLink.h" +#include "mozilla/ipc/ProtocolUtils.h" +#include "mozilla/ipc/NodeChannel.h" +#include "mozilla/ipc/NodeController.h" + +#include "mozilla/ipc/PIdleScheduler.h" +#include "mozilla/ipc/PBackground.h" +#include "mozilla/dom/PContent.h" + +using namespace mojo::core::ports; +using namespace mozilla::ipc; + +// Sync inject means that the actual fuzzing takes place on the I/O thread +// and hence it injects directly into the target NodeChannel. In async mode, +// we run the fuzzing on a separate thread and dispatch the runnable that +// injects the message back to the I/O thread. Both approaches seem to work +// and have advantages and disadvantages. Blocking the I/O thread means no +// IPC between other processes will interfere with our fuzzing in the meantime +// but blocking could also cause hangs when such IPC is required during the +// fuzzing runtime for some reason. +// #define MOZ_FUZZ_IPC_SYNC_INJECT 1 + +// For debugging purposes, it can be helpful to synchronize after each message +// rather than after each iteration, to see which messages are particularly +// slow or cause a hang. Without this, synchronization will occur at the end +// of each iteration as well as after each constructor message. +// #define MOZ_FUZZ_IPC_SYNC_AFTER_EACH_MSG + +namespace mozilla { +namespace fuzzing { + +IPCFuzzController::IPCFuzzController() : mMutex("IPCFuzzController") { + InitializeIPCTypes(); + + // We use 6 bits for port index selection without wrapping, so we just + // create 64 empty rows in our port matrix. Not all of these rows will + // be used though. + portNames.resize(64); + + // This is our port / toplevel actor ordering. Add new toplevel actors + // here to support them in the fuzzer. Do *NOT* change the order of + // these, as it will invalidate our fuzzing corpus. + portNameToIndex["PContent"] = 0; + portNameToIndex["PBackground"] = 1; + portNameToIndex["PBackgroundStarter"] = 2; + portNameToIndex["PCompositorManager"] = 3; + portNameToIndex["PImageBridge"] = 4; + portNameToIndex["PProcessHangMonitor"] = 5; + portNameToIndex["PProfiler"] = 6; + portNameToIndex["PVRManager"] = 7; + portNameToIndex["PCanvasManager"] = 8; +} + +// static +IPCFuzzController& IPCFuzzController::instance() { + static IPCFuzzController ifc; + return ifc; +} + +void IPCFuzzController::InitializeIPCTypes() { + const char* cons = "Constructor"; + size_t cons_len = strlen(cons); + + for (uint32_t start = 0; start < LastMsgIndex; ++start) { + uint32_t i; + for (i = (start << 16) + 1; i < ((start + 1) << 16); ++i) { + const char* name = IPC::StringFromIPCMessageType(i); + + if (name[0] == '<') break; + + size_t len = strlen(name); + if (len > cons_len && !memcmp(cons, name + len - cons_len, cons_len)) { + constructorTypes.insert(i); + } + } + + validMsgTypes[(ProtocolId)start] = i - ((start << 16) + 1); + } +} + +bool IPCFuzzController::GetRandomIPCMessageType(ProtocolId pId, + uint16_t typeOffset, + uint32_t* type) { + auto pIdEntry = validMsgTypes.find(pId); + if (pIdEntry == validMsgTypes.end()) { + return false; + } + + *type = + ((uint32_t)pIdEntry->first << 16) + 1 + (typeOffset % pIdEntry->second); + return true; +} + +void IPCFuzzController::OnActorConnected(IProtocol* protocol) { + if (!XRE_IsParentProcess()) { + return; + } + +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF("INFO: [OnActorConnected] ActorID %d Protocol: %s\n", + protocol->Id(), protocol->GetProtocolName()); +#endif + + MessageChannel* channel = protocol->ToplevelProtocol()->GetIPCChannel(); + + Maybe portName = channel->GetPortName(); + if (portName) { + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::OnActorConnected() Mutex try\n"); + // Called on background threads and modifies `actorIds`. + MutexAutoLock lock(mMutex); + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::OnActorConnected() Mutex locked\n"); + actorIds[*portName].emplace_back(protocol->Id(), protocol->GetProtocolId()); + + // Fix the port we will be using for at least the next 5 messages + useLastPortName = true; + lastActorPortName = *portName; + + // Use this actor for the next 5 messages + useLastActor = 5; + } else { + MOZ_FUZZING_NYX_PRINT("WARNING: No port name on actor?!\n"); + } +} + +void IPCFuzzController::OnActorDestroyed(IProtocol* protocol) { + if (!XRE_IsParentProcess()) { + return; + } + + MOZ_FUZZING_NYX_PRINTF("INFO: [OnActorDestroyed] ActorID %d Protocol: %s\n", + protocol->Id(), protocol->GetProtocolName()); + + MessageChannel* channel = protocol->ToplevelProtocol()->GetIPCChannel(); + + Maybe portName = channel->GetPortName(); + if (portName) { + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::OnActorDestroyed() Mutex try\n"); + // Called on background threads and modifies `actorIds`. + MutexAutoLock lock(mMutex); + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::OnActorDestroyed() Mutex locked\n"); + + for (auto iter = actorIds[*portName].begin(); + iter != actorIds[*portName].end();) { + if (iter->first == protocol->Id() && + iter->second == protocol->GetProtocolId()) { + iter = actorIds[*portName].erase(iter); + } else { + ++iter; + } + } + } else { + MOZ_FUZZING_NYX_PRINT("WARNING: No port name on destroyed actor?!\n"); + } +} + +void IPCFuzzController::AddToplevelActor(PortName name, ProtocolId protocolId) { + const char* protocolName = ProtocolIdToName(protocolId); + auto result = portNameToIndex.find(protocolName); + if (result == portNameToIndex.end()) { + MOZ_FUZZING_NYX_PRINTF( + "ERROR: [OnActorConnected] Unknown Top-Level Protocol: %s\n", + protocolName); + MOZ_FUZZING_NYX_ABORT("Unknown Top-Level Protocol\n"); + } + uint8_t portIndex = result->second; + portNames[portIndex].push_back(name); +} + +bool IPCFuzzController::ObserveIPCMessage(mozilla::ipc::NodeChannel* channel, + IPC::Message& aMessage) { + if (!mozilla::fuzzing::Nyx::instance().is_enabled("IPC_Generic")) { + // Fuzzer is not enabled. + return true; + } + + if (!XRE_IsParentProcess()) { + // For now we only care about things in the parent process. + return true; + } + + if (aMessage.IsFuzzMsg()) { + // Don't observe our own messages. If this is the first fuzzing message, + // we also block further non-fuzzing communication on that node. + if (!channel->mBlockSendRecv) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [NodeChannel::OnMessageReceived] Blocking further " + "communication on Port %lu %lu (seen fuzz msg)\n", + channel->GetName().v1, channel->GetName().v2); + channel->mBlockSendRecv = true; + } + return true; + } else if (aMessage.type() == dom::PContent::Msg_SignalFuzzingReady__ID) { + MOZ_FUZZING_NYX_PRINT("DEBUG: Ready message detected.\n"); + + // TODO: This is specific to PContent fuzzing. If we later want to fuzz + // a different process pair, we need additional signals here. + OnChildReady(); + + // The ready message indicates the right node name for us to work with + // and we should only ever receive it once. + if (haveTargetNodeName) { + MOZ_FUZZING_NYX_PRINT("ERROR: Received ready signal twice?!\n"); + MOZ_REALLY_CRASH(__LINE__); + } + + targetNodeName = channel->GetName(); + haveTargetNodeName = true; + + // We can also use this message as the base template for other messages + if (!this->sampleHeader.initLengthUninitialized( + sizeof(IPC::Message::Header))) { + MOZ_FUZZING_NYX_ABORT("sampleHeader.initLengthUninitialized failed\n"); + } + + memcpy(sampleHeader.begin(), aMessage.header(), + sizeof(IPC::Message::Header)); + } else if (haveTargetNodeName && targetNodeName != channel->GetName()) { + // Not our node, no need to observe + return true; + } else if (Nyx::instance().started()) { + // When fuzzing is already started, we shouldn't observe messages anymore. + if (!channel->mBlockSendRecv) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [NodeChannel::OnMessageReceived] Blocking further " + "communication on Port %lu %lu (fuzzing started)\n", + channel->GetName().v1, channel->GetName().v2); + channel->mBlockSendRecv = true; + } + return false; + } + + Vector footer; + + if (!footer.initLengthUninitialized(aMessage.event_footer_size())) { + MOZ_FUZZING_NYX_ABORT("footer.initLengthUninitialized failed\n"); + } + + if (!aMessage.ReadFooter(footer.begin(), footer.length(), false)) { + MOZ_FUZZING_NYX_ABORT("ERROR: ReadFooter() failed?!\n"); + } + + UniquePtr event = Event::Deserialize(footer.begin(), footer.length()); + + if (!event) { + MOZ_FUZZING_NYX_ABORT("ERROR: Failed to deserialize observed message?!\n"); + } + + if (event->type() == Event::kUserMessage) { + if (haveTargetNodeName && !fuzzingStartPending) { + bool missingActor = false; + + // Check if we have any entries in our port map that we haven't seen yet + // though `OnActorConnected`. That method is called on a background + // thread and this call will race with the I/O thread. + { + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::ObserveIPCMessage() Mutex try\n"); + // Called on the I/O thread and reads `portSeqNos`. + // + // IMPORTANT: We must give up any locks before entering `StartFuzzing`, + // as we will never return. This would cause a deadlock with new actors + // being created and `OnActorConnected` being called. + MutexAutoLock lock(mMutex); + + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: IPCFuzzController::ObserveIPCMessage() Mutex locked\n"); + + for (auto iter = portSeqNos.begin(); iter != portSeqNos.end(); ++iter) { + auto result = actorIds.find(iter->first); + if (result == actorIds.end()) { + // Make sure we only wait for actors that belong to us. + auto result = portNodeName.find(iter->first); + if (result->second == targetNodeName) { + missingActor = true; + break; + } + } + } + } + + if (missingActor) { + MOZ_FUZZING_NYX_PRINT( + "INFO: Delaying fuzzing start, missing actors...\n"); + } else if (!childReady) { + MOZ_FUZZING_NYX_PRINT( + "INFO: Delaying fuzzing start, waiting for child...\n"); + } else { + fuzzingStartPending = true; + StartFuzzing(channel, aMessage); + + // In the async case, we return and can already block the relevant + // communication. + if (targetNodeName == channel->GetName()) { + if (!channel->mBlockSendRecv) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: [NodeChannel::OnMessageReceived] Blocking further " + "communication on Port %lu %lu (fuzzing start pending)\n", + channel->GetName().v1, channel->GetName().v2); + channel->mBlockSendRecv = true; + } + + return false; + } + return true; + } + } + + // Add/update sequence numbers. We need to make sure to do this after our + // call to `StartFuzzing` because once we start fuzzing, the message will + // never actually be processed, so we run into a sequence number desync. + { + // Get the port name associated with this message + UserMessageEvent* userMsgEv = static_cast(event.get()); + PortName name = event->port_name(); + + // Called on the I/O thread and modifies `portSeqNos`. + MutexAutoLock lock(mMutex); + portSeqNos.insert_or_assign( + name, std::pair(aMessage.seqno(), + userMsgEv->sequence_num())); + + portNodeName.insert_or_assign(name, channel->GetName()); + } + } + + return true; +} + +bool IPCFuzzController::MakeTargetDecision( + uint8_t portIndex, uint8_t portInstanceIndex, uint8_t actorIndex, + uint16_t typeOffset, PortName* name, int32_t* seqno, uint64_t* fseqno, + int32_t* actorId, uint32_t* type, bool* is_cons, bool update) { + // Every possible toplevel actor type has a fixed number that + // we assign to it in the constructor of this class. Here, we + // use the lower 6 bits to select this toplevel actor type. + // This approach has the advantage that the tests will always + // select the same toplevel actor type deterministically, + // independent of the order they appeared and independent + // of the type of fuzzing we are doing. + auto portInstances = portNames[portIndex & 0x3f]; + if (!portInstances.size()) { + return false; + } + + if (useLastActor) { + useLastActor--; + *name = lastActorPortName; + + MOZ_FUZZING_NYX_PRINT("DEBUG: MakeTargetDecision: Pinned to last actor.\n"); + + // Once we stop pinning to the last actor, we need to decide if we + // want to keep the pinning on the port itself. We use one of the + // unused upper bits of portIndex for this purpose. + if (!useLastActor && (portIndex & (1 << 7))) { + MOZ_FUZZING_NYX_PRINT( + "DEBUG: MakeTargetDecision: Released pinning on last port.\n"); + useLastPortName = false; + } + } else if (useLastPortName) { + *name = lastActorPortName; + MOZ_FUZZING_NYX_PRINT("DEBUG: MakeTargetDecision: Pinned to last port.\n"); + } else { + *name = portInstances[portInstanceIndex % portInstances.size()]; + } + + // We should always have at least one actor per port + auto result = actorIds.find(*name); + if (result == actorIds.end()) { + MOZ_FUZZING_NYX_PRINT("ERROR: Couldn't find port in actors map?!\n"); + return false; + } + + // Find a random actor on this port + auto actors = result->second; + if (actors.empty()) { + MOZ_FUZZING_NYX_PRINT( + "ERROR: Couldn't find an actor for selected port?!\n"); + return false; + } + + auto seqNos = portSeqNos[*name]; + + // Hand out the correct sequence numbers + *seqno = seqNos.first - 1; + *fseqno = seqNos.second + 1; + + if (update) { + portSeqNos.insert_or_assign(*name, + std::pair(*seqno, *fseqno)); + } + + if (useLastActor) { + actorIndex = actors.size() - 1; + } else { + actorIndex %= actors.size(); + } + + ActorIdPair ids = actors[actorIndex]; + *actorId = ids.first; + + // If the actor ID is 0, then we are talking to the toplevel actor + // of this port. Hence we must set the ID to MSG_ROUTING_CONTROL. + if (!*actorId) { + *actorId = MSG_ROUTING_CONTROL; + } + + if (!this->GetRandomIPCMessageType(ids.second, typeOffset, type)) { + MOZ_FUZZING_NYX_PRINT("ERROR: GetRandomIPCMessageType failed?!\n"); + return false; + } + + *is_cons = false; + if (constructorTypes.find(*type) != constructorTypes.end()) { + *is_cons = true; + } + +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF( + "DEBUG: MakeTargetDecision: Protocol: %s msgType: %s\n", + ProtocolIdToName(ids.second), IPC::StringFromIPCMessageType(*type)); +#endif + + return true; +} + +void IPCFuzzController::OnMessageTaskStart() { messageStartCount++; } + +void IPCFuzzController::OnMessageTaskStop() { messageStopCount++; } + +void IPCFuzzController::OnPreFuzzMessageTaskRun() { messageTaskCount++; } +void IPCFuzzController::OnPreFuzzMessageTaskStop() { messageTaskCount--; } + +void IPCFuzzController::OnDropPeer(const char* reason = nullptr, + const char* file = nullptr, int line = 0) { + if (!XRE_IsParentProcess()) { + return; + } + + if (!Nyx::instance().started()) { + // It's possible to close a connection to some peer before we have even + // started fuzzing. We ignore these events until we are actually fuzzing. + return; + } + + MOZ_FUZZING_NYX_PRINT( + "ERROR: ======== END OF ITERATION (DROP_PEER) ========\n"); +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF("DEBUG: ======== %s:%d ========\n", file, line); +#endif + Nyx::instance().handle_event("MOZ_IPC_DROP_PEER", file, line, reason); + + if (Nyx::instance().is_replay()) { + // In replay mode, let's ignore drop peer to avoid races with it. + return; + } + + Nyx::instance().release(IPCFuzzController::instance().getMessageStopCount()); +} + +void IPCFuzzController::StartFuzzing(mozilla::ipc::NodeChannel* channel, + IPC::Message& aMessage) { + nodeChannel = channel; + + RefPtr runnable = new IPCFuzzLoop(); + +#if MOZ_FUZZ_IPC_SYNC_INJECT + runnable->Run(); +#else + nsCOMPtr newThread; + nsresult rv = + NS_NewNamedThread("IPCFuzzLoop", getter_AddRefs(newThread), runnable); + + if (NS_FAILED(rv)) { + MOZ_FUZZING_NYX_ABORT("ERROR: [StartFuzzing] NS_NewNamedThread failed?!\n"); + } +#endif +} + +IPCFuzzController::IPCFuzzLoop::IPCFuzzLoop() + : mozilla::Runnable("IPCFuzzLoop") {} + +NS_IMETHODIMP IPCFuzzController::IPCFuzzLoop::Run() { + MOZ_FUZZING_NYX_DEBUG("DEBUG: BEGIN IPCFuzzLoop::Run()\n"); + + const size_t maxMsgSize = 2048; + const size_t controlLen = 16; + + Vector buffer; + + RefPtr controller = NodeController::GetSingleton(); + + // TODO: The following code is full of data races. We need synchronization + // on the `IPCFuzzController` instance, because the I/O thread can call into + // this class via ObserveIPCMessages. The problem is that any such call + // must either be observed to update the sequence numbers, or the packet + // must be dropped already. + if (!IPCFuzzController::instance().haveTargetNodeName) { + MOZ_FUZZING_NYX_ABORT("ERROR: I don't have the target NodeName?!\n"); + } + + { + MOZ_FUZZING_NYX_DEBUG("DEBUG: IPCFuzzLoop::Run() Mutex try\n"); + // Called on the I/O thread and modifies `portSeqNos` and `actorIds`. + MutexAutoLock lock(IPCFuzzController::instance().mMutex); + MOZ_FUZZING_NYX_DEBUG("DEBUG: IPCFuzzLoop::Run() Mutex locked\n"); + + // The wait/delay logic in ObserveIPCMessage should ensure that we haven't + // seen any packets on ports for which we haven't received actor information + // yet, if those ports belong to our channel. However, we might also have + // seen ports not belonging to our channel, which we have to remove now. + for (auto iter = IPCFuzzController::instance().portSeqNos.begin(); + iter != IPCFuzzController::instance().portSeqNos.end();) { + auto result = IPCFuzzController::instance().actorIds.find(iter->first); + if (result == IPCFuzzController::instance().actorIds.end()) { + auto portNameResult = + IPCFuzzController::instance().portNodeName.find(iter->first); + if (portNameResult->second == + IPCFuzzController::instance().targetNodeName) { + MOZ_FUZZING_NYX_PRINT( + "ERROR: We should not have port map entries without a " + "corresponding " + "entry in our actors map\n"); + MOZ_REALLY_CRASH(__LINE__); + } else { + iter = IPCFuzzController::instance().portSeqNos.erase(iter); + } + } else { + ++iter; + } + } + + // TODO: Technically, at this point we only know that PContent (or whatever + // toplevel protocol we decided to synchronize on), is present. It might + // be possible that others aren't created yet and we are racing on this. + // + // Note: The delay logic mentioned above makes this less likely. Only actors + // which are created on-demand and which have not been referenced yet at all + // would be affected by such a race. + for (auto iter = IPCFuzzController::instance().actorIds.begin(); + iter != IPCFuzzController::instance().actorIds.end(); ++iter) { + bool isValidTarget = false; + Maybe status; + PortRef ref = controller->GetPort(iter->first); + if (ref.is_valid()) { + status = controller->GetStatus(ref); + if (status) { + isValidTarget = status->peer_node_name == + IPCFuzzController::instance().targetNodeName; + } + } + + auto result = IPCFuzzController::instance().portSeqNos.find(iter->first); + if (result == IPCFuzzController::instance().portSeqNos.end()) { + if (isValidTarget) { + MOZ_FUZZING_NYX_PRINTF( + "INFO: Using Port %lu %lu for protocol %s (*)\n", iter->first.v1, + iter->first.v2, ProtocolIdToName(iter->second[0].second)); + + // Normally the start sequence numbers would be -1 and 1, but our map + // does not record the next numbers, but the "last seen" state. So we + // have to adjust these so the next calculated sequence number pair + // matches the start sequence numbers. + IPCFuzzController::instance().portSeqNos.insert_or_assign( + iter->first, std::pair(0, 0)); + + IPCFuzzController::instance().AddToplevelActor( + iter->first, iter->second[0].second); + + } else { + MOZ_FUZZING_NYX_PRINTF( + "INFO: Removing Port %lu %lu for protocol %s (*)\n", + iter->first.v1, iter->first.v2, + ProtocolIdToName(iter->second[0].second)); + + // This toplevel actor does not belong to us, but we haven't added + // it to `portSeqNos`, so we don't have to remove it. + } + } else { + if (isValidTarget) { + MOZ_FUZZING_NYX_PRINTF("INFO: Using Port %lu %lu for protocol %s\n", + iter->first.v1, iter->first.v2, + ProtocolIdToName(iter->second[0].second)); + + IPCFuzzController::instance().AddToplevelActor( + iter->first, iter->second[0].second); + } else { + MOZ_FUZZING_NYX_PRINTF( + "INFO: Removing Port %lu %lu for protocol %s\n", iter->first.v1, + iter->first.v2, ProtocolIdToName(iter->second[0].second)); + + // This toplevel actor does not belong to us, so remove it. + IPCFuzzController::instance().portSeqNos.erase(result); + } + } + } + } + + IPCFuzzController::instance().runnableDone = false; + + SyncRunnable::DispatchToThread( + GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction("IPCFuzzController::StartFuzzing", [&]() -> void { + MOZ_FUZZING_NYX_PRINT("INFO: Main thread runnable start.\n"); + NS_ProcessPendingEvents(NS_GetCurrentThread()); + MOZ_FUZZING_NYX_PRINT("INFO: Main thread runnable done.\n"); + })); + + MOZ_FUZZING_NYX_PRINT("INFO: Performing snapshot...\n"); + Nyx::instance().start(); + + uint32_t expected_messages = 0; + + IPCFuzzController::instance().useLastActor = 0; + IPCFuzzController::instance().useLastPortName = false; + + if (!buffer.initLengthUninitialized(maxMsgSize)) { + MOZ_FUZZING_NYX_ABORT("ERROR: Failed to initialize buffer!\n"); + } + + for (int i = 0; i < 16; ++i) { + // Grab enough data to potentially fill our everything except the footer. + uint32_t bufsize = + Nyx::instance().get_data((uint8_t*)buffer.begin(), buffer.length()); + + if (bufsize == 0xFFFFFFFF) { + // Done constructing + MOZ_FUZZING_NYX_DEBUG("Iteration complete: Out of data.\n"); + break; + } + + // Payload must be int aligned + bufsize -= bufsize % 4; + + // Need at least a header and the control bytes. + if (bufsize < sizeof(IPC::Message::Header) + controlLen) { + MOZ_FUZZING_NYX_DEBUG("INFO: Not enough data to craft IPC message.\n"); + continue; + } + + buffer.shrinkTo(bufsize); + + const uint8_t* controlData = (uint8_t*)buffer.begin(); + + char* ipcMsgData = buffer.begin() + controlLen; + size_t ipcMsgLen = buffer.length() - controlLen; + + // Copy the header of the original message + memcpy(ipcMsgData, IPCFuzzController::instance().sampleHeader.begin(), + sizeof(IPC::Message::Header)); + + IPC::Message::Header* ipchdr = (IPC::Message::Header*)ipcMsgData; + + ipchdr->payload_size = ipcMsgLen - sizeof(IPC::Message::Header); + + PortName new_port_name; + int32_t new_seqno; + uint64_t new_fseqno; + + int32_t actorId; + uint32_t msgType; + bool isConstructor; + // Control Data Layout (16 byte) + // Byte 0 - Port Index (selects out of the valid ports seen) + // Byte 1 - Actor Index (selects one of the actors for that port) + // Byte 2 - Type Offset (select valid type for the specified actor) + // Byte 3 - ^- continued + // Byte 4 - Sync Bit + // Byte 5 - Optionally select a particular instance of the selected + // port type. Some toplevel protocols can have multiple + // instances running at the same time. + + uint8_t portIndex = controlData[0]; + uint8_t actorIndex = controlData[1]; + uint16_t typeOffset = *(uint16_t*)(&controlData[2]); + bool isSync = controlData[4] > 127; + uint8_t portInstanceIndex = controlData[5]; + + if (!IPCFuzzController::instance().MakeTargetDecision( + portIndex, portInstanceIndex, actorIndex, typeOffset, + &new_port_name, &new_seqno, &new_fseqno, &actorId, &msgType, + &isConstructor)) { + MOZ_FUZZING_NYX_DEBUG("DEBUG: MakeTargetDecision returned false.\n"); + continue; + } + + if (Nyx::instance().is_replay()) { + MOZ_FUZZING_NYX_PRINT("INFO: Replaying IPC packet with payload:\n"); + for (uint32_t i = 0; i < ipcMsgLen - sizeof(IPC::Message::Header); ++i) { + if (i % 16 == 0) { + MOZ_FUZZING_NYX_PRINT("\n "); + } + + MOZ_FUZZING_NYX_PRINTF( + "0x%02X ", + (unsigned char)(ipcMsgData[sizeof(IPC::Message::Header) + i])); + } + MOZ_FUZZING_NYX_PRINT("\n"); + } + + UniquePtr msg(new IPC::Message(ipcMsgData, ipcMsgLen)); + + if (isConstructor) { + MOZ_FUZZING_NYX_DEBUG("DEBUG: Sending constructor message...\n"); + msg->header()->flags.SetConstructor(); + } + + if (!isConstructor && isSync) { + MOZ_FUZZING_NYX_DEBUG("INFO: Sending sync message...\n"); + msg->header()->flags.SetSync(); + } + + msg->set_seqno(new_seqno); + msg->set_routing_id(actorId); + + // TODO: There is no setter for this. + msg->header()->type = msgType; + + // Create the footer + auto messageEvent = MakeUnique(0); + messageEvent->set_port_name(new_port_name); + messageEvent->set_sequence_num(new_fseqno); + + Vector footerBuffer; + (void)footerBuffer.initLengthUninitialized( + messageEvent->GetSerializedSize()); + messageEvent->Serialize(footerBuffer.begin()); + + msg->WriteFooter(footerBuffer.begin(), footerBuffer.length()); + msg->set_event_footer_size(footerBuffer.length()); + + // This marks the message as a fuzzing message. Without this, it will + // be ignored by MessageTask and also not even scheduled by NodeChannel + // in asynchronous mode. We use this to ignore any IPC activity that + // happens just while we are fuzzing. + msg->SetFuzzMsg(); + +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF( + "DEBUG: OnEventMessage iteration %d, EVS: %u Payload: %u.\n", i, + ipchdr->event_footer_size, ipchdr->payload_size); +#endif + +#ifdef FUZZ_DEBUG + MOZ_FUZZING_NYX_PRINTF("DEBUG: OnEventMessage: Port %lu %lu. Actor %d\n", + new_port_name.v1, new_port_name.v2, actorId); + MOZ_FUZZING_NYX_PRINTF( + "DEBUG: OnEventMessage: Flags: %u TxID: %d Handles: %u\n", + msg->header()->flags, msg->header()->txid, msg->header()->num_handles); +#endif + + // The number of messages we expect to see stopped. + expected_messages++; + +#if MOZ_FUZZ_IPC_SYNC_INJECT + // For synchronous injection, we just call OnMessageReceived directly. + IPCFuzzController::instance().nodeChannel->OnMessageReceived( + std::move(msg)); +#else + // For asynchronous injection, we have to post to the I/O thread instead. + XRE_GetIOMessageLoop()->PostTask(NS_NewRunnableFunction( + "NodeChannel::OnMessageReceived", + [msg = std::move(msg), + nodeChannel = + RefPtr{IPCFuzzController::instance().nodeChannel}]() mutable { + nodeChannel->OnMessageReceived(std::move(msg)); + })); +#endif + +#ifdef MOZ_FUZZ_IPC_SYNC_AFTER_EACH_MSG + MOZ_FUZZING_NYX_DEBUG("DEBUG: Synchronizing after message...\n"); + IPCFuzzController::instance().SynchronizeOnMessageExecution( + expected_messages); +#else + + if (isConstructor) { + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: Synchronizing due to constructor message...\n"); + IPCFuzzController::instance().SynchronizeOnMessageExecution( + expected_messages); + } +#endif + } + + MOZ_FUZZING_NYX_DEBUG("DEBUG: Synchronizing due to end of iteration...\n"); + IPCFuzzController::instance().SynchronizeOnMessageExecution( + expected_messages); + + MOZ_FUZZING_NYX_DEBUG( + "DEBUG: ======== END OF ITERATION (RELEASE) ========\n"); + + Nyx::instance().release(IPCFuzzController::instance().getMessageStopCount()); + + // Never reached. + return NS_OK; +} + +void IPCFuzzController::SynchronizeOnMessageExecution( + uint32_t expected_messages) { + // This synchronization will work in both the sync and async case. + // For the async case, it is important to wait for the exact stop count + // because the message task is not even started potentially when we + // read this loop. + int hang_timeout = 10 * 1000; + while (IPCFuzzController::instance().getMessageStopCount() != + expected_messages) { +#ifdef FUZZ_DEBUG + uint32_t count_stopped = + IPCFuzzController::instance().getMessageStopCount(); + uint32_t count_live = IPCFuzzController::instance().getMessageStartCount(); + MOZ_FUZZING_NYX_PRINTF( + "DEBUG: Post Constructor: %d stopped messages (%d live, %d " + "expected)!\n", + count_stopped, count_live, expected_messages); +#endif + PR_Sleep(PR_MillisecondsToInterval(50)); + hang_timeout -= 50; + + if (hang_timeout <= 0) { + Nyx::instance().handle_event("MOZ_TIMEOUT", nullptr, 0, nullptr); + MOZ_FUZZING_NYX_PRINT( + "ERROR: ======== END OF ITERATION (TIMEOUT) ========\n"); + Nyx::instance().release( + IPCFuzzController::instance().getMessageStopCount()); + } + } +} + +} // namespace fuzzing +} // namespace mozilla diff --git a/tools/fuzzing/ipc/IPCFuzzController.h b/tools/fuzzing/ipc/IPCFuzzController.h new file mode 100644 index 0000000000..faf89d9f81 --- /dev/null +++ b/tools/fuzzing/ipc/IPCFuzzController.h @@ -0,0 +1,193 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_ipc_IPCFuzzController_h +#define mozilla_ipc_IPCFuzzController_h + +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/HashTable.h" +#include "mozilla/Mutex.h" +#include "mozilla/fuzzing/Nyx.h" + +#include "nsIRunnable.h" +#include "nsThreadUtils.h" + +#include "chrome/common/ipc_message.h" +#include "mojo/core/ports/name.h" +#include "mojo/core/ports/event.h" + +#include "IPCMessageStart.h" + +#include +#include +#include +#include +#include + +#define MOZ_FUZZING_IPC_DROP_PEER(aReason) \ + mozilla::fuzzing::IPCFuzzController::instance().OnDropPeer( \ + aReason, __FILE__, __LINE__); + +#define MOZ_FUZZING_IPC_MT_CTOR() \ + mozilla::fuzzing::IPCFuzzController::instance().OnMessageTaskStart(); + +#define MOZ_FUZZING_IPC_MT_STOP() \ + mozilla::fuzzing::IPCFuzzController::instance().OnMessageTaskStop(); + +#define MOZ_FUZZING_IPC_PRE_FUZZ_MT_RUN() \ + mozilla::fuzzing::IPCFuzzController::instance().OnPreFuzzMessageTaskRun(); + +#define MOZ_FUZZING_IPC_PRE_FUZZ_MT_STOP() \ + mozilla::fuzzing::IPCFuzzController::instance().OnPreFuzzMessageTaskStop(); + +namespace mozilla { + +namespace ipc { +// We can't include ProtocolUtils.h here +class IProtocol; +typedef IPCMessageStart ProtocolId; + +class NodeChannel; +} // namespace ipc + +namespace fuzzing { + +class IPCFuzzController { + typedef std::pair SeqNoPair; + + typedef std::pair ActorIdPair; + + class IPCFuzzLoop final : public Runnable { + friend class IPCFuzzController; + + public: + NS_DECL_NSIRUNNABLE + + IPCFuzzLoop(); + + private: + ~IPCFuzzLoop() = default; + }; + + public: + static IPCFuzzController& instance(); + + void InitializeIPCTypes(); + bool GetRandomIPCMessageType(mozilla::ipc::ProtocolId pId, + uint16_t typeOffset, uint32_t* type); + + bool ObserveIPCMessage(mozilla::ipc::NodeChannel* channel, + IPC::Message& aMessage); + bool MakeTargetDecision(uint8_t portIndex, uint8_t portInstanceIndex, + uint8_t actorIndex, uint16_t typeOffset, + mojo::core::ports::PortName* name, int32_t* seqno, + uint64_t* fseqno, int32_t* actorId, uint32_t* type, + bool* is_cons, bool update = true); + + void OnActorConnected(mozilla::ipc::IProtocol* protocol); + void OnActorDestroyed(mozilla::ipc::IProtocol* protocol); + void OnDropPeer(const char* reason, const char* file, int line); + void OnMessageTaskStart(); + void OnMessageTaskStop(); + void OnPreFuzzMessageTaskRun(); + void OnPreFuzzMessageTaskStop(); + void OnChildReady() { childReady = true; } + void OnRunnableDone() { runnableDone = true; } + + uint32_t getPreFuzzMessageTaskCount() { return messageTaskCount; }; + uint32_t getMessageStartCount() { return messageStartCount; }; + uint32_t getMessageStopCount() { return messageStopCount; }; + + void StartFuzzing(mozilla::ipc::NodeChannel* channel, IPC::Message& aMessage); + + void SynchronizeOnMessageExecution(uint32_t expected_messages); + void AddToplevelActor(mojo::core::ports::PortName name, + mozilla::ipc::ProtocolId protocolId); + + private: + // This is a mapping from port name to a pair of last seen sequence numbers. + std::unordered_map portSeqNos; + + // This is a mapping from port name to node name. + std::unordered_map + portNodeName; + + // This maps each ProtocolId (IPCMessageStart) to the number of valid + // messages for that particular type. + std::unordered_map validMsgTypes; + + // This is a mapping from port name to pairs of actor Id and ProtocolId. + std::unordered_map> + actorIds; + + // If set, `lastActorPortName` is valid and fuzzing is pinned to this port. + Atomic useLastPortName; + + // Last port where a new actor appeared. Only valid with `useLastPortName`. + mojo::core::ports::PortName lastActorPortName; + + // Counter to indicate how long fuzzing should stay pinned to the last + // actor that appeared on `lastActorPortName`. + Atomic useLastActor; + + // This is the deterministic ordering of toplevel actors for fuzzing. + // In this matrix, each row (toplevel index) corresponds to one toplevel + // actor *type* while each entry in that row is an instance of that type, + // since some actors usually have multiple instances alive while others + // don't. For the exact ordering, please check the constructor for this + // class. + std::vector> portNames; + std::unordered_map portNameToIndex; + + // This is a set of all types that are constructors. + std::unordered_set constructorTypes; + + // This is the name of the target node. We select one Node based on a + // particular toplevel actor and then use this to pull in additional + // toplevel actors that are on the same node (i.e. belong to the same + // process pair). + mojo::core::ports::NodeName targetNodeName; + bool haveTargetNodeName = false; + + // This indicates that we have started the fuzzing thread and fuzzing will + // begin shortly. + bool fuzzingStartPending = false; + + // This is used as a signal from other threads that runnables we dispatched + // are completed. Right now we use this only when dispatching to the main + // thread to await the completion of all pending events. + Atomic runnableDone; + + // This is used to signal that the other process we are talking to is ready + // to start fuzzing. In the case of Parent <-> Child, a special IPC message + // is used to signal this. We might not be able to start fuzzing immediately + // hough if not all toplevel actors have been created yet. + Atomic childReady; + + // Current amount of pending message tasks. + Atomic messageStartCount; + Atomic messageStopCount; + + Atomic messageTaskCount; + + Vector sampleHeader; + + mozilla::ipc::NodeChannel* nodeChannel = nullptr; + + // This class is used both on the I/O and background threads as well as + // its own fuzzing thread. Those methods that alter non-threadsafe data + // structures need to aquire this mutex first. + Mutex mMutex; // MOZ_UNANNOTATED; + + IPCFuzzController(); + NYX_DISALLOW_COPY_AND_ASSIGN(IPCFuzzController); +}; + +} // namespace fuzzing +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/ipc/moz.build b/tools/fuzzing/ipc/moz.build new file mode 100644 index 0000000000..001c67ed37 --- /dev/null +++ b/tools/fuzzing/ipc/moz.build @@ -0,0 +1,27 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-ipc-protocol") + +LOCAL_INCLUDES += [ + "/dom/base", + "/dom/ipc", +] + +SOURCES += [ + "IPCFuzzController.cpp", +] + +EXPORTS.mozilla.fuzzing += [ + "IPCFuzzController.h", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul" + +# Add libFuzzer configuration directives +include("/tools/fuzzing/libfuzzer-config.mozbuild") diff --git a/tools/fuzzing/libfuzzer-config.mozbuild b/tools/fuzzing/libfuzzer-config.mozbuild new file mode 100644 index 0000000000..91998e0050 --- /dev/null +++ b/tools/fuzzing/libfuzzer-config.mozbuild @@ -0,0 +1,13 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +include('libfuzzer-flags.mozbuild') + +if CONFIG['FUZZING']: + if CONFIG['LIBFUZZER']: + # Add trace-pc coverage for libfuzzer + CFLAGS += libfuzzer_flags + CXXFLAGS += libfuzzer_flags diff --git a/tools/fuzzing/libfuzzer-flags.mozbuild b/tools/fuzzing/libfuzzer-flags.mozbuild new file mode 100644 index 0000000000..97f9c6cf45 --- /dev/null +++ b/tools/fuzzing/libfuzzer-flags.mozbuild @@ -0,0 +1,11 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +libfuzzer_flags = [] + +if CONFIG['LIBFUZZER_FLAGS']: + libfuzzer_flags += CONFIG['LIBFUZZER_FLAGS'] + diff --git a/tools/fuzzing/libfuzzer/FuzzerBuiltins.h b/tools/fuzzing/libfuzzer/FuzzerBuiltins.h new file mode 100644 index 0000000000..4c0ada8266 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerBuiltins.h @@ -0,0 +1,35 @@ +//===- FuzzerBuiltins.h - Internal header for builtins ----------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos around builtin functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_H +#define LLVM_FUZZER_BUILTINS_H + +#include "FuzzerPlatform.h" + +#if !LIBFUZZER_MSVC +#include + +#define GET_CALLER_PC() __builtin_return_address(0) + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); } +inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } +inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } + +inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); } +inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); } +inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); } + +} // namespace fuzzer + +#endif // !LIBFUZZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_H diff --git a/tools/fuzzing/libfuzzer/FuzzerBuiltinsMsvc.h b/tools/fuzzing/libfuzzer/FuzzerBuiltinsMsvc.h new file mode 100644 index 0000000000..c5bec9787d --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerBuiltinsMsvc.h @@ -0,0 +1,72 @@ +//===- FuzzerBuiltinsMSVC.h - Internal header for builtins ------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Wrapper functions and marcos that use intrinsics instead of builtin functions +// which cannot be compiled by MSVC. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_BUILTINS_MSVC_H +#define LLVM_FUZZER_BUILTINS_MSVC_H + +#include "FuzzerPlatform.h" + +#if LIBFUZZER_MSVC +#include +#include +#include + +// __builtin_return_address() cannot be compiled with MSVC. Use the equivalent +// from +#define GET_CALLER_PC() _ReturnAddress() + +namespace fuzzer { + +inline uint8_t Bswap(uint8_t x) { return x; } +// Use alternatives to __builtin functions from and on +// Windows since the builtins are not supported by MSVC. +inline uint16_t Bswap(uint16_t x) { return _byteswap_ushort(x); } +inline uint32_t Bswap(uint32_t x) { return _byteswap_ulong(x); } +inline uint64_t Bswap(uint64_t x) { return _byteswap_uint64(x); } + +// The functions below were mostly copied from +// compiler-rt/lib/builtins/int_lib.h which defines the __builtin functions used +// outside of Windows. +inline uint32_t Clzll(uint64_t X) { + unsigned long LeadZeroIdx = 0; + +#if !defined(_M_ARM) && !defined(_M_X64) + // Scan the high 32 bits. + if (_BitScanReverse(&LeadZeroIdx, static_cast(X >> 32))) + return static_cast(63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB. + // Scan the low 32 bits. + if (_BitScanReverse(&LeadZeroIdx, static_cast(X))) + return static_cast(63 - LeadZeroIdx); + +#else + if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; +#endif + return 64; +} + +inline uint32_t Clz(uint32_t X) { + unsigned long LeadZeroIdx = 0; + if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx; + return 32; +} + +inline int Popcountll(unsigned long long X) { +#if !defined(_M_ARM) && !defined(_M_X64) + return __popcnt(X) + __popcnt(X >> 32); +#else + return __popcnt64(X); +#endif +} + +} // namespace fuzzer + +#endif // LIBFUZER_MSVC +#endif // LLVM_FUZZER_BUILTINS_MSVC_H diff --git a/tools/fuzzing/libfuzzer/FuzzerCommand.h b/tools/fuzzing/libfuzzer/FuzzerCommand.h new file mode 100644 index 0000000000..87308864af --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerCommand.h @@ -0,0 +1,178 @@ +//===- FuzzerCommand.h - Interface representing a process -------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// FuzzerCommand represents a command to run in a subprocess. It allows callers +// to manage command line arguments and output and error streams. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_COMMAND_H +#define LLVM_FUZZER_COMMAND_H + +#include "FuzzerDefs.h" +#include "FuzzerIO.h" + +#include +#include +#include +#include + +namespace fuzzer { + +class Command final { +public: + // This command line flag is used to indicate that the remaining command line + // is immutable, meaning this flag effectively marks the end of the mutable + // argument list. + static inline const char *ignoreRemainingArgs() { + return "-ignore_remaining_args=1"; + } + + Command() : CombinedOutAndErr(false) {} + + explicit Command(const Vector &ArgsToAdd) + : Args(ArgsToAdd), CombinedOutAndErr(false) {} + + explicit Command(const Command &Other) + : Args(Other.Args), CombinedOutAndErr(Other.CombinedOutAndErr), + OutputFile(Other.OutputFile) {} + + Command &operator=(const Command &Other) { + Args = Other.Args; + CombinedOutAndErr = Other.CombinedOutAndErr; + OutputFile = Other.OutputFile; + return *this; + } + + ~Command() {} + + // Returns true if the given Arg is present in Args. Only checks up to + // "-ignore_remaining_args=1". + bool hasArgument(const std::string &Arg) const { + auto i = endMutableArgs(); + return std::find(Args.begin(), i, Arg) != i; + } + + // Gets all of the current command line arguments, **including** those after + // "-ignore-remaining-args=1". + const Vector &getArguments() const { return Args; } + + // Adds the given argument before "-ignore_remaining_args=1", or at the end + // if that flag isn't present. + void addArgument(const std::string &Arg) { + Args.insert(endMutableArgs(), Arg); + } + + // Adds all given arguments before "-ignore_remaining_args=1", or at the end + // if that flag isn't present. + void addArguments(const Vector &ArgsToAdd) { + Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end()); + } + + // Removes the given argument from the command argument list. Ignores any + // occurrences after "-ignore_remaining_args=1", if present. + void removeArgument(const std::string &Arg) { + auto i = endMutableArgs(); + Args.erase(std::remove(Args.begin(), i, Arg), i); + } + + // Like hasArgument, but checks for "-[Flag]=...". + bool hasFlag(const std::string &Flag) const { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + return std::any_of(Args.begin(), endMutableArgs(), IsMatch); + } + + // Returns the value of the first instance of a given flag, or an empty string + // if the flag isn't present. Ignores any occurrences after + // "-ignore_remaining_args=1", if present. + std::string getFlagValue(const std::string &Flag) const { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + auto i = endMutableArgs(); + auto j = std::find_if(Args.begin(), i, IsMatch); + std::string result; + if (j != i) { + result = j->substr(Arg.length()); + } + return result; + } + + // Like AddArgument, but adds "-[Flag]=[Value]". + void addFlag(const std::string &Flag, const std::string &Value) { + addArgument("-" + Flag + "=" + Value); + } + + // Like RemoveArgument, but removes "-[Flag]=...". + void removeFlag(const std::string &Flag) { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + auto i = endMutableArgs(); + Args.erase(std::remove_if(Args.begin(), i, IsMatch), i); + } + + // Returns whether the command's stdout is being written to an output file. + bool hasOutputFile() const { return !OutputFile.empty(); } + + // Returns the currently set output file. + const std::string &getOutputFile() const { return OutputFile; } + + // Configures the command to redirect its output to the name file. + void setOutputFile(const std::string &FileName) { OutputFile = FileName; } + + // Returns whether the command's stderr is redirected to stdout. + bool isOutAndErrCombined() const { return CombinedOutAndErr; } + + // Sets whether to redirect the command's stderr to its stdout. + void combineOutAndErr(bool combine = true) { CombinedOutAndErr = combine; } + + // Returns a string representation of the command. On many systems this will + // be the equivalent command line. + std::string toString() const { + std::stringstream SS; + for (auto arg : getArguments()) + SS << arg << " "; + if (hasOutputFile()) + SS << ">" << getOutputFile() << " "; + if (isOutAndErrCombined()) + SS << "2>&1 "; + std::string result = SS.str(); + if (!result.empty()) + result = result.substr(0, result.length() - 1); + return result; + } + +private: + Command(Command &&Other) = delete; + Command &operator=(Command &&Other) = delete; + + Vector::iterator endMutableArgs() { + return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); + } + + Vector::const_iterator endMutableArgs() const { + return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); + } + + // The command arguments. Args[0] is the command name. + Vector Args; + + // True indicates stderr is redirected to stdout. + bool CombinedOutAndErr; + + // If not empty, stdout is redirected to the named file. + std::string OutputFile; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_COMMAND_H diff --git a/tools/fuzzing/libfuzzer/FuzzerCorpus.h b/tools/fuzzing/libfuzzer/FuzzerCorpus.h new file mode 100644 index 0000000000..54d1e09ec6 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerCorpus.h @@ -0,0 +1,533 @@ +//===- FuzzerCorpus.h - Internal header for the Fuzzer ----------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::InputCorpus +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_CORPUS +#define LLVM_FUZZER_CORPUS + +#include "FuzzerDataFlowTrace.h" +#include "FuzzerDefs.h" +#include "FuzzerIO.h" +#include "FuzzerRandom.h" +#include "FuzzerSHA1.h" +#include "FuzzerTracePC.h" +#include +#include +#include +#include + +namespace fuzzer { + +struct InputInfo { + Unit U; // The actual input data. + uint8_t Sha1[kSHA1NumBytes]; // Checksum. + // Number of features that this input has and no smaller input has. + size_t NumFeatures = 0; + size_t Tmp = 0; // Used by ValidateFeatureSet. + // Stats. + size_t NumExecutedMutations = 0; + size_t NumSuccessfullMutations = 0; + bool MayDeleteFile = false; + bool Reduced = false; + bool HasFocusFunction = false; + Vector UniqFeatureSet; + Vector DataFlowTraceForFocusFunction; + // Power schedule. + bool NeedsEnergyUpdate = false; + double Energy = 0.0; + size_t SumIncidence = 0; + Vector> FeatureFreqs; + + // Delete feature Idx and its frequency from FeatureFreqs. + bool DeleteFeatureFreq(uint32_t Idx) { + if (FeatureFreqs.empty()) + return false; + + // Binary search over local feature frequencies sorted by index. + auto Lower = std::lower_bound(FeatureFreqs.begin(), FeatureFreqs.end(), + std::pair(Idx, 0)); + + if (Lower != FeatureFreqs.end() && Lower->first == Idx) { + FeatureFreqs.erase(Lower); + return true; + } + return false; + } + + // Assign more energy to a high-entropy seed, i.e., that reveals more + // information about the globally rare features in the neighborhood + // of the seed. Since we do not know the entropy of a seed that has + // never been executed we assign fresh seeds maximum entropy and + // let II->Energy approach the true entropy from above. + void UpdateEnergy(size_t GlobalNumberOfFeatures) { + Energy = 0.0; + SumIncidence = 0; + + // Apply add-one smoothing to locally discovered features. + for (auto F : FeatureFreqs) { + size_t LocalIncidence = F.second + 1; + Energy -= LocalIncidence * logl(LocalIncidence); + SumIncidence += LocalIncidence; + } + + // Apply add-one smoothing to locally undiscovered features. + // PreciseEnergy -= 0; // since logl(1.0) == 0) + SumIncidence += (GlobalNumberOfFeatures - FeatureFreqs.size()); + + // Add a single locally abundant feature apply add-one smoothing. + size_t AbdIncidence = NumExecutedMutations + 1; + Energy -= AbdIncidence * logl(AbdIncidence); + SumIncidence += AbdIncidence; + + // Normalize. + if (SumIncidence != 0) + Energy = (Energy / SumIncidence) + logl(SumIncidence); + } + + // Increment the frequency of the feature Idx. + void UpdateFeatureFrequency(uint32_t Idx) { + NeedsEnergyUpdate = true; + + // The local feature frequencies is an ordered vector of pairs. + // If there are no local feature frequencies, push_back preserves order. + // Set the feature frequency for feature Idx32 to 1. + if (FeatureFreqs.empty()) { + FeatureFreqs.push_back(std::pair(Idx, 1)); + return; + } + + // Binary search over local feature frequencies sorted by index. + auto Lower = std::lower_bound(FeatureFreqs.begin(), FeatureFreqs.end(), + std::pair(Idx, 0)); + + // If feature Idx32 already exists, increment its frequency. + // Otherwise, insert a new pair right after the next lower index. + if (Lower != FeatureFreqs.end() && Lower->first == Idx) { + Lower->second++; + } else { + FeatureFreqs.insert(Lower, std::pair(Idx, 1)); + } + } +}; + +struct EntropicOptions { + bool Enabled; + size_t NumberOfRarestFeatures; + size_t FeatureFrequencyThreshold; +}; + +class InputCorpus { + static const uint32_t kFeatureSetSize = 1 << 21; + static const uint8_t kMaxMutationFactor = 20; + static const size_t kSparseEnergyUpdates = 100; + + size_t NumExecutedMutations = 0; + + EntropicOptions Entropic; + +public: + InputCorpus(const std::string &OutputCorpus, EntropicOptions Entropic) + : Entropic(Entropic), OutputCorpus(OutputCorpus) { + memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature)); + memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature)); + } + ~InputCorpus() { + for (auto II : Inputs) + delete II; + } + size_t size() const { return Inputs.size(); } + size_t SizeInBytes() const { + size_t Res = 0; + for (auto II : Inputs) + Res += II->U.size(); + return Res; + } + size_t NumActiveUnits() const { + size_t Res = 0; + for (auto II : Inputs) + Res += !II->U.empty(); + return Res; + } + size_t MaxInputSize() const { + size_t Res = 0; + for (auto II : Inputs) + Res = std::max(Res, II->U.size()); + return Res; + } + void IncrementNumExecutedMutations() { NumExecutedMutations++; } + + size_t NumInputsThatTouchFocusFunction() { + return std::count_if(Inputs.begin(), Inputs.end(), [](const InputInfo *II) { + return II->HasFocusFunction; + }); + } + + size_t NumInputsWithDataFlowTrace() { + return std::count_if(Inputs.begin(), Inputs.end(), [](const InputInfo *II) { + return !II->DataFlowTraceForFocusFunction.empty(); + }); + } + + bool empty() const { return Inputs.empty(); } + const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; } + InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, + bool HasFocusFunction, + const Vector &FeatureSet, + const DataFlowTrace &DFT, const InputInfo *BaseII) { + assert(!U.empty()); + if (FeatureDebug) + Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures); + Inputs.push_back(new InputInfo()); + InputInfo &II = *Inputs.back(); + II.U = U; + II.NumFeatures = NumFeatures; + II.MayDeleteFile = MayDeleteFile; + II.UniqFeatureSet = FeatureSet; + II.HasFocusFunction = HasFocusFunction; + // Assign maximal energy to the new seed. + II.Energy = RareFeatures.empty() ? 1.0 : log(RareFeatures.size()); + II.SumIncidence = RareFeatures.size(); + II.NeedsEnergyUpdate = false; + std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end()); + ComputeSHA1(U.data(), U.size(), II.Sha1); + auto Sha1Str = Sha1ToString(II.Sha1); + Hashes.insert(Sha1Str); + if (HasFocusFunction) + if (auto V = DFT.Get(Sha1Str)) + II.DataFlowTraceForFocusFunction = *V; + // This is a gross heuristic. + // Ideally, when we add an element to a corpus we need to know its DFT. + // But if we don't, we'll use the DFT of its base input. + if (II.DataFlowTraceForFocusFunction.empty() && BaseII) + II.DataFlowTraceForFocusFunction = BaseII->DataFlowTraceForFocusFunction; + DistributionNeedsUpdate = true; + PrintCorpus(); + // ValidateFeatureSet(); + return &II; + } + + // Debug-only + void PrintUnit(const Unit &U) { + if (!FeatureDebug) return; + for (uint8_t C : U) { + if (C != 'F' && C != 'U' && C != 'Z') + C = '.'; + Printf("%c", C); + } + } + + // Debug-only + void PrintFeatureSet(const Vector &FeatureSet) { + if (!FeatureDebug) return; + Printf("{"); + for (uint32_t Feature: FeatureSet) + Printf("%u,", Feature); + Printf("}"); + } + + // Debug-only + void PrintCorpus() { + if (!FeatureDebug) return; + Printf("======= CORPUS:\n"); + int i = 0; + for (auto II : Inputs) { + if (std::find(II->U.begin(), II->U.end(), 'F') != II->U.end()) { + Printf("[%2d] ", i); + Printf("%s sz=%zd ", Sha1ToString(II->Sha1).c_str(), II->U.size()); + PrintUnit(II->U); + Printf(" "); + PrintFeatureSet(II->UniqFeatureSet); + Printf("\n"); + } + i++; + } + } + + void Replace(InputInfo *II, const Unit &U) { + assert(II->U.size() > U.size()); + Hashes.erase(Sha1ToString(II->Sha1)); + DeleteFile(*II); + ComputeSHA1(U.data(), U.size(), II->Sha1); + Hashes.insert(Sha1ToString(II->Sha1)); + II->U = U; + II->Reduced = true; + DistributionNeedsUpdate = true; + } + + bool HasUnit(const Unit &U) { return Hashes.count(Hash(U)); } + bool HasUnit(const std::string &H) { return Hashes.count(H); } + InputInfo &ChooseUnitToMutate(Random &Rand) { + InputInfo &II = *Inputs[ChooseUnitIdxToMutate(Rand)]; + assert(!II.U.empty()); + return II; + } + + // Returns an index of random unit from the corpus to mutate. + size_t ChooseUnitIdxToMutate(Random &Rand) { + UpdateCorpusDistribution(Rand); + size_t Idx = static_cast(CorpusDistribution(Rand)); + assert(Idx < Inputs.size()); + return Idx; + } + + void PrintStats() { + for (size_t i = 0; i < Inputs.size(); i++) { + const auto &II = *Inputs[i]; + Printf(" [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i, + Sha1ToString(II.Sha1).c_str(), II.U.size(), + II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction); + } + } + + void PrintFeatureSet() { + for (size_t i = 0; i < kFeatureSetSize; i++) { + if(size_t Sz = GetFeature(i)) + Printf("[%zd: id %zd sz%zd] ", i, SmallestElementPerFeature[i], Sz); + } + Printf("\n\t"); + for (size_t i = 0; i < Inputs.size(); i++) + if (size_t N = Inputs[i]->NumFeatures) + Printf(" %zd=>%zd ", i, N); + Printf("\n"); + } + + void DeleteFile(const InputInfo &II) { + if (!OutputCorpus.empty() && II.MayDeleteFile) + RemoveFile(DirPlusFile(OutputCorpus, Sha1ToString(II.Sha1))); + } + + void DeleteInput(size_t Idx) { + InputInfo &II = *Inputs[Idx]; + DeleteFile(II); + Unit().swap(II.U); + II.Energy = 0.0; + II.NeedsEnergyUpdate = false; + DistributionNeedsUpdate = true; + if (FeatureDebug) + Printf("EVICTED %zd\n", Idx); + } + + void AddRareFeature(uint32_t Idx) { + // Maintain *at least* TopXRarestFeatures many rare features + // and all features with a frequency below ConsideredRare. + // Remove all other features. + while (RareFeatures.size() > Entropic.NumberOfRarestFeatures && + FreqOfMostAbundantRareFeature > Entropic.FeatureFrequencyThreshold) { + + // Find most and second most abbundant feature. + uint32_t MostAbundantRareFeatureIndices[2] = {RareFeatures[0], + RareFeatures[0]}; + size_t Delete = 0; + for (size_t i = 0; i < RareFeatures.size(); i++) { + uint32_t Idx2 = RareFeatures[i]; + if (GlobalFeatureFreqs[Idx2] >= + GlobalFeatureFreqs[MostAbundantRareFeatureIndices[0]]) { + MostAbundantRareFeatureIndices[1] = MostAbundantRareFeatureIndices[0]; + MostAbundantRareFeatureIndices[0] = Idx2; + Delete = i; + } + } + + // Remove most abundant rare feature. + RareFeatures[Delete] = RareFeatures.back(); + RareFeatures.pop_back(); + + for (auto II : Inputs) { + if (II->DeleteFeatureFreq(MostAbundantRareFeatureIndices[0])) + II->NeedsEnergyUpdate = true; + } + + // Set 2nd most abundant as the new most abundant feature count. + FreqOfMostAbundantRareFeature = + GlobalFeatureFreqs[MostAbundantRareFeatureIndices[1]]; + } + + // Add rare feature, handle collisions, and update energy. + RareFeatures.push_back(Idx); + GlobalFeatureFreqs[Idx] = 0; + for (auto II : Inputs) { + II->DeleteFeatureFreq(Idx); + + // Apply add-one smoothing to this locally undiscovered feature. + // Zero energy seeds will never be fuzzed and remain zero energy. + if (II->Energy > 0.0) { + II->SumIncidence += 1; + II->Energy += logl(II->SumIncidence) / II->SumIncidence; + } + } + + DistributionNeedsUpdate = true; + } + + bool AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) { + assert(NewSize); + Idx = Idx % kFeatureSetSize; + uint32_t OldSize = GetFeature(Idx); + if (OldSize == 0 || (Shrink && OldSize > NewSize)) { + if (OldSize > 0) { + size_t OldIdx = SmallestElementPerFeature[Idx]; + InputInfo &II = *Inputs[OldIdx]; + assert(II.NumFeatures > 0); + II.NumFeatures--; + if (II.NumFeatures == 0) + DeleteInput(OldIdx); + } else { + NumAddedFeatures++; + if (Entropic.Enabled) + AddRareFeature((uint32_t)Idx); + } + NumUpdatedFeatures++; + if (FeatureDebug) + Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize); + SmallestElementPerFeature[Idx] = Inputs.size(); + InputSizesPerFeature[Idx] = NewSize; + return true; + } + return false; + } + + // Increment frequency of feature Idx globally and locally. + void UpdateFeatureFrequency(InputInfo *II, size_t Idx) { + uint32_t Idx32 = Idx % kFeatureSetSize; + + // Saturated increment. + if (GlobalFeatureFreqs[Idx32] == 0xFFFF) + return; + uint16_t Freq = GlobalFeatureFreqs[Idx32]++; + + // Skip if abundant. + if (Freq > FreqOfMostAbundantRareFeature || + std::find(RareFeatures.begin(), RareFeatures.end(), Idx32) == + RareFeatures.end()) + return; + + // Update global frequencies. + if (Freq == FreqOfMostAbundantRareFeature) + FreqOfMostAbundantRareFeature++; + + // Update local frequencies. + if (II) + II->UpdateFeatureFrequency(Idx32); + } + + size_t NumFeatures() const { return NumAddedFeatures; } + size_t NumFeatureUpdates() const { return NumUpdatedFeatures; } + +private: + + static const bool FeatureDebug = false; + + size_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } + + void ValidateFeatureSet() { + if (FeatureDebug) + PrintFeatureSet(); + for (size_t Idx = 0; Idx < kFeatureSetSize; Idx++) + if (GetFeature(Idx)) + Inputs[SmallestElementPerFeature[Idx]]->Tmp++; + for (auto II: Inputs) { + if (II->Tmp != II->NumFeatures) + Printf("ZZZ %zd %zd\n", II->Tmp, II->NumFeatures); + assert(II->Tmp == II->NumFeatures); + II->Tmp = 0; + } + } + + // Updates the probability distribution for the units in the corpus. + // Must be called whenever the corpus or unit weights are changed. + // + // Hypothesis: inputs that maximize information about globally rare features + // are interesting. + void UpdateCorpusDistribution(Random &Rand) { + // Skip update if no seeds or rare features were added/deleted. + // Sparse updates for local change of feature frequencies, + // i.e., randomly do not skip. + if (!DistributionNeedsUpdate && + (!Entropic.Enabled || Rand(kSparseEnergyUpdates))) + return; + + DistributionNeedsUpdate = false; + + size_t N = Inputs.size(); + assert(N); + Intervals.resize(N + 1); + Weights.resize(N); + std::iota(Intervals.begin(), Intervals.end(), 0); + + bool VanillaSchedule = true; + if (Entropic.Enabled) { + for (auto II : Inputs) { + if (II->NeedsEnergyUpdate && II->Energy != 0.0) { + II->NeedsEnergyUpdate = false; + II->UpdateEnergy(RareFeatures.size()); + } + } + + for (size_t i = 0; i < N; i++) { + + if (Inputs[i]->NumFeatures == 0) { + // If the seed doesn't represent any features, assign zero energy. + Weights[i] = 0.; + } else if (Inputs[i]->NumExecutedMutations / kMaxMutationFactor > + NumExecutedMutations / Inputs.size()) { + // If the seed was fuzzed a lot more than average, assign zero energy. + Weights[i] = 0.; + } else { + // Otherwise, simply assign the computed energy. + Weights[i] = Inputs[i]->Energy; + } + + // If energy for all seeds is zero, fall back to vanilla schedule. + if (Weights[i] > 0.0) + VanillaSchedule = false; + } + } + + if (VanillaSchedule) { + for (size_t i = 0; i < N; i++) + Weights[i] = Inputs[i]->NumFeatures + ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1) + : 0.; + } + + if (FeatureDebug) { + for (size_t i = 0; i < N; i++) + Printf("%zd ", Inputs[i]->NumFeatures); + Printf("SCORE\n"); + for (size_t i = 0; i < N; i++) + Printf("%f ", Weights[i]); + Printf("Weights\n"); + } + CorpusDistribution = std::piecewise_constant_distribution( + Intervals.begin(), Intervals.end(), Weights.begin()); + } + std::piecewise_constant_distribution CorpusDistribution; + + Vector Intervals; + Vector Weights; + + std::unordered_set Hashes; + Vector Inputs; + + size_t NumAddedFeatures = 0; + size_t NumUpdatedFeatures = 0; + uint32_t InputSizesPerFeature[kFeatureSetSize]; + uint32_t SmallestElementPerFeature[kFeatureSetSize]; + + bool DistributionNeedsUpdate = true; + uint16_t FreqOfMostAbundantRareFeature = 0; + uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {}; + Vector RareFeatures; + + std::string OutputCorpus; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_CORPUS diff --git a/tools/fuzzing/libfuzzer/FuzzerCrossOver.cpp b/tools/fuzzing/libfuzzer/FuzzerCrossOver.cpp new file mode 100644 index 0000000000..83d9f8d47c --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerCrossOver.cpp @@ -0,0 +1,51 @@ +//===- FuzzerCrossOver.cpp - Cross over two test inputs -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Cross over test inputs. +//===----------------------------------------------------------------------===// + +#include "FuzzerDefs.h" +#include "FuzzerMutate.h" +#include "FuzzerRandom.h" +#include + +namespace fuzzer { + +// Cross Data1 and Data2, store the result (up to MaxOutSize bytes) in Out. +size_t MutationDispatcher::CrossOver(const uint8_t *Data1, size_t Size1, + const uint8_t *Data2, size_t Size2, + uint8_t *Out, size_t MaxOutSize) { + assert(Size1 || Size2); + MaxOutSize = Rand(MaxOutSize) + 1; + size_t OutPos = 0; + size_t Pos1 = 0; + size_t Pos2 = 0; + size_t *InPos = &Pos1; + size_t InSize = Size1; + const uint8_t *Data = Data1; + bool CurrentlyUsingFirstData = true; + while (OutPos < MaxOutSize && (Pos1 < Size1 || Pos2 < Size2)) { + // Merge a part of Data into Out. + size_t OutSizeLeft = MaxOutSize - OutPos; + if (*InPos < InSize) { + size_t InSizeLeft = InSize - *InPos; + size_t MaxExtraSize = std::min(OutSizeLeft, InSizeLeft); + size_t ExtraSize = Rand(MaxExtraSize) + 1; + memcpy(Out + OutPos, Data + *InPos, ExtraSize); + OutPos += ExtraSize; + (*InPos) += ExtraSize; + } + // Use the other input data on the next iteration. + InPos = CurrentlyUsingFirstData ? &Pos2 : &Pos1; + InSize = CurrentlyUsingFirstData ? Size2 : Size1; + Data = CurrentlyUsingFirstData ? Data2 : Data1; + CurrentlyUsingFirstData = !CurrentlyUsingFirstData; + } + return OutPos; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp new file mode 100644 index 0000000000..06ea287a3c --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp @@ -0,0 +1,295 @@ +//===- FuzzerDataFlowTrace.cpp - DataFlowTrace ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::DataFlowTrace +//===----------------------------------------------------------------------===// + +#include "FuzzerDataFlowTrace.h" + +#include "FuzzerCommand.h" +#include "FuzzerIO.h" +#include "FuzzerRandom.h" +#include "FuzzerSHA1.h" +#include "FuzzerUtil.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fuzzer { +static const char *kFunctionsTxt = "functions.txt"; + +bool BlockCoverage::AppendCoverage(const std::string &S) { + std::stringstream SS(S); + return AppendCoverage(SS); +} + +// Coverage lines have this form: +// CN X Y Z T +// where N is the number of the function, T is the total number of instrumented +// BBs, and X,Y,Z, if present, are the indecies of covered BB. +// BB #0, which is the entry block, is not explicitly listed. +bool BlockCoverage::AppendCoverage(std::istream &IN) { + std::string L; + while (std::getline(IN, L, '\n')) { + if (L.empty()) + continue; + std::stringstream SS(L.c_str() + 1); + size_t FunctionId = 0; + SS >> FunctionId; + if (L[0] == 'F') { + FunctionsWithDFT.insert(FunctionId); + continue; + } + if (L[0] != 'C') continue; + Vector CoveredBlocks; + while (true) { + uint32_t BB = 0; + SS >> BB; + if (!SS) break; + CoveredBlocks.push_back(BB); + } + if (CoveredBlocks.empty()) return false; + uint32_t NumBlocks = CoveredBlocks.back(); + CoveredBlocks.pop_back(); + for (auto BB : CoveredBlocks) + if (BB >= NumBlocks) return false; + auto It = Functions.find(FunctionId); + auto &Counters = + It == Functions.end() + ? Functions.insert({FunctionId, Vector(NumBlocks)}) + .first->second + : It->second; + + if (Counters.size() != NumBlocks) return false; // wrong number of blocks. + + Counters[0]++; + for (auto BB : CoveredBlocks) + Counters[BB]++; + } + return true; +} + +// Assign weights to each function. +// General principles: +// * any uncovered function gets weight 0. +// * a function with lots of uncovered blocks gets bigger weight. +// * a function with a less frequently executed code gets bigger weight. +Vector BlockCoverage::FunctionWeights(size_t NumFunctions) const { + Vector Res(NumFunctions); + for (auto It : Functions) { + auto FunctionID = It.first; + auto Counters = It.second; + assert(FunctionID < NumFunctions); + auto &Weight = Res[FunctionID]; + // Give higher weight if the function has a DFT. + Weight = FunctionsWithDFT.count(FunctionID) ? 1000. : 1; + // Give higher weight to functions with less frequently seen basic blocks. + Weight /= SmallestNonZeroCounter(Counters); + // Give higher weight to functions with the most uncovered basic blocks. + Weight *= NumberOfUncoveredBlocks(Counters) + 1; + } + return Res; +} + +int DataFlowTrace::ReadCoverage(const std::string &DirPath) { + Vector Files; + int Res = GetSizedFilesFromDir(DirPath, &Files); + if (Res != 0) + return Res; + for (auto &SF : Files) { + auto Name = Basename(SF.File); + if (Name == kFunctionsTxt) continue; + if (!CorporaHashes.count(Name)) continue; + std::ifstream IF(SF.File); + Coverage.AppendCoverage(IF); + } + return 0; +} + +static void DFTStringAppendToVector(Vector *DFT, + const std::string &DFTString) { + assert(DFT->size() == DFTString.size()); + for (size_t I = 0, Len = DFT->size(); I < Len; I++) + (*DFT)[I] = DFTString[I] == '1'; +} + +// converts a string of '0' and '1' into a Vector +static Vector DFTStringToVector(const std::string &DFTString) { + Vector DFT(DFTString.size()); + DFTStringAppendToVector(&DFT, DFTString); + return DFT; +} + +static bool ParseError(const char *Err, const std::string &Line) { + Printf("DataFlowTrace: parse error: %s: Line: %s\n", Err, Line.c_str()); + return false; +} + +// TODO(metzman): replace std::string with std::string_view for +// better performance. Need to figure our how to use string_view on Windows. +static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum, + std::string *DFTString) { + if (!Line.empty() && Line[0] != 'F') + return false; // Ignore coverage. + size_t SpacePos = Line.find(' '); + if (SpacePos == std::string::npos) + return ParseError("no space in the trace line", Line); + if (Line.empty() || Line[0] != 'F') + return ParseError("the trace line doesn't start with 'F'", Line); + *FunctionNum = std::atol(Line.c_str() + 1); + const char *Beg = Line.c_str() + SpacePos + 1; + const char *End = Line.c_str() + Line.size(); + assert(Beg < End); + size_t Len = End - Beg; + for (size_t I = 0; I < Len; I++) { + if (Beg[I] != '0' && Beg[I] != '1') + return ParseError("the trace should contain only 0 or 1", Line); + } + *DFTString = Beg; + return true; +} + +int DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + Vector &CorporaFiles, Random &Rand) { + if (DirPath.empty()) return 0; + Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str()); + Vector Files; + int Res = GetSizedFilesFromDir(DirPath, &Files); + if (Res != 0) + return Res; + std::string L; + size_t FocusFuncIdx = SIZE_MAX; + Vector FunctionNames; + + // Collect the hashes of the corpus files. + for (auto &SF : CorporaFiles) + CorporaHashes.insert(Hash(FileToVector(SF.File))); + + // Read functions.txt + std::ifstream IF(DirPlusFile(DirPath, kFunctionsTxt)); + size_t NumFunctions = 0; + while (std::getline(IF, L, '\n')) { + FunctionNames.push_back(L); + NumFunctions++; + if (*FocusFunction == L) + FocusFuncIdx = NumFunctions - 1; + } + if (!NumFunctions) + return 0; + + if (*FocusFunction == "auto") { + // AUTOFOCUS works like this: + // * reads the coverage data from the DFT files. + // * assigns weights to functions based on coverage. + // * chooses a random function according to the weights. + Res = ReadCoverage(DirPath); + if (Res != 0) + return Res; + auto Weights = Coverage.FunctionWeights(NumFunctions); + Vector Intervals(NumFunctions + 1); + std::iota(Intervals.begin(), Intervals.end(), 0); + auto Distribution = std::piecewise_constant_distribution( + Intervals.begin(), Intervals.end(), Weights.begin()); + FocusFuncIdx = static_cast(Distribution(Rand)); + *FocusFunction = FunctionNames[FocusFuncIdx]; + assert(FocusFuncIdx < NumFunctions); + Printf("INFO: AUTOFOCUS: %zd %s\n", FocusFuncIdx, + FunctionNames[FocusFuncIdx].c_str()); + for (size_t i = 0; i < NumFunctions; i++) { + if (!Weights[i]) continue; + Printf(" [%zd] W %g\tBB-tot %u\tBB-cov %u\tEntryFreq %u:\t%s\n", i, + Weights[i], Coverage.GetNumberOfBlocks(i), + Coverage.GetNumberOfCoveredBlocks(i), Coverage.GetCounter(i, 0), + FunctionNames[i].c_str()); + } + } + + if (!NumFunctions || FocusFuncIdx == SIZE_MAX || Files.size() <= 1) + return 0; + + // Read traces. + size_t NumTraceFiles = 0; + size_t NumTracesWithFocusFunction = 0; + for (auto &SF : Files) { + auto Name = Basename(SF.File); + if (Name == kFunctionsTxt) continue; + if (!CorporaHashes.count(Name)) continue; // not in the corpus. + NumTraceFiles++; + // Printf("=== %s\n", Name.c_str()); + std::ifstream IF(SF.File); + while (std::getline(IF, L, '\n')) { + size_t FunctionNum = 0; + std::string DFTString; + if (ParseDFTLine(L, &FunctionNum, &DFTString) && + FunctionNum == FocusFuncIdx) { + NumTracesWithFocusFunction++; + + if (FunctionNum >= NumFunctions) { + ParseError("N is greater than the number of functions", L); + return 0; + } + Traces[Name] = DFTStringToVector(DFTString); + // Print just a few small traces. + if (NumTracesWithFocusFunction <= 3 && DFTString.size() <= 16) + Printf("%s => |%s|\n", Name.c_str(), std::string(DFTString).c_str()); + break; // No need to parse the following lines. + } + } + } + Printf("INFO: DataFlowTrace: %zd trace files, %zd functions, " + "%zd traces with focus function\n", + NumTraceFiles, NumFunctions, NumTracesWithFocusFunction); + return 0; +} + +int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, + const Vector &CorporaFiles) { + Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n", + DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size()); + if (CorporaFiles.empty()) { + Printf("ERROR: can't collect data flow without corpus provided."); + return 1; + } + + static char DFSanEnv[] = "DFSAN_OPTIONS=warn_unimplemented=0"; + putenv(DFSanEnv); + MkDir(DirPath); + for (auto &F : CorporaFiles) { + // For every input F we need to collect the data flow and the coverage. + // Data flow collection may fail if we request too many DFSan tags at once. + // So, we start from requesting all tags in range [0,Size) and if that fails + // we then request tags in [0,Size/2) and [Size/2, Size), and so on. + // Function number => DFT. + auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File))); + std::unordered_map> DFTMap; + std::unordered_set Cov; + Command Cmd; + Cmd.addArgument(DFTBinary); + Cmd.addArgument(F.File); + Cmd.addArgument(OutPath); + Printf("CMD: %s\n", Cmd.toString().c_str()); + ExecuteCommand(Cmd); + } + // Write functions.txt if it's currently empty or doesn't exist. + auto FunctionsTxtPath = DirPlusFile(DirPath, kFunctionsTxt); + if (FileToString(FunctionsTxtPath).empty()) { + Command Cmd; + Cmd.addArgument(DFTBinary); + Cmd.setOutputFile(FunctionsTxtPath); + ExecuteCommand(Cmd); + } + return 0; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h new file mode 100644 index 0000000000..767bad24f1 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h @@ -0,0 +1,135 @@ +//===- FuzzerDataFlowTrace.h - Internal header for the Fuzzer ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::DataFlowTrace; reads and handles a data-flow trace. +// +// A data flow trace is generated by e.g. dataflow/DataFlow.cpp +// and is stored on disk in a separate directory. +// +// The trace dir contains a file 'functions.txt' which lists function names, +// oner per line, e.g. +// ==> functions.txt <== +// Func2 +// LLVMFuzzerTestOneInput +// Func1 +// +// All other files in the dir are the traces, see dataflow/DataFlow.cpp. +// The name of the file is sha1 of the input used to generate the trace. +// +// Current status: +// the data is parsed and the summary is printed, but the data is not yet +// used in any other way. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_DATA_FLOW_TRACE +#define LLVM_FUZZER_DATA_FLOW_TRACE + +#include "FuzzerDefs.h" +#include "FuzzerIO.h" + +#include +#include +#include +#include + +namespace fuzzer { + +int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, + const Vector &CorporaFiles); + +class BlockCoverage { + public: + bool AppendCoverage(std::istream &IN); + bool AppendCoverage(const std::string &S); + + size_t NumCoveredFunctions() const { return Functions.size(); } + + uint32_t GetCounter(size_t FunctionId, size_t BasicBlockId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + if (BasicBlockId < Counters.size()) + return Counters[BasicBlockId]; + return 0; + } + + uint32_t GetNumberOfBlocks(size_t FunctionId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + return Counters.size(); + } + + uint32_t GetNumberOfCoveredBlocks(size_t FunctionId) { + auto It = Functions.find(FunctionId); + if (It == Functions.end()) return 0; + const auto &Counters = It->second; + uint32_t Result = 0; + for (auto Cnt: Counters) + if (Cnt) + Result++; + return Result; + } + + Vector FunctionWeights(size_t NumFunctions) const; + void clear() { Functions.clear(); } + + private: + + typedef Vector CoverageVector; + + uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const { + uint32_t Res = 0; + for (auto Cnt : Counters) + if (Cnt) + Res++; + return Res; + } + + uint32_t NumberOfUncoveredBlocks(const CoverageVector &Counters) const { + return Counters.size() - NumberOfCoveredBlocks(Counters); + } + + uint32_t SmallestNonZeroCounter(const CoverageVector &Counters) const { + assert(!Counters.empty()); + uint32_t Res = Counters[0]; + for (auto Cnt : Counters) + if (Cnt) + Res = Min(Res, Cnt); + assert(Res); + return Res; + } + + // Function ID => vector of counters. + // Each counter represents how many input files trigger the given basic block. + std::unordered_map Functions; + // Functions that have DFT entry. + std::unordered_set FunctionsWithDFT; +}; + +class DataFlowTrace { + public: + int ReadCoverage(const std::string &DirPath); + int Init(const std::string &DirPath, std::string *FocusFunction, + Vector &CorporaFiles, Random &Rand); + void Clear() { Traces.clear(); } + const Vector *Get(const std::string &InputSha1) const { + auto It = Traces.find(InputSha1); + if (It != Traces.end()) + return &It->second; + return nullptr; + } + + private: + // Input's sha1 => DFT for the FocusFunction. + std::unordered_map > Traces; + BlockCoverage Coverage; + std::unordered_set CorporaHashes; +}; +} // namespace fuzzer + +#endif // LLVM_FUZZER_DATA_FLOW_TRACE diff --git a/tools/fuzzing/libfuzzer/FuzzerDefs.h b/tools/fuzzing/libfuzzer/FuzzerDefs.h new file mode 100644 index 0000000000..3952ac51ef --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDefs.h @@ -0,0 +1,75 @@ +//===- FuzzerDefs.h - Internal header for the Fuzzer ------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Basic definitions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_DEFS_H +#define LLVM_FUZZER_DEFS_H + +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace fuzzer { + +template T Min(T a, T b) { return a < b ? a : b; } +template T Max(T a, T b) { return a > b ? a : b; } + +class Random; +class Dictionary; +class DictionaryEntry; +class MutationDispatcher; +struct FuzzingOptions; +class InputCorpus; +struct InputInfo; +struct ExternalFunctions; + +// Global interface to functions that may or may not be available. +extern ExternalFunctions *EF; + +// We are using a custom allocator to give a different symbol name to STL +// containers in order to avoid ODR violations. +template + class fuzzer_allocator: public std::allocator { + public: + fuzzer_allocator() = default; + + template + explicit fuzzer_allocator(const fuzzer_allocator&) {} + + template + struct rebind { typedef fuzzer_allocator other; }; + }; + +template +using Vector = std::vector>; + +template +using Set = std::set, fuzzer_allocator>; + +typedef Vector Unit; +typedef Vector UnitVector; +typedef int (*UserCallback)(const uint8_t *Data, size_t Size); + +int FuzzerDriver(int *argc, char ***argv, UserCallback Callback); + +uint8_t *ExtraCountersBegin(); +uint8_t *ExtraCountersEnd(); +void ClearExtraCounters(); + +extern bool RunningUserCallback; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_DEFS_H diff --git a/tools/fuzzing/libfuzzer/FuzzerDictionary.h b/tools/fuzzing/libfuzzer/FuzzerDictionary.h new file mode 100644 index 0000000000..301c5d9afe --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDictionary.h @@ -0,0 +1,118 @@ +//===- FuzzerDictionary.h - Internal header for the Fuzzer ------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::Dictionary +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_DICTIONARY_H +#define LLVM_FUZZER_DICTIONARY_H + +#include "FuzzerDefs.h" +#include "FuzzerIO.h" +#include "FuzzerUtil.h" +#include +#include + +namespace fuzzer { +// A simple POD sized array of bytes. +template class FixedWord { +public: + static const size_t kMaxSize = kMaxSizeT; + FixedWord() {} + FixedWord(const uint8_t *B, uint8_t S) { Set(B, S); } + + void Set(const uint8_t *B, uint8_t S) { + assert(S <= kMaxSize); + memcpy(Data, B, S); + Size = S; + } + + bool operator==(const FixedWord &w) const { + return Size == w.Size && 0 == memcmp(Data, w.Data, Size); + } + + static size_t GetMaxSize() { return kMaxSize; } + const uint8_t *data() const { return Data; } + uint8_t size() const { return Size; } + +private: + uint8_t Size = 0; + uint8_t Data[kMaxSize]; +}; + +typedef FixedWord<64> Word; + +class DictionaryEntry { + public: + DictionaryEntry() {} + DictionaryEntry(Word W) : W(W) {} + DictionaryEntry(Word W, size_t PositionHint) : W(W), PositionHint(PositionHint) {} + const Word &GetW() const { return W; } + + bool HasPositionHint() const { return PositionHint != std::numeric_limits::max(); } + size_t GetPositionHint() const { + assert(HasPositionHint()); + return PositionHint; + } + void IncUseCount() { UseCount++; } + void IncSuccessCount() { SuccessCount++; } + size_t GetUseCount() const { return UseCount; } + size_t GetSuccessCount() const {return SuccessCount; } + + void Print(const char *PrintAfter = "\n") { + PrintASCII(W.data(), W.size()); + if (HasPositionHint()) + Printf("@%zd", GetPositionHint()); + Printf("%s", PrintAfter); + } + +private: + Word W; + size_t PositionHint = std::numeric_limits::max(); + size_t UseCount = 0; + size_t SuccessCount = 0; +}; + +class Dictionary { + public: + static const size_t kMaxDictSize = 1 << 14; + + bool ContainsWord(const Word &W) const { + return std::any_of(begin(), end(), [&](const DictionaryEntry &DE) { + return DE.GetW() == W; + }); + } + const DictionaryEntry *begin() const { return &DE[0]; } + const DictionaryEntry *end() const { return begin() + Size; } + DictionaryEntry & operator[] (size_t Idx) { + assert(Idx < Size); + return DE[Idx]; + } + void push_back(DictionaryEntry DE) { + if (Size < kMaxDictSize) + this->DE[Size++] = DE; + } + void clear() { Size = 0; } + bool empty() const { return Size == 0; } + size_t size() const { return Size; } + +private: + DictionaryEntry DE[kMaxDictSize]; + size_t Size = 0; +}; + +// Parses one dictionary entry. +// If successful, write the enty to Unit and returns true, +// otherwise returns false. +bool ParseOneDictionaryEntry(const std::string &Str, Unit *U); +// Parses the dictionary file, fills Units, returns true iff all lines +// were parsed successfully. +bool ParseDictionaryFile(const std::string &Text, Vector *Units); + +} // namespace fuzzer + +#endif // LLVM_FUZZER_DICTIONARY_H diff --git a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp new file mode 100644 index 0000000000..bedad16efa --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp @@ -0,0 +1,888 @@ +//===- FuzzerDriver.cpp - FuzzerDriver function and flags -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// FuzzerDriver and flag parsing. +//===----------------------------------------------------------------------===// + +#include "FuzzerCommand.h" +#include "FuzzerCorpus.h" +#include "FuzzerFork.h" +#include "FuzzerIO.h" +#include "FuzzerInterface.h" +#include "FuzzerInternal.h" +#include "FuzzerMerge.h" +#include "FuzzerMutate.h" +#include "FuzzerPlatform.h" +#include "FuzzerRandom.h" +#include "FuzzerTracePC.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// This function should be present in the libFuzzer so that the client +// binary can test for its existence. +#if LIBFUZZER_MSVC +extern "C" void __libfuzzer_is_present() {} +#if defined(_M_IX86) || defined(__i386__) +#pragma comment(linker, "/include:___libfuzzer_is_present") +#else +#pragma comment(linker, "/include:__libfuzzer_is_present") +#endif +#else +extern "C" __attribute__((used)) void __libfuzzer_is_present() {} +#endif // LIBFUZZER_MSVC + +namespace fuzzer { + +// Program arguments. +struct FlagDescription { + const char *Name; + const char *Description; + int Default; + int *IntFlag; + const char **StrFlag; + unsigned int *UIntFlag; +}; + +struct { +#define FUZZER_DEPRECATED_FLAG(Name) +#define FUZZER_FLAG_INT(Name, Default, Description) int Name; +#define FUZZER_FLAG_UNSIGNED(Name, Default, Description) unsigned int Name; +#define FUZZER_FLAG_STRING(Name, Description) const char *Name; +#include "FuzzerFlags.def" +#undef FUZZER_DEPRECATED_FLAG +#undef FUZZER_FLAG_INT +#undef FUZZER_FLAG_UNSIGNED +#undef FUZZER_FLAG_STRING +} Flags; + +static const FlagDescription FlagDescriptions [] { +#define FUZZER_DEPRECATED_FLAG(Name) \ + {#Name, "Deprecated; don't use", 0, nullptr, nullptr, nullptr}, +#define FUZZER_FLAG_INT(Name, Default, Description) \ + {#Name, Description, Default, &Flags.Name, nullptr, nullptr}, +#define FUZZER_FLAG_UNSIGNED(Name, Default, Description) \ + {#Name, Description, static_cast(Default), \ + nullptr, nullptr, &Flags.Name}, +#define FUZZER_FLAG_STRING(Name, Description) \ + {#Name, Description, 0, nullptr, &Flags.Name, nullptr}, +#include "FuzzerFlags.def" +#undef FUZZER_DEPRECATED_FLAG +#undef FUZZER_FLAG_INT +#undef FUZZER_FLAG_UNSIGNED +#undef FUZZER_FLAG_STRING +}; + +static const size_t kNumFlags = + sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]); + +static Vector *Inputs; +static std::string *ProgName; + +static void PrintHelp() { + Printf("Usage:\n"); + auto Prog = ProgName->c_str(); + Printf("\nTo run fuzzing pass 0 or more directories.\n"); + Printf("%s [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ]\n", Prog); + + Printf("\nTo run individual tests without fuzzing pass 1 or more files:\n"); + Printf("%s [-flag1=val1 [-flag2=val2 ...] ] file1 [file2 ...]\n", Prog); + + Printf("\nFlags: (strictly in form -flag=value)\n"); + size_t MaxFlagLen = 0; + for (size_t F = 0; F < kNumFlags; F++) + MaxFlagLen = std::max(strlen(FlagDescriptions[F].Name), MaxFlagLen); + + for (size_t F = 0; F < kNumFlags; F++) { + const auto &D = FlagDescriptions[F]; + if (strstr(D.Description, "internal flag") == D.Description) continue; + Printf(" %s", D.Name); + for (size_t i = 0, n = MaxFlagLen - strlen(D.Name); i < n; i++) + Printf(" "); + Printf("\t"); + Printf("%d\t%s\n", D.Default, D.Description); + } + Printf("\nFlags starting with '--' will be ignored and " + "will be passed verbatim to subprocesses.\n"); +} + +static const char *FlagValue(const char *Param, const char *Name) { + size_t Len = strlen(Name); + if (Param[0] == '-' && strstr(Param + 1, Name) == Param + 1 && + Param[Len + 1] == '=') + return &Param[Len + 2]; + return nullptr; +} + +// Avoid calling stol as it triggers a bug in clang/glibc build. +static long MyStol(const char *Str) { + long Res = 0; + long Sign = 1; + if (*Str == '-') { + Str++; + Sign = -1; + } + for (size_t i = 0; Str[i]; i++) { + char Ch = Str[i]; + if (Ch < '0' || Ch > '9') + return Res; + Res = Res * 10 + (Ch - '0'); + } + return Res * Sign; +} + +static bool ParseOneFlag(const char *Param) { + if (Param[0] != '-') return false; + if (Param[1] == '-') { + static bool PrintedWarning = false; + if (!PrintedWarning) { + PrintedWarning = true; + Printf("INFO: libFuzzer ignores flags that start with '--'\n"); + } + for (size_t F = 0; F < kNumFlags; F++) + if (FlagValue(Param + 1, FlagDescriptions[F].Name)) + Printf("WARNING: did you mean '%s' (single dash)?\n", Param + 1); + return true; + } + for (size_t F = 0; F < kNumFlags; F++) { + const char *Name = FlagDescriptions[F].Name; + const char *Str = FlagValue(Param, Name); + if (Str) { + if (FlagDescriptions[F].IntFlag) { + int Val = MyStol(Str); + *FlagDescriptions[F].IntFlag = Val; + if (Flags.verbosity >= 2) + Printf("Flag: %s %d\n", Name, Val); + return true; + } else if (FlagDescriptions[F].UIntFlag) { + unsigned int Val = std::stoul(Str); + *FlagDescriptions[F].UIntFlag = Val; + if (Flags.verbosity >= 2) + Printf("Flag: %s %u\n", Name, Val); + return true; + } else if (FlagDescriptions[F].StrFlag) { + *FlagDescriptions[F].StrFlag = Str; + if (Flags.verbosity >= 2) + Printf("Flag: %s %s\n", Name, Str); + return true; + } else { // Deprecated flag. + Printf("Flag: %s: deprecated, don't use\n", Name); + return true; + } + } + } + Printf("\n\nWARNING: unrecognized flag '%s'; " + "use -help=1 to list all flags\n\n", Param); + return true; +} + +// We don't use any library to minimize dependencies. +static void ParseFlags(const Vector &Args, + const ExternalFunctions *EF) { + for (size_t F = 0; F < kNumFlags; F++) { + if (FlagDescriptions[F].IntFlag) + *FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default; + if (FlagDescriptions[F].UIntFlag) + *FlagDescriptions[F].UIntFlag = + static_cast(FlagDescriptions[F].Default); + if (FlagDescriptions[F].StrFlag) + *FlagDescriptions[F].StrFlag = nullptr; + } + + // Disable len_control by default, if LLVMFuzzerCustomMutator is used. + if (EF->LLVMFuzzerCustomMutator) { + Flags.len_control = 0; + Printf("INFO: found LLVMFuzzerCustomMutator (%p). " + "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator); + } + + Inputs = new Vector; + for (size_t A = 1; A < Args.size(); A++) { + if (ParseOneFlag(Args[A].c_str())) { + if (Flags.ignore_remaining_args) + break; + continue; + } + Inputs->push_back(Args[A]); + } +} + +static std::mutex Mu; + +static void PulseThread() { + while (true) { + SleepSeconds(600); + std::lock_guard Lock(Mu); + Printf("pulse...\n"); + } +} + +static void WorkerThread(const Command &BaseCmd, std::atomic *Counter, + unsigned NumJobs, std::atomic *HasErrors) { + while (true) { + unsigned C = (*Counter)++; + if (C >= NumJobs) break; + std::string Log = "fuzz-" + std::to_string(C) + ".log"; + Command Cmd(BaseCmd); + Cmd.setOutputFile(Log); + Cmd.combineOutAndErr(); + if (Flags.verbosity) { + std::string CommandLine = Cmd.toString(); + Printf("%s\n", CommandLine.c_str()); + } + int ExitCode = ExecuteCommand(Cmd); + if (ExitCode != 0) + *HasErrors = true; + std::lock_guard Lock(Mu); + Printf("================== Job %u exited with exit code %d ============\n", + C, ExitCode); + fuzzer::CopyFileToErr(Log); + } +} + +std::string CloneArgsWithoutX(const Vector &Args, + const char *X1, const char *X2) { + std::string Cmd; + for (auto &S : Args) { + if (FlagValue(S.c_str(), X1) || FlagValue(S.c_str(), X2)) + continue; + Cmd += S + " "; + } + return Cmd; +} + +static int RunInMultipleProcesses(const Vector &Args, + unsigned NumWorkers, unsigned NumJobs) { + std::atomic Counter(0); + std::atomic HasErrors(false); + Command Cmd(Args); + Cmd.removeFlag("jobs"); + Cmd.removeFlag("workers"); + Vector V; + std::thread Pulse(PulseThread); + Pulse.detach(); + for (unsigned i = 0; i < NumWorkers; i++) + V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); + for (auto &T : V) + T.join(); + return HasErrors ? 1 : 0; +} + +static void RssThread(Fuzzer *F, size_t RssLimitMb) { + while (true) { + SleepSeconds(1); + size_t Peak = GetPeakRSSMb(); + if (Peak > RssLimitMb) + F->RssLimitCallback(); + } +} + +static void StartRssThread(Fuzzer *F, size_t RssLimitMb) { + if (!RssLimitMb) + return; + std::thread T(RssThread, F, RssLimitMb); + T.detach(); +} + +int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) { + Unit U = FileToVector(InputFilePath); + if (MaxLen && MaxLen < U.size()) + U.resize(MaxLen); + F->ExecuteCallback(U.data(), U.size()); + F->TryDetectingAMemoryLeak(U.data(), U.size(), true); + return 0; +} + +static bool AllInputsAreFiles() { + if (Inputs->empty()) return false; + for (auto &Path : *Inputs) + if (!IsFile(Path)) + return false; + return true; +} + +static std::string GetDedupTokenFromCmdOutput(const std::string &S) { + auto Beg = S.find("DEDUP_TOKEN:"); + if (Beg == std::string::npos) + return ""; + auto End = S.find('\n', Beg); + if (End == std::string::npos) + return ""; + return S.substr(Beg, End - Beg); +} + +int CleanseCrashInput(const Vector &Args, + const FuzzingOptions &Options) { + if (Inputs->size() != 1 || !Flags.exact_artifact_path) { + Printf("ERROR: -cleanse_crash should be given one input file and" + " -exact_artifact_path\n"); + return 1; + } + std::string InputFilePath = Inputs->at(0); + std::string OutputFilePath = Flags.exact_artifact_path; + Command Cmd(Args); + Cmd.removeFlag("cleanse_crash"); + + assert(Cmd.hasArgument(InputFilePath)); + Cmd.removeArgument(InputFilePath); + + auto TmpFilePath = TempPath("CleanseCrashInput", ".repro"); + Cmd.addArgument(TmpFilePath); + Cmd.setOutputFile(getDevNull()); + Cmd.combineOutAndErr(); + + std::string CurrentFilePath = InputFilePath; + auto U = FileToVector(CurrentFilePath); + size_t Size = U.size(); + + const Vector ReplacementBytes = {' ', 0xff}; + for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) { + bool Changed = false; + for (size_t Idx = 0; Idx < Size; Idx++) { + Printf("CLEANSE[%d]: Trying to replace byte %zd of %zd\n", NumAttempts, + Idx, Size); + uint8_t OriginalByte = U[Idx]; + if (ReplacementBytes.end() != std::find(ReplacementBytes.begin(), + ReplacementBytes.end(), + OriginalByte)) + continue; + for (auto NewByte : ReplacementBytes) { + U[Idx] = NewByte; + WriteToFile(U, TmpFilePath); + auto ExitCode = ExecuteCommand(Cmd); + RemoveFile(TmpFilePath); + if (!ExitCode) { + U[Idx] = OriginalByte; + } else { + Changed = true; + Printf("CLEANSE: Replaced byte %zd with 0x%x\n", Idx, NewByte); + WriteToFile(U, OutputFilePath); + break; + } + } + } + if (!Changed) break; + } + return 0; +} + +int MinimizeCrashInput(const Vector &Args, + const FuzzingOptions &Options) { + if (Inputs->size() != 1) { + Printf("ERROR: -minimize_crash should be given one input file\n"); + return 1; + } + std::string InputFilePath = Inputs->at(0); + Command BaseCmd(Args); + BaseCmd.removeFlag("minimize_crash"); + BaseCmd.removeFlag("exact_artifact_path"); + assert(BaseCmd.hasArgument(InputFilePath)); + BaseCmd.removeArgument(InputFilePath); + if (Flags.runs <= 0 && Flags.max_total_time == 0) { + Printf("INFO: you need to specify -runs=N or " + "-max_total_time=N with -minimize_crash=1\n" + "INFO: defaulting to -max_total_time=600\n"); + BaseCmd.addFlag("max_total_time", "600"); + } + + BaseCmd.combineOutAndErr(); + + std::string CurrentFilePath = InputFilePath; + while (true) { + Unit U = FileToVector(CurrentFilePath); + Printf("CRASH_MIN: minimizing crash input: '%s' (%zd bytes)\n", + CurrentFilePath.c_str(), U.size()); + + Command Cmd(BaseCmd); + Cmd.addArgument(CurrentFilePath); + + Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str()); + std::string CmdOutput; + bool Success = ExecuteCommand(Cmd, &CmdOutput); + if (Success) { + Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); + return 1; + } + Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " + "it further\n", + CurrentFilePath.c_str(), U.size()); + auto DedupToken1 = GetDedupTokenFromCmdOutput(CmdOutput); + if (!DedupToken1.empty()) + Printf("CRASH_MIN: DedupToken1: %s\n", DedupToken1.c_str()); + + std::string ArtifactPath = + Flags.exact_artifact_path + ? Flags.exact_artifact_path + : Options.ArtifactPrefix + "minimized-from-" + Hash(U); + Cmd.addFlag("minimize_crash_internal_step", "1"); + Cmd.addFlag("exact_artifact_path", ArtifactPath); + Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str()); + CmdOutput.clear(); + Success = ExecuteCommand(Cmd, &CmdOutput); + Printf("%s", CmdOutput.c_str()); + if (Success) { + if (Flags.exact_artifact_path) { + CurrentFilePath = Flags.exact_artifact_path; + WriteToFile(U, CurrentFilePath); + } + Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n", + CurrentFilePath.c_str(), U.size()); + break; + } + auto DedupToken2 = GetDedupTokenFromCmdOutput(CmdOutput); + if (!DedupToken2.empty()) + Printf("CRASH_MIN: DedupToken2: %s\n", DedupToken2.c_str()); + + if (DedupToken1 != DedupToken2) { + if (Flags.exact_artifact_path) { + CurrentFilePath = Flags.exact_artifact_path; + WriteToFile(U, CurrentFilePath); + } + Printf("CRASH_MIN: mismatch in dedup tokens" + " (looks like a different bug). Won't minimize further\n"); + break; + } + + CurrentFilePath = ArtifactPath; + Printf("*********************************\n"); + } + return 0; +} + +int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { + assert(Inputs->size() == 1); + std::string InputFilePath = Inputs->at(0); + Unit U = FileToVector(InputFilePath); + Printf("INFO: Starting MinimizeCrashInputInternalStep: %zd\n", U.size()); + if (U.size() < 2) { + Printf("INFO: The input is small enough, exiting\n"); + return 0; + } + F->SetMaxInputLen(U.size()); + F->SetMaxMutationLen(U.size() - 1); + F->MinimizeCrashLoop(U); + Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n"); + return 0; +} + +int Merge(Fuzzer *F, FuzzingOptions &Options, const Vector &Args, + const Vector &Corpora, const char *CFPathOrNull) { + if (Corpora.size() < 2) { + Printf("INFO: Merge requires two or more corpus dirs\n"); + return 0; + } + + Vector OldCorpus, NewCorpus; + int Res = GetSizedFilesFromDir(Corpora[0], &OldCorpus); + if (Res != 0) + return Res; + for (size_t i = 1; i < Corpora.size(); i++) { + Res = GetSizedFilesFromDir(Corpora[i], &NewCorpus); + if (Res != 0) + return Res; + } + std::sort(OldCorpus.begin(), OldCorpus.end()); + std::sort(NewCorpus.begin(), NewCorpus.end()); + + std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt"); + Vector NewFiles; + Set NewFeatures, NewCov; + Res = CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, + {}, &NewCov, CFPath, true); + if (Res != 0) + return Res; + + if (F->isGracefulExitRequested()) + return 0; + for (auto &Path : NewFiles) + F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); + // We are done, delete the control file if it was a temporary one. + if (!Flags.merge_control_file) + RemoveFile(CFPath); + + return 0; +} + +int AnalyzeDictionary(Fuzzer *F, const Vector& Dict, + UnitVector& Corpus) { + Printf("Started dictionary minimization (up to %d tests)\n", + Dict.size() * Corpus.size() * 2); + + // Scores and usage count for each dictionary unit. + Vector Scores(Dict.size()); + Vector Usages(Dict.size()); + + Vector InitialFeatures; + Vector ModifiedFeatures; + for (auto &C : Corpus) { + // Get coverage for the testcase without modifications. + F->ExecuteCallback(C.data(), C.size()); + InitialFeatures.clear(); + TPC.CollectFeatures([&](size_t Feature) { + InitialFeatures.push_back(Feature); + }); + + for (size_t i = 0; i < Dict.size(); ++i) { + Vector Data = C; + auto StartPos = std::search(Data.begin(), Data.end(), + Dict[i].begin(), Dict[i].end()); + // Skip dictionary unit, if the testcase does not contain it. + if (StartPos == Data.end()) + continue; + + ++Usages[i]; + while (StartPos != Data.end()) { + // Replace all occurrences of dictionary unit in the testcase. + auto EndPos = StartPos + Dict[i].size(); + for (auto It = StartPos; It != EndPos; ++It) + *It ^= 0xFF; + + StartPos = std::search(EndPos, Data.end(), + Dict[i].begin(), Dict[i].end()); + } + + // Get coverage for testcase with masked occurrences of dictionary unit. + F->ExecuteCallback(Data.data(), Data.size()); + ModifiedFeatures.clear(); + TPC.CollectFeatures([&](size_t Feature) { + ModifiedFeatures.push_back(Feature); + }); + + if (InitialFeatures == ModifiedFeatures) + --Scores[i]; + else + Scores[i] += 2; + } + } + + Printf("###### Useless dictionary elements. ######\n"); + for (size_t i = 0; i < Dict.size(); ++i) { + // Dictionary units with positive score are treated as useful ones. + if (Scores[i] > 0) + continue; + + Printf("\""); + PrintASCII(Dict[i].data(), Dict[i].size(), "\""); + Printf(" # Score: %d, Used: %d\n", Scores[i], Usages[i]); + } + Printf("###### End of useless dictionary elements. ######\n"); + return 0; +} + +int ParseSeedInuts(const char *seed_inputs, Vector &Files) { + // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file + if (!seed_inputs) return 0; + std::string SeedInputs; + if (Flags.seed_inputs[0] == '@') + SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. + else + SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. + if (SeedInputs.empty()) { + Printf("seed_inputs is empty or @file does not exist.\n"); + return 1; + } + // Parse SeedInputs. + size_t comma_pos = 0; + while ((comma_pos = SeedInputs.find_last_of(',')) != std::string::npos) { + Files.push_back(SeedInputs.substr(comma_pos + 1)); + SeedInputs = SeedInputs.substr(0, comma_pos); + } + Files.push_back(SeedInputs); + return 0; +} + +static Vector ReadCorpora(const Vector &CorpusDirs, + const Vector &ExtraSeedFiles) { + Vector SizedFiles; + size_t LastNumFiles = 0; + for (auto &Dir : CorpusDirs) { + GetSizedFilesFromDir(Dir, &SizedFiles); + Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles, + Dir.c_str()); + LastNumFiles = SizedFiles.size(); + } + for (auto &File : ExtraSeedFiles) + if (auto Size = FileSize(File)) + SizedFiles.push_back({File, Size}); + return SizedFiles; +} + +int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + using namespace fuzzer; + assert(argc && argv && "Argument pointers cannot be nullptr"); + std::string Argv0((*argv)[0]); + if (!EF) + EF = new ExternalFunctions(); + if (EF->LLVMFuzzerInitialize) + EF->LLVMFuzzerInitialize(argc, argv); + if (EF->__msan_scoped_disable_interceptor_checks) + EF->__msan_scoped_disable_interceptor_checks(); + const Vector Args(*argv, *argv + *argc); + assert(!Args.empty()); + ProgName = new std::string(Args[0]); + if (Argv0 != *ProgName) { + Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n"); + return 1; + } + ParseFlags(Args, EF); + if (Flags.help) { + PrintHelp(); + return 0; + } + + if (Flags.close_fd_mask & 2) + DupAndCloseStderr(); + if (Flags.close_fd_mask & 1) + CloseStdout(); + + if (Flags.jobs > 0 && Flags.workers == 0) { + Flags.workers = std::min(NumberOfCpuCores() / 2, Flags.jobs); + if (Flags.workers > 1) + Printf("Running %u workers\n", Flags.workers); + } + + if (Flags.workers > 0 && Flags.jobs > 0) + return RunInMultipleProcesses(Args, Flags.workers, Flags.jobs); + + FuzzingOptions Options; + Options.Verbosity = Flags.verbosity; + Options.MaxLen = Flags.max_len; + Options.LenControl = Flags.len_control; + Options.UnitTimeoutSec = Flags.timeout; + Options.ErrorExitCode = Flags.error_exitcode; + Options.TimeoutExitCode = Flags.timeout_exitcode; + Options.IgnoreTimeouts = Flags.ignore_timeouts; + Options.IgnoreOOMs = Flags.ignore_ooms; + Options.IgnoreCrashes = Flags.ignore_crashes; + Options.MaxTotalTimeSec = Flags.max_total_time; + Options.DoCrossOver = Flags.cross_over; + Options.MutateDepth = Flags.mutate_depth; + Options.ReduceDepth = Flags.reduce_depth; + Options.UseCounters = Flags.use_counters; + Options.UseMemmem = Flags.use_memmem; + Options.UseCmp = Flags.use_cmp; + Options.UseValueProfile = Flags.use_value_profile; + Options.Shrink = Flags.shrink; + Options.ReduceInputs = Flags.reduce_inputs; + Options.ShuffleAtStartUp = Flags.shuffle; + Options.PreferSmall = Flags.prefer_small; + Options.ReloadIntervalSec = Flags.reload; + Options.OnlyASCII = Flags.only_ascii; + Options.DetectLeaks = Flags.detect_leaks; + Options.PurgeAllocatorIntervalSec = Flags.purge_allocator_interval; + Options.TraceMalloc = Flags.trace_malloc; + Options.RssLimitMb = Flags.rss_limit_mb; + Options.MallocLimitMb = Flags.malloc_limit_mb; + if (!Options.MallocLimitMb) + Options.MallocLimitMb = Options.RssLimitMb; + if (Flags.runs >= 0) + Options.MaxNumberOfRuns = Flags.runs; + if (!Inputs->empty() && !Flags.minimize_crash_internal_step) + Options.OutputCorpus = (*Inputs)[0]; + Options.ReportSlowUnits = Flags.report_slow_units; + if (Flags.artifact_prefix) + Options.ArtifactPrefix = Flags.artifact_prefix; + if (Flags.exact_artifact_path) + Options.ExactArtifactPath = Flags.exact_artifact_path; + Vector Dictionary; + if (Flags.dict) + if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) + return 1; + if (Flags.verbosity > 0 && !Dictionary.empty()) + Printf("Dictionary: %zd entries\n", Dictionary.size()); + bool RunIndividualFiles = AllInputsAreFiles(); + Options.SaveArtifacts = + !RunIndividualFiles || Flags.minimize_crash_internal_step; + Options.PrintNewCovPcs = Flags.print_pcs; + Options.PrintNewCovFuncs = Flags.print_funcs; + Options.PrintFinalStats = Flags.print_final_stats; + Options.PrintCorpusStats = Flags.print_corpus_stats; + Options.PrintCoverage = Flags.print_coverage; + if (Flags.exit_on_src_pos) + Options.ExitOnSrcPos = Flags.exit_on_src_pos; + if (Flags.exit_on_item) + Options.ExitOnItem = Flags.exit_on_item; + if (Flags.focus_function) + Options.FocusFunction = Flags.focus_function; + if (Flags.data_flow_trace) + Options.DataFlowTrace = Flags.data_flow_trace; + if (Flags.features_dir) + Options.FeaturesDir = Flags.features_dir; + if (Flags.collect_data_flow) + Options.CollectDataFlow = Flags.collect_data_flow; + if (Flags.stop_file) + Options.StopFile = Flags.stop_file; + Options.Entropic = Flags.entropic; + Options.EntropicFeatureFrequencyThreshold = + (size_t)Flags.entropic_feature_frequency_threshold; + Options.EntropicNumberOfRarestFeatures = + (size_t)Flags.entropic_number_of_rarest_features; + if (Options.Entropic) { + if (!Options.FocusFunction.empty()) { + Printf("ERROR: The parameters `--entropic` and `--focus_function` cannot " + "be used together.\n"); + return 1; + } + Printf("INFO: Running with entropic power schedule (0x%X, %d).\n", + Options.EntropicFeatureFrequencyThreshold, + Options.EntropicNumberOfRarestFeatures); + } + struct EntropicOptions Entropic; + Entropic.Enabled = Options.Entropic; + Entropic.FeatureFrequencyThreshold = + Options.EntropicFeatureFrequencyThreshold; + Entropic.NumberOfRarestFeatures = Options.EntropicNumberOfRarestFeatures; + + unsigned Seed = Flags.seed; + // Initialize Seed. + if (Seed == 0) + Seed = + std::chrono::system_clock::now().time_since_epoch().count() + GetPid(); + if (Flags.verbosity) + Printf("INFO: Seed: %u\n", Seed); + + if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) { + if (RunIndividualFiles) + return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, + ReadCorpora({}, *Inputs)); + else + return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, + ReadCorpora(*Inputs, {})); + } + + Random Rand(Seed); + auto *MD = new MutationDispatcher(Rand, Options); + auto *Corpus = new InputCorpus(Options.OutputCorpus, Entropic); + auto *F = new Fuzzer(Callback, *Corpus, *MD, Options); + + for (auto &U: Dictionary) + if (U.size() <= Word::GetMaxSize()) + MD->AddWordToManualDictionary(Word(U.data(), U.size())); + + // Threads are only supported by Chrome. Don't use them with emscripten + // for now. +#if !LIBFUZZER_EMSCRIPTEN + StartRssThread(F, Flags.rss_limit_mb); +#endif // LIBFUZZER_EMSCRIPTEN + + Options.HandleAbrt = Flags.handle_abrt; + Options.HandleBus = Flags.handle_bus; + Options.HandleFpe = Flags.handle_fpe; + Options.HandleIll = Flags.handle_ill; + Options.HandleInt = Flags.handle_int; + Options.HandleSegv = Flags.handle_segv; + Options.HandleTerm = Flags.handle_term; + Options.HandleXfsz = Flags.handle_xfsz; + Options.HandleUsr1 = Flags.handle_usr1; + Options.HandleUsr2 = Flags.handle_usr2; + SetSignalHandler(Options); + + std::atexit(Fuzzer::StaticExitCallback); + + if (Flags.minimize_crash) + return MinimizeCrashInput(Args, Options); + + if (Flags.minimize_crash_internal_step) + return MinimizeCrashInputInternalStep(F, Corpus); + + if (Flags.cleanse_crash) + return CleanseCrashInput(Args, Options); + + if (RunIndividualFiles) { + Options.SaveArtifacts = false; + int Runs = std::max(1, Flags.runs); + Printf("%s: Running %zd inputs %d time(s) each.\n", ProgName->c_str(), + Inputs->size(), Runs); + for (auto &Path : *Inputs) { + auto StartTime = system_clock::now(); + Printf("Running: %s\n", Path.c_str()); + for (int Iter = 0; Iter < Runs; Iter++) + RunOneTest(F, Path.c_str(), Options.MaxLen); + auto StopTime = system_clock::now(); + auto MS = duration_cast(StopTime - StartTime).count(); + Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS); + } + Printf("***\n" + "*** NOTE: fuzzing was not performed, you have only\n" + "*** executed the target code on a fixed set of inputs.\n" + "***\n"); + F->PrintFinalStats(); + return 0; + } + + if (Flags.fork) + return FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + + if (Flags.merge) + return Merge(F, Options, Args, *Inputs, Flags.merge_control_file); + + if (Flags.merge_inner) { + const size_t kDefaultMaxMergeLen = 1 << 20; + if (Options.MaxLen == 0) + F->SetMaxInputLen(kDefaultMaxMergeLen); + assert(Flags.merge_control_file); + return F->CrashResistantMergeInternalStep(Flags.merge_control_file); + } + + if (Flags.analyze_dict) { + size_t MaxLen = INT_MAX; // Large max length. + UnitVector InitialCorpus; + for (auto &Inp : *Inputs) { + Printf("Loading corpus dir: %s\n", Inp.c_str()); + ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, + MaxLen, /*ExitOnError=*/false); + } + + if (Dictionary.empty() || Inputs->empty()) { + Printf("ERROR: can't analyze dict without dict and corpus provided\n"); + return 1; + } + if (AnalyzeDictionary(F, Dictionary, InitialCorpus)) { + Printf("Dictionary analysis failed\n"); + return 1; + } + Printf("Dictionary analysis succeeded\n"); + return 0; + } + + { + Vector Files; + int Res = ParseSeedInuts(Flags.seed_inputs, Files); + if (Res != 0) + return Res; + auto CorporaFiles = ReadCorpora(*Inputs, Files); + Res = F->Loop(CorporaFiles); + if (Res != 0) + return Res; + if (F->isGracefulExitRequested()) + return 0; + } + + if (Flags.verbosity) + Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), + F->secondsSinceProcessStartUp()); + F->PrintFinalStats(); + + return 0; // Don't let F destroy itself. +} + +extern "C" ATTRIBUTE_INTERFACE int +LLVMFuzzerRunDriver(int *argc, char ***argv, + int (*UserCb)(const uint8_t *Data, size_t Size)) { + return FuzzerDriver(argc, argv, UserCb); +} + +// Storage for global ExternalFunctions object. +ExternalFunctions *EF = nullptr; + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctions.def b/tools/fuzzing/libfuzzer/FuzzerExtFunctions.def new file mode 100644 index 0000000000..51edf8444e --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctions.def @@ -0,0 +1,50 @@ +//===- FuzzerExtFunctions.def - External functions --------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// This defines the external function pointers that +// ``fuzzer::ExternalFunctions`` should contain and try to initialize. The +// EXT_FUNC macro must be defined at the point of inclusion. The signature of +// the macro is: +// +// EXT_FUNC(, , , ) +//===----------------------------------------------------------------------===// + +// Optional user functions +EXT_FUNC(LLVMFuzzerInitialize, int, (int *argc, char ***argv), false); +EXT_FUNC(LLVMFuzzerCustomMutator, size_t, + (uint8_t *Data, size_t Size, size_t MaxSize, unsigned int Seed), + false); +EXT_FUNC(LLVMFuzzerCustomCrossOver, size_t, + (const uint8_t *Data1, size_t Size1, + const uint8_t *Data2, size_t Size2, + uint8_t *Out, size_t MaxOutSize, unsigned int Seed), + false); + +// Sanitizer functions +EXT_FUNC(__lsan_enable, void, (), false); +EXT_FUNC(__lsan_disable, void, (), false); +EXT_FUNC(__lsan_do_recoverable_leak_check, int, (), false); +EXT_FUNC(__sanitizer_acquire_crash_state, int, (), true); +EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int, + (void (*malloc_hook)(const volatile void *, size_t), + void (*free_hook)(const volatile void *)), + false); +EXT_FUNC(__sanitizer_log_write, void, (const char *buf, size_t len), false); +EXT_FUNC(__sanitizer_purge_allocator, void, (), false); +EXT_FUNC(__sanitizer_print_memory_profile, void, (size_t, size_t), false); +EXT_FUNC(__sanitizer_print_stack_trace, void, (), true); +EXT_FUNC(__sanitizer_symbolize_pc, void, + (void *, const char *fmt, char *out_buf, size_t out_buf_size), false); +EXT_FUNC(__sanitizer_get_module_and_offset_for_pc, int, + (void *pc, char *module_path, + size_t module_path_len,void **pc_offset), false); +EXT_FUNC(__sanitizer_set_death_callback, void, (void (*)(void)), true); +EXT_FUNC(__sanitizer_set_report_fd, void, (void*), false); +EXT_FUNC(__msan_scoped_disable_interceptor_checks, void, (), false); +EXT_FUNC(__msan_scoped_enable_interceptor_checks, void, (), false); +EXT_FUNC(__msan_unpoison, void, (const volatile void *, size_t size), false); +EXT_FUNC(__msan_unpoison_param, void, (size_t n), false); diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctions.h b/tools/fuzzing/libfuzzer/FuzzerExtFunctions.h new file mode 100644 index 0000000000..c88aac4e67 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctions.h @@ -0,0 +1,34 @@ +//===- FuzzerExtFunctions.h - Interface to external functions ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Defines an interface to (possibly optional) functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_EXT_FUNCTIONS_H +#define LLVM_FUZZER_EXT_FUNCTIONS_H + +#include +#include + +namespace fuzzer { + +struct ExternalFunctions { + // Initialize function pointers. Functions that are not available will be set + // to nullptr. Do not call this constructor before ``main()`` has been + // entered. + ExternalFunctions(); + +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + RETURN_TYPE(*NAME) FUNC_SIG = nullptr + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +}; +} // namespace fuzzer + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsym.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsym.cpp new file mode 100644 index 0000000000..95233d2a10 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsym.cpp @@ -0,0 +1,51 @@ +//===- FuzzerExtFunctionsDlsym.cpp - Interface to external functions ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Implementation for operating systems that support dlsym(). We only use it on +// Apple platforms for now. We don't use this approach on Linux because it +// requires that clients of LibFuzzer pass ``--export-dynamic`` to the linker. +// That is a complication we don't wish to expose to clients right now. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_APPLE + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include + +using namespace fuzzer; + +template +static T GetFnPtr(const char *FnName, bool WarnIfMissing) { + dlerror(); // Clear any previous errors. + void *Fn = dlsym(RTLD_DEFAULT, FnName); + if (Fn == nullptr) { + if (WarnIfMissing) { + const char *ErrorMsg = dlerror(); + Printf("WARNING: Failed to find function \"%s\".", FnName); + if (ErrorMsg) + Printf(" Reason %s.", ErrorMsg); + Printf("\n"); + } + } + return reinterpret_cast(Fn); +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = GetFnPtr(#NAME, WARN) + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif // LIBFUZZER_APPLE diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp new file mode 100644 index 0000000000..24ddc57d47 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp @@ -0,0 +1,54 @@ +//===- FuzzerExtFunctionsWeak.cpp - Interface to external functions -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Implementation for Linux. This relies on the linker's support for weak +// symbols. We don't use this approach on Apple platforms because it requires +// clients of LibFuzzer to pass ``-U _`` to the linker to allow +// weak symbols to be undefined. That is a complication we don't want to expose +// to clients right now. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUCHSIA || \ + LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" + +extern "C" { +// Declare these symbols as weak to allow them to be optionally defined. +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + __attribute__((weak, visibility("default"))) RETURN_TYPE NAME FUNC_SIG + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +using namespace fuzzer; + +static void CheckFnPtr(void *FnPtr, const char *FnName, bool WarnIfMissing) { + if (FnPtr == nullptr && WarnIfMissing) { + Printf("WARNING: Failed to find function \"%s\".\n", FnName); + } +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = ::NAME; \ + CheckFnPtr(reinterpret_cast(reinterpret_cast(::NAME)), \ + #NAME, WARN); + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWindows.cpp b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWindows.cpp new file mode 100644 index 0000000000..688bad1d51 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWindows.cpp @@ -0,0 +1,82 @@ +//=== FuzzerExtWindows.cpp - Interface to external functions --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Implementation of FuzzerExtFunctions for Windows. Uses alternatename when +// compiled with MSVC. Uses weak aliases when compiled with clang. Unfortunately +// the method each compiler supports is not supported by the other. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" + +using namespace fuzzer; + +// Intermediate macro to ensure the parameter is expanded before stringified. +#define STRINGIFY_(A) #A +#define STRINGIFY(A) STRINGIFY_(A) + +#if LIBFUZZER_MSVC +// Copied from compiler-rt/lib/sanitizer_common/sanitizer_win_defs.h +#if defined(_M_IX86) || defined(__i386__) +#define WIN_SYM_PREFIX "_" +#else +#define WIN_SYM_PREFIX +#endif + +// Declare external functions as having alternativenames, so that we can +// determine if they are not defined. +#define EXTERNAL_FUNC(Name, Default) \ + __pragma(comment(linker, "/alternatename:" WIN_SYM_PREFIX STRINGIFY( \ + Name) "=" WIN_SYM_PREFIX STRINGIFY(Default))) +#else +// Declare external functions as weak to allow them to default to a specified +// function if not defined explicitly. We must use weak symbols because clang's +// support for alternatename is not 100%, see +// https://bugs.llvm.org/show_bug.cgi?id=40218 for more details. +#define EXTERNAL_FUNC(Name, Default) \ + __attribute__((weak, alias(STRINGIFY(Default)))) +#endif // LIBFUZZER_MSVC + +extern "C" { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + RETURN_TYPE NAME##Def FUNC_SIG { \ + Printf("ERROR: Function \"%s\" not defined.\n", #NAME); \ + exit(1); \ + } \ + EXTERNAL_FUNC(NAME, NAME##Def) RETURN_TYPE NAME FUNC_SIG + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +template +static T *GetFnPtr(T *Fun, T *FunDef, const char *FnName, bool WarnIfMissing) { + if (Fun == FunDef) { + if (WarnIfMissing) + Printf("WARNING: Failed to find function \"%s\".\n", FnName); + return nullptr; + } + return Fun; +} + +namespace fuzzer { + +ExternalFunctions::ExternalFunctions() { +#define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ + this->NAME = GetFnPtr(::NAME, ::NAME##Def, #NAME, WARN); + +#include "FuzzerExtFunctions.def" + +#undef EXT_FUNC +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp b/tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp new file mode 100644 index 0000000000..d36beba1b1 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp @@ -0,0 +1,42 @@ +//===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Extra coverage counters defined by user code. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" +#include + +#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ + LIBFUZZER_OPENBSD || LIBFUZZER_FUCHSIA || LIBFUZZER_EMSCRIPTEN +__attribute__((weak)) extern uint8_t __start___libfuzzer_extra_counters; +__attribute__((weak)) extern uint8_t __stop___libfuzzer_extra_counters; + +namespace fuzzer { +uint8_t *ExtraCountersBegin() { return &__start___libfuzzer_extra_counters; } +uint8_t *ExtraCountersEnd() { return &__stop___libfuzzer_extra_counters; } +ATTRIBUTE_NO_SANITIZE_ALL +void ClearExtraCounters() { // hand-written memset, don't asan-ify. + uintptr_t *Beg = reinterpret_cast(ExtraCountersBegin()); + uintptr_t *End = reinterpret_cast(ExtraCountersEnd()); + for (; Beg < End; Beg++) { + *Beg = 0; + __asm__ __volatile__("" : : : "memory"); + } +} + +} // namespace fuzzer + +#else +// TODO: implement for other platforms. +namespace fuzzer { +uint8_t *ExtraCountersBegin() { return nullptr; } +uint8_t *ExtraCountersEnd() { return nullptr; } +void ClearExtraCounters() {} +} // namespace fuzzer + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerFlags.def b/tools/fuzzing/libfuzzer/FuzzerFlags.def new file mode 100644 index 0000000000..832224a705 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerFlags.def @@ -0,0 +1,169 @@ +//===- FuzzerFlags.def - Run-time flags -------------------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Flags. FUZZER_FLAG_INT/FUZZER_FLAG_STRING macros should be defined at the +// point of inclusion. We are not using any flag parsing library for better +// portability and independence. +//===----------------------------------------------------------------------===// +FUZZER_FLAG_INT(verbosity, 1, "Verbosity level.") +FUZZER_FLAG_UNSIGNED(seed, 0, "Random seed. If 0, seed is generated.") +FUZZER_FLAG_INT(runs, -1, + "Number of individual test runs (-1 for infinite runs).") +FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. " + "If 0, libFuzzer tries to guess a good value based on the corpus " + "and reports it. ") +FUZZER_FLAG_INT(len_control, 100, "Try generating small inputs first, " + "then try larger inputs over time. Specifies the rate at which the length " + "limit is increased (smaller == faster). If 0, immediately try inputs with " + "size up to max_len. Default value is 0, if LLVMFuzzerCustomMutator is used.") +FUZZER_FLAG_STRING(seed_inputs, "A comma-separated list of input files " + "to use as an additional seed corpus. Alternatively, an \"@\" followed by " + "the name of a file containing the comma-separated list.") +FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") +FUZZER_FLAG_INT(mutate_depth, 5, + "Apply this number of consecutive mutations to each input.") +FUZZER_FLAG_INT(reduce_depth, 0, "Experimental/internal. " + "Reduce depth if mutations lose unique features") +FUZZER_FLAG_INT(shuffle, 1, "Shuffle inputs at startup") +FUZZER_FLAG_INT(prefer_small, 1, + "If 1, always prefer smaller inputs during the corpus shuffle.") +FUZZER_FLAG_INT( + timeout, 1200, + "Timeout in seconds (if positive). " + "If one unit runs more than this number of seconds the process will abort.") +FUZZER_FLAG_INT(error_exitcode, 77, "When libFuzzer itself reports a bug " + "this exit code will be used.") +FUZZER_FLAG_INT(timeout_exitcode, 70, "When libFuzzer reports a timeout " + "this exit code will be used.") +FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " + "time in seconds to run the fuzzer.") +FUZZER_FLAG_INT(help, 0, "Print help.") +FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " + "in a subprocess") +FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") +FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode") +FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode") +FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " + "merged into the 1-st corpus. Only interesting units will be taken. " + "This flag can be used to minimize a corpus.") +FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists") +FUZZER_FLAG_STRING(merge_inner, "internal flag") +FUZZER_FLAG_STRING(merge_control_file, + "Specify a control file used for the merge process. " + "If a merge process gets killed it tries to leave this file " + "in a state suitable for resuming the merge. " + "By default a temporary file will be used." + "The same file can be used for multistep merge process.") +FUZZER_FLAG_INT(minimize_crash, 0, "If 1, minimizes the provided" + " crash input. Use with -runs=N or -max_total_time=N to limit " + "the number attempts." + " Use with -exact_artifact_path to specify the output." + " Combine with ASAN_OPTIONS=dedup_token_length=3 (or similar) to ensure that" + " the minimized input triggers the same crash." + ) +FUZZER_FLAG_INT(cleanse_crash, 0, "If 1, tries to cleanse the provided" + " crash input to make it contain fewer original bytes." + " Use with -exact_artifact_path to specify the output." + ) +FUZZER_FLAG_INT(minimize_crash_internal_step, 0, "internal flag") +FUZZER_FLAG_STRING(features_dir, "internal flag. Used to dump feature sets on disk." + "Every time a new input is added to the corpus, a corresponding file in the features_dir" + " is created containing the unique features of that input." + " Features are stored in binary format.") +FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters") +FUZZER_FLAG_INT(use_memmem, 1, + "Use hints from intercepting memmem, strstr, etc") +FUZZER_FLAG_INT(use_value_profile, 0, + "Experimental. Use value profile to guide fuzzing.") +FUZZER_FLAG_INT(use_cmp, 1, "Use CMP traces to guide mutations") +FUZZER_FLAG_INT(shrink, 0, "Experimental. Try to shrink corpus inputs.") +FUZZER_FLAG_INT(reduce_inputs, 1, + "Try to reduce the size of inputs while preserving their full feature sets") +FUZZER_FLAG_UNSIGNED(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn" + " this number of jobs in separate worker processes" + " with stdout/stderr redirected to fuzz-JOB.log.") +FUZZER_FLAG_UNSIGNED(workers, 0, + "Number of simultaneous worker processes to run the jobs." + " If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.") +FUZZER_FLAG_INT(reload, 1, + "Reload the main corpus every seconds to get new units" + " discovered by other processes. If 0, disabled") +FUZZER_FLAG_INT(report_slow_units, 10, + "Report slowest units if they run for more than this number of seconds.") +FUZZER_FLAG_INT(only_ascii, 0, + "If 1, generate only ASCII (isprint+isspace) inputs.") +FUZZER_FLAG_STRING(dict, "Experimental. Use the dictionary file.") +FUZZER_FLAG_STRING(artifact_prefix, "Write fuzzing artifacts (crash, " + "timeout, or slow inputs) as " + "$(artifact_prefix)file") +FUZZER_FLAG_STRING(exact_artifact_path, + "Write the single artifact on failure (crash, timeout) " + "as $(exact_artifact_path). This overrides -artifact_prefix " + "and will not use checksum in the file name. Do not " + "use the same path for several parallel processes.") +FUZZER_FLAG_INT(print_pcs, 0, "If 1, print out newly covered PCs.") +FUZZER_FLAG_INT(print_funcs, 2, "If >=1, print out at most this number of " + "newly covered functions.") +FUZZER_FLAG_INT(print_final_stats, 0, "If 1, print statistics at exit.") +FUZZER_FLAG_INT(print_corpus_stats, 0, + "If 1, print statistics on corpus elements at exit.") +FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" + " at exit.") +FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated.") +FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") +FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") +FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.") +FUZZER_FLAG_INT(handle_ill, 1, "If 1, try to intercept SIGILL.") +FUZZER_FLAG_INT(handle_fpe, 1, "If 1, try to intercept SIGFPE.") +FUZZER_FLAG_INT(handle_int, 1, "If 1, try to intercept SIGINT.") +FUZZER_FLAG_INT(handle_term, 1, "If 1, try to intercept SIGTERM.") +FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.") +FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.") +FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.") +FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; " + "if 2, close stderr; if 3, close both. " + "Be careful, this will also close e.g. stderr of asan.") +FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled " + "try to detect memory leaks during fuzzing (i.e. not only at shut down).") +FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and " + "quarantines every seconds. When rss_limit_mb is specified (>0), " + "purging starts when RSS exceeds 50% of rss_limit_mb. Pass " + "purge_allocator_interval=-1 to disable this functionality.") +FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. " + "If >= 2 will also print stack traces.") +FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon" + "reaching this limit of RSS memory usage.") +FUZZER_FLAG_INT(malloc_limit_mb, 0, "If non-zero, the fuzzer will exit " + "if the target tries to allocate this number of Mb with one malloc call. " + "If zero (default) same limit as rss_limit_mb is applied.") +FUZZER_FLAG_STRING(exit_on_src_pos, "Exit if a newly found PC originates" + " from the given source location. Example: -exit_on_src_pos=foo.cc:123. " + "Used primarily for testing libFuzzer itself.") +FUZZER_FLAG_STRING(exit_on_item, "Exit if an item with a given sha1 sum" + " was added to the corpus. " + "Used primarily for testing libFuzzer itself.") +FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " + "after this one. Useful for fuzzers that need to do their own " + "argument parsing.") +FUZZER_FLAG_STRING(focus_function, "Experimental. " + "Fuzzing will focus on inputs that trigger calls to this function. " + "If -focus_function=auto and -data_flow_trace is used, libFuzzer " + "will choose the focus functions automatically.") +FUZZER_FLAG_INT(entropic, 0, "Experimental. Enables entropic power schedule.") +FUZZER_FLAG_INT(entropic_feature_frequency_threshold, 0xFF, "Experimental. If " + "entropic is enabled, all features which are observed less often than " + "the specified value are considered as rare.") +FUZZER_FLAG_INT(entropic_number_of_rarest_features, 100, "Experimental. If " + "entropic is enabled, we keep track of the frequencies only for the " + "Top-X least abundant features (union features that are considered as " + "rare).") + +FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") +FUZZER_DEPRECATED_FLAG(use_clang_coverage) +FUZZER_FLAG_STRING(data_flow_trace, "Experimental: use the data flow trace") +FUZZER_FLAG_STRING(collect_data_flow, + "Experimental: collect the data flow trace") diff --git a/tools/fuzzing/libfuzzer/FuzzerFork.cpp b/tools/fuzzing/libfuzzer/FuzzerFork.cpp new file mode 100644 index 0000000000..ee2a99a250 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerFork.cpp @@ -0,0 +1,427 @@ +//===- FuzzerFork.cpp - run fuzzing in separate subprocesses --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Spawn and orchestrate separate fuzzing processes. +//===----------------------------------------------------------------------===// + +#include "FuzzerCommand.h" +#include "FuzzerFork.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerMerge.h" +#include "FuzzerSHA1.h" +#include "FuzzerTracePC.h" +#include "FuzzerUtil.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fuzzer { + +struct Stats { + size_t number_of_executed_units = 0; + size_t peak_rss_mb = 0; + size_t average_exec_per_sec = 0; +}; + +static Stats ParseFinalStatsFromLog(const std::string &LogPath) { + std::ifstream In(LogPath); + std::string Line; + Stats Res; + struct { + const char *Name; + size_t *Var; + } NameVarPairs[] = { + {"stat::number_of_executed_units:", &Res.number_of_executed_units}, + {"stat::peak_rss_mb:", &Res.peak_rss_mb}, + {"stat::average_exec_per_sec:", &Res.average_exec_per_sec}, + {nullptr, nullptr}, + }; + while (std::getline(In, Line, '\n')) { + if (Line.find("stat::") != 0) continue; + std::istringstream ISS(Line); + std::string Name; + size_t Val; + ISS >> Name >> Val; + for (size_t i = 0; NameVarPairs[i].Name; i++) + if (Name == NameVarPairs[i].Name) + *NameVarPairs[i].Var = Val; + } + return Res; +} + +struct FuzzJob { + // Inputs. + Command Cmd; + std::string CorpusDir; + std::string FeaturesDir; + std::string LogPath; + std::string SeedListPath; + std::string CFPath; + size_t JobId; + + int DftTimeInSeconds = 0; + + // Fuzzing Outputs. + int ExitCode; + + ~FuzzJob() { + RemoveFile(CFPath); + RemoveFile(LogPath); + RemoveFile(SeedListPath); + RmDirRecursive(CorpusDir); + RmDirRecursive(FeaturesDir); + } +}; + +struct GlobalEnv { + Vector Args; + Vector CorpusDirs; + std::string MainCorpusDir; + std::string TempDir; + std::string DFTDir; + std::string DataFlowBinary; + Set Features, Cov; + Set FilesWithDFT; + Vector Files; + Random *Rand; + std::chrono::system_clock::time_point ProcessStartTime; + int Verbosity = 0; + + size_t NumTimeouts = 0; + size_t NumOOMs = 0; + size_t NumCrashes = 0; + + + size_t NumRuns = 0; + + std::string StopFile() { return DirPlusFile(TempDir, "STOP"); } + + size_t secondsSinceProcessStartUp() const { + return std::chrono::duration_cast( + std::chrono::system_clock::now() - ProcessStartTime) + .count(); + } + + FuzzJob *CreateNewJob(size_t JobId) { + Command Cmd(Args); + Cmd.removeFlag("fork"); + Cmd.removeFlag("runs"); + Cmd.removeFlag("collect_data_flow"); + for (auto &C : CorpusDirs) // Remove all corpora from the args. + Cmd.removeArgument(C); + Cmd.addFlag("reload", "0"); // working in an isolated dir, no reload. + Cmd.addFlag("print_final_stats", "1"); + Cmd.addFlag("print_funcs", "0"); // no need to spend time symbolizing. + Cmd.addFlag("max_total_time", std::to_string(std::min((size_t)300, JobId))); + Cmd.addFlag("stop_file", StopFile()); + if (!DataFlowBinary.empty()) { + Cmd.addFlag("data_flow_trace", DFTDir); + if (!Cmd.hasFlag("focus_function")) + Cmd.addFlag("focus_function", "auto"); + } + auto Job = new FuzzJob; + std::string Seeds; + if (size_t CorpusSubsetSize = + std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { + auto Time1 = std::chrono::system_clock::now(); + for (size_t i = 0; i < CorpusSubsetSize; i++) { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + auto Time2 = std::chrono::system_clock::now(); + Job->DftTimeInSeconds = duration_cast(Time2 - Time1).count(); + } + if (!Seeds.empty()) { + Job->SeedListPath = + DirPlusFile(TempDir, std::to_string(JobId) + ".seeds"); + WriteToFile(Seeds, Job->SeedListPath); + Cmd.addFlag("seed_inputs", "@" + Job->SeedListPath); + } + Job->LogPath = DirPlusFile(TempDir, std::to_string(JobId) + ".log"); + Job->CorpusDir = DirPlusFile(TempDir, "C" + std::to_string(JobId)); + Job->FeaturesDir = DirPlusFile(TempDir, "F" + std::to_string(JobId)); + Job->CFPath = DirPlusFile(TempDir, std::to_string(JobId) + ".merge"); + Job->JobId = JobId; + + + Cmd.addArgument(Job->CorpusDir); + Cmd.addFlag("features_dir", Job->FeaturesDir); + + for (auto &D : {Job->CorpusDir, Job->FeaturesDir}) { + RmDirRecursive(D); + MkDir(D); + } + + Cmd.setOutputFile(Job->LogPath); + Cmd.combineOutAndErr(); + + Job->Cmd = Cmd; + + if (Verbosity >= 2) + Printf("Job %zd/%p Created: %s\n", JobId, Job, + Job->Cmd.toString().c_str()); + // Start from very short runs and gradually increase them. + return Job; + } + + int RunOneMergeJob(FuzzJob *Job) { + auto Stats = ParseFinalStatsFromLog(Job->LogPath); + NumRuns += Stats.number_of_executed_units; + + Vector TempFiles, MergeCandidates; + // Read all newly created inputs and their feature sets. + // Choose only those inputs that have new features. + int Res = GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); + if (Res != 0) + return Res; + std::sort(TempFiles.begin(), TempFiles.end()); + for (auto &F : TempFiles) { + auto FeatureFile = F.File; + FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir); + auto FeatureBytes = FileToVector(FeatureFile, 0, false); + assert((FeatureBytes.size() % sizeof(uint32_t)) == 0); + Vector NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); + memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size()); + for (auto Ft : NewFeatures) { + if (!Features.count(Ft)) { + MergeCandidates.push_back(F); + break; + } + } + } + // if (!FilesToAdd.empty() || Job->ExitCode != 0) + Printf("#%zd: cov: %zd ft: %zd corp: %zd exec/s %zd " + "oom/timeout/crash: %zd/%zd/%zd time: %zds job: %zd dft_time: %d\n", + NumRuns, Cov.size(), Features.size(), Files.size(), + Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, + secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds); + + if (MergeCandidates.empty()) return 0; + + Vector FilesToAdd; + Set NewFeatures, NewCov; + CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, + &NewFeatures, Cov, &NewCov, Job->CFPath, false); + if (Fuzzer::isGracefulExitRequested()) + return 0; + for (auto &Path : FilesToAdd) { + auto U = FileToVector(Path); + auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); + WriteToFile(U, NewPath); + Files.push_back(NewPath); + } + Features.insert(NewFeatures.begin(), NewFeatures.end()); + Cov.insert(NewCov.begin(), NewCov.end()); + for (auto Idx : NewCov) + if (auto *TE = TPC.PCTableEntryByIdx(Idx)) + if (TPC.PcIsFuncEntry(TE)) + PrintPC(" NEW_FUNC: %p %F %L\n", "", + TPC.GetNextInstructionPc(TE->PC)); + return 0; + } + + + void CollectDFT(const std::string &InputPath) { + if (DataFlowBinary.empty()) return; + if (!FilesWithDFT.insert(InputPath).second) return; + Command Cmd(Args); + Cmd.removeFlag("fork"); + Cmd.removeFlag("runs"); + Cmd.addFlag("data_flow_trace", DFTDir); + Cmd.addArgument(InputPath); + for (auto &C : CorpusDirs) // Remove all corpora from the args. + Cmd.removeArgument(C); + Cmd.setOutputFile(DirPlusFile(TempDir, "dft.log")); + Cmd.combineOutAndErr(); + // Printf("CollectDFT: %s\n", Cmd.toString().c_str()); + ExecuteCommand(Cmd); + } + +}; + +struct JobQueue { + std::queue Qu; + std::mutex Mu; + std::condition_variable Cv; + + void Push(FuzzJob *Job) { + { + std::lock_guard Lock(Mu); + Qu.push(Job); + } + Cv.notify_one(); + } + FuzzJob *Pop() { + std::unique_lock Lk(Mu); + // std::lock_guard Lock(Mu); + Cv.wait(Lk, [&]{return !Qu.empty();}); + assert(!Qu.empty()); + auto Job = Qu.front(); + Qu.pop(); + return Job; + } +}; + +void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { + while (auto Job = FuzzQ->Pop()) { + // Printf("WorkerThread: job %p\n", Job); + Job->ExitCode = ExecuteCommand(Job->Cmd); + MergeQ->Push(Job); + } +} + +// This is just a skeleton of an experimental -fork=1 feature. +int FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector &Args, + const Vector &CorpusDirs, int NumJobs) { + Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); + + GlobalEnv Env; + Env.Args = Args; + Env.CorpusDirs = CorpusDirs; + Env.Rand = &Rand; + Env.Verbosity = Options.Verbosity; + Env.ProcessStartTime = std::chrono::system_clock::now(); + Env.DataFlowBinary = Options.CollectDataFlow; + + Vector SeedFiles; + int Res; + for (auto &Dir : CorpusDirs) { + Res = GetSizedFilesFromDir(Dir, &SeedFiles); + if (Res != 0) + return Res; + } + std::sort(SeedFiles.begin(), SeedFiles.end()); + Env.TempDir = TempPath("FuzzWithFork", ".dir"); + Env.DFTDir = DirPlusFile(Env.TempDir, "DFT"); + RmDirRecursive(Env.TempDir); // in case there is a leftover from old runs. + MkDir(Env.TempDir); + MkDir(Env.DFTDir); + + + if (CorpusDirs.empty()) + MkDir(Env.MainCorpusDir = DirPlusFile(Env.TempDir, "C")); + else + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + Res = CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, + {}, &Env.Cov, + CFPath, false); + if (Res != 0) + return Res; + if (Fuzzer::isGracefulExitRequested()) + return 0; + + RemoveFile(CFPath); + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, + Env.Files.size(), Env.TempDir.c_str()); + + int ExitCode = 0; + + JobQueue FuzzQ, MergeQ; + + auto StopJobs = [&]() { + for (int i = 0; i < NumJobs; i++) + FuzzQ.Push(nullptr); + MergeQ.Push(nullptr); + WriteToFile(Unit({1}), Env.StopFile()); + }; + + size_t JobId = 1; + Vector Threads; + for (int t = 0; t < NumJobs; t++) { + Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + + while (true) { + std::unique_ptr Job(MergeQ.Pop()); + if (!Job) + break; + ExitCode = Job->ExitCode; + if (ExitCode == Options.InterruptExitCode) { + Printf("==%lu== libFuzzer: a child was interrupted; exiting\n", GetPid()); + StopJobs(); + break; + } + if (Fuzzer::MaybeExitGracefully()) + return 0; + + Res = Env.RunOneMergeJob(Job.get()); + if (Res != 0) + return Res; + if (Fuzzer::isGracefulExitRequested()) + return 0; + + // Continue if our crash is one of the ignorred ones. + if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) + Env.NumTimeouts++; + else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode) + Env.NumOOMs++; + else if (ExitCode != 0) { + Env.NumCrashes++; + if (Options.IgnoreCrashes) { + std::ifstream In(Job->LogPath); + std::string Line; + while (std::getline(In, Line, '\n')) + if (Line.find("ERROR:") != Line.npos || + Line.find("runtime error:") != Line.npos) + Printf("%s\n", Line.c_str()); + } else { + // And exit if we don't ignore this crash. + Printf("INFO: log from the inner process:\n%s", + FileToString(Job->LogPath).c_str()); + StopJobs(); + break; + } + } + + // Stop if we are over the time budget. + // This is not precise, since other threads are still running + // and we will wait while joining them. + // We also don't stop instantly: other jobs need to finish. + if (Options.MaxTotalTimeSec > 0 && + Env.secondsSinceProcessStartUp() >= (size_t)Options.MaxTotalTimeSec) { + Printf("INFO: fuzzed for %zd seconds, wrapping up soon\n", + Env.secondsSinceProcessStartUp()); + StopJobs(); + break; + } + if (Env.NumRuns >= Options.MaxNumberOfRuns) { + Printf("INFO: fuzzed for %zd iterations, wrapping up soon\n", + Env.NumRuns); + StopJobs(); + break; + } + + FuzzQ.Push(Env.CreateNewJob(JobId++)); + } + + for (auto &T : Threads) + T.join(); + + // The workers have terminated. Don't try to remove the directory before they + // terminate to avoid a race condition preventing cleanup on Windows. + RmDirRecursive(Env.TempDir); + + // Use the exit code from the last child process. + Printf("INFO: exiting: %d time: %zds\n", ExitCode, + Env.secondsSinceProcessStartUp()); + return ExitCode; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerFork.h b/tools/fuzzing/libfuzzer/FuzzerFork.h new file mode 100644 index 0000000000..1352171ad4 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerFork.h @@ -0,0 +1,24 @@ +//===- FuzzerFork.h - run fuzzing in sub-processes --------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_FORK_H +#define LLVM_FUZZER_FORK_H + +#include "FuzzerDefs.h" +#include "FuzzerOptions.h" +#include "FuzzerRandom.h" + +#include + +namespace fuzzer { +int FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector &Args, + const Vector &CorpusDirs, int NumJobs); +} // namespace fuzzer + +#endif // LLVM_FUZZER_FORK_H diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.cpp b/tools/fuzzing/libfuzzer/FuzzerIO.cpp new file mode 100644 index 0000000000..6be2be67c6 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerIO.cpp @@ -0,0 +1,165 @@ +//===- FuzzerIO.cpp - IO utils. -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// IO functions. +//===----------------------------------------------------------------------===// + +#include "mozilla/Unused.h" +#include "FuzzerDefs.h" +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "FuzzerUtil.h" +#include +#include +#include +#include +#include +#include + +namespace fuzzer { + +static FILE *OutputFile = stderr; + +long GetEpoch(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return 0; // Can't stat, be conservative. + return St.st_mtime; +} + +Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { + std::ifstream T(Path, std::ios::binary); + if (ExitOnError && !T) { + Printf("No such directory: %s; exiting\n", Path.c_str()); + exit(1); + } + + T.seekg(0, T.end); + auto EndPos = T.tellg(); + if (EndPos < 0) return {}; + size_t FileLen = EndPos; + if (MaxSize) + FileLen = std::min(FileLen, MaxSize); + + T.seekg(0, T.beg); + Unit Res(FileLen); + T.read(reinterpret_cast(Res.data()), FileLen); + return Res; +} + +std::string FileToString(const std::string &Path) { + std::ifstream T(Path, std::ios::binary); + return std::string((std::istreambuf_iterator(T)), + std::istreambuf_iterator()); +} + +void CopyFileToErr(const std::string &Path) { + Printf("%s", FileToString(Path).c_str()); +} + +void WriteToFile(const Unit &U, const std::string &Path) { + WriteToFile(U.data(), U.size(), Path); +} + +void WriteToFile(const std::string &Data, const std::string &Path) { + WriteToFile(reinterpret_cast(Data.c_str()), Data.size(), + Path); +} + +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path) { + // Use raw C interface because this function may be called from a sig handler. + FILE *Out = fopen(Path.c_str(), "wb"); + if (!Out) return; + mozilla::Unused << fwrite(Data, sizeof(Data[0]), Size, Out); + fclose(Out); +} + +void ReadDirToVectorOfUnits(const char *Path, Vector *V, + long *Epoch, size_t MaxSize, bool ExitOnError) { + long E = Epoch ? *Epoch : 0; + Vector Files; + int Res = ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); + if (ExitOnError && Res != 0) + exit(Res); + size_t NumLoaded = 0; + for (size_t i = 0; i < Files.size(); i++) { + auto &X = Files[i]; + if (Epoch && GetEpoch(X) < E) continue; + NumLoaded++; + if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024) + Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path); + auto S = FileToVector(X, MaxSize, ExitOnError); + if (!S.empty()) + V->push_back(S); + } +} + + +int GetSizedFilesFromDir(const std::string &Dir, Vector *V) { + Vector Files; + int Res = ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); + if (Res != 0) + return Res; + for (auto &File : Files) + if (size_t Size = FileSize(File)) + V->push_back({File, Size}); + return 0; +} + +std::string DirPlusFile(const std::string &DirPath, + const std::string &FileName) { + return DirPath + GetSeparator() + FileName; +} + +void DupAndCloseStderr() { + int OutputFd = DuplicateFile(2); + if (OutputFd >= 0) { + FILE *NewOutputFile = OpenFile(OutputFd, "w"); + if (NewOutputFile) { + OutputFile = NewOutputFile; + if (EF->__sanitizer_set_report_fd) + EF->__sanitizer_set_report_fd( + reinterpret_cast(GetHandleFromFd(OutputFd))); + DiscardOutput(2); + } + } +} + +void CloseStdout() { + DiscardOutput(1); +} + +void Printf(const char *Fmt, ...) { + va_list ap; + va_start(ap, Fmt); + vfprintf(OutputFile, Fmt, ap); + va_end(ap); + fflush(OutputFile); +} + +void VPrintf(bool Verbose, const char *Fmt, ...) { + if (!Verbose) return; + va_list ap; + va_start(ap, Fmt); + vfprintf(OutputFile, Fmt, ap); + va_end(ap); + fflush(OutputFile); +} + +void RmDirRecursive(const std::string &Dir) { + IterateDirRecursive( + Dir, [](const std::string &Path) {}, + [](const std::string &Path) { RmDir(Path); }, + [](const std::string &Path) { RemoveFile(Path); }); +} + +std::string TempPath(const char *Prefix, const char *Extension) { + return DirPlusFile(TmpDir(), std::string("libFuzzerTemp.") + Prefix + + std::to_string(GetPid()) + Extension); +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.h b/tools/fuzzing/libfuzzer/FuzzerIO.h new file mode 100644 index 0000000000..6c90ba6373 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerIO.h @@ -0,0 +1,106 @@ +//===- FuzzerIO.h - Internal header for IO utils ----------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// IO interface. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_IO_H +#define LLVM_FUZZER_IO_H + +#include "FuzzerDefs.h" + +namespace fuzzer { + +long GetEpoch(const std::string &Path); + +Unit FileToVector(const std::string &Path, size_t MaxSize = 0, + bool ExitOnError = true); + +std::string FileToString(const std::string &Path); + +void CopyFileToErr(const std::string &Path); + +void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path); +// Write Data.c_str() to the file without terminating null character. +void WriteToFile(const std::string &Data, const std::string &Path); +void WriteToFile(const Unit &U, const std::string &Path); + +void ReadDirToVectorOfUnits(const char *Path, Vector *V, + long *Epoch, size_t MaxSize, bool ExitOnError); + +// Returns "Dir/FileName" or equivalent for the current OS. +std::string DirPlusFile(const std::string &DirPath, + const std::string &FileName); + +// Returns the name of the dir, similar to the 'dirname' utility. +std::string DirName(const std::string &FileName); + +// Returns path to a TmpDir. +std::string TmpDir(); + +std::string TempPath(const char *Prefix, const char *Extension); + +bool IsInterestingCoverageFile(const std::string &FileName); + +void DupAndCloseStderr(); + +void CloseStdout(); + +void Printf(const char *Fmt, ...); +void VPrintf(bool Verbose, const char *Fmt, ...); + +// Print using raw syscalls, useful when printing at early init stages. +void RawPrint(const char *Str); + +// Platform specific functions: +bool IsFile(const std::string &Path); +size_t FileSize(const std::string &Path); + +int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector *V, bool TopDir); + +void RmDirRecursive(const std::string &Dir); + +// Iterate files and dirs inside Dir, recursively. +// Call DirPreCallback/DirPostCallback on dirs before/after +// calling FileCallback on files. +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)); + +struct SizedFile { + std::string File; + size_t Size; + bool operator<(const SizedFile &B) const { return Size < B.Size; } +}; + +int GetSizedFilesFromDir(const std::string &Dir, Vector *V); + +char GetSeparator(); +// Similar to the basename utility: returns the file name w/o the dir prefix. +std::string Basename(const std::string &Path); + +FILE* OpenFile(int Fd, const char *Mode); + +int CloseFile(int Fd); + +int DuplicateFile(int Fd); + +void RemoveFile(const std::string &Path); +void RenameFile(const std::string &OldPath, const std::string &NewPath); + +intptr_t GetHandleFromFd(int fd); + +void MkDir(const std::string &Path); +void RmDir(const std::string &Path); + +const std::string &getDevNull(); + +} // namespace fuzzer + +#endif // LLVM_FUZZER_IO_H diff --git a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp new file mode 100644 index 0000000000..1a50295c01 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp @@ -0,0 +1,181 @@ +//===- FuzzerIOPosix.cpp - IO utils for Posix. ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// IO functions implementation using Posix API. +//===----------------------------------------------------------------------===// +#include "mozilla/Unused.h" +#include "FuzzerPlatform.h" +#if LIBFUZZER_POSIX || LIBFUZZER_FUCHSIA + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fuzzer { + +bool IsFile(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return false; + return S_ISREG(St.st_mode); +} + +static bool IsDirectory(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return false; + return S_ISDIR(St.st_mode); +} + +size_t FileSize(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return 0; + return St.st_size; +} + +std::string Basename(const std::string &Path) { + size_t Pos = Path.rfind(GetSeparator()); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + +int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector *V, bool TopDir) { + auto E = GetEpoch(Dir); + if (Epoch) + if (E && *Epoch >= E) return 0; + + DIR *D = opendir(Dir.c_str()); + if (!D) { + Printf("%s: %s; exiting\n", strerror(errno), Dir.c_str()); + return 1; + } + while (auto E = readdir(D)) { + std::string Path = DirPlusFile(Dir, E->d_name); + if (E->d_type == DT_REG || E->d_type == DT_LNK || + (E->d_type == DT_UNKNOWN && IsFile(Path))) + V->push_back(Path); + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && + *E->d_name != '.') { + int Res = ListFilesInDirRecursive(Path, Epoch, V, false); + if (Res != 0) + return Res; + } + } + closedir(D); + if (Epoch && TopDir) + *Epoch = E; + return 0; +} + + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + DirPreCallback(Dir); + DIR *D = opendir(Dir.c_str()); + if (!D) return; + while (auto E = readdir(D)) { + std::string Path = DirPlusFile(Dir, E->d_name); + if (E->d_type == DT_REG || E->d_type == DT_LNK || + (E->d_type == DT_UNKNOWN && IsFile(Path))) + FileCallback(Path); + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && + *E->d_name != '.') + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } + closedir(D); + DirPostCallback(Dir); +} + +char GetSeparator() { + return '/'; +} + +FILE* OpenFile(int Fd, const char* Mode) { + return fdopen(Fd, Mode); +} + +int CloseFile(int fd) { + return close(fd); +} + +int DuplicateFile(int Fd) { + return dup(Fd); +} + +void RemoveFile(const std::string &Path) { + unlink(Path.c_str()); +} + +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + +intptr_t GetHandleFromFd(int fd) { + return static_cast(fd); +} + +std::string DirName(const std::string &FileName) { + char *Tmp = new char[FileName.size() + 1]; + memcpy(Tmp, FileName.c_str(), FileName.size() + 1); + std::string Res = dirname(Tmp); + delete [] Tmp; + return Res; +} + +std::string TmpDir() { + if (auto Env = getenv("TMPDIR")) + return Env; + return "/tmp"; +} + +bool IsInterestingCoverageFile(const std::string &FileName) { + if (FileName.find("compiler-rt/lib/") != std::string::npos) + return false; // sanitizer internal. + if (FileName.find("/usr/lib/") != std::string::npos) + return false; + if (FileName.find("/usr/include/") != std::string::npos) + return false; + if (FileName == "") + return false; + return true; +} + +void RawPrint(const char *Str) { + mozilla::Unused << write(2, Str, strlen(Str)); +} + +void MkDir(const std::string &Path) { + mkdir(Path.c_str(), 0700); +} + +void RmDir(const std::string &Path) { + rmdir(Path.c_str()); +} + +const std::string &getDevNull() { + static const std::string devNull = "/dev/null"; + return devNull; +} + +} // namespace fuzzer + +#endif // LIBFUZZER_POSIX diff --git a/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp b/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp new file mode 100644 index 0000000000..0e977bd025 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp @@ -0,0 +1,417 @@ +//===- FuzzerIOWindows.cpp - IO utils for Windows. ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// IO functions implementation for Windows. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_WINDOWS + +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fuzzer { + +static bool IsFile(const std::string &Path, const DWORD &FileAttributes) { + + if (FileAttributes & FILE_ATTRIBUTE_NORMAL) + return true; + + if (FileAttributes & FILE_ATTRIBUTE_DIRECTORY) + return false; + + HANDLE FileHandle( + CreateFileA(Path.c_str(), 0, FILE_SHARE_READ, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, 0)); + + if (FileHandle == INVALID_HANDLE_VALUE) { + Printf("CreateFileA() failed for \"%s\" (Error code: %lu).\n", Path.c_str(), + GetLastError()); + return false; + } + + DWORD FileType = GetFileType(FileHandle); + + if (FileType == FILE_TYPE_UNKNOWN) { + Printf("GetFileType() failed for \"%s\" (Error code: %lu).\n", Path.c_str(), + GetLastError()); + CloseHandle(FileHandle); + return false; + } + + if (FileType != FILE_TYPE_DISK) { + CloseHandle(FileHandle); + return false; + } + + CloseHandle(FileHandle); + return true; +} + +bool IsFile(const std::string &Path) { + DWORD Att = GetFileAttributesA(Path.c_str()); + + if (Att == INVALID_FILE_ATTRIBUTES) { + Printf("GetFileAttributesA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), GetLastError()); + return false; + } + + return IsFile(Path, Att); +} + +static bool IsDir(DWORD FileAttrs) { + if (FileAttrs == INVALID_FILE_ATTRIBUTES) return false; + return FileAttrs & FILE_ATTRIBUTE_DIRECTORY; +} + +std::string Basename(const std::string &Path) { + size_t Pos = Path.find_last_of("/\\"); + if (Pos == std::string::npos) return Path; + assert(Pos < Path.size()); + return Path.substr(Pos + 1); +} + +size_t FileSize(const std::string &Path) { + WIN32_FILE_ATTRIBUTE_DATA attr; + if (!GetFileAttributesExA(Path.c_str(), GetFileExInfoStandard, &attr)) { + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) + Printf("GetFileAttributesExA() failed for \"%s\" (Error code: %lu).\n", + Path.c_str(), LastError); + return 0; + } + ULARGE_INTEGER size; + size.HighPart = attr.nFileSizeHigh; + size.LowPart = attr.nFileSizeLow; + return size.QuadPart; +} + +int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector *V, bool TopDir) { + int Res; + auto E = GetEpoch(Dir); + if (Epoch) + if (E && *Epoch >= E) return 0; + + std::string Path(Dir); + assert(!Path.empty()); + if (Path.back() != '\\') + Path.push_back('\\'); + Path.push_back('*'); + + // Get the first directory entry. + WIN32_FIND_DATAA FindInfo; + HANDLE FindHandle(FindFirstFileA(Path.c_str(), &FindInfo)); + if (FindHandle == INVALID_HANDLE_VALUE) + { + if (GetLastError() == ERROR_FILE_NOT_FOUND) + return 0; + Printf("No such file or directory: %s; exiting\n", Dir.c_str()); + return 1; + } + + do { + std::string FileName = DirPlusFile(Dir, FindInfo.cFileName); + + if (FindInfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + size_t FilenameLen = strlen(FindInfo.cFileName); + if ((FilenameLen == 1 && FindInfo.cFileName[0] == '.') || + (FilenameLen == 2 && FindInfo.cFileName[0] == '.' && + FindInfo.cFileName[1] == '.')) + continue; + + int Res = ListFilesInDirRecursive(FileName, Epoch, V, false); + if (Res != 0) + return Res; + } + else if (IsFile(FileName, FindInfo.dwFileAttributes)) + V->push_back(FileName); + } while (FindNextFileA(FindHandle, &FindInfo)); + + DWORD LastError = GetLastError(); + if (LastError != ERROR_NO_MORE_FILES) + Printf("FindNextFileA failed (Error code: %lu).\n", LastError); + + FindClose(FindHandle); + + if (Epoch && TopDir) + *Epoch = E; + return 0; +} + + +void IterateDirRecursive(const std::string &Dir, + void (*DirPreCallback)(const std::string &Dir), + void (*DirPostCallback)(const std::string &Dir), + void (*FileCallback)(const std::string &Dir)) { + // TODO(metzman): Implement ListFilesInDirRecursive via this function. + DirPreCallback(Dir); + + DWORD DirAttrs = GetFileAttributesA(Dir.c_str()); + if (!IsDir(DirAttrs)) return; + + std::string TargetDir(Dir); + assert(!TargetDir.empty()); + if (TargetDir.back() != '\\') TargetDir.push_back('\\'); + TargetDir.push_back('*'); + + WIN32_FIND_DATAA FindInfo; + // Find the directory's first file. + HANDLE FindHandle = FindFirstFileA(TargetDir.c_str(), &FindInfo); + if (FindHandle == INVALID_HANDLE_VALUE) { + DWORD LastError = GetLastError(); + if (LastError != ERROR_FILE_NOT_FOUND) { + // If the directory isn't empty, then something abnormal is going on. + Printf("FindFirstFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + } + return; + } + + do { + std::string Path = DirPlusFile(Dir, FindInfo.cFileName); + DWORD PathAttrs = FindInfo.dwFileAttributes; + if (IsDir(PathAttrs)) { + // Is Path the current directory (".") or the parent ("..")? + if (strcmp(FindInfo.cFileName, ".") == 0 || + strcmp(FindInfo.cFileName, "..") == 0) + continue; + IterateDirRecursive(Path, DirPreCallback, DirPostCallback, FileCallback); + } else if (PathAttrs != INVALID_FILE_ATTRIBUTES) { + FileCallback(Path); + } + } while (FindNextFileA(FindHandle, &FindInfo)); + + DWORD LastError = GetLastError(); + if (LastError != ERROR_NO_MORE_FILES) + Printf("FindNextFileA failed for %s (Error code: %lu).\n", Dir.c_str(), + LastError); + + FindClose(FindHandle); + DirPostCallback(Dir); +} + +char GetSeparator() { + return '\\'; +} + +FILE* OpenFile(int Fd, const char* Mode) { + return _fdopen(Fd, Mode); +} + +int CloseFile(int Fd) { + return _close(Fd); +} + +int DuplicateFile(int Fd) { + return _dup(Fd); +} + +void RemoveFile(const std::string &Path) { + _unlink(Path.c_str()); +} + +void RenameFile(const std::string &OldPath, const std::string &NewPath) { + rename(OldPath.c_str(), NewPath.c_str()); +} + +intptr_t GetHandleFromFd(int fd) { + return _get_osfhandle(fd); +} + +static bool IsSeparator(char C) { + return C == '\\' || C == '/'; +} + +// Parse disk designators, like "C:\". If Relative == true, also accepts: "C:". +// Returns number of characters considered if successful. +static size_t ParseDrive(const std::string &FileName, const size_t Offset, + bool Relative = true) { + if (Offset + 1 >= FileName.size() || FileName[Offset + 1] != ':') + return 0; + if (Offset + 2 >= FileName.size() || !IsSeparator(FileName[Offset + 2])) { + if (!Relative) // Accept relative path? + return 0; + else + return 2; + } + return 3; +} + +// Parse a file name, like: SomeFile.txt +// Returns number of characters considered if successful. +static size_t ParseFileName(const std::string &FileName, const size_t Offset) { + size_t Pos = Offset; + const size_t End = FileName.size(); + for(; Pos < End && !IsSeparator(FileName[Pos]); ++Pos) + ; + return Pos - Offset; +} + +// Parse a directory ending in separator, like: `SomeDir\` +// Returns number of characters considered if successful. +static size_t ParseDir(const std::string &FileName, const size_t Offset) { + size_t Pos = Offset; + const size_t End = FileName.size(); + if (Pos >= End || IsSeparator(FileName[Pos])) + return 0; + for(; Pos < End && !IsSeparator(FileName[Pos]); ++Pos) + ; + if (Pos >= End) + return 0; + ++Pos; // Include separator. + return Pos - Offset; +} + +// Parse a servername and share, like: `SomeServer\SomeShare\` +// Returns number of characters considered if successful. +static size_t ParseServerAndShare(const std::string &FileName, + const size_t Offset) { + size_t Pos = Offset, Res; + if (!(Res = ParseDir(FileName, Pos))) + return 0; + Pos += Res; + if (!(Res = ParseDir(FileName, Pos))) + return 0; + Pos += Res; + return Pos - Offset; +} + +// Parse the given Ref string from the position Offset, to exactly match the given +// string Patt. +// Returns number of characters considered if successful. +static size_t ParseCustomString(const std::string &Ref, size_t Offset, + const char *Patt) { + size_t Len = strlen(Patt); + if (Offset + Len > Ref.size()) + return 0; + return Ref.compare(Offset, Len, Patt) == 0 ? Len : 0; +} + +// Parse a location, like: +// \\?\UNC\Server\Share\ \\?\C:\ \\Server\Share\ \ C:\ C: +// Returns number of characters considered if successful. +static size_t ParseLocation(const std::string &FileName) { + size_t Pos = 0, Res; + + if ((Res = ParseCustomString(FileName, Pos, R"(\\?\)"))) { + Pos += Res; + if ((Res = ParseCustomString(FileName, Pos, R"(UNC\)"))) { + Pos += Res; + if ((Res = ParseServerAndShare(FileName, Pos))) + return Pos + Res; + return 0; + } + if ((Res = ParseDrive(FileName, Pos, false))) + return Pos + Res; + return 0; + } + + if (Pos < FileName.size() && IsSeparator(FileName[Pos])) { + ++Pos; + if (Pos < FileName.size() && IsSeparator(FileName[Pos])) { + ++Pos; + if ((Res = ParseServerAndShare(FileName, Pos))) + return Pos + Res; + return 0; + } + return Pos; + } + + if ((Res = ParseDrive(FileName, Pos))) + return Pos + Res; + + return Pos; +} + +std::string DirName(const std::string &FileName) { + size_t LocationLen = ParseLocation(FileName); + size_t DirLen = 0, Res; + while ((Res = ParseDir(FileName, LocationLen + DirLen))) + DirLen += Res; + size_t FileLen = ParseFileName(FileName, LocationLen + DirLen); + + if (LocationLen + DirLen + FileLen != FileName.size()) { + Printf("DirName() failed for \"%s\", invalid path.\n", FileName.c_str()); + exit(1); + } + + if (DirLen) { + --DirLen; // Remove trailing separator. + if (!FileLen) { // Path ended in separator. + assert(DirLen); + // Remove file name from Dir. + while (DirLen && !IsSeparator(FileName[LocationLen + DirLen - 1])) + --DirLen; + if (DirLen) // Remove trailing separator. + --DirLen; + } + } + + if (!LocationLen) { // Relative path. + if (!DirLen) + return "."; + return std::string(".\\").append(FileName, 0, DirLen); + } + + return FileName.substr(0, LocationLen + DirLen); +} + +std::string TmpDir() { + std::string Tmp; + Tmp.resize(MAX_PATH + 1); + DWORD Size = GetTempPathA(Tmp.size(), &Tmp[0]); + if (Size == 0) { + Printf("Couldn't get Tmp path.\n"); + exit(1); + } + Tmp.resize(Size); + return Tmp; +} + +bool IsInterestingCoverageFile(const std::string &FileName) { + if (FileName.find("Program Files") != std::string::npos) + return false; + if (FileName.find("compiler-rt\\lib\\") != std::string::npos) + return false; // sanitizer internal. + if (FileName == "") + return false; + return true; +} + +void RawPrint(const char *Str) { + _write(2, Str, strlen(Str)); +} + +void MkDir(const std::string &Path) { + if (CreateDirectoryA(Path.c_str(), nullptr)) return; + Printf("CreateDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +void RmDir(const std::string &Path) { + if (RemoveDirectoryA(Path.c_str())) return; + Printf("RemoveDirectoryA failed for %s (Error code: %lu).\n", Path.c_str(), + GetLastError()); +} + +const std::string &getDevNull() { + static const std::string devNull = "NUL"; + return devNull; +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerInterceptors.cpp b/tools/fuzzing/libfuzzer/FuzzerInterceptors.cpp new file mode 100644 index 0000000000..a1a64780de --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerInterceptors.cpp @@ -0,0 +1,235 @@ +//===-- FuzzerInterceptors.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Intercept certain libc functions to aid fuzzing. +// Linked only when other RTs that define their own interceptors are not linked. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" + +#if LIBFUZZER_LINUX + +#define GET_CALLER_PC() __builtin_return_address(0) + +#define PTR_TO_REAL(x) real_##x +#define REAL(x) __interception::PTR_TO_REAL(x) +#define FUNC_TYPE(x) x##_type +#define DEFINE_REAL(ret_type, func, ...) \ + typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ + namespace __interception { \ + FUNC_TYPE(func) PTR_TO_REAL(func); \ + } + +#include +#include +#include // for dlsym() +#include + +static void *getFuncAddr(const char *name, uintptr_t wrapper_addr) { + void *addr = dlsym(RTLD_NEXT, name); + if (!addr) { + // If the lookup using RTLD_NEXT failed, the sanitizer runtime library is + // later in the library search order than the DSO that we are trying to + // intercept, which means that we cannot intercept this function. We still + // want the address of the real definition, though, so look it up using + // RTLD_DEFAULT. + addr = dlsym(RTLD_DEFAULT, name); + + // In case `name' is not loaded, dlsym ends up finding the actual wrapper. + // We don't want to intercept the wrapper and have it point to itself. + if (reinterpret_cast(addr) == wrapper_addr) + addr = nullptr; + } + return addr; +} + +static int FuzzerInited = 0; +static bool FuzzerInitIsRunning; + +static void fuzzerInit(); + +static void ensureFuzzerInited() { + assert(!FuzzerInitIsRunning); + if (!FuzzerInited) { + fuzzerInit(); + } +} + +static int internal_strcmp_strncmp(const char *s1, const char *s2, bool strncmp, + size_t n) { + size_t i = 0; + while (true) { + if (strncmp) { + if (i == n) + break; + i++; + } + unsigned c1 = *s1; + unsigned c2 = *s2; + if (c1 != c2) + return (c1 < c2) ? -1 : 1; + if (c1 == 0) + break; + s1++; + s2++; + } + return 0; +} + +static int internal_strncmp(const char *s1, const char *s2, size_t n) { + return internal_strcmp_strncmp(s1, s2, true, n); +} + +static int internal_strcmp(const char *s1, const char *s2) { + return internal_strcmp_strncmp(s1, s2, false, 0); +} + +static int internal_memcmp(const void *s1, const void *s2, size_t n) { + const uint8_t *t1 = static_cast(s1); + const uint8_t *t2 = static_cast(s2); + for (size_t i = 0; i < n; ++i, ++t1, ++t2) + if (*t1 != *t2) + return *t1 < *t2 ? -1 : 1; + return 0; +} + +static size_t internal_strlen(const char *s) { + size_t i = 0; + while (s[i]) + i++; + return i; +} + +static char *internal_strstr(const char *haystack, const char *needle) { + // This is O(N^2), but we are not using it in hot places. + size_t len1 = internal_strlen(haystack); + size_t len2 = internal_strlen(needle); + if (len1 < len2) + return nullptr; + for (size_t pos = 0; pos <= len1 - len2; pos++) { + if (internal_memcmp(haystack + pos, needle, len2) == 0) + return const_cast(haystack) + pos; + } + return nullptr; +} + +extern "C" { + +DEFINE_REAL(int, bcmp, const void *, const void *, size_t) +DEFINE_REAL(int, memcmp, const void *, const void *, size_t) +DEFINE_REAL(int, strncmp, const char *, const char *, size_t) +DEFINE_REAL(int, strcmp, const char *, const char *) +DEFINE_REAL(int, strncasecmp, const char *, const char *, size_t) +DEFINE_REAL(int, strcasecmp, const char *, const char *) +DEFINE_REAL(char *, strstr, const char *, const char *) +DEFINE_REAL(char *, strcasestr, const char *, const char *) +DEFINE_REAL(void *, memmem, const void *, size_t, const void *, size_t) + +ATTRIBUTE_INTERFACE int bcmp(const char *s1, const char *s2, size_t n) { + if (!FuzzerInited) + return internal_memcmp(s1, s2, n); + int result = REAL(bcmp)(s1, s2, n); + __sanitizer_weak_hook_memcmp(GET_CALLER_PC(), s1, s2, n, result); + return result; +} + +ATTRIBUTE_INTERFACE int memcmp(const void *s1, const void *s2, size_t n) { + if (!FuzzerInited) + return internal_memcmp(s1, s2, n); + int result = REAL(memcmp)(s1, s2, n); + __sanitizer_weak_hook_memcmp(GET_CALLER_PC(), s1, s2, n, result); + return result; +} + +ATTRIBUTE_INTERFACE int strncmp(const char *s1, const char *s2, size_t n) { + if (!FuzzerInited) + return internal_strncmp(s1, s2, n); + int result = REAL(strncmp)(s1, s2, n); + __sanitizer_weak_hook_strncmp(GET_CALLER_PC(), s1, s2, n, result); + return result; +} + +ATTRIBUTE_INTERFACE int strcmp(const char *s1, const char *s2) { + if (!FuzzerInited) + return internal_strcmp(s1, s2); + int result = REAL(strcmp)(s1, s2); + __sanitizer_weak_hook_strcmp(GET_CALLER_PC(), s1, s2, result); + return result; +} + +ATTRIBUTE_INTERFACE int strncasecmp(const char *s1, const char *s2, size_t n) { + ensureFuzzerInited(); + int result = REAL(strncasecmp)(s1, s2, n); + __sanitizer_weak_hook_strncasecmp(GET_CALLER_PC(), s1, s2, n, result); + return result; +} + +ATTRIBUTE_INTERFACE int strcasecmp(const char *s1, const char *s2) { + ensureFuzzerInited(); + int result = REAL(strcasecmp)(s1, s2); + __sanitizer_weak_hook_strcasecmp(GET_CALLER_PC(), s1, s2, result); + return result; +} + +ATTRIBUTE_INTERFACE char *strstr(const char *s1, const char *s2) { + if (!FuzzerInited) + return internal_strstr(s1, s2); + char *result = REAL(strstr)(s1, s2); + __sanitizer_weak_hook_strstr(GET_CALLER_PC(), s1, s2, result); + return result; +} + +ATTRIBUTE_INTERFACE char *strcasestr(const char *s1, const char *s2) { + ensureFuzzerInited(); + char *result = REAL(strcasestr)(s1, s2); + __sanitizer_weak_hook_strcasestr(GET_CALLER_PC(), s1, s2, result); + return result; +} + +ATTRIBUTE_INTERFACE +void *memmem(const void *s1, size_t len1, const void *s2, size_t len2) { + ensureFuzzerInited(); + void *result = REAL(memmem)(s1, len1, s2, len2); + __sanitizer_weak_hook_memmem(GET_CALLER_PC(), s1, len1, s2, len2, result); + return result; +} + +__attribute__((section(".preinit_array"), + used)) static void (*__local_fuzzer_preinit)(void) = fuzzerInit; + +} // extern "C" + +static void fuzzerInit() { + assert(!FuzzerInitIsRunning); + if (FuzzerInited) + return; + FuzzerInitIsRunning = true; + + REAL(bcmp) = reinterpret_cast( + getFuncAddr("bcmp", reinterpret_cast(&bcmp))); + REAL(memcmp) = reinterpret_cast( + getFuncAddr("memcmp", reinterpret_cast(&memcmp))); + REAL(strncmp) = reinterpret_cast( + getFuncAddr("strncmp", reinterpret_cast(&strncmp))); + REAL(strcmp) = reinterpret_cast( + getFuncAddr("strcmp", reinterpret_cast(&strcmp))); + REAL(strncasecmp) = reinterpret_cast( + getFuncAddr("strncasecmp", reinterpret_cast(&strncasecmp))); + REAL(strcasecmp) = reinterpret_cast( + getFuncAddr("strcasecmp", reinterpret_cast(&strcasecmp))); + REAL(strstr) = reinterpret_cast( + getFuncAddr("strstr", reinterpret_cast(&strstr))); + REAL(strcasestr) = reinterpret_cast( + getFuncAddr("strcasestr", reinterpret_cast(&strcasestr))); + REAL(memmem) = reinterpret_cast( + getFuncAddr("memmem", reinterpret_cast(&memmem))); + + FuzzerInitIsRunning = false; + FuzzerInited = 1; +} + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerInterface.h b/tools/fuzzing/libfuzzer/FuzzerInterface.h new file mode 100644 index 0000000000..4f62822eac --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerInterface.h @@ -0,0 +1,79 @@ +//===- FuzzerInterface.h - Interface header for the Fuzzer ------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Define the interface between libFuzzer and the library being tested. +//===----------------------------------------------------------------------===// + +// NOTE: the libFuzzer interface is thin and in the majority of cases +// you should not include this file into your target. In 95% of cases +// all you need is to define the following function in your file: +// extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); + +// WARNING: keep the interface in C. + +#ifndef LLVM_FUZZER_INTERFACE_H +#define LLVM_FUZZER_INTERFACE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +// Define FUZZER_INTERFACE_VISIBILITY to set default visibility in a way that +// doesn't break MSVC. +#if defined(_WIN32) +#define FUZZER_INTERFACE_VISIBILITY __declspec(dllexport) +#else +#define FUZZER_INTERFACE_VISIBILITY __attribute__((visibility("default"))) +#endif + +// Mandatory user-provided target function. +// Executes the code under test with [Data, Data+Size) as the input. +// libFuzzer will invoke this function *many* times with different inputs. +// Must return 0. +FUZZER_INTERFACE_VISIBILITY int +LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); + +// Optional user-provided initialization function. +// If provided, this function will be called by libFuzzer once at startup. +// It may read and modify argc/argv. +// Must return 0. +FUZZER_INTERFACE_VISIBILITY int LLVMFuzzerInitialize(int *argc, char ***argv); + +// Optional user-provided custom mutator. +// Mutates raw data in [Data, Data+Size) inplace. +// Returns the new size, which is not greater than MaxSize. +// Given the same Seed produces the same mutation. +FUZZER_INTERFACE_VISIBILITY size_t +LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, + unsigned int Seed); + +// Optional user-provided custom cross-over function. +// Combines pieces of Data1 & Data2 together into Out. +// Returns the new size, which is not greater than MaxOutSize. +// Should produce the same mutation given the same Seed. +FUZZER_INTERFACE_VISIBILITY size_t +LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, + const uint8_t *Data2, size_t Size2, uint8_t *Out, + size_t MaxOutSize, unsigned int Seed); + +// Experimental, may go away in future. +// libFuzzer-provided function to be used inside LLVMFuzzerCustomMutator. +// Mutates raw data in [Data, Data+Size) inplace. +// Returns the new size, which is not greater than MaxSize. +FUZZER_INTERFACE_VISIBILITY size_t +LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize); + +#undef FUZZER_INTERFACE_VISIBILITY + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // LLVM_FUZZER_INTERFACE_H diff --git a/tools/fuzzing/libfuzzer/FuzzerInternal.h b/tools/fuzzing/libfuzzer/FuzzerInternal.h new file mode 100644 index 0000000000..cc2650b58e --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h @@ -0,0 +1,175 @@ +//===- FuzzerInternal.h - Internal header for the Fuzzer --------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Define the main class fuzzer::Fuzzer and most functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_INTERNAL_H +#define LLVM_FUZZER_INTERNAL_H + +#include "FuzzerDataFlowTrace.h" +#include "FuzzerDefs.h" +#include "FuzzerExtFunctions.h" +#include "FuzzerInterface.h" +#include "FuzzerOptions.h" +#include "FuzzerSHA1.h" +#include "FuzzerValueBitMap.h" +#include +#include +#include +#include +#include +#include + +namespace fuzzer { + +using namespace std::chrono; + +class Fuzzer { +public: + + Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, + FuzzingOptions Options); + ~Fuzzer(); + int Loop(Vector &CorporaFiles); + int ReadAndExecuteSeedCorpora(Vector &CorporaFiles); + void MinimizeCrashLoop(const Unit &U); + void RereadOutputCorpus(size_t MaxSize); + + size_t secondsSinceProcessStartUp() { + return duration_cast(system_clock::now() - ProcessStartTime) + .count(); + } + + bool TimedOut() { + return Options.MaxTotalTimeSec > 0 && + secondsSinceProcessStartUp() > + static_cast(Options.MaxTotalTimeSec); + } + + size_t execPerSec() { + size_t Seconds = secondsSinceProcessStartUp(); + return Seconds ? TotalNumberOfRuns / Seconds : 0; + } + + size_t getTotalNumberOfRuns() { return TotalNumberOfRuns; } + + static void StaticAlarmCallback(); + static void StaticCrashSignalCallback(); + static void StaticExitCallback(); + static void StaticInterruptCallback(); + static void StaticFileSizeExceedCallback(); + static void StaticGracefulExitCallback(); + + static void GracefullyExit(); + static bool isGracefulExitRequested(); + + int ExecuteCallback(const uint8_t *Data, size_t Size); + bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, + InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); + + // Merge Corpora[1:] into Corpora[0]. + void Merge(const Vector &Corpora); + int CrashResistantMergeInternalStep(const std::string &ControlFilePath); + MutationDispatcher &GetMD() { return MD; } + void PrintFinalStats(); + void SetMaxInputLen(size_t MaxInputLen); + void SetMaxMutationLen(size_t MaxMutationLen); + void RssLimitCallback(); + + bool InFuzzingThread() const { return IsMyThread; } + size_t GetCurrentUnitInFuzzingThead(const uint8_t **Data) const; + void TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size, + bool DuringInitialCorpusExecution); + + void HandleMalloc(size_t Size); + static bool MaybeExitGracefully(); + std::string WriteToOutputCorpus(const Unit &U); + +private: + void AlarmCallback(); + void CrashCallback(); + void ExitCallback(); + void CrashOnOverwrittenData(); + void InterruptCallback(); + bool MutateAndTestOne(); + void PurgeAllocator(); + void ReportNewCoverage(InputInfo *II, const Unit &U); + void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size); + void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix); + void PrintStats(const char *Where, const char *End = "\n", size_t Units = 0, + size_t Features = 0); + void PrintStatusForNewUnit(const Unit &U, const char *Text); + void CheckExitOnSrcPosOrItem(); + + static void StaticDeathCallback(); + void DumpCurrentUnit(const char *Prefix); + void DeathCallback(); + + void AllocateCurrentUnitData(); + uint8_t *CurrentUnitData = nullptr; + std::atomic CurrentUnitSize; + uint8_t BaseSha1[kSHA1NumBytes]; // Checksum of the base unit. + + bool GracefulExitRequested = false; + + size_t TotalNumberOfRuns = 0; + size_t NumberOfNewUnitsAdded = 0; + + size_t LastCorpusUpdateRun = 0; + + bool HasMoreMallocsThanFrees = false; + size_t NumberOfLeakDetectionAttempts = 0; + + system_clock::time_point LastAllocatorPurgeAttemptTime = system_clock::now(); + + UserCallback CB; + InputCorpus &Corpus; + MutationDispatcher &MD; + FuzzingOptions Options; + DataFlowTrace DFT; + + system_clock::time_point ProcessStartTime = system_clock::now(); + system_clock::time_point UnitStartTime, UnitStopTime; + long TimeOfLongestUnitInSeconds = 0; + long EpochOfLastReadOfOutputCorpus = 0; + + size_t MaxInputLen = 0; + size_t MaxMutationLen = 0; + size_t TmpMaxMutationLen = 0; + + Vector UniqFeatureSetTmp; + + // Need to know our own thread. + static thread_local bool IsMyThread; +}; + +struct ScopedEnableMsanInterceptorChecks { + ScopedEnableMsanInterceptorChecks() { + if (EF->__msan_scoped_enable_interceptor_checks) + EF->__msan_scoped_enable_interceptor_checks(); + } + ~ScopedEnableMsanInterceptorChecks() { + if (EF->__msan_scoped_disable_interceptor_checks) + EF->__msan_scoped_disable_interceptor_checks(); + } +}; + +struct ScopedDisableMsanInterceptorChecks { + ScopedDisableMsanInterceptorChecks() { + if (EF->__msan_scoped_disable_interceptor_checks) + EF->__msan_scoped_disable_interceptor_checks(); + } + ~ScopedDisableMsanInterceptorChecks() { + if (EF->__msan_scoped_enable_interceptor_checks) + EF->__msan_scoped_enable_interceptor_checks(); + } +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_INTERNAL_H diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp new file mode 100644 index 0000000000..e7dfc187db --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp @@ -0,0 +1,901 @@ +//===- FuzzerLoop.cpp - Fuzzer's main loop --------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Fuzzer's main loop. +//===----------------------------------------------------------------------===// + +#include "FuzzerCorpus.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerMutate.h" +#include "FuzzerPlatform.h" +#include "FuzzerRandom.h" +#include "FuzzerTracePC.h" +#include +#include +#include +#include +#include + +#if defined(__has_include) +#if __has_include() +#include +#endif +#endif + +#define NO_SANITIZE_MEMORY +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#undef NO_SANITIZE_MEMORY +#define NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +#endif +#endif + +namespace fuzzer { +static const size_t kMaxUnitSizeToPrint = 256; + +thread_local bool Fuzzer::IsMyThread; + +bool RunningUserCallback = false; + +// Only one Fuzzer per process. +static Fuzzer *F; + +// Leak detection is expensive, so we first check if there were more mallocs +// than frees (using the sanitizer malloc hooks) and only then try to call lsan. +struct MallocFreeTracer { + void Start(int TraceLevel) { + this->TraceLevel = TraceLevel; + if (TraceLevel) + Printf("MallocFreeTracer: START\n"); + Mallocs = 0; + Frees = 0; + } + // Returns true if there were more mallocs than frees. + bool Stop() { + if (TraceLevel) + Printf("MallocFreeTracer: STOP %zd %zd (%s)\n", Mallocs.load(), + Frees.load(), Mallocs == Frees ? "same" : "DIFFERENT"); + bool Result = Mallocs > Frees; + Mallocs = 0; + Frees = 0; + TraceLevel = 0; + return Result; + } + std::atomic Mallocs; + std::atomic Frees; + int TraceLevel = 0; + + std::recursive_mutex TraceMutex; + bool TraceDisabled = false; +}; + +static MallocFreeTracer AllocTracer; + +// Locks printing and avoids nested hooks triggered from mallocs/frees in +// sanitizer. +class TraceLock { +public: + TraceLock() : Lock(AllocTracer.TraceMutex) { + AllocTracer.TraceDisabled = !AllocTracer.TraceDisabled; + } + ~TraceLock() { AllocTracer.TraceDisabled = !AllocTracer.TraceDisabled; } + + bool IsDisabled() const { + // This is already inverted value. + return !AllocTracer.TraceDisabled; + } + +private: + std::lock_guard Lock; +}; + +ATTRIBUTE_NO_SANITIZE_MEMORY +void MallocHook(const volatile void *ptr, size_t size) { + size_t N = AllocTracer.Mallocs++; + F->HandleMalloc(size); + if (int TraceLevel = AllocTracer.TraceLevel) { + TraceLock Lock; + if (Lock.IsDisabled()) + return; + Printf("MALLOC[%zd] %p %zd\n", N, ptr, size); + if (TraceLevel >= 2 && EF) + PrintStackTrace(); + } +} + +ATTRIBUTE_NO_SANITIZE_MEMORY +void FreeHook(const volatile void *ptr) { + size_t N = AllocTracer.Frees++; + if (int TraceLevel = AllocTracer.TraceLevel) { + TraceLock Lock; + if (Lock.IsDisabled()) + return; + Printf("FREE[%zd] %p\n", N, ptr); + if (TraceLevel >= 2 && EF) + PrintStackTrace(); + } +} + +// Crash on a single malloc that exceeds the rss limit. +void Fuzzer::HandleMalloc(size_t Size) { + if (!Options.MallocLimitMb || (Size >> 20) < (size_t)Options.MallocLimitMb) + return; + Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(), + Size); + Printf(" To change the out-of-memory limit use -rss_limit_mb=\n\n"); + PrintStackTrace(); + DumpCurrentUnit("oom-"); + Printf("SUMMARY: libFuzzer: out-of-memory\n"); + PrintFinalStats(); + _Exit(Options.OOMExitCode); // Stop right now. +} + +Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, + FuzzingOptions Options) + : CB(CB), Corpus(Corpus), MD(MD), Options(Options) { + if (EF->__sanitizer_set_death_callback) + EF->__sanitizer_set_death_callback(StaticDeathCallback); + assert(!F); + F = this; + TPC.ResetMaps(); + IsMyThread = true; + if (Options.DetectLeaks && EF->__sanitizer_install_malloc_and_free_hooks) + EF->__sanitizer_install_malloc_and_free_hooks(MallocHook, FreeHook); + TPC.SetUseCounters(Options.UseCounters); + TPC.SetUseValueProfileMask(Options.UseValueProfile); + + if (Options.Verbosity) + TPC.PrintModuleInfo(); + if (!Options.OutputCorpus.empty() && Options.ReloadIntervalSec) + EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus); + MaxInputLen = MaxMutationLen = Options.MaxLen; + TmpMaxMutationLen = 0; // Will be set once we load the corpus. + AllocateCurrentUnitData(); + CurrentUnitSize = 0; + memset(BaseSha1, 0, sizeof(BaseSha1)); +} + +Fuzzer::~Fuzzer() {} + +void Fuzzer::AllocateCurrentUnitData() { + if (CurrentUnitData || MaxInputLen == 0) + return; + CurrentUnitData = new uint8_t[MaxInputLen]; +} + +void Fuzzer::StaticDeathCallback() { + assert(F); + F->DeathCallback(); +} + +void Fuzzer::DumpCurrentUnit(const char *Prefix) { + if (!CurrentUnitData) + return; // Happens when running individual inputs. + ScopedDisableMsanInterceptorChecks S; + MD.PrintMutationSequence(); + Printf("; base unit: %s\n", Sha1ToString(BaseSha1).c_str()); + size_t UnitSize = CurrentUnitSize; + if (UnitSize <= kMaxUnitSizeToPrint) { + PrintHexArray(CurrentUnitData, UnitSize, "\n"); + PrintASCII(CurrentUnitData, UnitSize, "\n"); + } + WriteUnitToFileWithPrefix({CurrentUnitData, CurrentUnitData + UnitSize}, + Prefix); +} + +NO_SANITIZE_MEMORY +void Fuzzer::DeathCallback() { + DumpCurrentUnit("crash-"); + PrintFinalStats(); +} + +void Fuzzer::StaticAlarmCallback() { + assert(F); + F->AlarmCallback(); +} + +void Fuzzer::StaticCrashSignalCallback() { + assert(F); + F->CrashCallback(); +} + +void Fuzzer::StaticExitCallback() { + assert(F); + F->ExitCallback(); +} + +void Fuzzer::StaticInterruptCallback() { + assert(F); + F->InterruptCallback(); +} + +void Fuzzer::StaticGracefulExitCallback() { + assert(F); + F->GracefulExitRequested = true; + Printf("INFO: signal received, trying to exit gracefully\n"); +} + +void Fuzzer::StaticFileSizeExceedCallback() { + Printf("==%lu== ERROR: libFuzzer: file size exceeded\n", GetPid()); + exit(1); +} + +void Fuzzer::CrashCallback() { + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; + Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid()); + PrintStackTrace(); + Printf("NOTE: libFuzzer has rudimentary signal handlers.\n" + " Combine libFuzzer with AddressSanitizer or similar for better " + "crash reports.\n"); + Printf("SUMMARY: libFuzzer: deadly signal\n"); + DumpCurrentUnit("crash-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); // Stop right now. +} + +void Fuzzer::ExitCallback() { + if (!RunningUserCallback) + return; // This exit did not come from the user callback + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; + Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid()); + PrintStackTrace(); + Printf("SUMMARY: libFuzzer: fuzz target exited\n"); + DumpCurrentUnit("crash-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); +} + +bool Fuzzer::MaybeExitGracefully() { + if (!F->GracefulExitRequested) return false; + Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid()); + RmDirRecursive(TempPath("FuzzWithFork", ".dir")); + F->PrintFinalStats(); + return true; +} + +void Fuzzer::GracefullyExit() { + F->GracefulExitRequested = true; +} + +bool Fuzzer::isGracefulExitRequested() { + return F->GracefulExitRequested; +} + +void Fuzzer::InterruptCallback() { + Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); + PrintFinalStats(); + ScopedDisableMsanInterceptorChecks S; // RmDirRecursive may call opendir(). + RmDirRecursive(TempPath("FuzzWithFork", ".dir")); + // Stop right now, don't perform any at-exit actions. + _Exit(Options.InterruptExitCode); +} + +NO_SANITIZE_MEMORY +void Fuzzer::AlarmCallback() { + assert(Options.UnitTimeoutSec > 0); + // In Windows and Fuchsia, Alarm callback is executed by a different thread. + // NetBSD's current behavior needs this change too. +#if !LIBFUZZER_WINDOWS && !LIBFUZZER_NETBSD && !LIBFUZZER_FUCHSIA + if (!InFuzzingThread()) + return; +#endif + if (!RunningUserCallback) + return; // We have not started running units yet. + size_t Seconds = + duration_cast(system_clock::now() - UnitStartTime).count(); + if (Seconds == 0) + return; + if (Options.Verbosity >= 2) + Printf("AlarmCallback %zd\n", Seconds); + if (Seconds >= (size_t)Options.UnitTimeoutSec) { + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; + Printf("ALARM: working on the last Unit for %zd seconds\n", Seconds); + Printf(" and the timeout value is %d (use -timeout=N to change)\n", + Options.UnitTimeoutSec); + DumpCurrentUnit("timeout-"); + Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(), + Seconds); + PrintStackTrace(); + Printf("SUMMARY: libFuzzer: timeout\n"); + PrintFinalStats(); + _Exit(Options.TimeoutExitCode); // Stop right now. + } +} + +void Fuzzer::RssLimitCallback() { + if (EF->__sanitizer_acquire_crash_state && + !EF->__sanitizer_acquire_crash_state()) + return; + Printf( + "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n", + GetPid(), GetPeakRSSMb(), Options.RssLimitMb); + Printf(" To change the out-of-memory limit use -rss_limit_mb=\n\n"); + PrintMemoryProfile(); + DumpCurrentUnit("oom-"); + Printf("SUMMARY: libFuzzer: out-of-memory\n"); + PrintFinalStats(); + _Exit(Options.OOMExitCode); // Stop right now. +} + +void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units, + size_t Features) { + size_t ExecPerSec = execPerSec(); + if (!Options.Verbosity) + return; + Printf("#%zd\t%s", TotalNumberOfRuns, Where); + if (size_t N = TPC.GetTotalPCCoverage()) + Printf(" cov: %zd", N); + if (size_t N = Features ? Features : Corpus.NumFeatures()) + Printf(" ft: %zd", N); + if (!Corpus.empty()) { + Printf(" corp: %zd", Corpus.NumActiveUnits()); + if (size_t N = Corpus.SizeInBytes()) { + if (N < (1 << 14)) + Printf("/%zdb", N); + else if (N < (1 << 24)) + Printf("/%zdKb", N >> 10); + else + Printf("/%zdMb", N >> 20); + } + if (size_t FF = Corpus.NumInputsThatTouchFocusFunction()) + Printf(" focus: %zd", FF); + } + if (TmpMaxMutationLen) + Printf(" lim: %zd", TmpMaxMutationLen); + if (Units) + Printf(" units: %zd", Units); + + Printf(" exec/s: %zd", ExecPerSec); + Printf(" rss: %zdMb", GetPeakRSSMb()); + Printf("%s", End); +} + +void Fuzzer::PrintFinalStats() { + if (Options.PrintCoverage) + TPC.PrintCoverage(); + if (Options.PrintCorpusStats) + Corpus.PrintStats(); + if (!Options.PrintFinalStats) + return; + size_t ExecPerSec = execPerSec(); + Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns); + Printf("stat::average_exec_per_sec: %zd\n", ExecPerSec); + Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded); + Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds); + Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb()); +} + +void Fuzzer::SetMaxInputLen(size_t MaxInputLen) { + assert(this->MaxInputLen == 0); // Can only reset MaxInputLen from 0 to non-0. + assert(MaxInputLen); + this->MaxInputLen = MaxInputLen; + this->MaxMutationLen = MaxInputLen; + AllocateCurrentUnitData(); + Printf("INFO: -max_len is not provided; " + "libFuzzer will not generate inputs larger than %zd bytes\n", + MaxInputLen); +} + +void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { + assert(MaxMutationLen && MaxMutationLen <= MaxInputLen); + this->MaxMutationLen = MaxMutationLen; +} + +void Fuzzer::CheckExitOnSrcPosOrItem() { + if (!Options.ExitOnSrcPos.empty()) { + static auto *PCsSet = new Set; + auto HandlePC = [&](const TracePC::PCTableEntry *TE) { + if (!PCsSet->insert(TE->PC).second) + return; + std::string Descr = DescribePC("%F %L", TE->PC + 1); + if (Descr.find(Options.ExitOnSrcPos) != std::string::npos) { + Printf("INFO: found line matching '%s', exiting.\n", + Options.ExitOnSrcPos.c_str()); + _Exit(0); + } + }; + TPC.ForEachObservedPC(HandlePC); + } + if (!Options.ExitOnItem.empty()) { + if (Corpus.HasUnit(Options.ExitOnItem)) { + Printf("INFO: found item with checksum '%s', exiting.\n", + Options.ExitOnItem.c_str()); + _Exit(0); + } + } +} + +void Fuzzer::RereadOutputCorpus(size_t MaxSize) { + if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) + return; + Vector AdditionalCorpus; + ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus, + &EpochOfLastReadOfOutputCorpus, MaxSize, + /*ExitOnError*/ false); + if (Options.Verbosity >= 2) + Printf("Reload: read %zd new units.\n", AdditionalCorpus.size()); + bool Reloaded = false; + for (auto &U : AdditionalCorpus) { + if (U.size() > MaxSize) + U.resize(MaxSize); + if (!Corpus.HasUnit(U)) { + if (RunOne(U.data(), U.size())) { + CheckExitOnSrcPosOrItem(); + Reloaded = true; + } + } + } + if (Reloaded) + PrintStats("RELOAD"); +} + +void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { + auto TimeOfUnit = + duration_cast(UnitStopTime - UnitStartTime).count(); + if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && + secondsSinceProcessStartUp() >= 2) + PrintStats("pulse "); + if (TimeOfUnit > TimeOfLongestUnitInSeconds * 1.1 && + TimeOfUnit >= Options.ReportSlowUnits) { + TimeOfLongestUnitInSeconds = TimeOfUnit; + Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds); + WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-"); + } +} + +static void WriteFeatureSetToFile(const std::string &FeaturesDir, + const std::string &FileName, + const Vector &FeatureSet) { + if (FeaturesDir.empty() || FeatureSet.empty()) return; + WriteToFile(reinterpret_cast(FeatureSet.data()), + FeatureSet.size() * sizeof(FeatureSet[0]), + DirPlusFile(FeaturesDir, FileName)); +} + +static void RenameFeatureSetFile(const std::string &FeaturesDir, + const std::string &OldFile, + const std::string &NewFile) { + if (FeaturesDir.empty()) return; + RenameFile(DirPlusFile(FeaturesDir, OldFile), + DirPlusFile(FeaturesDir, NewFile)); +} + +bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, + InputInfo *II, bool *FoundUniqFeatures) { + if (!Size) + return false; + + if (ExecuteCallback(Data, Size) > 0) { + return false; + } + + UniqFeatureSetTmp.clear(); + size_t FoundUniqFeaturesOfII = 0; + size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); + TPC.CollectFeatures([&](size_t Feature) { + if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + UniqFeatureSetTmp.push_back(Feature); + if (Options.Entropic) + Corpus.UpdateFeatureFrequency(II, Feature); + if (Options.ReduceInputs && II) + if (std::binary_search(II->UniqFeatureSet.begin(), + II->UniqFeatureSet.end(), Feature)) + FoundUniqFeaturesOfII++; + }); + if (FoundUniqFeatures) + *FoundUniqFeatures = FoundUniqFeaturesOfII; + PrintPulseAndReportSlowInput(Data, Size); + size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; + if (NumNewFeatures) { + TPC.UpdateObservedPCs(); + auto NewII = Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, + MayDeleteFile, TPC.ObservedFocusFunction(), + UniqFeatureSetTmp, DFT, II); + WriteFeatureSetToFile(Options.FeaturesDir, Sha1ToString(NewII->Sha1), + NewII->UniqFeatureSet); + return true; + } + if (II && FoundUniqFeaturesOfII && + II->DataFlowTraceForFocusFunction.empty() && + FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && + II->U.size() > Size) { + auto OldFeaturesFile = Sha1ToString(II->Sha1); + Corpus.Replace(II, {Data, Data + Size}); + RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile, + Sha1ToString(II->Sha1)); + return true; + } + return false; +} + +size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const { + assert(InFuzzingThread()); + *Data = CurrentUnitData; + return CurrentUnitSize; +} + +void Fuzzer::CrashOnOverwrittenData() { + Printf("==%d== ERROR: libFuzzer: fuzz target overwrites its const input\n", + GetPid()); + PrintStackTrace(); + Printf("SUMMARY: libFuzzer: overwrites-const-input\n"); + DumpCurrentUnit("crash-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); // Stop right now. +} + +// Compare two arrays, but not all bytes if the arrays are large. +static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) { + const size_t Limit = 64; + if (Size <= 64) + return !memcmp(A, B, Size); + // Compare first and last Limit/2 bytes. + return !memcmp(A, B, Limit / 2) && + !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); +} + +int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { + TPC.RecordInitialStack(); + TotalNumberOfRuns++; + assert(InFuzzingThread()); + // We copy the contents of Unit into a separate heap buffer + // so that we reliably find buffer overflows in it. + uint8_t *DataCopy = new uint8_t[Size]; + memcpy(DataCopy, Data, Size); + if (EF->__msan_unpoison) + EF->__msan_unpoison(DataCopy, Size); + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(2); + if (CurrentUnitData && CurrentUnitData != Data) + memcpy(CurrentUnitData, Data, Size); + CurrentUnitSize = Size; + int Res = 0; + { + ScopedEnableMsanInterceptorChecks S; + AllocTracer.Start(Options.TraceMalloc); + UnitStartTime = system_clock::now(); + TPC.ResetMaps(); + RunningUserCallback = true; + Res = CB(DataCopy, Size); + RunningUserCallback = false; + UnitStopTime = system_clock::now(); + assert(Res >= 0); + HasMoreMallocsThanFrees = AllocTracer.Stop(); + } + if (!LooseMemeq(DataCopy, Data, Size)) + CrashOnOverwrittenData(); + CurrentUnitSize = 0; + delete[] DataCopy; + return Res; +} + +std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { + if (Options.OnlyASCII) + assert(IsASCII(U)); + if (Options.OutputCorpus.empty()) + return ""; + std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); + WriteToFile(U, Path); + if (Options.Verbosity >= 2) + Printf("Written %zd bytes to %s\n", U.size(), Path.c_str()); + return Path; +} + +void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) { + if (!Options.SaveArtifacts) + return; + std::string Path = Options.ArtifactPrefix + Prefix + Hash(U); + if (!Options.ExactArtifactPath.empty()) + Path = Options.ExactArtifactPath; // Overrides ArtifactPrefix. + WriteToFile(U, Path); + Printf("artifact_prefix='%s'; Test unit written to %s\n", + Options.ArtifactPrefix.c_str(), Path.c_str()); + if (U.size() <= kMaxUnitSizeToPrint) + Printf("Base64: %s\n", Base64(U).c_str()); +} + +void Fuzzer::PrintStatusForNewUnit(const Unit &U, const char *Text) { + if (!Options.PrintNEW) + return; + PrintStats(Text, ""); + if (Options.Verbosity) { + Printf(" L: %zd/%zd ", U.size(), Corpus.MaxInputSize()); + MD.PrintMutationSequence(); + Printf("\n"); + } +} + +void Fuzzer::ReportNewCoverage(InputInfo *II, const Unit &U) { + II->NumSuccessfullMutations++; + MD.RecordSuccessfulMutationSequence(); + PrintStatusForNewUnit(U, II->Reduced ? "REDUCE" : "NEW "); + WriteToOutputCorpus(U); + NumberOfNewUnitsAdded++; + CheckExitOnSrcPosOrItem(); // Check only after the unit is saved to corpus. + LastCorpusUpdateRun = TotalNumberOfRuns; +} + +// Tries detecting a memory leak on the particular input that we have just +// executed before calling this function. +void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size, + bool DuringInitialCorpusExecution) { + if (!HasMoreMallocsThanFrees) + return; // mallocs==frees, a leak is unlikely. + if (!Options.DetectLeaks) + return; + if (!DuringInitialCorpusExecution && + TotalNumberOfRuns >= Options.MaxNumberOfRuns) + return; + if (!&(EF->__lsan_enable) || !&(EF->__lsan_disable) || + !(EF->__lsan_do_recoverable_leak_check)) + return; // No lsan. + // Run the target once again, but with lsan disabled so that if there is + // a real leak we do not report it twice. + EF->__lsan_disable(); + ExecuteCallback(Data, Size); + EF->__lsan_enable(); + if (!HasMoreMallocsThanFrees) + return; // a leak is unlikely. + if (NumberOfLeakDetectionAttempts++ > 1000) { + Options.DetectLeaks = false; + Printf("INFO: libFuzzer disabled leak detection after every mutation.\n" + " Most likely the target function accumulates allocated\n" + " memory in a global state w/o actually leaking it.\n" + " You may try running this binary with -trace_malloc=[12]" + " to get a trace of mallocs and frees.\n" + " If LeakSanitizer is enabled in this process it will still\n" + " run on the process shutdown.\n"); + return; + } + // Now perform the actual lsan pass. This is expensive and we must ensure + // we don't call it too often. + if (EF->__lsan_do_recoverable_leak_check()) { // Leak is found, report it. + if (DuringInitialCorpusExecution) + Printf("\nINFO: a leak has been found in the initial corpus.\n\n"); + Printf("INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.\n\n"); + CurrentUnitSize = Size; + DumpCurrentUnit("leak-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on. + } +} + +bool Fuzzer::MutateAndTestOne() { + MD.StartMutationSequence(); + + auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); + if (Options.DoCrossOver) + MD.SetCrossOverWith(&Corpus.ChooseUnitToMutate(MD.GetRand()).U); + const auto &U = II.U; + memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1)); + assert(CurrentUnitData); + size_t Size = U.size(); + assert(Size <= MaxInputLen && "Oversized Unit"); + memcpy(CurrentUnitData, U.data(), Size); + + assert(MaxMutationLen > 0); + + size_t CurrentMaxMutationLen = + Min(MaxMutationLen, Max(U.size(), TmpMaxMutationLen)); + assert(CurrentMaxMutationLen > 0); + + for (int i = 0; i < Options.MutateDepth; i++) { + if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) + break; + if (MaybeExitGracefully()) return true; + size_t NewSize = 0; + if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() && + Size <= CurrentMaxMutationLen) + NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size, + II.DataFlowTraceForFocusFunction); + + // If MutateWithMask either failed or wasn't called, call default Mutate. + if (!NewSize) + NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); + + if (!NewSize) + continue; + + assert(NewSize > 0 && "Mutator returned empty unit"); + assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); + Size = NewSize; + II.NumExecutedMutations++; + Corpus.IncrementNumExecutedMutations(); + + bool FoundUniqFeatures = false; + bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II, + &FoundUniqFeatures); + TryDetectingAMemoryLeak(CurrentUnitData, Size, + /*DuringInitialCorpusExecution*/ false); + if (NewCov) { + ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size}); + break; // We will mutate this input more in the next rounds. + } + if (Options.ReduceDepth && !FoundUniqFeatures) + break; + } + + II.NeedsEnergyUpdate = true; + return false; +} + +void Fuzzer::PurgeAllocator() { + if (Options.PurgeAllocatorIntervalSec < 0 || !EF->__sanitizer_purge_allocator) + return; + if (duration_cast(system_clock::now() - + LastAllocatorPurgeAttemptTime) + .count() < Options.PurgeAllocatorIntervalSec) + return; + + if (Options.RssLimitMb <= 0 || + GetPeakRSSMb() > static_cast(Options.RssLimitMb) / 2) + EF->__sanitizer_purge_allocator(); + + LastAllocatorPurgeAttemptTime = system_clock::now(); +} + +int Fuzzer::ReadAndExecuteSeedCorpora(Vector &CorporaFiles) { + const size_t kMaxSaneLen = 1 << 20; + const size_t kMinDefaultLen = 4096; + size_t MaxSize = 0; + size_t MinSize = -1; + size_t TotalSize = 0; + for (auto &File : CorporaFiles) { + MaxSize = Max(File.Size, MaxSize); + MinSize = Min(File.Size, MinSize); + TotalSize += File.Size; + } + if (Options.MaxLen == 0) + SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxSize), kMaxSaneLen)); + assert(MaxInputLen > 0); + + // Test the callback with empty input and never try it again. + uint8_t dummy = 0; + ExecuteCallback(&dummy, 0); + + if (CorporaFiles.empty()) { + Printf("INFO: A corpus is not provided, starting from an empty corpus\n"); + Unit U({'\n'}); // Valid ASCII input. + RunOne(U.data(), U.size()); + } else { + Printf("INFO: seed corpus: files: %zd min: %zdb max: %zdb total: %zdb" + " rss: %zdMb\n", + CorporaFiles.size(), MinSize, MaxSize, TotalSize, GetPeakRSSMb()); + if (Options.ShuffleAtStartUp) + std::shuffle(CorporaFiles.begin(), CorporaFiles.end(), MD.GetRand()); + + if (Options.PreferSmall) { + std::stable_sort(CorporaFiles.begin(), CorporaFiles.end()); + assert(CorporaFiles.front().Size <= CorporaFiles.back().Size); + } + + // Load and execute inputs one by one. + for (auto &SF : CorporaFiles) { + auto U = FileToVector(SF.File, MaxInputLen, /*ExitOnError=*/false); + assert(U.size() <= MaxInputLen); + RunOne(U.data(), U.size()); + CheckExitOnSrcPosOrItem(); + TryDetectingAMemoryLeak(U.data(), U.size(), + /*DuringInitialCorpusExecution*/ true); + } + } + + PrintStats("INITED"); + if (!Options.FocusFunction.empty()) { + Printf("INFO: %zd/%zd inputs touch the focus function\n", + Corpus.NumInputsThatTouchFocusFunction(), Corpus.size()); + if (!Options.DataFlowTrace.empty()) + Printf("INFO: %zd/%zd inputs have the Data Flow Trace\n", + Corpus.NumInputsWithDataFlowTrace(), + Corpus.NumInputsThatTouchFocusFunction()); + } + + if (Corpus.empty() && Options.MaxNumberOfRuns) { + Printf("ERROR: no interesting inputs were found. " + "Is the code instrumented for coverage? Exiting.\n"); + return 1; + } + return 0; +} + +int Fuzzer::Loop(Vector &CorporaFiles) { + auto FocusFunctionOrAuto = Options.FocusFunction; + int Res = DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, + MD.GetRand()); + if (Res != 0) + return Res; + Res = TPC.SetFocusFunction(FocusFunctionOrAuto); + if (Res != 0) + return Res; + Res = ReadAndExecuteSeedCorpora(CorporaFiles); + if (Res != 0) + return Res; + DFT.Clear(); // No need for DFT any more. + TPC.SetPrintNewPCs(Options.PrintNewCovPcs); + TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs); + system_clock::time_point LastCorpusReload = system_clock::now(); + + TmpMaxMutationLen = + Min(MaxMutationLen, Max(size_t(4), Corpus.MaxInputSize())); + + while (true) { + auto Now = system_clock::now(); + if (!Options.StopFile.empty() && + !FileToVector(Options.StopFile, 1, false).empty()) + break; + if (duration_cast(Now - LastCorpusReload).count() >= + Options.ReloadIntervalSec) { + RereadOutputCorpus(MaxInputLen); + LastCorpusReload = system_clock::now(); + } + if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) + break; + if (TimedOut()) + break; + + // Update TmpMaxMutationLen + if (Options.LenControl) { + if (TmpMaxMutationLen < MaxMutationLen && + TotalNumberOfRuns - LastCorpusUpdateRun > + Options.LenControl * Log(TmpMaxMutationLen)) { + TmpMaxMutationLen = + Min(MaxMutationLen, TmpMaxMutationLen + Log(TmpMaxMutationLen)); + LastCorpusUpdateRun = TotalNumberOfRuns; + } + } else { + TmpMaxMutationLen = MaxMutationLen; + } + + // Perform several mutations and runs. + if (MutateAndTestOne()) + return 0; + + PurgeAllocator(); + } + + PrintStats("DONE ", "\n"); + MD.PrintRecommendedDictionary(); + return 0; +} + +void Fuzzer::MinimizeCrashLoop(const Unit &U) { + if (U.size() <= 1) + return; + while (!TimedOut() && TotalNumberOfRuns < Options.MaxNumberOfRuns) { + MD.StartMutationSequence(); + memcpy(CurrentUnitData, U.data(), U.size()); + for (int i = 0; i < Options.MutateDepth; i++) { + size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen); + assert(NewSize <= MaxMutationLen); + if (!NewSize) + continue; + ExecuteCallback(CurrentUnitData, NewSize); + PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); + TryDetectingAMemoryLeak(CurrentUnitData, NewSize, + /*DuringInitialCorpusExecution*/ false); + } + } +} + +} // namespace fuzzer + +extern "C" { + +ATTRIBUTE_INTERFACE size_t +LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { + assert(fuzzer::F); + return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize); +} + +} // extern "C" diff --git a/tools/fuzzing/libfuzzer/FuzzerMain.cpp b/tools/fuzzing/libfuzzer/FuzzerMain.cpp new file mode 100644 index 0000000000..75f2f8e75c --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMain.cpp @@ -0,0 +1,21 @@ +//===- FuzzerMain.cpp - main() function and flags -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// main() and flags. +//===----------------------------------------------------------------------===// + +#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" + +extern "C" { +// This function should be defined by the user. +int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); +} // extern "C" + +ATTRIBUTE_INTERFACE int main(int argc, char **argv) { + return fuzzer::FuzzerDriver(&argc, &argv, LLVMFuzzerTestOneInput); +} diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp new file mode 100644 index 0000000000..0a185c7325 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp @@ -0,0 +1,419 @@ +//===- FuzzerMerge.cpp - merging corpora ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Merging corpora. +//===----------------------------------------------------------------------===// + +#include "FuzzerCommand.h" +#include "FuzzerMerge.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerTracePC.h" +#include "FuzzerUtil.h" + +#include +#include +#include +#include +#include + +namespace fuzzer { + +bool Merger::Parse(const std::string &Str, bool ParseCoverage) { + std::istringstream SS(Str); + return Parse(SS, ParseCoverage); +} + +int Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { + if (!Parse(IS, ParseCoverage)) { + Printf("MERGE: failed to parse the control file (unexpected error)\n"); + return 1; + } + return 0; +} + +// The control file example: +// +// 3 # The number of inputs +// 1 # The number of inputs in the first corpus, <= the previous number +// file0 +// file1 +// file2 # One file name per line. +// STARTED 0 123 # FileID, file size +// FT 0 1 4 6 8 # FileID COV1 COV2 ... +// COV 0 7 8 9 # FileID COV1 COV1 +// STARTED 1 456 # If FT is missing, the input crashed while processing. +// STARTED 2 567 +// FT 2 8 9 +// COV 2 11 12 +bool Merger::Parse(std::istream &IS, bool ParseCoverage) { + LastFailure.clear(); + std::string Line; + + // Parse NumFiles. + if (!std::getline(IS, Line, '\n')) return false; + std::istringstream L1(Line); + size_t NumFiles = 0; + L1 >> NumFiles; + if (NumFiles == 0 || NumFiles > 10000000) return false; + + // Parse NumFilesInFirstCorpus. + if (!std::getline(IS, Line, '\n')) return false; + std::istringstream L2(Line); + NumFilesInFirstCorpus = NumFiles + 1; + L2 >> NumFilesInFirstCorpus; + if (NumFilesInFirstCorpus > NumFiles) return false; + + // Parse file names. + Files.resize(NumFiles); + for (size_t i = 0; i < NumFiles; i++) + if (!std::getline(IS, Files[i].Name, '\n')) + return false; + + // Parse STARTED, FT, and COV lines. + size_t ExpectedStartMarker = 0; + const size_t kInvalidStartMarker = -1; + size_t LastSeenStartMarker = kInvalidStartMarker; + Vector TmpFeatures; + Set PCs; + while (std::getline(IS, Line, '\n')) { + std::istringstream ISS1(Line); + std::string Marker; + size_t N; + ISS1 >> Marker; + ISS1 >> N; + if (Marker == "STARTED") { + // STARTED FILE_ID FILE_SIZE + if (ExpectedStartMarker != N) + return false; + ISS1 >> Files[ExpectedStartMarker].Size; + LastSeenStartMarker = ExpectedStartMarker; + assert(ExpectedStartMarker < Files.size()); + ExpectedStartMarker++; + } else if (Marker == "FT") { + // FT FILE_ID COV1 COV2 COV3 ... + size_t CurrentFileIdx = N; + if (CurrentFileIdx != LastSeenStartMarker) + return false; + LastSeenStartMarker = kInvalidStartMarker; + if (ParseCoverage) { + TmpFeatures.clear(); // use a vector from outer scope to avoid resizes. + while (ISS1 >> N) + TmpFeatures.push_back(N); + std::sort(TmpFeatures.begin(), TmpFeatures.end()); + Files[CurrentFileIdx].Features = TmpFeatures; + } + } else if (Marker == "COV") { + size_t CurrentFileIdx = N; + if (ParseCoverage) + while (ISS1 >> N) + if (PCs.insert(N).second) + Files[CurrentFileIdx].Cov.push_back(N); + } else { + return false; + } + } + if (LastSeenStartMarker != kInvalidStartMarker) + LastFailure = Files[LastSeenStartMarker].Name; + + FirstNotProcessedFile = ExpectedStartMarker; + return true; +} + +size_t Merger::ApproximateMemoryConsumption() const { + size_t Res = 0; + for (const auto &F: Files) + Res += sizeof(F) + F.Features.size() * sizeof(F.Features[0]); + return Res; +} + +// Decides which files need to be merged (add those to NewFiles). +// Returns the number of new features added. +size_t Merger::Merge(const Set &InitialFeatures, + Set *NewFeatures, + const Set &InitialCov, Set *NewCov, + Vector *NewFiles) { + NewFiles->clear(); + assert(NumFilesInFirstCorpus <= Files.size()); + Set AllFeatures = InitialFeatures; + + // What features are in the initial corpus? + for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { + auto &Cur = Files[i].Features; + AllFeatures.insert(Cur.begin(), Cur.end()); + } + // Remove all features that we already know from all other inputs. + for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { + auto &Cur = Files[i].Features; + Vector Tmp; + std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(), + AllFeatures.end(), std::inserter(Tmp, Tmp.begin())); + Cur.swap(Tmp); + } + + // Sort. Give preference to + // * smaller files + // * files with more features. + std::sort(Files.begin() + NumFilesInFirstCorpus, Files.end(), + [&](const MergeFileInfo &a, const MergeFileInfo &b) -> bool { + if (a.Size != b.Size) + return a.Size < b.Size; + return a.Features.size() > b.Features.size(); + }); + + // One greedy pass: add the file's features to AllFeatures. + // If new features were added, add this file to NewFiles. + for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { + auto &Cur = Files[i].Features; + // Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(), + // Files[i].Size, Cur.size()); + bool FoundNewFeatures = false; + for (auto Fe: Cur) { + if (AllFeatures.insert(Fe).second) { + FoundNewFeatures = true; + NewFeatures->insert(Fe); + } + } + if (FoundNewFeatures) + NewFiles->push_back(Files[i].Name); + for (auto Cov : Files[i].Cov) + if (InitialCov.find(Cov) == InitialCov.end()) + NewCov->insert(Cov); + } + return NewFeatures->size(); +} + +Set Merger::AllFeatures() const { + Set S; + for (auto &File : Files) + S.insert(File.Features.begin(), File.Features.end()); + return S; +} + +// Inner process. May crash if the target crashes. +int Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { + Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); + Merger M; + std::ifstream IF(CFPath); + int Res = M.ParseOrExit(IF, false); + if (Res != 0) + return Res; + IF.close(); + if (!M.LastFailure.empty()) + Printf("MERGE-INNER: '%s' caused a failure at the previous merge step\n", + M.LastFailure.c_str()); + + Printf("MERGE-INNER: %zd total files;" + " %zd processed earlier; will process %zd files now\n", + M.Files.size(), M.FirstNotProcessedFile, + M.Files.size() - M.FirstNotProcessedFile); + + std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); + Set AllFeatures; + auto PrintStatsWrapper = [this, &AllFeatures](const char* Where) { + this->PrintStats(Where, "\n", 0, AllFeatures.size()); + }; + Set AllPCs; + for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { + if (Fuzzer::MaybeExitGracefully()) + return 0; + auto U = FileToVector(M.Files[i].Name); + if (U.size() > MaxInputLen) { + U.resize(MaxInputLen); + U.shrink_to_fit(); + } + + // Write the pre-run marker. + OF << "STARTED " << i << " " << U.size() << "\n"; + OF.flush(); // Flush is important since Command::Execute may crash. + // Run. + TPC.ResetMaps(); + if (ExecuteCallback(U.data(), U.size()) > 0) { + continue; + } + // Collect coverage. We are iterating over the files in this order: + // * First, files in the initial corpus ordered by size, smallest first. + // * Then, all other files, smallest first. + // So it makes no sense to record all features for all files, instead we + // only record features that were not seen before. + Set UniqFeatures; + TPC.CollectFeatures([&](size_t Feature) { + if (AllFeatures.insert(Feature).second) + UniqFeatures.insert(Feature); + }); + TPC.UpdateObservedPCs(); + // Show stats. + if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) + PrintStatsWrapper("pulse "); + if (TotalNumberOfRuns == M.NumFilesInFirstCorpus) + PrintStatsWrapper("LOADED"); + // Write the post-run marker and the coverage. + OF << "FT " << i; + for (size_t F : UniqFeatures) + OF << " " << F; + OF << "\n"; + OF << "COV " << i; + TPC.ForEachObservedPC([&](const TracePC::PCTableEntry *TE) { + if (AllPCs.insert(TE).second) + OF << " " << TPC.PCTableEntryIdx(TE); + }); + OF << "\n"; + OF.flush(); + } + PrintStatsWrapper("DONE "); + return 0; +} + +static int WriteNewControlFile(const std::string &CFPath, + const Vector &OldCorpus, + const Vector &NewCorpus, + const Vector &KnownFiles, + size_t &NumFiles) { + std::unordered_set FilesToSkip; + for (auto &SF: KnownFiles) + FilesToSkip.insert(SF.Name); + + Vector FilesToUse; + auto MaybeUseFile = [=, &FilesToUse](std::string Name) { + if (FilesToSkip.find(Name) == FilesToSkip.end()) + FilesToUse.push_back(Name); + }; + for (auto &SF: OldCorpus) + MaybeUseFile(SF.File); + auto FilesToUseFromOldCorpus = FilesToUse.size(); + for (auto &SF: NewCorpus) + MaybeUseFile(SF.File); + + RemoveFile(CFPath); + std::ofstream ControlFile(CFPath); + ControlFile << FilesToUse.size() << "\n"; + ControlFile << FilesToUseFromOldCorpus << "\n"; + for (auto &FN: FilesToUse) + ControlFile << FN << "\n"; + + if (!ControlFile) { + Printf("MERGE-OUTER: failed to write to the control file: %s\n", + CFPath.c_str()); + return 1; + } + + NumFiles = FilesToUse.size(); + return 0; +} + +// Outer process. Does not call the target code and thus should not fail. +int CrashResistantMerge(const Vector &Args, + const Vector &OldCorpus, + const Vector &NewCorpus, + Vector *NewFiles, + const Set &InitialFeatures, + Set *NewFeatures, + const Set &InitialCov, + Set *NewCov, + const std::string &CFPath, + bool V /*Verbose*/) { + if (NewCorpus.empty() && OldCorpus.empty()) return 0; // Nothing to merge. + size_t NumAttempts = 0; + int Res; + Vector KnownFiles; + if (FileSize(CFPath)) { + VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", + CFPath.c_str()); + Merger M; + std::ifstream IF(CFPath); + if (M.Parse(IF, /*ParseCoverage=*/true)) { + VPrintf(V, "MERGE-OUTER: control file ok, %zd files total," + " first not processed file %zd\n", + M.Files.size(), M.FirstNotProcessedFile); + if (!M.LastFailure.empty()) + VPrintf(V, "MERGE-OUTER: '%s' will be skipped as unlucky " + "(merge has stumbled on it the last time)\n", + M.LastFailure.c_str()); + if (M.FirstNotProcessedFile >= M.Files.size()) { + // Merge has already been completed with the given merge control file. + if (M.Files.size() == OldCorpus.size() + NewCorpus.size()) { + VPrintf( + V, + "MERGE-OUTER: nothing to do, merge has been completed before\n"); + Fuzzer::GracefullyExit(); + return 0; + } + + // Number of input files likely changed, start merge from scratch, but + // reuse coverage information from the given merge control file. + VPrintf( + V, + "MERGE-OUTER: starting merge from scratch, but reusing coverage " + "information from the given control file\n"); + KnownFiles = M.Files; + } else { + // There is a merge in progress, continue. + NumAttempts = M.Files.size() - M.FirstNotProcessedFile; + } + } else { + VPrintf(V, "MERGE-OUTER: bad control file, will overwrite it\n"); + } + } + + if (!NumAttempts) { + // The supplied control file is empty or bad, create a fresh one. + VPrintf(V, "MERGE-OUTER: " + "%zd files, %zd in the initial corpus, %zd processed earlier\n", + OldCorpus.size() + NewCorpus.size(), OldCorpus.size(), + KnownFiles.size()); + Res = WriteNewControlFile(CFPath, OldCorpus, NewCorpus, KnownFiles, NumAttempts); + if (Res != 0) + return Res; + } + + // Execute the inner process until it passes. + // Every inner process should execute at least one input. + Command BaseCmd(Args); + BaseCmd.removeFlag("merge"); + BaseCmd.removeFlag("fork"); + BaseCmd.removeFlag("collect_data_flow"); + for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { + if (Fuzzer::MaybeExitGracefully()) + return 0; + VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); + Command Cmd(BaseCmd); + Cmd.addFlag("merge_control_file", CFPath); + Cmd.addFlag("merge_inner", "1"); + if (!V) { + Cmd.setOutputFile(getDevNull()); + Cmd.combineOutAndErr(); + } + auto ExitCode = ExecuteCommand(Cmd); + if (!ExitCode) { + VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); + break; + } + } + // Read the control file and do the merge. + Merger M; + std::ifstream IF(CFPath); + IF.seekg(0, IF.end); + VPrintf(V, "MERGE-OUTER: the control file has %zd bytes\n", + (size_t)IF.tellg()); + IF.seekg(0, IF.beg); + Res = M.ParseOrExit(IF, true); + if (Res != 0) + return Res; + IF.close(); + VPrintf(V, + "MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", + M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); + + M.Files.insert(M.Files.end(), KnownFiles.begin(), KnownFiles.end()); + M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " + "%zd new coverage edges\n", + NewFiles->size(), NewFeatures->size(), NewCov->size()); + return 0; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.h b/tools/fuzzing/libfuzzer/FuzzerMerge.h new file mode 100644 index 0000000000..6dc1c4c45a --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMerge.h @@ -0,0 +1,87 @@ +//===- FuzzerMerge.h - merging corpa ----------------------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Merging Corpora. +// +// The task: +// Take the existing corpus (possibly empty) and merge new inputs into +// it so that only inputs with new coverage ('features') are added. +// The process should tolerate the crashes, OOMs, leaks, etc. +// +// Algorithm: +// The outer process collects the set of files and writes their names +// into a temporary "control" file, then repeatedly launches the inner +// process until all inputs are processed. +// The outer process does not actually execute the target code. +// +// The inner process reads the control file and sees a) list of all the inputs +// and b) the last processed input. Then it starts processing the inputs one +// by one. Before processing every input it writes one line to control file: +// STARTED INPUT_ID INPUT_SIZE +// After processing an input it writes the following lines: +// FT INPUT_ID Feature1 Feature2 Feature3 ... +// COV INPUT_ID Coverage1 Coverage2 Coverage3 ... +// If a crash happens while processing an input the last line in the control +// file will be "STARTED INPUT_ID" and so the next process will know +// where to resume. +// +// Once all inputs are processed by the inner process(es) the outer process +// reads the control files and does the merge based entirely on the contents +// of control file. +// It uses a single pass greedy algorithm choosing first the smallest inputs +// within the same size the inputs that have more new features. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_MERGE_H +#define LLVM_FUZZER_MERGE_H + +#include "FuzzerDefs.h" + +#include +#include +#include +#include + +namespace fuzzer { + +struct MergeFileInfo { + std::string Name; + size_t Size = 0; + Vector Features, Cov; +}; + +struct Merger { + Vector Files; + size_t NumFilesInFirstCorpus = 0; + size_t FirstNotProcessedFile = 0; + std::string LastFailure; + + bool Parse(std::istream &IS, bool ParseCoverage); + bool Parse(const std::string &Str, bool ParseCoverage); + int ParseOrExit(std::istream &IS, bool ParseCoverage); + size_t Merge(const Set &InitialFeatures, Set *NewFeatures, + const Set &InitialCov, Set *NewCov, + Vector *NewFiles); + size_t ApproximateMemoryConsumption() const; + Set AllFeatures() const; +}; + +int CrashResistantMerge(const Vector &Args, + const Vector &OldCorpus, + const Vector &NewCorpus, + Vector *NewFiles, + const Set &InitialFeatures, + Set *NewFeatures, + const Set &InitialCov, + Set *NewCov, + const std::string &CFPath, + bool Verbose); + +} // namespace fuzzer + +#endif // LLVM_FUZZER_MERGE_H diff --git a/tools/fuzzing/libfuzzer/FuzzerMutate.cpp b/tools/fuzzing/libfuzzer/FuzzerMutate.cpp new file mode 100644 index 0000000000..29541eac5d --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMutate.cpp @@ -0,0 +1,562 @@ +//===- FuzzerMutate.cpp - Mutate a test input -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Mutate a test input. +//===----------------------------------------------------------------------===// + +#include "FuzzerDefs.h" +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "FuzzerMutate.h" +#include "FuzzerOptions.h" +#include "FuzzerTracePC.h" + +namespace fuzzer { + +const size_t Dictionary::kMaxDictSize; + +static void PrintASCII(const Word &W, const char *PrintAfter) { + PrintASCII(W.data(), W.size(), PrintAfter); +} + +MutationDispatcher::MutationDispatcher(Random &Rand, + const FuzzingOptions &Options) + : Rand(Rand), Options(Options) { + DefaultMutators.insert( + DefaultMutators.begin(), + { + {&MutationDispatcher::Mutate_EraseBytes, "EraseBytes"}, + {&MutationDispatcher::Mutate_InsertByte, "InsertByte"}, + {&MutationDispatcher::Mutate_InsertRepeatedBytes, + "InsertRepeatedBytes"}, + {&MutationDispatcher::Mutate_ChangeByte, "ChangeByte"}, + {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, + {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, + {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, + {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, + {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, + {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, + {&MutationDispatcher::Mutate_AddWordFromManualDictionary, + "ManualDict"}, + {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, + "PersAutoDict"}, + }); + if(Options.UseCmp) + DefaultMutators.push_back( + {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); + + if (EF->LLVMFuzzerCustomMutator) + Mutators.push_back({&MutationDispatcher::Mutate_Custom, "Custom"}); + else + Mutators = DefaultMutators; + + if (EF->LLVMFuzzerCustomCrossOver) + Mutators.push_back( + {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); +} + +static char RandCh(Random &Rand) { + if (Rand.RandBool()) return Rand(256); + const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00"; + return Special[Rand(sizeof(Special) - 1)]; +} + +size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, + size_t MaxSize) { + return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand()); +} + +size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size == 0) + return 0; + if (!CrossOverWith) return 0; + const Unit &Other = *CrossOverWith; + if (Other.empty()) + return 0; + CustomCrossOverInPlaceHere.resize(MaxSize); + auto &U = CustomCrossOverInPlaceHere; + size_t NewSize = EF->LLVMFuzzerCustomCrossOver( + Data, Size, Other.data(), Other.size(), U.data(), U.size(), Rand.Rand()); + if (!NewSize) + return 0; + assert(NewSize <= MaxSize && "CustomCrossOver returned overisized unit"); + memcpy(Data, U.data(), NewSize); + return NewSize; +} + +size_t MutationDispatcher::Mutate_ShuffleBytes(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize || Size == 0) return 0; + size_t ShuffleAmount = + Rand(std::min(Size, (size_t)8)) + 1; // [1,8] and <= Size. + size_t ShuffleStart = Rand(Size - ShuffleAmount); + assert(ShuffleStart + ShuffleAmount <= Size); + std::shuffle(Data + ShuffleStart, Data + ShuffleStart + ShuffleAmount, Rand); + return Size; +} + +size_t MutationDispatcher::Mutate_EraseBytes(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size <= 1) return 0; + size_t N = Rand(Size / 2) + 1; + assert(N < Size); + size_t Idx = Rand(Size - N + 1); + // Erase Data[Idx:Idx+N]. + memmove(Data + Idx, Data + Idx + N, Size - Idx - N); + // Printf("Erase: %zd %zd => %zd; Idx %zd\n", N, Size, Size - N, Idx); + return Size - N; +} + +size_t MutationDispatcher::Mutate_InsertByte(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size >= MaxSize) return 0; + size_t Idx = Rand(Size + 1); + // Insert new value at Data[Idx]. + memmove(Data + Idx + 1, Data + Idx, Size - Idx); + Data[Idx] = RandCh(Rand); + return Size + 1; +} + +size_t MutationDispatcher::Mutate_InsertRepeatedBytes(uint8_t *Data, + size_t Size, + size_t MaxSize) { + const size_t kMinBytesToInsert = 3; + if (Size + kMinBytesToInsert >= MaxSize) return 0; + size_t MaxBytesToInsert = std::min(MaxSize - Size, (size_t)128); + size_t N = Rand(MaxBytesToInsert - kMinBytesToInsert + 1) + kMinBytesToInsert; + assert(Size + N <= MaxSize && N); + size_t Idx = Rand(Size + 1); + // Insert new values at Data[Idx]. + memmove(Data + Idx + N, Data + Idx, Size - Idx); + // Give preference to 0x00 and 0xff. + uint8_t Byte = Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255); + for (size_t i = 0; i < N; i++) + Data[Idx + i] = Byte; + return Size + N; +} + +size_t MutationDispatcher::Mutate_ChangeByte(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + size_t Idx = Rand(Size); + Data[Idx] = RandCh(Rand); + return Size; +} + +size_t MutationDispatcher::Mutate_ChangeBit(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + size_t Idx = Rand(Size); + Data[Idx] ^= 1 << Rand(8); + return Size; +} + +size_t MutationDispatcher::Mutate_AddWordFromManualDictionary(uint8_t *Data, + size_t Size, + size_t MaxSize) { + return AddWordFromDictionary(ManualDictionary, Data, Size, MaxSize); +} + +size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size, + size_t MaxSize, + DictionaryEntry &DE) { + const Word &W = DE.GetW(); + bool UsePositionHint = DE.HasPositionHint() && + DE.GetPositionHint() + W.size() < Size && + Rand.RandBool(); + if (Rand.RandBool()) { // Insert W. + if (Size + W.size() > MaxSize) return 0; + size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1); + memmove(Data + Idx + W.size(), Data + Idx, Size - Idx); + memcpy(Data + Idx, W.data(), W.size()); + Size += W.size(); + } else { // Overwrite some bytes with W. + if (W.size() > Size) return 0; + size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size()); + memcpy(Data + Idx, W.data(), W.size()); + } + return Size; +} + +// Somewhere in the past we have observed a comparison instructions +// with arguments Arg1 Arg2. This function tries to guess a dictionary +// entry that will satisfy that comparison. +// It first tries to find one of the arguments (possibly swapped) in the +// input and if it succeeds it creates a DE with a position hint. +// Otherwise it creates a DE with one of the arguments w/o a position hint. +DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( + const void *Arg1, const void *Arg2, + const void *Arg1Mutation, const void *Arg2Mutation, + size_t ArgSize, const uint8_t *Data, + size_t Size) { + bool HandleFirst = Rand.RandBool(); + const void *ExistingBytes, *DesiredBytes; + Word W; + const uint8_t *End = Data + Size; + for (int Arg = 0; Arg < 2; Arg++) { + ExistingBytes = HandleFirst ? Arg1 : Arg2; + DesiredBytes = HandleFirst ? Arg2Mutation : Arg1Mutation; + HandleFirst = !HandleFirst; + W.Set(reinterpret_cast(DesiredBytes), ArgSize); + const size_t kMaxNumPositions = 8; + size_t Positions[kMaxNumPositions]; + size_t NumPositions = 0; + for (const uint8_t *Cur = Data; + Cur < End && NumPositions < kMaxNumPositions; Cur++) { + Cur = + (const uint8_t *)SearchMemory(Cur, End - Cur, ExistingBytes, ArgSize); + if (!Cur) break; + Positions[NumPositions++] = Cur - Data; + } + if (!NumPositions) continue; + return DictionaryEntry(W, Positions[Rand(NumPositions)]); + } + DictionaryEntry DE(W); + return DE; +} + + +template +DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( + T Arg1, T Arg2, const uint8_t *Data, size_t Size) { + if (Rand.RandBool()) Arg1 = Bswap(Arg1); + if (Rand.RandBool()) Arg2 = Bswap(Arg2); + T Arg1Mutation = Arg1 + Rand(-1, 1); + T Arg2Mutation = Arg2 + Rand(-1, 1); + return MakeDictionaryEntryFromCMP(&Arg1, &Arg2, &Arg1Mutation, &Arg2Mutation, + sizeof(Arg1), Data, Size); +} + +DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( + const Word &Arg1, const Word &Arg2, const uint8_t *Data, size_t Size) { + return MakeDictionaryEntryFromCMP(Arg1.data(), Arg2.data(), Arg1.data(), + Arg2.data(), Arg1.size(), Data, Size); +} + +size_t MutationDispatcher::Mutate_AddWordFromTORC( + uint8_t *Data, size_t Size, size_t MaxSize) { + Word W; + DictionaryEntry DE; + switch (Rand(4)) { + case 0: { + auto X = TPC.TORC8.Get(Rand.Rand()); + DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); + } break; + case 1: { + auto X = TPC.TORC4.Get(Rand.Rand()); + if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool()) + DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data, Size); + else + DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); + } break; + case 2: { + auto X = TPC.TORCW.Get(Rand.Rand()); + DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); + } break; + case 3: if (Options.UseMemmem) { + auto X = TPC.MMT.Get(Rand.Rand()); + DE = DictionaryEntry(X); + } break; + default: + assert(0); + } + if (!DE.GetW().size()) return 0; + Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE); + if (!Size) return 0; + DictionaryEntry &DERef = + CmpDictionaryEntriesDeque[CmpDictionaryEntriesDequeIdx++ % + kCmpDictionaryEntriesDequeSize]; + DERef = DE; + CurrentDictionaryEntrySequence.push_back(&DERef); + return Size; +} + +size_t MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary( + uint8_t *Data, size_t Size, size_t MaxSize) { + return AddWordFromDictionary(PersistentAutoDictionary, Data, Size, MaxSize); +} + +size_t MutationDispatcher::AddWordFromDictionary(Dictionary &D, uint8_t *Data, + size_t Size, size_t MaxSize) { + if (Size > MaxSize) return 0; + if (D.empty()) return 0; + DictionaryEntry &DE = D[Rand(D.size())]; + Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE); + if (!Size) return 0; + DE.IncUseCount(); + CurrentDictionaryEntrySequence.push_back(&DE); + return Size; +} + +// Overwrites part of To[0,ToSize) with a part of From[0,FromSize). +// Returns ToSize. +size_t MutationDispatcher::CopyPartOf(const uint8_t *From, size_t FromSize, + uint8_t *To, size_t ToSize) { + // Copy From[FromBeg, FromBeg + CopySize) into To[ToBeg, ToBeg + CopySize). + size_t ToBeg = Rand(ToSize); + size_t CopySize = Rand(ToSize - ToBeg) + 1; + assert(ToBeg + CopySize <= ToSize); + CopySize = std::min(CopySize, FromSize); + size_t FromBeg = Rand(FromSize - CopySize + 1); + assert(FromBeg + CopySize <= FromSize); + memmove(To + ToBeg, From + FromBeg, CopySize); + return ToSize; +} + +// Inserts part of From[0,ToSize) into To. +// Returns new size of To on success or 0 on failure. +size_t MutationDispatcher::InsertPartOf(const uint8_t *From, size_t FromSize, + uint8_t *To, size_t ToSize, + size_t MaxToSize) { + if (ToSize >= MaxToSize) return 0; + size_t AvailableSpace = MaxToSize - ToSize; + size_t MaxCopySize = std::min(AvailableSpace, FromSize); + size_t CopySize = Rand(MaxCopySize) + 1; + size_t FromBeg = Rand(FromSize - CopySize + 1); + assert(FromBeg + CopySize <= FromSize); + size_t ToInsertPos = Rand(ToSize + 1); + assert(ToInsertPos + CopySize <= MaxToSize); + size_t TailSize = ToSize - ToInsertPos; + if (To == From) { + MutateInPlaceHere.resize(MaxToSize); + memcpy(MutateInPlaceHere.data(), From + FromBeg, CopySize); + memmove(To + ToInsertPos + CopySize, To + ToInsertPos, TailSize); + memmove(To + ToInsertPos, MutateInPlaceHere.data(), CopySize); + } else { + memmove(To + ToInsertPos + CopySize, To + ToInsertPos, TailSize); + memmove(To + ToInsertPos, From + FromBeg, CopySize); + } + return ToSize + CopySize; +} + +size_t MutationDispatcher::Mutate_CopyPart(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize || Size == 0) return 0; + // If Size == MaxSize, `InsertPartOf(...)` will + // fail so there's no point using it in this case. + if (Size == MaxSize || Rand.RandBool()) + return CopyPartOf(Data, Size, Data, Size); + else + return InsertPartOf(Data, Size, Data, Size, MaxSize); +} + +size_t MutationDispatcher::Mutate_ChangeASCIIInteger(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + size_t B = Rand(Size); + while (B < Size && !isdigit(Data[B])) B++; + if (B == Size) return 0; + size_t E = B; + while (E < Size && isdigit(Data[E])) E++; + assert(B < E); + // now we have digits in [B, E). + // strtol and friends don't accept non-zero-teminated data, parse it manually. + uint64_t Val = Data[B] - '0'; + for (size_t i = B + 1; i < E; i++) + Val = Val * 10 + Data[i] - '0'; + + // Mutate the integer value. + switch(Rand(5)) { + case 0: Val++; break; + case 1: Val--; break; + case 2: Val /= 2; break; + case 3: Val *= 2; break; + case 4: Val = Rand(Val * Val); break; + default: assert(0); + } + // Just replace the bytes with the new ones, don't bother moving bytes. + for (size_t i = B; i < E; i++) { + size_t Idx = E + B - i - 1; + assert(Idx >= B && Idx < E); + Data[Idx] = (Val % 10) + '0'; + Val /= 10; + } + return Size; +} + +template +size_t ChangeBinaryInteger(uint8_t *Data, size_t Size, Random &Rand) { + if (Size < sizeof(T)) return 0; + size_t Off = Rand(Size - sizeof(T) + 1); + assert(Off + sizeof(T) <= Size); + T Val; + if (Off < 64 && !Rand(4)) { + Val = Size; + if (Rand.RandBool()) + Val = Bswap(Val); + } else { + memcpy(&Val, Data + Off, sizeof(Val)); + T Add = Rand(21); + Add -= 10; + if (Rand.RandBool()) + Val = Bswap(T(Bswap(Val) + Add)); // Add assuming different endiannes. + else + Val = Val + Add; // Add assuming current endiannes. + if (Add == 0 || Rand.RandBool()) // Maybe negate. + Val = -Val; + } + memcpy(Data + Off, &Val, sizeof(Val)); + return Size; +} + +size_t MutationDispatcher::Mutate_ChangeBinaryInteger(uint8_t *Data, + size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + switch (Rand(4)) { + case 3: return ChangeBinaryInteger(Data, Size, Rand); + case 2: return ChangeBinaryInteger(Data, Size, Rand); + case 1: return ChangeBinaryInteger(Data, Size, Rand); + case 0: return ChangeBinaryInteger(Data, Size, Rand); + default: assert(0); + } + return 0; +} + +size_t MutationDispatcher::Mutate_CrossOver(uint8_t *Data, size_t Size, + size_t MaxSize) { + if (Size > MaxSize) return 0; + if (Size == 0) return 0; + if (!CrossOverWith) return 0; + const Unit &O = *CrossOverWith; + if (O.empty()) return 0; + MutateInPlaceHere.resize(MaxSize); + auto &U = MutateInPlaceHere; + size_t NewSize = 0; + switch(Rand(3)) { + case 0: + NewSize = CrossOver(Data, Size, O.data(), O.size(), U.data(), U.size()); + break; + case 1: + NewSize = InsertPartOf(O.data(), O.size(), U.data(), U.size(), MaxSize); + if (!NewSize) + NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size()); + break; + case 2: + NewSize = CopyPartOf(O.data(), O.size(), U.data(), U.size()); + break; + default: assert(0); + } + assert(NewSize > 0 && "CrossOver returned empty unit"); + assert(NewSize <= MaxSize && "CrossOver returned overisized unit"); + memcpy(Data, U.data(), NewSize); + return NewSize; +} + +void MutationDispatcher::StartMutationSequence() { + CurrentMutatorSequence.clear(); + CurrentDictionaryEntrySequence.clear(); +} + +// Copy successful dictionary entries to PersistentAutoDictionary. +void MutationDispatcher::RecordSuccessfulMutationSequence() { + for (auto DE : CurrentDictionaryEntrySequence) { + // PersistentAutoDictionary.AddWithSuccessCountOne(DE); + DE->IncSuccessCount(); + assert(DE->GetW().size()); + // Linear search is fine here as this happens seldom. + if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) + PersistentAutoDictionary.push_back({DE->GetW(), 1}); + } +} + +void MutationDispatcher::PrintRecommendedDictionary() { + Vector V; + for (auto &DE : PersistentAutoDictionary) + if (!ManualDictionary.ContainsWord(DE.GetW())) + V.push_back(DE); + if (V.empty()) return; + Printf("###### Recommended dictionary. ######\n"); + for (auto &DE: V) { + assert(DE.GetW().size()); + Printf("\""); + PrintASCII(DE.GetW(), "\""); + Printf(" # Uses: %zd\n", DE.GetUseCount()); + } + Printf("###### End of recommended dictionary. ######\n"); +} + +void MutationDispatcher::PrintMutationSequence() { + Printf("MS: %zd ", CurrentMutatorSequence.size()); + for (auto M : CurrentMutatorSequence) + Printf("%s-", M.Name); + if (!CurrentDictionaryEntrySequence.empty()) { + Printf(" DE: "); + for (auto DE : CurrentDictionaryEntrySequence) { + Printf("\""); + PrintASCII(DE->GetW(), "\"-"); + } + } +} + +size_t MutationDispatcher::Mutate(uint8_t *Data, size_t Size, size_t MaxSize) { + return MutateImpl(Data, Size, MaxSize, Mutators); +} + +size_t MutationDispatcher::DefaultMutate(uint8_t *Data, size_t Size, + size_t MaxSize) { + return MutateImpl(Data, Size, MaxSize, DefaultMutators); +} + +// Mutates Data in place, returns new size. +size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, + size_t MaxSize, + Vector &Mutators) { + assert(MaxSize > 0); + // Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize), + // in which case they will return 0. + // Try several times before returning un-mutated data. + for (int Iter = 0; Iter < 100; Iter++) { + auto M = Mutators[Rand(Mutators.size())]; + size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize); + if (NewSize && NewSize <= MaxSize) { + if (Options.OnlyASCII) + ToASCII(Data, NewSize); + CurrentMutatorSequence.push_back(M); + return NewSize; + } + } + *Data = ' '; + return 1; // Fallback, should not happen frequently. +} + +// Mask represents the set of Data bytes that are worth mutating. +size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, + size_t MaxSize, + const Vector &Mask) { + size_t MaskedSize = std::min(Size, Mask.size()); + // * Copy the worthy bytes into a temporary array T + // * Mutate T + // * Copy T back. + // This is totally unoptimized. + auto &T = MutateWithMaskTemp; + if (T.size() < Size) + T.resize(Size); + size_t OneBits = 0; + for (size_t I = 0; I < MaskedSize; I++) + if (Mask[I]) + T[OneBits++] = Data[I]; + + if (!OneBits) return 0; + assert(!T.empty()); + size_t NewSize = Mutate(T.data(), OneBits, OneBits); + assert(NewSize <= OneBits); + (void)NewSize; + // Even if NewSize < OneBits we still use all OneBits bytes. + for (size_t I = 0, J = 0; I < MaskedSize; I++) + if (Mask[I]) + Data[I] = T[J++]; + return Size; +} + +void MutationDispatcher::AddWordToManualDictionary(const Word &W) { + ManualDictionary.push_back( + {W, std::numeric_limits::max()}); +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerMutate.h b/tools/fuzzing/libfuzzer/FuzzerMutate.h new file mode 100644 index 0000000000..6cbce80276 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerMutate.h @@ -0,0 +1,156 @@ +//===- FuzzerMutate.h - Internal header for the Fuzzer ----------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::MutationDispatcher +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_MUTATE_H +#define LLVM_FUZZER_MUTATE_H + +#include "FuzzerDefs.h" +#include "FuzzerDictionary.h" +#include "FuzzerOptions.h" +#include "FuzzerRandom.h" + +namespace fuzzer { + +class MutationDispatcher { +public: + MutationDispatcher(Random &Rand, const FuzzingOptions &Options); + ~MutationDispatcher() {} + /// Indicate that we are about to start a new sequence of mutations. + void StartMutationSequence(); + /// Print the current sequence of mutations. + void PrintMutationSequence(); + /// Indicate that the current sequence of mutations was successful. + void RecordSuccessfulMutationSequence(); + /// Mutates data by invoking user-provided mutator. + size_t Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by invoking user-provided crossover. + size_t Mutate_CustomCrossOver(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by shuffling bytes. + size_t Mutate_ShuffleBytes(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by erasing bytes. + size_t Mutate_EraseBytes(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by inserting a byte. + size_t Mutate_InsertByte(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by inserting several repeated bytes. + size_t Mutate_InsertRepeatedBytes(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by chanding one byte. + size_t Mutate_ChangeByte(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by chanding one bit. + size_t Mutate_ChangeBit(uint8_t *Data, size_t Size, size_t MaxSize); + /// Mutates data by copying/inserting a part of data into a different place. + size_t Mutate_CopyPart(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Mutates data by adding a word from the manual dictionary. + size_t Mutate_AddWordFromManualDictionary(uint8_t *Data, size_t Size, + size_t MaxSize); + + /// Mutates data by adding a word from the TORC. + size_t Mutate_AddWordFromTORC(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Mutates data by adding a word from the persistent automatic dictionary. + size_t Mutate_AddWordFromPersistentAutoDictionary(uint8_t *Data, size_t Size, + size_t MaxSize); + + /// Tries to find an ASCII integer in Data, changes it to another ASCII int. + size_t Mutate_ChangeASCIIInteger(uint8_t *Data, size_t Size, size_t MaxSize); + /// Change a 1-, 2-, 4-, or 8-byte integer in interesting ways. + size_t Mutate_ChangeBinaryInteger(uint8_t *Data, size_t Size, size_t MaxSize); + + /// CrossOver Data with CrossOverWith. + size_t Mutate_CrossOver(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Applies one of the configured mutations. + /// Returns the new size of data which could be up to MaxSize. + size_t Mutate(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Applies one of the configured mutations to the bytes of Data + /// that have '1' in Mask. + /// Mask.size() should be >= Size. + size_t MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, + const Vector &Mask); + + /// Applies one of the default mutations. Provided as a service + /// to mutation authors. + size_t DefaultMutate(uint8_t *Data, size_t Size, size_t MaxSize); + + /// Creates a cross-over of two pieces of Data, returns its size. + size_t CrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2, + size_t Size2, uint8_t *Out, size_t MaxOutSize); + + void AddWordToManualDictionary(const Word &W); + + void PrintRecommendedDictionary(); + + void SetCrossOverWith(const Unit *U) { CrossOverWith = U; } + + Random &GetRand() { return Rand; } + + private: + struct Mutator { + size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); + const char *Name; + }; + + size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, + size_t MaxSize); + size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, + Vector &Mutators); + + size_t InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, + size_t ToSize, size_t MaxToSize); + size_t CopyPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, + size_t ToSize); + size_t ApplyDictionaryEntry(uint8_t *Data, size_t Size, size_t MaxSize, + DictionaryEntry &DE); + + template + DictionaryEntry MakeDictionaryEntryFromCMP(T Arg1, T Arg2, + const uint8_t *Data, size_t Size); + DictionaryEntry MakeDictionaryEntryFromCMP(const Word &Arg1, const Word &Arg2, + const uint8_t *Data, size_t Size); + DictionaryEntry MakeDictionaryEntryFromCMP(const void *Arg1, const void *Arg2, + const void *Arg1Mutation, + const void *Arg2Mutation, + size_t ArgSize, + const uint8_t *Data, size_t Size); + + Random &Rand; + const FuzzingOptions Options; + + // Dictionary provided by the user via -dict=DICT_FILE. + Dictionary ManualDictionary; + // Temporary dictionary modified by the fuzzer itself, + // recreated periodically. + Dictionary TempAutoDictionary; + // Persistent dictionary modified by the fuzzer, consists of + // entries that led to successful discoveries in the past mutations. + Dictionary PersistentAutoDictionary; + + Vector CurrentDictionaryEntrySequence; + + static const size_t kCmpDictionaryEntriesDequeSize = 16; + DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; + size_t CmpDictionaryEntriesDequeIdx = 0; + + const Unit *CrossOverWith = nullptr; + Vector MutateInPlaceHere; + Vector MutateWithMaskTemp; + // CustomCrossOver needs its own buffer as a custom implementation may call + // LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere. + Vector CustomCrossOverInPlaceHere; + + Vector Mutators; + Vector DefaultMutators; + Vector CurrentMutatorSequence; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_MUTATE_H diff --git a/tools/fuzzing/libfuzzer/FuzzerOptions.h b/tools/fuzzing/libfuzzer/FuzzerOptions.h new file mode 100644 index 0000000000..9d975bd61f --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerOptions.h @@ -0,0 +1,85 @@ +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::FuzzingOptions +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_OPTIONS_H +#define LLVM_FUZZER_OPTIONS_H + +#include "FuzzerDefs.h" + +namespace fuzzer { + +struct FuzzingOptions { + int Verbosity = 1; + size_t MaxLen = 0; + size_t LenControl = 1000; + int UnitTimeoutSec = 300; + int TimeoutExitCode = 70; + int OOMExitCode = 71; + int InterruptExitCode = 72; + int ErrorExitCode = 77; + bool IgnoreTimeouts = true; + bool IgnoreOOMs = true; + bool IgnoreCrashes = false; + int MaxTotalTimeSec = 0; + int RssLimitMb = 0; + int MallocLimitMb = 0; + bool DoCrossOver = true; + int MutateDepth = 5; + bool ReduceDepth = false; + bool UseCounters = false; + bool UseMemmem = true; + bool UseCmp = false; + int UseValueProfile = false; + bool Shrink = false; + bool ReduceInputs = false; + int ReloadIntervalSec = 1; + bool ShuffleAtStartUp = true; + bool PreferSmall = true; + size_t MaxNumberOfRuns = -1L; + int ReportSlowUnits = 10; + bool OnlyASCII = false; + bool Entropic = false; + size_t EntropicFeatureFrequencyThreshold = 0xFF; + size_t EntropicNumberOfRarestFeatures = 100; + std::string OutputCorpus; + std::string ArtifactPrefix = "./"; + std::string ExactArtifactPath; + std::string ExitOnSrcPos; + std::string ExitOnItem; + std::string FocusFunction; + std::string DataFlowTrace; + std::string CollectDataFlow; + std::string FeaturesDir; + std::string StopFile; + bool SaveArtifacts = true; + bool PrintNEW = true; // Print a status line when new units are found; + bool PrintNewCovPcs = false; + int PrintNewCovFuncs = 0; + bool PrintFinalStats = false; + bool PrintCorpusStats = false; + bool PrintCoverage = false; + bool DumpCoverage = false; + bool DetectLeaks = true; + int PurgeAllocatorIntervalSec = 1; + int TraceMalloc = 0; + bool HandleAbrt = false; + bool HandleBus = false; + bool HandleFpe = false; + bool HandleIll = false; + bool HandleInt = false; + bool HandleSegv = false; + bool HandleTerm = false; + bool HandleXfsz = false; + bool HandleUsr1 = false; + bool HandleUsr2 = false; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_OPTIONS_H diff --git a/tools/fuzzing/libfuzzer/FuzzerPlatform.h b/tools/fuzzing/libfuzzer/FuzzerPlatform.h new file mode 100644 index 0000000000..8befdb882c --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerPlatform.h @@ -0,0 +1,163 @@ +//===-- FuzzerPlatform.h --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Common platform macros. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_PLATFORM_H +#define LLVM_FUZZER_PLATFORM_H + +// Platform detection. +#ifdef __linux__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 1 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __APPLE__ +#define LIBFUZZER_APPLE 1 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __NetBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 1 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __FreeBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 1 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __OpenBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 1 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif _WIN32 +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 1 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __Fuchsia__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 1 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 0 +#elif __EMSCRIPTEN__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_OPENBSD 0 +#define LIBFUZZER_WINDOWS 0 +#define LIBFUZZER_EMSCRIPTEN 1 +#else +#error "Support for your platform has not been implemented" +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +// MSVC compiler is being used. +#define LIBFUZZER_MSVC 1 +#else +#define LIBFUZZER_MSVC 0 +#endif + +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#define LIBFUZZER_POSIX \ + (LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD || \ + LIBFUZZER_FREEBSD || LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN) + +#ifdef __x86_64 +#if __has_attribute(target) +#define ATTRIBUTE_TARGET_POPCNT __attribute__((target("popcnt"))) +#else +#define ATTRIBUTE_TARGET_POPCNT +#endif +#else +#define ATTRIBUTE_TARGET_POPCNT +#endif + +#ifdef __clang__ // avoid gcc warning. +#if __has_attribute(no_sanitize) +#define ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) +#else +#define ATTRIBUTE_NO_SANITIZE_MEMORY +#endif +#define ALWAYS_INLINE __attribute__((always_inline)) +#else +#define ATTRIBUTE_NO_SANITIZE_MEMORY +#define ALWAYS_INLINE +#endif // __clang__ + +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_NO_SANITIZE_ADDRESS +#else +#define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +#endif + +#if LIBFUZZER_WINDOWS +#define ATTRIBUTE_ALIGNED(X) __declspec(align(X)) +#define ATTRIBUTE_INTERFACE __declspec(dllexport) +// This is used for __sancov_lowest_stack which is needed for +// -fsanitize-coverage=stack-depth. That feature is not yet available on +// Windows, so make the symbol static to avoid linking errors. +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC static +#define ATTRIBUTE_NOINLINE __declspec(noinline) +#else +#define ATTRIBUTE_ALIGNED(X) __attribute__((aligned(X))) +#define ATTRIBUTE_INTERFACE __attribute__((visibility("default"))) +#define ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC \ + ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) thread_local + +#define ATTRIBUTE_NOINLINE __attribute__((noinline)) +#endif + +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_ADDRESS +#elif __has_feature(memory_sanitizer) +#define ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_NO_SANITIZE_MEMORY +#else +#define ATTRIBUTE_NO_SANITIZE_ALL +#endif +#else +#define ATTRIBUTE_NO_SANITIZE_ALL +#endif + +#endif // LLVM_FUZZER_PLATFORM_H diff --git a/tools/fuzzing/libfuzzer/FuzzerRandom.h b/tools/fuzzing/libfuzzer/FuzzerRandom.h new file mode 100644 index 0000000000..659283eee2 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerRandom.h @@ -0,0 +1,38 @@ +//===- FuzzerRandom.h - Internal header for the Fuzzer ----------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::Random +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_RANDOM_H +#define LLVM_FUZZER_RANDOM_H + +#include + +namespace fuzzer { +class Random : public std::minstd_rand { + public: + Random(unsigned int seed) : std::minstd_rand(seed) {} + result_type operator()() { return this->std::minstd_rand::operator()(); } + size_t Rand() { return this->operator()(); } + size_t RandBool() { return Rand() % 2; } + size_t SkewTowardsLast(size_t n) { + size_t T = this->operator()(n * n); + size_t Res = sqrt(T); + return Res; + } + size_t operator()(size_t n) { return n ? Rand() % n : 0; } + intptr_t operator()(intptr_t From, intptr_t To) { + assert(From < To); + intptr_t RangeSize = To - From + 1; + return operator()(RangeSize) + From; + } +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_RANDOM_H diff --git a/tools/fuzzing/libfuzzer/FuzzerSHA1.cpp b/tools/fuzzing/libfuzzer/FuzzerSHA1.cpp new file mode 100644 index 0000000000..2005dc7003 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerSHA1.cpp @@ -0,0 +1,223 @@ +//===- FuzzerSHA1.h - Private copy of the SHA1 implementation ---*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// This code is taken from public domain +// (http://oauth.googlecode.com/svn/code/c/liboauth/src/sha1.c) +// and modified by adding anonymous namespace, adding an interface +// function fuzzer::ComputeSHA1() and removing unnecessary code. +// +// lib/Fuzzer can not use SHA1 implementation from openssl because +// openssl may not be available and because we may be fuzzing openssl itself. +// For the same reason we do not want to depend on SHA1 from LLVM tree. +//===----------------------------------------------------------------------===// + +#include "FuzzerSHA1.h" +#include "FuzzerDefs.h" +#include "FuzzerPlatform.h" + +/* This code is public-domain - it is based on libcrypt + * placed in the public domain by Wei Dai and other contributors. + */ + +#include +#include +#include +#include + +namespace { // Added for LibFuzzer + +#ifdef __BIG_ENDIAN__ +# define SHA_BIG_ENDIAN +// Windows is always little endian and MSVC doesn't have +#elif defined __LITTLE_ENDIAN__ || LIBFUZZER_WINDOWS +/* override */ +#elif defined __BYTE_ORDER +# if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define SHA_BIG_ENDIAN +# endif +#else // ! defined __LITTLE_ENDIAN__ +# include // machine/endian.h +# if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +# define SHA_BIG_ENDIAN +# endif +#endif + + +/* header */ + +#define HASH_LENGTH 20 +#define BLOCK_LENGTH 64 + +typedef struct sha1nfo { + uint32_t buffer[BLOCK_LENGTH/4]; + uint32_t state[HASH_LENGTH/4]; + uint32_t byteCount; + uint8_t bufferOffset; + uint8_t keyBuffer[BLOCK_LENGTH]; + uint8_t innerHash[HASH_LENGTH]; +} sha1nfo; + +/* public API - prototypes - TODO: doxygen*/ + +/** + */ +void sha1_init(sha1nfo *s); +/** + */ +void sha1_writebyte(sha1nfo *s, uint8_t data); +/** + */ +void sha1_write(sha1nfo *s, const char *data, size_t len); +/** + */ +uint8_t* sha1_result(sha1nfo *s); + + +/* code */ +#define SHA1_K0 0x5a827999 +#define SHA1_K20 0x6ed9eba1 +#define SHA1_K40 0x8f1bbcdc +#define SHA1_K60 0xca62c1d6 + +void sha1_init(sha1nfo *s) { + s->state[0] = 0x67452301; + s->state[1] = 0xefcdab89; + s->state[2] = 0x98badcfe; + s->state[3] = 0x10325476; + s->state[4] = 0xc3d2e1f0; + s->byteCount = 0; + s->bufferOffset = 0; +} + +uint32_t sha1_rol32(uint32_t number, uint8_t bits) { + return ((number << bits) | (number >> (32-bits))); +} + +void sha1_hashBlock(sha1nfo *s) { + uint8_t i; + uint32_t a,b,c,d,e,t; + + a=s->state[0]; + b=s->state[1]; + c=s->state[2]; + d=s->state[3]; + e=s->state[4]; + for (i=0; i<80; i++) { + if (i>=16) { + t = s->buffer[(i+13)&15] ^ s->buffer[(i+8)&15] ^ s->buffer[(i+2)&15] ^ s->buffer[i&15]; + s->buffer[i&15] = sha1_rol32(t,1); + } + if (i<20) { + t = (d ^ (b & (c ^ d))) + SHA1_K0; + } else if (i<40) { + t = (b ^ c ^ d) + SHA1_K20; + } else if (i<60) { + t = ((b & c) | (d & (b | c))) + SHA1_K40; + } else { + t = (b ^ c ^ d) + SHA1_K60; + } + t+=sha1_rol32(a,5) + e + s->buffer[i&15]; + e=d; + d=c; + c=sha1_rol32(b,30); + b=a; + a=t; + } + s->state[0] += a; + s->state[1] += b; + s->state[2] += c; + s->state[3] += d; + s->state[4] += e; +} + +void sha1_addUncounted(sha1nfo *s, uint8_t data) { + uint8_t * const b = (uint8_t*) s->buffer; +#ifdef SHA_BIG_ENDIAN + b[s->bufferOffset] = data; +#else + b[s->bufferOffset ^ 3] = data; +#endif + s->bufferOffset++; + if (s->bufferOffset == BLOCK_LENGTH) { + sha1_hashBlock(s); + s->bufferOffset = 0; + } +} + +void sha1_writebyte(sha1nfo *s, uint8_t data) { + ++s->byteCount; + sha1_addUncounted(s, data); +} + +void sha1_write(sha1nfo *s, const char *data, size_t len) { + for (;len--;) sha1_writebyte(s, (uint8_t) *data++); +} + +void sha1_pad(sha1nfo *s) { + // Implement SHA-1 padding (fips180-2 §5.1.1) + + // Pad with 0x80 followed by 0x00 until the end of the block + sha1_addUncounted(s, 0x80); + while (s->bufferOffset != 56) sha1_addUncounted(s, 0x00); + + // Append length in the last 8 bytes + sha1_addUncounted(s, 0); // We're only using 32 bit lengths + sha1_addUncounted(s, 0); // But SHA-1 supports 64 bit lengths + sha1_addUncounted(s, 0); // So zero pad the top bits + sha1_addUncounted(s, s->byteCount >> 29); // Shifting to multiply by 8 + sha1_addUncounted(s, s->byteCount >> 21); // as SHA-1 supports bitstreams as well as + sha1_addUncounted(s, s->byteCount >> 13); // byte. + sha1_addUncounted(s, s->byteCount >> 5); + sha1_addUncounted(s, s->byteCount << 3); +} + +uint8_t* sha1_result(sha1nfo *s) { + // Pad to complete the last block + sha1_pad(s); + +#ifndef SHA_BIG_ENDIAN + // Swap byte order back + int i; + for (i=0; i<5; i++) { + s->state[i]= + (((s->state[i])<<24)& 0xff000000) + | (((s->state[i])<<8) & 0x00ff0000) + | (((s->state[i])>>8) & 0x0000ff00) + | (((s->state[i])>>24)& 0x000000ff); + } +#endif + + // Return pointer to hash (20 characters) + return (uint8_t*) s->state; +} + +} // namespace; Added for LibFuzzer + +namespace fuzzer { + +// The rest is added for LibFuzzer +void ComputeSHA1(const uint8_t *Data, size_t Len, uint8_t *Out) { + sha1nfo s; + sha1_init(&s); + sha1_write(&s, (const char*)Data, Len); + memcpy(Out, sha1_result(&s), HASH_LENGTH); +} + +std::string Sha1ToString(const uint8_t Sha1[kSHA1NumBytes]) { + std::stringstream SS; + for (int i = 0; i < kSHA1NumBytes; i++) + SS << std::hex << std::setfill('0') << std::setw(2) << (unsigned)Sha1[i]; + return SS.str(); +} + +std::string Hash(const Unit &U) { + uint8_t Hash[kSHA1NumBytes]; + ComputeSHA1(U.data(), U.size(), Hash); + return Sha1ToString(Hash); +} + +} diff --git a/tools/fuzzing/libfuzzer/FuzzerSHA1.h b/tools/fuzzing/libfuzzer/FuzzerSHA1.h new file mode 100644 index 0000000000..05cbacda87 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerSHA1.h @@ -0,0 +1,32 @@ +//===- FuzzerSHA1.h - Internal header for the SHA1 utils --------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// SHA1 utils. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_SHA1_H +#define LLVM_FUZZER_SHA1_H + +#include "FuzzerDefs.h" +#include +#include + +namespace fuzzer { + +// Private copy of SHA1 implementation. +static const int kSHA1NumBytes = 20; + +// Computes SHA1 hash of 'Len' bytes in 'Data', writes kSHA1NumBytes to 'Out'. +void ComputeSHA1(const uint8_t *Data, size_t Len, uint8_t *Out); + +std::string Sha1ToString(const uint8_t Sha1[kSHA1NumBytes]); + +std::string Hash(const Unit &U); + +} // namespace fuzzer + +#endif // LLVM_FUZZER_SHA1_H diff --git a/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp b/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp new file mode 100644 index 0000000000..fbceda39bc --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp @@ -0,0 +1,657 @@ +//===- FuzzerTracePC.cpp - PC tracing--------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Trace PCs. +// This module implements __sanitizer_cov_trace_pc_guard[_init], +// the callback required for -fsanitize-coverage=trace-pc-guard instrumentation. +// +//===----------------------------------------------------------------------===// + +#include "FuzzerTracePC.h" +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" +#include "FuzzerCorpus.h" +#include "FuzzerDefs.h" +#include "FuzzerDictionary.h" +#include "FuzzerExtFunctions.h" +#include "FuzzerIO.h" +#include "FuzzerPlatform.h" +#include "FuzzerUtil.h" +#include "FuzzerValueBitMap.h" +#include + +// Used by -fsanitize-coverage=stack-depth to track stack depth +ATTRIBUTES_INTERFACE_TLS_INITIAL_EXEC uintptr_t __sancov_lowest_stack; + +namespace fuzzer { + +TracePC TPC; + +size_t TracePC::GetTotalPCCoverage() { + return ObservedPCs.size(); +} + + +void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) { + if (Start == Stop) return; + if (NumModules && + Modules[NumModules - 1].Start() == Start) + return; + assert(NumModules < + sizeof(Modules) / sizeof(Modules[0])); + auto &M = Modules[NumModules++]; + uint8_t *AlignedStart = RoundUpByPage(Start); + uint8_t *AlignedStop = RoundDownByPage(Stop); + size_t NumFullPages = AlignedStop > AlignedStart ? + (AlignedStop - AlignedStart) / PageSize() : 0; + bool NeedFirst = Start < AlignedStart || !NumFullPages; + bool NeedLast = Stop > AlignedStop && AlignedStop >= AlignedStart; + M.NumRegions = NumFullPages + NeedFirst + NeedLast;; + assert(M.NumRegions > 0); + M.Regions = new Module::Region[M.NumRegions]; + assert(M.Regions); + size_t R = 0; + if (NeedFirst) + M.Regions[R++] = {Start, std::min(Stop, AlignedStart), true, false}; + for (uint8_t *P = AlignedStart; P < AlignedStop; P += PageSize()) + M.Regions[R++] = {P, P + PageSize(), true, true}; + if (NeedLast) + M.Regions[R++] = {AlignedStop, Stop, true, false}; + assert(R == M.NumRegions); + assert(M.Size() == (size_t)(Stop - Start)); + assert(M.Stop() == Stop); + assert(M.Start() == Start); + NumInline8bitCounters += M.Size(); +} + +void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) { + const PCTableEntry *B = reinterpret_cast(Start); + const PCTableEntry *E = reinterpret_cast(Stop); + if (NumPCTables && ModulePCTable[NumPCTables - 1].Start == B) return; + assert(NumPCTables < sizeof(ModulePCTable) / sizeof(ModulePCTable[0])); + ModulePCTable[NumPCTables++] = {B, E}; + NumPCsInPCTables += E - B; +} + +void TracePC::PrintModuleInfo() { + if (NumModules) { + Printf("INFO: Loaded %zd modules (%zd inline 8-bit counters): ", + NumModules, NumInline8bitCounters); + for (size_t i = 0; i < NumModules; i++) + Printf("%zd [%p, %p), ", Modules[i].Size(), Modules[i].Start(), + Modules[i].Stop()); + Printf("\n"); + } + if (NumPCTables) { + Printf("INFO: Loaded %zd PC tables (%zd PCs): ", NumPCTables, + NumPCsInPCTables); + for (size_t i = 0; i < NumPCTables; i++) { + Printf("%zd [%p,%p), ", ModulePCTable[i].Stop - ModulePCTable[i].Start, + ModulePCTable[i].Start, ModulePCTable[i].Stop); + } + Printf("\n"); + + if (NumInline8bitCounters && NumInline8bitCounters != NumPCsInPCTables) { + Printf("ERROR: The size of coverage PC tables does not match the\n" + "number of instrumented PCs. This might be a compiler bug,\n" + "please contact the libFuzzer developers.\n" + "Also check https://bugs.llvm.org/show_bug.cgi?id=34636\n" + "for possible workarounds (tl;dr: don't use the old GNU ld)\n"); + _Exit(1); + } + } + if (size_t NumExtraCounters = ExtraCountersEnd() - ExtraCountersBegin()) + Printf("INFO: %zd Extra Counters\n", NumExtraCounters); +} + +ATTRIBUTE_NO_SANITIZE_ALL +void TracePC::HandleCallerCallee(uintptr_t Caller, uintptr_t Callee) { + const uintptr_t kBits = 12; + const uintptr_t kMask = (1 << kBits) - 1; + uintptr_t Idx = (Caller & kMask) | ((Callee & kMask) << kBits); + ValueProfileMap.AddValueModPrime(Idx); +} + +/// \return the address of the previous instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.h` +inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { +#if defined(__arm__) + // T32 (Thumb) branch instructions might be 16 or 32 bit long, + // so we return (pc-2) in that case in order to be safe. + // For A32 mode we return (pc-4) because all instructions are 32 bit long. + return (PC - 3) & (~1); +#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__aarch64__) + // PCs are always 4 byte aligned. + return PC - 4; +#elif defined(__sparc__) || defined(__mips__) + return PC - 8; +#else + return PC - 1; +#endif +} + +/// \return the address of the next instruction. +/// Note: the logic is copied from `sanitizer_common/sanitizer_stacktrace.cpp` +ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { +#if defined(__mips__) + return PC + 8; +#elif defined(__powerpc__) || defined(__sparc__) || defined(__arm__) || \ + defined(__aarch64__) + return PC + 4; +#else + return PC + 1; +#endif +} + +void TracePC::UpdateObservedPCs() { + Vector CoveredFuncs; + auto ObservePC = [&](const PCTableEntry *TE) { + if (ObservedPCs.insert(TE).second && DoPrintNewPCs) { + PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", + GetNextInstructionPc(TE->PC)); + Printf("\n"); + } + }; + + auto Observe = [&](const PCTableEntry *TE) { + if (PcIsFuncEntry(TE)) + if (++ObservedFuncs[TE->PC] == 1 && NumPrintNewFuncs) + CoveredFuncs.push_back(TE->PC); + ObservePC(TE); + }; + + if (NumPCsInPCTables) { + if (NumInline8bitCounters == NumPCsInPCTables) { + for (size_t i = 0; i < NumModules; i++) { + auto &M = Modules[i]; + assert(M.Size() == + (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); + for (size_t r = 0; r < M.NumRegions; r++) { + auto &R = M.Regions[r]; + if (!R.Enabled) continue; + for (uint8_t *P = R.Start; P < R.Stop; P++) + if (*P) + Observe(&ModulePCTable[i].Start[M.Idx(P)]); + } + } + } + } + + for (size_t i = 0, N = Min(CoveredFuncs.size(), NumPrintNewFuncs); i < N; + i++) { + Printf("\tNEW_FUNC[%zd/%zd]: ", i + 1, CoveredFuncs.size()); + PrintPC("%p %F %L", "%p", GetNextInstructionPc(CoveredFuncs[i])); + Printf("\n"); + } +} + +uintptr_t TracePC::PCTableEntryIdx(const PCTableEntry *TE) { + size_t TotalTEs = 0; + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + if (TE >= M.Start && TE < M.Stop) + return TotalTEs + TE - M.Start; + TotalTEs += M.Stop - M.Start; + } + assert(0); + return 0; +} + +const TracePC::PCTableEntry *TracePC::PCTableEntryByIdx(uintptr_t Idx) { + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + size_t Size = M.Stop - M.Start; + if (Idx < Size) return &M.Start[Idx]; + Idx -= Size; + } + return nullptr; +} + +static std::string GetModuleName(uintptr_t PC) { + char ModulePathRaw[4096] = ""; // What's PATH_MAX in portable C++? + void *OffsetRaw = nullptr; + if (!EF->__sanitizer_get_module_and_offset_for_pc( + reinterpret_cast(PC), ModulePathRaw, + sizeof(ModulePathRaw), &OffsetRaw)) + return ""; + return ModulePathRaw; +} + +template +void TracePC::IterateCoveredFunctions(CallBack CB) { + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + assert(M.Start < M.Stop); + auto ModuleName = GetModuleName(M.Start->PC); + for (auto NextFE = M.Start; NextFE < M.Stop; ) { + auto FE = NextFE; + assert(PcIsFuncEntry(FE) && "Not a function entry point"); + do { + NextFE++; + } while (NextFE < M.Stop && !(PcIsFuncEntry(NextFE))); + CB(FE, NextFE, ObservedFuncs[FE->PC]); + } + } +} + +int TracePC::SetFocusFunction(const std::string &FuncName) { + // This function should be called once. + assert(!FocusFunctionCounterPtr); + // "auto" is not a valid function name. If this function is called with "auto" + // that means the auto focus functionality failed. + if (FuncName.empty() || FuncName == "auto") + return 0; + for (size_t M = 0; M < NumModules; M++) { + auto &PCTE = ModulePCTable[M]; + size_t N = PCTE.Stop - PCTE.Start; + for (size_t I = 0; I < N; I++) { + if (!(PcIsFuncEntry(&PCTE.Start[I]))) continue; // not a function entry. + auto Name = DescribePC("%F", GetNextInstructionPc(PCTE.Start[I].PC)); + if (Name[0] == 'i' && Name[1] == 'n' && Name[2] == ' ') + Name = Name.substr(3, std::string::npos); + if (FuncName != Name) continue; + Printf("INFO: Focus function is set to '%s'\n", Name.c_str()); + FocusFunctionCounterPtr = Modules[M].Start() + I; + return 0; + } + } + + Printf("ERROR: Failed to set focus function. Make sure the function name is " + "valid (%s) and symbolization is enabled.\n", FuncName.c_str()); + return 1; +} + +bool TracePC::ObservedFocusFunction() { + return FocusFunctionCounterPtr && *FocusFunctionCounterPtr; +} + +void TracePC::PrintCoverage() { + if (!EF->__sanitizer_symbolize_pc || + !EF->__sanitizer_get_module_and_offset_for_pc) { + Printf("INFO: __sanitizer_symbolize_pc or " + "__sanitizer_get_module_and_offset_for_pc is not available," + " not printing coverage\n"); + return; + } + Printf("COVERAGE:\n"); + auto CoveredFunctionCallback = [&](const PCTableEntry *First, + const PCTableEntry *Last, + uintptr_t Counter) { + assert(First < Last); + auto VisualizePC = GetNextInstructionPc(First->PC); + std::string FileStr = DescribePC("%s", VisualizePC); + if (!IsInterestingCoverageFile(FileStr)) + return; + std::string FunctionStr = DescribePC("%F", VisualizePC); + if (FunctionStr.find("in ") == 0) + FunctionStr = FunctionStr.substr(3); + std::string LineStr = DescribePC("%l", VisualizePC); + size_t NumEdges = Last - First; + Vector UncoveredPCs; + for (auto TE = First; TE < Last; TE++) + if (!ObservedPCs.count(TE)) + UncoveredPCs.push_back(TE->PC); + Printf("%sCOVERED_FUNC: hits: %zd", Counter ? "" : "UN", Counter); + Printf(" edges: %zd/%zd", NumEdges - UncoveredPCs.size(), NumEdges); + Printf(" %s %s:%s\n", FunctionStr.c_str(), FileStr.c_str(), + LineStr.c_str()); + if (Counter) + for (auto PC : UncoveredPCs) + Printf(" UNCOVERED_PC: %s\n", + DescribePC("%s:%l", GetNextInstructionPc(PC)).c_str()); + }; + + IterateCoveredFunctions(CoveredFunctionCallback); +} + +// Value profile. +// We keep track of various values that affect control flow. +// These values are inserted into a bit-set-based hash map. +// Every new bit in the map is treated as a new coverage. +// +// For memcmp/strcmp/etc the interesting value is the length of the common +// prefix of the parameters. +// For cmp instructions the interesting value is a XOR of the parameters. +// The interesting value is mixed up with the PC and is then added to the map. + +ATTRIBUTE_NO_SANITIZE_ALL +void TracePC::AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, + size_t n, bool StopAtZero) { + if (!n) return; + size_t Len = std::min(n, Word::GetMaxSize()); + const uint8_t *A1 = reinterpret_cast(s1); + const uint8_t *A2 = reinterpret_cast(s2); + uint8_t B1[Word::kMaxSize]; + uint8_t B2[Word::kMaxSize]; + // Copy the data into locals in this non-msan-instrumented function + // to avoid msan complaining further. + size_t Hash = 0; // Compute some simple hash of both strings. + for (size_t i = 0; i < Len; i++) { + B1[i] = A1[i]; + B2[i] = A2[i]; + size_t T = B1[i]; + Hash ^= (T << 8) | B2[i]; + } + size_t I = 0; + uint8_t HammingDistance = 0; + for (; I < Len; I++) { + if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) { + HammingDistance = Popcountll(B1[I] ^ B2[I]); + break; + } + } + size_t PC = reinterpret_cast(caller_pc); + size_t Idx = (PC & 4095) | (I << 12); + Idx += HammingDistance; + ValueProfileMap.AddValue(Idx); + TORCW.Insert(Idx ^ Hash, Word(B1, Len), Word(B2, Len)); +} + +template +ATTRIBUTE_TARGET_POPCNT ALWAYS_INLINE +ATTRIBUTE_NO_SANITIZE_ALL +void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) { + uint64_t ArgXor = Arg1 ^ Arg2; + if (sizeof(T) == 4) + TORC4.Insert(ArgXor, Arg1, Arg2); + else if (sizeof(T) == 8) + TORC8.Insert(ArgXor, Arg1, Arg2); + uint64_t HammingDistance = Popcountll(ArgXor); // [0,64] + uint64_t AbsoluteDistance = (Arg1 == Arg2 ? 0 : Clzll(Arg1 - Arg2) + 1); + ValueProfileMap.AddValue(PC * 128 + HammingDistance); + ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance); +} + +static size_t InternalStrnlen(const char *S, size_t MaxLen) { + size_t Len = 0; + for (; Len < MaxLen && S[Len]; Len++) {} + return Len; +} + +// Finds min of (strlen(S1), strlen(S2)). +// Needed bacause one of these strings may actually be non-zero terminated. +static size_t InternalStrnlen2(const char *S1, const char *S2) { + size_t Len = 0; + for (; S1[Len] && S2[Len]; Len++) {} + return Len; +} + +void TracePC::ClearInlineCounters() { + IterateCounterRegions([](const Module::Region &R){ + if (R.Enabled) + memset(R.Start, 0, R.Stop - R.Start); + }); +} + +ATTRIBUTE_NO_SANITIZE_ALL +void TracePC::RecordInitialStack() { + int stack; + __sancov_lowest_stack = InitialStack = reinterpret_cast(&stack); +} + +uintptr_t TracePC::GetMaxStackOffset() const { + return InitialStack - __sancov_lowest_stack; // Stack grows down +} + +void WarnAboutDeprecatedInstrumentation(const char *flag) { + // Use RawPrint because Printf cannot be used on Windows before OutputFile is + // initialized. + RawPrint(flag); + RawPrint( + " is no longer supported by libFuzzer.\n" + "Please either migrate to a compiler that supports -fsanitize=fuzzer\n" + "or use an older version of libFuzzer\n"); + exit(1); +} + +} // namespace fuzzer + +extern "C" { +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); +} + +// Best-effort support for -fsanitize-coverage=trace-pc, which is available +// in both Clang and GCC. +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +void __sanitizer_cov_trace_pc() { + fuzzer::WarnAboutDeprecatedInstrumentation("-fsanitize-coverage=trace-pc"); +} + +ATTRIBUTE_INTERFACE +void __sanitizer_cov_trace_pc_guard_init(uint32_t *Start, uint32_t *Stop) { + fuzzer::WarnAboutDeprecatedInstrumentation( + "-fsanitize-coverage=trace-pc-guard"); +} + +ATTRIBUTE_INTERFACE +void __sanitizer_cov_8bit_counters_init(uint8_t *Start, uint8_t *Stop) { + fuzzer::TPC.HandleInline8bitCountersInit(Start, Stop); +} + +ATTRIBUTE_INTERFACE +void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + const uintptr_t *pcs_end) { + fuzzer::TPC.HandlePCsInit(pcs_beg, pcs_end); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +void __sanitizer_cov_trace_pc_indir(uintptr_t Callee) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCallerCallee(PC, Callee); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_cmp8(uint64_t Arg1, uint64_t Arg2) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +// Now the __sanitizer_cov_trace_const_cmp[1248] callbacks just mimic +// the behaviour of __sanitizer_cov_trace_cmp[1248] ones. This, however, +// should be changed later to make full use of instrumentation. +void __sanitizer_cov_trace_const_cmp8(uint64_t Arg1, uint64_t Arg2) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_cmp4(uint32_t Arg1, uint32_t Arg2) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_const_cmp4(uint32_t Arg1, uint32_t Arg2) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_cmp2(uint16_t Arg1, uint16_t Arg2) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_const_cmp2(uint16_t Arg1, uint16_t Arg2) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_cmp1(uint8_t Arg1, uint8_t Arg2) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_const_cmp1(uint8_t Arg1, uint8_t Arg2) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases) { + uint64_t N = Cases[0]; + uint64_t ValSizeInBits = Cases[1]; + uint64_t *Vals = Cases + 2; + // Skip the most common and the most boring case: all switch values are small. + // We may want to skip this at compile-time, but it will make the + // instrumentation less general. + if (Vals[N - 1] < 256) + return; + // Also skip small inputs values, they won't give good signal. + if (Val < 256) + return; + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + size_t i; + uint64_t Smaller = 0; + uint64_t Larger = ~(uint64_t)0; + // Find two switch values such that Smaller < Val < Larger. + // Use 0 and 0xfff..f as the defaults. + for (i = 0; i < N; i++) { + if (Val < Vals[i]) { + Larger = Vals[i]; + break; + } + if (Val > Vals[i]) Smaller = Vals[i]; + } + + // Apply HandleCmp to {Val,Smaller} and {Val, Larger}, + // use i as the PC modifier for HandleCmp. + if (ValSizeInBits == 16) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast(Val), + (uint16_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast(Val), + (uint16_t)(Larger)); + } else if (ValSizeInBits == 32) { + fuzzer::TPC.HandleCmp(PC + 2 * i, static_cast(Val), + (uint32_t)(Smaller)); + fuzzer::TPC.HandleCmp(PC + 2 * i + 1, static_cast(Val), + (uint32_t)(Larger)); + } else { + fuzzer::TPC.HandleCmp(PC + 2*i, Val, Smaller); + fuzzer::TPC.HandleCmp(PC + 2*i + 1, Val, Larger); + } +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_div4(uint32_t Val) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Val, (uint32_t)0); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_div8(uint64_t Val) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Val, (uint64_t)0); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_gep(uintptr_t Idx) { + uintptr_t PC = reinterpret_cast(GET_CALLER_PC()); + fuzzer::TPC.HandleCmp(PC, Idx, (uintptr_t)0); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *s1, + const void *s2, size_t n, int result) { + if (!fuzzer::RunningUserCallback) return; + if (result == 0) return; // No reason to mutate. + if (n <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/false); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1, + const char *s2, size_t n, int result) { + if (!fuzzer::RunningUserCallback) return; + if (result == 0) return; // No reason to mutate. + size_t Len1 = fuzzer::InternalStrnlen(s1, n); + size_t Len2 = fuzzer::InternalStrnlen(s2, n); + n = std::min(n, Len1); + n = std::min(n, Len2); + if (n <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/true); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcmp(void *caller_pc, const char *s1, + const char *s2, int result) { + if (!fuzzer::RunningUserCallback) return; + if (result == 0) return; // No reason to mutate. + size_t N = fuzzer::InternalStrnlen2(s1, s2); + if (N <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, N, /*StopAtZero*/true); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result) { + if (!fuzzer::RunningUserCallback) return; + return __sanitizer_weak_hook_strncmp(called_pc, s1, s2, n, result); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, + const char *s2, int result) { + if (!fuzzer::RunningUserCallback) return; + return __sanitizer_weak_hook_strcmp(called_pc, s1, s2, result); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, + const char *s2, char *result) { + if (!fuzzer::RunningUserCallback) return; + fuzzer::TPC.MMT.Add(reinterpret_cast(s2), strlen(s2)); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, + const char *s2, char *result) { + if (!fuzzer::RunningUserCallback) return; + fuzzer::TPC.MMT.Add(reinterpret_cast(s2), strlen(s2)); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_memmem(void *called_pc, const void *s1, size_t len1, + const void *s2, size_t len2, void *result) { + if (!fuzzer::RunningUserCallback) return; + fuzzer::TPC.MMT.Add(reinterpret_cast(s2), len2); +} +} // extern "C" diff --git a/tools/fuzzing/libfuzzer/FuzzerTracePC.h b/tools/fuzzing/libfuzzer/FuzzerTracePC.h new file mode 100644 index 0000000000..b46ebb909d --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerTracePC.h @@ -0,0 +1,289 @@ +//===- FuzzerTracePC.h - Internal header for the Fuzzer ---------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// fuzzer::TracePC +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_TRACE_PC +#define LLVM_FUZZER_TRACE_PC + +#include "FuzzerDefs.h" +#include "FuzzerDictionary.h" +#include "FuzzerValueBitMap.h" + +#include +#include + +namespace fuzzer { + +// TableOfRecentCompares (TORC) remembers the most recently performed +// comparisons of type T. +// We record the arguments of CMP instructions in this table unconditionally +// because it seems cheaper this way than to compute some expensive +// conditions inside __sanitizer_cov_trace_cmp*. +// After the unit has been executed we may decide to use the contents of +// this table to populate a Dictionary. +template +struct TableOfRecentCompares { + static const size_t kSize = kSizeT; + struct Pair { + T A, B; + }; + ATTRIBUTE_NO_SANITIZE_ALL + void Insert(size_t Idx, const T &Arg1, const T &Arg2) { + Idx = Idx % kSize; + Table[Idx].A = Arg1; + Table[Idx].B = Arg2; + } + + Pair Get(size_t I) { return Table[I % kSize]; } + + Pair Table[kSize]; +}; + +template +struct MemMemTable { + static const size_t kSize = kSizeT; + Word MemMemWords[kSize]; + Word EmptyWord; + + void Add(const uint8_t *Data, size_t Size) { + if (Size <= 2) return; + Size = std::min(Size, Word::GetMaxSize()); + size_t Idx = SimpleFastHash(Data, Size) % kSize; + MemMemWords[Idx].Set(Data, Size); + } + const Word &Get(size_t Idx) { + for (size_t i = 0; i < kSize; i++) { + const Word &W = MemMemWords[(Idx + i) % kSize]; + if (W.size()) return W; + } + EmptyWord.Set(nullptr, 0); + return EmptyWord; + } +}; + +class TracePC { + public: + void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop); + void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop); + void HandleCallerCallee(uintptr_t Caller, uintptr_t Callee); + template void HandleCmp(uintptr_t PC, T Arg1, T Arg2); + size_t GetTotalPCCoverage(); + void SetUseCounters(bool UC) { UseCounters = UC; } + void SetUseValueProfileMask(uint32_t VPMask) { UseValueProfileMask = VPMask; } + void SetPrintNewPCs(bool P) { DoPrintNewPCs = P; } + void SetPrintNewFuncs(size_t P) { NumPrintNewFuncs = P; } + void UpdateObservedPCs(); + template void CollectFeatures(Callback CB) const; + + void ResetMaps() { + ValueProfileMap.Reset(); + ClearExtraCounters(); + ClearInlineCounters(); + } + + void ClearInlineCounters(); + + void UpdateFeatureSet(size_t CurrentElementIdx, size_t CurrentElementSize); + void PrintFeatureSet(); + + void PrintModuleInfo(); + + void PrintCoverage(); + + template + void IterateCoveredFunctions(CallBack CB); + + void AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, + size_t n, bool StopAtZero); + + TableOfRecentCompares TORC4; + TableOfRecentCompares TORC8; + TableOfRecentCompares TORCW; + MemMemTable<1024> MMT; + + void RecordInitialStack(); + uintptr_t GetMaxStackOffset() const; + + template + void ForEachObservedPC(CallBack CB) { + for (auto PC : ObservedPCs) + CB(PC); + } + + int SetFocusFunction(const std::string &FuncName); + bool ObservedFocusFunction(); + + struct PCTableEntry { + uintptr_t PC, PCFlags; + }; + + uintptr_t PCTableEntryIdx(const PCTableEntry *TE); + const PCTableEntry *PCTableEntryByIdx(uintptr_t Idx); + static uintptr_t GetNextInstructionPc(uintptr_t PC); + bool PcIsFuncEntry(const PCTableEntry *TE) { return TE->PCFlags & 1; } + +private: + bool UseCounters = false; + uint32_t UseValueProfileMask = false; + bool DoPrintNewPCs = false; + size_t NumPrintNewFuncs = 0; + + // Module represents the array of 8-bit counters split into regions + // such that every region, except maybe the first and the last one, is one + // full page. + struct Module { + struct Region { + uint8_t *Start, *Stop; + bool Enabled; + bool OneFullPage; + }; + Region *Regions; + size_t NumRegions; + uint8_t *Start() { return Regions[0].Start; } + uint8_t *Stop() { return Regions[NumRegions - 1].Stop; } + size_t Size() { return Stop() - Start(); } + size_t Idx(uint8_t *P) { + assert(P >= Start() && P < Stop()); + return P - Start(); + } + }; + + Module Modules[4096]; + size_t NumModules; // linker-initialized. + size_t NumInline8bitCounters; + + template + void IterateCounterRegions(Callback CB) { + for (size_t m = 0; m < NumModules; m++) + for (size_t r = 0; r < Modules[m].NumRegions; r++) + CB(Modules[m].Regions[r]); + } + + struct { const PCTableEntry *Start, *Stop; } ModulePCTable[4096]; + size_t NumPCTables; + size_t NumPCsInPCTables; + + Set ObservedPCs; + std::unordered_map ObservedFuncs; // PC => Counter. + + uint8_t *FocusFunctionCounterPtr = nullptr; + + ValueBitMap ValueProfileMap; + uintptr_t InitialStack; +}; + +template +// void Callback(size_t FirstFeature, size_t Idx, uint8_t Value); +ATTRIBUTE_NO_SANITIZE_ALL +size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, + size_t FirstFeature, Callback Handle8bitCounter) { + typedef uintptr_t LargeType; + const size_t Step = sizeof(LargeType) / sizeof(uint8_t); + const size_t StepMask = Step - 1; + auto P = Begin; + // Iterate by 1 byte until either the alignment boundary or the end. + for (; reinterpret_cast(P) & StepMask && P < End; P++) + if (uint8_t V = *P) + Handle8bitCounter(FirstFeature, P - Begin, V); + + // Iterate by Step bytes at a time. + for (; P < End; P += Step) + if (LargeType Bundle = *reinterpret_cast(P)) + for (size_t I = 0; I < Step; I++, Bundle >>= 8) + if (uint8_t V = Bundle & 0xff) + Handle8bitCounter(FirstFeature, P - Begin + I, V); + + // Iterate by 1 byte until the end. + for (; P < End; P++) + if (uint8_t V = *P) + Handle8bitCounter(FirstFeature, P - Begin, V); + return End - Begin; +} + +// Given a non-zero Counter returns a number in the range [0,7]. +template +unsigned CounterToFeature(T Counter) { + // Returns a feature number by placing Counters into buckets as illustrated + // below. + // + // Counter bucket: [1] [2] [3] [4-7] [8-15] [16-31] [32-127] [128+] + // Feature number: 0 1 2 3 4 5 6 7 + // + // This is a heuristic taken from AFL (see + // http://lcamtuf.coredump.cx/afl/technical_details.txt). + // + // This implementation may change in the future so clients should + // not rely on it. + assert(Counter); + unsigned Bit = 0; + /**/ if (Counter >= 128) Bit = 7; + else if (Counter >= 32) Bit = 6; + else if (Counter >= 16) Bit = 5; + else if (Counter >= 8) Bit = 4; + else if (Counter >= 4) Bit = 3; + else if (Counter >= 3) Bit = 2; + else if (Counter >= 2) Bit = 1; + return Bit; +} + +template // void Callback(size_t Feature) +ATTRIBUTE_NO_SANITIZE_ADDRESS +ATTRIBUTE_NOINLINE +void TracePC::CollectFeatures(Callback HandleFeature) const { + auto Handle8bitCounter = [&](size_t FirstFeature, + size_t Idx, uint8_t Counter) { + if (UseCounters) + HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Counter)); + else + HandleFeature(FirstFeature + Idx); + }; + + size_t FirstFeature = 0; + + for (size_t i = 0; i < NumModules; i++) { + for (size_t r = 0; r < Modules[i].NumRegions; r++) { + if (!Modules[i].Regions[r].Enabled) continue; + FirstFeature += 8 * ForEachNonZeroByte(Modules[i].Regions[r].Start, + Modules[i].Regions[r].Stop, + FirstFeature, Handle8bitCounter); + } + } + + FirstFeature += + 8 * ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), + FirstFeature, Handle8bitCounter); + + if (UseValueProfileMask) { + ValueProfileMap.ForEach([&](size_t Idx) { + HandleFeature(FirstFeature + Idx); + }); + FirstFeature += ValueProfileMap.SizeInBits(); + } + + // Step function, grows similar to 8 * Log_2(A). + auto StackDepthStepFunction = [](uint32_t A) -> uint32_t { + if (!A) return A; + uint32_t Log2 = Log(A); + if (Log2 < 3) return A; + Log2 -= 3; + return (Log2 + 1) * 8 + ((A >> Log2) & 7); + }; + assert(StackDepthStepFunction(1024) == 64); + assert(StackDepthStepFunction(1024 * 4) == 80); + assert(StackDepthStepFunction(1024 * 1024) == 144); + + if (auto MaxStackOffset = GetMaxStackOffset()) + HandleFeature(FirstFeature + StackDepthStepFunction(MaxStackOffset / 8)); +} + +extern TracePC TPC; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_TRACE_PC diff --git a/tools/fuzzing/libfuzzer/FuzzerUtil.cpp b/tools/fuzzing/libfuzzer/FuzzerUtil.cpp new file mode 100644 index 0000000000..7eecb68d07 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtil.cpp @@ -0,0 +1,236 @@ +//===- FuzzerUtil.cpp - Misc utils ----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils. +//===----------------------------------------------------------------------===// + +#include "FuzzerUtil.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fuzzer { + +void PrintHexArray(const uint8_t *Data, size_t Size, + const char *PrintAfter) { + for (size_t i = 0; i < Size; i++) + Printf("0x%x,", (unsigned)Data[i]); + Printf("%s", PrintAfter); +} + +void Print(const Unit &v, const char *PrintAfter) { + PrintHexArray(v.data(), v.size(), PrintAfter); +} + +void PrintASCIIByte(uint8_t Byte) { + if (Byte == '\\') + Printf("\\\\"); + else if (Byte == '"') + Printf("\\\""); + else if (Byte >= 32 && Byte < 127) + Printf("%c", Byte); + else + Printf("\\x%02x", Byte); +} + +void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) { + for (size_t i = 0; i < Size; i++) + PrintASCIIByte(Data[i]); + Printf("%s", PrintAfter); +} + +void PrintASCII(const Unit &U, const char *PrintAfter) { + PrintASCII(U.data(), U.size(), PrintAfter); +} + +bool ToASCII(uint8_t *Data, size_t Size) { + bool Changed = false; + for (size_t i = 0; i < Size; i++) { + uint8_t &X = Data[i]; + auto NewX = X; + NewX &= 127; + if (!isspace(NewX) && !isprint(NewX)) + NewX = ' '; + Changed |= NewX != X; + X = NewX; + } + return Changed; +} + +bool IsASCII(const Unit &U) { return IsASCII(U.data(), U.size()); } + +bool IsASCII(const uint8_t *Data, size_t Size) { + for (size_t i = 0; i < Size; i++) + if (!(isprint(Data[i]) || isspace(Data[i]))) return false; + return true; +} + +bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) { + U->clear(); + if (Str.empty()) return false; + size_t L = 0, R = Str.size() - 1; // We are parsing the range [L,R]. + // Skip spaces from both sides. + while (L < R && isspace(Str[L])) L++; + while (R > L && isspace(Str[R])) R--; + if (R - L < 2) return false; + // Check the closing " + if (Str[R] != '"') return false; + R--; + // Find the opening " + while (L < R && Str[L] != '"') L++; + if (L >= R) return false; + assert(Str[L] == '\"'); + L++; + assert(L <= R); + for (size_t Pos = L; Pos <= R; Pos++) { + uint8_t V = (uint8_t)Str[Pos]; + if (!isprint(V) && !isspace(V)) return false; + if (V =='\\') { + // Handle '\\' + if (Pos + 1 <= R && (Str[Pos + 1] == '\\' || Str[Pos + 1] == '"')) { + U->push_back(Str[Pos + 1]); + Pos++; + continue; + } + // Handle '\xAB' + if (Pos + 3 <= R && Str[Pos + 1] == 'x' + && isxdigit(Str[Pos + 2]) && isxdigit(Str[Pos + 3])) { + char Hex[] = "0xAA"; + Hex[2] = Str[Pos + 2]; + Hex[3] = Str[Pos + 3]; + U->push_back(strtol(Hex, nullptr, 16)); + Pos += 3; + continue; + } + return false; // Invalid escape. + } else { + // Any other character. + U->push_back(V); + } + } + return true; +} + +bool ParseDictionaryFile(const std::string &Text, Vector *Units) { + if (Text.empty()) { + Printf("ParseDictionaryFile: file does not exist or is empty\n"); + return false; + } + std::istringstream ISS(Text); + Units->clear(); + Unit U; + int LineNo = 0; + std::string S; + while (std::getline(ISS, S, '\n')) { + LineNo++; + size_t Pos = 0; + while (Pos < S.size() && isspace(S[Pos])) Pos++; // Skip spaces. + if (Pos == S.size()) continue; // Empty line. + if (S[Pos] == '#') continue; // Comment line. + if (ParseOneDictionaryEntry(S, &U)) { + Units->push_back(U); + } else { + Printf("ParseDictionaryFile: error in line %d\n\t\t%s\n", LineNo, + S.c_str()); + return false; + } + } + return true; +} + +// Code duplicated (and tested) in llvm/include/llvm/Support/Base64.h +std::string Base64(const Unit &U) { + static const char Table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + std::string Buffer; + Buffer.resize(((U.size() + 2) / 3) * 4); + + size_t i = 0, j = 0; + for (size_t n = U.size() / 3 * 3; i < n; i += 3, j += 4) { + uint32_t x = ((unsigned char)U[i] << 16) | ((unsigned char)U[i + 1] << 8) | + (unsigned char)U[i + 2]; + Buffer[j + 0] = Table[(x >> 18) & 63]; + Buffer[j + 1] = Table[(x >> 12) & 63]; + Buffer[j + 2] = Table[(x >> 6) & 63]; + Buffer[j + 3] = Table[x & 63]; + } + if (i + 1 == U.size()) { + uint32_t x = ((unsigned char)U[i] << 16); + Buffer[j + 0] = Table[(x >> 18) & 63]; + Buffer[j + 1] = Table[(x >> 12) & 63]; + Buffer[j + 2] = '='; + Buffer[j + 3] = '='; + } else if (i + 2 == U.size()) { + uint32_t x = ((unsigned char)U[i] << 16) | ((unsigned char)U[i + 1] << 8); + Buffer[j + 0] = Table[(x >> 18) & 63]; + Buffer[j + 1] = Table[(x >> 12) & 63]; + Buffer[j + 2] = Table[(x >> 6) & 63]; + Buffer[j + 3] = '='; + } + return Buffer; +} + +static std::mutex SymbolizeMutex; + +std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC) { + std::unique_lock l(SymbolizeMutex, std::try_to_lock); + if (!EF->__sanitizer_symbolize_pc || !l.owns_lock()) + return ""; + char PcDescr[1024] = {}; + EF->__sanitizer_symbolize_pc(reinterpret_cast(PC), + SymbolizedFMT, PcDescr, sizeof(PcDescr)); + PcDescr[sizeof(PcDescr) - 1] = 0; // Just in case. + return PcDescr; +} + +void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC) { + if (EF->__sanitizer_symbolize_pc) + Printf("%s", DescribePC(SymbolizedFMT, PC).c_str()); + else + Printf(FallbackFMT, PC); +} + +void PrintStackTrace() { + std::unique_lock l(SymbolizeMutex, std::try_to_lock); + if (EF->__sanitizer_print_stack_trace && l.owns_lock()) + EF->__sanitizer_print_stack_trace(); +} + +void PrintMemoryProfile() { + std::unique_lock l(SymbolizeMutex, std::try_to_lock); + if (EF->__sanitizer_print_memory_profile && l.owns_lock()) + EF->__sanitizer_print_memory_profile(95, 8); +} + +unsigned NumberOfCpuCores() { + unsigned N = std::thread::hardware_concurrency(); + if (!N) { + Printf("WARNING: std::thread::hardware_concurrency not well defined for " + "your platform. Assuming CPU count of 1.\n"); + N = 1; + } + return N; +} + +size_t SimpleFastHash(const uint8_t *Data, size_t Size) { + size_t Res = 0; + for (size_t i = 0; i < Size; i++) + Res = Res * 11 + Data[i]; + return Res; +} + +} // namespace fuzzer diff --git a/tools/fuzzing/libfuzzer/FuzzerUtil.h b/tools/fuzzing/libfuzzer/FuzzerUtil.h new file mode 100644 index 0000000000..4ae3583830 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtil.h @@ -0,0 +1,111 @@ +//===- FuzzerUtil.h - Internal header for the Fuzzer Utils ------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Util functions. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_UTIL_H +#define LLVM_FUZZER_UTIL_H + +#include "FuzzerBuiltins.h" +#include "FuzzerBuiltinsMsvc.h" +#include "FuzzerCommand.h" +#include "FuzzerDefs.h" + +namespace fuzzer { + +void PrintHexArray(const Unit &U, const char *PrintAfter = ""); + +void PrintHexArray(const uint8_t *Data, size_t Size, + const char *PrintAfter = ""); + +void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter = ""); + +void PrintASCII(const Unit &U, const char *PrintAfter = ""); + +// Changes U to contain only ASCII (isprint+isspace) characters. +// Returns true iff U has been changed. +bool ToASCII(uint8_t *Data, size_t Size); + +bool IsASCII(const Unit &U); + +bool IsASCII(const uint8_t *Data, size_t Size); + +std::string Base64(const Unit &U); + +void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC); + +std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC); + +void PrintStackTrace(); + +void PrintMemoryProfile(); + +unsigned NumberOfCpuCores(); + +// Platform specific functions. +void SetSignalHandler(const FuzzingOptions& Options); + +void SleepSeconds(int Seconds); + +unsigned long GetPid(); + +size_t GetPeakRSSMb(); + +int ExecuteCommand(const Command &Cmd); +bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput); + +// Fuchsia does not have popen/pclose. +FILE *OpenProcessPipe(const char *Command, const char *Mode); +int CloseProcessPipe(FILE *F); + +const void *SearchMemory(const void *haystack, size_t haystacklen, + const void *needle, size_t needlelen); + +std::string CloneArgsWithoutX(const Vector &Args, + const char *X1, const char *X2); + +inline std::string CloneArgsWithoutX(const Vector &Args, + const char *X) { + return CloneArgsWithoutX(Args, X, X); +} + +inline std::pair SplitBefore(std::string X, + std::string S) { + auto Pos = S.find(X); + if (Pos == std::string::npos) + return std::make_pair(S, ""); + return std::make_pair(S.substr(0, Pos), S.substr(Pos)); +} + +void DiscardOutput(int Fd); + +std::string DisassembleCmd(const std::string &FileName); + +std::string SearchRegexCmd(const std::string &Regex); + +size_t SimpleFastHash(const uint8_t *Data, size_t Size); + +inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } + +inline size_t PageSize() { return 4096; } +inline uint8_t *RoundUpByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast(P); + size_t Mask = PageSize() - 1; + X = (X + Mask) & ~Mask; + return reinterpret_cast(X); +} +inline uint8_t *RoundDownByPage(uint8_t *P) { + uintptr_t X = reinterpret_cast(P); + size_t Mask = PageSize() - 1; + X = X & ~Mask; + return reinterpret_cast(X); +} + +} // namespace fuzzer + +#endif // LLVM_FUZZER_UTIL_H diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp new file mode 100644 index 0000000000..a5bed658a4 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp @@ -0,0 +1,170 @@ +//===- FuzzerUtilDarwin.cpp - Misc utils ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils for Darwin. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_APPLE +#include "FuzzerCommand.h" +#include "FuzzerIO.h" +#include +#include +#include +#include +#include +#include +#include + +// There is no header for this on macOS so declare here +extern "C" char **environ; + +namespace fuzzer { + +static std::mutex SignalMutex; +// Global variables used to keep track of how signal handling should be +// restored. They should **not** be accessed without holding `SignalMutex`. +static int ActiveThreadCount = 0; +static struct sigaction OldSigIntAction; +static struct sigaction OldSigQuitAction; +static sigset_t OldBlockedSignalsSet; + +// This is a reimplementation of Libc's `system()`. On Darwin the Libc +// implementation contains a mutex which prevents it from being used +// concurrently. This implementation **can** be used concurrently. It sets the +// signal handlers when the first thread enters and restores them when the last +// thread finishes execution of the function and ensures this is not racey by +// using a mutex. +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + posix_spawnattr_t SpawnAttributes; + if (posix_spawnattr_init(&SpawnAttributes)) + return -1; + // Block and ignore signals of the current process when the first thread + // enters. + { + std::lock_guard Lock(SignalMutex); + if (ActiveThreadCount == 0) { + static struct sigaction IgnoreSignalAction; + sigset_t BlockedSignalsSet; + memset(&IgnoreSignalAction, 0, sizeof(IgnoreSignalAction)); + IgnoreSignalAction.sa_handler = SIG_IGN; + + if (sigaction(SIGINT, &IgnoreSignalAction, &OldSigIntAction) == -1) { + Printf("Failed to ignore SIGINT\n"); + (void)posix_spawnattr_destroy(&SpawnAttributes); + return -1; + } + if (sigaction(SIGQUIT, &IgnoreSignalAction, &OldSigQuitAction) == -1) { + Printf("Failed to ignore SIGQUIT\n"); + // Try our best to restore the signal handlers. + (void)sigaction(SIGINT, &OldSigIntAction, NULL); + (void)posix_spawnattr_destroy(&SpawnAttributes); + return -1; + } + + (void)sigemptyset(&BlockedSignalsSet); + (void)sigaddset(&BlockedSignalsSet, SIGCHLD); + if (sigprocmask(SIG_BLOCK, &BlockedSignalsSet, &OldBlockedSignalsSet) == + -1) { + Printf("Failed to block SIGCHLD\n"); + // Try our best to restore the signal handlers. + (void)sigaction(SIGQUIT, &OldSigQuitAction, NULL); + (void)sigaction(SIGINT, &OldSigIntAction, NULL); + (void)posix_spawnattr_destroy(&SpawnAttributes); + return -1; + } + } + ++ActiveThreadCount; + } + + // NOTE: Do not introduce any new `return` statements past this + // point. It is important that `ActiveThreadCount` always be decremented + // when leaving this function. + + // Make sure the child process uses the default handlers for the + // following signals rather than inheriting what the parent has. + sigset_t DefaultSigSet; + (void)sigemptyset(&DefaultSigSet); + (void)sigaddset(&DefaultSigSet, SIGQUIT); + (void)sigaddset(&DefaultSigSet, SIGINT); + (void)posix_spawnattr_setsigdefault(&SpawnAttributes, &DefaultSigSet); + // Make sure the child process doesn't block SIGCHLD + (void)posix_spawnattr_setsigmask(&SpawnAttributes, &OldBlockedSignalsSet); + short SpawnFlags = POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK; + (void)posix_spawnattr_setflags(&SpawnAttributes, SpawnFlags); + + pid_t Pid; + char **Environ = environ; // Read from global + const char *CommandCStr = CmdLine.c_str(); + char *const Argv[] = { + strdup("sh"), + strdup("-c"), + strdup(CommandCStr), + NULL + }; + int ErrorCode = 0, ProcessStatus = 0; + // FIXME: We probably shouldn't hardcode the shell path. + ErrorCode = posix_spawn(&Pid, "/bin/sh", NULL, &SpawnAttributes, + Argv, Environ); + (void)posix_spawnattr_destroy(&SpawnAttributes); + if (!ErrorCode) { + pid_t SavedPid = Pid; + do { + // Repeat until call completes uninterrupted. + Pid = waitpid(SavedPid, &ProcessStatus, /*options=*/0); + } while (Pid == -1 && errno == EINTR); + if (Pid == -1) { + // Fail for some other reason. + ProcessStatus = -1; + } + } else if (ErrorCode == ENOMEM || ErrorCode == EAGAIN) { + // Fork failure. + ProcessStatus = -1; + } else { + // Shell execution failure. + ProcessStatus = W_EXITCODE(127, 0); + } + for (unsigned i = 0, n = sizeof(Argv) / sizeof(Argv[0]); i < n; ++i) + free(Argv[i]); + + // Restore the signal handlers of the current process when the last thread + // using this function finishes. + { + std::lock_guard Lock(SignalMutex); + --ActiveThreadCount; + if (ActiveThreadCount == 0) { + bool FailedRestore = false; + if (sigaction(SIGINT, &OldSigIntAction, NULL) == -1) { + Printf("Failed to restore SIGINT handling\n"); + FailedRestore = true; + } + if (sigaction(SIGQUIT, &OldSigQuitAction, NULL) == -1) { + Printf("Failed to restore SIGQUIT handling\n"); + FailedRestore = true; + } + if (sigprocmask(SIG_BLOCK, &OldBlockedSignalsSet, NULL) == -1) { + Printf("Failed to unblock SIGCHLD\n"); + FailedRestore = true; + } + if (FailedRestore) + ProcessStatus = -1; + } + } + return ProcessStatus; +} + +void DiscardOutput(int Fd) { + FILE* Temp = fopen("/dev/null", "w"); + if (!Temp) + return; + dup2(fileno(Temp), Fd); + fclose(Temp); +} + +} // namespace fuzzer + +#endif // LIBFUZZER_APPLE diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilFuchsia.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilFuchsia.cpp new file mode 100644 index 0000000000..190fb78666 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilFuchsia.cpp @@ -0,0 +1,565 @@ +//===- FuzzerUtilFuchsia.cpp - Misc utils for Fuchsia. --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils implementation using Fuchsia/Zircon APIs. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" + +#if LIBFUZZER_FUCHSIA + +#include "FuzzerInternal.h" +#include "FuzzerUtil.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace fuzzer { + +// Given that Fuchsia doesn't have the POSIX signals that libFuzzer was written +// around, the general approach is to spin up dedicated threads to watch for +// each requested condition (alarm, interrupt, crash). Of these, the crash +// handler is the most involved, as it requires resuming the crashed thread in +// order to invoke the sanitizers to get the needed state. + +// Forward declaration of assembly trampoline needed to resume crashed threads. +// This appears to have external linkage to C++, which is why it's not in the +// anonymous namespace. The assembly definition inside MakeTrampoline() +// actually defines the symbol with internal linkage only. +void CrashTrampolineAsm() __asm__("CrashTrampolineAsm"); + +namespace { + +// Helper function to handle Zircon syscall failures. +void ExitOnErr(zx_status_t Status, const char *Syscall) { + if (Status != ZX_OK) { + Printf("libFuzzer: %s failed: %s\n", Syscall, + _zx_status_get_string(Status)); + exit(1); + } +} + +void AlarmHandler(int Seconds) { + while (true) { + SleepSeconds(Seconds); + Fuzzer::StaticAlarmCallback(); + } +} + +void InterruptHandler() { + fd_set readfds; + // Ctrl-C sends ETX in Zircon. + do { + FD_ZERO(&readfds); + FD_SET(STDIN_FILENO, &readfds); + select(STDIN_FILENO + 1, &readfds, nullptr, nullptr, nullptr); + } while(!FD_ISSET(STDIN_FILENO, &readfds) || getchar() != 0x03); + Fuzzer::StaticInterruptCallback(); +} + +// CFAOffset is used to reference the stack pointer before entering the +// trampoline (Stack Pointer + CFAOffset = prev Stack Pointer). Before jumping +// to the trampoline we copy all the registers onto the stack. We need to make +// sure that the new stack has enough space to store all the registers. +// +// The trampoline holds CFI information regarding the registers stored in the +// stack, which is then used by the unwinder to restore them. +#if defined(__x86_64__) +// In x86_64 the crashing function might also be using the red zone (128 bytes +// on top of their rsp). +constexpr size_t CFAOffset = 128 + sizeof(zx_thread_state_general_regs_t); +#elif defined(__aarch64__) +// In aarch64 we need to always have the stack pointer aligned to 16 bytes, so we +// make sure that we are keeping that same alignment. +constexpr size_t CFAOffset = (sizeof(zx_thread_state_general_regs_t) + 15) & -(uintptr_t)16; +#endif + +// For the crash handler, we need to call Fuzzer::StaticCrashSignalCallback +// without POSIX signal handlers. To achieve this, we use an assembly function +// to add the necessary CFI unwinding information and a C function to bridge +// from that back into C++. + +// FIXME: This works as a short-term solution, but this code really shouldn't be +// architecture dependent. A better long term solution is to implement remote +// unwinding and expose the necessary APIs through sanitizer_common and/or ASAN +// to allow the exception handling thread to gather the crash state directly. +// +// Alternatively, Fuchsia may in future actually implement basic signal +// handling for the machine trap signals. +#if defined(__x86_64__) +#define FOREACH_REGISTER(OP_REG, OP_NUM) \ + OP_REG(rax) \ + OP_REG(rbx) \ + OP_REG(rcx) \ + OP_REG(rdx) \ + OP_REG(rsi) \ + OP_REG(rdi) \ + OP_REG(rbp) \ + OP_REG(rsp) \ + OP_REG(r8) \ + OP_REG(r9) \ + OP_REG(r10) \ + OP_REG(r11) \ + OP_REG(r12) \ + OP_REG(r13) \ + OP_REG(r14) \ + OP_REG(r15) \ + OP_REG(rip) + +#elif defined(__aarch64__) +#define FOREACH_REGISTER(OP_REG, OP_NUM) \ + OP_NUM(0) \ + OP_NUM(1) \ + OP_NUM(2) \ + OP_NUM(3) \ + OP_NUM(4) \ + OP_NUM(5) \ + OP_NUM(6) \ + OP_NUM(7) \ + OP_NUM(8) \ + OP_NUM(9) \ + OP_NUM(10) \ + OP_NUM(11) \ + OP_NUM(12) \ + OP_NUM(13) \ + OP_NUM(14) \ + OP_NUM(15) \ + OP_NUM(16) \ + OP_NUM(17) \ + OP_NUM(18) \ + OP_NUM(19) \ + OP_NUM(20) \ + OP_NUM(21) \ + OP_NUM(22) \ + OP_NUM(23) \ + OP_NUM(24) \ + OP_NUM(25) \ + OP_NUM(26) \ + OP_NUM(27) \ + OP_NUM(28) \ + OP_NUM(29) \ + OP_REG(sp) + +#else +#error "Unsupported architecture for fuzzing on Fuchsia" +#endif + +// Produces a CFI directive for the named or numbered register. +// The value used refers to an assembler immediate operand with the same name +// as the register (see ASM_OPERAND_REG). +#define CFI_OFFSET_REG(reg) ".cfi_offset " #reg ", %c[" #reg "]\n" +#define CFI_OFFSET_NUM(num) CFI_OFFSET_REG(x##num) + +// Produces an assembler immediate operand for the named or numbered register. +// This operand contains the offset of the register relative to the CFA. +#define ASM_OPERAND_REG(reg) \ + [reg] "i"(offsetof(zx_thread_state_general_regs_t, reg) - CFAOffset), +#define ASM_OPERAND_NUM(num) \ + [x##num] "i"(offsetof(zx_thread_state_general_regs_t, r[num]) - CFAOffset), + +// Trampoline to bridge from the assembly below to the static C++ crash +// callback. +__attribute__((noreturn)) +static void StaticCrashHandler() { + Fuzzer::StaticCrashSignalCallback(); + for (;;) { + _Exit(1); + } +} + +// Creates the trampoline with the necessary CFI information to unwind through +// to the crashing call stack: +// * Defining the CFA so that it points to the stack pointer at the point +// of crash. +// * Storing all registers at the point of crash in the stack and refer to them +// via CFI information (relative to the CFA). +// * Setting the return column so the unwinder knows how to continue unwinding. +// * (x86_64) making sure rsp is aligned before calling StaticCrashHandler. +// * Calling StaticCrashHandler that will trigger the unwinder. +// +// The __attribute__((used)) is necessary because the function +// is never called; it's just a container around the assembly to allow it to +// use operands for compile-time computed constants. +__attribute__((used)) +void MakeTrampoline() { + __asm__(".cfi_endproc\n" + ".pushsection .text.CrashTrampolineAsm\n" + ".type CrashTrampolineAsm,STT_FUNC\n" +"CrashTrampolineAsm:\n" + ".cfi_startproc simple\n" + ".cfi_signal_frame\n" +#if defined(__x86_64__) + ".cfi_return_column rip\n" + ".cfi_def_cfa rsp, %c[CFAOffset]\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + "mov %%rsp, %%rbp\n" + ".cfi_def_cfa_register rbp\n" + "andq $-16, %%rsp\n" + "call %c[StaticCrashHandler]\n" + "ud2\n" +#elif defined(__aarch64__) + ".cfi_return_column 33\n" + ".cfi_def_cfa sp, %c[CFAOffset]\n" + FOREACH_REGISTER(CFI_OFFSET_REG, CFI_OFFSET_NUM) + ".cfi_offset 33, %c[pc]\n" + ".cfi_offset 30, %c[lr]\n" + "bl %c[StaticCrashHandler]\n" + "brk 1\n" +#else +#error "Unsupported architecture for fuzzing on Fuchsia" +#endif + ".cfi_endproc\n" + ".size CrashTrampolineAsm, . - CrashTrampolineAsm\n" + ".popsection\n" + ".cfi_startproc\n" + : // No outputs + : FOREACH_REGISTER(ASM_OPERAND_REG, ASM_OPERAND_NUM) +#if defined(__aarch64__) + ASM_OPERAND_REG(pc) + ASM_OPERAND_REG(lr) +#endif + [StaticCrashHandler] "i" (StaticCrashHandler), + [CFAOffset] "i" (CFAOffset)); +} + +void CrashHandler(zx_handle_t *Event) { + // This structure is used to ensure we close handles to objects we create in + // this handler. + struct ScopedHandle { + ~ScopedHandle() { _zx_handle_close(Handle); } + zx_handle_t Handle = ZX_HANDLE_INVALID; + }; + + // Create the exception channel. We need to claim to be a "debugger" so the + // kernel will allow us to modify and resume dying threads (see below). Once + // the channel is set, we can signal the main thread to continue and wait + // for the exception to arrive. + ScopedHandle Channel; + zx_handle_t Self = _zx_process_self(); + ExitOnErr(_zx_task_create_exception_channel( + Self, ZX_EXCEPTION_CHANNEL_DEBUGGER, &Channel.Handle), + "_zx_task_create_exception_channel"); + + ExitOnErr(_zx_object_signal(*Event, 0, ZX_USER_SIGNAL_0), + "_zx_object_signal"); + + // This thread lives as long as the process in order to keep handling + // crashes. In practice, the first crashed thread to reach the end of the + // StaticCrashHandler will end the process. + while (true) { + ExitOnErr(_zx_object_wait_one(Channel.Handle, ZX_CHANNEL_READABLE, + ZX_TIME_INFINITE, nullptr), + "_zx_object_wait_one"); + + zx_exception_info_t ExceptionInfo; + ScopedHandle Exception; + ExitOnErr(_zx_channel_read(Channel.Handle, 0, &ExceptionInfo, + &Exception.Handle, sizeof(ExceptionInfo), 1, + nullptr, nullptr), + "_zx_channel_read"); + + // Ignore informational synthetic exceptions. + if (ZX_EXCP_THREAD_STARTING == ExceptionInfo.type || + ZX_EXCP_THREAD_EXITING == ExceptionInfo.type || + ZX_EXCP_PROCESS_STARTING == ExceptionInfo.type) { + continue; + } + + // At this point, we want to get the state of the crashing thread, but + // libFuzzer and the sanitizers assume this will happen from that same + // thread via a POSIX signal handler. "Resurrecting" the thread in the + // middle of the appropriate callback is as simple as forcibly setting the + // instruction pointer/program counter, provided we NEVER EVER return from + // that function (since otherwise our stack will not be valid). + ScopedHandle Thread; + ExitOnErr(_zx_exception_get_thread(Exception.Handle, &Thread.Handle), + "_zx_exception_get_thread"); + + zx_thread_state_general_regs_t GeneralRegisters; + ExitOnErr(_zx_thread_read_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, + &GeneralRegisters, + sizeof(GeneralRegisters)), + "_zx_thread_read_state"); + + // To unwind properly, we need to push the crashing thread's register state + // onto the stack and jump into a trampoline with CFI instructions on how + // to restore it. +#if defined(__x86_64__) + uintptr_t StackPtr = GeneralRegisters.rsp - CFAOffset; + __unsanitized_memcpy(reinterpret_cast(StackPtr), &GeneralRegisters, + sizeof(GeneralRegisters)); + GeneralRegisters.rsp = StackPtr; + GeneralRegisters.rip = reinterpret_cast(CrashTrampolineAsm); + +#elif defined(__aarch64__) + uintptr_t StackPtr = GeneralRegisters.sp - CFAOffset; + __unsanitized_memcpy(reinterpret_cast(StackPtr), &GeneralRegisters, + sizeof(GeneralRegisters)); + GeneralRegisters.sp = StackPtr; + GeneralRegisters.pc = reinterpret_cast(CrashTrampolineAsm); + +#else +#error "Unsupported architecture for fuzzing on Fuchsia" +#endif + + // Now force the crashing thread's state. + ExitOnErr( + _zx_thread_write_state(Thread.Handle, ZX_THREAD_STATE_GENERAL_REGS, + &GeneralRegisters, sizeof(GeneralRegisters)), + "_zx_thread_write_state"); + + // Set the exception to HANDLED so it resumes the thread on close. + uint32_t ExceptionState = ZX_EXCEPTION_STATE_HANDLED; + ExitOnErr(_zx_object_set_property(Exception.Handle, ZX_PROP_EXCEPTION_STATE, + &ExceptionState, sizeof(ExceptionState)), + "zx_object_set_property"); + } +} + +} // namespace + +// Platform specific functions. +void SetSignalHandler(const FuzzingOptions &Options) { + // Make sure information from libFuzzer and the sanitizers are easy to + // reassemble. `__sanitizer_log_write` has the added benefit of ensuring the + // DSO map is always available for the symbolizer. + // A uint64_t fits in 20 chars, so 64 is plenty. + char Buf[64]; + memset(Buf, 0, sizeof(Buf)); + snprintf(Buf, sizeof(Buf), "==%lu== INFO: libFuzzer starting.\n", GetPid()); + if (EF->__sanitizer_log_write) + __sanitizer_log_write(Buf, sizeof(Buf)); + Printf("%s", Buf); + + // Set up alarm handler if needed. + if (Options.UnitTimeoutSec > 0) { + std::thread T(AlarmHandler, Options.UnitTimeoutSec / 2 + 1); + T.detach(); + } + + // Set up interrupt handler if needed. + if (Options.HandleInt || Options.HandleTerm) { + std::thread T(InterruptHandler); + T.detach(); + } + + // Early exit if no crash handler needed. + if (!Options.HandleSegv && !Options.HandleBus && !Options.HandleIll && + !Options.HandleFpe && !Options.HandleAbrt) + return; + + // Set up the crash handler and wait until it is ready before proceeding. + zx_handle_t Event; + ExitOnErr(_zx_event_create(0, &Event), "_zx_event_create"); + + std::thread T(CrashHandler, &Event); + zx_status_t Status = + _zx_object_wait_one(Event, ZX_USER_SIGNAL_0, ZX_TIME_INFINITE, nullptr); + _zx_handle_close(Event); + ExitOnErr(Status, "_zx_object_wait_one"); + + T.detach(); +} + +void SleepSeconds(int Seconds) { + _zx_nanosleep(_zx_deadline_after(ZX_SEC(Seconds))); +} + +unsigned long GetPid() { + zx_status_t rc; + zx_info_handle_basic_t Info; + if ((rc = _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &Info, + sizeof(Info), NULL, NULL)) != ZX_OK) { + Printf("libFuzzer: unable to get info about self: %s\n", + _zx_status_get_string(rc)); + exit(1); + } + return Info.koid; +} + +size_t GetPeakRSSMb() { + zx_status_t rc; + zx_info_task_stats_t Info; + if ((rc = _zx_object_get_info(_zx_process_self(), ZX_INFO_TASK_STATS, &Info, + sizeof(Info), NULL, NULL)) != ZX_OK) { + Printf("libFuzzer: unable to get info about self: %s\n", + _zx_status_get_string(rc)); + exit(1); + } + return (Info.mem_private_bytes + Info.mem_shared_bytes) >> 20; +} + +template +class RunOnDestruction { + public: + explicit RunOnDestruction(Fn fn) : fn_(fn) {} + ~RunOnDestruction() { fn_(); } + + private: + Fn fn_; +}; + +template +RunOnDestruction at_scope_exit(Fn fn) { + return RunOnDestruction(fn); +} + +static fdio_spawn_action_t clone_fd_action(int localFd, int targetFd) { + return { + .action = FDIO_SPAWN_ACTION_CLONE_FD, + .fd = + { + .local_fd = localFd, + .target_fd = targetFd, + }, + }; +} + +int ExecuteCommand(const Command &Cmd) { + zx_status_t rc; + + // Convert arguments to C array + auto Args = Cmd.getArguments(); + size_t Argc = Args.size(); + assert(Argc != 0); + std::unique_ptr Argv(new const char *[Argc + 1]); + for (size_t i = 0; i < Argc; ++i) + Argv[i] = Args[i].c_str(); + Argv[Argc] = nullptr; + + // Determine output. On Fuchsia, the fuzzer is typically run as a component + // that lacks a mutable working directory. Fortunately, when this is the case + // a mutable output directory must be specified using "-artifact_prefix=...", + // so write the log file(s) there. + // However, we don't want to apply this logic for absolute paths. + int FdOut = STDOUT_FILENO; + bool discardStdout = false; + bool discardStderr = false; + + if (Cmd.hasOutputFile()) { + std::string Path = Cmd.getOutputFile(); + if (Path == getDevNull()) { + // On Fuchsia, there's no "/dev/null" like-file, so we + // just don't copy the FDs into the spawned process. + discardStdout = true; + } else { + bool IsAbsolutePath = Path.length() > 1 && Path[0] == '/'; + if (!IsAbsolutePath && Cmd.hasFlag("artifact_prefix")) + Path = Cmd.getFlagValue("artifact_prefix") + "/" + Path; + + FdOut = open(Path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); + if (FdOut == -1) { + Printf("libFuzzer: failed to open %s: %s\n", Path.c_str(), + strerror(errno)); + return ZX_ERR_IO; + } + } + } + auto CloseFdOut = at_scope_exit([FdOut]() { + if (FdOut != STDOUT_FILENO) + close(FdOut); + }); + + // Determine stderr + int FdErr = STDERR_FILENO; + if (Cmd.isOutAndErrCombined()) { + FdErr = FdOut; + if (discardStdout) + discardStderr = true; + } + + // Clone the file descriptors into the new process + std::vector SpawnActions; + SpawnActions.push_back(clone_fd_action(STDIN_FILENO, STDIN_FILENO)); + + if (!discardStdout) + SpawnActions.push_back(clone_fd_action(FdOut, STDOUT_FILENO)); + if (!discardStderr) + SpawnActions.push_back(clone_fd_action(FdErr, STDERR_FILENO)); + + // Start the process. + char ErrorMsg[FDIO_SPAWN_ERR_MSG_MAX_LENGTH]; + zx_handle_t ProcessHandle = ZX_HANDLE_INVALID; + rc = fdio_spawn_etc(ZX_HANDLE_INVALID, + FDIO_SPAWN_CLONE_ALL & (~FDIO_SPAWN_CLONE_STDIO), Argv[0], + Argv.get(), nullptr, SpawnActions.size(), + SpawnActions.data(), &ProcessHandle, ErrorMsg); + + if (rc != ZX_OK) { + Printf("libFuzzer: failed to launch '%s': %s, %s\n", Argv[0], ErrorMsg, + _zx_status_get_string(rc)); + return rc; + } + auto CloseHandle = at_scope_exit([&]() { _zx_handle_close(ProcessHandle); }); + + // Now join the process and return the exit status. + if ((rc = _zx_object_wait_one(ProcessHandle, ZX_PROCESS_TERMINATED, + ZX_TIME_INFINITE, nullptr)) != ZX_OK) { + Printf("libFuzzer: failed to join '%s': %s\n", Argv[0], + _zx_status_get_string(rc)); + return rc; + } + + zx_info_process_t Info; + if ((rc = _zx_object_get_info(ProcessHandle, ZX_INFO_PROCESS, &Info, + sizeof(Info), nullptr, nullptr)) != ZX_OK) { + Printf("libFuzzer: unable to get return code from '%s': %s\n", Argv[0], + _zx_status_get_string(rc)); + return rc; + } + + return Info.return_code; +} + +bool ExecuteCommand(const Command &BaseCmd, std::string *CmdOutput) { + auto LogFilePath = TempPath("SimPopenOut", ".txt"); + Command Cmd(BaseCmd); + Cmd.setOutputFile(LogFilePath); + int Ret = ExecuteCommand(Cmd); + *CmdOutput = FileToString(LogFilePath); + RemoveFile(LogFilePath); + return Ret == 0; +} + +const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, + size_t PattLen) { + return memmem(Data, DataLen, Patt, PattLen); +} + +// In fuchsia, accessing /dev/null is not supported. There's nothing +// similar to a file that discards everything that is written to it. +// The way of doing something similar in fuchsia is by using +// fdio_null_create and binding that to a file descriptor. +void DiscardOutput(int Fd) { + fdio_t *fdio_null = fdio_null_create(); + if (fdio_null == nullptr) return; + int nullfd = fdio_bind_to_fd(fdio_null, -1, 0); + if (nullfd < 0) return; + dup2(nullfd, Fd); +} + +} // namespace fuzzer + +#endif // LIBFUZZER_FUCHSIA diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp new file mode 100644 index 0000000000..95490b992e --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp @@ -0,0 +1,41 @@ +//===- FuzzerUtilLinux.cpp - Misc utils for Linux. ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils for Linux. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD || \ + LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN +#include "FuzzerCommand.h" + +#include +#include +#include +#include + + +namespace fuzzer { + +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + int exit_code = system(CmdLine.c_str()); + if (WIFEXITED(exit_code)) + return WEXITSTATUS(exit_code); + return exit_code; +} + +void DiscardOutput(int Fd) { + FILE* Temp = fopen("/dev/null", "w"); + if (!Temp) + return; + dup2(fileno(Temp), Fd); + fclose(Temp); +} + +} // namespace fuzzer + +#endif diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp new file mode 100644 index 0000000000..fc57b724db --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp @@ -0,0 +1,185 @@ +//===- FuzzerUtilPosix.cpp - Misc utils for Posix. ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils implementation using Posix API. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_POSIX +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include "FuzzerTracePC.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fuzzer { + +static void AlarmHandler(int, siginfo_t *, void *) { + Fuzzer::StaticAlarmCallback(); +} + +static void (*upstream_segv_handler)(int, siginfo_t *, void *); + +static void SegvHandler(int sig, siginfo_t *si, void *ucontext) { + assert(si->si_signo == SIGSEGV); + if (upstream_segv_handler) + return upstream_segv_handler(sig, si, ucontext); + Fuzzer::StaticCrashSignalCallback(); +} + +static void CrashHandler(int, siginfo_t *, void *) { + Fuzzer::StaticCrashSignalCallback(); +} + +static void InterruptHandler(int, siginfo_t *, void *) { + Fuzzer::StaticInterruptCallback(); +} + +static void GracefulExitHandler(int, siginfo_t *, void *) { + Fuzzer::StaticGracefulExitCallback(); +} + +static void FileSizeExceedHandler(int, siginfo_t *, void *) { + Fuzzer::StaticFileSizeExceedCallback(); +} + +static void SetSigaction(int signum, + void (*callback)(int, siginfo_t *, void *)) { + struct sigaction sigact = {}; + if (sigaction(signum, nullptr, &sigact)) { + Printf("libFuzzer: sigaction failed with %d\n", errno); + exit(1); + } + if (sigact.sa_flags & SA_SIGINFO) { + if (sigact.sa_sigaction) { + if (signum != SIGSEGV) + return; + upstream_segv_handler = sigact.sa_sigaction; + } + } else { + if (sigact.sa_handler != SIG_DFL && sigact.sa_handler != SIG_IGN && + sigact.sa_handler != SIG_ERR) + return; + } + + sigact = {}; + sigact.sa_flags = SA_SIGINFO; + sigact.sa_sigaction = callback; + if (sigaction(signum, &sigact, 0)) { + Printf("libFuzzer: sigaction failed with %d\n", errno); + exit(1); + } +} + +// Return true on success, false otherwise. +bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput) { + FILE *Pipe = popen(Cmd.toString().c_str(), "r"); + if (!Pipe) + return false; + + if (CmdOutput) { + char TmpBuffer[128]; + while (fgets(TmpBuffer, sizeof(TmpBuffer), Pipe)) + CmdOutput->append(TmpBuffer); + } + return pclose(Pipe) == 0; +} + +void SetTimer(int Seconds) { + struct itimerval T { + {Seconds, 0}, { Seconds, 0 } + }; + if (setitimer(ITIMER_REAL, &T, nullptr)) { + Printf("libFuzzer: setitimer failed with %d\n", errno); + exit(1); + } + SetSigaction(SIGALRM, AlarmHandler); +} + +void SetSignalHandler(const FuzzingOptions& Options) { + // setitimer is not implemented in emscripten. + if (Options.UnitTimeoutSec > 0 && !LIBFUZZER_EMSCRIPTEN) + SetTimer(Options.UnitTimeoutSec / 2 + 1); + if (Options.HandleInt) + SetSigaction(SIGINT, InterruptHandler); + if (Options.HandleTerm) + SetSigaction(SIGTERM, InterruptHandler); + if (Options.HandleSegv) + SetSigaction(SIGSEGV, SegvHandler); + if (Options.HandleBus) + SetSigaction(SIGBUS, CrashHandler); + if (Options.HandleAbrt) + SetSigaction(SIGABRT, CrashHandler); + if (Options.HandleIll) + SetSigaction(SIGILL, CrashHandler); + if (Options.HandleFpe) + SetSigaction(SIGFPE, CrashHandler); + if (Options.HandleXfsz) + SetSigaction(SIGXFSZ, FileSizeExceedHandler); + if (Options.HandleUsr1) + SetSigaction(SIGUSR1, GracefulExitHandler); + if (Options.HandleUsr2) + SetSigaction(SIGUSR2, GracefulExitHandler); +} + +void SleepSeconds(int Seconds) { + sleep(Seconds); // Use C API to avoid coverage from instrumented libc++. +} + +unsigned long GetPid() { return (unsigned long)getpid(); } + +size_t GetPeakRSSMb() { + struct rusage usage; + if (getrusage(RUSAGE_SELF, &usage)) + return 0; + if (LIBFUZZER_LINUX || LIBFUZZER_FREEBSD || LIBFUZZER_NETBSD || + LIBFUZZER_OPENBSD || LIBFUZZER_EMSCRIPTEN) { + // ru_maxrss is in KiB + return usage.ru_maxrss >> 10; + } else if (LIBFUZZER_APPLE) { + // ru_maxrss is in bytes + return usage.ru_maxrss >> 20; + } + assert(0 && "GetPeakRSSMb() is not implemented for your platform"); + return 0; +} + +FILE *OpenProcessPipe(const char *Command, const char *Mode) { + return popen(Command, Mode); +} + +int CloseProcessPipe(FILE *F) { + return pclose(F); +} + +const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, + size_t PattLen) { + return memmem(Data, DataLen, Patt, PattLen); +} + +std::string DisassembleCmd(const std::string &FileName) { + return "objdump -d " + FileName; +} + +std::string SearchRegexCmd(const std::string &Regex) { + return "grep '" + Regex + "'"; +} + +} // namespace fuzzer + +#endif // LIBFUZZER_POSIX diff --git a/tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp b/tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp new file mode 100644 index 0000000000..6c693e3d7e --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp @@ -0,0 +1,221 @@ +//===- FuzzerUtilWindows.cpp - Misc utils for Windows. --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Misc utils implementation for Windows. +//===----------------------------------------------------------------------===// +#include "FuzzerPlatform.h" +#if LIBFUZZER_WINDOWS +#include "FuzzerCommand.h" +#include "FuzzerIO.h" +#include "FuzzerInternal.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// This must be included after windows.h. +#include + +namespace fuzzer { + +static const FuzzingOptions* HandlerOpt = nullptr; + +static LONG CALLBACK ExceptionHandler(PEXCEPTION_POINTERS ExceptionInfo) { + switch (ExceptionInfo->ExceptionRecord->ExceptionCode) { + case EXCEPTION_ACCESS_VIOLATION: + case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: + case EXCEPTION_STACK_OVERFLOW: + if (HandlerOpt->HandleSegv) + Fuzzer::StaticCrashSignalCallback(); + break; + case EXCEPTION_DATATYPE_MISALIGNMENT: + case EXCEPTION_IN_PAGE_ERROR: + if (HandlerOpt->HandleBus) + Fuzzer::StaticCrashSignalCallback(); + break; + case EXCEPTION_ILLEGAL_INSTRUCTION: + case EXCEPTION_PRIV_INSTRUCTION: + if (HandlerOpt->HandleIll) + Fuzzer::StaticCrashSignalCallback(); + break; + case EXCEPTION_FLT_DENORMAL_OPERAND: + case EXCEPTION_FLT_DIVIDE_BY_ZERO: + case EXCEPTION_FLT_INEXACT_RESULT: + case EXCEPTION_FLT_INVALID_OPERATION: + case EXCEPTION_FLT_OVERFLOW: + case EXCEPTION_FLT_STACK_CHECK: + case EXCEPTION_FLT_UNDERFLOW: + case EXCEPTION_INT_DIVIDE_BY_ZERO: + case EXCEPTION_INT_OVERFLOW: + if (HandlerOpt->HandleFpe) + Fuzzer::StaticCrashSignalCallback(); + break; + // TODO: handle (Options.HandleXfsz) + } + return EXCEPTION_CONTINUE_SEARCH; +} + +BOOL WINAPI CtrlHandler(DWORD dwCtrlType) { + switch (dwCtrlType) { + case CTRL_C_EVENT: + if (HandlerOpt->HandleInt) + Fuzzer::StaticInterruptCallback(); + return TRUE; + case CTRL_BREAK_EVENT: + if (HandlerOpt->HandleTerm) + Fuzzer::StaticInterruptCallback(); + return TRUE; + } + return FALSE; +} + +void CALLBACK AlarmHandler(PVOID, BOOLEAN) { + Fuzzer::StaticAlarmCallback(); +} + +class TimerQ { + HANDLE TimerQueue; + public: + TimerQ() : TimerQueue(NULL) {} + ~TimerQ() { + if (TimerQueue) + DeleteTimerQueueEx(TimerQueue, NULL); + } + void SetTimer(int Seconds) { + if (!TimerQueue) { + TimerQueue = CreateTimerQueue(); + if (!TimerQueue) { + Printf("libFuzzer: CreateTimerQueue failed.\n"); + exit(1); + } + } + HANDLE Timer; + if (!CreateTimerQueueTimer(&Timer, TimerQueue, AlarmHandler, NULL, + Seconds*1000, Seconds*1000, 0)) { + Printf("libFuzzer: CreateTimerQueueTimer failed.\n"); + exit(1); + } + } +}; + +static TimerQ Timer; + +static void CrashHandler(int) { Fuzzer::StaticCrashSignalCallback(); } + +void SetSignalHandler(const FuzzingOptions& Options) { + HandlerOpt = &Options; + + if (Options.UnitTimeoutSec > 0) + Timer.SetTimer(Options.UnitTimeoutSec / 2 + 1); + + if (Options.HandleInt || Options.HandleTerm) + if (!SetConsoleCtrlHandler(CtrlHandler, TRUE)) { + DWORD LastError = GetLastError(); + Printf("libFuzzer: SetConsoleCtrlHandler failed (Error code: %lu).\n", + LastError); + exit(1); + } + + if (Options.HandleSegv || Options.HandleBus || Options.HandleIll || + Options.HandleFpe) + SetUnhandledExceptionFilter(ExceptionHandler); + + if (Options.HandleAbrt) + if (SIG_ERR == signal(SIGABRT, CrashHandler)) { + Printf("libFuzzer: signal failed with %d\n", errno); + exit(1); + } +} + +void SleepSeconds(int Seconds) { Sleep(Seconds * 1000); } + +unsigned long GetPid() { return GetCurrentProcessId(); } + +size_t GetPeakRSSMb() { + PROCESS_MEMORY_COUNTERS info; + if (!GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info))) + return 0; + return info.PeakWorkingSetSize >> 20; +} + +FILE *OpenProcessPipe(const char *Command, const char *Mode) { + return _popen(Command, Mode); +} + +int CloseProcessPipe(FILE *F) { + return _pclose(F); +} + +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + return system(CmdLine.c_str()); +} + +bool ExecuteCommand(const Command &Cmd, std::string *CmdOutput) { + FILE *Pipe = _popen(Cmd.toString().c_str(), "r"); + if (!Pipe) + return false; + + if (CmdOutput) { + char TmpBuffer[128]; + while (fgets(TmpBuffer, sizeof(TmpBuffer), Pipe)) + CmdOutput->append(TmpBuffer); + } + return _pclose(Pipe) == 0; +} + +const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, + size_t PattLen) { + // TODO: make this implementation more efficient. + const char *Cdata = (const char *)Data; + const char *Cpatt = (const char *)Patt; + + if (!Data || !Patt || DataLen == 0 || PattLen == 0 || DataLen < PattLen) + return NULL; + + if (PattLen == 1) + return memchr(Data, *Cpatt, DataLen); + + const char *End = Cdata + DataLen - PattLen + 1; + + for (const char *It = Cdata; It < End; ++It) + if (It[0] == Cpatt[0] && memcmp(It, Cpatt, PattLen) == 0) + return It; + + return NULL; +} + +std::string DisassembleCmd(const std::string &FileName) { + Vector command_vector; + command_vector.push_back("dumpbin /summary > nul"); + if (ExecuteCommand(Command(command_vector)) == 0) + return "dumpbin /disasm " + FileName; + Printf("libFuzzer: couldn't find tool to disassemble (dumpbin)\n"); + exit(1); +} + +std::string SearchRegexCmd(const std::string &Regex) { + return "findstr /r \"" + Regex + "\""; +} + +void DiscardOutput(int Fd) { + FILE* Temp = fopen("nul", "w"); + if (!Temp) + return; + _dup2(_fileno(Temp), Fd); + fclose(Temp); +} + +} // namespace fuzzer + +#endif // LIBFUZZER_WINDOWS diff --git a/tools/fuzzing/libfuzzer/FuzzerValueBitMap.h b/tools/fuzzing/libfuzzer/FuzzerValueBitMap.h new file mode 100644 index 0000000000..ddbfe200af --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerValueBitMap.h @@ -0,0 +1,73 @@ +//===- FuzzerValueBitMap.h - INTERNAL - Bit map -----------------*- C++ -* ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// ValueBitMap. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_VALUE_BIT_MAP_H +#define LLVM_FUZZER_VALUE_BIT_MAP_H + +#include "FuzzerPlatform.h" +#include + +namespace fuzzer { + +// A bit map containing kMapSizeInWords bits. +struct ValueBitMap { + static const size_t kMapSizeInBits = 1 << 16; + static const size_t kMapPrimeMod = 65371; // Largest Prime < kMapSizeInBits; + static const size_t kBitsInWord = (sizeof(uintptr_t) * 8); + static const size_t kMapSizeInWords = kMapSizeInBits / kBitsInWord; + public: + + // Clears all bits. + void Reset() { memset(Map, 0, sizeof(Map)); } + + // Computes a hash function of Value and sets the corresponding bit. + // Returns true if the bit was changed from 0 to 1. + ATTRIBUTE_NO_SANITIZE_ALL + inline bool AddValue(uintptr_t Value) { + uintptr_t Idx = Value % kMapSizeInBits; + uintptr_t WordIdx = Idx / kBitsInWord; + uintptr_t BitIdx = Idx % kBitsInWord; + uintptr_t Old = Map[WordIdx]; + uintptr_t New = Old | (1ULL << BitIdx); + Map[WordIdx] = New; + return New != Old; + } + + ATTRIBUTE_NO_SANITIZE_ALL + inline bool AddValueModPrime(uintptr_t Value) { + return AddValue(Value % kMapPrimeMod); + } + + inline bool Get(uintptr_t Idx) { + assert(Idx < kMapSizeInBits); + uintptr_t WordIdx = Idx / kBitsInWord; + uintptr_t BitIdx = Idx % kBitsInWord; + return Map[WordIdx] & (1ULL << BitIdx); + } + + size_t SizeInBits() const { return kMapSizeInBits; } + + template + ATTRIBUTE_NO_SANITIZE_ALL + void ForEach(Callback CB) const { + for (size_t i = 0; i < kMapSizeInWords; i++) + if (uintptr_t M = Map[i]) + for (size_t j = 0; j < sizeof(M) * 8; j++) + if (M & ((uintptr_t)1 << j)) + CB(i * sizeof(M) * 8 + j); + } + + private: + ATTRIBUTE_ALIGNED(512) uintptr_t Map[kMapSizeInWords]; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_VALUE_BIT_MAP_H diff --git a/tools/fuzzing/libfuzzer/LICENSE.TXT b/tools/fuzzing/libfuzzer/LICENSE.TXT new file mode 100644 index 0000000000..555c8bb952 --- /dev/null +++ b/tools/fuzzing/libfuzzer/LICENSE.TXT @@ -0,0 +1,68 @@ +============================================================================== +LLVM Release License +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2003-2016 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== +Copyrights and Licenses for Third Party Software Distributed with LLVM: +============================================================================== +The LLVM software contains code written by third parties. Such software will +have its own individual LICENSE.TXT file in the directory in which it appears. +This file will describe the copyrights, license, and restrictions which apply +to that code. + +The disclaimer of warranty in the University of Illinois Open Source License +applies to all code in the LLVM Distribution, and nothing in any of the +other licenses gives permission to use the names of the LLVM Team or the +University of Illinois to endorse or promote products derived from this +Software. + +The following pieces of software have additional or alternate copyrights, +licenses, and/or restrictions: + +Program Directory +------- --------- +Google Test llvm/utils/unittest/googletest +OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} +pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} +ARM contributions llvm/lib/Target/ARM/LICENSE.TXT +md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h diff --git a/tools/fuzzing/libfuzzer/clone_libfuzzer.sh b/tools/fuzzing/libfuzzer/clone_libfuzzer.sh new file mode 100755 index 0000000000..ca01df2978 --- /dev/null +++ b/tools/fuzzing/libfuzzer/clone_libfuzzer.sh @@ -0,0 +1,36 @@ +#!/bin/bash -e + +# Optionally get revision from cmd line +[ $1 ] && REVISION=$1 || REVISION=76d07503f0c69f6632e6d8d4736e2a4cb4055a92 + +mkdir tmp +git clone --single-branch --no-checkout --shallow-since "2020-07-01" https://github.com/llvm/llvm-project tmp + +(cd tmp && git reset --hard $REVISION) + +# libFuzzer source files +CPPS=($(ls tmp/compiler-rt/lib/fuzzer/*.cpp | sort -r)) +CPPS=(${CPPS[@]##*/}) +CPPS=(${CPPS[@]##FuzzerMain*}) # ignored +CPPS=(${CPPS[@]##FuzzerInterceptors*}) # ignored + +# Update SOURCES entries +sed -e "/^SOURCES/,/^]/ {/'/d}" -i moz.build +for CPP in ${CPPS[@]}; do sed -e "/^SOURCES/ a \\ '${CPP}'," -i moz.build; done + +# Remove previous files +rm *.{cpp,h,def} + +# Copy files +cp tmp/compiler-rt/lib/fuzzer/*.{cpp,h,def} . + +# Apply local patches +for patch in patches/*.patch +do + patch -p4 < $patch +done + +# Remove the temporary directory +rm -Rf tmp/ + +echo "Updated libFuzzer to ${REVISION}" diff --git a/tools/fuzzing/libfuzzer/moz.build b/tools/fuzzing/libfuzzer/moz.build new file mode 100644 index 0000000000..a04169df57 --- /dev/null +++ b/tools/fuzzing/libfuzzer/moz.build @@ -0,0 +1,55 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library('fuzzer') + +EXPORTS += [ + 'FuzzerDefs.h', + 'FuzzerExtFunctions.def', + 'FuzzerExtFunctions.h', +] + +SOURCES += [ + 'FuzzerCrossOver.cpp', + 'FuzzerDataFlowTrace.cpp', + 'FuzzerDriver.cpp', + 'FuzzerExtFunctionsDlsym.cpp', + 'FuzzerExtFunctionsWeak.cpp', + 'FuzzerExtFunctionsWindows.cpp', + 'FuzzerExtraCounters.cpp', + 'FuzzerFork.cpp', + 'FuzzerIO.cpp', + 'FuzzerIOPosix.cpp', + 'FuzzerIOWindows.cpp', + 'FuzzerLoop.cpp', + 'FuzzerMerge.cpp', + 'FuzzerMutate.cpp', + 'FuzzerSHA1.cpp', + 'FuzzerTracePC.cpp', + 'FuzzerUtil.cpp', + 'FuzzerUtilDarwin.cpp', + 'FuzzerUtilFuchsia.cpp', + 'FuzzerUtilLinux.cpp', + 'FuzzerUtilPosix.cpp', + 'FuzzerUtilWindows.cpp', +] + +if CONFIG['CC_TYPE'] == 'clang': + CXXFLAGS += ['-Wno-unreachable-code-return'] + +# According to the LLVM docs, LibFuzzer isn't supposed to be built with any +# sanitizer flags and in fact, building it with ASan coverage currently causes +# Clang 3.9+ to crash, so we filter out all sanitizer-related flags here. +for flags_var in ('OS_CFLAGS', 'OS_CXXFLAGS'): + COMPILE_FLAGS[flags_var] = [ + f for f in COMPILE_FLAGS.get(flags_var, []) + if not f.startswith('-fsanitize') + ] + +LINK_FLAGS['OS'] = [ + f for f in LINK_FLAGS.get('OS', []) + if not f.startswith('-fsanitize') +] diff --git a/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch b/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch new file mode 100644 index 0000000000..34562e27a5 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/10-ef-runtime.patch @@ -0,0 +1,31 @@ +# HG changeset patch +# User Christian Holler +# Date 1596126054 -7200 +# Thu Jul 30 18:20:54 2020 +0200 +# Node ID 8a2a26b33d516c43c366b2f24d731d27d9843349 +# Parent 997c4109edd112695097fd8c55cbacd976cab24a +[libFuzzer] Allow external functions to be defined at runtime + +diff --git a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp +@@ -608,17 +608,18 @@ static Vector ReadCorpora(con + SizedFiles.push_back({File, Size}); + return SizedFiles; + } + + int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + using namespace fuzzer; + assert(argc && argv && "Argument pointers cannot be nullptr"); + std::string Argv0((*argv)[0]); +- EF = new ExternalFunctions(); ++ if (!EF) ++ EF = new ExternalFunctions(); + if (EF->LLVMFuzzerInitialize) + EF->LLVMFuzzerInitialize(argc, argv); + if (EF->__msan_scoped_disable_interceptor_checks) + EF->__msan_scoped_disable_interceptor_checks(); + const Vector Args(*argv, *argv + *argc); + assert(!Args.empty()); + ProgName = new std::string(Args[0]); + if (Argv0 != *ProgName) { diff --git a/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch b/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch new file mode 100644 index 0000000000..650444442c --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/11-callback-rv.patch @@ -0,0 +1,132 @@ +# HG changeset patch +# User Christian Holler +# Date 1596126448 -7200 +# Thu Jul 30 18:27:28 2020 +0200 +# Node ID ea198a0331a6db043cb5978512226977514104db +# Parent 8a2a26b33d516c43c366b2f24d731d27d9843349 +[libFuzzer] Change libFuzzer callback contract to allow positive return values + +diff --git a/tools/fuzzing/libfuzzer/FuzzerInternal.h b/tools/fuzzing/libfuzzer/FuzzerInternal.h +--- a/tools/fuzzing/libfuzzer/FuzzerInternal.h ++++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h +@@ -60,17 +60,17 @@ public: + + static void StaticAlarmCallback(); + static void StaticCrashSignalCallback(); + static void StaticExitCallback(); + static void StaticInterruptCallback(); + static void StaticFileSizeExceedCallback(); + static void StaticGracefulExitCallback(); + +- void ExecuteCallback(const uint8_t *Data, size_t Size); ++ int ExecuteCallback(const uint8_t *Data, size_t Size); + bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, + InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); + + // Merge Corpora[1:] into Corpora[0]. + void Merge(const Vector &Corpora); + void CrashResistantMergeInternalStep(const std::string &ControlFilePath); + MutationDispatcher &GetMD() { return MD; } + void PrintFinalStats(); +diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +@@ -463,17 +463,19 @@ static void RenameFeatureSetFile(const s + DirPlusFile(FeaturesDir, NewFile)); + } + + bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, + InputInfo *II, bool *FoundUniqFeatures) { + if (!Size) + return false; + +- ExecuteCallback(Data, Size); ++ if (ExecuteCallback(Data, Size) > 0) { ++ return false; ++ } + + UniqFeatureSetTmp.clear(); + size_t FoundUniqFeaturesOfII = 0; + size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); + TPC.CollectFeatures([&](size_t Feature) { + if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + UniqFeatureSetTmp.push_back(Feature); + if (Options.Entropic) +@@ -530,48 +532,49 @@ static bool LooseMemeq(const uint8_t *A, + const size_t Limit = 64; + if (Size <= 64) + return !memcmp(A, B, Size); + // Compare first and last Limit/2 bytes. + return !memcmp(A, B, Limit / 2) && + !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); + } + +-void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { ++int Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { + TPC.RecordInitialStack(); + TotalNumberOfRuns++; + assert(InFuzzingThread()); + // We copy the contents of Unit into a separate heap buffer + // so that we reliably find buffer overflows in it. + uint8_t *DataCopy = new uint8_t[Size]; + memcpy(DataCopy, Data, Size); + if (EF->__msan_unpoison) + EF->__msan_unpoison(DataCopy, Size); + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(2); + if (CurrentUnitData && CurrentUnitData != Data) + memcpy(CurrentUnitData, Data, Size); + CurrentUnitSize = Size; ++ int Res = 0; + { + ScopedEnableMsanInterceptorChecks S; + AllocTracer.Start(Options.TraceMalloc); + UnitStartTime = system_clock::now(); + TPC.ResetMaps(); + RunningUserCallback = true; +- int Res = CB(DataCopy, Size); ++ Res = CB(DataCopy, Size); + RunningUserCallback = false; + UnitStopTime = system_clock::now(); +- (void)Res; +- assert(Res == 0); ++ assert(Res >= 0); + HasMoreMallocsThanFrees = AllocTracer.Stop(); + } + if (!LooseMemeq(DataCopy, Data, Size)) + CrashOnOverwrittenData(); + CurrentUnitSize = 0; + delete[] DataCopy; ++ return Res; + } + + std::string Fuzzer::WriteToOutputCorpus(const Unit &U) { + if (Options.OnlyASCII) + assert(IsASCII(U)); + if (Options.OutputCorpus.empty()) + return ""; + std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); +diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp +@@ -223,17 +223,19 @@ void Fuzzer::CrashResistantMergeInternal + U.shrink_to_fit(); + } + + // Write the pre-run marker. + OF << "STARTED " << i << " " << U.size() << "\n"; + OF.flush(); // Flush is important since Command::Execute may crash. + // Run. + TPC.ResetMaps(); +- ExecuteCallback(U.data(), U.size()); ++ if (ExecuteCallback(U.data(), U.size()) > 0) { ++ continue; ++ } + // Collect coverage. We are iterating over the files in this order: + // * First, files in the initial corpus ordered by size, smallest first. + // * Then, all other files, smallest first. + // So it makes no sense to record all features for all files, instead we + // only record features that were not seen before. + Set UniqFeatures; + TPC.CollectFeatures([&](size_t Feature) { + if (AllFeatures.insert(Feature).second) diff --git a/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch b/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch new file mode 100644 index 0000000000..2457c1f046 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/12-custom-mutator-fail.patch @@ -0,0 +1,53 @@ +# HG changeset patch +# User Christian Holler +# Date 1596126768 -7200 +# Thu Jul 30 18:32:48 2020 +0200 +# Node ID 64e7d096fa77a62b71a306b2c5383b8f75ac4945 +# Parent ea198a0331a6db043cb5978512226977514104db +[libFuzzer] Allow custom mutators to fail + +diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +@@ -690,16 +690,20 @@ void Fuzzer::MutateAndTestOne() { + if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() && + Size <= CurrentMaxMutationLen) + NewSize = MD.MutateWithMask(CurrentUnitData, Size, Size, + II.DataFlowTraceForFocusFunction); + + // If MutateWithMask either failed or wasn't called, call default Mutate. + if (!NewSize) + NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); ++ ++ if (!NewSize) ++ continue; ++ + assert(NewSize > 0 && "Mutator returned empty unit"); + assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); + Size = NewSize; + II.NumExecutedMutations++; + Corpus.IncrementNumExecutedMutations(); + + bool FoundUniqFeatures = false; + bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II, +@@ -850,17 +854,19 @@ void Fuzzer::Loop(Vector &Cor + void Fuzzer::MinimizeCrashLoop(const Unit &U) { + if (U.size() <= 1) + return; + while (!TimedOut() && TotalNumberOfRuns < Options.MaxNumberOfRuns) { + MD.StartMutationSequence(); + memcpy(CurrentUnitData, U.data(), U.size()); + for (int i = 0; i < Options.MutateDepth; i++) { + size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen); +- assert(NewSize > 0 && NewSize <= MaxMutationLen); ++ assert(NewSize <= MaxMutationLen); ++ if (!NewSize) ++ continue; + ExecuteCallback(CurrentUnitData, NewSize); + PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); + TryDetectingAMemoryLeak(CurrentUnitData, NewSize, + /*DuringInitialCorpusExecution*/ false); + } + } + } + diff --git a/tools/fuzzing/libfuzzer/patches/13-unused-write.patch b/tools/fuzzing/libfuzzer/patches/13-unused-write.patch new file mode 100644 index 0000000000..f8ef7a5ac9 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/13-unused-write.patch @@ -0,0 +1,88 @@ +# HG changeset patch +# User Christian Holler +# Date 1596126946 -7200 +# Thu Jul 30 18:35:46 2020 +0200 +# Node ID 6c779ec81530b6784a714063af66085681ab7318 +# Parent 64e7d096fa77a62b71a306b2c5383b8f75ac4945 +[libFuzzer] Suppress warnings about unused return values + +diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.cpp b/tools/fuzzing/libfuzzer/FuzzerIO.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerIO.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerIO.cpp +@@ -3,16 +3,17 @@ + // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + // See https://llvm.org/LICENSE.txt for license information. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // + //===----------------------------------------------------------------------===// + // IO functions. + //===----------------------------------------------------------------------===// + ++#include "mozilla/Unused.h" + #include "FuzzerDefs.h" + #include "FuzzerExtFunctions.h" + #include "FuzzerIO.h" + #include "FuzzerUtil.h" + #include + #include + #include + #include +@@ -68,17 +69,17 @@ void WriteToFile(const std::string &Data + WriteToFile(reinterpret_cast(Data.c_str()), Data.size(), + Path); + } + + void WriteToFile(const uint8_t *Data, size_t Size, const std::string &Path) { + // Use raw C interface because this function may be called from a sig handler. + FILE *Out = fopen(Path.c_str(), "wb"); + if (!Out) return; +- fwrite(Data, sizeof(Data[0]), Size, Out); ++ mozilla::Unused << fwrite(Data, sizeof(Data[0]), Size, Out); + fclose(Out); + } + + void ReadDirToVectorOfUnits(const char *Path, Vector *V, + long *Epoch, size_t MaxSize, bool ExitOnError) { + long E = Epoch ? *Epoch : 0; + Vector Files; + ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); +diff --git a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp +--- a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp +@@ -2,16 +2,17 @@ + // + // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + // See https://llvm.org/LICENSE.txt for license information. + // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + // + //===----------------------------------------------------------------------===// + // IO functions implementation using Posix API. + //===----------------------------------------------------------------------===// ++#include "mozilla/Unused.h" + #include "FuzzerPlatform.h" + #if LIBFUZZER_POSIX || LIBFUZZER_FUCHSIA + + #include "FuzzerExtFunctions.h" + #include "FuzzerIO.h" + #include + #include + #include +@@ -150,17 +151,17 @@ bool IsInterestingCoverageFile(const std + if (FileName.find("/usr/include/") != std::string::npos) + return false; + if (FileName == "") + return false; + return true; + } + + void RawPrint(const char *Str) { +- write(2, Str, strlen(Str)); ++ mozilla::Unused << write(2, Str, strlen(Str)); + } + + void MkDir(const std::string &Path) { + mkdir(Path.c_str(), 0700); + } + + void RmDir(const std::string &Path) { + rmdir(Path.c_str()); diff --git a/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch b/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch new file mode 100644 index 0000000000..8e02273193 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/14-explicit-allocator.patch @@ -0,0 +1,30 @@ +# HG changeset patch +# User Christian Holler +# Date 1596126981 -7200 +# Thu Jul 30 18:36:21 2020 +0200 +# Node ID 069dfa3715b1d30905ff0ea1c0f66db88ce146f9 +# Parent 6c779ec81530b6784a714063af66085681ab7318 +[libFuzzer] Make fuzzer_allocator explicit + +diff --git a/tools/fuzzing/libfuzzer/FuzzerDefs.h b/tools/fuzzing/libfuzzer/FuzzerDefs.h +--- a/tools/fuzzing/libfuzzer/FuzzerDefs.h ++++ b/tools/fuzzing/libfuzzer/FuzzerDefs.h +@@ -41,17 +41,17 @@ extern ExternalFunctions *EF; + // We are using a custom allocator to give a different symbol name to STL + // containers in order to avoid ODR violations. + template + class fuzzer_allocator: public std::allocator { + public: + fuzzer_allocator() = default; + + template +- fuzzer_allocator(const fuzzer_allocator&) {} ++ explicit fuzzer_allocator(const fuzzer_allocator&) {} + + template + struct rebind { typedef fuzzer_allocator other; }; + }; + + template + using Vector = std::vector>; + diff --git a/tools/fuzzing/libfuzzer/patches/15-return-to-exit.patch b/tools/fuzzing/libfuzzer/patches/15-return-to-exit.patch new file mode 100644 index 0000000000..4adb9f59a9 --- /dev/null +++ b/tools/fuzzing/libfuzzer/patches/15-return-to-exit.patch @@ -0,0 +1,968 @@ +commit f80733b3b1b5e05e7dfd7a071f60050fe20108c3 +Author: Jesse Schwartzentruber +Date: Mon Mar 1 15:47:38 2021 -0500 + + [libfuzzer] In most cases, return instead of exit(). + +diff --git a/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp +index 0e9cdf7e66b1..06ea287a3cfe 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.cpp +@@ -102,9 +102,11 @@ Vector BlockCoverage::FunctionWeights(size_t NumFunctions) const { + return Res; + } + +-void DataFlowTrace::ReadCoverage(const std::string &DirPath) { ++int DataFlowTrace::ReadCoverage(const std::string &DirPath) { + Vector Files; +- GetSizedFilesFromDir(DirPath, &Files); ++ int Res = GetSizedFilesFromDir(DirPath, &Files); ++ if (Res != 0) ++ return Res; + for (auto &SF : Files) { + auto Name = Basename(SF.File); + if (Name == kFunctionsTxt) continue; +@@ -112,6 +114,7 @@ void DataFlowTrace::ReadCoverage(const std::string &DirPath) { + std::ifstream IF(SF.File); + Coverage.AppendCoverage(IF); + } ++ return 0; + } + + static void DFTStringAppendToVector(Vector *DFT, +@@ -157,12 +160,14 @@ static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum, + return true; + } + +-bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, ++int DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + Vector &CorporaFiles, Random &Rand) { +- if (DirPath.empty()) return false; ++ if (DirPath.empty()) return 0; + Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str()); + Vector Files; +- GetSizedFilesFromDir(DirPath, &Files); ++ int Res = GetSizedFilesFromDir(DirPath, &Files); ++ if (Res != 0) ++ return Res; + std::string L; + size_t FocusFuncIdx = SIZE_MAX; + Vector FunctionNames; +@@ -181,14 +186,16 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + FocusFuncIdx = NumFunctions - 1; + } + if (!NumFunctions) +- return false; ++ return 0; + + if (*FocusFunction == "auto") { + // AUTOFOCUS works like this: + // * reads the coverage data from the DFT files. + // * assigns weights to functions based on coverage. + // * chooses a random function according to the weights. +- ReadCoverage(DirPath); ++ Res = ReadCoverage(DirPath); ++ if (Res != 0) ++ return Res; + auto Weights = Coverage.FunctionWeights(NumFunctions); + Vector Intervals(NumFunctions + 1); + std::iota(Intervals.begin(), Intervals.end(), 0); +@@ -209,7 +216,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + } + + if (!NumFunctions || FocusFuncIdx == SIZE_MAX || Files.size() <= 1) +- return false; ++ return 0; + + // Read traces. + size_t NumTraceFiles = 0; +@@ -228,8 +235,10 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + FunctionNum == FocusFuncIdx) { + NumTracesWithFocusFunction++; + +- if (FunctionNum >= NumFunctions) +- return ParseError("N is greater than the number of functions", L); ++ if (FunctionNum >= NumFunctions) { ++ ParseError("N is greater than the number of functions", L); ++ return 0; ++ } + Traces[Name] = DFTStringToVector(DFTString); + // Print just a few small traces. + if (NumTracesWithFocusFunction <= 3 && DFTString.size() <= 16) +@@ -241,7 +250,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, + Printf("INFO: DataFlowTrace: %zd trace files, %zd functions, " + "%zd traces with focus function\n", + NumTraceFiles, NumFunctions, NumTracesWithFocusFunction); +- return NumTraceFiles > 0; ++ return 0; + } + + int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, +diff --git a/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h +index d6e3de30a4ef..767bad24f1d0 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h ++++ b/tools/fuzzing/libfuzzer/FuzzerDataFlowTrace.h +@@ -113,8 +113,8 @@ class BlockCoverage { + + class DataFlowTrace { + public: +- void ReadCoverage(const std::string &DirPath); +- bool Init(const std::string &DirPath, std::string *FocusFunction, ++ int ReadCoverage(const std::string &DirPath); ++ int Init(const std::string &DirPath, std::string *FocusFunction, + Vector &CorporaFiles, Random &Rand); + void Clear() { Traces.clear(); } + const Vector *Get(const std::string &InputSha1) const { +diff --git a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp +index cd720200848b..bedad16efa7b 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp +@@ -326,7 +326,7 @@ int CleanseCrashInput(const Vector &Args, + if (Inputs->size() != 1 || !Flags.exact_artifact_path) { + Printf("ERROR: -cleanse_crash should be given one input file and" + " -exact_artifact_path\n"); +- exit(1); ++ return 1; + } + std::string InputFilePath = Inputs->at(0); + std::string OutputFilePath = Flags.exact_artifact_path; +@@ -380,7 +380,7 @@ int MinimizeCrashInput(const Vector &Args, + const FuzzingOptions &Options) { + if (Inputs->size() != 1) { + Printf("ERROR: -minimize_crash should be given one input file\n"); +- exit(1); ++ return 1; + } + std::string InputFilePath = Inputs->at(0); + Command BaseCmd(Args); +@@ -411,7 +411,7 @@ int MinimizeCrashInput(const Vector &Args, + bool Success = ExecuteCommand(Cmd, &CmdOutput); + if (Success) { + Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); +- exit(1); ++ return 1; + } + Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " + "it further\n", +@@ -466,42 +466,51 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { + Printf("INFO: Starting MinimizeCrashInputInternalStep: %zd\n", U.size()); + if (U.size() < 2) { + Printf("INFO: The input is small enough, exiting\n"); +- exit(0); ++ return 0; + } + F->SetMaxInputLen(U.size()); + F->SetMaxMutationLen(U.size() - 1); + F->MinimizeCrashLoop(U); + Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n"); +- exit(0); + return 0; + } + +-void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector &Args, ++int Merge(Fuzzer *F, FuzzingOptions &Options, const Vector &Args, + const Vector &Corpora, const char *CFPathOrNull) { + if (Corpora.size() < 2) { + Printf("INFO: Merge requires two or more corpus dirs\n"); +- exit(0); ++ return 0; + } + + Vector OldCorpus, NewCorpus; +- GetSizedFilesFromDir(Corpora[0], &OldCorpus); +- for (size_t i = 1; i < Corpora.size(); i++) +- GetSizedFilesFromDir(Corpora[i], &NewCorpus); ++ int Res = GetSizedFilesFromDir(Corpora[0], &OldCorpus); ++ if (Res != 0) ++ return Res; ++ for (size_t i = 1; i < Corpora.size(); i++) { ++ Res = GetSizedFilesFromDir(Corpora[i], &NewCorpus); ++ if (Res != 0) ++ return Res; ++ } + std::sort(OldCorpus.begin(), OldCorpus.end()); + std::sort(NewCorpus.begin(), NewCorpus.end()); + + std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt"); + Vector NewFiles; + Set NewFeatures, NewCov; +- CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, ++ Res = CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, + {}, &NewCov, CFPath, true); ++ if (Res != 0) ++ return Res; ++ ++ if (F->isGracefulExitRequested()) ++ return 0; + for (auto &Path : NewFiles) + F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); + // We are done, delete the control file if it was a temporary one. + if (!Flags.merge_control_file) + RemoveFile(CFPath); + +- exit(0); ++ return 0; + } + + int AnalyzeDictionary(Fuzzer *F, const Vector& Dict, +@@ -570,10 +579,9 @@ int AnalyzeDictionary(Fuzzer *F, const Vector& Dict, + return 0; + } + +-Vector ParseSeedInuts(const char *seed_inputs) { ++int ParseSeedInuts(const char *seed_inputs, Vector &Files) { + // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file +- Vector Files; +- if (!seed_inputs) return Files; ++ if (!seed_inputs) return 0; + std::string SeedInputs; + if (Flags.seed_inputs[0] == '@') + SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. +@@ -581,7 +589,7 @@ Vector ParseSeedInuts(const char *seed_inputs) { + SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. + if (SeedInputs.empty()) { + Printf("seed_inputs is empty or @file does not exist.\n"); +- exit(1); ++ return 1; + } + // Parse SeedInputs. + size_t comma_pos = 0; +@@ -590,7 +598,7 @@ Vector ParseSeedInuts(const char *seed_inputs) { + SeedInputs = SeedInputs.substr(0, comma_pos); + } + Files.push_back(SeedInputs); +- return Files; ++ return 0; + } + + static Vector ReadCorpora(const Vector &CorpusDirs, +@@ -624,7 +632,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + ProgName = new std::string(Args[0]); + if (Argv0 != *ProgName) { + Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n"); +- exit(1); ++ return 1; + } + ParseFlags(Args, EF); + if (Flags.help) { +@@ -723,7 +731,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + if (!Options.FocusFunction.empty()) { + Printf("ERROR: The parameters `--entropic` and `--focus_function` cannot " + "be used together.\n"); +- exit(1); ++ return 1; + } + Printf("INFO: Running with entropic power schedule (0x%X, %d).\n", + Options.EntropicFeatureFrequencyThreshold, +@@ -809,22 +817,21 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + "*** executed the target code on a fixed set of inputs.\n" + "***\n"); + F->PrintFinalStats(); +- exit(0); ++ return 0; + } + + if (Flags.fork) +- FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); ++ return FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); + + if (Flags.merge) +- Merge(F, Options, Args, *Inputs, Flags.merge_control_file); ++ return Merge(F, Options, Args, *Inputs, Flags.merge_control_file); + + if (Flags.merge_inner) { + const size_t kDefaultMaxMergeLen = 1 << 20; + if (Options.MaxLen == 0) + F->SetMaxInputLen(kDefaultMaxMergeLen); + assert(Flags.merge_control_file); +- F->CrashResistantMergeInternalStep(Flags.merge_control_file); +- exit(0); ++ return F->CrashResistantMergeInternalStep(Flags.merge_control_file); + } + + if (Flags.analyze_dict) { +@@ -842,21 +849,31 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { + } + if (AnalyzeDictionary(F, Dictionary, InitialCorpus)) { + Printf("Dictionary analysis failed\n"); +- exit(1); ++ return 1; + } + Printf("Dictionary analysis succeeded\n"); +- exit(0); ++ return 0; + } + +- auto CorporaFiles = ReadCorpora(*Inputs, ParseSeedInuts(Flags.seed_inputs)); +- F->Loop(CorporaFiles); ++ { ++ Vector Files; ++ int Res = ParseSeedInuts(Flags.seed_inputs, Files); ++ if (Res != 0) ++ return Res; ++ auto CorporaFiles = ReadCorpora(*Inputs, Files); ++ Res = F->Loop(CorporaFiles); ++ if (Res != 0) ++ return Res; ++ if (F->isGracefulExitRequested()) ++ return 0; ++ } + + if (Flags.verbosity) + Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), + F->secondsSinceProcessStartUp()); + F->PrintFinalStats(); + +- exit(0); // Don't let F destroy itself. ++ return 0; // Don't let F destroy itself. + } + + extern "C" ATTRIBUTE_INTERFACE int +diff --git a/tools/fuzzing/libfuzzer/FuzzerFork.cpp b/tools/fuzzing/libfuzzer/FuzzerFork.cpp +index d9e6b79443e0..ee2a99a250c1 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerFork.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerFork.cpp +@@ -177,14 +177,16 @@ struct GlobalEnv { + return Job; + } + +- void RunOneMergeJob(FuzzJob *Job) { ++ int RunOneMergeJob(FuzzJob *Job) { + auto Stats = ParseFinalStatsFromLog(Job->LogPath); + NumRuns += Stats.number_of_executed_units; + + Vector TempFiles, MergeCandidates; + // Read all newly created inputs and their feature sets. + // Choose only those inputs that have new features. +- GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); ++ int Res = GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); ++ if (Res != 0) ++ return Res; + std::sort(TempFiles.begin(), TempFiles.end()); + for (auto &F : TempFiles) { + auto FeatureFile = F.File; +@@ -207,12 +209,14 @@ struct GlobalEnv { + Stats.average_exec_per_sec, NumOOMs, NumTimeouts, NumCrashes, + secondsSinceProcessStartUp(), Job->JobId, Job->DftTimeInSeconds); + +- if (MergeCandidates.empty()) return; ++ if (MergeCandidates.empty()) return 0; + + Vector FilesToAdd; + Set NewFeatures, NewCov; + CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, + &NewFeatures, Cov, &NewCov, Job->CFPath, false); ++ if (Fuzzer::isGracefulExitRequested()) ++ return 0; + for (auto &Path : FilesToAdd) { + auto U = FileToVector(Path); + auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); +@@ -226,7 +230,7 @@ struct GlobalEnv { + if (TPC.PcIsFuncEntry(TE)) + PrintPC(" NEW_FUNC: %p %F %L\n", "", + TPC.GetNextInstructionPc(TE->PC)); +- ++ return 0; + } + + +@@ -280,7 +284,7 @@ void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { + } + + // This is just a skeleton of an experimental -fork=1 feature. +-void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, ++int FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector &Args, + const Vector &CorpusDirs, int NumJobs) { + Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); +@@ -294,8 +298,12 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + Env.DataFlowBinary = Options.CollectDataFlow; + + Vector SeedFiles; +- for (auto &Dir : CorpusDirs) +- GetSizedFilesFromDir(Dir, &SeedFiles); ++ int Res; ++ for (auto &Dir : CorpusDirs) { ++ Res = GetSizedFilesFromDir(Dir, &SeedFiles); ++ if (Res != 0) ++ return Res; ++ } + std::sort(SeedFiles.begin(), SeedFiles.end()); + Env.TempDir = TempPath("FuzzWithFork", ".dir"); + Env.DFTDir = DirPlusFile(Env.TempDir, "DFT"); +@@ -310,9 +318,14 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + Env.MainCorpusDir = CorpusDirs[0]; + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); +- CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, ++ Res = CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, + {}, &Env.Cov, + CFPath, false); ++ if (Res != 0) ++ return Res; ++ if (Fuzzer::isGracefulExitRequested()) ++ return 0; ++ + RemoveFile(CFPath); + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, + Env.Files.size(), Env.TempDir.c_str()); +@@ -345,9 +358,14 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + StopJobs(); + break; + } +- Fuzzer::MaybeExitGracefully(); ++ if (Fuzzer::MaybeExitGracefully()) ++ return 0; + +- Env.RunOneMergeJob(Job.get()); ++ Res = Env.RunOneMergeJob(Job.get()); ++ if (Res != 0) ++ return Res; ++ if (Fuzzer::isGracefulExitRequested()) ++ return 0; + + // Continue if our crash is one of the ignorred ones. + if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) +@@ -403,7 +421,7 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + // Use the exit code from the last child process. + Printf("INFO: exiting: %d time: %zds\n", ExitCode, + Env.secondsSinceProcessStartUp()); +- exit(ExitCode); ++ return ExitCode; + } + + } // namespace fuzzer +diff --git a/tools/fuzzing/libfuzzer/FuzzerFork.h b/tools/fuzzing/libfuzzer/FuzzerFork.h +index b29a43e13fbc..1352171ad49d 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerFork.h ++++ b/tools/fuzzing/libfuzzer/FuzzerFork.h +@@ -16,7 +16,7 @@ + #include + + namespace fuzzer { +-void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, ++int FuzzWithFork(Random &Rand, const FuzzingOptions &Options, + const Vector &Args, + const Vector &CorpusDirs, int NumJobs); + } // namespace fuzzer +diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.cpp b/tools/fuzzing/libfuzzer/FuzzerIO.cpp +index 0053ef39f2b9..6be2be67c691 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerIO.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerIO.cpp +@@ -82,7 +82,9 @@ void ReadDirToVectorOfUnits(const char *Path, Vector *V, + long *Epoch, size_t MaxSize, bool ExitOnError) { + long E = Epoch ? *Epoch : 0; + Vector Files; +- ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); ++ int Res = ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); ++ if (ExitOnError && Res != 0) ++ exit(Res); + size_t NumLoaded = 0; + for (size_t i = 0; i < Files.size(); i++) { + auto &X = Files[i]; +@@ -97,12 +99,15 @@ void ReadDirToVectorOfUnits(const char *Path, Vector *V, + } + + +-void GetSizedFilesFromDir(const std::string &Dir, Vector *V) { ++int GetSizedFilesFromDir(const std::string &Dir, Vector *V) { + Vector Files; +- ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); ++ int Res = ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); ++ if (Res != 0) ++ return Res; + for (auto &File : Files) + if (size_t Size = FileSize(File)) + V->push_back({File, Size}); ++ return 0; + } + + std::string DirPlusFile(const std::string &DirPath, +diff --git a/tools/fuzzing/libfuzzer/FuzzerIO.h b/tools/fuzzing/libfuzzer/FuzzerIO.h +index 6e4368b971fa..6c90ba637322 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerIO.h ++++ b/tools/fuzzing/libfuzzer/FuzzerIO.h +@@ -60,7 +60,7 @@ void RawPrint(const char *Str); + bool IsFile(const std::string &Path); + size_t FileSize(const std::string &Path); + +-void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, ++int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector *V, bool TopDir); + + void RmDirRecursive(const std::string &Dir); +@@ -79,7 +79,7 @@ struct SizedFile { + bool operator<(const SizedFile &B) const { return Size < B.Size; } + }; + +-void GetSizedFilesFromDir(const std::string &Dir, Vector *V); ++int GetSizedFilesFromDir(const std::string &Dir, Vector *V); + + char GetSeparator(); + // Similar to the basename utility: returns the file name w/o the dir prefix. +diff --git a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp +index 4b453d286c80..1a50295c010f 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp +@@ -53,16 +53,16 @@ std::string Basename(const std::string &Path) { + return Path.substr(Pos + 1); + } + +-void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, ++int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector *V, bool TopDir) { + auto E = GetEpoch(Dir); + if (Epoch) +- if (E && *Epoch >= E) return; ++ if (E && *Epoch >= E) return 0; + + DIR *D = opendir(Dir.c_str()); + if (!D) { + Printf("%s: %s; exiting\n", strerror(errno), Dir.c_str()); +- exit(1); ++ return 1; + } + while (auto E = readdir(D)) { + std::string Path = DirPlusFile(Dir, E->d_name); +@@ -71,12 +71,16 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + V->push_back(Path); + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && +- *E->d_name != '.') +- ListFilesInDirRecursive(Path, Epoch, V, false); ++ *E->d_name != '.') { ++ int Res = ListFilesInDirRecursive(Path, Epoch, V, false); ++ if (Res != 0) ++ return Res; ++ } + } + closedir(D); + if (Epoch && TopDir) + *Epoch = E; ++ return 0; + } + + +diff --git a/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp b/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp +index 651283a551cf..0e977bd02557 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp +@@ -98,11 +98,12 @@ size_t FileSize(const std::string &Path) { + return size.QuadPart; + } + +-void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, ++int ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + Vector *V, bool TopDir) { ++ int Res; + auto E = GetEpoch(Dir); + if (Epoch) +- if (E && *Epoch >= E) return; ++ if (E && *Epoch >= E) return 0; + + std::string Path(Dir); + assert(!Path.empty()); +@@ -116,9 +117,9 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + if (FindHandle == INVALID_HANDLE_VALUE) + { + if (GetLastError() == ERROR_FILE_NOT_FOUND) +- return; ++ return 0; + Printf("No such file or directory: %s; exiting\n", Dir.c_str()); +- exit(1); ++ return 1; + } + + do { +@@ -131,7 +132,9 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + FindInfo.cFileName[1] == '.')) + continue; + +- ListFilesInDirRecursive(FileName, Epoch, V, false); ++ int Res = ListFilesInDirRecursive(FileName, Epoch, V, false); ++ if (Res != 0) ++ return Res; + } + else if (IsFile(FileName, FindInfo.dwFileAttributes)) + V->push_back(FileName); +@@ -145,6 +148,7 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, + + if (Epoch && TopDir) + *Epoch = E; ++ return 0; + } + + +diff --git a/tools/fuzzing/libfuzzer/FuzzerInternal.h b/tools/fuzzing/libfuzzer/FuzzerInternal.h +index 1f7d671ed848..cc2650b58ef1 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerInternal.h ++++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h +@@ -35,8 +35,8 @@ public: + Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, + FuzzingOptions Options); + ~Fuzzer(); +- void Loop(Vector &CorporaFiles); +- void ReadAndExecuteSeedCorpora(Vector &CorporaFiles); ++ int Loop(Vector &CorporaFiles); ++ int ReadAndExecuteSeedCorpora(Vector &CorporaFiles); + void MinimizeCrashLoop(const Unit &U); + void RereadOutputCorpus(size_t MaxSize); + +@@ -65,13 +65,16 @@ public: + static void StaticFileSizeExceedCallback(); + static void StaticGracefulExitCallback(); + ++ static void GracefullyExit(); ++ static bool isGracefulExitRequested(); ++ + int ExecuteCallback(const uint8_t *Data, size_t Size); + bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, + InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); + + // Merge Corpora[1:] into Corpora[0]. + void Merge(const Vector &Corpora); +- void CrashResistantMergeInternalStep(const std::string &ControlFilePath); ++ int CrashResistantMergeInternalStep(const std::string &ControlFilePath); + MutationDispatcher &GetMD() { return MD; } + void PrintFinalStats(); + void SetMaxInputLen(size_t MaxInputLen); +@@ -84,7 +87,7 @@ public: + bool DuringInitialCorpusExecution); + + void HandleMalloc(size_t Size); +- static void MaybeExitGracefully(); ++ static bool MaybeExitGracefully(); + std::string WriteToOutputCorpus(const Unit &U); + + private: +@@ -93,7 +96,7 @@ private: + void ExitCallback(); + void CrashOnOverwrittenData(); + void InterruptCallback(); +- void MutateAndTestOne(); ++ bool MutateAndTestOne(); + void PurgeAllocator(); + void ReportNewCoverage(InputInfo *II, const Unit &U); + void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size); +diff --git a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +index 4c4e8c271b1f..e7dfc187dbfe 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +@@ -254,12 +254,20 @@ void Fuzzer::ExitCallback() { + _Exit(Options.ErrorExitCode); + } + +-void Fuzzer::MaybeExitGracefully() { +- if (!F->GracefulExitRequested) return; ++bool Fuzzer::MaybeExitGracefully() { ++ if (!F->GracefulExitRequested) return false; + Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid()); + RmDirRecursive(TempPath("FuzzWithFork", ".dir")); + F->PrintFinalStats(); +- _Exit(0); ++ return true; ++} ++ ++void Fuzzer::GracefullyExit() { ++ F->GracefulExitRequested = true; ++} ++ ++bool Fuzzer::isGracefulExitRequested() { ++ return F->GracefulExitRequested; + } + + void Fuzzer::InterruptCallback() { +@@ -663,7 +671,7 @@ void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size, + } + } + +-void Fuzzer::MutateAndTestOne() { ++bool Fuzzer::MutateAndTestOne() { + MD.StartMutationSequence(); + + auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); +@@ -685,7 +693,7 @@ void Fuzzer::MutateAndTestOne() { + for (int i = 0; i < Options.MutateDepth; i++) { + if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) + break; +- MaybeExitGracefully(); ++ if (MaybeExitGracefully()) return true; + size_t NewSize = 0; + if (II.HasFocusFunction && !II.DataFlowTraceForFocusFunction.empty() && + Size <= CurrentMaxMutationLen) +@@ -719,6 +727,7 @@ void Fuzzer::MutateAndTestOne() { + } + + II.NeedsEnergyUpdate = true; ++ return false; + } + + void Fuzzer::PurgeAllocator() { +@@ -736,7 +745,7 @@ void Fuzzer::PurgeAllocator() { + LastAllocatorPurgeAttemptTime = system_clock::now(); + } + +-void Fuzzer::ReadAndExecuteSeedCorpora(Vector &CorporaFiles) { ++int Fuzzer::ReadAndExecuteSeedCorpora(Vector &CorporaFiles) { + const size_t kMaxSaneLen = 1 << 20; + const size_t kMinDefaultLen = 4096; + size_t MaxSize = 0; +@@ -795,16 +804,23 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector &CorporaFiles) { + if (Corpus.empty() && Options.MaxNumberOfRuns) { + Printf("ERROR: no interesting inputs were found. " + "Is the code instrumented for coverage? Exiting.\n"); +- exit(1); ++ return 1; + } ++ return 0; + } + +-void Fuzzer::Loop(Vector &CorporaFiles) { ++int Fuzzer::Loop(Vector &CorporaFiles) { + auto FocusFunctionOrAuto = Options.FocusFunction; +- DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, ++ int Res = DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, + MD.GetRand()); +- TPC.SetFocusFunction(FocusFunctionOrAuto); +- ReadAndExecuteSeedCorpora(CorporaFiles); ++ if (Res != 0) ++ return Res; ++ Res = TPC.SetFocusFunction(FocusFunctionOrAuto); ++ if (Res != 0) ++ return Res; ++ Res = ReadAndExecuteSeedCorpora(CorporaFiles); ++ if (Res != 0) ++ return Res; + DFT.Clear(); // No need for DFT any more. + TPC.SetPrintNewPCs(Options.PrintNewCovPcs); + TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs); +@@ -842,13 +858,15 @@ void Fuzzer::Loop(Vector &CorporaFiles) { + } + + // Perform several mutations and runs. +- MutateAndTestOne(); ++ if (MutateAndTestOne()) ++ return 0; + + PurgeAllocator(); + } + + PrintStats("DONE ", "\n"); + MD.PrintRecommendedDictionary(); ++ return 0; + } + + void Fuzzer::MinimizeCrashLoop(const Unit &U) { +diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp +index 919eea848580..0a185c7325bb 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp +@@ -28,11 +28,12 @@ bool Merger::Parse(const std::string &Str, bool ParseCoverage) { + return Parse(SS, ParseCoverage); + } + +-void Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { ++int Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) { + if (!Parse(IS, ParseCoverage)) { + Printf("MERGE: failed to parse the control file (unexpected error)\n"); +- exit(1); ++ return 1; + } ++ return 0; + } + + // The control file example: +@@ -194,11 +195,13 @@ Set Merger::AllFeatures() const { + } + + // Inner process. May crash if the target crashes. +-void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { ++int Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { + Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); + Merger M; + std::ifstream IF(CFPath); +- M.ParseOrExit(IF, false); ++ int Res = M.ParseOrExit(IF, false); ++ if (Res != 0) ++ return Res; + IF.close(); + if (!M.LastFailure.empty()) + Printf("MERGE-INNER: '%s' caused a failure at the previous merge step\n", +@@ -216,7 +219,8 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { + }; + Set AllPCs; + for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { +- Fuzzer::MaybeExitGracefully(); ++ if (Fuzzer::MaybeExitGracefully()) ++ return 0; + auto U = FileToVector(M.Files[i].Name); + if (U.size() > MaxInputLen) { + U.resize(MaxInputLen); +@@ -261,12 +265,14 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { + OF.flush(); + } + PrintStatsWrapper("DONE "); ++ return 0; + } + +-static size_t WriteNewControlFile(const std::string &CFPath, ++static int WriteNewControlFile(const std::string &CFPath, + const Vector &OldCorpus, + const Vector &NewCorpus, +- const Vector &KnownFiles) { ++ const Vector &KnownFiles, ++ size_t &NumFiles) { + std::unordered_set FilesToSkip; + for (auto &SF: KnownFiles) + FilesToSkip.insert(SF.Name); +@@ -292,14 +298,15 @@ static size_t WriteNewControlFile(const std::string &CFPath, + if (!ControlFile) { + Printf("MERGE-OUTER: failed to write to the control file: %s\n", + CFPath.c_str()); +- exit(1); ++ return 1; + } + +- return FilesToUse.size(); ++ NumFiles = FilesToUse.size(); ++ return 0; + } + + // Outer process. Does not call the target code and thus should not fail. +-void CrashResistantMerge(const Vector &Args, ++int CrashResistantMerge(const Vector &Args, + const Vector &OldCorpus, + const Vector &NewCorpus, + Vector *NewFiles, +@@ -309,8 +316,9 @@ void CrashResistantMerge(const Vector &Args, + Set *NewCov, + const std::string &CFPath, + bool V /*Verbose*/) { +- if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. ++ if (NewCorpus.empty() && OldCorpus.empty()) return 0; // Nothing to merge. + size_t NumAttempts = 0; ++ int Res; + Vector KnownFiles; + if (FileSize(CFPath)) { + VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", +@@ -331,7 +339,8 @@ void CrashResistantMerge(const Vector &Args, + VPrintf( + V, + "MERGE-OUTER: nothing to do, merge has been completed before\n"); +- exit(0); ++ Fuzzer::GracefullyExit(); ++ return 0; + } + + // Number of input files likely changed, start merge from scratch, but +@@ -356,7 +365,9 @@ void CrashResistantMerge(const Vector &Args, + "%zd files, %zd in the initial corpus, %zd processed earlier\n", + OldCorpus.size() + NewCorpus.size(), OldCorpus.size(), + KnownFiles.size()); +- NumAttempts = WriteNewControlFile(CFPath, OldCorpus, NewCorpus, KnownFiles); ++ Res = WriteNewControlFile(CFPath, OldCorpus, NewCorpus, KnownFiles, NumAttempts); ++ if (Res != 0) ++ return Res; + } + + // Execute the inner process until it passes. +@@ -366,7 +377,8 @@ void CrashResistantMerge(const Vector &Args, + BaseCmd.removeFlag("fork"); + BaseCmd.removeFlag("collect_data_flow"); + for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { +- Fuzzer::MaybeExitGracefully(); ++ if (Fuzzer::MaybeExitGracefully()) ++ return 0; + VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); + Command Cmd(BaseCmd); + Cmd.addFlag("merge_control_file", CFPath); +@@ -388,7 +400,9 @@ void CrashResistantMerge(const Vector &Args, + VPrintf(V, "MERGE-OUTER: the control file has %zd bytes\n", + (size_t)IF.tellg()); + IF.seekg(0, IF.beg); +- M.ParseOrExit(IF, true); ++ Res = M.ParseOrExit(IF, true); ++ if (Res != 0) ++ return Res; + IF.close(); + VPrintf(V, + "MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", +@@ -399,6 +413,7 @@ void CrashResistantMerge(const Vector &Args, + VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " + "%zd new coverage edges\n", + NewFiles->size(), NewFeatures->size(), NewCov->size()); ++ return 0; + } + + } // namespace fuzzer +diff --git a/tools/fuzzing/libfuzzer/FuzzerMerge.h b/tools/fuzzing/libfuzzer/FuzzerMerge.h +index e0c6bc539bdb..6dc1c4c45abf 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerMerge.h ++++ b/tools/fuzzing/libfuzzer/FuzzerMerge.h +@@ -63,7 +63,7 @@ struct Merger { + + bool Parse(std::istream &IS, bool ParseCoverage); + bool Parse(const std::string &Str, bool ParseCoverage); +- void ParseOrExit(std::istream &IS, bool ParseCoverage); ++ int ParseOrExit(std::istream &IS, bool ParseCoverage); + size_t Merge(const Set &InitialFeatures, Set *NewFeatures, + const Set &InitialCov, Set *NewCov, + Vector *NewFiles); +@@ -71,7 +71,7 @@ struct Merger { + Set AllFeatures() const; + }; + +-void CrashResistantMerge(const Vector &Args, ++int CrashResistantMerge(const Vector &Args, + const Vector &OldCorpus, + const Vector &NewCorpus, + Vector *NewFiles, +diff --git a/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp b/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp +index b2ca7693e540..fbceda39bc22 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp ++++ b/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp +@@ -238,13 +238,13 @@ void TracePC::IterateCoveredFunctions(CallBack CB) { + } + } + +-void TracePC::SetFocusFunction(const std::string &FuncName) { ++int TracePC::SetFocusFunction(const std::string &FuncName) { + // This function should be called once. + assert(!FocusFunctionCounterPtr); + // "auto" is not a valid function name. If this function is called with "auto" + // that means the auto focus functionality failed. + if (FuncName.empty() || FuncName == "auto") +- return; ++ return 0; + for (size_t M = 0; M < NumModules; M++) { + auto &PCTE = ModulePCTable[M]; + size_t N = PCTE.Stop - PCTE.Start; +@@ -256,13 +256,13 @@ void TracePC::SetFocusFunction(const std::string &FuncName) { + if (FuncName != Name) continue; + Printf("INFO: Focus function is set to '%s'\n", Name.c_str()); + FocusFunctionCounterPtr = Modules[M].Start() + I; +- return; ++ return 0; + } + } + + Printf("ERROR: Failed to set focus function. Make sure the function name is " + "valid (%s) and symbolization is enabled.\n", FuncName.c_str()); +- exit(1); ++ return 1; + } + + bool TracePC::ObservedFocusFunction() { +diff --git a/tools/fuzzing/libfuzzer/FuzzerTracePC.h b/tools/fuzzing/libfuzzer/FuzzerTracePC.h +index 501f3b544971..b46ebb909dbf 100644 +--- a/tools/fuzzing/libfuzzer/FuzzerTracePC.h ++++ b/tools/fuzzing/libfuzzer/FuzzerTracePC.h +@@ -116,7 +116,7 @@ class TracePC { + CB(PC); + } + +- void SetFocusFunction(const std::string &FuncName); ++ int SetFocusFunction(const std::string &FuncName); + bool ObservedFocusFunction(); + + struct PCTableEntry { diff --git a/tools/fuzzing/messagemanager/MessageManagerFuzzer.cpp b/tools/fuzzing/messagemanager/MessageManagerFuzzer.cpp new file mode 100644 index 0000000000..6a4475641f --- /dev/null +++ b/tools/fuzzing/messagemanager/MessageManagerFuzzer.cpp @@ -0,0 +1,327 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include +#include +#include "FuzzingTraits.h" +#include "jsapi.h" +#include "jsfriendapi.h" +#include "js/CharacterEncoding.h" +#include "js/Exception.h" +#include "js/PropertyAndElement.h" // JS_Enumerate, JS_GetProperty, JS_GetPropertyById, JS_SetProperty, JS_SetPropertyById +#include "prenv.h" +#include "MessageManagerFuzzer.h" +#include "mozilla/ErrorResult.h" +#include "nsComponentManagerUtils.h" +#include "nsDebug.h" +#include "nsError.h" +#include "nsFrameMessageManager.h" +#include "nsJSUtils.h" +#include "nsXULAppAPI.h" +#include "nsNetCID.h" +#include "nsString.h" +#include "nsUnicharUtils.h" +#include "nsIFile.h" +#include "nsIFileStreams.h" +#include "nsILineInputStream.h" +#include "nsLocalFile.h" +#include "nsTArray.h" + +#ifdef IsLoggingEnabled +// This is defined in the Windows SDK urlmon.h +# undef IsLoggingEnabled +#endif + +#define MESSAGEMANAGER_FUZZER_DEFAULT_MUTATION_PROBABILITY 2 +#define MSGMGR_FUZZER_LOG(fmt, args...) \ + if (MessageManagerFuzzer::IsLoggingEnabled()) { \ + printf_stderr("[MessageManagerFuzzer] " fmt "\n", ##args); \ + } + +namespace mozilla { +namespace dom { + +using namespace fuzzing; +using namespace ipc; + +/* static */ +void MessageManagerFuzzer::ReadFile(const char* path, + nsTArray& aArray) { + nsCOMPtr file; + nsresult rv = + NS_NewLocalFile(NS_ConvertUTF8toUTF16(path), true, getter_AddRefs(file)); + NS_ENSURE_SUCCESS_VOID(rv); + + bool exists = false; + rv = file->Exists(&exists); + if (NS_FAILED(rv) || !exists) { + return; + } + + nsCOMPtr fileStream( + do_CreateInstance(NS_LOCALFILEINPUTSTREAM_CONTRACTID, &rv)); + NS_ENSURE_SUCCESS_VOID(rv); + + rv = fileStream->Init(file, -1, -1, false); + NS_ENSURE_SUCCESS_VOID(rv); + + nsCOMPtr lineStream(do_QueryInterface(fileStream, &rv)); + NS_ENSURE_SUCCESS_VOID(rv); + + nsAutoCString line; + bool more = true; + do { + rv = lineStream->ReadLine(line, &more); + NS_ENSURE_SUCCESS_VOID(rv); + aArray.AppendElement(line); + } while (more); +} + +/* static */ +bool MessageManagerFuzzer::IsMessageNameBlacklisted( + const nsAString& aMessageName) { + static bool sFileLoaded = false; + static nsTArray valuesInFile; + + if (!sFileLoaded) { + ReadFile(PR_GetEnv("MESSAGEMANAGER_FUZZER_BLACKLIST"), valuesInFile); + sFileLoaded = true; + } + + if (valuesInFile.Length() == 0) { + return false; + } + + return valuesInFile.Contains(NS_ConvertUTF16toUTF8(aMessageName).get()); +} + +/* static */ +nsCString MessageManagerFuzzer::GetFuzzValueFromFile() { + static bool sFileLoaded = false; + static nsTArray valuesInFile; + + if (!sFileLoaded) { + ReadFile(PR_GetEnv("MESSAGEMANAGER_FUZZER_STRINGSFILE"), valuesInFile); + sFileLoaded = true; + } + + // If something goes wrong with importing the file we return an empty string. + if (valuesInFile.Length() == 0) { + return nsCString(); + } + + unsigned randIdx = RandomIntegerRange(0, valuesInFile.Length()); + return valuesInFile.ElementAt(randIdx); +} + +/* static */ +void MessageManagerFuzzer::MutateObject(JSContext* aCx, + JS::Handle aValue, + unsigned short int aRecursionCounter) { + JS::Rooted object(aCx, &aValue.toObject()); + JS::Rooted ids(aCx, JS::IdVector(aCx)); + + if (!JS_Enumerate(aCx, object, &ids)) { + return; + } + + for (size_t i = 0, n = ids.length(); i < n; i++) { + // Retrieve Property name. + nsAutoJSString propName; + if (!propName.init(aCx, ids[i])) { + continue; + } + MSGMGR_FUZZER_LOG("%*s- Property: %s", aRecursionCounter * 4, "", + NS_ConvertUTF16toUTF8(propName).get()); + + // The likelihood when a value gets fuzzed of this object. + if (!FuzzingTraits::Sometimes(DefaultMutationProbability())) { + continue; + } + + // Retrieve Property value. + JS::Rooted propertyValue(aCx); + JS_GetPropertyById(aCx, object, ids[i], &propertyValue); + + JS::Rooted newPropValue(aCx); + MutateValue(aCx, propertyValue, &newPropValue, aRecursionCounter); + + JS_SetPropertyById(aCx, object, ids[i], newPropValue); + } +} + +/* static */ +bool MessageManagerFuzzer::MutateValue( + JSContext* aCx, JS::Handle aValue, + JS::MutableHandle aOutMutationValue, + unsigned short int aRecursionCounter) { + if (aValue.isInt32()) { + if (FuzzingTraits::Sometimes(DefaultMutationProbability() * 2)) { + aOutMutationValue.set(JS::Int32Value(RandomNumericLimit())); + } else { + aOutMutationValue.set(JS::Int32Value(RandomInteger())); + } + MSGMGR_FUZZER_LOG("%*s! Mutated value of type |int32|: '%d' to '%d'", + aRecursionCounter * 4, "", aValue.toInt32(), + aOutMutationValue.toInt32()); + return true; + } + + if (aValue.isDouble()) { + aOutMutationValue.set(JS::DoubleValue(RandomFloatingPoint())); + MSGMGR_FUZZER_LOG("%*s! Mutated value of type |double|: '%f' to '%f'", + aRecursionCounter * 4, "", aValue.toDouble(), + aOutMutationValue.toDouble()); + return true; + } + + if (aValue.isBoolean()) { + aOutMutationValue.set(JS::BooleanValue(bool(RandomIntegerRange(0, 2)))); + MSGMGR_FUZZER_LOG("%*s! Mutated value of type |boolean|: '%d' to '%d'", + aRecursionCounter * 4, "", aValue.toBoolean(), + aOutMutationValue.toBoolean()); + return true; + } + + if (aValue.isString()) { + nsCString x = GetFuzzValueFromFile(); + if (x.IsEmpty()) { + return false; + } + JSString* str = JS_NewStringCopyZ(aCx, x.get()); + aOutMutationValue.set(JS::StringValue(str)); + JS::Rooted rootedValue(aCx, aValue.toString()); + JS::UniqueChars valueChars = JS_EncodeStringToUTF8(aCx, rootedValue); + MSGMGR_FUZZER_LOG("%*s! Mutated value of type |string|: '%s' to '%s'", + aRecursionCounter * 4, "", valueChars.get(), x.get()); + return true; + } + + if (aValue.isObject()) { + aRecursionCounter++; + MSGMGR_FUZZER_LOG("%*s", aRecursionCounter * 4, + ""); + MutateObject(aCx, aValue, aRecursionCounter); + aOutMutationValue.set(aValue); + return true; + } + + return false; +} + +/* static */ +bool MessageManagerFuzzer::Mutate(JSContext* aCx, const nsAString& aMessageName, + ipc::StructuredCloneData* aData, + const JS::Value& aTransfer) { + MSGMGR_FUZZER_LOG("Message: %s in process: %d", + NS_ConvertUTF16toUTF8(aMessageName).get(), + XRE_GetProcessType()); + + unsigned short int aRecursionCounter = 0; + ErrorResult rv; + JS::Rooted t(aCx, aTransfer); + + /* Read original StructuredCloneData. */ + JS::Rooted scdContent(aCx); + aData->Read(aCx, &scdContent, rv); + if (NS_WARN_IF(rv.Failed())) { + rv.SuppressException(); + JS_ClearPendingException(aCx); + return false; + } + + JS::Rooted scdMutationContent(aCx); + bool isMutated = + MutateValue(aCx, scdContent, &scdMutationContent, aRecursionCounter); + + /* Write mutated StructuredCloneData. */ + ipc::StructuredCloneData mutatedStructuredCloneData; + mutatedStructuredCloneData.Write(aCx, scdMutationContent, t, + JS::CloneDataPolicy(), rv); + if (NS_WARN_IF(rv.Failed())) { + rv.SuppressException(); + JS_ClearPendingException(aCx); + return false; + } + + // See: https://bugzilla.mozilla.org/show_bug.cgi?id=1346040 + aData->Copy(mutatedStructuredCloneData); + + /* Mutated and successfully written to StructuredCloneData object. */ + if (isMutated) { + JS::Rooted str(aCx, JS_ValueToSource(aCx, scdMutationContent)); + JS::UniqueChars strChars = JS_EncodeStringToUTF8(aCx, str); + MSGMGR_FUZZER_LOG("Mutated '%s' Message: %s", + NS_ConvertUTF16toUTF8(aMessageName).get(), + strChars.get()); + } + + return true; +} + +/* static */ +unsigned int MessageManagerFuzzer::DefaultMutationProbability() { + static unsigned long sPropValue = + MESSAGEMANAGER_FUZZER_DEFAULT_MUTATION_PROBABILITY; + static bool sInitialized = false; + + if (sInitialized) { + return sPropValue; + } + sInitialized = true; + + // Defines the likelihood of fuzzing a message. + const char* probability = + PR_GetEnv("MESSAGEMANAGER_FUZZER_MUTATION_PROBABILITY"); + if (probability) { + long n = std::strtol(probability, nullptr, 10); + if (n != 0) { + sPropValue = n; + return sPropValue; + } + } + + return sPropValue; +} + +/* static */ +bool MessageManagerFuzzer::IsLoggingEnabled() { + static bool sInitialized = false; + static bool sIsLoggingEnabled = false; + + if (!sInitialized) { + sIsLoggingEnabled = !!PR_GetEnv("MESSAGEMANAGER_FUZZER_ENABLE_LOGGING"); + sInitialized = true; + } + + return sIsLoggingEnabled; +} + +/* static */ +bool MessageManagerFuzzer::IsEnabled() { + return !!PR_GetEnv("MESSAGEMANAGER_FUZZER_ENABLE") && XRE_IsContentProcess(); +} + +/* static */ +void MessageManagerFuzzer::TryMutate(JSContext* aCx, + const nsAString& aMessageName, + ipc::StructuredCloneData* aData, + const JS::Value& aTransfer) { + if (!IsEnabled()) { + return; + } + + if (IsMessageNameBlacklisted(aMessageName)) { + MSGMGR_FUZZER_LOG("Blacklisted message: %s", + NS_ConvertUTF16toUTF8(aMessageName).get()); + return; + } + + Mutate(aCx, aMessageName, aData, aTransfer); +} + +} // namespace dom +} // namespace mozilla diff --git a/tools/fuzzing/messagemanager/MessageManagerFuzzer.h b/tools/fuzzing/messagemanager/MessageManagerFuzzer.h new file mode 100644 index 0000000000..fd7835055a --- /dev/null +++ b/tools/fuzzing/messagemanager/MessageManagerFuzzer.h @@ -0,0 +1,63 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_MessageManagerFuzzer_h__ +#define mozilla_dom_MessageManagerFuzzer_h__ + +#include "jspubtd.h" +#include "nsAString.h" +#include "nsTArray.h" + +namespace mozilla { +namespace dom { + +namespace ipc { +class StructuredCloneData; +} + +/* +Exposed environment variables: +MESSAGEMANAGER_FUZZER_ENABLE=1 +MESSAGEMANAGER_FUZZER_ENABLE_LOGGING=1 (optional) +MESSAGEMANAGER_FUZZER_MUTATION_PROBABILITY=2 (optional) +MESSAGEMANAGER_FUZZER_STRINGSFILE= (optional) +MESSAGEMANAGER_FUZZER_BLACKLIST= (optional) +*/ + +#ifdef IsLoggingEnabled +// This is defined in the Windows SDK urlmon.h +# undef IsLoggingEnabled +#endif + +class MessageManagerFuzzer { + public: + static void TryMutate(JSContext* aCx, const nsAString& aMessageName, + ipc::StructuredCloneData* aData, + const JS::Value& aTransfer); + + private: + static void ReadFile(const char* path, nsTArray& aArray); + static nsCString GetFuzzValueFromFile(); + static bool IsMessageNameBlacklisted(const nsAString& aMessageName); + static bool Mutate(JSContext* aCx, const nsAString& aMessageName, + ipc::StructuredCloneData* aData, + const JS::Value& aTransfer); + static void Mutate(JSContext* aCx, JS::Rooted& aMutation); + static void MutateObject(JSContext* aCx, JS::Handle aValue, + unsigned short int aRecursionCounter); + static bool MutateValue(JSContext* aCx, JS::Handle aValue, + JS::MutableHandle aOutMutationValue, + unsigned short int aRecursionCounter); + static unsigned int DefaultMutationProbability(); + static nsAutoString ReadJSON(JSContext* aCx, const JS::Value& aJSON); + static bool IsEnabled(); + static bool IsLoggingEnabled(); +}; + +} // namespace dom +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/messagemanager/moz.build b/tools/fuzzing/messagemanager/moz.build new file mode 100644 index 0000000000..9fc49d7b73 --- /dev/null +++ b/tools/fuzzing/messagemanager/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += ["MessageManagerFuzzer.cpp"] + +EXPORTS += ["MessageManagerFuzzer.h"] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/moz.build b/tools/fuzzing/moz.build new file mode 100644 index 0000000000..37dc85ddd1 --- /dev/null +++ b/tools/fuzzing/moz.build @@ -0,0 +1,33 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + "interface", + "registry", +] + +if not CONFIG["JS_STANDALONE"]: + DIRS += [ + "common", + "messagemanager", + "shmem", + ] + + if CONFIG["FUZZING_SNAPSHOT"]: + DIRS += [ + "ipc", + "nyx", + ] + + if CONFIG["LIBFUZZER"]: + DIRS += [ + "rust", + ] + +if CONFIG["LIBFUZZER"]: + DIRS += [ + "libfuzzer", + ] diff --git a/tools/fuzzing/nyx/Nyx.cpp b/tools/fuzzing/nyx/Nyx.cpp new file mode 100644 index 0000000000..5c48d3342b --- /dev/null +++ b/tools/fuzzing/nyx/Nyx.cpp @@ -0,0 +1,206 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Assertions.h" +#include "mozilla/Unused.h" +#include "mozilla/Vector.h" +#include "mozilla/fuzzing/Nyx.h" +#include "prinrval.h" +#include "prthread.h" + +#include +#include + +#include + +namespace mozilla { +namespace fuzzing { + +Nyx::Nyx() {} + +// static +Nyx& Nyx::instance() { + static Nyx nyx; + return nyx; +} + +extern "C" { +MOZ_EXPORT __attribute__((weak)) void nyx_start(void); +MOZ_EXPORT __attribute__((weak)) uint32_t nyx_get_next_fuzz_data(void*, + uint32_t); +MOZ_EXPORT __attribute__((weak)) void nyx_release(uint32_t); +MOZ_EXPORT __attribute__((weak)) void nyx_handle_event(const char*, const char*, + int, const char*); +MOZ_EXPORT __attribute__((weak)) void nyx_puts(const char*); +} + +/* + * In this macro, we must avoid calling MOZ_CRASH and friends, as these + * calls will redirect to the Nyx event handler routines. If the library + * is not properly preloaded, we will crash in the process. Instead, emit + * a descriptive error and then force a crash that won't be redirected. + */ +#define NYX_CHECK_API(func) \ + if (!func) { \ + fprintf( \ + stderr, \ + "Error: Nyx library must be in LD_PRELOAD. Missing function \"%s\"\n", \ + #func); \ + MOZ_REALLY_CRASH(__LINE__); \ + } + +void Nyx::start(void) { + MOZ_RELEASE_ASSERT(!mInited); + mInited = true; + + // Check if we are in replay mode. + char* testFilePtr = getenv("MOZ_FUZZ_TESTFILE"); + if (testFilePtr) { + mReplayMode = true; + + MOZ_FUZZING_NYX_PRINT("[Replay Mode] Reading data file...\n"); + + std::string testFile(testFilePtr); + std::ifstream is; + is.open(testFile, std::ios::binary); + + uint64_t chksum, num_ops, num_data, op_offset, data_offset; + + // If only C++ supported streaming operators on binary files... + is.read(reinterpret_cast(&chksum), sizeof(uint64_t)); + is.read(reinterpret_cast(&num_ops), sizeof(uint64_t)); + is.read(reinterpret_cast(&num_data), sizeof(uint64_t)); + is.read(reinterpret_cast(&op_offset), sizeof(uint64_t)); + is.read(reinterpret_cast(&data_offset), sizeof(uint64_t)); + + if (!is.good()) { + MOZ_FUZZING_NYX_PRINT("[Replay Mode] Error reading input file.\n"); + _exit(1); + } + + is.seekg(data_offset); + + // The data chunks we receive through Nyx are stored in the data + // section of the testfile as chunks prefixed with a 16-bit data + // length. We read all chunks and store them away to simulate how + // we originally received the data via Nyx. + + while (is.good()) { + uint16_t pktsize; + is.read(reinterpret_cast(&pktsize), sizeof(uint16_t)); + + if (!is.good()) { + break; + } + + auto buffer = new Vector(); + + mozilla::Unused << buffer->initLengthUninitialized(pktsize); + is.read(reinterpret_cast(buffer->begin()), buffer->length()); + + MOZ_FUZZING_NYX_PRINTF("[Replay Mode] Read data packet of size %zu\n", + buffer->length()); + + mReplayBuffers.push_back(buffer); + } + + if (!mReplayBuffers.size()) { + MOZ_FUZZING_NYX_PRINT("[Replay Mode] Error: No buffers read.\n"); + _exit(1); + } + + is.close(); + + if (!!getenv("MOZ_FUZZ_WAIT_BEFORE_REPLAY")) { + // This can be useful in some cases to reproduce intermittent issues. + PR_Sleep(PR_MillisecondsToInterval(5000)); + } + + return; + } + + NYX_CHECK_API(nyx_start); + NYX_CHECK_API(nyx_get_next_fuzz_data); + NYX_CHECK_API(nyx_release); + NYX_CHECK_API(nyx_handle_event); + NYX_CHECK_API(nyx_puts); + + nyx_start(); +} + +bool Nyx::started(void) { return mInited; } + +bool Nyx::is_enabled(const char* identifier) { + static char* fuzzer = getenv("NYX_FUZZER"); + if (!fuzzer || strcmp(fuzzer, identifier)) { + return false; + } + return true; +} + +bool Nyx::is_replay() { return mReplayMode; } + +uint32_t Nyx::get_data(uint8_t* data, uint32_t size) { + MOZ_RELEASE_ASSERT(mInited); + + if (mReplayMode) { + if (!mReplayBuffers.size()) { + return 0xFFFFFFFF; + } + + Vector* buffer = mReplayBuffers.front(); + mReplayBuffers.pop_front(); + + size = std::min(size, (uint32_t)buffer->length()); + memcpy(data, buffer->begin(), size); + + delete buffer; + + return size; + } + + return nyx_get_next_fuzz_data(data, size); +} + +void Nyx::release(uint32_t iterations) { + MOZ_RELEASE_ASSERT(mInited); + + if (mReplayMode) { + MOZ_FUZZING_NYX_PRINT("[Replay Mode] Nyx::release() called.\n"); + + // If we reach this point in replay mode, we are essentially done. + // Let's wait a bit further for things to settle and then exit. + PR_Sleep(PR_MillisecondsToInterval(5000)); + _exit(1); + } + + MOZ_FUZZING_NYX_DEBUG("[DEBUG] Nyx::release() called.\n"); + + nyx_release(iterations); +} + +void Nyx::handle_event(const char* type, const char* file, int line, + const char* reason) { + if (mReplayMode) { + MOZ_FUZZING_NYX_PRINTF( + "[Replay Mode] Nyx::handle_event() called: %s at %s:%d : %s\n", type, + file, line, reason); + return; + } + + if (mInited) { + nyx_handle_event(type, file, line, reason); + } else { + // We can have events such as MOZ_CRASH even before we snapshot. + // Output some useful information to make it clear where it happened. + MOZ_FUZZING_NYX_PRINTF( + "[ERROR] PRE SNAPSHOT Nyx::handle_event() called: %s at %s:%d : %s\n", + type, file, line, reason); + } +} + +} // namespace fuzzing +} // namespace mozilla diff --git a/tools/fuzzing/nyx/Nyx.h b/tools/fuzzing/nyx/Nyx.h new file mode 100644 index 0000000000..e29f5f687a --- /dev/null +++ b/tools/fuzzing/nyx/Nyx.h @@ -0,0 +1,54 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_fuzzing_Nyx_h +#define mozilla_fuzzing_Nyx_h + +#include +#include +#include + +#ifndef NYX_DISALLOW_COPY_AND_ASSIGN +# define NYX_DISALLOW_COPY_AND_ASSIGN(T) \ + T(const T&); \ + void operator=(const T&) +#endif + +namespace mozilla { + +class MallocAllocPolicy; +template +class Vector; + +namespace fuzzing { + +class Nyx { + public: + static Nyx& instance(); + + void start(void); + bool started(void); + bool is_enabled(const char* identifier); + bool is_replay(); + uint32_t get_data(uint8_t* data, uint32_t size); + void release(uint32_t iterations = 1); + void handle_event(const char* type, const char* file, int line, + const char* reason); + + private: + std::atomic mInited; + + std::atomic mReplayMode; + std::list*> mReplayBuffers; + + Nyx(); + NYX_DISALLOW_COPY_AND_ASSIGN(Nyx); +}; + +} // namespace fuzzing +} // namespace mozilla + +#endif /* mozilla_fuzzing_Nyx_h */ diff --git a/tools/fuzzing/nyx/NyxWrapper.h b/tools/fuzzing/nyx/NyxWrapper.h new file mode 100644 index 0000000000..0bb6e0fc46 --- /dev/null +++ b/tools/fuzzing/nyx/NyxWrapper.h @@ -0,0 +1,33 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_fuzzing_NyxWrapper_h +#define mozilla_fuzzing_NyxWrapper_h + +#include "mozilla/Types.h" + +/* + * We need this event handler definition both in C and C++ to differentiate + * the various flavors of controlled aborts (e.g. MOZ_DIAGNOSTIC_ASSERT + * vs. MOZ_RELEASE_ASSERT). Hence we can't use the higher level C++ API + * for this and directly redirect to the Nyx event handler in the preloaded + * library. + */ + +#ifdef __cplusplus +extern "C" { +#endif +MOZ_EXPORT __attribute__((weak)) void nyx_handle_event(const char* type, + const char* file, + int line, + const char* reason); + +MOZ_EXPORT __attribute__((weak)) void nyx_puts(const char*); +#ifdef __cplusplus +} +#endif + +#endif /* mozilla_fuzzing_NyxWrapper_h */ diff --git a/tools/fuzzing/nyx/moz.build b/tools/fuzzing/nyx/moz.build new file mode 100644 index 0000000000..74c8ec8515 --- /dev/null +++ b/tools/fuzzing/nyx/moz.build @@ -0,0 +1,16 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SOURCES += [ + "Nyx.cpp", +] + +EXPORTS.mozilla.fuzzing += [ + "Nyx.h", + "NyxWrapper.h", +] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/registry/FuzzerRegistry.cpp b/tools/fuzzing/registry/FuzzerRegistry.cpp new file mode 100644 index 0000000000..ed4b0c7fa8 --- /dev/null +++ b/tools/fuzzing/registry/FuzzerRegistry.cpp @@ -0,0 +1,26 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzerRegistry.h" + +namespace mozilla { + +FuzzerRegistry& FuzzerRegistry::getInstance() { + static FuzzerRegistry instance; + return instance; +} + +void FuzzerRegistry::registerModule(std::string moduleName, + FuzzerInitFunc initFunc, + FuzzerTestingFunc testingFunc) { + moduleMap.insert(std::pair( + moduleName, FuzzerFunctions(initFunc, testingFunc))); +} + +FuzzerFunctions FuzzerRegistry::getModuleFunctions(std::string& moduleName) { + return moduleMap[moduleName]; +} + +} // namespace mozilla diff --git a/tools/fuzzing/registry/FuzzerRegistry.h b/tools/fuzzing/registry/FuzzerRegistry.h new file mode 100644 index 0000000000..5976ddc5b6 --- /dev/null +++ b/tools/fuzzing/registry/FuzzerRegistry.h @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _FuzzerRegistry_h__ +#define _FuzzerRegistry_h__ + +#include +#include +#include +#include + +#include "mozilla/Attributes.h" +#include "mozilla/Types.h" + +typedef int (*FuzzerInitFunc)(int*, char***); +typedef int (*FuzzerTestingFunc)(const uint8_t*, size_t); + +typedef int (*LibFuzzerDriver)(int*, char***, FuzzerTestingFunc); + +namespace mozilla { + +typedef std::pair FuzzerFunctions; + +class FuzzerRegistry { + public: + MOZ_EXPORT static FuzzerRegistry& getInstance(); + MOZ_EXPORT void registerModule(std::string moduleName, + FuzzerInitFunc initFunc, + FuzzerTestingFunc testingFunc); + MOZ_EXPORT FuzzerFunctions getModuleFunctions(std::string& moduleName); + + FuzzerRegistry(FuzzerRegistry const&) = delete; + void operator=(FuzzerRegistry const&) = delete; + + private: + FuzzerRegistry(){}; + std::map moduleMap; +}; + +} // namespace mozilla + +#endif // _FuzzerRegistry_h__ diff --git a/tools/fuzzing/registry/moz.build b/tools/fuzzing/registry/moz.build new file mode 100644 index 0000000000..4aa005a56e --- /dev/null +++ b/tools/fuzzing/registry/moz.build @@ -0,0 +1,20 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-registry") + +SOURCES += [ + "FuzzerRegistry.cpp", +] + +EXPORTS += [ + "FuzzerRegistry.h", +] + +if CONFIG["JS_STANDALONE"]: + FINAL_LIBRARY = "js" +else: + FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/rust/Cargo.toml b/tools/fuzzing/rust/Cargo.toml new file mode 100644 index 0000000000..de0830e976 --- /dev/null +++ b/tools/fuzzing/rust/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "gecko-fuzz-targets" +version = "0.1.0" +authors = ["fuzzing@mozilla.com"] + +[dependencies] +libc = "0.2" +tempfile = "3" +lazy_static = "1.4.0" +rkv = { version = "0.18", features = ["with-fuzzer-no-link"] } +lmdb-rkv = { version = "0.14", features = ["with-fuzzer-no-link"] } diff --git a/tools/fuzzing/rust/RustFuzzingTargets.cpp b/tools/fuzzing/rust/RustFuzzingTargets.cpp new file mode 100644 index 0000000000..7b4dd6c86c --- /dev/null +++ b/tools/fuzzing/rust/RustFuzzingTargets.cpp @@ -0,0 +1,15 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzingInterface.h" +#include "RustFuzzingTargets.h" + +int FuzzingInitDummy(int* argc, char*** argv) { return 0; } + +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_db_file, RkvDbFile); +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_db_name, RkvDbName); +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_key_write, RkvKeyWrite); +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_val_write, RkvValWrite); +MOZ_FUZZING_INTERFACE_RAW(FuzzingInitDummy, fuzz_rkv_calls, RkvCalls); diff --git a/tools/fuzzing/rust/RustFuzzingTargets.h b/tools/fuzzing/rust/RustFuzzingTargets.h new file mode 100644 index 0000000000..a9c1439598 --- /dev/null +++ b/tools/fuzzing/rust/RustFuzzingTargets.h @@ -0,0 +1,26 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface definitions for fuzzing rust modules + */ + +#ifndef RustFuzzingTargets_h__ +#define RustFuzzingTargets_h__ + +#include +#include + +extern "C" { + +int fuzz_rkv_db_file(const uint8_t* raw_data, size_t size); +int fuzz_rkv_db_name(const uint8_t* raw_data, size_t size); +int fuzz_rkv_key_write(const uint8_t* raw_data, size_t size); +int fuzz_rkv_val_write(const uint8_t* raw_data, size_t size); +int fuzz_rkv_calls(const uint8_t* raw_data, size_t size); + +} // extern "C" + +#endif // RustFuzzingTargets_h__ diff --git a/tools/fuzzing/rust/moz.build b/tools/fuzzing/rust/moz.build new file mode 100644 index 0000000000..aa6aaa29dc --- /dev/null +++ b/tools/fuzzing/rust/moz.build @@ -0,0 +1,16 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library("fuzzer-rust-targets") + +SOURCES += [ + "RustFuzzingTargets.cpp", +] + +# Add libFuzzer configuration directives +include("/tools/fuzzing/libfuzzer-config.mozbuild") + +FINAL_LIBRARY = "xul-gtest" diff --git a/tools/fuzzing/rust/src/lib.rs b/tools/fuzzing/rust/src/lib.rs new file mode 100644 index 0000000000..25c9195fb8 --- /dev/null +++ b/tools/fuzzing/rust/src/lib.rs @@ -0,0 +1,341 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#[macro_use] +extern crate lazy_static; +extern crate libc; +extern crate lmdb; +extern crate rkv; +extern crate tempfile; + +use rkv::backend::{ + BackendEnvironmentBuilder, SafeMode, SafeModeDatabase, SafeModeEnvironment, + SafeModeRoTransaction, SafeModeRwTransaction, +}; +use std::fs; +use std::fs::File; +use std::io::Write; +use std::iter; +use std::path::Path; +use std::sync::Arc; +use std::thread; +use tempfile::Builder; + +type Rkv = rkv::Rkv; +type SingleStore = rkv::SingleStore; + +fn eat_lmdb_err(value: Result) -> Result, rkv::StoreError> { + match value { + Ok(value) => Ok(Some(value)), + Err(rkv::StoreError::LmdbError(_)) => Ok(None), + Err(err) => { + println!("Not a crash, but an error outside LMDB: {}", err); + println!("A refined fuzzing test, or changes to RKV, might be required."); + Err(err) + } + } +} + +fn panic_with_err(err: rkv::StoreError) { + println!("Got error: {}", err); + Result::Err(err).unwrap() +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_db_file(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + + // First 8192 bytes are for the lock file. + if data.len() < 8192 { + return 0; + } + let (lock, db) = data.split_at(8192); + let mut lock_file = File::create("data.lock").unwrap(); + lock_file.write_all(lock).unwrap(); + let mut db_file = File::create("data.mdb").unwrap(); + db_file.write_all(db).unwrap(); + + let env = Rkv::with_capacity::(Path::new("."), 2).unwrap(); + let store = env + .open_single("test", rkv::StoreOptions::create()) + .unwrap(); + + let reader = env.read().unwrap(); + eat_lmdb_err(store.get(&reader, &[0])).unwrap(); + + 0 +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_db_name(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + + let root = Builder::new().prefix("fuzz_rkv_db_name").tempdir().unwrap(); + fs::create_dir_all(root.path()).unwrap(); + + let env = Rkv::new::(root.path()).unwrap(); + let name = String::from_utf8_lossy(data); + println!("Checking string: '{:?}'", name); + // Some strings are invalid database names, and are handled as store errors. + // Ignore those errors, but not others. + let store = eat_lmdb_err(env.open_single(name.as_ref(), rkv::StoreOptions::create())).unwrap(); + + if let Some(store) = store { + let reader = env.read().unwrap(); + eat_lmdb_err(store.get(&reader, &[0])).unwrap(); + }; + + 0 +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_key_write(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + + let root = Builder::new() + .prefix("fuzz_rkv_key_write") + .tempdir() + .unwrap(); + fs::create_dir_all(root.path()).unwrap(); + + let env = Rkv::new::(root.path()).unwrap(); + let store = env + .open_single("test", rkv::StoreOptions::create()) + .unwrap(); + + let mut writer = env.write().unwrap(); + // Some data are invalid values, and are handled as store errors. + // Ignore those errors, but not others. + eat_lmdb_err(store.put(&mut writer, data, &rkv::Value::Str("val"))).unwrap(); + + 0 +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_val_write(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + + let root = Builder::new() + .prefix("fuzz_rkv_val_write") + .tempdir() + .unwrap(); + fs::create_dir_all(root.path()).unwrap(); + + let env = Rkv::new::(root.path()).unwrap(); + let store = env + .open_single("test", rkv::StoreOptions::create()) + .unwrap(); + + let mut writer = env.write().unwrap(); + let string = String::from_utf8_lossy(data); + let value = rkv::Value::Str(&string); + store.put(&mut writer, "key", &value).unwrap(); + + 0 +} + +lazy_static! { + static ref STATIC_DATA: Vec = { + let sizes = vec![4, 16, 128, 512, 1024]; + let mut data = Vec::new(); + + for (i, s) in sizes.into_iter().enumerate() { + let bytes = iter::repeat('a' as u8 + i as u8).take(s).collect(); + data.push(String::from_utf8(bytes).unwrap()); + } + + data + }; +} + +#[no_mangle] +pub extern "C" fn fuzz_rkv_calls(raw_data: *const u8, size: libc::size_t) -> libc::c_int { + let data = unsafe { std::slice::from_raw_parts(raw_data as *const u8, size as usize) }; + let mut fuzz = data.iter().copied(); + + fn maybe_do>(fuzz: &mut I, f: impl FnOnce(&mut I) -> ()) -> Option<()> { + match fuzz.next().map(|byte| byte % 2) { + Some(0) => Some(f(fuzz)), + _ => None, + } + } + + fn maybe_abort>( + fuzz: &mut I, + read: rkv::Reader, + ) -> Result<(), rkv::StoreError> { + match fuzz.next().map(|byte| byte % 2) { + Some(0) => Ok(read.abort()), + _ => Ok(()), + } + } + + fn maybe_commit>( + fuzz: &mut I, + write: rkv::Writer, + ) -> Result<(), rkv::StoreError> { + match fuzz.next().map(|byte| byte % 3) { + Some(0) => write.commit(), + Some(1) => Ok(write.abort()), + _ => Ok(()), + } + } + + fn get_static_data<'a, I: Iterator>(fuzz: &mut I) -> Option<&'a String> { + fuzz.next().map(|byte| { + let data = &*STATIC_DATA; + let n = byte as usize; + data.get(n % data.len()).unwrap() + }) + } + + fn get_fuzz_data + Clone>( + fuzz: &mut I, + max_len: usize, + ) -> Option> { + fuzz.next().map(|byte| { + let n = byte as usize; + fuzz.clone().take((n * n) % max_len).collect() + }) + } + + fn get_any_data + Clone>( + fuzz: &mut I, + max_len: usize, + ) -> Option> { + match fuzz.next().map(|byte| byte % 2) { + Some(0) => get_static_data(fuzz).map(|v| v.clone().into_bytes()), + Some(1) => get_fuzz_data(fuzz, max_len), + _ => None, + } + } + + fn store_put + Clone>(fuzz: &mut I, env: &Rkv, store: &SingleStore) { + let key = match get_any_data(fuzz, 1024) { + Some(key) => key, + None => return, + }; + let value = match get_any_data(fuzz, std::usize::MAX) { + Some(value) => value, + None => return, + }; + + let mut writer = env.write().unwrap(); + let mut full = false; + + match store.put(&mut writer, key, &rkv::Value::Blob(&value)) { + Ok(_) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::BadValSize)) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::MapFull)) => full = true, + Err(err) => panic_with_err(err), + }; + + if full { + writer.abort(); + store_resize(fuzz, env); + } else { + maybe_commit(fuzz, writer).unwrap(); + } + } + + fn store_get + Clone>(fuzz: &mut I, env: &Rkv, store: &SingleStore) { + let key = match get_any_data(fuzz, 1024) { + Some(key) => key, + None => return, + }; + + let mut reader = match env.read() { + Ok(reader) => reader, + Err(rkv::StoreError::LmdbError(lmdb::Error::ReadersFull)) => return, + Err(err) => return panic_with_err(err), + }; + + match store.get(&mut reader, key) { + Ok(_) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::BadValSize)) => {} + Err(err) => panic_with_err(err), + }; + + maybe_abort(fuzz, reader).unwrap(); + } + + fn store_delete + Clone>(fuzz: &mut I, env: &Rkv, store: &SingleStore) { + let key = match get_any_data(fuzz, 1024) { + Some(key) => key, + None => return, + }; + + let mut writer = env.write().unwrap(); + + match store.delete(&mut writer, key) { + Ok(_) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::BadValSize)) => {} + Err(rkv::StoreError::LmdbError(lmdb::Error::NotFound)) => {} + Err(err) => panic_with_err(err), + }; + + maybe_commit(fuzz, writer).unwrap(); + } + + fn store_resize>(fuzz: &mut I, env: &Rkv) { + let n = fuzz.next().unwrap_or(1) as usize; + env.set_map_size(1_048_576 * (n % 100)).unwrap() // 1,048,576 bytes, i.e. 1MiB. + } + + let root = Builder::new().prefix("fuzz_rkv_calls").tempdir().unwrap(); + fs::create_dir_all(root.path()).unwrap(); + + let mut builder: SafeMode = Rkv::environment_builder(); + builder.set_max_dbs(1); // need at least one db + + maybe_do(&mut fuzz, |fuzz| { + let n = fuzz.next().unwrap_or(126) as u32; // default + builder.set_max_readers(1 + n); + }); + maybe_do(&mut fuzz, |fuzz| { + let n = fuzz.next().unwrap_or(0) as u32; + builder.set_max_dbs(1 + n); + }); + maybe_do(&mut fuzz, |fuzz| { + let n = fuzz.next().unwrap_or(1) as usize; + builder.set_map_size(1_048_576 * (n % 100)); // 1,048,576 bytes, i.e. 1MiB. + }); + + let env = Rkv::from_builder(root.path(), builder).unwrap(); + let store = env + .open_single("test", rkv::StoreOptions::create()) + .unwrap(); + + let shared_env = Arc::new(env); + let shared_store = Arc::new(store); + + let use_threads = fuzz.next().map(|byte| byte % 10 == 0).unwrap_or(false); + let max_threads = if use_threads { 16 } else { 1 }; + let num_threads = fuzz.next().unwrap_or(0) as usize % max_threads + 1; + let chunk_size = fuzz.len() / num_threads; + + let threads = (0..num_threads).map(|_| { + let env = shared_env.clone(); + let store = shared_store.clone(); + + let chunk: Vec<_> = fuzz.by_ref().take(chunk_size).collect(); + let mut fuzz = chunk.into_iter(); + + thread::spawn(move || loop { + match fuzz.next().map(|byte| byte % 4) { + Some(0) => store_put(&mut fuzz, &env, &store), + Some(1) => store_get(&mut fuzz, &env, &store), + Some(2) => store_delete(&mut fuzz, &env, &store), + Some(3) => store_resize(&mut fuzz, &env), + _ => break, + } + }) + }); + + for handle in threads { + handle.join().unwrap() + } + + 0 +} diff --git a/tools/fuzzing/shmem/SharedMemoryFuzzer.cpp b/tools/fuzzing/shmem/SharedMemoryFuzzer.cpp new file mode 100644 index 0000000000..49a79fa975 --- /dev/null +++ b/tools/fuzzing/shmem/SharedMemoryFuzzer.cpp @@ -0,0 +1,122 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FuzzingMutate.h" +#include "FuzzingTraits.h" +#include "nsDebug.h" +#include "prenv.h" +#include "SharedMemoryFuzzer.h" + +#define SHMEM_FUZZER_DEFAULT_MUTATION_PROBABILITY 2 +#define SHMEM_FUZZER_DEFAULT_MUTATION_FACTOR 500 +#define SHMEM_FUZZER_LOG(fmt, args...) \ + if (SharedMemoryFuzzer::IsLoggingEnabled()) { \ + printf_stderr("[SharedMemoryFuzzer] " fmt "\n", ##args); \ + } + +namespace mozilla { +namespace ipc { + +using namespace fuzzing; + +/* static */ +bool SharedMemoryFuzzer::IsLoggingEnabled() { + static bool sInitialized = false; + static bool sIsLoggingEnabled = false; + + if (!sInitialized) { + sIsLoggingEnabled = !!PR_GetEnv("SHMEM_FUZZER_ENABLE_LOGGING"); + sInitialized = true; + } + return sIsLoggingEnabled; +} + +/* static */ +bool SharedMemoryFuzzer::IsEnabled() { + static bool sInitialized = false; + static bool sIsFuzzerEnabled = false; + + if (!sInitialized) { + sIsFuzzerEnabled = !!PR_GetEnv("SHMEM_FUZZER_ENABLE"); + } + return sIsFuzzerEnabled; +} + +/* static */ +uint64_t SharedMemoryFuzzer::MutationProbability() { + static uint64_t sPropValue = SHMEM_FUZZER_DEFAULT_MUTATION_PROBABILITY; + static bool sInitialized = false; + + if (sInitialized) { + return sPropValue; + } + sInitialized = true; + + const char* probability = PR_GetEnv("SHMEM_FUZZER_MUTATION_PROBABILITY"); + if (probability) { + long n = std::strtol(probability, nullptr, 10); + if (n != 0) { + sPropValue = n; + return sPropValue; + } + } + return sPropValue; +} + +/* static */ +uint64_t SharedMemoryFuzzer::MutationFactor() { + static uint64_t sPropValue = SHMEM_FUZZER_DEFAULT_MUTATION_FACTOR; + static bool sInitialized = false; + + if (sInitialized) { + return sPropValue; + } + sInitialized = true; + + const char* factor = PR_GetEnv("SHMEM_FUZZER_MUTATION_FACTOR"); + if (factor) { + long n = strtol(factor, nullptr, 10); + if (n != 0) { + sPropValue = n; + return sPropValue; + } + } + return sPropValue; +} + +/* static */ +void* SharedMemoryFuzzer::MutateSharedMemory(void* aMemory, size_t aSize) { + if (!IsEnabled()) { + return aMemory; + } + + if (aSize == 0) { + /* Shmem opened from foreign handle. */ + SHMEM_FUZZER_LOG("shmem is of size 0."); + return aMemory; + } + + if (!aMemory) { + /* Memory space is not mapped. */ + SHMEM_FUZZER_LOG("shmem memory space is not mapped."); + return aMemory; + } + + // The likelihood when a value gets fuzzed of this object. + if (!FuzzingTraits::Sometimes(MutationProbability())) { + return aMemory; + } + + const size_t max = FuzzingTraits::Frequency(aSize, MutationFactor()); + SHMEM_FUZZER_LOG("shmem of size: %zu / mutations: %zu", aSize, max); + for (size_t i = 0; i < max; i++) { + FuzzingMutate::ChangeBit((uint8_t*)aMemory, aSize); + } + return aMemory; +} + +} // namespace ipc +} // namespace mozilla diff --git a/tools/fuzzing/shmem/SharedMemoryFuzzer.h b/tools/fuzzing/shmem/SharedMemoryFuzzer.h new file mode 100644 index 0000000000..bd862edf6a --- /dev/null +++ b/tools/fuzzing/shmem/SharedMemoryFuzzer.h @@ -0,0 +1,38 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_SharedMemoryFuzzer_h +#define mozilla_dom_SharedMemoryFuzzer_h + +#include +#include + +namespace mozilla { +namespace ipc { + +/* + * Exposed environment variables: + * SHMEM_FUZZER_ENABLE=1 + * SHMEM_FUZZER_ENABLE_LOGGING=1 (optional) + * SHMEM_FUZZER_MUTATION_PROBABILITY=2 (optional) + * SHMEM_FUZZER_MUTATION_FACTOR=500 (optional) + */ + +class SharedMemoryFuzzer { + public: + static void* MutateSharedMemory(void* aMemory, size_t aSize); + + private: + static uint64_t MutationProbability(); + static uint64_t MutationFactor(); + static bool IsEnabled(); + static bool IsLoggingEnabled(); +}; + +} // namespace ipc +} // namespace mozilla + +#endif diff --git a/tools/fuzzing/shmem/moz.build b/tools/fuzzing/shmem/moz.build new file mode 100644 index 0000000000..ee9c549920 --- /dev/null +++ b/tools/fuzzing/shmem/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += ["SharedMemoryFuzzer.cpp"] + +EXPORTS.mozilla.ipc += ["SharedMemoryFuzzer.h"] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/smoke/grizzly_requirements.in b/tools/fuzzing/smoke/grizzly_requirements.in new file mode 100644 index 0000000000..96d49442a0 --- /dev/null +++ b/tools/fuzzing/smoke/grizzly_requirements.in @@ -0,0 +1,2 @@ +grizzly-framework==0.16.2 +-r ../../../build/psutil_requirements.txt diff --git a/tools/fuzzing/smoke/grizzly_requirements.txt b/tools/fuzzing/smoke/grizzly_requirements.txt new file mode 100644 index 0000000000..a6687e5426 --- /dev/null +++ b/tools/fuzzing/smoke/grizzly_requirements.txt @@ -0,0 +1,127 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --generate-hashes --output-file=tools/fuzzing/smoke/grizzly_requirements.txt tools/fuzzing/smoke/grizzly_requirements.in +# +certifi==2021.10.8 \ + --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ + --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 + # via requests +charset-normalizer==2.0.8 \ + --hash=sha256:735e240d9a8506778cd7a453d97e817e536bb1fc29f4f6961ce297b9c7a917b0 \ + --hash=sha256:83fcdeb225499d6344c8f7f34684c2981270beacc32ede2e669e94f7fa544405 + # via requests +cssbeautifier==1.14.0 \ + --hash=sha256:20be1f47f20762db32c78124ff44d351ba13894fa8e7cfe34014b672f9f6ecb2 + # via grizzly-framework +editorconfig==0.12.3 \ + --hash=sha256:6b0851425aa875b08b16789ee0eeadbd4ab59666e9ebe728e526314c4a2e52c1 + # via + # cssbeautifier + # jsbeautifier +fasteners==0.16.3 \ + --hash=sha256:8408e52656455977053871990bd25824d85803b9417aa348f10ba29ef0c751f7 + # via + # fuzzmanager + # grizzly-framework +ffpuppet==0.9.2 \ + --hash=sha256:340ec47fddc274c97e0b9d9e56d46885e96a07a45a265fd2fa5b21af7f655947 \ + --hash=sha256:41e93d2f3d8d3230822fd4962a85e66246a70927528dce76d765e0ffd859bf69 + # via grizzly-framework +fuzzmanager==0.4.1 \ + --hash=sha256:2bb17b5a725d8d6f03eb9979a75416a1137b7456c95a581a50c7b542fbf3d174 + # via grizzly-framework +grizzly-framework==0.16.2 \ + --hash=sha256:3270fd705a933c4d197784c6e403389c3edb791d09ce26b8a7d7c98934bbc87b \ + --hash=sha256:b01503d1a3a0f8a3fb65cfbc2397a1b1bd4d524389a24ce8eaf00613d374a91b + # via -r tools/fuzzing/smoke/grizzly_requirements.in +idna==3.3 \ + --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \ + --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d + # via requests +jsbeautifier==1.14.0 \ + --hash=sha256:84fdb008d8af89619269a6aca702288b48f837a99427a0f529aa57ecfb36ee3c + # via + # cssbeautifier + # grizzly-framework +lithium-reducer==0.6.1 \ + --hash=sha256:ea2f77f496fc57bcb4d74209210c2ec84b1b327a7b707f98655f85575b6fcc16 + # via grizzly-framework +prefpicker==1.1.2 \ + --hash=sha256:1404cb0e7c07acca060a09fcc3d9203ae2d8784741b6fe97600a707b9b3ff75e \ + --hash=sha256:ea25b33e92a342a0bf2c38f5588dd4baf9d381f70faf7ae2145ca5cad49a2644 + # via grizzly-framework +psutil==5.9.4 \ + --hash=sha256:149555f59a69b33f056ba1c4eb22bb7bf24332ce631c44a319cec09f876aaeff \ + --hash=sha256:16653106f3b59386ffe10e0bad3bb6299e169d5327d3f187614b1cb8f24cf2e1 \ + --hash=sha256:3d7f9739eb435d4b1338944abe23f49584bde5395f27487d2ee25ad9a8774a62 \ + --hash=sha256:3ff89f9b835100a825b14c2808a106b6fdcc4b15483141482a12c725e7f78549 \ + --hash=sha256:54c0d3d8e0078b7666984e11b12b88af2db11d11249a8ac8920dd5ef68a66e08 \ + --hash=sha256:54d5b184728298f2ca8567bf83c422b706200bcbbfafdc06718264f9393cfeb7 \ + --hash=sha256:6001c809253a29599bc0dfd5179d9f8a5779f9dffea1da0f13c53ee568115e1e \ + --hash=sha256:68908971daf802203f3d37e78d3f8831b6d1014864d7a85937941bb35f09aefe \ + --hash=sha256:6b92c532979bafc2df23ddc785ed116fced1f492ad90a6830cf24f4d1ea27d24 \ + --hash=sha256:852dd5d9f8a47169fe62fd4a971aa07859476c2ba22c2254d4a1baa4e10b95ad \ + --hash=sha256:9120cd39dca5c5e1c54b59a41d205023d436799b1c8c4d3ff71af18535728e94 \ + --hash=sha256:c1ca331af862803a42677c120aff8a814a804e09832f166f226bfd22b56feee8 \ + --hash=sha256:efeae04f9516907be44904cc7ce08defb6b665128992a56957abc9b61dca94b7 \ + --hash=sha256:fd8522436a6ada7b4aad6638662966de0d61d241cb821239b2ae7013d41a43d4 + # via + # -r tools/fuzzing/smoke/../../../build/psutil_requirements.txt + # ffpuppet + # grizzly-framework +pyyaml==6.0 \ + --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \ + --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \ + --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \ + --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \ + --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \ + --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \ + --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \ + --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \ + --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \ + --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \ + --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \ + --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \ + --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \ + --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \ + --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \ + --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \ + --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \ + --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \ + --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \ + --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \ + --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \ + --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \ + --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \ + --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \ + --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \ + --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \ + --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \ + --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \ + --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \ + --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \ + --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \ + --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \ + --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5 + # via prefpicker +requests==2.26.0 \ + --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \ + --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7 + # via fuzzmanager +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via + # cssbeautifier + # fasteners + # fuzzmanager + # jsbeautifier +urllib3==1.26.7 \ + --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece \ + --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844 + # via requests +xvfbwrapper==0.2.9 \ + --hash=sha256:bcf4ae571941b40254faf7a73432dfc119ad21ce688f1fdec533067037ecfc24 + # via ffpuppet diff --git a/tools/fuzzing/smoke/js.py b/tools/fuzzing/smoke/js.py new file mode 100755 index 0000000000..d6ad08eb6a --- /dev/null +++ b/tools/fuzzing/smoke/js.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Hello I am a fake jsshell for testing purpose. +Add more features! +""" +import argparse +import sys + + +def run(): + parser = argparse.ArgumentParser(description="Process some integers.") + parser.add_argument("-e", type=str, default=None) + + parser.add_argument("--fuzzing-safe", action="store_true", default=False) + + args = parser.parse_args() + + if args.e is not None: + if "crash()" in args.e: + sys.exit(1) + + +if __name__ == "__main__": + run() diff --git a/tools/fuzzing/smoke/python.ini b/tools/fuzzing/smoke/python.ini new file mode 100644 index 0000000000..6701224f36 --- /dev/null +++ b/tools/fuzzing/smoke/python.ini @@ -0,0 +1,5 @@ +[DEFAULT] +subsuite = fuzzing + +[test_grizzly.py] +requirements = tools/fuzzing/smoke/grizzly_requirements.txt diff --git a/tools/fuzzing/smoke/smoke.py b/tools/fuzzing/smoke/smoke.py new file mode 100644 index 0000000000..e328be71d6 --- /dev/null +++ b/tools/fuzzing/smoke/smoke.py @@ -0,0 +1,71 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" Smoke test script for Fuzzing + +This script can be used to perform simple calls using `jsshell` +or whatever other tools you may add. + +The call is done via `taskcluster/ci/fuzzing/kind.yml` and +files contained in the `target.jsshell.zip` and `target.fuzztest.tests.tar.gz` +build artifacts are downloaded to run things. + +Everything included in this directory will be added in +`target.fuzztest.tests.tar.gz` at build time, so you can add more scripts and +tools if you need. They will be located in `$MOZ_FETCHES_DIR` and follow the +same directory structure than the source tree. +""" +import os +import os.path +import shlex +import subprocess +import sys +from distutils.spawn import find_executable + + +def run_jsshell(command, label=None): + """Invokes `jsshell` with command. + + This function will use the `JSSHELL` environment variable, + and fallback to a `js` executable if it finds one + """ + shell = os.environ.get("JSSHELL") + if shell is None: + shell = find_executable("js") + if shell is None: + raise FileNotFoundError(shell) + else: + if not os.path.exists(shell) or not os.path.isfile(shell): + raise FileNotFoundError(shell) + + if label is None: + label = command + sys.stdout.write(label) + cmd = [shell] + shlex.split(command) + sys.stdout.flush() + try: + subprocess.check_call(cmd) + finally: + sys.stdout.write("\n") + sys.stdout.flush() + + +def smoke_test(): + # first, let's make sure it catches crashes so we don't have false + # positives. + try: + run_jsshell("-e 'crash();'", "Testing for crash\n") + except subprocess.CalledProcessError: + pass + else: + raise Exception("Could not get the process to crash") + + # now let's proceed with some tests + run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'", "Simple Fuzzing...") + + # add more smoke tests here + + +if __name__ == "__main__": + # if this calls raises an error, the job will turn red in the CI. + smoke_test() diff --git a/tools/fuzzing/smoke/test_grizzly.py b/tools/fuzzing/smoke/test_grizzly.py new file mode 100644 index 0000000000..2c8f406222 --- /dev/null +++ b/tools/fuzzing/smoke/test_grizzly.py @@ -0,0 +1,42 @@ +import os +import os.path +import sys +from subprocess import check_call + +import mozunit +import pytest +from moztest.selftest import fixtures + +MOZ_AUTOMATION = bool(os.getenv("MOZ_AUTOMATION", "0") == "1") + + +def test_grizzly_smoke(): + ffbin = fixtures.binary() + + if MOZ_AUTOMATION: + assert os.path.exists( + ffbin + ), "Missing Firefox build. Build it, or set GECKO_BINARY_PATH" + + elif not os.path.exists(ffbin): + pytest.skip("Missing Firefox build. Build it, or set GECKO_BINARY_PATH") + + check_call( + [ + sys.executable, + "-m", + "grizzly", + ffbin, + "no-op", + "--headless", + "--smoke-test", + "--limit", + "10", + "--relaunch", + "5", + ], + ) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/fuzzing/smoke/tests.py b/tools/fuzzing/smoke/tests.py new file mode 100644 index 0000000000..bc06e2427b --- /dev/null +++ b/tools/fuzzing/smoke/tests.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os +from contextlib import contextmanager + +import pytest +import smoke + +JS = os.path.join(os.path.dirname(__file__), "js.py") + + +@contextmanager +def fake_js(): + os.environ["JSSHELL"] = JS + try: + yield + finally: + del os.environ["JSSHELL"] + + +def test_run_no_jsshell(): + with pytest.raises(FileNotFoundError): + smoke.run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'") + + +def test_run_jsshell_set(): + with fake_js(): + smoke.run_jsshell("--fuzzing-safe -e 'print(\"PASSED\")'") + + +def test_smoke_test(): + with fake_js(): + smoke.smoke_test() diff --git a/tools/github-sync/converter.py b/tools/github-sync/converter.py new file mode 100755 index 0000000000..57db9286c1 --- /dev/null +++ b/tools/github-sync/converter.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import subprocess +import sys + +import hglib +import pygit2 + +DEBUG = False + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def debugprint(*args, **kwargs): + if DEBUG: + eprint(*args, **kwargs) + + +class HgCommit: + def __init__(self, parent1, parent2): + self.parents = [] + if parent1 == NULL_PARENT_REV: + raise Exception( + "Encountered a hg changeset with no parents! We don't handle this...." + ) + self.parents.append(parent1) + if parent2 != NULL_PARENT_REV: + self.parents.append(parent2) + self.touches_sync_code = False + self.children = [] + + def add_child(self, rev): + self.children.append(rev) + + +class GitCommit: + def __init__(self, hg_rev, commit_obj): + self.hg_rev = hg_rev + self.commit_obj = commit_obj + + +def load_git_repository(): + commit_map = dict() + # First, scan the tags for "mozilla-xxx" that keep track of manually synchronized changes + sync_tags = filter( + lambda ref: ref.startswith("refs/tags/mozilla-"), + list(downstream_git_repo.references), + ) + for desc in sync_tags: + commit = downstream_git_repo.lookup_reference(desc).peel() + # cut out the revision hash from the output + hg_rev = desc[18:] + commit_map[hg_rev] = GitCommit(hg_rev, commit) + debugprint("Loaded pre-existing tag hg %s -> git %s" % (hg_rev, commit.oid)) + + # Next, scan the commits for a specific message format + re_commitmsg = re.compile( + r"^\[(ghsync|wrupdater)\] From https://hg.mozilla.org/mozilla-central/rev/([0-9a-fA-F]+)$", + re.MULTILINE, + ) + for commit in downstream_git_repo.walk(downstream_git_repo.head.target): + m = re_commitmsg.search(commit.message) + if not m: + continue + hg_rev = m.group(2) + commit_map[hg_rev] = GitCommit(hg_rev, commit) + debugprint("Loaded pre-existing commit hg %s -> git %s" % (hg_rev, commit.oid)) + return commit_map + + +def timeof(git_commit): + return git_commit.commit_obj.commit_time + git_commit.commit_obj.commit_time_offset + + +def find_newest_commit(commit_map): + newest_hg_rev = None + newest_commit_time = None + + for hg_rev, git_commit in commit_map.items(): + if newest_hg_rev is None or timeof(git_commit) > newest_commit_time: + newest_hg_rev = hg_rev + newest_commit_time = timeof(git_commit) + + return newest_hg_rev + + +def get_single_rev(revset): + output = subprocess.check_output( + ["hg", "log", "-r", revset, "--template", "{node}"] + ) + output = str(output, "ascii") + return output + + +def get_multiple_revs(revset, template): + output = subprocess.check_output( + ["hg", "log", "-r", revset, "--template", template + "\\n"] + ) + for line in output.splitlines(): + yield str(line, "ascii") + + +def get_base_hg_rev(commit_map): + base_hg_rev = find_newest_commit(commit_map) + eprint("Using %s as base hg revision" % base_hg_rev) + return base_hg_rev + + +def load_hg_commits(commits, query): + for cset in get_multiple_revs(query, "{node} {p1node} {p2node}"): + tokens = cset.split() + commits[tokens[0]] = HgCommit(tokens[1], tokens[2]) + return commits + + +def get_real_base_hg_rev(hg_data, commit_map): + # Some of the HG commits we want to port to github may have landed on codelines + # that branched off central prior to base_hg_rev. So when we create the git + # equivalents, they will have parents that are not the HEAD of the git repo, + # but instead will be descendants of older commits in the git repo. In order + # to do this correctly, we need to find the hg-equivalents of all of those + # possible git parents. So first we identify all the "tail" hg revisions in + # our hg_data set (think "tail" as in opposite of "head" which is the tipmost + # commit). The "tail" hg revisions are the ones for which we don't have their + # ancestors in hg_data. + tails = [] + for (rev, cset) in hg_data.items(): + for parent in cset.parents: + if parent not in hg_data: + tails.append(rev) + eprint("Found hg tail revisions %s" % tails) + # Then we find their common ancestor, which will be some ancestor of base_hg_rev + # from which those codelines. + if len(tails) == 0: + common_ancestor = get_single_rev(".") + else: + common_ancestor = get_single_rev("ancestor(" + ",".join(tails) + ")") + eprint("Found common ancestor of tail revisions: %s" % common_ancestor) + + # And then we find the newest git commit whose hg-equivalent is an ancestor of + # that common ancestor, to make sure we are starting from a known hg/git + # commit pair. + for git_commit in sorted(commit_map.values(), key=timeof, reverse=True): + new_base = get_single_rev( + "ancestor(" + common_ancestor + "," + git_commit.hg_rev + ")" + ) + if new_base == common_ancestor: + eprint( + "Pre-existing git commit %s from hg rev %s is descendant of common ancestor; %s" + % ( + git_commit.commit_obj.id, + git_commit.hg_rev, + "walking back further...", + ) + ) + continue + if new_base != git_commit.hg_rev: + eprint( + "Pre-existing git commit %s from hg rev %s is on sibling branch" + " of common ancestor; %s" + % ( + git_commit.commit_obj.id, + git_commit.hg_rev, + "walking back further...", + ) + ) + continue + eprint( + "Pre-existing git commit %s from hg rev %s is sufficiently old; stopping walk" + % (git_commit.commit_obj.id, git_commit.hg_rev) + ) + common_ancestor = new_base + break + + return common_ancestor + + +# Now we prune out all the uninteresting changesets from hg_commits. The +# uninteresting ones are ones that don't touch the target code, are not merges, +# and are not referenced by mozilla tags in the git repo. +# We do this by rewriting the parents to the "interesting" ancestor. +def prune_boring(rev): + while rev in hg_commits: + parent_pruned = False + for i in range(len(hg_commits[rev].parents)): + parent_rev = hg_commits[rev].parents[i] + if parent_rev not in hg_commits: + continue + if hg_commits[parent_rev].touches_sync_code: + continue + if len(hg_commits[parent_rev].parents) > 1: + continue + if parent_rev in hg_to_git_commit_map: + continue + + # If we get here, then `parent_rev` is a boring revision and we can + # prune it. Connect `rev` to its grandparent, and prune the parent + grandparent_rev = hg_commits[parent_rev].parents[0] + hg_commits[rev].parents[i] = grandparent_rev + # eprint("Pruned %s as boring parent of %s, using %s now" % + # (parent_rev, rev, grandparent_rev)) + parent_pruned = True + + if parent_pruned: + # If we pruned a parent, process `rev` again as we might want to + # prune more parents + continue + + # Collapse identical parents, because if the parents are identical + # we don't need to keep multiple copies of them. + hg_commits[rev].parents = list(dict.fromkeys(hg_commits[rev].parents)) + + # If we get here, all of `rev`s parents are interesting, so we can't + # prune them. Move up to the parent rev and start processing that, or + # if we have multiple parents then recurse on those nodes. + if len(hg_commits[rev].parents) == 1: + rev = hg_commits[rev].parents[0] + continue + + for parent_rev in hg_commits[rev].parents: + prune_boring(parent_rev) + return + + +class FakeCommit: + def __init__(self, oid): + self.oid = oid + + +def fake_commit(hg_rev, parent1, parent2): + if parent1 is None: + eprint("ERROR: Trying to build on None") + exit(1) + oid = "githash_%s" % hash(parent1) + eprint("Fake-built %s" % oid) + return FakeCommit(oid) + + +def build_tree(builder, treedata): + for (name, value) in treedata.items(): + if isinstance(value, dict): + subbuilder = downstream_git_repo.TreeBuilder() + build_tree(subbuilder, value) + builder.insert(name, subbuilder.write(), pygit2.GIT_FILEMODE_TREE) + else: + (filemode, contents) = value + blob_oid = downstream_git_repo.create_blob(contents) + builder.insert(name, blob_oid, filemode) + + +def author_to_signature(author): + pieces = author.strip().split("<") + if len(pieces) != 2 or pieces[1][-1] != ">": + # We could probably handle this better + return pygit2.Signature(author, "") + name = pieces[0].strip() + email = pieces[1][:-1].strip() + return pygit2.Signature(name, email) + + +def real_commit(hg_rev, parent1, parent2): + filetree = dict() + manifest = mozilla_hg_repo.manifest(rev=hg_rev) + for (nodeid, permission, executable, symlink, filename) in manifest: + if not filename.startswith(relative_path.encode("utf-8")): + continue + if symlink: + filemode = pygit2.GIT_FILEMODE_LINK + elif executable: + filemode = pygit2.GIT_FILEMODE_BLOB_EXECUTABLE + else: + filemode = pygit2.GIT_FILEMODE_BLOB + filecontent = mozilla_hg_repo.cat([filename], rev=hg_rev) + subtree = filetree + for component in filename.split(b"/")[2:-1]: + subtree = subtree.setdefault(component.decode("latin-1"), dict()) + filename = filename.split(b"/")[-1] + subtree[filename.decode("latin-1")] = (filemode, filecontent) + + builder = downstream_git_repo.TreeBuilder() + build_tree(builder, filetree) + tree_oid = builder.write() + + parent1_obj = downstream_git_repo.get(parent1) + if parent1_obj.tree_id == tree_oid: + eprint("Early-exit; tree matched that of parent git commit %s" % parent1) + return parent1_obj + + if parent2 is not None: + parent2_obj = downstream_git_repo.get(parent2) + if parent2_obj.tree_id == tree_oid: + eprint("Early-exit; tree matched that of parent git commit %s" % parent2) + return parent2_obj + + hg_rev_obj = mozilla_hg_repo.log(revrange=hg_rev, limit=1)[0] + commit_author = hg_rev_obj[4].decode("latin-1") + commit_message = hg_rev_obj[5].decode("latin-1") + commit_message += ( + "\n\n[ghsync] From https://hg.mozilla.org/mozilla-central/rev/%s" % hg_rev + + "\n" + ) + + parents = [parent1] + if parent2 is not None: + parents.append(parent2) + commit_oid = downstream_git_repo.create_commit( + None, + author_to_signature(commit_author), + author_to_signature(commit_author), + commit_message, + tree_oid, + parents, + ) + eprint("Built git commit %s" % commit_oid) + return downstream_git_repo.get(commit_oid) + + +def try_commit(hg_rev, parent1, parent2=None): + if False: + return fake_commit(hg_rev, parent1, parent2) + else: + return real_commit(hg_rev, parent1, parent2) + + +def build_git_commits(rev): + debugprint("build_git_commit(%s)..." % rev) + if rev in hg_to_git_commit_map: + debugprint(" maps to %s" % hg_to_git_commit_map[rev].commit_obj.oid) + return hg_to_git_commit_map[rev].commit_obj.oid + + if rev not in hg_commits: + debugprint(" not in hg_commits") + return None + + if len(hg_commits[rev].parents) == 1: + git_parent = build_git_commits(hg_commits[rev].parents[0]) + if not hg_commits[rev].touches_sync_code: + eprint( + "WARNING: Found rev %s that is non-merge and not related to the target" + % rev + ) + return git_parent + eprint("Building git equivalent for %s on top of %s" % (rev, git_parent)) + commit_obj = try_commit(rev, git_parent) + hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj) + debugprint(" built %s as %s" % (rev, commit_obj.oid)) + return commit_obj.oid + + git_parent_1 = build_git_commits(hg_commits[rev].parents[0]) + git_parent_2 = build_git_commits(hg_commits[rev].parents[1]) + if git_parent_1 is None or git_parent_2 is None or git_parent_1 == git_parent_2: + git_parent = git_parent_1 if git_parent_2 is None else git_parent_2 + if not hg_commits[rev].touches_sync_code: + debugprint( + " %s is merge with no parents or doesn't touch WR, returning %s" + % (rev, git_parent) + ) + return git_parent + + eprint( + "WARNING: Found merge rev %s whose parents have identical target code" + ", but modifies the target" % rev + ) + eprint("Building git equivalent for %s on top of %s" % (rev, git_parent)) + commit_obj = try_commit(rev, git_parent) + hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj) + debugprint(" built %s as %s" % (rev, commit_obj.oid)) + return commit_obj.oid + + # An actual merge + eprint( + "Building git equivalent for %s on top of %s, %s" + % (rev, git_parent_1, git_parent_2) + ) + commit_obj = try_commit(rev, git_parent_1, git_parent_2) + hg_to_git_commit_map[rev] = GitCommit(rev, commit_obj) + debugprint(" built %s as %s" % (rev, commit_obj.oid)) + return commit_obj.oid + + +def pretty_print(rev, cset): + desc = " %s" % rev + desc += " parents: %s" % cset.parents + if rev in hg_to_git_commit_map: + desc += " git: %s" % hg_to_git_commit_map[rev].commit_obj.oid + if rev == hg_tip: + desc += " (tip)" + return desc + + +if len(sys.argv) < 3: + eprint("Usage: %s " % sys.argv[0]) + eprint("Current dir must be the mozilla hg repo") + exit(1) + +local_checkout_path = sys.argv[1] +relative_path = sys.argv[2] +mozilla_hg_path = os.getcwd() +NULL_PARENT_REV = "0000000000000000000000000000000000000000" + +downstream_git_repo = pygit2.Repository(pygit2.discover_repository(local_checkout_path)) +mozilla_hg_repo = hglib.open(mozilla_hg_path) +hg_to_git_commit_map = load_git_repository() +base_hg_rev = get_base_hg_rev(hg_to_git_commit_map) +if base_hg_rev is None: + eprint("Found no sync commits or 'mozilla-xxx' tags") + exit(1) + +hg_commits = load_hg_commits(dict(), "only(.," + base_hg_rev + ")") +eprint("Initial set has %s changesets" % len(hg_commits)) +base_hg_rev = get_real_base_hg_rev(hg_commits, hg_to_git_commit_map) +eprint("Using hg rev %s as common ancestor of all interesting changesets" % base_hg_rev) + +# Refresh hg_commits with our wider dataset +hg_tip = get_single_rev(".") +wider_range = "%s::%s" % (base_hg_rev, hg_tip) +hg_commits = load_hg_commits(hg_commits, wider_range) +eprint("Updated set has %s changesets" % len(hg_commits)) + +if DEBUG: + eprint("Graph of descendants of %s" % base_hg_rev) + output = subprocess.check_output( + [ + "hg", + "log", + "--graph", + "-r", + "descendants(" + base_hg_rev + ")", + "--template", + "{node} {desc|firstline}\\n", + ] + ) + for line in output.splitlines(): + eprint(line.decode("utf-8", "ignore")) + +# Also flag any changes that touch the project +query = "(" + wider_range + ') & file("glob:' + relative_path + '/**")' +for cset in get_multiple_revs(query, "{node}"): + debugprint("Changeset %s modifies %s" % (cset, relative_path)) + hg_commits[cset].touches_sync_code = True +eprint( + "Identified %s changesets that touch the target code" + % sum([1 if v.touches_sync_code else 0 for (k, v) in hg_commits.items()]) +) + +prune_boring(hg_tip) + +# hg_tip itself might be boring +if not hg_commits[hg_tip].touches_sync_code and len(hg_commits[hg_tip].parents) == 1: + new_tip = hg_commits[hg_tip].parents[0] + eprint("Pruned tip %s as boring, using %s now" % (hg_tip, new_tip)) + hg_tip = new_tip + +eprint("--- Interesting changesets ---") +for (rev, cset) in hg_commits.items(): + if cset.touches_sync_code or len(cset.parents) > 1 or rev in hg_to_git_commit_map: + eprint(pretty_print(rev, cset)) +if DEBUG: + eprint("--- Other changesets (not really interesting) ---") + for (rev, cset) in hg_commits.items(): + if not ( + cset.touches_sync_code + or len(cset.parents) > 1 + or rev in hg_to_git_commit_map + ): + eprint(pretty_print(rev, cset)) + +git_tip = build_git_commits(hg_tip) +if git_tip is None: + eprint("No new changesets generated, exiting.") +else: + downstream_git_repo.create_reference("refs/heads/github-sync", git_tip, force=True) + eprint("Updated github-sync branch to %s, done!" % git_tip) diff --git a/tools/github-sync/read-json.py b/tools/github-sync/read-json.py new file mode 100755 index 0000000000..87264d7df4 --- /dev/null +++ b/tools/github-sync/read-json.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import sys + +j = json.load(sys.stdin) +components = sys.argv[1].split("/") + + +def next_match(json_fragment, components): + if len(components) == 0: + yield json_fragment + else: + component = components[0] + if type(json_fragment) == list: + if component == "*": + for item in json_fragment: + yield from next_match(item, components[1:]) + else: + component = int(component) + if component >= len(j): + sys.exit(1) + yield from next_match(json_fragment[component], components[1:]) + elif type(json_fragment) == dict: + if component == "*": + for key in sorted(json_fragment.keys()): + yield from next_match(json_fragment[key], components[1:]) + elif component not in json_fragment: + sys.exit(1) + else: + yield from next_match(json_fragment[component], components[1:]) + + +for match in list(next_match(j, components)): + if type(match) == dict: + print(" ".join(match.keys())) + else: + print(match) diff --git a/tools/github-sync/readme.md b/tools/github-sync/readme.md new file mode 100644 index 0000000000..d691071336 --- /dev/null +++ b/tools/github-sync/readme.md @@ -0,0 +1,106 @@ +# Github synchronization scripts + +This tool aims to help synchronizing changes from mozilla-central to Github on pushes. +This is useful for Gecko sub-projects that have Github mirrors, like `gfx/wr` linking to `https://github.com/servo/webrender`. +Originally, the tools were developed in `https://github.com/staktrace/wrupdater`, +then got moved under `gfx/wr/ci-scripts/wrupdater`, +and finally migrated here while also abstracting away from WebRender specifically. + +The main entry point is the `sync-to-github.sh` script that is called with the following arguments: + 1. name of the project, matching the repository under `https://github.com/moz-gfx` user (e.g. `webrender`) + 2. relative folder in mozilla-central, which is the upstream for the changes (e.g. `gfx/wr`) + 3. downstream repository specified as "organization/project-name" (e.g. `servo/webrender`) + 4. name to call for auto-approving the pull request (e.g. `bors` or `@bors-servo`) + +It creates a staging directory at `~/.ghsync` if one doesn't already exist, +and clones the the downstream repo into it. +The script also requires the `GECKO_PATH` environment variable +to point to a mercurial clone of `mozilla-central`, and access to the +taskcluster secrets service to get a Github API token. + +The `sync-to-github.sh` script does some setup steps but the bulk of the actual work +is done by the `converter.py` script. This script scans the mercurial +repository for new changes to the relative folder in m-c, +and adds commits to the git repository corresponding to those changes. +There are some details in the implementation that make it more robust +than simply exporting patches and attempting to reapply them; +in particular it builds a commit tree structure that mirrors what is found in +the `mozilla-central` repository with respect to branches and merges. +So if conflicting changes land on autoland and inbound, and then get +merged, the git repository commits will have the same structure with +a fork/merge in the commit history. This was discovered to be +necessary after a previous version ran into multiple cases where +the simple patch approach didn't really work. + +One of the actions the `converter.py` takes is to find the last sync point +between Github and mozilla-central. This is done based on the following markers: + - commit message containing the string "[ghsync] From https://hg.mozilla.org/mozilla-central/rev/xxx" + - commit message containing the string "[wrupdater] From https://hg.mozilla.org/mozilla-central/rev/xxx" + - commit with tag "mozilla-xxx" +(where xxx is always a mozilla-central hg revision identifier). + +Once the converter is done converting, the `sync-to-github.sh` script +finishes the process by pushing the new commits to the `github-sync` branch +of the `https://github.com/moz-gfx/` repository, +and generating a pull request against the downstream repository. It also +leaves a comment on the PR that triggers testing and automatic merge of the PR. +If there is already a pull request (perhaps from a previous run) the +pre-existing PR is force-updated instead. This allows for graceful +handling of scenarios where the PR failed to get merged (e.g. due to +CI failures on the Github side). + +The script is intended to by run by taskcluster for any changes that +touch the relative folder that land on `mozilla-central`. This may mean +that multiple instances of this script run concurrently, or even out +of order (i.e. the task for an older m-c push runs after the task for +a newer m-c push). The script was written with these possibilities in +mind and should be able to eventually recover from any such scenario +automatically (although it may take additional changes to mozilla-central +for such recovery to occur). That being said, the number of pathological +scenarios here is quite large and they were not really tested. + +## Ownership and access + +When this tool is run in Firefox CI, it needs to have push permissions to +the `moz-gfx` github user's account. It gets this permission via a secret token +stored in the Firefox CI taskcluster secrets service. If you need to update +the token, you need to find somebody who is a member of the +[webrender-ci access group](https://people.mozilla.org/a/webrender-ci/). The +Google Drive associated with that access group has additional documentation +on the `moz-gfx` github user and the secret token. + +## Debugging + +To debug the converter.py script, you need to have a hg checkout of +mozilla-central, let's assume it's at $MOZILLA. First create a virtualenv +with the right dependencies installed: + +``` +mkdir -p $HOME/.ghsync +virtualenv --python=python3 $HOME/.ghsync/venv +source $HOME/.ghsync/venv/bin/activate +pip3 install -r $MOZILLA/taskcluster/docker/github-sync/requirements.txt +``` + +Also create a checkout of the downstream github repo and set up a `github-sync` +branch to the point where you want port commits to. For example, for WebRender +you'd do: + +``` +cd $HOME/.ghsync +git clone https://github.com/servo/webrender +cd webrender +git checkout -b github-sync master +``` + +(You can set the github-sync branch to a past revision if you want to replicate +a failure that already got committed). + +Then run the converter from your hg checkout: + +``` +cd $MOZILLA +tools/github-sync/converter.py $HOME/.ghsync/webrender gfx/wr +``` + +You can set the DEBUG variable in the script to True to get more output. diff --git a/tools/github-sync/sync-to-github.sh b/tools/github-sync/sync-to-github.sh new file mode 100755 index 0000000000..d677649748 --- /dev/null +++ b/tools/github-sync/sync-to-github.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +# Do NOT set -x here, since that will expose a secret API token! +set -o errexit +set -o nounset +set -o pipefail + +if [[ "$(uname)" != "Linux" ]]; then + echo "Error: this script must be run on Linux due to readlink semantics" + exit 1 +fi + +# GECKO_PATH should definitely be set +if [[ -z "${GECKO_PATH}" ]]; then + echo "Error: GECKO_PATH must point to a hg clone of mozilla-central" + exit 1 +fi + +# Internal variables, don't fiddle with these +MYSELF=$(readlink -f ${0}) +MYDIR=$(dirname "${MYSELF}") +WORKDIR="${HOME}/.ghsync" +TMPDIR="${WORKDIR}/tmp" + +NAME="$1" +RELATIVE_PATH="$2" +DOWNSTREAM_REPO="$3" +BORS="$4" +BRANCH="github-sync" + +mkdir -p "${TMPDIR}" + +# Bring the project clone to a known good up-to-date state +if [[ ! -d "${WORKDIR}/${NAME}" ]]; then + echo "Setting up ${NAME} repo..." + git clone "https://github.com/${DOWNSTREAM_REPO}" "${WORKDIR}/${NAME}" + pushd "${WORKDIR}/${NAME}" + git remote add moz-gfx https://github.com/moz-gfx/${NAME} + popd +else + echo "Updating ${NAME} repo..." + pushd "${WORKDIR}/${NAME}" + git checkout master + git pull + popd +fi + +if [[ -n "${GITHUB_SECRET:-}" ]]; then + echo "Obtaining github API token..." + # Be careful, GITHUB_TOKEN is secret, so don't log it (or any variables + # built using it). + GITHUB_TOKEN=$( + curl -sSfL "$TASKCLUSTER_PROXY_URL/secrets/v1/secret/${GITHUB_SECRET}" | + ${MYDIR}/read-json.py "secret/token" + ) + AUTH="moz-gfx:${GITHUB_TOKEN}" + CURL_AUTH="Authorization: bearer ${GITHUB_TOKEN}" +fi + +echo "Pushing base ${BRANCH} branch..." +pushd "${WORKDIR}/${NAME}" +git fetch moz-gfx +git checkout -B ${BRANCH} moz-gfx/${BRANCH} || git checkout -B ${BRANCH} master + +if [[ -n "${GITHUB_SECRET:-}" ]]; then + # git may emit error messages that contain the URL, so let's sanitize them + # or we might leak the auth token to the task log. + git push "https://${AUTH}@github.com/moz-gfx/${NAME}" \ + "${BRANCH}:${BRANCH}" 2>&1 | sed -e "s/${AUTH}/_SANITIZED_/g" + # Re-fetch to update the remote moz-gfx/$BRANCH branch in the local repo; + # normally the push does this but we use a fully-qualified URL for + # pushing so it doesn't happen. + git fetch moz-gfx +fi +popd + +# Run the converter +echo "Running converter..." +pushd "${GECKO_PATH}" +"${MYDIR}/converter.py" "${WORKDIR}/${NAME}" "${RELATIVE_PATH}" +popd + +# Check to see if we have changes that need pushing +echo "Checking for new changes..." +pushd "${WORKDIR}/${NAME}" +PATCHCOUNT=$(git log --oneline moz-gfx/${BRANCH}..${BRANCH}| wc -l) +if [[ ${PATCHCOUNT} -eq 0 ]]; then + echo "No new patches found, aborting..." + exit 0 +fi + +# Log the new changes, just for logging purposes +echo "Here are the new changes:" +git log --graph --stat moz-gfx/${BRANCH}..${BRANCH} + +# Collect PR numbers of PRs opened on Github and merged to m-c +set +e +FIXES=$( + git log master..${BRANCH} | + grep "\[import_pr\] From https://github.com/${DOWNSTREAM_REPO}/pull" | + sed -e "s%.*pull/% Fixes #%" | + uniq | + tr '\n' ',' +) +echo "${FIXES}" +set -e + +if [[ -z "${GITHUB_SECRET:-}" ]]; then + echo "Running in try push, exiting now" + exit 0 +fi + +echo "Pushing new changes to moz-gfx..." +# git may emit error messages that contain the URL, so let's sanitize them +# or we might leak the auth token to the task log. +git push "https://${AUTH}@github.com/moz-gfx/${NAME}" +${BRANCH}:${BRANCH} \ + 2>&1 | sed -e "s/${AUTH}/_SANITIZED_/g" + +CURL_HEADER="Accept: application/vnd.github.v3+json" +CURL=(curl -sSfL -H "${CURL_HEADER}" -H "${CURL_AUTH}") +# URL extracted here mostly to make servo-tidy happy with line lengths +API_URL="https://api.github.com/repos/${DOWNSTREAM_REPO}" + +# Check if there's an existing PR open +echo "Listing pre-existing pull requests..." +"${CURL[@]}" "${API_URL}/pulls?head=moz-gfx:${BRANCH}" | + tee "${TMPDIR}/pr.get" +set +e +COMMENT_URL=$(cat "${TMPDIR}/pr.get" | ${MYDIR}/read-json.py "0/comments_url") +HAS_COMMENT_URL="${?}" +set -e + +if [[ ${HAS_COMMENT_URL} -ne 0 ]]; then + echo "Pull request not found, creating..." + # The PR doesn't exist yet, so let's create it + ( echo -n '{ "title": "Sync changes from mozilla-central '"${RELATIVE_PATH}"'"' + echo -n ', "body": "'"${FIXES}"'"' + echo -n ', "head": "moz-gfx:'"${BRANCH}"'"' + echo -n ', "base": "master" }' + ) > "${TMPDIR}/pr.create" + "${CURL[@]}" -d "@${TMPDIR}/pr.create" "${API_URL}/pulls" | + tee "${TMPDIR}/pr.response" + COMMENT_URL=$( + cat "${TMPDIR}/pr.response" | + ${MYDIR}/read-json.py "comments_url" + ) +fi + +# At this point COMMENTS_URL should be set, so leave a comment to tell bors +# to merge the PR. +echo "Posting r+ comment to ${COMMENT_URL}..." +echo '{ "body": "'"$BORS"' r=auto" }' > "${TMPDIR}/bors_rplus" +"${CURL[@]}" -d "@${TMPDIR}/bors_rplus" "${COMMENT_URL}" + +echo "All done!" diff --git a/tools/jprof/README.html b/tools/jprof/README.html new file mode 100644 index 0000000000..ac25acc5ef --- /dev/null +++ b/tools/jprof/README.html @@ -0,0 +1,330 @@ + + + +The Jprof Profiler + + +
+

The Jprof Profiler

+ +jim_nance@yahoo.com

+Recent (4/2011) updates Randell Jesup (see bugzilla for contact info) + +


+ +Introduction | Operation | +Setup | Usage | +Interpretation + +
+
+ +

Introduction

+ +Jprof is a profiling tool. I am writing it because I need to find out +where mozilla is spending its time, and there do not seem to be any +profilers for Linux that can handle threads and/or shared libraries. +This code is based heavily on Kipp Hickman's leaky. + +

Operation

+ +Jprof operates by installing a timer which periodically interrupts mozilla. +When this timer goes off, the jprof code inside mozilla walks the function call +stack to determine which code was executing and saves the results into the +jprof-log and jprof-map files. By collecting a large +number of these call stacks, it is possible to deduce where mozilla is spending +its time. + +

Setup

+ +

Configure your mozilla with jprof support by adding +--enable-jprof to your configure options (eg adding +ac_add_options --enable-jprof to your .mozconfig) and +making sure that you do not have the +--enable-strip configure option set -- jprof needs symbols to +operate. On many architectures with GCC, you'll need to add +--enable-optimize="-O3 -fno-omit-frame-pointer" or the +equivalent to ensure frame pointer generation in the compiler you're using.

+ +

Finally, build mozilla with your new configuration. Now you can run jprof.

+ +

Usage

+
 jprof [-v] [-t] [-e exclude] [-i include] [-s stackdepth] [--last] [--all] [--start n [--end m]] [--output-dir dir] prog log [log2 ...]
+Options: +
    +
  • -s depth : Limit depth looked at from captured stack + frames
  • +
  • -v : Output some information about the symbols, memory map, etc.
  • +
  • -t or --threads : Group output according to thread. May require external + LD_PRELOAD library to help force sampling of spawned threads; jprof + may capture the main thread only. See gprof-helper; + it may need adaption for jprof.
  • +
  • --only-thread id : Only output data for thread 'id'
  • +
  • -e exclusion : Allows excluding specific stack frames
  • +
  • -i inclusion : Allows including specific stack frames
  • +
  • --last : Only process data from the last 'section' of sampling + (starting at the last PROF)
  • +
  • --start N : Start processing data at 'section' N
  • +
  • --end N : Stop processing data at 'section' N
  • +
  • --output-dir dir : Store generated .html files in the given directory
  • +
+The behavior of jprof is determined by the value of the JPROF_FLAGS environment +variable. This environment variable can be composed of several substrings +which have the following meanings: +
    +
  • JP_START : Install the signal handler, and start sending the + timer signals. + +
  • JP_DEFER : Install the signal handler, but don't start sending + the timer signals. The user must start the signals by sending the first + one (with kill -PROF, or with kill -ALRM if + JP_REALTIME is used, or with kill -POLL (also known as kill -IO) if JP_RTC_HZ is used). + +
  • JP_FIRST=x : Wait x seconds before starting the timer + +
  • JP_PERIOD=y : Set timer to interrupt every y seconds. Only + values of y greater than or equal to 0.001 are supported. Default is + 0.050 (50ms). + +
  • JP_REALTIME : Do the profiling in intervals of real time rather + than intervals of time used by the mozilla process (and the kernel + when doing work for mozilla). This could probably lead to weird + results (you'll see whatever runs when mozilla is waiting for events), + but is needed to see time spent in the X server. + +
  • JP_RTC_HZ=freq : This option, only available on Linux if the + kernel is built with RTC support, makes jprof use the RTC timer instead of + using its own timer. This option, like JP_REALTIME, uses intervals of real + time. This option overrides JP_PERIOD. freq is the frequency + at which the timer should fire, measured in Hz. It must be a power of 2. + The maximal frequency allowed by the kernel can be changed by writing to + /proc/sys/dev/rtc/max-user-freq; the maximum value it can be + set to is 8192. Note that /dev/rtc will need to be readable + by the Firefox process; making that file world-readable is a simple way to + accomplish that. + +
  • JP_CIRCULAR=size : This tells jprof to store samples in a + circular buffer of the given size, which then will be saved (appended) + to disk when SIGUSR1 is received or JProfStopProfiling is done. If the + buffer overflows, the oldest entries will be evicted until there's + space for the new entry.

    + + SIGUSR2 will cause the circular buffer to be cleared. + +

  • JP_FILENAME=basefilename : This is the filename used for + saving the log files to; the default is "jprof-log". If Electrolysis + is used, each process after the first will have the process ID + added ("jprof-log-3212"); + +
+ +

Starting and stopping jprof from JavaScript

+

+A build with jprof enabled adds four functions to the Window object:

+JProfStartProfiling() and JProfStopProfiling(): When used with JP_DEFER, these +allow one to start and stop the timer just around whatever critical section is +being profiled.

+JProfClearCircular() and JProfSaveCircular(): +These clear the circular buffer and save the buffer (without stopping), respectively.

+ +

Examples of JPROF_FLAGS usage

+
    + +
  • To make the timer start firing 3 seconds after the program is started and + fire every 25 milliseconds of program time use: +
    +        setenv JPROF_FLAGS "JP_START JP_FIRST=3 JP_PERIOD=0.025" 
    + +
  • To make the timer start on your signal and fire every 1 millisecond of + program time use: +
    +        setenv JPROF_FLAGS "JP_DEFER JP_PERIOD=0.001" 
    + +
  • To make the timer start on your signal and fire every 10 milliseconds of + wall-clock time use: +
    +        setenv JPROF_FLAGS "JP_DEFER JP_PERIOD=0.010 JP_REALTIME" 
    + +
  • To make the timer start on your signal and fire at 8192 Hz in wall-clock + time use: +
    +        setenv JPROF_FLAGS "JP_DEFER JP_RTC_HZ=8192" 
    + +
  • To make the timer start on JProfStartProfiling() and run continously + with a 1ms sample rate until told to stop, then save the last 1MB of + data: +
    +        setenv JPROF_FLAGS "JP_DEFER JP_CIRCULAR=1048576 JP_PERIOD=0.001" 
    + +
+ +

Pausing profiles

+ +

jprof can be paused at any time by sending a SIGUSR1 to mozilla (kill +-USR1). This will cause the timer signals to stop and jprof-map to be +written, but it will not close jprof-log. Combining SIGUSR1 with the JP_DEFER +option allows profiling of one sequence of actions by starting the timer right +before starting the actions and stopping the timer right afterward. + +

After a SIGUSR1, sending another timer signal (SIGPROF, SIGALRM, or SIGPOLL (aka SIGIO), +depending on the mode) can be used to continue writing data to the same +output. + +

SIGUSR2 will cause the circular buffer to be cleared, if it's in use. +This is useful right before running a test when you're using a large, +continuous circular buffer, or programmatically at the start of an action +which might take too long (JProfClearCircular()). + +

Looking at the results

+ +Now that we have jprof-log and jprof-map files, we +can use the jprof executable is used to turn them into readable output. To do +this jprof needs the name of the mozilla binary and the log file. It deduces +the name of the map file: + +
+  ./jprof /home/user/mozilla/objdir/dist/bin/firefox ./jprof-log > tmp.html
+
+ +This will generate the file tmp.html which you should view in a +web browser. + +
+  ./jprof --output-dir=/tmp /home/user/mozilla/objdir/dist/bin/firefox ./jprof-log*
+
+ +This will generate a set of files in /tmp for each process. + + +

Interpretation

+ + +The Jprof output is split into a flat portion and a hierarchical portion. +There are links to each section at the top of the page. It is typically +easier to analyze the profile by starting with the flat output and following +the links contained in the flat output up to the hierarchical output. + +

Flat output

+ +The flat portion of the profile indicates which functions were executing +when the timer was going off. It is displayed as a list of functions names +on the right and the number of times that function was interrupted on the +left. The list is sorted by decreasing interrupt count. For example: + +
+Total hit count: 151603
+Count %Total  Function Name
+
+8806   5.8     __libc_poll
+2254   1.5     __i686.get_pc_thunk.bx
+2053   1.4     _int_malloc
+1777   1.2     ComputedStyle::GetStyleData(nsStyleStructID)
+1600   1.1     __libc_malloc
+1552   1.0     nsCOMPtr_base::~nsCOMPtr_base()
+
+ +This shows that of the 151603 times the timer fired, 1777 (1.2% of the total) were inside ComputedStyle::GetStyleData() and 1552 (1.0% of the total) were in the nsCOMPtr_base destructor. + +

+In general, the functions with the highest count are the functions which +are taking the most time. + +

+The function names are linked to the entry for that function in the +hierarchical profile, which is described in the next section. + +

Hierarchical output

+ +The hierarchical output is divided up into sections, with each section +corresponding to one function. A typical section looks something like +this: + +
+ index  Count         Hits      Function Name
+                           545 (46.4%) nsBlockFrame::ReflowInlineFrames(nsBlockReflowState&, nsLineList_iterator, int*)
+                           100 (8.5%)  nsBlockFrame::ReflowDirtyLines(nsBlockReflowState&)
+ 72870      4 (0.3%)       645 (54.9%) nsBlockFrame::DoReflowInlineFrames(nsBlockReflowState&, nsLineLayout&, nsLineList_iterator, nsFlowAreaRect&, int&, nsFloatManager::SavedState*, int*, LineReflowStatus*, int)
+                           545 (46.4%) nsBlockFrame::ReflowInlineFrame(nsBlockReflowState&, nsLineLayout&, nsLineList_iterator, nsIFrame*, LineReflowStatus*)
+                            83 (7.1%)  nsBlockFrame::PlaceLine(nsBlockReflowState&, nsLineLayout&, nsLineList_iterator, nsFloatManager::SavedState*, nsRect&, int&, int*)
+                             9 (0.8%)  nsLineLayout::BeginLineReflow(int, int, int, int, int, int)
+                             1 (0.1%)  nsTextFrame::GetType() const
+                             1 (0.1%)  nsLineLayout::RelativePositionFrames(nsOverflowAreas&)
+                             1 (0.1%)  __i686.get_pc_thunk.bx
+                             1 (0.1%)  PL_ArenaAllocate
+
+ +The information this block tells us is: + +
    +
  • There were 4 profiler hits in nsBlockFrame::DoReflowInlineFrames +
  • There were 645 profiler hits in or under nsBlockFrame::DoReflowInlineFrames. Of these: +
      +
    • 545 were in or under nsBlockFrame::ReflowInlineFrame +
    • 83 were in or under nsBlockFrame::PlaceLine +
    • 9 were in or under nsLineLayout::BeginLineReflow +
    • 1 was in or under nsTextFrame::GetType +
    • 1 was in or under nsLineLayout::RelativePositionFrames +
    • 1 was in or under __i686.get_pc_thunk.bx +
    • 1 was in or under PL_ArenaAllocate +
    +
  • Of these 645 calls into nsBlockFrame::DoReflowInlineFrames: +
      +
    • 545 came from nsBlockFrame::ReflowInlineFrames +
    • 100 came from nsBlockFrame::ReflowDirtyLines +
    +
+ + +The rest of this section explains how to read this information off from the jprof output. + +

This block corresponds to the function nsBlockFrame::DoReflowInlineFrames, which is +therefore bolded and not a link. The name of this function is preceded by +five numbers which have the following meaning. The number on the left (72870) +is the index number, and is not important. The next number (4) and the +percentage following (0.3%) are the number +of times this function was interrupted by the timer and the percentage of +the total hits that is. The last number pair ("645 (54.9%)") +are the number of times this function was in the call stack when the timer went +off. That is, the timer went off while we were in code that was ultimately +called from nsBlockFrame::DoReflowInlineFrames. +

For our example we can see that our function was in the call stack for +645 interrupt ticks, but we were only the function that was running when +the interrupt arrived 4 times. +

+The functions listed above the line for nsBlockFrame::DoReflowInlineFrames are its +callers. The numbers to the left of these function names are the numbers of +times these functions were in the call stack as callers of +nsBlockFrame::DoReflowInlineFrames. In our example, we were called 545 times by +nsBlockFrame::ReflowInlineFrames and 100 times by +nsBlockFrame::ReflowDirtyLines. +

+The functions listed below the line for nsBlockFrame::DoReflowInlineFrames are its +callees. The numbers to the left of the function names are the numbers of +times these functions were in the callstack as callees of +nsBlockFrame::DoReflowInlineFrames and the corresponding percentages. In our example, of the 645 profiler hits under nsBlockFrame::DoReflowInlineFrames 545 were under nsBlockFrame::ReflowInlineFrame, 83 were under nsBlockFrame::PlaceLine, and so forth.

+ +NOTE: If there are loops of execution or recursion, the numbers will +not add up and percentages can exceed 100%. If a function directly calls +itself "(self)" will be appended to the line, but indirect recursion will +not be marked. + +

Bugs

+The current build of Jprof has only been tested under Ubuntu 8.04 LTS, but +should work under any fairly modern linux distribution using GCC/GLIBC. +Please update this document with any known compatibilities/incompatibilities. +

+If you get an error:

Inconsistency detected by ld.so: dl-open.c: 260: dl_open_worker: Assertion `_dl_debug_initialize (0, args->nsid)->r_state == RT_CONSISTENT' failed! +

that means you've hit a timing hole in the version of glibc you're +running. See Redhat bug 4578. + + + + diff --git a/tools/jprof/bfd.cpp b/tools/jprof/bfd.cpp new file mode 100644 index 0000000000..6be8fde760 --- /dev/null +++ b/tools/jprof/bfd.cpp @@ -0,0 +1,221 @@ +// vim:ts=8:sw=2:et: +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "leaky.h" + +#ifdef USE_BFD +# include +# include +# include +# include +# include +# include +# include + +static bfd* try_debug_file(const char* filename, unsigned long crc32) { + int fd = open(filename, O_RDONLY); + if (fd < 0) return nullptr; + + unsigned char buf[4 * 1024]; + unsigned long crc = 0; + + while (1) { + ssize_t count = read(fd, buf, sizeof(buf)); + if (count <= 0) break; + + crc = bfd_calc_gnu_debuglink_crc32(crc, buf, count); + } + + close(fd); + + if (crc != crc32) return nullptr; + + bfd* object = bfd_openr(filename, nullptr); + if (!bfd_check_format(object, bfd_object)) { + bfd_close(object); + return nullptr; + } + + return object; +} + +static bfd* find_debug_file(bfd* lib, const char* aFileName) { + // check for a separate debug file with symbols + asection* sect = bfd_get_section_by_name(lib, ".gnu_debuglink"); + + if (!sect) return nullptr; + + bfd_size_type debuglinkSize = bfd_section_size(objfile->obfd, sect); + + char* debuglink = new char[debuglinkSize]; + bfd_get_section_contents(lib, sect, debuglink, 0, debuglinkSize); + + // crc checksum is aligned to 4 bytes, and after the NUL. + int crc_offset = (int(strlen(debuglink)) & ~3) + 4; + unsigned long crc32 = bfd_get_32(lib, debuglink + crc_offset); + + // directory component + char* dirbuf = strdup(aFileName); + const char* dir = dirname(dirbuf); + + static const char debug_subdir[] = ".debug"; + // This is gdb's default global debugging info directory, but gdb can + // be instructed to use a different directory. + static const char global_debug_dir[] = "/usr/lib/debug"; + + char* filename = + new char[strlen(global_debug_dir) + strlen(dir) + crc_offset + 3]; + + // /path/debuglink + sprintf(filename, "%s/%s", dir, debuglink); + bfd* debugFile = try_debug_file(filename, crc32); + if (!debugFile) { + // /path/.debug/debuglink + sprintf(filename, "%s/%s/%s", dir, debug_subdir, debuglink); + debugFile = try_debug_file(filename, crc32); + if (!debugFile) { + // /usr/lib/debug/path/debuglink + sprintf(filename, "%s/%s/%s", global_debug_dir, dir, debuglink); + debugFile = try_debug_file(filename, crc32); + } + } + + delete[] filename; + free(dirbuf); + delete[] debuglink; + + return debugFile; +} + +// Use an indirect array to avoid copying tons of objects +Symbol** leaky::ExtendSymbols(int num) { + long n = numExternalSymbols + num; + + externalSymbols = (Symbol**)realloc(externalSymbols, + (size_t)(sizeof(externalSymbols[0]) * n)); + Symbol* new_array = new Symbol[n]; + for (int i = 0; i < num; i++) { + externalSymbols[i + numExternalSymbols] = &new_array[i]; + } + lastSymbol = externalSymbols + n; + Symbol** sp = externalSymbols + numExternalSymbols; + numExternalSymbols = n; + return sp; +} + +# define NEXT_SYMBOL \ + do { \ + sp++; \ + if (sp >= lastSymbol) { \ + sp = ExtendSymbols(16384); \ + } \ + } while (0) + +void leaky::ReadSymbols(const char* aFileName, u_long aBaseAddress) { + int initialSymbols = usefulSymbols; + if (nullptr == externalSymbols) { + externalSymbols = (Symbol**)calloc(sizeof(Symbol*), 10000); + Symbol* new_array = new Symbol[10000]; + for (int i = 0; i < 10000; i++) { + externalSymbols[i] = &new_array[i]; + } + numExternalSymbols = 10000; + } + Symbol** sp = externalSymbols + usefulSymbols; + lastSymbol = externalSymbols + numExternalSymbols; + + // Create a dummy symbol for the library so, if it doesn't have any + // symbols, we show it by library. + (*sp)->Init(aFileName, aBaseAddress); + NEXT_SYMBOL; + + bfd_boolean kDynamic = (bfd_boolean) false; + + static int firstTime = 1; + if (firstTime) { + firstTime = 0; + bfd_init(); + } + + bfd* lib = bfd_openr(aFileName, nullptr); + if (nullptr == lib) { + return; + } + if (!bfd_check_format(lib, bfd_object)) { + bfd_close(lib); + return; + } + + bfd* symbolFile = find_debug_file(lib, aFileName); + + // read mini symbols + PTR minisyms; + unsigned int size; + long symcount = 0; + + if (symbolFile) { + symcount = bfd_read_minisymbols(symbolFile, kDynamic, &minisyms, &size); + if (symcount == 0) { + bfd_close(symbolFile); + } else { + bfd_close(lib); + } + } + if (symcount == 0) { + symcount = bfd_read_minisymbols(lib, kDynamic, &minisyms, &size); + if (symcount == 0) { + // symtab is empty; try dynamic symbols + kDynamic = (bfd_boolean) true; + symcount = bfd_read_minisymbols(lib, kDynamic, &minisyms, &size); + } + symbolFile = lib; + } + + asymbol* store; + store = bfd_make_empty_symbol(symbolFile); + + // Scan symbols + size_t demangle_buffer_size = 128; + char* demangle_buffer = (char*)malloc(demangle_buffer_size); + bfd_byte* from = (bfd_byte*)minisyms; + bfd_byte* fromend = from + symcount * size; + for (; from < fromend; from += size) { + asymbol* sym; + sym = + bfd_minisymbol_to_symbol(symbolFile, kDynamic, (const PTR)from, store); + + symbol_info syminfo; + bfd_get_symbol_info(symbolFile, sym, &syminfo); + + // if ((syminfo.type == 'T') || (syminfo.type == 't')) { + const char* nm = bfd_asymbol_name(sym); + if (nm && nm[0]) { + char* dnm = nullptr; + if (strncmp("__thunk", nm, 7)) { + dnm = + abi::__cxa_demangle(nm, demangle_buffer, &demangle_buffer_size, 0); + if (dnm) { + demangle_buffer = dnm; + } + } + (*sp)->Init(dnm ? dnm : nm, syminfo.value + aBaseAddress); + NEXT_SYMBOL; + } + // } + } + + free(demangle_buffer); + demangle_buffer = nullptr; + + bfd_close(symbolFile); + + int interesting = sp - externalSymbols; + if (!quiet) { + printf("%s provided %d symbols\n", aFileName, interesting - initialSymbols); + } + usefulSymbols = interesting; +} + +#endif /* USE_BFD */ diff --git a/tools/jprof/coff.cpp b/tools/jprof/coff.cpp new file mode 100644 index 0000000000..0efa83960c --- /dev/null +++ b/tools/jprof/coff.cpp @@ -0,0 +1,93 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "leaky.h" + +#ifdef USE_COFF + +# define LANGUAGE_C +# include +# include +# include +# include +# include +# include +# include + +# ifdef IRIX4 +extern "C" { +extern char* demangle(char const* in); +}; +# else +# include +# endif + +static char* Demangle(char* rawName) { +# ifdef IRIX4 + return strdup(demangle(rawName)); +# else + char namebuf[4000]; + demangle(rawName, namebuf); + return strdup(namebuf); +# endif +} + +void leaky::readSymbols(const char* fileName) { + LDFILE* ldptr; + + ldptr = ldopen(fileName, nullptr); + if (!ldptr) { + fprintf(stderr, "%s: unable to open \"%s\"\n", applicationName, fileName); + exit(-1); + } + if (PSYMTAB(ldptr) == 0) { + fprintf(stderr, "%s: \"%s\": has no symbol table\n", applicationName, + fileName); + exit(-1); + } + + long isymMax = SYMHEADER(ldptr).isymMax; + long iextMax = SYMHEADER(ldptr).iextMax; + long iMax = isymMax + iextMax; + + long alloced = 10000; + Symbol* syms = (Symbol*)malloc(sizeof(Symbol) * 10000); + Symbol* sp = syms; + Symbol* last = syms + alloced; + SYMR symr; + + for (long isym = 0; isym < iMax; isym++) { + if (ldtbread(ldptr, isym, &symr) != SUCCESS) { + fprintf(stderr, "%s: can't read symbol #%d\n", applicationName, isym); + exit(-1); + } + if (isym < isymMax) { + if ((symr.st == stStaticProc) || + ((symr.st == stProc) && + ((symr.sc == scText) || (symr.sc == scAbs))) || + ((symr.st == stBlock) && (symr.sc == scText))) { + // Text symbol. Set name field to point to the symbol name + sp->name = Demangle(ldgetname(ldptr, &symr)); + sp->address = symr.value; + sp++; + if (sp >= last) { + long n = alloced + 10000; + syms = (Symbol*)realloc(syms, (size_t)(sizeof(Symbol) * n)); + last = syms + n; + sp = syms + alloced; + alloced = n; + } + } + } + } + + int interesting = sp - syms; + if (!quiet) { + printf("Total of %d symbols\n", interesting); + } + usefulSymbols = interesting; + externalSymbols = syms; +} + +#endif /* USE_COFF */ diff --git a/tools/jprof/elf.cpp b/tools/jprof/elf.cpp new file mode 100644 index 0000000000..c2e00f60da --- /dev/null +++ b/tools/jprof/elf.cpp @@ -0,0 +1,128 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "leaky.h" + +#ifdef USE_ELF + +# include "leaky.h" +# include +# include +# include +# include +# include +# include + +void leaky::readSymbols(const char* fileName) { + int fd = ::open(fileName, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "%s: unable to open \"%s\"\n", applicationName, fileName); + exit(-1); + } + + elf_version(EV_CURRENT); + Elf* elf = elf_begin(fd, ELF_C_READ, 0); + if (!elf) { + fprintf(stderr, "%s: \"%s\": has no symbol table\n", applicationName, + fileName); + exit(-1); + } + + long alloced = 10000; + Symbol* syms = (Symbol*)malloc(sizeof(Symbol) * 10000); + Symbol* sp = syms; + Symbol* last = syms + alloced; + + // Get each of the relevant sections and add them to the list of + // symbols. + Elf32_Ehdr* ehdr = elf32_getehdr(elf); + if (!ehdr) { + fprintf(stderr, "%s: elf library lossage\n", applicationName); + exit(-1); + } +# if 0 + Elf32_Half ndx = ehdr->e_shstrndx; +# endif + + Elf_Scn* scn = 0; + int strtabndx = -1; + for (int i = 1; (scn = elf_nextscn(elf, scn)) != 0; i++) { + Elf32_Shdr* shdr = elf32_getshdr(scn); +# if 0 + char *name = elf_strptr(elf, ndx, (size_t) shdr->sh_name); + printf("Section %s (%d 0x%x)\n", name ? name : "(null)", + shdr->sh_type, shdr->sh_type); +# endif + if (shdr->sh_type == SHT_STRTAB) { + /* We assume here that string tables preceed symbol tables... */ + strtabndx = i; + continue; + } +# if 0 + if (shdr->sh_type == SHT_DYNAMIC) { + /* Dynamic */ + Elf_Data *data = elf_getdata(scn, 0); + if (!data || !data->d_size) { + printf("No data..."); + continue; + } + + Elf32_Dyn *dyn = (Elf32_Dyn*) data->d_buf; + Elf32_Dyn *lastdyn = + (Elf32_Dyn*) ((char*) data->d_buf + data->d_size); + for (; dyn < lastdyn; dyn++) { + printf("tag=%d value=0x%x\n", dyn->d_tag, dyn->d_un.d_val); + } + } else +# endif + if ((shdr->sh_type == SHT_SYMTAB) || (shdr->sh_type == SHT_DYNSYM)) { + /* Symbol table */ + Elf_Data* data = elf_getdata(scn, 0); + if (!data || !data->d_size) { + printf("No data..."); + continue; + } + + /* In theory we now have the symbols... */ + Elf32_Sym* esym = (Elf32_Sym*)data->d_buf; + Elf32_Sym* lastsym = (Elf32_Sym*)((char*)data->d_buf + data->d_size); + for (; esym < lastsym; esym++) { +# if 0 + char *nm = elf_strptr(elf, strtabndx, (size_t)esym->st_name); + printf("%20s 0x%08x %02x %02x\n", + nm, esym->st_value, ELF32_ST_BIND(esym->st_info), + ELF32_ST_TYPE(esym->st_info)); +# endif + if ((esym->st_value == 0) || + (ELF32_ST_BIND(esym->st_info) == STB_WEAK) || + (ELF32_ST_BIND(esym->st_info) == STB_NUM) || + (ELF32_ST_TYPE(esym->st_info) != STT_FUNC)) { + continue; + } +# if 1 + char* nm = elf_strptr(elf, strtabndx, (size_t)esym->st_name); +# endif + sp->name = nm ? strdup(nm) : "(no name)"; + sp->address = esym->st_value; + sp++; + if (sp >= last) { + long n = alloced + 10000; + syms = (Symbol*)realloc(syms, (size_t)(sizeof(Symbol) * n)); + last = syms + n; + sp = syms + alloced; + alloced = n; + } + } + } + } + + int interesting = sp - syms; + if (!quiet) { + printf("Total of %d symbols\n", interesting); + } + usefulSymbols = interesting; + externalSymbols = syms; +} + +#endif /* USE_ELF */ diff --git a/tools/jprof/intcnt.cpp b/tools/jprof/intcnt.cpp new file mode 100644 index 0000000000..a87b6ccf74 --- /dev/null +++ b/tools/jprof/intcnt.cpp @@ -0,0 +1,69 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "intcnt.h" + +IntCount::IntCount() : numInts(0), iPair(nullptr) {} +IntCount::~IntCount() { delete[] iPair; } +int IntCount::getSize() { return numInts; } +int IntCount::getCount(int pos) { return iPair[pos].cnt; } +int IntCount::getIndex(int pos) { return iPair[pos].idx; } + +void IntCount::clear() { + delete[] iPair; + iPair = new IntPair[0]; + numInts = 0; +} + +int IntCount::countAdd(int index, int increment) { + if (numInts) { + // Do a binary search to find the element + int divPoint = 0; + + if (index > iPair[numInts - 1].idx) { + divPoint = numInts; + } else if (index < iPair[0].idx) { + divPoint = 0; + } else { + int low = 0, high = numInts - 1; + int mid = (low + high) / 2; + while (1) { + mid = (low + high) / 2; + + if (index < iPair[mid].idx) { + high = mid; + } else if (index > iPair[mid].idx) { + if (mid < numInts - 1 && index < iPair[mid + 1].idx) { + divPoint = mid + 1; + break; + } else { + low = mid + 1; + } + } else if (index == iPair[mid].idx) { + return iPair[mid].cnt += increment; + } + } + } + + int i; + IntPair* tpair = new IntPair[numInts + 1]; + for (i = 0; i < divPoint; i++) { + tpair[i] = iPair[i]; + } + for (i = divPoint; i < numInts; i++) { + tpair[i + 1] = iPair[i]; + } + ++numInts; + delete[] iPair; + iPair = tpair; + iPair[divPoint].idx = index; + iPair[divPoint].cnt = increment; + return increment; + } else { + iPair = new IntPair[1]; + numInts = 1; + iPair[0].idx = index; + return iPair[0].cnt = increment; + } +} diff --git a/tools/jprof/intcnt.h b/tools/jprof/intcnt.h new file mode 100644 index 0000000000..2cf9ec1ff1 --- /dev/null +++ b/tools/jprof/intcnt.h @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef INTCNT_H +#define INTCNT_H + +class IntCount { + public: + IntCount(); + ~IntCount(); + void clear(); + int countAdd(int index, int increment = 1); + int countGet(int index); + int getSize(); + int getCount(int pos); + int getIndex(int pos); + + IntCount(const IntCount& old) { + numInts = old.numInts; + if (numInts > 0) { + iPair = new IntPair[numInts]; + for (int i = 0; i < numInts; i++) { + iPair[i] = old.iPair[i]; + } + } else { + iPair = nullptr; + } + } + + private: + int numInts; + struct IntPair { + int idx; + int cnt; + }* iPair; +}; + +#endif diff --git a/tools/jprof/jprofsig b/tools/jprof/jprofsig new file mode 100755 index 0000000000..02226fc4b6 --- /dev/null +++ b/tools/jprof/jprofsig @@ -0,0 +1,46 @@ +#!/bin/sh +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Find Mozilla PID and send it a signal, to be used +# with the jprof tool. +# + +jpsignal_usage() { + echo "Usage: jprofsig [start|stop]" + exit 1 +} + +if [ $# != 1 ]; then + echo "Wrong number of arguments." + jpsignal_usage +fi + +jpsignal_arg="$1" + +# Find & print mozilla PID +tmpmoz=`ps aux | grep mozilla-bin | head -1 | awk '{ print $2 }'` +echo "Mozilla PID = $tmpmoz" + +# See how we were called. +case "$jpsignal_arg" in + start) + if [ "$JP_REALTIME" = 1 ]; then + kill -ALRM $tmpmoz + else + # Normal, non-realtime mode. + kill -PROF $tmpmoz + fi + ;; + stop) + kill -USR1 $tmpmoz + ;; + *) + jpsignal_usage + exit 1 +esac + +exit 0 diff --git a/tools/jprof/leaky.cpp b/tools/jprof/leaky.cpp new file mode 100644 index 0000000000..91e9d8aa82 --- /dev/null +++ b/tools/jprof/leaky.cpp @@ -0,0 +1,827 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "leaky.h" +#include "intcnt.h" + +#include +#include +#include +#include +#include +#include +#ifndef NTO +# include +#endif +#include +#include +#include + +#ifdef NTO +# include +#endif + +#ifndef FALSE +# define FALSE 0 +#endif +#ifndef TRUE +# define TRUE 1 +#endif + +static const u_int DefaultBuckets = 10007; // arbitrary, but prime +static const u_int MaxBuckets = 1000003; // arbitrary, but prime + +//---------------------------------------------------------------------- + +int main(int argc, char** argv) { + leaky* l = new leaky; + + l->initialize(argc, argv); + l->outputfd = stdout; + + for (int i = 0; i < l->numLogFiles; i++) { + if (l->output_dir || l->numLogFiles > 1) { + char name[2048]; // XXX fix + if (l->output_dir) + snprintf(name, sizeof(name), "%s/%s.html", l->output_dir, + argv[l->logFileIndex + i]); + else + snprintf(name, sizeof(name), "%s.html", argv[l->logFileIndex + i]); + + fprintf(stderr, "opening %s\n", name); + l->outputfd = fopen(name, "w"); + // if an error we won't process the file + } + if (l->outputfd) { // paranoia + l->open(argv[l->logFileIndex + i]); + + if (l->outputfd != stderr) { + fclose(l->outputfd); + l->outputfd = nullptr; + } + } + } + + return 0; +} + +char* htmlify(const char* in) { + const char* p = in; + char *out, *q; + int n = 0; + size_t newlen; + + // Count the number of '<' and '>' in the input. + while ((p = strpbrk(p, "<>"))) { + ++n; + ++p; + } + + // Knowing the number of '<' and '>', we can calculate the space + // needed for the output string. + newlen = strlen(in) + n * 3 + 1; + out = new char[newlen]; + + // Copy the input to the output, with substitutions. + p = in; + q = out; + do { + if (*p == '<') { + strcpy(q, "<"); + q += 4; + } else if (*p == '>') { + strcpy(q, ">"); + q += 4; + } else { + *q++ = *p; + } + p++; + } while (*p); + *q = '\0'; + + return out; +} + +leaky::leaky() { + applicationName = nullptr; + progFile = nullptr; + + quiet = true; + showAddress = false; + showThreads = false; + stackDepth = 100000; + onlyThread = 0; + cleo = false; + + mappedLogFile = -1; + firstLogEntry = lastLogEntry = 0; + + sfd = -1; + externalSymbols = 0; + usefulSymbols = 0; + numExternalSymbols = 0; + lowestSymbolAddr = 0; + highestSymbolAddr = 0; + + loadMap = nullptr; + + collect_last = false; + collect_start = -1; + collect_end = -1; +} + +leaky::~leaky() {} + +void leaky::usageError() { + fprintf(stderr, + "Usage: %s [-v] [-t] [-e exclude] [-i include] [-s stackdepth] " + "[--last] [--all] [--start n [--end m]] [--cleo] [--output-dir dir] " + "prog log [log2 ...]\n", + (char*)applicationName); + fprintf( + stderr, + "\t-v: verbose\n" + "\t-t | --threads: split threads\n" + "\t--only-thread n: only profile thread N\n" + "\t-i include-id: stack must include specified id\n" + "\t-e exclude-id: stack must NOT include specified id\n" + "\t-s stackdepth: Limit depth looked at from captured stack frames\n" + "\t--last: only profile the last capture section\n" + "\t--start n [--end m]: profile n to m (or end) capture sections\n" + "\t--cleo: format output for 'cleopatra' display\n" + "\t--output-dir dir: write output files to dir\n" + "\tIf there's one log, output goes to stdout unless --output-dir is set\n" + "\tIf there are more than one log, output files will be named with .html " + "added\n"); + exit(-1); +} + +static struct option longopts[] = { + {"threads", 0, nullptr, 't'}, {"only-thread", 1, nullptr, 'T'}, + {"last", 0, nullptr, 'l'}, {"start", 1, nullptr, 'x'}, + {"end", 1, nullptr, 'n'}, {"cleo", 0, nullptr, 'c'}, + {"output-dir", 1, nullptr, 'd'}, {nullptr, 0, nullptr, 0}, +}; + +void leaky::initialize(int argc, char** argv) { + applicationName = argv[0]; + applicationName = strrchr(applicationName, '/'); + if (!applicationName) { + applicationName = argv[0]; + } else { + applicationName++; + } + + int arg; + int errflg = 0; + int longindex = 0; + + onlyThread = 0; + output_dir = nullptr; + cleo = false; + + // XXX tons of cruft here left over from tracemalloc + // XXX The -- options shouldn't need short versions, or they should be + // documented + while (((arg = getopt_long(argc, argv, "adEe:gh:i:r:Rs:tT:qvx:ln:", longopts, + &longindex)) != -1)) { + switch (arg) { + case '?': + default: + fprintf(stderr, "error: unknown option %c\n", optopt); + errflg++; + break; + case 'a': + break; + case 'A': // not implemented + showAddress = true; + break; + case 'c': + cleo = true; + break; + case 'd': + output_dir = optarg; // reference to an argv pointer + break; + case 'R': + break; + case 'e': + exclusions.add(optarg); + break; + case 'g': + break; + case 'r': // not implemented + roots.add(optarg); + if (!includes.IsEmpty()) { + errflg++; + } + break; + case 'i': + includes.add(optarg); + if (!roots.IsEmpty()) { + errflg++; + } + break; + case 'h': + break; + case 's': + stackDepth = atoi(optarg); + if (stackDepth < 2) { + stackDepth = 2; + } + break; + case 'x': + // --start + collect_start = atoi(optarg); + break; + case 'n': + // --end + collect_end = atoi(optarg); + break; + case 'l': + // --last + collect_last = true; + break; + case 'q': + break; + case 'v': + quiet = !quiet; + break; + case 't': + showThreads = true; + break; + case 'T': + showThreads = true; + onlyThread = atoi(optarg); + break; + } + } + if (errflg || ((argc - optind) < 2)) { + usageError(); + } + progFile = argv[optind++]; + logFileIndex = optind; + numLogFiles = argc - optind; + if (!quiet) fprintf(stderr, "numlogfiles = %d\n", numLogFiles); +} + +static void* mapFile(int fd, u_int flags, off_t* sz) { + struct stat sb; + if (fstat(fd, &sb) < 0) { + perror("fstat"); + exit(-1); + } + void* base = mmap(0, (int)sb.st_size, flags, MAP_PRIVATE, fd, 0); + if (!base) { + perror("mmap"); + exit(-1); + } + *sz = sb.st_size; + return base; +} + +void leaky::LoadMap() { + malloc_map_entry mme; + char name[1000]; + + if (!loadMap) { + // all files use the same map + int fd = ::open(M_MAPFILE, O_RDONLY); + if (fd < 0) { + perror("open: " M_MAPFILE); + exit(-1); + } + for (;;) { + int nb = read(fd, &mme, sizeof(mme)); + if (nb != sizeof(mme)) break; + nb = read(fd, name, mme.nameLen); + if (nb != (int)mme.nameLen) break; + name[mme.nameLen] = 0; + if (!quiet) { + fprintf(stderr, "%s @ %lx\n", name, mme.address); + } + + LoadMapEntry* lme = new LoadMapEntry; + lme->address = mme.address; + lme->name = strdup(name); + lme->next = loadMap; + loadMap = lme; + } + close(fd); + } +} + +void leaky::open(char* logFile) { + int threadArray[100]; // should auto-expand + int last_thread = -1; + int numThreads = 0; + int section = -1; + bool collecting = false; + + LoadMap(); + + setupSymbols(progFile); + + // open up the log file + if (mappedLogFile) ::close(mappedLogFile); + + mappedLogFile = ::open(logFile, O_RDONLY); + if (mappedLogFile < 0) { + perror("open"); + exit(-1); + } + off_t size; + firstLogEntry = (malloc_log_entry*)mapFile(mappedLogFile, PROT_READ, &size); + lastLogEntry = (malloc_log_entry*)((char*)firstLogEntry + size); + + if (!collect_last || collect_start < 0) { + collecting = true; + } + + // First, restrict it to the capture sections specified (all, last, start/end) + // This loop walks through all the call stacks we recorded + for (malloc_log_entry* lep = firstLogEntry; lep < lastLogEntry; + lep = reinterpret_cast(&lep->pcs[lep->numpcs])) { + if (lep->flags & JP_FIRST_AFTER_PAUSE) { + section++; + if (collect_last) { + firstLogEntry = lep; + numThreads = 0; + collecting = true; + } + if (collect_start == section) { + collecting = true; + firstLogEntry = lep; + } + if (collect_end == section) { + collecting = false; + lastLogEntry = lep; + } + if (!quiet) + fprintf(stderr, "New section %d: first=%p, last=%p, collecting=%d\n", + section, (void*)firstLogEntry, (void*)lastLogEntry, collecting); + } + + // Capture thread info at the same time + + // Find all the threads captured + + // pthread/linux docs say the signal can be delivered to any thread in + // the process. In practice, it appears in Linux that it's always + // delivered to the thread that called setitimer(), and each thread can + // have a separate itimer. There's a support library for gprof that + // overlays pthread_create() to set timers in any threads you spawn. + if (showThreads && collecting) { + if (lep->thread != last_thread) { + int i; + for (i = 0; i < numThreads; i++) { + if (lep->thread == threadArray[i]) break; + } + if (i == numThreads && + i < (int)(sizeof(threadArray) / sizeof(threadArray[0]))) { + threadArray[i] = lep->thread; + numThreads++; + if (!quiet) fprintf(stderr, "new thread %d\n", lep->thread); + } + } + } + } + if (!quiet) + fprintf(stderr, + "Done collecting: sections %d: first=%p, last=%p, numThreads=%d\n", + section, (void*)firstLogEntry, (void*)lastLogEntry, numThreads); + + if (!cleo) { + fprintf(outputfd, + "Jprof Profile Report\n"); + fprintf(outputfd, "

Jprof Profile Report

\n"); + } + + if (showThreads) { + fprintf(stderr, "Num threads %d\n", numThreads); + + if (!cleo) { + fprintf(outputfd, "
Threads:

\n");
+      for (int i = 0; i < numThreads; i++) {
+        fprintf(outputfd, "   %d  ", threadArray[i],
+                threadArray[i]);
+        if ((i + 1) % 10 == 0) fprintf(outputfd, "
\n"); + } + fprintf(outputfd, "
"); + } + + for (int i = 0; i < numThreads; i++) { + if (!onlyThread || onlyThread == threadArray[i]) analyze(threadArray[i]); + } + } else { + analyze(0); + } + + if (!cleo) fprintf(outputfd, "\n"); +} + +//---------------------------------------------------------------------- + +static int symbolOrder(void const* a, void const* b) { + Symbol const** ap = (Symbol const**)a; + Symbol const** bp = (Symbol const**)b; + return (*ap)->address == (*bp)->address + ? 0 + : ((*ap)->address > (*bp)->address ? 1 : -1); +} + +void leaky::ReadSharedLibrarySymbols() { + LoadMapEntry* lme = loadMap; + while (nullptr != lme) { + ReadSymbols(lme->name, lme->address); + lme = lme->next; + } +} + +void leaky::setupSymbols(const char* fileName) { + if (usefulSymbols == 0) { + // only read once! + + // Read in symbols from the program + ReadSymbols(fileName, 0); + + // Read in symbols from the .so's + ReadSharedLibrarySymbols(); + + if (!quiet) { + fprintf(stderr, "A total of %d symbols were loaded\n", usefulSymbols); + } + + // Now sort them + qsort(externalSymbols, usefulSymbols, sizeof(Symbol*), symbolOrder); + lowestSymbolAddr = externalSymbols[0]->address; + highestSymbolAddr = externalSymbols[usefulSymbols - 1]->address; + } +} + +// Binary search the table, looking for a symbol that covers this +// address. +int leaky::findSymbolIndex(u_long addr) { + u_int base = 0; + u_int limit = usefulSymbols - 1; + Symbol** end = &externalSymbols[limit]; + while (base <= limit) { + u_int midPoint = (base + limit) >> 1; + Symbol** sp = &externalSymbols[midPoint]; + if (addr < (*sp)->address) { + if (midPoint == 0) { + return -1; + } + limit = midPoint - 1; + } else { + if (sp + 1 < end) { + if (addr < (*(sp + 1))->address) { + return midPoint; + } + } else { + return midPoint; + } + base = midPoint + 1; + } + } + return -1; +} + +Symbol* leaky::findSymbol(u_long addr) { + int idx = findSymbolIndex(addr); + + if (idx < 0) { + return nullptr; + } else { + return externalSymbols[idx]; + } +} + +//---------------------------------------------------------------------- + +bool leaky::excluded(malloc_log_entry* lep) { + if (exclusions.IsEmpty()) { + return false; + } + + char** pcp = &lep->pcs[0]; + u_int n = lep->numpcs; + for (u_int i = 0; i < n; i++, pcp++) { + Symbol* sp = findSymbol((u_long)*pcp); + if (sp && exclusions.contains(sp->name)) { + return true; + } + } + return false; +} + +bool leaky::included(malloc_log_entry* lep) { + if (includes.IsEmpty()) { + return true; + } + + char** pcp = &lep->pcs[0]; + u_int n = lep->numpcs; + for (u_int i = 0; i < n; i++, pcp++) { + Symbol* sp = findSymbol((u_long)*pcp); + if (sp && includes.contains(sp->name)) { + return true; + } + } + return false; +} + +//---------------------------------------------------------------------- + +void leaky::displayStackTrace(FILE* out, malloc_log_entry* lep) { + char** pcp = &lep->pcs[0]; + u_int n = (lep->numpcs < stackDepth) ? lep->numpcs : stackDepth; + for (u_int i = 0; i < n; i++, pcp++) { + u_long addr = (u_long)*pcp; + Symbol* sp = findSymbol(addr); + if (sp) { + fputs(sp->name, out); + if (showAddress) { + fprintf(out, "[%p]", (char*)addr); + } + } else { + fprintf(out, "<%p>", (char*)addr); + } + fputc(' ', out); + } + fputc('\n', out); +} + +void leaky::dumpEntryToLog(malloc_log_entry* lep) { + printf("%ld\t", lep->delTime); + printf(" --> "); + displayStackTrace(outputfd, lep); +} + +void leaky::generateReportHTML(FILE* fp, int* countArray, int count, + int thread) { + fprintf(fp, "
"); + if (showThreads) { + fprintf(fp, "
Thread: %d

", thread, + thread); + } + fprintf( + fp, + "flat | hierarchical", + thread, thread); + fprintf(fp, "

\n"); + + int totalTimerHits = count; + int* rankingTable = new int[usefulSymbols]; + + for (int cnt = usefulSymbols; --cnt >= 0; rankingTable[cnt] = cnt) + ; + + // Drat. I would use ::qsort() but I would need a global variable and my + // intro-pascal professor threatened to flunk anyone who used globals. + // She damaged me for life :-) (That was 1986. See how much influence + // she had. I don't remember her name but I always feel guilty about globals) + + // Shell Sort. 581130733 is the max 31 bit value of h = 3h+1 + int mx, i, h; + for (mx = usefulSymbols / 9, h = 581130733; h > 0; h /= 3) { + if (h < mx) { + for (i = h - 1; i < usefulSymbols; i++) { + int j, tmp = rankingTable[i], val = countArray[tmp]; + for (j = i; (j >= h) && (countArray[rankingTable[j - h]] < val); + j -= h) { + rankingTable[j] = rankingTable[j - h]; + } + rankingTable[j] = tmp; + } + } + } + + // Ok, We are sorted now. Let's go through the table until we get to + // functions that were never called. Right now we don't do much inside + // this loop. Later we can get callers and callees into it like gprof + // does + fprintf(fp, + "

Hierarchical Profile


\n", + thread); + fprintf(fp, "
\n");
+  fprintf(fp, "%6s %6s         %4s      %s\n", "index", "Count", "Hits",
+          "Function Name");
+
+  for (i = 0; i < usefulSymbols && countArray[rankingTable[i]] > 0; i++) {
+    Symbol** sp = &externalSymbols[rankingTable[i]];
+
+    (*sp)->cntP.printReport(fp, this, rankingTable[i], totalTimerHits);
+
+    char* symname = htmlify((*sp)->name);
+    fprintf(fp,
+            "%6d %6d (%3.1f%%)%s %8d (%3.1f%%)%s %s\n",
+            rankingTable[i], (*sp)->timerHit,
+            ((*sp)->timerHit * 1000 / totalTimerHits) / 10.0,
+            ((*sp)->timerHit * 1000 / totalTimerHits) / 10.0 >= 10.0 ? "" : " ",
+            rankingTable[i], countArray[rankingTable[i]],
+            (countArray[rankingTable[i]] * 1000 / totalTimerHits) / 10.0,
+            (countArray[rankingTable[i]] * 1000 / totalTimerHits) / 10.0 >= 10.0
+                ? ""
+                : " ",
+            symname);
+    delete[] symname;
+
+    (*sp)->cntC.printReport(fp, this, rankingTable[i], totalTimerHits);
+
+    fprintf(fp, "
\n"); + } + fprintf(fp, "
\n"); + + // OK, Now we want to print the flat profile. To do this we resort on + // the hit count. + + // Cut-N-Paste Shell sort from above. The Ranking Table has already been + // populated, so we do not have to reinitialize it. + for (mx = usefulSymbols / 9, h = 581130733; h > 0; h /= 3) { + if (h < mx) { + for (i = h - 1; i < usefulSymbols; i++) { + int j, tmp = rankingTable[i], val = externalSymbols[tmp]->timerHit; + for (j = i; + (j >= h) && (externalSymbols[rankingTable[j - h]]->timerHit < val); + j -= h) { + rankingTable[j] = rankingTable[j - h]; + } + rankingTable[j] = tmp; + } + } + } + + // Pre-count up total counter hits, to get a percentage. + // I wanted the total before walking the list, if this + // double-pass over externalSymbols gets slow we can + // do single-pass and print this out after the loop finishes. + totalTimerHits = 0; + for (i = 0; + i < usefulSymbols && externalSymbols[rankingTable[i]]->timerHit > 0; + i++) { + Symbol** sp = &externalSymbols[rankingTable[i]]; + totalTimerHits += (*sp)->timerHit; + } + if (totalTimerHits == 0) totalTimerHits = 1; + + if (totalTimerHits != count) + fprintf(stderr, "Hit count mismatch: count=%d; totalTimerHits=%d", count, + totalTimerHits); + + fprintf(fp, + "

Flat Profile


\n", + thread); + fprintf(fp, "
\n");
+
+  fprintf(fp, "Total hit count: %d\n", totalTimerHits);
+  fprintf(fp, "Count %%Total  Function Name\n");
+  // Now loop for as long as we have timer hits
+  for (i = 0;
+       i < usefulSymbols && externalSymbols[rankingTable[i]]->timerHit > 0;
+       i++) {
+    Symbol** sp = &externalSymbols[rankingTable[i]];
+
+    char* symname = htmlify((*sp)->name);
+    fprintf(fp, "%3d   %-2.1f     %s\n", rankingTable[i],
+            (*sp)->timerHit,
+            ((float)(*sp)->timerHit / (float)totalTimerHits) * 100.0, symname);
+    delete[] symname;
+  }
+}
+
+void leaky::analyze(int thread) {
+  int* countArray = new int[usefulSymbols];
+  int* flagArray = new int[usefulSymbols];
+
+  // Zero our function call counter
+  memset(countArray, 0, sizeof(countArray[0]) * usefulSymbols);
+
+  // reset hit counts
+  for (int i = 0; i < usefulSymbols; i++) {
+    externalSymbols[i]->timerHit = 0;
+    externalSymbols[i]->regClear();
+  }
+
+  // The flag array is used to prevent counting symbols multiple times
+  // if functions are called recursively.  In order to keep from having
+  // to zero it on each pass through the loop, we mark it with the value
+  // of stacks on each trip through the loop.  This means we can determine
+  // if we have seen this symbol for this stack trace w/o having to reset
+  // from the prior stacktrace.
+  memset(flagArray, -1, sizeof(flagArray[0]) * usefulSymbols);
+
+  if (cleo) fprintf(outputfd, "m-Start\n");
+
+  // This loop walks through all the call stacks we recorded
+  // --last, --start and --end can restrict it, as can excludes/includes
+  stacks = 0;
+  for (malloc_log_entry* lep = firstLogEntry; lep < lastLogEntry;
+       lep = reinterpret_cast(&lep->pcs[lep->numpcs])) {
+    if ((thread != 0 && lep->thread != thread) || excluded(lep) ||
+        !included(lep)) {
+      continue;
+    }
+
+    ++stacks;  // How many stack frames did we collect
+
+    u_int n = (lep->numpcs < stackDepth) ? lep->numpcs : stackDepth;
+    char** pcp = &lep->pcs[n - 1];
+    int idx = -1, parrentIdx = -1;  // Init idx incase n==0
+    if (cleo) {
+      // This loop walks through every symbol in the call stack.  By walking it
+      // backwards we know who called the function when we get there.
+      char type = 's';
+      for (int i = n - 1; i >= 0; --i, --pcp) {
+        idx = findSymbolIndex(reinterpret_cast(*pcp));
+
+        if (idx >= 0) {
+          // Skip over bogus __restore_rt frames that realtime profiling
+          // can introduce.
+          if (i > 0 && !strcmp(externalSymbols[idx]->name, "__restore_rt")) {
+            --pcp;
+            --i;
+            idx = findSymbolIndex(reinterpret_cast(*pcp));
+            if (idx < 0) {
+              continue;
+            }
+          }
+          Symbol** sp = &externalSymbols[idx];
+          char* symname = htmlify((*sp)->name);
+          fprintf(outputfd, "%c-%s\n", type, symname);
+          delete[] symname;
+        }
+        // else can't find symbol - ignore
+        type = 'c';
+      }
+    } else {
+      // This loop walks through every symbol in the call stack.  By walking it
+      // backwards we know who called the function when we get there.
+      for (int i = n - 1; i >= 0; --i, --pcp) {
+        idx = findSymbolIndex(reinterpret_cast(*pcp));
+
+        if (idx >= 0) {
+          // Skip over bogus __restore_rt frames that realtime profiling
+          // can introduce.
+          if (i > 0 && !strcmp(externalSymbols[idx]->name, "__restore_rt")) {
+            --pcp;
+            --i;
+            idx = findSymbolIndex(reinterpret_cast(*pcp));
+            if (idx < 0) {
+              continue;
+            }
+          }
+
+          // If we have not seen this symbol before count it and mark it as seen
+          if (flagArray[idx] != stacks && ((flagArray[idx] = stacks) || true)) {
+            ++countArray[idx];
+          }
+
+          // We know who we are and we know who our parrent is.  Count this
+          if (parrentIdx >= 0) {
+            externalSymbols[parrentIdx]->regChild(idx);
+            externalSymbols[idx]->regParrent(parrentIdx);
+          }
+          // inside if() so an unknown in the middle of a stack won't break
+          // the link!
+          parrentIdx = idx;
+        }
+      }
+
+      // idx should be the function that we were in when we received the signal.
+      if (idx >= 0) {
+        ++externalSymbols[idx]->timerHit;
+      }
+    }
+  }
+  if (!cleo) generateReportHTML(outputfd, countArray, stacks, thread);
+}
+
+void FunctionCount::printReport(FILE* fp, leaky* lk, int parent, int total) {
+  const char* fmt =
+      "                      %8d (%3.1f%%)%s %s%s\n";
+
+  int nmax, tmax = ((~0U) >> 1);
+
+  do {
+    nmax = 0;
+    for (int j = getSize(); --j >= 0;) {
+      int cnt = getCount(j);
+      if (cnt == tmax) {
+        int idx = getIndex(j);
+        char* symname = htmlify(lk->indexToName(idx));
+        fprintf(fp, fmt, idx, getCount(j), getCount(j) * 100.0 / total,
+                getCount(j) * 100.0 / total >= 10.0 ? "" : " ", symname,
+                parent == idx ? " (self)" : "");
+        delete[] symname;
+      } else if (cnt < tmax && cnt > nmax) {
+        nmax = cnt;
+      }
+    }
+  } while ((tmax = nmax) > 0);
+}
diff --git a/tools/jprof/leaky.h b/tools/jprof/leaky.h
new file mode 100644
index 0000000000..6c9beb7b42
--- /dev/null
+++ b/tools/jprof/leaky.h
@@ -0,0 +1,124 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef __leaky_h_
+#define __leaky_h_
+
+#include "config.h"
+#include 
+#include 
+#include 
+#include "libmalloc.h"
+#include "strset.h"
+#include "intcnt.h"
+
+typedef unsigned int u_int;
+
+struct Symbol;
+struct leaky;
+
+class FunctionCount : public IntCount {
+ public:
+  void printReport(FILE* fp, leaky* lk, int parent, int total);
+};
+
+struct Symbol {
+  char* name;
+  u_long address;
+  int timerHit;
+  FunctionCount cntP, cntC;
+
+  int regChild(int id) { return cntC.countAdd(id, 1); }
+  int regParrent(int id) { return cntP.countAdd(id, 1); }
+  void regClear() {
+    cntC.clear();
+    cntP.clear();
+  }
+
+  Symbol() : timerHit(0) {}
+  void Init(const char* aName, u_long aAddress) {
+    name = aName ? strdup(aName) : (char*)"";
+    address = aAddress;
+  }
+};
+
+struct LoadMapEntry {
+  char* name;      // name of .so
+  u_long address;  // base address where it was mapped in
+  LoadMapEntry* next;
+};
+
+struct leaky {
+  leaky();
+  ~leaky();
+
+  void initialize(int argc, char** argv);
+  void open(char* arg);
+
+  char* applicationName;
+  int logFileIndex;
+  int numLogFiles;
+  char* progFile;
+  FILE* outputfd;
+
+  bool quiet;
+  bool showAddress;
+  bool showThreads;
+  bool cleo;
+  u_int stackDepth;
+  int onlyThread;
+  char* output_dir;
+
+  int mappedLogFile;
+  malloc_log_entry* firstLogEntry;
+  malloc_log_entry* lastLogEntry;
+
+  int stacks;
+
+  int sfd;
+  Symbol** externalSymbols;
+  Symbol** lastSymbol;
+  int usefulSymbols;
+  int numExternalSymbols;
+  StrSet exclusions;
+  u_long lowestSymbolAddr;
+  u_long highestSymbolAddr;
+
+  LoadMapEntry* loadMap;
+
+  bool collect_last;
+  int collect_start;
+  int collect_end;
+
+  StrSet roots;
+  StrSet includes;
+
+  void usageError();
+
+  void LoadMap();
+
+  void analyze(int thread);
+
+  void dumpEntryToLog(malloc_log_entry* lep);
+
+  void insertAddress(u_long address, malloc_log_entry* lep);
+  void removeAddress(u_long address, malloc_log_entry* lep);
+
+  void displayStackTrace(FILE* out, malloc_log_entry* lep);
+
+  Symbol** ExtendSymbols(int num);
+  void ReadSymbols(const char* fileName, u_long aBaseAddress);
+  void ReadSharedLibrarySymbols();
+  void setupSymbols(const char* fileName);
+  Symbol* findSymbol(u_long address);
+  bool excluded(malloc_log_entry* lep);
+  bool included(malloc_log_entry* lep);
+  const char* indexToName(int idx) { return externalSymbols[idx]->name; }
+
+ private:
+  void generateReportHTML(FILE* fp, int* countArray, int count, int thread);
+  int findSymbolIndex(u_long address);
+};
+
+#endif /* __leaky_h_ */
diff --git a/tools/jprof/moz.build b/tools/jprof/moz.build
new file mode 100644
index 0000000000..2aa4a8c294
--- /dev/null
+++ b/tools/jprof/moz.build
@@ -0,0 +1,28 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+DIRS += ["stub"]
+
+Program("jprof")
+
+SOURCES += [
+    "bfd.cpp",
+    "coff.cpp",
+    "elf.cpp",
+    "intcnt.cpp",
+    "leaky.cpp",
+    "strset.cpp",
+]
+
+LOCAL_INCLUDES += [
+    "stub",
+]
+
+OS_LIBS += [
+    "dl",
+    "bfd",
+    "iberty",
+]
diff --git a/tools/jprof/split-profile.py b/tools/jprof/split-profile.py
new file mode 100755
index 0000000000..c280c130c2
--- /dev/null
+++ b/tools/jprof/split-profile.py
@@ -0,0 +1,156 @@
+#!/usr/bin/python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This program splits up a jprof profile into multiple files based on a
+# list of functions in a text file.  First, a complete profile is
+# generated.  Then, for each line in the text file, a profile is
+# generated containing only stacks that go through that line, and also
+# excluding all stacks in earlier lines in the text file.  This means
+# that the text file, from start to end, is splitting out pieces of the
+# profile in their own file.  Finally, a final profile containing the
+# remainder is produced.
+
+# The program takes four arguments:
+#   (1) The path to jprof.
+#   (2) The path to the text file describing the splits.  The output
+#       will be placed in the same directory as this file.
+#   (3) The program that was profiled.
+#   (4) The jprof-log file generated by the profile, to be split up.
+# (Really, all arguments from (3) and later are passed through to
+# jprof, so additional arguments could be provided if you want to pass
+# additional arguments to jprof.)
+
+# In slightly more detail:
+#
+# This script uses jprof's includes (-i) and excludes (-e) options to
+# split profiles into segments.  It takes as input a single text file,
+# and from that text file creates a series of jprof profiles in the
+# directory the text file is in.
+#
+# The input file format looks like the following:
+#
+#   poll g_main_poll
+#   GetRuleCascade CSSRuleProcessor::GetRuleCascade(nsPresContext *, nsAtom *)
+#   RuleProcessorData RuleProcessorData::RuleProcessorData
+#        (nsPresContext *, nsIContent *, nsRuleWalker *, nsCompatibility *)
+#
+#
+# From this input file, the script will construct a profile called
+# jprof-0.html that contains the whole profile, a profile called
+# jprof-1-poll.html that includes only stacks with g_main_poll, a
+# profile called jprof-2-GetRuleCascade.html that includes only stacks
+# that have GetRuleCascade and do not have g_main_poll, a profile called
+# jprof-3-RuleProcessorData.html that includes only stacks that have the
+# RuleProcessorData constructor and do not have GetRuleCascade or
+# g_main_poll, and a profile called jprof-4.html that includes only
+# stacks that do not have any of the three functions in them.
+#
+# This means that all of the segments of the profile, except
+# jprof-0.html, are mutually exclusive.  Thus clever ordering of the
+# functions in the input file can lead to a logical splitting of the
+# profile into segments.
+
+import os.path
+import subprocess
+import sys
+
+if len(sys.argv) < 5:
+    sys.stderr.write("Expected arguments:    \n")
+    sys.exit(1)
+
+jprof = sys.argv[1]
+splitfile = sys.argv[2]
+passthrough = sys.argv[3:]
+
+for f in [jprof, splitfile]:
+    if not os.path.isfile(f):
+        sys.stderr.write("could not find file: {0}\n".format(f))
+        sys.exit(1)
+
+
+def read_splits(splitfile):
+    """
+    Read splitfile (each line of which contains a name, a space, and
+    then a function name to split on), and return a list of pairs
+    representing exactly that.  (Note that the name cannot contain
+    spaces, but the function name can, and often does.)
+    """
+
+    def line_to_split(line):
+        line = line.strip("\r\n")
+        idx = line.index(" ")
+        return (line[0:idx], line[idx + 1 :])
+
+    io = open(splitfile, "r")
+    result = [line_to_split(line) for line in io]
+    io.close()
+    return result
+
+
+splits = read_splits(splitfile)
+
+
+def generate_profile(options, destfile):
+    """
+    Run jprof to generate one split of the profile.
+    """
+    args = [jprof] + options + passthrough
+    print("Generating {}".format(destfile))
+    destio = open(destfile, "w")
+    # jprof expects the "jprof-map" file to be in its current working directory
+    cwd = None
+    for option in passthrough:
+        if option.find("jprof-log"):
+            cwd = os.path.dirname(option)
+    if cwd is None:
+        raise Exception("no jprof-log option given")
+    process = subprocess.Popen(args, stdout=destio, cwd=cwd)
+    process.wait()
+    destio.close()
+    if process.returncode != 0:
+        os.remove(destfile)
+        sys.stderr.write(
+            "Error {0} from command:\n  {1}\n".format(
+                process.returncode, " ".join(args)
+            )
+        )
+        sys.exit(process.returncode)
+
+
+def output_filename(number, splitname):
+    """
+    Return the filename (absolute path) we should use to output the
+    profile segment with the given number and splitname.  Splitname
+    should be None for the complete profile and the remainder.
+    """
+
+    def pad_count(i):
+        result = str(i)
+        # 0-pad to the same length
+        result = "0" * (len(str(len(splits) + 1)) - len(result)) + result
+        return result
+
+    name = pad_count(number)
+    if splitname is not None:
+        name += "-" + splitname
+
+    return os.path.join(os.path.dirname(splitfile), "jprof-{0}.html".format(name))
+
+
+# generate the complete profile
+generate_profile([], output_filename(0, None))
+
+# generate the listed splits
+count = 1
+excludes = []
+for (splitname, splitfunction) in splits:
+    generate_profile(
+        excludes + ["-i" + splitfunction], output_filename(count, splitname)
+    )
+    excludes += ["-e" + splitfunction]
+    count = count + 1
+
+# generate the remainder after the splits
+generate_profile(excludes, output_filename(count, None))
diff --git a/tools/jprof/strset.cpp b/tools/jprof/strset.cpp
new file mode 100644
index 0000000000..514b8c03e0
--- /dev/null
+++ b/tools/jprof/strset.cpp
@@ -0,0 +1,37 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "strset.h"
+#include 
+#include 
+
+StrSet::StrSet() {
+  strings = 0;
+  numstrings = 0;
+}
+
+void StrSet::add(const char* s) {
+  if (strings) {
+    strings = (char**)realloc(strings, (numstrings + 1) * sizeof(char*));
+  } else {
+    strings = (char**)malloc(sizeof(char*));
+  }
+  strings[numstrings] = strdup(s);
+  numstrings++;
+}
+
+int StrSet::contains(const char* s) {
+  char** sp = strings;
+  int i = numstrings;
+
+  while (--i >= 0) {
+    char* ss = *sp++;
+    if (ss[0] == s[0]) {
+      if (strcmp(ss, s) == 0) {
+        return 1;
+      }
+    }
+  }
+  return 0;
+}
diff --git a/tools/jprof/strset.h b/tools/jprof/strset.h
new file mode 100644
index 0000000000..681ed22a25
--- /dev/null
+++ b/tools/jprof/strset.h
@@ -0,0 +1,19 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef __strset_h_
+#define __strset_h_
+
+struct StrSet {
+  StrSet();
+
+  void add(const char* string);
+  int contains(const char* string);
+  bool IsEmpty() const { return 0 == numstrings; }
+
+  char** strings;
+  int numstrings;
+};
+
+#endif /* __strset_h_ */
diff --git a/tools/jprof/stub/Makefile.in b/tools/jprof/stub/Makefile.in
new file mode 100644
index 0000000000..8e6b6b8f8d
--- /dev/null
+++ b/tools/jprof/stub/Makefile.in
@@ -0,0 +1,8 @@
+#! gmake
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# override optimization
+MOZ_OPTIMIZE_FLAGS = -fno-omit-frame-pointer
diff --git a/tools/jprof/stub/config.h b/tools/jprof/stub/config.h
new file mode 100644
index 0000000000..6e5789452a
--- /dev/null
+++ b/tools/jprof/stub/config.h
@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef config_h___
+#define config_h___
+
+#define MAX_STACK_CRAWL 500
+#define M_LOGFILE "jprof-log"
+#define M_MAPFILE "jprof-map"
+
+#if defined(linux) || defined(NTO)
+#  define USE_BFD
+#  undef NEED_WRAPPERS
+
+#endif /* linux */
+
+#endif /* config_h___ */
diff --git a/tools/jprof/stub/jprof.h b/tools/jprof/stub/jprof.h
new file mode 100644
index 0000000000..c118760750
--- /dev/null
+++ b/tools/jprof/stub/jprof.h
@@ -0,0 +1,17 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jprof_h___
+#define jprof_h___
+#include "nscore.h"
+
+#ifdef _IMPL_JPPROF_API
+#  define JPROF_API(type) NS_EXPORT_(type)
+#else
+#  define JPROF_API(type) NS_IMPORT_(type)
+#endif
+
+JPROF_API(void) setupProfilingStuff(void);
+
+#endif /* jprof_h___ */
diff --git a/tools/jprof/stub/libmalloc.cpp b/tools/jprof/stub/libmalloc.cpp
new file mode 100644
index 0000000000..3003543a93
--- /dev/null
+++ b/tools/jprof/stub/libmalloc.cpp
@@ -0,0 +1,740 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+// vim:cindent:sw=4:et:ts=8:
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The linux glibc hides part of sigaction if _POSIX_SOURCE is defined
+#if defined(linux)
+#  undef _POSIX_SOURCE
+#  undef _SVID_SOURCE
+#  ifndef _GNU_SOURCE
+#    define _GNU_SOURCE
+#  endif
+#endif
+
+#include 
+#if defined(linux)
+#  include 
+#  include 
+#endif
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "libmalloc.h"
+#include "jprof.h"
+#include 
+#include 
+#include 
+
+#ifdef NTO
+#  include 
+extern r_debug _r_debug;
+#else
+#  include 
+#endif
+
+#define USE_GLIBC_BACKTRACE 1
+// To debug, use #define JPROF_STATIC
+#define JPROF_STATIC static
+
+static int gLogFD = -1;
+static pthread_t main_thread;
+
+static bool gIsChild = false;
+static int gFilenamePID;
+
+static void startSignalCounter(unsigned long millisec);
+static int enableRTCSignals(bool enable);
+
+//----------------------------------------------------------------------
+// replace use of atexit()
+
+static void DumpAddressMap();
+
+struct JprofShutdown {
+  JprofShutdown() {}
+  ~JprofShutdown() { DumpAddressMap(); }
+};
+
+static void RegisterJprofShutdown() {
+  // This instanciates the dummy class above, and will trigger the class
+  // destructor when libxul is unloaded. This is equivalent to atexit(),
+  // but gracefully handles dlclose().
+  static JprofShutdown t;
+}
+
+#if defined(i386) || defined(_i386) || defined(__x86_64__)
+JPROF_STATIC void CrawlStack(malloc_log_entry* me, void* stack_top,
+                             void* top_instr_ptr) {
+#  if USE_GLIBC_BACKTRACE
+  // This probably works on more than x86!  But we need a way to get the
+  // top instruction pointer, which is kindof arch-specific
+  void* array[500];
+  int cnt, i;
+  u_long numpcs = 0;
+
+  // This is from glibc.  A more generic version might use
+  // libunwind and/or CaptureStackBackTrace() on Windows
+  cnt = backtrace(&array[0], sizeof(array) / sizeof(array[0]));
+
+  // StackHook->JprofLog->CrawlStack
+  // Then we have sigaction, which replaced top_instr_ptr
+  array[3] = top_instr_ptr;
+  for (i = 3; i < cnt; i++) {
+    me->pcs[numpcs++] = (char*)array[i];
+  }
+  me->numpcs = numpcs;
+
+#  else
+  // original code - this breaks on many platforms
+  void** bp;
+#    if defined(__i386)
+  __asm__("movl %%ebp, %0" : "=g"(bp));
+#    elif defined(__x86_64__)
+  __asm__("movq %%rbp, %0" : "=g"(bp));
+#    else
+  // It would be nice if this worked uniformly, but at least on i386 and
+  // x86_64, it stopped working with gcc 4.1, because it points to the
+  // end of the saved registers instead of the start.
+  bp = __builtin_frame_address(0);
+#    endif
+  u_long numpcs = 0;
+  bool tracing = false;
+
+  me->pcs[numpcs++] = (char*)top_instr_ptr;
+
+  while (numpcs < MAX_STACK_CRAWL) {
+    void** nextbp = (void**)*bp++;
+    void* pc = *bp;
+    if (nextbp < bp) {
+      break;
+    }
+    if (tracing) {
+      // Skip the signal handling.
+      me->pcs[numpcs++] = (char*)pc;
+    } else if (pc == top_instr_ptr) {
+      tracing = true;
+    }
+    bp = nextbp;
+  }
+  me->numpcs = numpcs;
+#  endif
+}
+#endif
+
+//----------------------------------------------------------------------
+
+static int rtcHz;
+static int rtcFD = -1;
+static bool circular = false;
+
+#if defined(linux) || defined(NTO)
+static void DumpAddressMap() {
+  // Turn off the timer so we don't get interrupts during shutdown
+#  if defined(linux)
+  if (rtcHz) {
+    enableRTCSignals(false);
+  } else
+#  endif
+  {
+    startSignalCounter(0);
+  }
+
+  char filename[2048];
+  if (gIsChild)
+    snprintf(filename, sizeof(filename), "%s-%d", M_MAPFILE, gFilenamePID);
+  else
+    snprintf(filename, sizeof(filename), "%s", M_MAPFILE);
+
+  int mfd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0666);
+  if (mfd >= 0) {
+    malloc_map_entry mme;
+    link_map* map = _r_debug.r_map;
+    while (nullptr != map) {
+      if (map->l_name && *map->l_name) {
+        mme.nameLen = strlen(map->l_name);
+        mme.address = map->l_addr;
+        write(mfd, &mme, sizeof(mme));
+        write(mfd, map->l_name, mme.nameLen);
+#  if 0
+	write(1, map->l_name, mme.nameLen);
+	write(1, "\n", 1);
+#  endif
+      }
+      map = map->l_next;
+    }
+    close(mfd);
+  }
+}
+#endif
+
+static bool was_paused = true;
+
+JPROF_STATIC void JprofBufferDump();
+JPROF_STATIC void JprofBufferClear();
+
+static void ClearProfilingHook(int signum) {
+  if (circular) {
+    JprofBufferClear();
+    puts("Jprof: cleared circular buffer.");
+  }
+}
+
+static void EndProfilingHook(int signum) {
+  if (circular) JprofBufferDump();
+
+  DumpAddressMap();
+  was_paused = true;
+  puts("Jprof: profiling paused.");
+}
+
+//----------------------------------------------------------------------
+// proper usage would be a template, including the function to find the
+// size of an entry, or include a size header explicitly to each entry.
+#if defined(linux)
+#  define DUMB_LOCK() pthread_mutex_lock(&mutex);
+#  define DUMB_UNLOCK() pthread_mutex_unlock(&mutex);
+#else
+#  define DUMB_LOCK() FIXME()
+#  define DUMB_UNLOCK() FIXME()
+#endif
+
+class DumbCircularBuffer {
+ public:
+  DumbCircularBuffer(size_t init_buffer_size) {
+    used = 0;
+    buffer_size = init_buffer_size;
+    buffer = (unsigned char*)malloc(buffer_size);
+    head = tail = buffer;
+
+#if defined(linux)
+    pthread_mutexattr_t mAttr;
+    pthread_mutexattr_settype(&mAttr, PTHREAD_MUTEX_RECURSIVE_NP);
+    pthread_mutex_init(&mutex, &mAttr);
+    pthread_mutexattr_destroy(&mAttr);
+#endif
+  }
+  ~DumbCircularBuffer() {
+    free(buffer);
+#if defined(linux)
+    pthread_mutex_destroy(&mutex);
+#endif
+  }
+
+  void clear() {
+    DUMB_LOCK();
+    head = tail;
+    used = 0;
+    DUMB_UNLOCK();
+  }
+
+  bool empty() { return head == tail; }
+
+  size_t space_available() {
+    size_t result;
+    DUMB_LOCK();
+    if (tail > head)
+      result = buffer_size - (tail - head) - 1;
+    else
+      result = head - tail - 1;
+    DUMB_UNLOCK();
+    return result;
+  }
+
+  void drop(size_t size) {
+    // assumes correctness!
+    DUMB_LOCK();
+    head += size;
+    if (head >= &buffer[buffer_size]) head -= buffer_size;
+    used--;
+    DUMB_UNLOCK();
+  }
+
+  bool insert(void* data, size_t size) {
+    // can fail if not enough space in the entire buffer
+    DUMB_LOCK();
+    if (space_available() < size) return false;
+
+    size_t max_without_wrap = &buffer[buffer_size] - tail;
+    size_t initial = size > max_without_wrap ? max_without_wrap : size;
+#if DEBUG_CIRCULAR
+    fprintf(stderr, "insert(%d): max_without_wrap %d, size %d, initial %d\n",
+            used, max_without_wrap, size, initial);
+#endif
+    memcpy(tail, data, initial);
+    tail += initial;
+    data = ((char*)data) + initial;
+    size -= initial;
+    if (size != 0) {
+#if DEBUG_CIRCULAR
+      fprintf(stderr, "wrapping by %d bytes\n", size);
+#endif
+      memcpy(buffer, data, size);
+      tail = &(((unsigned char*)buffer)[size]);
+    }
+
+    used++;
+    DUMB_UNLOCK();
+
+    return true;
+  }
+
+  // for external access to the buffer (saving)
+  void lock() { DUMB_LOCK(); }
+
+  void unlock() { DUMB_UNLOCK(); }
+
+  // XXX These really shouldn't be public...
+  unsigned char* head;
+  unsigned char* tail;
+  unsigned int used;
+  unsigned char* buffer;
+  size_t buffer_size;
+
+ private:
+  pthread_mutex_t mutex;
+};
+
+class DumbCircularBuffer* JprofBuffer;
+
+JPROF_STATIC void JprofBufferInit(size_t size) {
+  JprofBuffer = new DumbCircularBuffer(size);
+}
+
+JPROF_STATIC void JprofBufferClear() {
+  fprintf(stderr, "Told to clear JPROF circular buffer\n");
+  JprofBuffer->clear();
+}
+
+JPROF_STATIC size_t JprofEntrySizeof(malloc_log_entry* me) {
+  return offsetof(malloc_log_entry, pcs) + me->numpcs * sizeof(char*);
+}
+
+JPROF_STATIC void JprofBufferAppend(malloc_log_entry* me) {
+  size_t size = JprofEntrySizeof(me);
+
+  do {
+    while (JprofBuffer->space_available() < size && JprofBuffer->used > 0) {
+#if DEBUG_CIRCULAR
+      fprintf(
+          stderr,
+          "dropping entry: %d in use, %d free, need %d, size_to_free = %d\n",
+          JprofBuffer->used, JprofBuffer->space_available(), size,
+          JprofEntrySizeof((malloc_log_entry*)JprofBuffer->head));
+#endif
+      JprofBuffer->drop(JprofEntrySizeof((malloc_log_entry*)JprofBuffer->head));
+    }
+    if (JprofBuffer->space_available() < size) return;
+
+  } while (!JprofBuffer->insert(me, size));
+}
+
+JPROF_STATIC void JprofBufferDump() {
+  JprofBuffer->lock();
+#if DEBUG_CIRCULAR
+  fprintf(
+      stderr, "dumping JP_CIRCULAR buffer, %d of %d bytes\n",
+      JprofBuffer->tail > JprofBuffer->head
+          ? JprofBuffer->tail - JprofBuffer->head
+          : JprofBuffer->buffer_size + JprofBuffer->tail - JprofBuffer->head,
+      JprofBuffer->buffer_size);
+#endif
+  if (JprofBuffer->tail >= JprofBuffer->head) {
+    write(gLogFD, JprofBuffer->head, JprofBuffer->tail - JprofBuffer->head);
+  } else {
+    write(gLogFD, JprofBuffer->head,
+          &(JprofBuffer->buffer[JprofBuffer->buffer_size]) - JprofBuffer->head);
+    write(gLogFD, JprofBuffer->buffer, JprofBuffer->tail - JprofBuffer->buffer);
+  }
+  JprofBuffer->clear();
+  JprofBuffer->unlock();
+}
+
+//----------------------------------------------------------------------
+
+JPROF_STATIC void JprofLog(u_long aTime, void* stack_top, void* top_instr_ptr) {
+  // Static is simply to make debugging tolerable
+  static malloc_log_entry me;
+
+  me.delTime = aTime;
+  me.thread = syscall(SYS_gettid);  // gettid();
+  if (was_paused) {
+    me.flags = JP_FIRST_AFTER_PAUSE;
+    was_paused = 0;
+  } else {
+    me.flags = 0;
+  }
+
+  CrawlStack(&me, stack_top, top_instr_ptr);
+
+#ifndef NTO
+  if (circular) {
+    JprofBufferAppend(&me);
+  } else {
+    write(gLogFD, &me, JprofEntrySizeof(&me));
+  }
+#else
+  printf("Neutrino is missing the pcs member of malloc_log_entry!! \n");
+#endif
+}
+
+static int realTime;
+
+/* Lets interrupt at 10 Hz.  This is so my log files don't get too large.
+ * This can be changed to a faster value latter.  This timer is not
+ * programmed to reset, even though it is capable of doing so.  This is
+ * to keep from getting interrupts from inside of the handler.
+ */
+static void startSignalCounter(unsigned long millisec) {
+  struct itimerval tvalue;
+
+  tvalue.it_interval.tv_sec = 0;
+  tvalue.it_interval.tv_usec = 0;
+  tvalue.it_value.tv_sec = millisec / 1000;
+  tvalue.it_value.tv_usec = (millisec % 1000) * 1000;
+
+  if (realTime) {
+    setitimer(ITIMER_REAL, &tvalue, nullptr);
+  } else {
+    setitimer(ITIMER_PROF, &tvalue, nullptr);
+  }
+}
+
+static long timerMilliSec = 50;
+
+#if defined(linux)
+static int setupRTCSignals(int hz, struct sigaction* sap) {
+  /* global */ rtcFD = open("/dev/rtc", O_RDONLY);
+  if (rtcFD < 0) {
+    perror("JPROF_RTC setup: open(\"/dev/rtc\", O_RDONLY)");
+    return 0;
+  }
+
+  if (sigaction(SIGIO, sap, nullptr) == -1) {
+    perror("JPROF_RTC setup: sigaction(SIGIO)");
+    return 0;
+  }
+
+  if (ioctl(rtcFD, RTC_IRQP_SET, hz) == -1) {
+    perror("JPROF_RTC setup: ioctl(/dev/rtc, RTC_IRQP_SET, $JPROF_RTC_HZ)");
+    return 0;
+  }
+
+  if (ioctl(rtcFD, RTC_PIE_ON, 0) == -1) {
+    perror("JPROF_RTC setup: ioctl(/dev/rtc, RTC_PIE_ON)");
+    return 0;
+  }
+
+  if (fcntl(rtcFD, F_SETSIG, 0) == -1) {
+    perror("JPROF_RTC setup: fcntl(/dev/rtc, F_SETSIG, 0)");
+    return 0;
+  }
+
+  if (fcntl(rtcFD, F_SETOWN, getpid()) == -1) {
+    perror("JPROF_RTC setup: fcntl(/dev/rtc, F_SETOWN, getpid())");
+    return 0;
+  }
+
+  return 1;
+}
+
+static int enableRTCSignals(bool enable) {
+  static bool enabled = false;
+  if (enabled == enable) {
+    return 0;
+  }
+  enabled = enable;
+
+  int flags = fcntl(rtcFD, F_GETFL);
+  if (flags < 0) {
+    perror("JPROF_RTC setup: fcntl(/dev/rtc, F_GETFL)");
+    return 0;
+  }
+
+  if (enable) {
+    flags |= FASYNC;
+  } else {
+    flags &= ~FASYNC;
+  }
+
+  if (fcntl(rtcFD, F_SETFL, flags) == -1) {
+    if (enable) {
+      perror("JPROF_RTC setup: fcntl(/dev/rtc, F_SETFL, flags | FASYNC)");
+    } else {
+      perror("JPROF_RTC setup: fcntl(/dev/rtc, F_SETFL, flags & ~FASYNC)");
+    }
+    return 0;
+  }
+
+  return 1;
+}
+#endif
+
+JPROF_STATIC void StackHook(int signum, siginfo_t* info, void* ucontext) {
+  static struct timeval tFirst;
+  static int first = 1;
+  size_t millisec = 0;
+
+#if defined(linux)
+  if (rtcHz && pthread_self() != main_thread) {
+    // Only collect stack data on the main thread, for now.
+    return;
+  }
+#endif
+
+  if (first && !(first = 0)) {
+    puts("Jprof: received first signal");
+#if defined(linux)
+    if (rtcHz) {
+      enableRTCSignals(true);
+    } else
+#endif
+    {
+      gettimeofday(&tFirst, 0);
+      millisec = 0;
+    }
+  } else {
+#if defined(linux)
+    if (rtcHz) {
+      enableRTCSignals(true);
+    } else
+#endif
+    {
+      struct timeval tNow;
+      gettimeofday(&tNow, 0);
+      double usec = 1e6 * (tNow.tv_sec - tFirst.tv_sec);
+      usec += (tNow.tv_usec - tFirst.tv_usec);
+      millisec = static_cast(usec * 1e-3);
+    }
+  }
+
+  gregset_t& gregs = ((ucontext_t*)ucontext)->uc_mcontext.gregs;
+#ifdef __x86_64__
+  JprofLog(millisec, (void*)gregs[REG_RSP], (void*)gregs[REG_RIP]);
+#else
+  JprofLog(millisec, (void*)gregs[REG_ESP], (void*)gregs[REG_EIP]);
+#endif
+
+  if (!rtcHz) startSignalCounter(timerMilliSec);
+}
+
+NS_EXPORT_(void) setupProfilingStuff(void) {
+  static int gFirstTime = 1;
+  char filename[2048];  // XXX fix
+
+  if (gFirstTime && !(gFirstTime = 0)) {
+    int startTimer = 1;
+    int doNotStart = 1;
+    int firstDelay = 0;
+    int append = O_TRUNC;
+    char* tst = getenv("JPROF_FLAGS");
+
+    /* Options from JPROF_FLAGS environment variable:
+     *   JP_DEFER  -> Wait for a SIGPROF (or SIGALRM, if JP_REALTIME
+     *               is set) from userland before starting
+     *               to generate them internally
+     *   JP_START  -> Install the signal handler
+     *   JP_PERIOD -> Time between profiler ticks
+     *   JP_FIRST  -> Extra delay before starting
+     *   JP_REALTIME -> Take stack traces in intervals of real time
+     *               rather than time used by the process (and the
+     *               system for the process).  This is useful for
+     *               finding time spent by the X server.
+     *   JP_APPEND -> Append to jprof-log rather than overwriting it.
+     *               This is somewhat risky since it depends on the
+     *               address map staying constant across multiple runs.
+     *   JP_FILENAME -> base filename to use when saving logs.  Note that
+     *               this does not affect the mapfile.
+     *   JP_CIRCULAR -> use a circular buffer of size N, write/clear on SIGUSR1
+     *
+     * JPROF_ISCHILD is set if this is not the first process.
+     */
+
+    circular = false;
+
+    if (tst) {
+      if (strstr(tst, "JP_DEFER")) {
+        doNotStart = 0;
+        startTimer = 0;
+      }
+      if (strstr(tst, "JP_START")) doNotStart = 0;
+      if (strstr(tst, "JP_REALTIME")) realTime = 1;
+      if (strstr(tst, "JP_APPEND")) append = O_APPEND;
+
+      char* delay = strstr(tst, "JP_PERIOD=");
+      if (delay) {
+        double tmp = strtod(delay + strlen("JP_PERIOD="), nullptr);
+        if (tmp >= 1e-3) {
+          timerMilliSec = static_cast(1000 * tmp);
+        } else {
+          fprintf(stderr, "JP_PERIOD of %g less than 0.001 (1ms), using 1ms\n",
+                  tmp);
+          timerMilliSec = 1;
+        }
+      }
+
+      char* circular_op = strstr(tst, "JP_CIRCULAR=");
+      if (circular_op) {
+        size_t size = atol(circular_op + strlen("JP_CIRCULAR="));
+        if (size < 1000) {
+          fprintf(stderr, "JP_CIRCULAR of %lu less than 1000, using 10000\n",
+                  (unsigned long)size);
+          size = 10000;
+        }
+        JprofBufferInit(size);
+        fprintf(stderr, "JP_CIRCULAR buffer of %lu bytes\n",
+                (unsigned long)size);
+        circular = true;
+      }
+
+      char* first = strstr(tst, "JP_FIRST=");
+      if (first) {
+        firstDelay = atol(first + strlen("JP_FIRST="));
+      }
+
+      char* rtc = strstr(tst, "JP_RTC_HZ=");
+      if (rtc) {
+#if defined(linux)
+        rtcHz = atol(rtc + strlen("JP_RTC_HZ="));
+        timerMilliSec = 0; /* This makes JP_FIRST work right. */
+        realTime = 1;      /* It's the _R_TC and all.  ;) */
+
+#  define IS_POWER_OF_TWO(x) (((x) & ((x)-1)) == 0)
+
+        if (!IS_POWER_OF_TWO(rtcHz) || rtcHz < 2) {
+          fprintf(stderr,
+                  "JP_RTC_HZ must be power of two and >= 2, "
+                  "but %d was provided; using default of 2048\n",
+                  rtcHz);
+          rtcHz = 2048;
+        }
+#else
+        fputs(
+            "JP_RTC_HZ found, but RTC profiling only supported on "
+            "Linux!\n",
+            stderr);
+
+#endif
+      }
+      const char* f = strstr(tst, "JP_FILENAME=");
+      if (f)
+        f = f + strlen("JP_FILENAME=");
+      else
+        f = M_LOGFILE;
+
+      char* is_child = getenv("JPROF_ISCHILD");
+      if (!is_child) setenv("JPROF_ISCHILD", "", 0);
+      gIsChild = !!is_child;
+
+      gFilenamePID = syscall(SYS_gettid);  // gettid();
+      if (is_child)
+        snprintf(filename, sizeof(filename), "%s-%d", f, gFilenamePID);
+      else
+        snprintf(filename, sizeof(filename), "%s", f);
+
+      // XXX FIX! inherit current capture state!
+    }
+
+    if (!doNotStart) {
+      if (gLogFD < 0) {
+        gLogFD = open(filename, O_CREAT | O_WRONLY | append, 0666);
+        if (gLogFD < 0) {
+          fprintf(stderr, "Unable to create " M_LOGFILE);
+          perror(":");
+        } else {
+          struct sigaction action;
+          sigset_t mset;
+
+          // Dump out the address map when we terminate
+          RegisterJprofShutdown();
+
+          main_thread = pthread_self();
+          // fprintf(stderr,"jprof: main_thread = %u\n",
+          //        (unsigned int)main_thread);
+
+          // FIX!  probably should block these against each other
+          // Very unlikely.
+          sigemptyset(&mset);
+          action.sa_handler = nullptr;
+          action.sa_sigaction = StackHook;
+          action.sa_mask = mset;
+          action.sa_flags = SA_RESTART | SA_SIGINFO;
+#if defined(linux)
+          if (rtcHz) {
+            if (!setupRTCSignals(rtcHz, &action)) {
+              fputs(
+                  "jprof: Error initializing RTC, NOT "
+                  "profiling\n",
+                  stderr);
+              return;
+            }
+          }
+
+          if (!rtcHz || firstDelay != 0)
+#endif
+          {
+            if (realTime) {
+              sigaction(SIGALRM, &action, nullptr);
+            }
+          }
+          // enable PROF in all cases to simplify JP_DEFER/pause/restart
+          sigaction(SIGPROF, &action, nullptr);
+
+          // make it so a SIGUSR1 will stop the profiling
+          // Note:  It currently does not close the logfile.
+          // This could be configurable (so that it could
+          // later be reopened).
+
+          struct sigaction stop_action;
+          stop_action.sa_handler = EndProfilingHook;
+          stop_action.sa_mask = mset;
+          stop_action.sa_flags = SA_RESTART;
+          sigaction(SIGUSR1, &stop_action, nullptr);
+
+          // make it so a SIGUSR2 will clear the circular buffer
+
+          stop_action.sa_handler = ClearProfilingHook;
+          stop_action.sa_mask = mset;
+          stop_action.sa_flags = SA_RESTART;
+          sigaction(SIGUSR2, &stop_action, nullptr);
+
+          printf(
+              "Jprof: Initialized signal handler and set "
+              "timer for %lu %s, %d s "
+              "initial delay\n",
+              rtcHz ? rtcHz : timerMilliSec, rtcHz ? "Hz" : "ms", firstDelay);
+
+          if (startTimer) {
+#if defined(linux)
+            /* If we have an initial delay we can just use
+               startSignalCounter to set up a timer to fire the
+               first stackHook after that delay.  When that happens
+               we'll go and switch to RTC profiling. */
+            if (rtcHz && firstDelay == 0) {
+              puts("Jprof: enabled RTC signals");
+              enableRTCSignals(true);
+            } else
+#endif
+            {
+              puts("Jprof: started timer");
+              startSignalCounter(firstDelay * 1000 + timerMilliSec);
+            }
+          }
+        }
+      }
+    }
+  } else {
+    printf("setupProfilingStuff() called multiple times\n");
+  }
+}
diff --git a/tools/jprof/stub/libmalloc.h b/tools/jprof/stub/libmalloc.h
new file mode 100644
index 0000000000..a78b35ade8
--- /dev/null
+++ b/tools/jprof/stub/libmalloc.h
@@ -0,0 +1,45 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef libmalloc_h___
+#define libmalloc_h___
+
+#include 
+#include 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "config.h"
+
+typedef unsigned long u_long;
+
+// For me->flags
+#define JP_FIRST_AFTER_PAUSE 1
+
+// Format of a jprof log entry. This is what's written out to the
+// "jprof-log" file.
+// It's called malloc_log_entry because the history of jprof is that
+// it's a modified version of tracemalloc.
+struct malloc_log_entry {
+  u_long delTime;
+  u_long numpcs;
+  unsigned int flags;
+  int thread;
+  char* pcs[MAX_STACK_CRAWL];
+};
+
+// Format of a malloc map entry; after this struct is nameLen+1 bytes of
+// name data.
+struct malloc_map_entry {
+  u_long nameLen;
+  u_long address;  // base address
+};
+
+#ifdef __cplusplus
+} /* end of extern "C" */
+#endif
+
+#endif /* libmalloc_h___ */
diff --git a/tools/jprof/stub/moz.build b/tools/jprof/stub/moz.build
new file mode 100644
index 0000000000..692c6ea37f
--- /dev/null
+++ b/tools/jprof/stub/moz.build
@@ -0,0 +1,17 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXPORTS += [
+    "jprof.h",
+]
+
+SOURCES += [
+    "libmalloc.cpp",
+]
+
+SharedLibrary("jprof")
+
+DEFINES["_IMPL_JPROF_API"] = True
diff --git a/tools/leak-gauge/leak-gauge.html b/tools/leak-gauge/leak-gauge.html
new file mode 100644
index 0000000000..7eb2de10c1
--- /dev/null
+++ b/tools/leak-gauge/leak-gauge.html
@@ -0,0 +1,320 @@
+
+
+
+  
+    
+    Leak Gauge
+
+    
+    
+  
+  
+    

Leak Gauge

+ +
+$Id: leak-gauge.html,v 1.8 2008/02/08 19:55:34 dbaron%dbaron.org Exp $
+ +

+ This script is designed to help testers isolate and simplify testcases for + many classes of leaks (those that involve large graphs of core data + structures) in Mozilla-based browsers. It is designed to print information + about what has leaked by processing a log taken while running the browser. + Such a log can be taken over a long session of normal browsing and then + the log can be processed to find sites that leak. Once a site is known to + leak, the logging can then be repeated to figure out under what conditions + the leak occurs. +

+ +

The way to create this log is to set the environment variables:

+
  MOZ_LOG=DOMLeak:5,DocumentLeak:5,nsDocShellLeak:5,NodeInfoManagerLeak:5
+  MOZ_LOG_FILE=nspr.log     (or any other filename of your choice)
+

in your shell and then run the program.

+
    +
  • + In a Windows command prompt, set environment variables with +
        set VAR=value
    +
  • +
  • + In an sh-based shell such as bash, set environment variables with +
        export VAR=value
    +
  • +
  • + In a csh-based shell such as tcsh, set environment variables with +
        setenv VAR value
    +
  • +
+ +

+ Once you have this log from a complete run of the browser (you have to + exit; otherwise it will look like everything leaked), you can load this + page (be careful not to overwrite the log when starting the browser to + load this page) and enter the filename of the log: +

+ +

+ +

+ Then you'll see the output below, which will tell you which of certain + core objects leaked and the URLs associated with those objects. +

+ + diff --git a/tools/leak-gauge/leak-gauge.pl b/tools/leak-gauge/leak-gauge.pl new file mode 100755 index 0000000000..76ac597df1 --- /dev/null +++ b/tools/leak-gauge/leak-gauge.pl @@ -0,0 +1,239 @@ +#!/usr/bin/perl -w +# vim:sw=4:ts=4:et: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# $Id: leak-gauge.pl,v 1.8 2008/02/08 19:55:03 dbaron%dbaron.org Exp $ +# This script is designed to help testers isolate and simplify testcases +# for many classes of leaks (those that involve large graphs of core +# data structures) in Mozilla-based browsers. It is designed to print +# information about what has leaked by processing a log taken while +# running the browser. Such a log can be taken over a long session of +# normal browsing and then the log can be processed to find sites that +# leak. Once a site is known to leak, the logging can then be repeated +# to figure out under what conditions the leak occurs. +# +# The way to create this log is to set the environment variables: +# MOZ_LOG=DOMLeak:5,DocumentLeak:5,nsDocShellLeak:5,NodeInfoManagerLeak:5 +# MOZ_LOG_FILE=nspr.log (or any other filename of your choice) +# in your shell and then run the program. +# * In a Windows command prompt, set environment variables with +# set VAR=value +# * In an sh-based shell such as bash, set environment variables with +# export VAR=value +# * In a csh-based shell such as tcsh, set environment variables with +# setenv VAR value +# +# Then, after you have exited the browser, run this perl script over the +# log. Either of the following commands should work: +# perl ./path/to/leak-gauge.pl nspr.log +# cat nspr.log | perl ./path/to/leak-gauge.pl +# and it will tell you which of certain core objects leaked and the URLs +# associated with those objects. + + +# Nobody said I'm not allowed to write my own object system in perl. No +# classes here. Just objects and methods. +sub call { + my $func = shift; + my $obj = shift; + my $funcref = ${$obj}{$func}; + &$funcref($obj, @_); +} + +# A hash of objects (keyed by the first word of the line in the log) +# that have two public methods, handle_line and dump (to be called using +# call, above), along with any private data they need. +my $handlers = { + "DOMWINDOW" => { + count => 0, + windows => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + my $windows = ${$self}{windows}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + $rest =~ / outer=([0-9a-f]*)$/ || die "outer expected"; + my $outer = $1; + ${$windows}{$addr} = { outer => $1 }; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$windows}{$addr}; + } elsif ($verb eq "SetNewDocument") { + $rest =~ /^ (.*)$/ || die "URI expected"; + my $uri = ($1); + ${${$windows}{$addr}}{$uri} = 1; + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $windows = ${$self}{windows}; + foreach my $addr (keys(%{$windows})) { + my $winobj = ${$windows}{$addr}; + my $outer = delete ${$winobj}{outer}; + print "Leaked " . ($outer eq "0" ? "outer" : "inner") . + " window $addr " . + ($outer eq "0" ? "" : "(outer $outer) ") . + "at address $addr.\n"; + foreach my $uri (keys(%{$winobj})) { + print " ... with URI \"$uri\".\n"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $windows = ${$self}{windows}; + print 'Leaked ' . keys(%{$windows}) . ' out of ' . + ${$self}{count} . " DOM Windows\n"; + } + }, + "DOCUMENT" => { + count => 0, + docs => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + # This doesn't work; I don't have time to figure out why not. + # my $docs = ${$self}{docs}; + my $docs = ${$handlers}{"DOCUMENT"}{docs}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + ${$docs}{$addr} = {}; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$docs}{$addr}; + } elsif ($verb eq "ResetToURI" || + $verb eq "StartDocumentLoad") { + $rest =~ /^ (.*)$/ || die "URI expected"; + my $uri = $1; + my $doc_info = ${$docs}{$addr}; + ${$doc_info}{$uri} = 1; + if (exists(${$doc_info}{"nim"})) { + ${$doc_info}{"nim"}{$uri} = 1; + } + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $docs = ${$self}{docs}; + foreach my $addr (keys(%{$docs})) { + print "Leaked document at address $addr.\n"; + foreach my $uri (keys(%{${$docs}{$addr}})) { + print " ... with URI \"$uri\".\n" unless $uri eq "nim"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $docs = ${$self}{docs}; + print 'Leaked ' . keys(%{$docs}) . ' out of ' . + ${$self}{count} . " documents\n"; + } + }, + "DOCSHELL" => { + count => 0, + shells => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + my $shells = ${$self}{shells}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + ${$shells}{$addr} = {}; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$shells}{$addr}; + } elsif ($verb eq "InternalLoad" || + $verb eq "SetCurrentURI") { + $rest =~ /^ (.*)$/ || die "URI expected"; + my $uri = $1; + ${${$shells}{$addr}}{$uri} = 1; + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $shells = ${$self}{shells}; + foreach my $addr (keys(%{$shells})) { + print "Leaked docshell at address $addr.\n"; + foreach my $uri (keys(%{${$shells}{$addr}})) { + print " ... which loaded URI \"$uri\".\n"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $shells = ${$self}{shells}; + print 'Leaked ' . keys(%{$shells}) . ' out of ' . + ${$self}{count} . " docshells\n"; + } + }, + "NODEINFOMANAGER" => { + count => 0, + nims => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + my $nims = ${$self}{nims}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + ${$nims}{$addr} = {}; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$nims}{$addr}; + } elsif ($verb eq "Init") { + $rest =~ /^ document=(.*)$/ || + die "document pointer expected"; + my $doc = $1; + if ($doc ne "0") { + my $nim_info = ${$nims}{$addr}; + my $doc_info = ${$handlers}{"DOCUMENT"}{docs}{$doc}; + foreach my $uri (keys(%{$doc_info})) { + ${$nim_info}{$uri} = 1; + } + ${$doc_info}{"nim"} = $nim_info; + } + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $nims = ${$self}{nims}; + foreach my $addr (keys(%{$nims})) { + print "Leaked content nodes associated with node info manager at address $addr.\n"; + foreach my $uri (keys(%{${$nims}{$addr}})) { + print " ... with document URI \"$uri\".\n"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $nims = ${$self}{nims}; + print 'Leaked content nodes within ' . keys(%{$nims}) . ' out of ' . + ${$self}{count} . " documents\n"; + } + } +}; + +while (<>) { + # strip off initial "-", thread id, and thread pointer; separate + # first word and rest + if (/^\-?[0-9]*\[[0-9a-f]*\]: (\S*) ([^\n\r]*)[\n\r]*$/) { + my ($handler, $data) = ($1, $2); + if (defined(${$handlers}{$handler})) { + call("handle_line", ${$handlers}{$handler}, $data); + } + } +} + +foreach my $key (keys(%{$handlers})) { + call("dump", ${$handlers}{$key}); +} +print "Summary:\n"; +foreach my $key (keys(%{$handlers})) { + call("summary", ${$handlers}{$key}); +} diff --git a/tools/lint/android-api-lint.yml b/tools/lint/android-api-lint.yml new file mode 100644 index 0000000000..0b81e79a83 --- /dev/null +++ b/tools/lint/android-api-lint.yml @@ -0,0 +1,15 @@ +--- +android-api-lint: + description: Android api-lint + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:api_lint + setup: android.lints:setup diff --git a/tools/lint/android-checkstyle.yml b/tools/lint/android-checkstyle.yml new file mode 100644 index 0000000000..abd4974e6a --- /dev/null +++ b/tools/lint/android-checkstyle.yml @@ -0,0 +1,15 @@ +--- +android-checkstyle: + description: Android checkstyle + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:checkstyle + setup: android.lints:setup diff --git a/tools/lint/android-format.yml b/tools/lint/android-format.yml new file mode 100644 index 0000000000..cacf3ff2d7 --- /dev/null +++ b/tools/lint/android-format.yml @@ -0,0 +1,15 @@ +--- +android-format: + description: Android formatting lint + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:format + setup: android.lints:setup diff --git a/tools/lint/android-javadoc.yml b/tools/lint/android-javadoc.yml new file mode 100644 index 0000000000..a0811a08cd --- /dev/null +++ b/tools/lint/android-javadoc.yml @@ -0,0 +1,15 @@ +--- +android-javadoc: + description: Android javadoc + include: ['mobile/android/geckoview'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:javadoc + setup: android.lints:setup diff --git a/tools/lint/android-lint.yml b/tools/lint/android-lint.yml new file mode 100644 index 0000000000..6a1dbe7b74 --- /dev/null +++ b/tools/lint/android-lint.yml @@ -0,0 +1,15 @@ +--- +android-lint: + description: Android lint + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:lint + setup: android.lints:setup diff --git a/tools/lint/android-test.yml b/tools/lint/android-test.yml new file mode 100644 index 0000000000..65739295dd --- /dev/null +++ b/tools/lint/android-test.yml @@ -0,0 +1,15 @@ +--- +android-test: + description: Android test + include: ['mobile/android'] + exclude: [] + extensions: ['java', 'kt'] + support-files: + - 'mobile/android/**/Makefile.in' + - 'mobile/android/config/**' + - 'mobile/android/gradle.configure' + - 'mobile/android/**/moz.build' + - '**/*.gradle' + type: global + payload: android.lints:test + setup: android.lints:setup diff --git a/tools/lint/android/__init__.py b/tools/lint/android/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/lint/android/lints.py b/tools/lint/android/lints.py new file mode 100644 index 0000000000..a0ec59e805 --- /dev/null +++ b/tools/lint/android/lints.py @@ -0,0 +1,421 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import glob +import itertools +import json +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +import mozpack.path as mozpath +import six +from mozlint import result +from mozpack.files import FileFinder + +# The Gradle target invocations are serialized with a simple locking file scheme. It's fine for +# them to take a while, since the first will compile all the Java, etc, and then perform +# potentially expensive static analyses. +GRADLE_LOCK_MAX_WAIT_SECONDS = 20 * 60 + + +def setup(root, **setupargs): + if setupargs.get("substs", {}).get("MOZ_BUILD_APP") != "mobile/android": + return 1 + + if "topobjdir" not in setupargs: + print( + "Skipping {}: a configured Android build is required!".format( + setupargs["name"] + ) + ) + return 1 + + return 0 + + +def gradle(log, topsrcdir=None, topobjdir=None, tasks=[], extra_args=[], verbose=True): + sys.path.insert(0, os.path.join(topsrcdir, "mobile", "android")) + from gradle import gradle_lock + + with gradle_lock(topobjdir, max_wait_seconds=GRADLE_LOCK_MAX_WAIT_SECONDS): + # The android-lint parameter can be used by gradle tasks to run special + # logic when they are run for a lint using + # project.hasProperty('android-lint') + cmd_args = ( + [ + sys.executable, + os.path.join(topsrcdir, "mach"), + "gradle", + "--verbose", + "-Pandroid-lint", + "--", + ] + + tasks + + extra_args + ) + + cmd = " ".join(six.moves.shlex_quote(arg) for arg in cmd_args) + log.debug(cmd) + + # Gradle and mozprocess do not get along well, so we use subprocess + # directly. + proc = subprocess.Popen(cmd_args, cwd=topsrcdir) + status = None + # Leave it to the subprocess to handle Ctrl+C. If it terminates as a result + # of Ctrl+C, proc.wait() will return a status code, and, we get out of the + # loop. If it doesn't, like e.g. gdb, we continue waiting. + while status is None: + try: + status = proc.wait() + except KeyboardInterrupt: + pass + + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + raise + + return proc.returncode + + +def format(config, fix=None, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + if fix: + tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FIX_TASKS"] + else: + tasks = lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_CHECK_TASKS"] + + ret = gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=tasks, + extra_args=lintargs.get("extra_args") or [], + ) + + results = [] + for path in lintargs["substs"]["GRADLE_ANDROID_FORMAT_LINT_FOLDERS"]: + folder = os.path.join( + topobjdir, "gradle", "build", path, "spotless", "spotlessJava" + ) + for filename in glob.iglob(folder + "/**/*.java", recursive=True): + err = { + "rule": "spotless-java", + "path": os.path.join( + topsrcdir, path, mozpath.relpath(filename, folder) + ), + "lineno": 0, + "column": 0, + "message": "Formatting error, please run ./mach lint -l android-format --fix", + "level": "error", + } + results.append(result.from_config(config, **err)) + folder = os.path.join( + topobjdir, "gradle", "build", path, "spotless", "spotlessKotlin" + ) + for filename in glob.iglob(folder + "/**/*.kt", recursive=True): + err = { + "rule": "spotless-kt", + "path": os.path.join( + topsrcdir, path, mozpath.relpath(filename, folder) + ), + "lineno": 0, + "column": 0, + "message": "Formatting error, please run ./mach lint -l android-format --fix", + "level": "error", + } + results.append(result.from_config(config, **err)) + + if len(results) == 0 and ret != 0: + # spotless seems to hit unfixed error. + err = { + "rule": "spotless", + "path": "", + "lineno": 0, + "column": 0, + "message": "Unexpected error", + "level": "error", + } + results.append(result.from_config(config, **err)) + + # If --fix was passed, we just report the number of files that were changed + if fix: + return {"results": [], "fixed": len(results)} + return results + + +def api_lint(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + folder = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_APILINT_FOLDER"] + + results = [] + + with open(os.path.join(topobjdir, folder, "apilint-result.json")) as f: + issues = json.load(f) + + for rule in ("compat_failures", "failures"): + for r in issues[rule]: + err = { + "rule": r["rule"] if rule == "failures" else "compat_failures", + "path": r["file"], + "lineno": int(r["line"]), + "column": int(r.get("column") or 0), + "message": r["msg"], + "level": "error" if r["error"] else "warning", + } + results.append(result.from_config(config, **err)) + + for r in issues["api_changes"]: + err = { + "rule": "api_changes", + "path": r["file"], + "lineno": int(r["line"]), + "column": int(r.get("column") or 0), + "message": "Unexpected api change. Please run ./mach gradle {} for more " + "information".format( + " ".join(lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"]) + ), + } + results.append(result.from_config(config, **err)) + + return results + + +def javadoc(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + output_files = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_OUTPUT_FILES"] + + results = [] + + for output_file in output_files: + with open(os.path.join(topobjdir, output_file)) as f: + # Like: '[{"path":"/absolute/path/to/topsrcdir/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/ContentBlocking.java","lineno":"462","level":"warning","message":"no @return"}]'. # NOQA: E501 + issues = json.load(f) + + for issue in issues: + # We want warnings to be errors for linting purposes. + # TODO: Bug 1316188 - resolve missing javadoc comments + issue["level"] = ( + "error" if issue["message"] != ": no comment" else "warning" + ) + results.append(result.from_config(config, **issue)) + + return results + + +def lint(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_LINT_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + # It's surprising that this is the App variant name, but this is "withoutGeckoBinariesDebug" + # right now and the GeckoView variant name is "withGeckoBinariesDebug". This will be addressed + # as we unify variants. + path = os.path.join( + lintargs["topobjdir"], + "gradle/build/mobile/android/geckoview/reports", + "lint-results-{}.xml".format( + lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"] + ), + ) + tree = ET.parse(open(path, "rt")) + root = tree.getroot() + + results = [] + + for issue in root.findall("issue"): + location = issue[0] + if "third_party" in location.get("file") or "thirdparty" in location.get( + "file" + ): + continue + err = { + "level": issue.get("severity").lower(), + "rule": issue.get("id"), + "message": issue.get("message"), + "path": location.get("file"), + "lineno": int(location.get("line") or 0), + } + results.append(result.from_config(config, **err)) + + return results + + +def _parse_checkstyle_output(config, topsrcdir=None, report_path=None): + tree = ET.parse(open(report_path, "rt")) + root = tree.getroot() + + for file in root.findall("file"): + + for error in file.findall("error"): + # Like . # NOQA: E501 + err = { + "level": "error", + "rule": error.get("source"), + "message": error.get("message"), + "path": file.get("name"), + "lineno": int(error.get("line") or 0), + "column": int(error.get("column") or 0), + } + yield result.from_config(config, **err) + + +def checkstyle(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + results = [] + + for relative_path in lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_OUTPUT_FILES"]: + report_path = os.path.join(lintargs["topobjdir"], relative_path) + results.extend( + _parse_checkstyle_output( + config, topsrcdir=lintargs["root"], report_path=report_path + ) + ) + + return results + + +def _parse_android_test_results(config, topsrcdir=None, report_dir=None): + # A brute force way to turn a Java FQN into a path on disk. Assumes Java + # and Kotlin sources are in mobile/android for performance and simplicity. + sourcepath_finder = FileFinder(os.path.join(topsrcdir, "mobile", "android")) + + finder = FileFinder(report_dir) + reports = list(finder.find("TEST-*.xml")) + if not reports: + raise RuntimeError("No reports found under {}".format(report_dir)) + + for report, _ in reports: + tree = ET.parse(open(os.path.join(finder.base, report), "rt")) + root = tree.getroot() + + class_name = root.get( + "name" + ) # Like 'org.mozilla.gecko.permissions.TestPermissions'. + path = ( + "**/" + class_name.replace(".", "/") + ".*" + ) # Like '**/org/mozilla/gecko/permissions/TestPermissions.*'. # NOQA: E501 + + for testcase in root.findall("testcase"): + function_name = testcase.get("name") + + # Schema cribbed from http://llg.cubic.org/docs/junit/. + for unexpected in itertools.chain( + testcase.findall("error"), testcase.findall("failure") + ): + sourcepaths = list(sourcepath_finder.find(path)) + if not sourcepaths: + raise RuntimeError( + "No sourcepath found for class {class_name}".format( + class_name=class_name + ) + ) + + for sourcepath, _ in sourcepaths: + lineno = 0 + message = unexpected.get("message") + # Turn '... at org.mozilla.gecko.permissions.TestPermissions.testMultipleRequestsAreQueuedAndDispatchedSequentially(TestPermissions.java:118)' into 118. # NOQA: E501 + pattern = r"at {class_name}\.{function_name}\(.*:(\d+)\)" + pattern = pattern.format( + class_name=class_name, function_name=function_name + ) + match = re.search(pattern, message) + if match: + lineno = int(match.group(1)) + else: + msg = "No source line found for {class_name}.{function_name}".format( + class_name=class_name, function_name=function_name + ) + raise RuntimeError(msg) + + err = { + "level": "error", + "rule": unexpected.get("type"), + "message": message, + "path": os.path.join( + topsrcdir, "mobile", "android", sourcepath + ), + "lineno": lineno, + } + yield result.from_config(config, **err) + + +def test(config, **lintargs): + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] + + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_TEST_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) + + results = [] + + def capitalize(s): + # Can't use str.capitalize because it lower cases trailing letters. + return (s[0].upper() + s[1:]) if s else "" + + pairs = [("geckoview", lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"])] + for project, variant in pairs: + report_dir = os.path.join( + lintargs["topobjdir"], + "gradle/build/mobile/android/{}/test-results/test{}UnitTest".format( + project, capitalize(variant) + ), + ) + results.extend( + _parse_android_test_results( + config, topsrcdir=lintargs["root"], report_dir=report_dir + ) + ) + + return results diff --git a/tools/lint/black.yml b/tools/lint/black.yml new file mode 100644 index 0000000000..4c0f8533cc --- /dev/null +++ b/tools/lint/black.yml @@ -0,0 +1,18 @@ +--- +black: + description: Reformat python + exclude: + - gfx/harfbuzz/src/meson.build + - '**/*.mako.py' + - python/mozbuild/mozbuild/test/frontend/data/reader-error-syntax/moz.build + - testing/mozharness/configs/test/test_malformed.py + - testing/web-platform/tests + extensions: + - build + - configure + - py + support-files: + - 'tools/lint/python/**' + type: external + payload: python.black:lint + setup: python.black:setup diff --git a/tools/lint/clang-format.yml b/tools/lint/clang-format.yml new file mode 100644 index 0000000000..bcdf9d8591 --- /dev/null +++ b/tools/lint/clang-format.yml @@ -0,0 +1,12 @@ +--- +clang-format: + description: Reformat C/C++ + include: + - '.' + extensions: ['cpp', 'c', 'cc', 'h', 'm', 'mm'] + support-files: + - 'tools/lint/clang-format/**' + type: external + payload: clang-format:lint + code_review_warnings: false + setup: clang-format:setup diff --git a/tools/lint/clang-format/__init__.py b/tools/lint/clang-format/__init__.py new file mode 100644 index 0000000000..d0a81f7b06 --- /dev/null +++ b/tools/lint/clang-format/__init__.py @@ -0,0 +1,237 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import signal +import sys +import xml.etree.ElementTree as ET + +from mozboot.util import get_tools_dir +from mozlint import result +from mozlint.pathutils import expand_exclusions +from mozprocess import ProcessHandler + +CLANG_FORMAT_NOT_FOUND = """ +Could not find clang-format! It should've been installed automatically - \ +please report a bug here: +https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox%20Build%20System&component=Lint%20and%20Formatting +""".strip() + + +def setup(root, mach_command_context, **lintargs): + if get_clang_format_binary(): + return 0 + + from mozbuild.code_analysis.mach_commands import get_clang_tools + + rc, _ = get_clang_tools(mach_command_context) + if rc: + return 1 + + +class ClangFormatProcess(ProcessHandler): + def __init__(self, config, *args, **kwargs): + self.config = config + kwargs["stream"] = False + kwargs["universal_newlines"] = True + ProcessHandler.__init__(self, *args, **kwargs) + + def run(self, *args, **kwargs): + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + ProcessHandler.run(self, *args, **kwargs) + signal.signal(signal.SIGINT, orig) + + +def run_process(config, cmd): + proc = ClangFormatProcess(config, cmd) + proc.run() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + + return proc.output + + +def get_clang_format_binary(): + """ + Returns the path of the first clang-format binary available + if not found returns None + """ + binary = os.environ.get("CLANG_FORMAT") + if binary: + return binary + + clang_tools_path = os.path.join(get_tools_dir(), "clang-tools") + bin_path = os.path.join(clang_tools_path, "clang-tidy", "bin") + binary = os.path.join(bin_path, "clang-format") + + if sys.platform.startswith("win"): + binary += ".exe" + + if not os.path.isfile(binary): + return None + + return binary + + +def is_ignored_path(ignored_dir_re, topsrcdir, f): + # Remove up to topsrcdir in pathname and match + if f.startswith(topsrcdir + "/"): + match_f = f[len(topsrcdir + "/") :] + else: + match_f = f + return re.match(ignored_dir_re, match_f) + + +def remove_ignored_path(paths, topsrcdir, log): + path_to_third_party = os.path.join(topsrcdir, ".clang-format-ignore") + + ignored_dir = [] + with open(path_to_third_party, "r") as fh: + for line in fh: + # In case it starts with a space + line = line.strip() + # Remove comments and empty lines + if line.startswith("#") or len(line) == 0: + continue + # The regexp is to make sure we are managing relative paths + ignored_dir.append(r"^[\./]*" + line.rstrip()) + + # Generates the list of regexp + ignored_dir_re = "(%s)" % "|".join(ignored_dir) + + path_list = [] + for f in paths: + if is_ignored_path(ignored_dir_re, topsrcdir, f): + # Early exit if we have provided an ignored directory + log.debug("Ignored third party code '{0}'".format(f)) + continue + path_list.append(f) + + return path_list + + +def lint(paths, config, fix=None, **lintargs): + log = lintargs["log"] + paths = list(expand_exclusions(paths, config, lintargs["root"])) + + # We ignored some specific files for a bunch of reasons. + # Not using excluding to avoid duplication + if lintargs.get("use_filters", True): + paths = remove_ignored_path(paths, lintargs["root"], log) + + # An empty path array can occur when the user passes in `-n`. If we don't + # return early in this case, rustfmt will attempt to read stdin and hang. + if not paths: + return [] + + binary = get_clang_format_binary() + + if not binary: + print(CLANG_FORMAT_NOT_FOUND) + if "MOZ_AUTOMATION" in os.environ: + return 1 + return [] + + cmd_args = [binary] + + base_command = cmd_args + ["--version"] + version = run_process(config, base_command) + log.debug("Version: {}".format(version)) + + cmd_args.append("--output-replacements-xml") + base_command = cmd_args + paths + log.debug("Command: {}".format(" ".join(cmd_args))) + output = run_process(config, base_command) + + def replacement(parser): + for end, e in parser.read_events(): + assert end == "end" + if e.tag == "replacement": + item = {k: int(v) for k, v in e.items()} + assert sorted(item.keys()) == ["length", "offset"] + item["with"] = (e.text or "").encode("utf-8") + yield item + + # When given multiple paths as input, --output-replacements-xml + # will output one xml per path, in the order they are given, but + # XML parsers don't know how to handle that, so do it manually. + parser = None + replacements = [] + for line in output: + if line.startswith("") + + def condition(self, payload, line, config): + if not line.startswith("#include"): + return False + + if self.regex.search(line, re.I): + return not self.regex.search(line) + + +def lint(paths, config, **lintargs): + results = [] + + m = MinGWCapitalization() + for path in paths: + results.extend(m._lint(path, config, **lintargs)) + return results diff --git a/tools/lint/cpp/mingw-headers.txt b/tools/lint/cpp/mingw-headers.txt new file mode 100644 index 0000000000..5ee737c393 --- /dev/null +++ b/tools/lint/cpp/mingw-headers.txt @@ -0,0 +1,1452 @@ +accctrl.h +aclapi.h +aclui.h +acpiioct.h +activation.h +activaut.h +activdbg100.h +activdbg.h +activecf.h +activeds.h +activprof.h +activscp.h +adc.h +adhoc.h +admex.h +adoctint.h +adodef.h +adogpool_backcompat.h +adogpool.h +adoguids.h +adoid.h +adoint_backcompat.h +adoint.h +adojet.h +adomd.h +adptif.h +adsdb.h +adserr.h +adshlp.h +adsiid.h +adsnms.h +adsprop.h +adssts.h +adtgen.h +advpub.h +afilter.h +af_irda.h +afxres.h +agtctl.h +agterr.h +agtsvr.h +alg.h +alink.h +amaudio.h +amstream.h +amtvuids.h +amvideo.h +apdevpkey.h +apisetcconv.h +apiset.h +appmgmt.h +aqadmtyp.h +asptlb.h +assert.h +atacct.h +atalkwsh.h +atm.h +atsmedia.h +audevcod.h +audioapotypes.h +audioclient.h +audioendpoints.h +audioengineendpoint.h +audiopolicy.h +audiosessiontypes.h +austream.h +authif.h +authz.h +aux_ulib.h +avifmt.h +aviriff.h +avrfsdk.h +avrt.h +axextendenums.h +azroles.h +basetsd.h +basetyps.h +batclass.h +bcrypt.h +bdaiface_enums.h +bdaiface.h +bdamedia.h +bdasup.h +bdatypes.h +bemapiset.h +bh.h +bidispl.h +bits1_5.h +bits2_0.h +bitscfg.h +bits.h +bitsmsg.h +blberr.h +bluetoothapis.h +_bsd_types.h +bthdef.h +bthsdpdef.h +bugcodes.h +callobj.h +cardmod.h +casetup.h +cchannel.h +cdefs.h +cderr.h +cdoexerr.h +cdoex.h +cdoexm.h +cdoexstr.h +cdonts.h +cdosyserr.h +cdosys.h +cdosysstr.h +celib.h +certadm.h +certbase.h +certbcli.h +certcli.h +certenc.h +certenroll.h +certexit.h +certif.h +certmod.h +certpol.h +certreqd.h +certsrv.h +certview.h +cfg.h +cfgmgr32.h +cguid.h +chanmgr.h +cierror.h +classpnp.h +clfs.h +clfsmgmt.h +clfsmgmtw32.h +clfsw32.h +client.h +cluadmex.h +clusapi.h +cluscfgguids.h +cluscfgserver.h +cluscfgwizard.h +cmdtree.h +cmnquery.h +codecapi.h +colordlg.h +comadmin.h +combaseapi.h +comcat.h +comdef.h +comdefsp.h +comip.h +comlite.h +commapi.h +commctrl.h +commdlg.h +commoncontrols.h +complex.h +compobj.h +compressapi.h +compstui.h +comsvcs.h +comutil.h +confpriv.h +conio.h +conio_s.h +control.h +corecrt_startup.h +corerror.h +corewrappers.h +cor.h +corhdr.h +correg.h +cplext.h +cpl.h +credssp.h +crtdbg.h +crtdbg_s.h +crtdefs.h +cryptuiapi.h +cryptxml.h +cscapi.h +cscobj.h +csq.h +ctfutb.h +ctxtcall.h +ctype.h +custcntl.h +_cygwin.h +d2d1_1.h +d2d1_1helper.h +d2d1effectauthor.h +d2d1effecthelpers.h +d2d1effects.h +d2d1.h +d2d1helper.h +d2dbasetypes.h +d2derr.h +d3d10_1.h +d3d10_1shader.h +d3d10effect.h +d3d10.h +d3d10misc.h +d3d10sdklayers.h +d3d10shader.h +d3d11_1.h +d3d11_2.h +d3d11_3.h +d3d11_4.h +d3d11.h +d3d11sdklayers.h +d3d11shader.h +d3d8caps.h +d3d8.h +d3d8types.h +d3d9caps.h +d3d9.h +d3d9types.h +d3dcaps.h +d3dcommon.h +d3dcompiler.h +d3d.h +d3dhalex.h +d3dhal.h +d3dnthal.h +d3drmdef.h +d3drm.h +d3drmobj.h +d3dtypes.h +d3dx9anim.h +d3dx9core.h +d3dx9effect.h +d3dx9.h +d3dx9math.h +d3dx9mesh.h +d3dx9shader.h +d3dx9shape.h +d3dx9tex.h +d3dx9xof.h +d4drvif.h +d4iface.h +daogetrw.h +datapath.h +datetimeapi.h +davclnt.h +dbdaoerr.h +_dbdao.h +dbdaoid.h +dbdaoint.h +dbgautoattach.h +_dbg_common.h +dbgeng.h +dbghelp.h +_dbg_LOAD_IMAGE.h +dbgprop.h +dbt.h +dciddi.h +dciman.h +dcommon.h +dcompanimation.h +dcomp.h +dcomptypes.h +dde.h +ddeml.h +dderror.h +ddkernel.h +ddkmapi.h +ddrawgdi.h +ddraw.h +ddrawi.h +ddrawint.h +ddstream.h +debugapi.h +delayimp.h +devguid.h +devicetopology.h +devioctl.h +devpkey.h +devpropdef.h +dhcpcsdk.h +dhcpsapi.h +dhcpssdk.h +dhcpv6csdk.h +dhtmldid.h +dhtmled.h +dhtmliid.h +digitalv.h +dimm.h +dinput.h +direct.h +dirent.h +dir.h +diskguid.h +dispatch.h +dispdib.h +dispex.h +dlcapi.h +dlgs.h +dls1.h +dls2.h +dmdls.h +dmemmgr.h +dmerror.h +dmksctrl.h +dmodshow.h +dmo.h +dmoreg.h +dmort.h +dmplugin.h +dmusbuff.h +dmusicc.h +dmusicf.h +dmusici.h +dmusicks.h +dmusics.h +docobjectservice.h +docobj.h +documenttarget.h +domdid.h +dos.h +downloadmgr.h +dpaddr.h +dpapi.h +dpfilter.h +dplay8.h +dplay.h +dplobby8.h +dplobby.h +dpnathlp.h +driverspecs.h +drivinit.h +drmexternals.h +drmk.h +dsadmin.h +dsclient.h +dsconf.h +dsdriver.h +dsgetdc.h +dshow.h +dskquota.h +dsound.h +dsquery.h +dsrole.h +dssec.h +dtchelp.h +dvbsiparser.h +dvdevcod.h +dvdmedia.h +dvec.h +dvobj.h +dvp.h +dwmapi.h +dwrite_1.h +dwrite_2.h +dwrite_3.h +dwrite.h +dxapi.h +dxdiag.h +dxerr8.h +dxerr9.h +dxfile.h +dxgi1_2.h +dxgi1_3.h +dxgi1_4.h +dxgi1_5.h +dxgicommon.h +dxgiformat.h +dxgi.h +dxgitype.h +dxtmpl.h +dxva2api.h +dxva.h +dxvahd.h +eapauthenticatoractiondefine.h +eapauthenticatortypes.h +eaphosterror.h +eaphostpeerconfigapis.h +eaphostpeertypes.h +eapmethodauthenticatorapis.h +eapmethodpeerapis.h +eapmethodtypes.h +eappapis.h +eaptypes.h +edevdefs.h +eh.h +ehstorapi.h +elscore.h +emostore.h +emptyvc.h +endpointvolume.h +errhandlingapi.h +errno.h +error.h +errorrep.h +errors.h +esent.h +evcode.h +evcoll.h +eventsys.h +evntcons.h +evntprov.h +evntrace.h +evr9.h +evr.h +exchform.h +excpt.h +exdisp.h +exdispid.h +fci.h +fcntl.h +fdi.h +_fd_types.h +fenv.h +fibersapi.h +fileapi.h +fileextd.h +file.h +filehc.h +filter.h +filterr.h +float.h +fltdefs.h +fltsafe.h +fltuser.h +fltuserstructures.h +fltwinerror.h +fpieee.h +fsrmenums.h +fsrmerr.h +fsrm.h +fsrmpipeline.h +fsrmquota.h +fsrmreports.h +fsrmscreen.h +ftsiface.h +ftw.h +functiondiscoveryapi.h +functiondiscoverycategories.h +functiondiscoveryconstraints.h +functiondiscoverykeys_devpkey.h +functiondiscoverykeys.h +functiondiscoverynotification.h +fusion.h +fvec.h +fwpmtypes.h +fwpmu.h +fwptypes.h +gb18030.h +gdiplusbase.h +gdiplusbrush.h +gdipluscolor.h +gdipluscolormatrix.h +gdipluseffects.h +gdiplusenums.h +gdiplusflat.h +gdiplusgpstubs.h +gdiplusgraphics.h +gdiplus.h +gdiplusheaders.h +gdiplusimageattributes.h +gdiplusimagecodec.h +gdiplusimaging.h +gdiplusimpl.h +gdiplusinit.h +gdipluslinecaps.h +gdiplusmatrix.h +gdiplusmem.h +gdiplusmetafile.h +gdiplusmetaheader.h +gdipluspath.h +gdipluspen.h +gdipluspixelformats.h +gdiplusstringformat.h +gdiplustypes.h +getopt.h +glaux.h +glcorearb.h +glext.h +gl.h +glu.h +glxext.h +gpedit.h +gpio.h +gpmgmt.h +guiddef.h +h323priv.h +handleapi.h +heapapi.h +hidclass.h +hidpi.h +hidsdi.h +hidusage.h +highlevelmonitorconfigurationapi.h +hlguids.h +hliface.h +hlink.h +hostinfo.h +hstring.h +htiface.h +htiframe.h +htmlguid.h +htmlhelp.h +httpext.h +httpfilt.h +http.h +httprequestid.h +hubbusif.h +ia64reg.h +iaccess.h +iadmext.h +iadmw.h +iads.h +icftypes.h +icm.h +icmpapi.h +icodecapi.h +icrsint.h +i_cryptasn1tls.h +ide.h +identitycommon.h +identitystore.h +idf.h +idispids.h +iedial.h +ieeefp.h +ieverp.h +ifdef.h +iiisext.h +iiis.h +iimgctx.h +iiscnfg.h +iisrsta.h +iketypes.h +imagehlp.h +ime.h +imessage.h +imm.h +in6addr.h +inaddr.h +indexsrv.h +inetreg.h +inetsdk.h +infstr.h +initguid.h +initoid.h +inputscope.h +inspectable.h +interlockedapi.h +internal.h +intrin.h +intrin-impl.h +intsafe.h +intshcut.h +inttypes.h +invkprxy.h +ioaccess.h +ioapiset.h +ioevent.h +io.h +ipexport.h +iphlpapi.h +ipifcons.h +ipinfoid.h +ipmib.h +_ip_mreq1.h +ipmsp.h +iprtrmib.h +ipsectypes.h +_ip_types.h +iptypes.h +ipxconst.h +ipxrip.h +ipxrtdef.h +ipxsap.h +ipxtfflt.h +iscsidsc.h +isguids.h +issper16.h +issperr.h +isysmon.h +ivec.h +iwamreg.h +jobapi.h +kbdmou.h +kcom.h +knownfolders.h +ksdebug.h +ksguid.h +ks.h +ksmedia.h +ksproxy.h +ksuuids.h +ktmtypes.h +ktmw32.h +kxia64.h +l2cmn.h +libgen.h +libloaderapi.h +limits.h +lmaccess.h +lmalert.h +lmapibuf.h +lmat.h +lmaudit.h +lmconfig.h +lmcons.h +lmdfs.h +lmerr.h +lmerrlog.h +lm.h +lmjoin.h +lmmsg.h +lmon.h +lmremutl.h +lmrepl.h +lmserver.h +lmshare.h +lmsname.h +lmstats.h +lmsvc.h +lmuseflg.h +lmuse.h +lmwksta.h +loadperf.h +locale.h +locationapi.h +locking.h +lpmapi.h +lzexpand.h +madcapcl.h +magnification.h +mailmsgprops.h +malloc.h +manipulations.h +mapicode.h +mapidbg.h +mapidefs.h +mapiform.h +mapiguid.h +mapi.h +mapihook.h +mapinls.h +mapioid.h +mapispi.h +mapitags.h +mapiutil.h +mapival.h +mapiwin.h +mapiwz.h +mapix.h +math.h +mbctype.h +mbstring.h +mbstring_s.h +mcd.h +mce.h +mciavi.h +mcx.h +mdcommsg.h +mddefw.h +mdhcp.h +mdmsg.h +mediaerr.h +mediaobj.h +medparam.h +mem.h +memoryapi.h +memory.h +mergemod.h +mfapi.h +mferror.h +mfidl.h +mfmp2dlna.h +mfobjects.h +mfplay.h +mfreadwrite.h +mftransform.h +mgm.h +mgmtapi.h +midles.h +mimedisp.h +mimeinfo.h +_mingw_ddk.h +_mingw_directx.h +_mingw_dxhelper.h +_mingw_mac.h +_mingw_off_t.h +_mingw_print_pop.h +_mingw_print_push.h +_mingw_secapi.h +_mingw_stat64.h +_mingw_stdarg.h +_mingw_unicode.h +miniport.h +minitape.h +minmax.h +minwinbase.h +minwindef.h +mlang.h +mmc.h +mmcobj.h +mmdeviceapi.h +mmreg.h +mmstream.h +mmsystem.h +mobsync.h +module.h +moniker.h +mountdev.h +mountmgr.h +mpeg2bits.h +mpeg2data.h +mpeg2psiparser.h +mpeg2structs.h +mprapi.h +mprerror.h +mq.h +mqmail.h +mqoai.h +msacmdlg.h +msacm.h +msado15.h +msasn1.h +msber.h +mscat.h +mschapp.h +msclus.h +mscoree.h +msctf.h +msctfmonitorapi.h +msdadc.h +msdaguid.h +msdaipper.h +msdaipp.h +msdaora.h +msdaosp.h +msdasc.h +msdasql.h +msdatsrc.h +msdrmdefs.h +msdrm.h +msdshape.h +msfs.h +mshtmcid.h +mshtmdid.h +mshtmhst.h +mshtmlc.h +mshtml.h +msidefs.h +msi.h +msimcntl.h +msimcsdk.h +msinkaut.h +msiquery.h +msoav.h +msopc.h +mspab.h +mspaddr.h +mspbase.h +mspcall.h +mspcoll.h +mspenum.h +msp.h +msplog.h +msports.h +mspst.h +mspstrm.h +mspterm.h +mspthrd.h +msptrmac.h +msptrmar.h +msptrmvc.h +msputils.h +msrdc.h +msremote.h +mssip.h +msstkppg.h +mstask.h +mstcpip.h +msterr.h +mswsock.h +msxml2did.h +msxml2.h +msxmldid.h +msxml.h +mtsadmin.h +mtsevents.h +mtsgrp.h +mtxadmin.h +mtxattr.h +mtxdm.h +mtx.h +muiload.h +multimon.h +multinfo.h +mxdc.h +namedpipeapi.h +namespaceapi.h +napcertrelyingparty.h +napcommon.h +napenforcementclient.h +napmanagement.h +napmicrosoftvendorids.h +napprotocol.h +napservermanagement.h +napsystemhealthagent.h +napsystemhealthvalidator.h +naptypes.h +naputil.h +nb30.h +ncrypt.h +ndattrib.h +ndfapi.h +ndhelper.h +ndisguid.h +ndis.h +ndistapi.h +ndiswan.h +ndkinfo.h +ndr64types.h +ndrtypes.h +netcon.h +neterr.h +netevent.h +netfw.h +netioapi.h +netlistmgr.h +netmon.h +netpnp.h +netprov.h +nettypes.h +newapis.h +newdev.h +new.h +nldef.h +nmsupp.h +npapi.h +nsemail.h +nspapi.h +ntagp.h +ntdd1394.h +ntdd8042.h +ntddbeep.h +ntddcdrm.h +ntddcdvd.h +ntddchgr.h +ntdddisk.h +ntddft.h +ntddkbd.h +ntddk.h +ntddmmc.h +ntddmodm.h +ntddmou.h +ntddndis.h +ntddpar.h +ntddpcm.h +ntddpsch.h +ntddscsi.h +ntddser.h +ntddsnd.h +ntddstor.h +ntddtape.h +ntddtdi.h +ntddvdeo.h +ntddvol.h +ntdef.h +ntdsapi.h +ntdsbcli.h +ntdsbmsg.h +ntgdi.h +ntifs.h +ntimage.h +ntiologc.h +ntldap.h +ntmsapi.h +ntmsmli.h +ntnls.h +ntpoapi.h +ntquery.h +ntsdexts.h +ntsecapi.h +ntsecpkg.h +ntstatus.h +ntstrsafe.h +ntverp.h +oaidl.h +objbase.h +objectarray.h +objerror.h +objidlbase.h +objidl.h +objsafe.h +objsel.h +ocidl.h +ocmm.h +odbcinst.h +odbcss.h +ole2.h +ole2ver.h +oleacc.h +oleauto.h +olectl.h +olectlid.h +oledbdep.h +oledberr.h +oledbguid.h +oledb.h +oledlg.h +ole.h +oleidl.h +oletx2xa.h +opmapi.h +oprghdlr.h +optary.h +p2p.h +packoff.h +packon.h +parallel.h +param.h +parser.h +patchapi.h +patchwiz.h +pathcch.h +pbt.h +pchannel.h +pciprop.h +pcrt32.h +pdh.h +pdhmsg.h +penwin.h +perflib.h +perhist.h +persist.h +pfhook.h +pgobootrun.h +physicalmonitorenumerationapi.h +pla.h +pnrpdef.h +pnrpns.h +poclass.h +polarity.h +_pop_BOOL.h +poppack.h +portabledeviceconnectapi.h +portabledevicetypes.h +portcls.h +powrprof.h +prnasnot.h +prntfont.h +processenv.h +process.h +processthreadsapi.h +processtopologyapi.h +profileapi.h +profile.h +profinfo.h +propidl.h +propkeydef.h +propkey.h +propsys.h +propvarutil.h +prsht.h +psapi.h +pshpack1.h +pshpack2.h +pshpack4.h +pshpack8.h +pshpck16.h +pstore.h +pthread_signal.h +pthread_time.h +pthread_unistd.h +punknown.h +_push_BOOL.h +qedit.h +qmgr.h +qnetwork.h +qos2.h +qos.h +qosname.h +qospol.h +qossp.h +rasdlg.h +raseapif.h +raserror.h +ras.h +rassapi.h +rasshost.h +ratings.h +rdpencomapi.h +realtimeapiset.h +reason.h +recguids.h +reconcil.h +regbag.h +regstr.h +rend.h +resapi.h +restartmanager.h +richedit.h +richole.h +rkeysvcc.h +rnderr.h +roapi.h +routprot.h +rpcasync.h +rpcdce.h +rpcdcep.h +rpc.h +rpcndr.h +rpcnsi.h +rpcnsip.h +rpcnterr.h +rpcproxy.h +rpcsal.h +rpcssl.h +rrascfg.h +rtcapi.h +rtccore.h +rtcerr.h +rtinfo.h +rtm.h +rtmv2.h +rtutils.h +sal.h +sapi51.h +sapi53.h +sapi54.h +sapi.h +sas.h +sbe.h +scarddat.h +scarderr.h +scardmgr.h +scardsrv.h +scardssp.h +scesvc.h +schannel.h +schedule.h +schemadef.h +schnlsp.h +scode.h +scrnsave.h +scrptids.h +scsi.h +scsiscan.h +scsiwmi.h +sddl.h +sdkddkver.h +sdoias.h +sdpblb.h +sdperr.h +search.h +search_s.h +secext.h +securityappcontainer.h +securitybaseapi.h +security.h +sehmap.h +sensapi.h +sensevts.h +sens.h +sensorsapi.h +sensors.h +servprov.h +setjmpex.h +setjmp.h +setupapi.h +sfc.h +shappmgr.h +share.h +shdeprecated.h +shdispid.h +shellapi.h +sherrors.h +shfolder.h +shldisp.h +shlguid.h +shlobj.h +shlwapi.h +shobjidl.h +shtypes.h +signal.h +simpdata.h +simpdc.h +sipbase.h +sisbkup.h +slerror.h +slpublic.h +smbus.h +smpab.h +smpms.h +smpxp.h +smtpguid.h +smx.h +snmp.h +_socket_types.h +softpub.h +specstrings.h +sperror.h +sphelper.h +sporder.h +sql_1.h +sqlext.h +sql.h +sqloledb.h +sqltypes.h +sqlucode.h +srb.h +srrestoreptapi.h +srv.h +sspguid.h +sspi.h +sspserr.h +sspsidl.h +stat.h +stdarg.h +stddef.h +stdexcpt.h +stdint.h +stdio.h +stdio_s.h +stdlib.h +stdlib_s.h +stdunk.h +stierr.h +sti.h +stireg.h +stllock.h +stm.h +storage.h +storduid.h +storport.h +storprop.h +stralign.h +stralign_s.h +stringapiset.h +string.h +string_s.h +strings.h +strmif.h +strmini.h +strsafe.h +structuredquerycondition.h +subauth.h +subsmgr.h +svcguid.h +svrapi.h +swenum.h +synchapi.h +sysinfoapi.h +syslimits.h +systemtopologyapi.h +t2embapi.h +tabflicks.h +tapi3cc.h +tapi3ds.h +tapi3err.h +tapi3.h +tapi3if.h +tapi.h +taskschd.h +tbs.h +tcerror.h +tcguid.h +tchar.h +tchar_s.h +tcpestats.h +tcpmib.h +tdh.h +tdi.h +tdiinfo.h +tdikrnl.h +tdistat.h +termmgr.h +textserv.h +textstor.h +threadpoolapiset.h +threadpoollegacyapiset.h +timeb.h +timeb_s.h +time.h +timeprov.h +_timeval.h +timezoneapi.h +tlbref.h +tlhelp32.h +tlogstg.h +tmschema.h +tnef.h +tom.h +tpcshrd.h +traffic.h +transact.h +triedcid.h +triediid.h +triedit.h +tsattrs.h +tspi.h +tssbx.h +tsuserex.h +tuner.h +tvout.h +txcoord.h +txctx.h +txdtc.h +txfw32.h +typeinfo.h +types.h +uastrfnc.h +uchar.h +udpmib.h +uianimation.h +uiautomationclient.h +uiautomationcoreapi.h +uiautomationcore.h +uiautomation.h +uiviewsettingsinterop.h +umx.h +unistd.h +unknown.h +unknwnbase.h +unknwn.h +upssvc.h +urlhist.h +urlmon.h +usb100.h +usb200.h +usbbusif.h +usbcamdi.h +usbdi.h +usbdlib.h +usbdrivr.h +usb.h +usbioctl.h +usbiodef.h +usbkern.h +usbprint.h +usbprotocoldefs.h +usbrpmif.h +usbscan.h +usbspec.h +usbstorioctl.h +usbuser.h +userenv.h +usp10.h +utilapiset.h +utime.h +uuids.h +uxtheme.h +vadefs.h +varargs.h +_varenum.h +vcr.h +vdmdbg.h +vds.h +vdslun.h +versionhelpers.h +vfw.h +vfwmsgs.h +videoagp.h +video.h +virtdisk.h +vmr9.h +vsadmin.h +vsbackup.h +vsmgmt.h +vsprov.h +vss.h +vsstyle.h +vssym32.h +vswriter.h +w32api.h +wabapi.h +wabcode.h +wabdefs.h +wab.h +wabiab.h +wabmem.h +wabnot.h +wabtags.h +wabutil.h +wbemads.h +wbemcli.h +wbemdisp.h +wbemidl.h +wbemprov.h +wbemtran.h +wchar.h +wchar_s.h +wcmconfig.h +wcsplugin.h +wct.h +wctype.h +wdmguid.h +wdm.h +wdsbp.h +wdsclientapi.h +wdspxe.h +wdstci.h +wdstpdi.h +wdstptmgmt.h +werapi.h +wfext.h +wglext.h +wiadef.h +wiadevd.h +wia.h +wiavideo.h +winable.h +winapifamily.h +winbase.h +winber.h +wincodec.h +wincon.h +wincred.h +wincrypt.h +winddi.h +winddiui.h +windef.h +windns.h +windot11.h +windows.foundation.h +windows.h +windows.security.cryptography.h +windows.storage.h +windows.storage.streams.h +windows.system.threading.h +windowsx.h +winefs.h +winerror.h +winevt.h +wingdi.h +winhttp.h +wininet.h +winineti.h +winioctl.h +winldap.h +winnetwk.h +winnls32.h +winnls.h +winnt.h +winperf.h +winreg.h +winresrc.h +winsafer.h +winsatcominterfacei.h +winscard.h +winsdkver.h +winsmcrd.h +winsnmp.h +winsock2.h +winsock.h +winsplp.h +winspool.h +winstring.h +winsvc.h +winsxs.h +winsync.h +winternl.h +wintrust.h +winusb.h +winusbio.h +winuser.h +winver.h +winwlx.h +wlanapi.h +wlanihvtypes.h +wlantypes.h +wmcodecdsp.h +wmcontainer.h +wmdrmsdk.h +wmiatlprov.h +wmidata.h +wmilib.h +wmistr.h +wmiutils.h +wmsbuffer.h +wmsdkidl.h +wnnc.h +wow64apiset.h +wownt16.h +wownt32.h +wpapi.h +wpapimsg.h +wpcapi.h +wpcevent.h +wpcrsmsg.h +wpftpmsg.h +wppstmsg.h +wpspihlp.h +wptypes.h +wpwizmsg.h +wrl.h +_ws1_undef.h +ws2atm.h +ws2bth.h +ws2def.h +ws2dnet.h +ws2ipdef.h +ws2san.h +ws2spi.h +ws2tcpip.h +_wsadata.h +_wsa_errnos.h +wsdapi.h +wsdattachment.h +wsdbase.h +wsdclient.h +wsddisco.h +wsdhost.h +wsdtypes.h +wsdutil.h +wsdxmldom.h +wsdxml.h +wshisotp.h +wsipv6ok.h +wsipx.h +wsmandisp.h +wsman.h +wsnetbs.h +wsnwlink.h +wspiapi.h +wsrm.h +wsvns.h +wtsapi32.h +wtypesbase.h +wtypes.h +xa.h +xcmcext.h +xcmc.h +xcmcmsx2.h +xcmcmsxt.h +xenroll.h +xfilter.h +xinput.h +xlocinfo.h +xmath.h +_xmitfile.h +xmldomdid.h +xmldsodid.h +xmllite.h +xmltrnsf.h +xolehlp.h +xpsdigitalsignature.h +xpsobjectmodel_1.h +xpsobjectmodel.h +xpsprint.h +xpsrassvc.h +ymath.h +yvals.h +zmouse.h diff --git a/tools/lint/eslint.yml b/tools/lint/eslint.yml new file mode 100644 index 0000000000..6946ef672a --- /dev/null +++ b/tools/lint/eslint.yml @@ -0,0 +1,32 @@ +--- +eslint: + description: JavaScript linter + # ESLint infra handles its own path filtering, so just include cwd + include: ['.'] + exclude: [] + # When adding to this list, consider updating hooks_js_format.py as well. + extensions: ['mjs', 'js', 'jsm', 'json', 'jsx', 'html', 'sjs', 'xhtml'] + support-files: + - '**/.eslintrc.js' + - '.eslintrc-test-paths.js' + - '.eslintignore' + - 'tools/lint/eslint/**' + # Files that can influence global variables + - 'browser/base/content/nsContextMenu.js' + - 'browser/base/content/utilityOverlay.js' + - 'browser/components/customizableui/content/panelUI.js' + - 'browser/components/downloads/content/downloads.js' + - 'browser/components/downloads/content/indicator.js' + - 'testing/mochitest/tests/SimpleTest/EventUtils.js' + - 'testing/mochitest/tests/SimpleTest/MockObjects.js' + - 'testing/mochitest/tests/SimpleTest/SimpleTest.js' + - 'testing/mochitest/tests/SimpleTest/WindowSnapshot.js' + - 'toolkit/components/printing/content/printUtils.js' + - 'toolkit/components/viewsource/content/viewSourceUtils.js' + - 'toolkit/content/contentAreaUtils.js' + - 'toolkit/content/editMenuOverlay.js' + - 'toolkit/content/globalOverlay.js' + - 'toolkit/modules/Services.jsm' + type: external + payload: eslint:lint + setup: eslint:setup diff --git a/tools/lint/eslint/.eslintrc.js b/tools/lint/eslint/.eslintrc.js new file mode 100644 index 0000000000..4a245d9de2 --- /dev/null +++ b/tools/lint/eslint/.eslintrc.js @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + plugins: ["eslint-plugin"], + extends: ["plugin:eslint-plugin/recommended"], + // eslint-plugin-mozilla runs under node, so we need a more restrictive + // environment / parser setup here than the rest of mozilla-central. + env: { + browser: false, + node: true, + }, + parser: "espree", + parserOptions: { + ecmaVersion: 12, + }, + + rules: { + camelcase: ["error", { properties: "never" }], + "handle-callback-err": ["error", "er"], + "no-shadow": "error", + "no-undef-init": "error", + "one-var": ["error", "never"], + strict: ["error", "global"], + "eslint-plugin/prefer-message-ids": "off", + }, +}; diff --git a/tools/lint/eslint/__init__.py b/tools/lint/eslint/__init__.py new file mode 100644 index 0000000000..0bd320d691 --- /dev/null +++ b/tools/lint/eslint/__init__.py @@ -0,0 +1,280 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import signal +import subprocess +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "eslint")) +from mozbuild.nodeutil import find_node_executable +from mozlint import result + +from eslint import setup_helper + +ESLINT_ERROR_MESSAGE = """ +An error occurred running eslint. Please check the following error messages: + +{} +""".strip() + +ESLINT_NOT_FOUND_MESSAGE = """ +Could not find eslint! We looked at the --binary option, at the ESLINT +environment variable, and then at your local node_modules path. Please Install +eslint and needed plugins with: + +mach eslint --setup + +and try again. +""".strip() + +PRETTIER_ERROR_MESSAGE = """ +An error occurred running prettier. Please check the following error messages: + +{} +""".strip() + +PRETTIER_FORMATTING_MESSAGE = ( + "This file needs formatting with Prettier (use 'mach lint --fix ')." +) + + +def setup(root, **lintargs): + setup_helper.set_project_root(root) + + if not setup_helper.check_node_executables_valid(): + return 1 + + return setup_helper.eslint_maybe_setup() + + +def lint(paths, config, binary=None, fix=None, rules=[], setup=None, **lintargs): + """Run eslint.""" + log = lintargs["log"] + setup_helper.set_project_root(lintargs["root"]) + module_path = setup_helper.get_project_root() + + # Valid binaries are: + # - Any provided by the binary argument. + # - Any pointed at by the ESLINT environmental variable. + # - Those provided by |mach lint --setup|. + + if not binary: + binary, _ = find_node_executable() + + if not binary: + print(ESLINT_NOT_FOUND_MESSAGE) + return 1 + + extra_args = lintargs.get("extra_args") or [] + exclude_args = [] + for path in config.get("exclude", []): + exclude_args.extend( + ["--ignore-pattern", os.path.relpath(path, lintargs["root"])] + ) + + for rule in rules: + extra_args.extend(["--rule", rule]) + + # First run ESLint + cmd_args = ( + [ + binary, + os.path.join(module_path, "node_modules", "eslint", "bin", "eslint.js"), + # This keeps ext as a single argument. + "--ext", + "[{}]".format(",".join(config["extensions"])), + "--format", + "json", + "--no-error-on-unmatched-pattern", + ] + + rules + + extra_args + + exclude_args + + paths + ) + + if fix: + # eslint requires that --fix be set before the --ext argument. + cmd_args.insert(2, "--fix") + + log.debug("ESLint command: {}".format(" ".join(cmd_args))) + + result = run(cmd_args, config) + if result == 1: + return result + + # Then run Prettier + cmd_args = ( + [ + binary, + os.path.join(module_path, "node_modules", "prettier", "bin-prettier.js"), + "--list-different", + "--no-error-on-unmatched-pattern", + ] + + extra_args + # Prettier does not support exclude arguments. + # + exclude_args + + paths + ) + log.debug("Prettier command: {}".format(" ".join(cmd_args))) + + if fix: + cmd_args.append("--write") + + prettier_result = run_prettier(cmd_args, config, fix) + if prettier_result == 1: + return prettier_result + + result["results"].extend(prettier_result["results"]) + result["fixed"] = result["fixed"] + prettier_result["fixed"] + return result + + +def run(cmd_args, config): + + shell = False + if ( + os.environ.get("MSYSTEM") in ("MINGW32", "MINGW64") + or "MOZILLABUILD" in os.environ + ): + # The eslint binary needs to be run from a shell with msys + shell = True + encoding = "utf-8" + + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = subprocess.Popen( + cmd_args, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + signal.signal(signal.SIGINT, orig) + + try: + output, errors = proc.communicate() + except KeyboardInterrupt: + proc.kill() + return {"results": [], "fixed": 0} + + if errors: + errors = errors.decode(encoding, "replace") + print(ESLINT_ERROR_MESSAGE.format(errors)) + + if proc.returncode >= 2: + return 1 + + if not output: + return {"results": [], "fixed": 0} # no output means success + output = output.decode(encoding, "replace") + try: + jsonresult = json.loads(output) + except ValueError: + print(ESLINT_ERROR_MESSAGE.format(output)) + return 1 + + results = [] + fixed = 0 + for obj in jsonresult: + errors = obj["messages"] + # This will return a count of files fixed, rather than issues fixed, as + # that is the only count we have. + if "output" in obj: + fixed = fixed + 1 + + for err in errors: + err.update( + { + "hint": err.get("fix"), + "level": "error" if err["severity"] == 2 else "warning", + "lineno": err.get("line") or 0, + "path": obj["filePath"], + "rule": err.get("ruleId"), + } + ) + results.append(result.from_config(config, **err)) + + return {"results": results, "fixed": fixed} + + +def run_prettier(cmd_args, config, fix): + shell = False + if is_windows(): + # The eslint binary needs to be run from a shell with msys + shell = True + encoding = "utf-8" + + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = subprocess.Popen( + cmd_args, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + signal.signal(signal.SIGINT, orig) + + try: + output, errors = proc.communicate() + except KeyboardInterrupt: + proc.kill() + return {"results": [], "fixed": 0} + + if errors: + errors = errors.decode(encoding, "replace").strip().split("\n") + errors = [ + error + for error in errors + # Unknown options are not an issue for Prettier, this avoids + # errors during tests. + if not ("Ignored unknown option" in error) + ] + if len(errors): + print(PRETTIER_ERROR_MESSAGE.format("\n".join(errors))) + + if not output: + # If we have errors, but no output, we assume something really bad happened. + if errors and len(errors): + return 1 + + return {"results": [], "fixed": 0} # no output means success + + output = output.decode(encoding, "replace").splitlines() + + results = [] + fixed = 0 + + if fix: + # When Prettier is running in fix mode, it outputs the list of files + # that have been fixed, so sum them up here. + # If it can't fix files, it will throw an error, which will be handled + # above. + fixed = len(output) + else: + # When in "check" mode, Prettier will output the list of files that + # need changing, so we'll wrap them in our result structure here. + for file in output: + if not file: + continue + + file = os.path.abspath(file) + results.append( + result.from_config( + config, + **{ + "name": "eslint", + "path": file, + "message": PRETTIER_FORMATTING_MESSAGE, + "level": "error", + "rule": "prettier", + "lineno": 0, + "column": 0, + } + ) + ) + + return {"results": results, "fixed": fixed} + + +def is_windows(): + return ( + os.environ.get("MSYSTEM") in ("MINGW32", "MINGW64") + or "MOZILLABUILD" in os.environ + ) diff --git a/tools/lint/eslint/eslint-plugin-mozilla/.npmignore b/tools/lint/eslint/eslint-plugin-mozilla/.npmignore new file mode 100644 index 0000000000..3713448c7a --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/.npmignore @@ -0,0 +1,8 @@ +.eslintrc.js +.npmignore +node_modules +reporters +scripts +tests +package-lock.json +update.sh diff --git a/tools/lint/eslint/eslint-plugin-mozilla/LICENSE b/tools/lint/eslint/eslint-plugin-mozilla/LICENSE new file mode 100644 index 0000000000..e87a115e46 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/tools/lint/eslint/eslint-plugin-mozilla/README.md b/tools/lint/eslint/eslint-plugin-mozilla/README.md new file mode 100644 index 0000000000..650507754e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/README.md @@ -0,0 +1,56 @@ +# eslint-plugin-mozilla + +A collection of rules that help enforce JavaScript coding standard in the Mozilla project. + +These are primarily developed and used within the Firefox build system ([mozilla-central](https://hg.mozilla.org/mozilla-central/)), but are made available for other +related projects to use as well. + +## Installation + +### Within mozilla-central: + +``` +$ ./mach eslint --setup +``` + +### Outside mozilla-central: + +Install ESLint [ESLint](http://eslint.org): + +``` +$ npm i eslint --save-dev +``` + +Next, install `eslint-plugin-mozilla`: + +``` +$ npm install eslint-plugin-mozilla --save-dev +``` + +## Documentation + +For details about the rules, please see the [firefox documentation page](http://firefox-source-docs.mozilla.org/tools/lint/linters/eslint-plugin-mozilla.html). + +## Source Code + +The sources can be found at: + +* Code: https://searchfox.org/mozilla-central/source/tools/lint/eslint/eslint-plugin-mozilla +* Documentation: https://searchfox.org/mozilla-central/source/docs/code-quality/lint/linters + +## Bugs + +Please file bugs in Bugzilla in the Lint component of the Testing product. + +* [Existing bugs](https://bugzilla.mozilla.org/buglist.cgi?resolution=---&query_format=advanced&component=Lint&product=Testing) +* [New bugs](https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=Lint) + +## Tests + +The tests can only be run from within mozilla-central. To run the tests: + +``` +./mach eslint --setup +cd tools/lint/eslint/eslint-plugin-mozilla +npm run test +``` diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/.eslintrc.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/.eslintrc.js new file mode 100644 index 0000000000..76df4134f5 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/.eslintrc.js @@ -0,0 +1,8 @@ +"use strict"; + +module.exports = { + rules: { + // Require object keys to be sorted. + "sort-keys": "error", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/browser-test.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/browser-test.js new file mode 100644 index 0000000000..28b51f9ed3 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/browser-test.js @@ -0,0 +1,91 @@ +// Parent config file for all browser-chrome files. +"use strict"; + +module.exports = { + env: { + browser: true, + "mozilla/browser-window": true, + "mozilla/simpletest": true, + // "node": true + }, + + // All globals made available in the test environment. + globals: { + // `$` is defined in SimpleTest.js + $: false, + Assert: false, + BrowserTestUtils: false, + ContentTask: false, + ContentTaskUtils: false, + EventUtils: false, + IOUtils: false, + PathUtils: false, + PromiseDebugging: false, + SpecialPowers: false, + TestUtils: false, + addLoadEvent: false, + add_setup: false, + add_task: false, + content: false, + executeSoon: false, + expectUncaughtException: false, + export_assertions: false, + extractJarToTmp: false, + finish: false, + gTestPath: false, + getChromeDir: false, + getJar: false, + getResolvedURI: false, + getRootDirectory: false, + getTestFilePath: false, + ignoreAllUncaughtExceptions: false, + info: false, + is: false, + isnot: false, + ok: false, + record: false, + registerCleanupFunction: false, + requestLongerTimeout: false, + setExpectedFailuresForSelfTest: false, + stringContains: false, + stringMatches: false, + todo: false, + todo_is: false, + todo_isnot: false, + waitForClipboard: false, + waitForExplicitFinish: false, + waitForFocus: false, + }, + + plugins: ["mozilla", "@microsoft/sdl"], + + rules: { + // No using of insecure url, so no http urls + "@microsoft/sdl/no-insecure-url": [ + "error", + { + exceptions: [ + "^http:\\/\\/mochi\\.test?.*", + "^http:\\/\\/localhost?.*", + "^http:\\/\\/127\\.0\\.0\\.1?.*", + // Exempt xmlns urls + "^http:\\/\\/www\\.w3\\.org?.*", + "^http:\\/\\/www\\.mozilla\\.org\\/keymaster\\/gatekeeper?.*", + // Exempt urls that start with ftp or ws. + "^ws:?.*", + "^ftp:?.*", + ], + varExceptions: ["insecure?.*"], + }, + ], + "mozilla/import-content-task-globals": "error", + "mozilla/import-headjs-globals": "error", + "mozilla/mark-test-function-used": "error", + "mozilla/no-addtask-setup": "error", + "mozilla/no-arbitrary-setTimeout": "error", + "mozilla/no-redeclare-with-import-autofix": [ + "error", + { errorForNonImports: false }, + ], + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/chrome-test.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/chrome-test.js new file mode 100644 index 0000000000..fa1e8d6190 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/chrome-test.js @@ -0,0 +1,61 @@ +// Parent config file for all mochitest files. +"use strict"; + +module.exports = { + env: { + browser: true, + "mozilla/browser-window": true, + }, + + // All globals made available in the test environment. + globals: { + // SpecialPowers is injected into the window object via SimpleTest.js + SpecialPowers: false, + extractJarToTmp: false, + getChromeDir: false, + getJar: false, + getResolvedURI: false, + getRootDirectory: false, + }, + + overrides: [ + { + env: { + // Ideally we wouldn't be using the simpletest env here, but our uses of + // js files mean we pick up everything from the global scope, which could + // be any one of a number of html files. So we just allow the basics... + "mozilla/simpletest": true, + }, + files: ["*.js"], + }, + ], + + plugins: ["mozilla", "@microsoft/sdl"], + + rules: { + // No using of insecure url, so no http urls + "@microsoft/sdl/no-insecure-url": [ + "error", + { + exceptions: [ + "^http:\\/\\/mochi\\.test?.*", + "^http:\\/\\/localhost?.*", + "^http:\\/\\/127\\.0\\.0\\.1?.*", + // Exempt xmlns urls + "^http:\\/\\/www\\.w3\\.org?.*", + "^http:\\/\\/www\\.mozilla\\.org\\/keymaster\\/gatekeeper?.*", + // Exempt urls that start with ftp or ws. + "^ws:?.*", + "^ftp:?.*", + ], + varExceptions: ["insecure?.*"], + }, + ], + "mozilla/import-content-task-globals": "error", + "mozilla/import-headjs-globals": "error", + "mozilla/mark-test-function-used": "error", + // We mis-predict globals for HTML test files in directories shared + // with browser tests. + "mozilla/no-redeclare-with-import-autofix": "off", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/mochitest-test.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/mochitest-test.js new file mode 100644 index 0000000000..3f72193d4d --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/mochitest-test.js @@ -0,0 +1,59 @@ +// Parent config file for all mochitest files. +"use strict"; + +module.exports = { + env: { + browser: true, + }, + + // All globals made available in the test environment. + globals: { + // SpecialPowers is injected into the window object via SimpleTest.js + SpecialPowers: false, + }, + + overrides: [ + { + env: { + // Ideally we wouldn't be using the simpletest env here, but our uses of + // js files mean we pick up everything from the global scope, which could + // be any one of a number of html files. So we just allow the basics... + "mozilla/simpletest": true, + }, + files: ["*.js"], + }, + ], + plugins: ["mozilla", "@microsoft/sdl"], + + rules: { + // No using of insecure url, so no http urls + "@microsoft/sdl/no-insecure-url": [ + "error", + { + exceptions: [ + "^http:\\/\\/mochi\\.test?.*", + "^http:\\/\\/mochi\\.xorigin-test?.*", + "^http:\\/\\/localhost?.*", + "^http:\\/\\/127\\.0\\.0\\.1?.*", + // Exempt xmlns urls + "^http:\\/\\/www\\.w3\\.org?.*", + "^http:\\/\\/www\\.mozilla\\.org\\/keymaster\\/gatekeeper?.*", + // Exempt urls that start with ftp or ws. + "^ws:?.*", + "^ftp:?.*", + ], + varExceptions: ["insecure?.*"], + }, + ], + "mozilla/import-content-task-globals": "error", + "mozilla/import-headjs-globals": "error", + "mozilla/mark-test-function-used": "error", + // Turn off no-define-cc-etc for mochitests as these don't have Cc etc defined in the + // global scope. + "mozilla/no-define-cc-etc": "off", + // We mis-predict globals for HTML test files in directories shared + // with browser tests, so don't try to "fix" imports that are needed. + "mozilla/no-redeclare-with-import-autofix": "off", + "no-shadow": "error", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js new file mode 100644 index 0000000000..a7d8c067f0 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/recommended.js @@ -0,0 +1,356 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +/** + * The configuration is based on eslint:recommended config. The details for all + * the ESLint rules, and which ones are in the recommended configuration can + * be found here: + * + * https://eslint.org/docs/rules/ + */ +module.exports = { + env: { + browser: true, + es2021: true, + "mozilla/privileged": true, + "mozilla/specific": true, + }, + + // The prettier configuration here comes from eslint-config-prettier and + // turns off all of ESLint's rules related to formatting. + extends: ["eslint:recommended", "prettier"], + + overrides: [ + { + // System mjs files and jsm files are not loaded in the browser scope, + // so we turn that off for those. Though we do have our own special + // environment for them. + env: { + browser: false, + "mozilla/jsm": true, + }, + files: ["**/*.sys.mjs", "**/*.jsm", "**/*.jsm.js"], + rules: { + "mozilla/lazy-getter-object-name": "error", + "mozilla/reject-eager-module-in-lazy-getter": "error", + "mozilla/reject-global-this": "error", + "mozilla/reject-globalThis-modification": "error", + // For all system modules, we expect no properties to need importing, + // hence reject everything. + "mozilla/reject-importGlobalProperties": ["error", "everything"], + "mozilla/reject-mixing-eager-and-lazy": "error", + "mozilla/reject-top-level-await": "error", + // TODO: Bug 1575506 turn `builtinGlobals` on here. + // We can enable builtinGlobals for jsms due to their scopes. + "no-redeclare": ["error", { builtinGlobals: false }], + }, + }, + { + files: ["**/*.mjs", "**/*.jsm"], + rules: { + // Modules are far easier to check for no-unused-vars on a global scope, + // than our content files. Hence we turn that on here. + "no-unused-vars": [ + "error", + { + args: "none", + vars: "all", + }, + ], + }, + }, + { + files: ["**/*.sys.mjs"], + rules: { + "mozilla/use-static-import": "error", + }, + }, + { + excludedFiles: ["**/*.sys.mjs"], + files: ["**/*.mjs"], + rules: { + "mozilla/reject-import-system-module-from-non-system": "error", + "mozilla/reject-lazy-imports-into-globals": "error", + "no-shadow": ["error", { allow: ["event"], builtinGlobals: true }], + }, + }, + { + files: ["**/*.mjs"], + rules: { + "mozilla/use-static-import": "error", + // This rule defaults to not allowing "use strict" in module files since + // they are always loaded in strict mode. + strict: "error", + }, + }, + { + files: ["**/*.jsm", "**/*.jsm.js"], + rules: { + "mozilla/mark-exported-symbols-as-used": "error", + }, + }, + { + env: { + browser: false, + "mozilla/privileged": false, + "mozilla/sjs": true, + }, + files: ["**/*.sjs"], + rules: { + // TODO Bug 1501127: sjs files have their own sandbox, and do not inherit + // the Window backstage pass directly. Turn this rule off for sjs files for + // now until we develop a solution. + "mozilla/reject-importGlobalProperties": "off", + }, + }, + ], + + parserOptions: { + ecmaVersion: 12, + }, + + // When adding items to this file please check for effects on sub-directories. + plugins: ["html", "fetch-options", "no-unsanitized"], + + // When adding items to this file please check for effects on all of toolkit + // and browser + rules: { + // This may conflict with prettier, so we turn it off. + "arrow-body-style": "off", + + // Warn about cyclomatic complexity in functions. + // XXX Get this down to 20? + complexity: ["error", 34], + + // Functions must always return something or nothing + "consistent-return": "error", + + // XXX This rule line should be removed to enable it. See bug 1487642. + // Require super() calls in constructors + "constructor-super": "off", + + // Require braces around blocks that start a new line + curly: ["error", "all"], + + // Encourage the use of dot notation whenever possible. + "dot-notation": "error", + + // XXX This rule should be enabled, see Bug 1557040 + // No credentials submitted with fetch calls + "fetch-options/no-fetch-credentials": "off", + + // XXX This rule line should be removed to enable it. See bug 1487642. + // Enforce return statements in getters + "getter-return": "off", + + // Don't enforce the maximum depth that blocks can be nested. The complexity + // rule is a better rule to check this. + "max-depth": "off", + + // Maximum depth callbacks can be nested. + "max-nested-callbacks": ["error", 10], + + "mozilla/avoid-removeChild": "error", + "mozilla/consistent-if-bracing": "error", + "mozilla/import-browser-window-globals": "error", + "mozilla/import-globals": "error", + "mozilla/no-compare-against-boolean-literals": "error", + "mozilla/no-cu-reportError": "error", + "mozilla/no-define-cc-etc": "error", + "mozilla/no-throw-cr-literal": "error", + "mozilla/no-useless-parameters": "error", + "mozilla/no-useless-removeEventListener": "error", + "mozilla/prefer-boolean-length-check": "error", + "mozilla/prefer-formatValues": "error", + "mozilla/reject-addtask-only": "error", + "mozilla/reject-chromeutils-import-params": "error", + "mozilla/reject-importGlobalProperties": ["error", "allownonwebidl"], + "mozilla/reject-multiple-getters-calls": "error", + "mozilla/reject-scriptableunicodeconverter": "warn", + "mozilla/rejects-requires-await": "error", + "mozilla/use-cc-etc": "error", + "mozilla/use-chromeutils-generateqi": "error", + "mozilla/use-chromeutils-import": "error", + "mozilla/use-default-preference-values": "error", + "mozilla/use-includes-instead-of-indexOf": "error", + "mozilla/use-isInstance": "error", + "mozilla/use-ownerGlobal": "error", + "mozilla/use-returnValue": "error", + "mozilla/use-services": "error", + "mozilla/valid-lazy": "error", + "mozilla/valid-services": "error", + + // Use [] instead of Array() + "no-array-constructor": "error", + + // Disallow use of arguments.caller or arguments.callee. + "no-caller": "error", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Disallow lexical declarations in case clauses + "no-case-declarations": "off", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Disallow the use of console + "no-console": "off", + + // Disallows expressions where the operation doesn't affect the value. + "no-constant-binary-expression": "error", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Disallow constant expressions in conditions + "no-constant-condition": "off", + + // No duplicate keys in object declarations + "no-dupe-keys": "error", + + // If an if block ends with a return no need for an else block + "no-else-return": "error", + + // No empty statements + "no-empty": ["error", { allowEmptyCatch: true }], + + // Disallow eval and setInteral/setTimeout with strings + "no-eval": "error", + + // Disallow unnecessary calls to .bind() + "no-extra-bind": "error", + + // Disallow fallthrough of case statements + "no-fallthrough": [ + "error", + { + // The eslint rule doesn't allow for case-insensitive regex option. + // The following pattern allows for a dash between "fall through" as + // well as alternate spelling of "fall thru". The pattern also allows + // for an optional "s" at the end of "fall" ("falls through"). + commentPattern: + "[Ff][Aa][Ll][Ll][Ss]?[\\s-]?([Tt][Hh][Rr][Oo][Uu][Gg][Hh]|[Tt][Hh][Rr][Uu])", + }, + ], + + // Disallow assignments to native objects or read-only global variables + "no-global-assign": "error", + + // Disallow eval and setInteral/setTimeout with strings + "no-implied-eval": "error", + + // This has been superseded since we're using ES6. + // Disallow variable or function declarations in nested blocks + "no-inner-declarations": "off", + + // Disallow the use of the __iterator__ property + "no-iterator": "error", + + // No labels + "no-labels": "error", + + // Disallow unnecessary nested blocks + "no-lone-blocks": "error", + + // No single if block inside an else block + "no-lonely-if": "error", + + // Disallow the use of number literals that immediately lose precision at runtime when converted to JS Number + "no-loss-of-precision": "error", + + // Nested ternary statements are confusing + "no-nested-ternary": "error", + + // Use {} instead of new Object() + "no-new-object": "error", + + // Disallow use of new wrappers + "no-new-wrappers": "error", + + // We don't want this, see bug 1551829 + "no-prototype-builtins": "off", + + // Disable builtinGlobals for no-redeclare as this conflicts with our + // globals declarations especially for browser window. + "no-redeclare": ["error", { builtinGlobals: false }], + + // Disallow use of event global. + "no-restricted-globals": ["error", "event"], + + // Disallows unnecessary `return await ...`. + "no-return-await": "error", + + // No unnecessary comparisons + "no-self-compare": "error", + + // No comma sequenced statements + "no-sequences": "error", + + // No declaring variables from an outer scope + // "no-shadow": "error", + + // No declaring variables that hide things like arguments + "no-shadow-restricted-names": "error", + + // Disallow throwing literals (eg. throw "error" instead of + // throw new Error("error")). + "no-throw-literal": "error", + + // Disallow the use of Boolean literals in conditional expressions. + "no-unneeded-ternary": "error", + + // No unsanitized use of innerHTML=, document.write() etc. + // cf. https://github.com/mozilla/eslint-plugin-no-unsanitized#rule-details + "no-unsanitized/method": "error", + "no-unsanitized/property": "error", + + // No declaring variables that are never used + "no-unused-vars": [ + "error", + { + args: "none", + vars: "local", + }, + ], + + // No using variables before defined + // "no-use-before-define": ["error", "nofunc"], + + // Disallow unnecessary .call() and .apply() + "no-useless-call": "error", + + // Don't concatenate string literals together (unless they span multiple + // lines) + "no-useless-concat": "error", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Disallow unnecessary escape characters + "no-useless-escape": "off", + + // Disallow redundant return statements + "no-useless-return": "error", + + // No using with + "no-with": "error", + + // Require object-literal shorthand with ES6 method syntax + "object-shorthand": ["error", "always", { avoidQuotes: true }], + + // This may conflict with prettier, so turn it off. + "prefer-arrow-callback": "off", + + // This generates too many false positives that are not easy to work around, + // and false positives seem to be inherent in the rule. + "require-atomic-updates": "off", + + // XXX Bug 1487642 - decide if we want to enable this or not. + // Require generator functions to contain yield + "require-yield": "off", + }, + + // To avoid bad interactions of the html plugin with the xml preprocessor in + // eslint-plugin-mozilla, we turn off processing of the html plugin for .xml + // files. + settings: { + "html/xml-extensions": [".xhtml"], + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/require-jsdoc.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/require-jsdoc.js new file mode 100644 index 0000000000..0a037ab159 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/require-jsdoc.js @@ -0,0 +1,32 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + plugins: ["jsdoc"], + + rules: { + "jsdoc/require-jsdoc": [ + "error", + { + require: { + ClassDeclaration: true, + FunctionDeclaration: false, + }, + }, + ], + "jsdoc/require-param": "error", + "jsdoc/require-param-description": "error", + "jsdoc/require-param-name": "error", + "jsdoc/require-property": "error", + "jsdoc/require-property-description": "error", + "jsdoc/require-property-name": "error", + "jsdoc/require-property-type": "error", + "jsdoc/require-returns": "error", + "jsdoc/require-returns-check": "error", + "jsdoc/require-yields": "error", + "jsdoc/require-yields-check": "error", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/valid-jsdoc.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/valid-jsdoc.js new file mode 100644 index 0000000000..485dddfbd4 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/valid-jsdoc.js @@ -0,0 +1,25 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + plugins: ["jsdoc"], + + rules: { + "jsdoc/check-access": "error", + // Handled by prettier + // "jsdoc/check-alignment": "error", + "jsdoc/check-param-names": "error", + "jsdoc/check-property-names": "error", + "jsdoc/check-tag-names": "error", + "jsdoc/check-types": "error", + "jsdoc/empty-tags": "error", + "jsdoc/newline-after-description": "error", + "jsdoc/no-multi-asterisks": "error", + "jsdoc/require-param-type": "error", + "jsdoc/require-returns-type": "error", + "jsdoc/valid-types": "error", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/xpcshell-test.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/xpcshell-test.js new file mode 100644 index 0000000000..492ecc1ae9 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/configs/xpcshell-test.js @@ -0,0 +1,50 @@ +// Parent config file for all xpcshell files. +"use strict"; + +module.exports = { + env: { + browser: false, + "mozilla/privileged": true, + "mozilla/xpcshell": true, + }, + + overrides: [ + { + // If it is a head file, we turn off global unused variable checks, as it + // would require searching the other test files to know if they are used or not. + // This would be expensive and slow, and it isn't worth it for head files. + // We could get developers to declare as exported, but that doesn't seem worth it. + files: "head*.js", + rules: { + "no-unused-vars": [ + "error", + { + args: "none", + vars: "local", + }, + ], + }, + }, + { + // No declaring variables that are never used + files: "test*.js", + rules: { + "no-unused-vars": [ + "error", + { + args: "none", + vars: "all", + }, + ], + }, + }, + ], + + rules: { + "mozilla/import-headjs-globals": "error", + "mozilla/mark-test-function-used": "error", + "mozilla/no-arbitrary-setTimeout": "error", + "mozilla/no-useless-run-test": "error", + "no-shadow": "error", + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/browser-window.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/browser-window.js new file mode 100644 index 0000000000..01f7d512b6 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/browser-window.js @@ -0,0 +1,119 @@ +/** + * @fileoverview Defines the environment when in the browser.xhtml window. + * Imports many globals from various files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var fs = require("fs"); +var helpers = require("../helpers"); +var { getScriptGlobals } = require("./utils"); + +// When updating EXTRA_SCRIPTS or MAPPINGS, be sure to also update the +// 'support-files' config in `tools/lint/eslint.yml`. + +// These are scripts not loaded from browser.xhtml or global-scripts.inc +// but via other includes. +const EXTRA_SCRIPTS = [ + "browser/base/content/nsContextMenu.js", + "browser/components/downloads/content/downloads.js", + "browser/components/downloads/content/indicator.js", + "toolkit/content/customElements.js", + "toolkit/content/editMenuOverlay.js", +]; + +const extraDefinitions = [ + // Via Components.utils, defineModuleGetter, defineLazyModuleGetters or + // defineLazyScriptGetter (and map to + // single) variable. + { name: "XPCOMUtils", writable: false }, + { name: "Task", writable: false }, + { name: "windowGlobalChild", writable: false }, + // structuredClone is a new global that would be defined for the `browser` + // environment in ESLint, but only Firefox has implemented it currently and so + // it isn't in ESLint's globals yet. + // https://developer.mozilla.org/docs/Web/API/structuredClone + { name: "structuredClone", writable: false }, +]; + +// Some files in global-scripts.inc need mapping to specific locations. +const MAPPINGS = { + "printUtils.js": "toolkit/components/printing/content/printUtils.js", + "panelUI.js": "browser/components/customizableui/content/panelUI.js", + "viewSourceUtils.js": + "toolkit/components/viewsource/content/viewSourceUtils.js", + "browserPlacesViews.js": + "browser/components/places/content/browserPlacesViews.js", + "places-tree.js": "browser/components/places/content/places-tree.js", + "places-menupopup.js": + "browser/components/places/content/places-menupopup.js", +}; + +const globalScriptsRegExp = + /^\s*Services.scriptloader.loadSubScript\(\"(.*?)\", this\);$/; + +function getGlobalScriptIncludes(scriptPath) { + let fileData; + try { + fileData = fs.readFileSync(scriptPath, { encoding: "utf8" }); + } catch (ex) { + // The file isn't present, so this isn't an m-c repository. + return null; + } + + fileData = fileData.split("\n"); + + let result = []; + + for (let line of fileData) { + let match = line.match(globalScriptsRegExp); + if (match) { + let sourceFile = match[1] + .replace( + "chrome://browser/content/search/", + "browser/components/search/content/" + ) + .replace( + "chrome://browser/content/screenshots/", + "browser/components/screenshots/content/" + ) + .replace("chrome://browser/content/", "browser/base/content/") + .replace("chrome://global/content/", "toolkit/content/"); + + for (let mapping of Object.getOwnPropertyNames(MAPPINGS)) { + if (sourceFile.includes(mapping)) { + sourceFile = MAPPINGS[mapping]; + } + } + + result.push(sourceFile); + } + } + + return result; +} + +function getGlobalScripts() { + let results = []; + for (let scriptPath of helpers.globalScriptPaths) { + results = results.concat(getGlobalScriptIncludes(scriptPath)); + } + return results; +} + +module.exports = getScriptGlobals( + "browser-window", + getGlobalScripts().concat(EXTRA_SCRIPTS), + extraDefinitions, + { + browserjsScripts: getGlobalScripts().concat(EXTRA_SCRIPTS), + } +); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-script.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-script.js new file mode 100644 index 0000000000..9b0ae54a2e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-script.js @@ -0,0 +1,28 @@ +/** + * @fileoverview Defines the environment for SpecialPowers chrome script. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var { globals } = require("./special-powers-sandbox"); +var util = require("util"); + +module.exports = { + globals: util._extend( + { + // testing/specialpowers/content/SpecialPowersParent.sys.mjs + + // SPLoadChromeScript block + createWindowlessBrowser: false, + sendAsyncMessage: false, + addMessageListener: false, + removeMessageListener: false, + actorParent: false, + }, + globals + ), +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-worker.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-worker.js new file mode 100644 index 0000000000..db5759b26c --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/chrome-worker.js @@ -0,0 +1,25 @@ +/** + * @fileoverview Defines the environment for chrome workers. This differs + * from normal workers by the fact that `ctypes` can be accessed + * as well. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var globals = require("globals"); +var util = require("util"); + +var workerGlobals = util._extend( + { + ctypes: false, + }, + globals.worker +); + +module.exports = { + globals: workerGlobals, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/frame-script.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/frame-script.js new file mode 100644 index 0000000000..7ac5c941cf --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/frame-script.js @@ -0,0 +1,39 @@ +/** + * @fileoverview Defines the environment for frame scripts. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // dom/chrome-webidl/MessageManager.webidl + + // MessageManagerGlobal + dump: false, + atob: false, + btoa: false, + + // MessageListenerManagerMixin + addMessageListener: false, + removeMessageListener: false, + addWeakMessageListener: false, + removeWeakMessageListener: false, + + // MessageSenderMixin + sendAsyncMessage: false, + processMessageManager: false, + remoteType: false, + + // SyncMessageSenderMixin + sendSyncMessage: false, + + // ContentFrameMessageManager + content: false, + docShell: false, + tabEventTarget: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/jsm.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/jsm.js new file mode 100644 index 0000000000..1ff73c04d9 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/jsm.js @@ -0,0 +1,31 @@ +/** + * @fileoverview Defines the environment for jsm files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // These globals are hard-coded and available in .jsm scopes. + // https://searchfox.org/mozilla-central/rev/dcb0cfb66e4ed3b9c7fbef1e80572426ff5f3c3a/js/xpconnect/loader/mozJSModuleLoader.cpp#222-223 + // Although `debug` is allowed for jsm files, this is non-standard and something + // we don't want to allow in mjs files. Hence it is not included here. + atob: false, + btoa: false, + dump: false, + // The WebAssembly global is available in most (if not all) contexts where + // JS can run. It's definitely available in JSMs. So even if this is not + // the perfect place to add it, it's not wrong, and we can move it later. + WebAssembly: false, + // These are hard-coded and available in .jsm scopes. + // See BackstagePass::Resolve. + fetch: false, + crypto: false, + indexedDB: false, + structuredClone: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/privileged.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/privileged.js new file mode 100644 index 0000000000..13e81f7cef --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/privileged.js @@ -0,0 +1,803 @@ +/** + * @fileoverview Defines the environment for privileges JS files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // Intl and WebAssembly are available everywhere but are not webIDL definitions. + Intl: false, + WebAssembly: false, + // This list of items is currently obtained manually from the list of + // mozilla::dom::constructor::id::ID enumerations in an object directory + // generated dom/bindings/RegisterBindings.cpp + APZHitResultFlags: false, + AbortController: false, + AbortSignal: false, + AccessibleNode: false, + Addon: false, + AddonEvent: false, + AddonInstall: false, + AddonManager: true, + AddonManagerPermissions: false, + AnalyserNode: false, + Animation: false, + AnimationEffect: false, + AnimationEvent: false, + AnimationPlaybackEvent: false, + AnimationTimeline: false, + AnonymousContent: false, + Attr: false, + AudioBuffer: false, + AudioBufferSourceNode: false, + AudioContext: false, + AudioDestinationNode: false, + AudioListener: false, + AudioNode: false, + AudioParam: false, + AudioParamMap: false, + AudioProcessingEvent: false, + AudioScheduledSourceNode: false, + AudioTrack: false, + AudioTrackList: false, + AudioWorklet: false, + AudioWorkletNode: false, + AuthenticatorAssertionResponse: false, + AuthenticatorAttestationResponse: false, + AuthenticatorResponse: false, + BarProp: false, + BaseAudioContext: false, + BatteryManager: false, + BeforeUnloadEvent: false, + BiquadFilterNode: false, + Blob: false, + BlobEvent: false, + BoxObject: false, + BroadcastChannel: false, + BrowsingContext: false, + ByteLengthQueuingStrategy: false, + CanonicalBrowsingContext: false, + CDATASection: false, + CSS: false, + CSS2Properties: false, + CSSAnimation: false, + CSSConditionRule: false, + CSSCounterStyleRule: false, + CSSFontFaceRule: false, + CSSFontFeatureValuesRule: false, + CSSGroupingRule: false, + CSSImportRule: false, + CSSKeyframeRule: false, + CSSKeyframesRule: false, + CSSMediaRule: false, + CSSMozDocumentRule: false, + CSSNamespaceRule: false, + CSSPageRule: false, + CSSPseudoElement: false, + CSSRule: false, + CSSRuleList: false, + CSSStyleDeclaration: false, + CSSStyleRule: false, + CSSStyleSheet: false, + CSSSupportsRule: false, + CSSTransition: false, + Cache: false, + CacheStorage: false, + CanvasCaptureMediaStream: false, + CanvasGradient: false, + CanvasPattern: false, + CanvasRenderingContext2D: false, + CaretPosition: false, + CaretStateChangedEvent: false, + ChannelMergerNode: false, + ChannelSplitterNode: false, + ChannelWrapper: false, + CharacterData: false, + CheckerboardReportService: false, + ChildProcessMessageManager: false, + ChildSHistory: false, + ChromeMessageBroadcaster: false, + ChromeMessageSender: false, + ChromeNodeList: false, + ChromeUtils: false, + ChromeWorker: false, + Clipboard: false, + ClipboardEvent: false, + ClonedErrorHolder: false, + CloseEvent: false, + CommandEvent: false, + Comment: false, + CompositionEvent: false, + ConsoleInstance: false, + ConstantSourceNode: false, + ContentFrameMessageManager: false, + ContentProcessMessageManager: false, + ConvolverNode: false, + CountQueuingStrategy: false, + CreateOfferRequest: false, + Credential: false, + CredentialsContainer: false, + Crypto: false, + CryptoKey: false, + CustomElementRegistry: false, + CustomEvent: false, + DOMError: false, + DOMException: false, + DOMImplementation: false, + DOMLocalization: false, + DOMMatrix: false, + DOMMatrixReadOnly: false, + DOMParser: false, + DOMPoint: false, + DOMPointReadOnly: false, + DOMQuad: false, + DOMRect: false, + DOMRectList: false, + DOMRectReadOnly: false, + DOMRequest: false, + DOMStringList: false, + DOMStringMap: false, + DOMTokenList: false, + DataTransfer: false, + DataTransferItem: false, + DataTransferItemList: false, + DebuggerNotificationObserver: false, + DelayNode: false, + DeprecationReportBody: false, + DeviceLightEvent: false, + DeviceMotionEvent: false, + DeviceOrientationEvent: false, + DeviceProximityEvent: false, + Directory: false, + Document: false, + DocumentFragment: false, + DocumentTimeline: false, + DocumentType: false, + DominatorTree: false, + DragEvent: false, + DynamicsCompressorNode: false, + Element: false, + ErrorEvent: false, + Event: false, + EventSource: false, + EventTarget: false, + FeaturePolicyViolationReportBody: false, + FetchObserver: false, + File: false, + FileList: false, + FileReader: false, + FileSystem: false, + FileSystemDirectoryEntry: false, + FileSystemDirectoryReader: false, + FileSystemEntry: false, + FileSystemFileEntry: false, + Flex: false, + FlexItemValues: false, + FlexLineValues: false, + FluentBundle: false, + FluentResource: false, + FocusEvent: false, + FontFace: false, + FontFaceSet: false, + FontFaceSetLoadEvent: false, + FormData: false, + FrameCrashedEvent: false, + FrameLoader: false, + GainNode: false, + Gamepad: false, + GamepadAxisMoveEvent: false, + GamepadButton: false, + GamepadButtonEvent: false, + GamepadEvent: false, + GamepadHapticActuator: false, + GamepadPose: false, + GamepadServiceTest: false, + Glean: false, + GleanPings: false, + Grid: false, + GridArea: false, + GridDimension: false, + GridLine: false, + GridLines: false, + GridTrack: false, + GridTracks: false, + HTMLAllCollection: false, + HTMLAnchorElement: false, + HTMLAreaElement: false, + HTMLAudioElement: false, + Audio: false, + HTMLBRElement: false, + HTMLBaseElement: false, + HTMLBodyElement: false, + HTMLButtonElement: false, + HTMLCanvasElement: false, + HTMLCollection: false, + HTMLDListElement: false, + HTMLDataElement: false, + HTMLDataListElement: false, + HTMLDetailsElement: false, + HTMLDialogElement: false, + HTMLDirectoryElement: false, + HTMLDivElement: false, + HTMLDocument: false, + HTMLElement: false, + HTMLEmbedElement: false, + HTMLFieldSetElement: false, + HTMLFontElement: false, + HTMLFormControlsCollection: false, + HTMLFormElement: false, + HTMLFrameElement: false, + HTMLFrameSetElement: false, + HTMLHRElement: false, + HTMLHeadElement: false, + HTMLHeadingElement: false, + HTMLHtmlElement: false, + HTMLIFrameElement: false, + HTMLImageElement: false, + Image: false, + HTMLInputElement: false, + HTMLLIElement: false, + HTMLLabelElement: false, + HTMLLegendElement: false, + HTMLLinkElement: false, + HTMLMapElement: false, + HTMLMarqueeElement: false, + HTMLMediaElement: false, + HTMLMenuElement: false, + HTMLMenuItemElement: false, + HTMLMetaElement: false, + HTMLMeterElement: false, + HTMLModElement: false, + HTMLOListElement: false, + HTMLObjectElement: false, + HTMLOptGroupElement: false, + HTMLOptionElement: false, + Option: false, + HTMLOptionsCollection: false, + HTMLOutputElement: false, + HTMLParagraphElement: false, + HTMLParamElement: false, + HTMLPictureElement: false, + HTMLPreElement: false, + HTMLProgressElement: false, + HTMLQuoteElement: false, + HTMLScriptElement: false, + HTMLSelectElement: false, + HTMLSlotElement: false, + HTMLSourceElement: false, + HTMLSpanElement: false, + HTMLStyleElement: false, + HTMLTableCaptionElement: false, + HTMLTableCellElement: false, + HTMLTableColElement: false, + HTMLTableElement: false, + HTMLTableRowElement: false, + HTMLTableSectionElement: false, + HTMLTemplateElement: false, + HTMLTextAreaElement: false, + HTMLTimeElement: false, + HTMLTitleElement: false, + HTMLTrackElement: false, + HTMLUListElement: false, + HTMLUnknownElement: false, + HTMLVideoElement: false, + HashChangeEvent: false, + Headers: false, + HeapSnapshot: false, + History: false, + IDBCursor: false, + IDBCursorWithValue: false, + IDBDatabase: false, + IDBFactory: false, + IDBFileHandle: false, + IDBFileRequest: false, + IDBIndex: false, + IDBKeyRange: false, + IDBLocaleAwareKeyRange: false, + IDBMutableFile: false, + IDBObjectStore: false, + IDBOpenDBRequest: false, + IDBRequest: false, + IDBTransaction: false, + IDBVersionChangeEvent: false, + IIRFilterNode: false, + IdleDeadline: false, + ImageBitmap: false, + ImageBitmapRenderingContext: false, + ImageCapture: false, + ImageCaptureErrorEvent: false, + ImageData: false, + ImageDocument: false, + InputEvent: false, + InspectorFontFace: false, + InspectorUtils: false, + InstallTriggerImpl: false, + IntersectionObserver: false, + IntersectionObserverEntry: false, + IOUtils: false, + JSProcessActorChild: false, + JSProcessActorParent: false, + JSWindowActorChild: false, + JSWindowActorParent: false, + KeyEvent: false, + KeyboardEvent: false, + KeyframeEffect: false, + L10nFileSource: false, + L10nRegistry: false, + Localization: false, + Location: false, + MIDIAccess: false, + MIDIConnectionEvent: false, + MIDIInput: false, + MIDIInputMap: false, + MIDIMessageEvent: false, + MIDIOutput: false, + MIDIOutputMap: false, + MIDIPort: false, + MatchGlob: false, + MatchPattern: false, + MatchPatternSet: false, + MediaCapabilities: false, + MediaCapabilitiesInfo: false, + MediaControlService: false, + MediaDeviceInfo: false, + MediaDevices: false, + MediaElementAudioSourceNode: false, + MediaEncryptedEvent: false, + MediaError: false, + MediaKeyError: false, + MediaKeyMessageEvent: false, + MediaKeySession: false, + MediaKeyStatusMap: false, + MediaKeySystemAccess: false, + MediaKeys: false, + MediaList: false, + MediaQueryList: false, + MediaQueryListEvent: false, + MediaRecorder: false, + MediaRecorderErrorEvent: false, + MediaSource: false, + MediaStream: false, + MediaStreamAudioDestinationNode: false, + MediaStreamAudioSourceNode: false, + MediaStreamEvent: false, + MediaStreamTrack: false, + MediaStreamTrackEvent: false, + MerchantValidationEvent: false, + MessageBroadcaster: false, + MessageChannel: false, + MessageEvent: false, + MessageListenerManager: false, + MessagePort: false, + MessageSender: false, + MimeType: false, + MimeTypeArray: false, + MouseEvent: false, + MouseScrollEvent: false, + MozCanvasPrintState: false, + MozDocumentMatcher: false, + MozDocumentObserver: false, + MozQueryInterface: false, + MozSharedMap: false, + MozSharedMapChangeEvent: false, + MozStorageAsyncStatementParams: false, + MozStorageStatementParams: false, + MozStorageStatementRow: false, + MozWritableSharedMap: false, + MutationEvent: false, + MutationObserver: false, + MutationRecord: false, + NamedNodeMap: false, + Navigator: false, + NetworkInformation: false, + Node: false, + NodeFilter: false, + NodeIterator: false, + NodeList: false, + Notification: false, + NotifyPaintEvent: false, + OfflineAudioCompletionEvent: false, + OfflineAudioContext: false, + OfflineResourceList: false, + OffscreenCanvas: false, + OscillatorNode: false, + PageTransitionEvent: false, + PaintRequest: false, + PaintRequestList: false, + PannerNode: false, + ParentProcessMessageManager: false, + Path2D: false, + PathUtils: false, + PaymentAddress: false, + PaymentMethodChangeEvent: false, + PaymentRequest: false, + PaymentRequestUpdateEvent: false, + PaymentResponse: false, + PeerConnectionImpl: false, + PeerConnectionObserver: false, + Performance: false, + PerformanceEntry: false, + PerformanceEntryEvent: false, + PerformanceMark: false, + PerformanceMeasure: false, + PerformanceNavigation: false, + PerformanceNavigationTiming: false, + PerformanceObserver: false, + PerformanceObserverEntryList: false, + PerformanceResourceTiming: false, + PerformanceServerTiming: false, + PerformanceTiming: false, + PeriodicWave: false, + PermissionStatus: false, + Permissions: false, + PlacesBookmark: false, + PlacesBookmarkAddition: false, + PlacesBookmarkGuid: false, + PlacesBookmarkKeyword: false, + PlacesBookmarkMoved: false, + PlacesBookmarkRemoved: false, + PlacesBookmarkTags: false, + PlacesBookmarkTime: false, + PlacesBookmarkTitle: false, + PlacesBookmarkUrl: false, + PlacesEvent: false, + PlacesHistoryCleared: false, + PlacesObservers: false, + PlacesPurgeCaches: false, + PlacesRanking: false, + PlacesVisit: false, + PlacesVisitRemoved: false, + PlacesVisitTitle: false, + PlacesWeakCallbackWrapper: false, + Plugin: false, + PluginArray: false, + PluginCrashedEvent: false, + PointerEvent: false, + PopStateEvent: false, + PopupBlockedEvent: false, + PrecompiledScript: false, + Presentation: false, + PresentationAvailability: false, + PresentationConnection: false, + PresentationConnectionAvailableEvent: false, + PresentationConnectionCloseEvent: false, + PresentationConnectionList: false, + PresentationReceiver: false, + PresentationRequest: false, + PrioEncoder: false, + ProcessMessageManager: false, + ProcessingInstruction: false, + ProgressEvent: false, + PromiseDebugging: false, + PromiseRejectionEvent: false, + PublicKeyCredential: false, + PushManager: false, + PushManagerImpl: false, + PushSubscription: false, + PushSubscriptionOptions: false, + RTCCertificate: false, + RTCDTMFSender: false, + RTCDTMFToneChangeEvent: false, + RTCDataChannel: false, + RTCDataChannelEvent: false, + RTCIceCandidate: false, + RTCPeerConnection: false, + RTCPeerConnectionIceEvent: false, + RTCPeerConnectionStatic: false, + RTCRtpReceiver: false, + RTCRtpSender: false, + RTCRtpTransceiver: false, + RTCSessionDescription: false, + RTCStatsReport: false, + RTCTrackEvent: false, + RadioNodeList: false, + Range: false, + ReadableStreamBYOBReader: false, + ReadableStreamBYOBRequest: false, + ReadableByteStreamController: false, + ReadableStream: false, + ReadableStreamDefaultController: false, + ReadableStreamDefaultReader: false, + Report: false, + ReportBody: false, + ReportingObserver: false, + Request: false, + Response: false, + SessionStoreUtils: false, + SVGAElement: false, + SVGAngle: false, + SVGAnimateElement: false, + SVGAnimateMotionElement: false, + SVGAnimateTransformElement: false, + SVGAnimatedAngle: false, + SVGAnimatedBoolean: false, + SVGAnimatedEnumeration: false, + SVGAnimatedInteger: false, + SVGAnimatedLength: false, + SVGAnimatedLengthList: false, + SVGAnimatedNumber: false, + SVGAnimatedNumberList: false, + SVGAnimatedPreserveAspectRatio: false, + SVGAnimatedRect: false, + SVGAnimatedString: false, + SVGAnimatedTransformList: false, + SVGAnimationElement: false, + SVGCircleElement: false, + SVGClipPathElement: false, + SVGComponentTransferFunctionElement: false, + SVGDefsElement: false, + SVGDescElement: false, + SVGElement: false, + SVGEllipseElement: false, + SVGFEBlendElement: false, + SVGFEColorMatrixElement: false, + SVGFEComponentTransferElement: false, + SVGFECompositeElement: false, + SVGFEConvolveMatrixElement: false, + SVGFEDiffuseLightingElement: false, + SVGFEDisplacementMapElement: false, + SVGFEDistantLightElement: false, + SVGFEDropShadowElement: false, + SVGFEFloodElement: false, + SVGFEFuncAElement: false, + SVGFEFuncBElement: false, + SVGFEFuncGElement: false, + SVGFEFuncRElement: false, + SVGFEGaussianBlurElement: false, + SVGFEImageElement: false, + SVGFEMergeElement: false, + SVGFEMergeNodeElement: false, + SVGFEMorphologyElement: false, + SVGFEOffsetElement: false, + SVGFEPointLightElement: false, + SVGFESpecularLightingElement: false, + SVGFESpotLightElement: false, + SVGFETileElement: false, + SVGFETurbulenceElement: false, + SVGFilterElement: false, + SVGForeignObjectElement: false, + SVGGElement: false, + SVGGeometryElement: false, + SVGGradientElement: false, + SVGGraphicsElement: false, + SVGImageElement: false, + SVGLength: false, + SVGLengthList: false, + SVGLineElement: false, + SVGLinearGradientElement: false, + SVGMPathElement: false, + SVGMarkerElement: false, + SVGMaskElement: false, + SVGMatrix: false, + SVGMetadataElement: false, + SVGNumber: false, + SVGNumberList: false, + SVGPathElement: false, + SVGPathSegList: false, + SVGPatternElement: false, + SVGPoint: false, + SVGPointList: false, + SVGPolygonElement: false, + SVGPolylineElement: false, + SVGPreserveAspectRatio: false, + SVGRadialGradientElement: false, + SVGRect: false, + SVGRectElement: false, + SVGSVGElement: false, + SVGScriptElement: false, + SVGSetElement: false, + SVGStopElement: false, + SVGStringList: false, + SVGStyleElement: false, + SVGSwitchElement: false, + SVGSymbolElement: false, + SVGTSpanElement: false, + SVGTextContentElement: false, + SVGTextElement: false, + SVGTextPathElement: false, + SVGTextPositioningElement: false, + SVGTitleElement: false, + SVGTransform: false, + SVGTransformList: false, + SVGUnitTypes: false, + SVGUseElement: false, + SVGViewElement: false, + SVGZoomAndPan: false, + Screen: false, + ScreenLuminance: false, + ScreenOrientation: false, + ScriptProcessorNode: false, + ScrollAreaEvent: false, + ScrollViewChangeEvent: false, + SecurityPolicyViolationEvent: false, + Selection: false, + ServiceWorker: false, + ServiceWorkerContainer: false, + ServiceWorkerRegistration: false, + ShadowRoot: false, + SharedWorker: false, + SimpleGestureEvent: false, + SourceBuffer: false, + SourceBufferList: false, + SpeechGrammar: false, + SpeechGrammarList: false, + SpeechRecognition: false, + SpeechRecognitionAlternative: false, + SpeechRecognitionError: false, + SpeechRecognitionEvent: false, + SpeechRecognitionResult: false, + SpeechRecognitionResultList: false, + SpeechSynthesis: false, + SpeechSynthesisErrorEvent: false, + SpeechSynthesisEvent: false, + SpeechSynthesisUtterance: false, + SpeechSynthesisVoice: false, + StereoPannerNode: false, + Storage: false, + StorageEvent: false, + StorageManager: false, + StreamFilter: false, + StreamFilterDataEvent: false, + StructuredCloneHolder: false, + StructuredCloneTester: false, + StyleSheet: false, + StyleSheetApplicableStateChangeEvent: false, + StyleSheetList: false, + SubtleCrypto: false, + SyncMessageSender: false, + TCPServerSocket: false, + TCPServerSocketEvent: false, + TCPSocket: false, + TCPSocketErrorEvent: false, + TCPSocketEvent: false, + TelemetryStopwatch: false, + TestingDeprecatedInterface: false, + Text: false, + TextClause: false, + TextDecoder: false, + TextEncoder: false, + TextMetrics: false, + TextTrack: false, + TextTrackCue: false, + TextTrackCueList: false, + TextTrackList: false, + TimeEvent: false, + TimeRanges: false, + Touch: false, + TouchEvent: false, + TouchList: false, + TrackEvent: false, + TransceiverImpl: false, + TransformStream: false, + TransformStreamDefaultController: false, + TransitionEvent: false, + TreeColumn: false, + TreeColumns: false, + TreeContentView: false, + TreeWalker: false, + U2F: false, + UDPMessageEvent: false, + UDPSocket: false, + UIEvent: false, + URL: false, + URLSearchParams: false, + UserInteraction: false, + UserProximityEvent: false, + VRDisplay: false, + VRDisplayCapabilities: false, + VRDisplayEvent: false, + VREyeParameters: false, + VRFieldOfView: false, + VRFrameData: false, + VRMockController: false, + VRMockDisplay: false, + VRPose: false, + VRServiceTest: false, + VRStageParameters: false, + VRSubmitFrameResult: false, + VTTCue: false, + VTTRegion: false, + ValidityState: false, + VideoPlaybackQuality: false, + VideoTrack: false, + VideoTrackList: false, + VisualViewport: false, + WaveShaperNode: false, + WebExtensionContentScript: false, + WebExtensionPolicy: false, + WebGL2RenderingContext: false, + WebGLActiveInfo: false, + WebGLBuffer: false, + WebGLContextEvent: false, + WebGLFramebuffer: false, + WebGLProgram: false, + WebGLQuery: false, + WebGLRenderbuffer: false, + WebGLRenderingContext: false, + WebGLSampler: false, + WebGLShader: false, + WebGLShaderPrecisionFormat: false, + WebGLSync: false, + WebGLTexture: false, + WebGLTransformFeedback: false, + WebGLUniformLocation: false, + WebGLVertexArrayObject: false, + WebGPU: false, + WebGPUAdapter: false, + WebGPUAttachmentState: false, + WebGPUBindGroup: false, + WebGPUBindGroupLayout: false, + WebGPUBindingType: false, + WebGPUBlendFactor: false, + WebGPUBlendOperation: false, + WebGPUBlendState: false, + WebGPUBuffer: false, + WebGPUBufferUsage: false, + WebGPUColorWriteBits: false, + WebGPUCommandBuffer: false, + WebGPUCommandEncoder: false, + WebGPUCompareFunction: false, + WebGPUComputePipeline: false, + WebGPUDepthStencilState: false, + WebGPUDevice: false, + WebGPUFence: false, + WebGPUFilterMode: false, + WebGPUIndexFormat: false, + WebGPUInputState: false, + WebGPUInputStepMode: false, + WebGPULoadOp: false, + WebGPULogEntry: false, + WebGPUPipelineLayout: false, + WebGPUPrimitiveTopology: false, + WebGPUQueue: false, + WebGPURenderPipeline: false, + WebGPUSampler: false, + WebGPUShaderModule: false, + WebGPUShaderStage: false, + WebGPUShaderStageBit: false, + WebGPUStencilOperation: false, + WebGPUStoreOp: false, + WebGPUSwapChain: false, + WebGPUTexture: false, + WebGPUTextureDimension: false, + WebGPUTextureFormat: false, + WebGPUTextureUsage: false, + WebGPUTextureView: false, + WebGPUVertexFormat: false, + WebKitCSSMatrix: false, + WebSocket: false, + WebrtcGlobalInformation: false, + WheelEvent: false, + Window: false, + WindowGlobalChild: false, + WindowGlobalParent: false, + WindowRoot: false, + Worker: false, + Worklet: false, + WritableStream: false, + WritableStreamDefaultController: false, + WritableStreamDefaultWriter: false, + XMLDocument: false, + XMLHttpRequest: false, + XMLHttpRequestEventTarget: false, + XMLHttpRequestUpload: false, + XMLSerializer: false, + XPathEvaluator: false, + XPathExpression: false, + XPathResult: false, + XSLTProcessor: false, + XULCommandEvent: false, + XULElement: false, + XULFrameElement: false, + XULMenuElement: false, + XULPopupElement: false, + XULScrollElement: false, + XULTextElement: false, + console: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/process-script.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/process-script.js new file mode 100644 index 0000000000..f329a6650b --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/process-script.js @@ -0,0 +1,38 @@ +/** + * @fileoverview Defines the environment for process scripts. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // dom/chrome-webidl/MessageManager.webidl + + // MessageManagerGlobal + dump: false, + atob: false, + btoa: false, + + // MessageListenerManagerMixin + addMessageListener: false, + removeMessageListener: false, + addWeakMessageListener: false, + removeWeakMessageListener: false, + + // MessageSenderMixin + sendAsyncMessage: false, + processMessageManager: false, + remoteType: false, + + // SyncMessageSenderMixin + sendSyncMessage: false, + + // ContentProcessMessageManager + initialProcessData: false, + sharedData: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/remote-page.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/remote-page.js new file mode 100644 index 0000000000..5b362ea2e0 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/remote-page.js @@ -0,0 +1,43 @@ +/** + * @fileoverview Defines the environment for remote page. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + atob: false, + btoa: false, + RPMAddTRRExcludedDomain: false, + RPMGetAppBuildID: false, + RPMGetInnerMostURI: false, + RPMGetIntPref: false, + RPMGetStringPref: false, + RPMGetBoolPref: false, + RPMSetBoolPref: false, + RPMGetFormatURLPref: false, + RPMIsTRROnlyFailure: false, + RPMIsFirefox: false, + RPMIsNativeFallbackFailure: false, + RPMIsWindowPrivate: false, + RPMSendAsyncMessage: false, + RPMSendQuery: false, + RPMAddMessageListener: false, + RPMRecordTelemetryEvent: false, + RPMCheckAlternateHostAvailable: false, + RPMAddToHistogram: false, + RPMRemoveMessageListener: false, + RPMGetHttpResponseHeader: false, + RPMTryPingSecureWWWLink: false, + RPMOpenSecureWWWLink: false, + RPMOpenPreferences: false, + RPMGetTRRSkipReason: false, + RPMGetTRRDomain: false, + RPMIsSiteSpecificTRRError: false, + RPMSetTRRDisabledLoadFlags: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/simpletest.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/simpletest.js new file mode 100644 index 0000000000..2f5dd5c33e --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/simpletest.js @@ -0,0 +1,35 @@ +/** + * @fileoverview Defines the environment for scripts that use the SimpleTest + * mochitest harness. Imports the globals from the relevant files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var path = require("path"); +var { getScriptGlobals } = require("./utils"); + +// When updating this list, be sure to also update the 'support-files' config +// in `tools/lint/eslint.yml`. +const simpleTestFiles = [ + "AccessibilityUtils.js", + "ExtensionTestUtils.js", + "EventUtils.js", + "MockObjects.js", + "SimpleTest.js", + "WindowSnapshot.js", + "paint_listener.js", +]; +const simpleTestPath = "testing/mochitest/tests/SimpleTest"; + +module.exports = getScriptGlobals( + "simpletest", + simpleTestFiles.map(file => path.join(simpleTestPath, file)) +); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/sjs.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/sjs.js new file mode 100644 index 0000000000..995c3173d2 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/sjs.js @@ -0,0 +1,30 @@ +/** + * @fileoverview Defines the environment for sjs files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // All these variables are hard-coded to be available for sjs scopes only. + // https://searchfox.org/mozilla-central/rev/26a1b0fce12e6dd495a954c542bb1e7bd6e0d548/netwerk/test/httpserver/httpd.js#2879 + atob: false, + btoa: false, + ChromeUtils: false, + dump: false, + getState: false, + setState: false, + getSharedState: false, + setSharedState: false, + getObjectState: false, + setObjectState: false, + registerPathHandler: false, + Services: false, + // importScripts is also available. + importScripts: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/special-powers-sandbox.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/special-powers-sandbox.js new file mode 100644 index 0000000000..5a28c91883 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/special-powers-sandbox.js @@ -0,0 +1,46 @@ +/** + * @fileoverview Defines the environment for SpecialPowers sandbox. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + // wantComponents defaults to true, + Components: false, + Ci: false, + Cr: false, + Cc: false, + Cu: false, + Services: false, + + // testing/specialpowers/content/SpecialPowersSandbox.sys.mjs + + // SANDBOX_GLOBALS + Blob: false, + ChromeUtils: false, + FileReader: false, + TextDecoder: false, + TextEncoder: false, + URL: false, + + // EXTRA_IMPORTS + EventUtils: false, + + // SpecialPowersSandbox constructor + assert: false, + Assert: false, + BrowsingContext: false, + InspectorUtils: false, + ok: false, + is: false, + isnot: false, + todo: false, + todo_is: false, + info: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/specific.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/specific.js new file mode 100644 index 0000000000..23ebcb5bb1 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/specific.js @@ -0,0 +1,31 @@ +/** + * @fileoverview Defines the environment for the Firefox browser. Allows global + * variables which are non-standard and specific to Firefox. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +module.exports = { + globals: { + Cc: false, + ChromeUtils: false, + Ci: false, + Components: false, + Cr: false, + Cu: false, + Debugger: false, + InstallTrigger: false, + // https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/InternalError + InternalError: true, + Services: false, + // https://developer.mozilla.org/docs/Web/API/Window/dump + dump: true, + openDialog: false, + // https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/uneval + uneval: false, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/utils.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/utils.js new file mode 100644 index 0000000000..aeda690ba5 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/utils.js @@ -0,0 +1,62 @@ +/** + * @fileoverview Provides utilities for setting up environments. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var path = require("path"); +var helpers = require("../helpers"); +var globals = require("../globals"); + +/** + * Obtains the globals for a list of files. + * + * @param {Array.} files + * The array of files to get globals for. The paths are relative to the topsrcdir. + * @returns {Object} + * Returns an object with keys of the global names and values of if they are + * writable or not. + */ +function getGlobalsForScripts(environmentName, files, extraDefinitions) { + let fileGlobals = extraDefinitions; + const root = helpers.rootDir; + for (const file of files) { + const fileName = path.join(root, file); + try { + fileGlobals = fileGlobals.concat(globals.getGlobalsForFile(fileName)); + } catch (e) { + console.error(`Could not load globals from file ${fileName}: ${e}`); + console.error( + `You may need to update the mappings for the ${environmentName} environment` + ); + throw new Error(`Could not load globals from file ${fileName}: ${e}`); + } + } + + var globalObjects = {}; + for (let global of fileGlobals) { + globalObjects[global.name] = global.writable; + } + return globalObjects; +} + +module.exports = { + getScriptGlobals( + environmentName, + files, + extraDefinitions = [], + extraEnv = {} + ) { + if (helpers.isMozillaCentralBased()) { + return { + globals: getGlobalsForScripts(environmentName, files, extraDefinitions), + ...extraEnv, + }; + } + return helpers.getSavedEnvironmentItems(environmentName); + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/xpcshell.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/xpcshell.js new file mode 100644 index 0000000000..408bc2e277 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/environments/xpcshell.js @@ -0,0 +1,59 @@ +/** + * @fileoverview Defines the environment for xpcshell test files. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +var { getScriptGlobals } = require("./utils"); + +const extraGlobals = [ + // Defined in XPCShellImpl.cpp + "print", + "readline", + "load", + "quit", + "dumpXPC", + "dump", + "gc", + "gczeal", + "options", + "sendCommand", + "atob", + "btoa", + "setInterruptCallback", + "simulateNoScriptActivity", + "registerXPCTestComponents", + + // Assert.sys.mjs globals. + "setReporter", + "report", + "ok", + "equal", + "notEqual", + "deepEqual", + "notDeepEqual", + "strictEqual", + "notStrictEqual", + "throws", + "rejects", + "greater", + "greaterOrEqual", + "less", + "lessOrEqual", + // TestingFunctions.cpp globals + "allocationMarker", + "byteSize", + "saveStack", +]; + +module.exports = getScriptGlobals( + "xpcshell", + ["testing/xpcshell/head.js"], + extraGlobals.map(g => { + return { name: g, writable: false }; + }) +); diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js new file mode 100644 index 0000000000..aa3d4258ae --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js @@ -0,0 +1,665 @@ +/** + * @fileoverview functions for scanning an AST for globals including + * traversing referenced scripts. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const path = require("path"); +const fs = require("fs"); +const helpers = require("./helpers"); +const htmlparser = require("htmlparser2"); + +const callExpressionDefinitions = [ + /^loader\.lazyGetter\((?:globalThis|this), "(\w+)"/, + /^loader\.lazyServiceGetter\((?:globalThis|this), "(\w+)"/, + /^loader\.lazyRequireGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyModuleGetter\((?:globalThis|this), "(\w+)"/, + /^ChromeUtils\.defineLazyGetter\((?:globalThis|this), "(\w+)"/, + /^ChromeUtils\.defineModuleGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyPreferenceGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyProxy\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyScriptGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineLazyServiceGetter\((?:globalThis|this), "(\w+)"/, + /^XPCOMUtils\.defineConstant\((?:globalThis|this), "(\w+)"/, + /^DevToolsUtils\.defineLazyModuleGetter\((?:globalThis|this), "(\w+)"/, + /^DevToolsUtils\.defineLazyGetter\((?:globalThis|this), "(\w+)"/, + /^Object\.defineProperty\((?:globalThis|this), "(\w+)"/, + /^Reflect\.defineProperty\((?:globalThis|this), "(\w+)"/, + /^this\.__defineGetter__\("(\w+)"/, +]; + +const callExpressionMultiDefinitions = [ + "XPCOMUtils.defineLazyGlobalGetters(this,", + "XPCOMUtils.defineLazyGlobalGetters(globalThis,", + "XPCOMUtils.defineLazyModuleGetters(this,", + "XPCOMUtils.defineLazyModuleGetters(globalThis,", + "XPCOMUtils.defineLazyServiceGetters(this,", + "XPCOMUtils.defineLazyServiceGetters(globalThis,", + "ChromeUtils.defineESModuleGetters(this,", + "ChromeUtils.defineESModuleGetters(globalThis,", + "loader.lazyRequireGetter(this,", + "loader.lazyRequireGetter(globalThis,", +]; + +const subScriptMatches = [ + /Services\.scriptloader\.loadSubScript\("(.*?)", this\)/, +]; + +const workerImportFilenameMatch = /(.*\/)*((.*?)\.jsm?)/; + +/** + * Parses a list of "name:boolean_value" or/and "name" options divided by comma + * or whitespace. + * + * This function was copied from eslint.js + * + * @param {string} string The string to parse. + * @param {Comment} comment The comment node which has the string. + * @returns {Object} Result map object of names and boolean values + */ +function parseBooleanConfig(string, comment) { + let items = {}; + + // Collapse whitespace around : to make parsing easier + string = string.replace(/\s*:\s*/g, ":"); + // Collapse whitespace around , + string = string.replace(/\s*,\s*/g, ","); + + string.split(/\s|,+/).forEach(function (name) { + if (!name) { + return; + } + + let pos = name.indexOf(":"); + let value; + if (pos !== -1) { + value = name.substring(pos + 1, name.length); + name = name.substring(0, pos); + } + + items[name] = { + value: value === "true", + comment, + }; + }); + + return items; +} + +/** + * Global discovery can require parsing many files. This map of + * {String} => {Object} caches what globals were discovered for a file path. + */ +const globalCache = new Map(); + +/** + * Global discovery can occasionally meet circular dependencies due to the way + * js files are included via html/xhtml files etc. This set is used to avoid + * getting into loops whilst the discovery is in progress. + */ +var globalDiscoveryInProgressForFiles = new Set(); + +/** + * When looking for globals in HTML files, it can be common to have more than + * one script tag with inline javascript. These will normally be called together, + * so we store the globals for just the last HTML file processed. + */ +var lastHTMLGlobals = {}; + +/** + * Attempts to convert an CallExpressions that look like module imports + * into global variable definitions. + * + * @param {Object} node + * The AST node to convert. + * @param {boolean} isGlobal + * True if the current node is in the global scope. + * + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ +function convertCallExpressionToGlobals(node, isGlobal) { + let express = node.expression; + if ( + express.type === "CallExpression" && + express.callee.type === "MemberExpression" && + express.callee.object && + express.callee.object.type === "Identifier" && + express.arguments.length === 1 && + express.arguments[0].type === "ArrayExpression" && + express.callee.property.type === "Identifier" && + express.callee.property.name === "importGlobalProperties" + ) { + return express.arguments[0].elements.map(literal => { + return { + explicit: true, + name: literal.value, + writable: false, + }; + }); + } + + let source; + try { + source = helpers.getASTSource(node); + } catch (e) { + return []; + } + + // The definition matches below must be in the global scope for us to define + // a global, so bail out early if we're not a global. + if (!isGlobal) { + return []; + } + + for (let reg of subScriptMatches) { + let match = source.match(reg); + if (match) { + return getGlobalsForScript(match[1], "script").map(g => { + // We don't want any loadSubScript globals to be explicit, as this + // could trigger no-unused-vars when importing multiple variables + // from a script and not using all of them. + g.explicit = false; + return g; + }); + } + } + + for (let reg of callExpressionDefinitions) { + let match = source.match(reg); + if (match) { + return [{ name: match[1], writable: true, explicit: true }]; + } + } + + if ( + callExpressionMultiDefinitions.some(expr => source.startsWith(expr)) && + node.expression.arguments[1] + ) { + let arg = node.expression.arguments[1]; + if (arg.type === "ObjectExpression") { + return arg.properties + .map(p => ({ + name: p.type === "Property" && p.key.name, + writable: true, + explicit: true, + })) + .filter(g => g.name); + } + if (arg.type === "ArrayExpression") { + return arg.elements + .map(p => ({ + name: p.type === "Literal" && p.value, + writable: true, + explicit: true, + })) + .filter(g => typeof g.name == "string"); + } + } + + if ( + node.expression.callee.type == "MemberExpression" && + node.expression.callee.property.type == "Identifier" && + node.expression.callee.property.name == "defineLazyScriptGetter" + ) { + // The case where we have a single symbol as a string has already been + // handled by the regexp, so we have an array of symbols here. + return node.expression.arguments[1].elements.map(n => ({ + name: n.value, + writable: true, + explicit: true, + })); + } + + return []; +} + +/** + * Attempts to convert an AssignmentExpression into a global variable + * definition if it applies to `this` in the global scope. + * + * @param {Object} node + * The AST node to convert. + * @param {boolean} isGlobal + * True if the current node is in the global scope. + * + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ +function convertThisAssignmentExpressionToGlobals(node, isGlobal) { + if ( + isGlobal && + node.expression.left && + node.expression.left.object && + node.expression.left.object.type === "ThisExpression" && + node.expression.left.property && + node.expression.left.property.type === "Identifier" + ) { + return [{ name: node.expression.left.property.name, writable: true }]; + } + return []; +} + +/** + * Attempts to convert an ExpressionStatement to likely global variable + * definitions. + * + * @param {Object} node + * The AST node to convert. + * @param {boolean} isGlobal + * True if the current node is in the global scope. + * + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ +function convertWorkerExpressionToGlobals(node, isGlobal, dirname) { + let results = []; + let expr = node.expression; + + if ( + node.expression.type === "CallExpression" && + expr.callee && + expr.callee.type === "Identifier" && + expr.callee.name === "importScripts" + ) { + for (var arg of expr.arguments) { + var match = arg.value && arg.value.match(workerImportFilenameMatch); + if (match) { + if (!match[1]) { + let filePath = path.resolve(dirname, match[2]); + if (fs.existsSync(filePath)) { + let additionalGlobals = module.exports.getGlobalsForFile(filePath); + results = results.concat(additionalGlobals); + } + } + // Import with relative/absolute path should explicitly use + // `import-globals-from` comment. + } + } + } + + return results; +} + +/** + * Attempts to load the globals for a given script. + * + * @param {string} src + * The source path or url of the script to look for. + * @param {string} type + * The type of the current file (script/module). + * @param {string} [dir] + * The directory of the current file. + * @returns {object[]} + * An array of objects with details of the globals in them. + */ +function getGlobalsForScript(src, type, dir) { + let scriptName; + if (src.includes("http:")) { + // We don't handle this currently as the paths are complex to match. + } else if (src.startsWith("chrome://mochikit/content/")) { + // Various ways referencing test files. + src = src.replace("chrome://mochikit/content/", "/"); + scriptName = path.join(helpers.rootDir, "testing", "mochitest", src); + } else if (src.startsWith("chrome://mochitests/content/browser")) { + src = src.replace("chrome://mochitests/content/browser", ""); + scriptName = path.join(helpers.rootDir, src); + } else if (src.includes("SimpleTest")) { + // This is another way of referencing test files... + scriptName = path.join(helpers.rootDir, "testing", "mochitest", src); + } else if (src.startsWith("/tests/")) { + scriptName = path.join(helpers.rootDir, src.substring(7)); + } else if (dir) { + // Fallback to hoping this is a relative path. + scriptName = path.join(dir, src); + } + if (scriptName && fs.existsSync(scriptName)) { + return module.exports.getGlobalsForFile(scriptName, { + ecmaVersion: helpers.getECMAVersion(), + sourceType: type, + }); + } + return []; +} + +/** + * An object that returns found globals for given AST node types. Each prototype + * property should be named for a node type and accepts a node parameter and a + * parents parameter which is a list of the parent nodes of the current node. + * Each returns an array of globals found. + * + * @param {String} filePath + * The absolute path of the file being parsed. + */ +function GlobalsForNode(filePath, context) { + this.path = filePath; + this.context = context; + + if (this.path) { + this.dirname = path.dirname(this.path); + } else { + this.dirname = null; + } +} + +GlobalsForNode.prototype = { + Program(node) { + let globals = []; + for (let comment of node.comments) { + if (comment.type !== "Block") { + continue; + } + let value = comment.value.trim(); + value = value.replace(/\n/g, ""); + + // We have to discover any globals that ESLint would have defined through + // comment directives. + let match = /^globals?\s+(.+)/.exec(value); + if (match) { + let values = parseBooleanConfig(match[1].trim(), node); + for (let name of Object.keys(values)) { + globals.push({ + name, + writable: values[name].value, + }); + } + // We matched globals, so we won't match import-globals-from. + continue; + } + + match = /^import-globals-from\s+(.+)$/.exec(value); + if (!match) { + continue; + } + + if (!this.dirname) { + // If this is testing context without path, ignore import. + return globals; + } + + let filePath = match[1].trim(); + + if (filePath.endsWith(".mjs")) { + if (this.context) { + this.context.report( + comment, + "import-globals-from does not support module files - use a direct import instead" + ); + } else { + // Fall back to throwing an error, as we do not have a context in all situations, + // e.g. when loading the environment. + throw new Error( + "import-globals-from does not support module files - use a direct import instead" + ); + } + continue; + } + + if (!path.isAbsolute(filePath)) { + filePath = path.resolve(this.dirname, filePath); + } else { + filePath = path.join(helpers.rootDir, filePath); + } + globals = globals.concat(module.exports.getGlobalsForFile(filePath)); + } + + return globals; + }, + + ExpressionStatement(node, parents, globalScope) { + let isGlobal = helpers.getIsGlobalThis(parents); + let globals = []; + + // Note: We check the expression types here and only call the necessary + // functions to aid performance. + if (node.expression.type === "AssignmentExpression") { + globals = convertThisAssignmentExpressionToGlobals(node, isGlobal); + } else if (node.expression.type === "CallExpression") { + globals = convertCallExpressionToGlobals(node, isGlobal); + } + + // Here we assume that if importScripts is set in the global scope, then + // this is a worker. It would be nice if eslint gave us a way of getting + // the environment directly. + // + // If this is testing context without path, ignore import. + if (globalScope && globalScope.set.get("importScripts") && this.dirname) { + let workerDetails = convertWorkerExpressionToGlobals( + node, + isGlobal, + this.dirname + ); + globals = globals.concat(workerDetails); + } + + return globals; + }, +}; + +module.exports = { + /** + * Returns all globals for a given file. Recursively searches through + * import-globals-from directives and also includes globals defined by + * standard eslint directives. + * + * @param {String} filePath + * The absolute path of the file to be parsed. + * @param {Object} astOptions + * Extra options to pass to the parser. + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ + getGlobalsForFile(filePath, astOptions = {}) { + if (globalCache.has(filePath)) { + return globalCache.get(filePath); + } + + if (globalDiscoveryInProgressForFiles.has(filePath)) { + // We're already processing this file, so return an empty set for now - + // the initial processing will pick up on the globals for this file. + return []; + } + globalDiscoveryInProgressForFiles.add(filePath); + + let content = fs.readFileSync(filePath, "utf8"); + + // Parse the content into an AST + let { ast, scopeManager, visitorKeys } = helpers.parseCode( + content, + astOptions + ); + + // Discover global declarations + let globalScope = scopeManager.acquire(ast); + + let globals = Object.keys(globalScope.variables).map(v => ({ + name: globalScope.variables[v].name, + writable: true, + })); + + // Walk over the AST to find any of our custom globals + let handler = new GlobalsForNode(filePath); + + helpers.walkAST(ast, visitorKeys, (type, node, parents) => { + if (type in handler) { + let newGlobals = handler[type](node, parents, globalScope); + globals.push.apply(globals, newGlobals); + } + }); + + globalCache.set(filePath, globals); + + globalDiscoveryInProgressForFiles.delete(filePath); + return globals; + }, + + /** + * Returns all globals for a code. + * This is only for testing. + * + * @param {String} code + * The JS code + * @param {Object} astOptions + * Extra options to pass to the parser. + * @return {Array} + * An array of objects that contain details about the globals: + * - {String} name + * The name of the global. + * - {Boolean} writable + * If the global is writeable or not. + */ + getGlobalsForCode(code, astOptions = {}) { + // Parse the content into an AST + let { ast, scopeManager, visitorKeys } = helpers.parseCode( + code, + astOptions, + { useBabel: false } + ); + + // Discover global declarations + let globalScope = scopeManager.acquire(ast); + + let globals = Object.keys(globalScope.variables).map(v => ({ + name: globalScope.variables[v].name, + writable: true, + })); + + // Walk over the AST to find any of our custom globals + let handler = new GlobalsForNode(null); + + helpers.walkAST(ast, visitorKeys, (type, node, parents) => { + if (type in handler) { + let newGlobals = handler[type](node, parents, globalScope); + globals.push.apply(globals, newGlobals); + } + }); + + return globals; + }, + + /** + * Returns all the globals for an html file that are defined by imported + * scripts (i.e. + + + Do some work for 500ms. + + diff --git a/tools/profiler/tests/browser/firefox-logo-nightly.svg b/tools/profiler/tests/browser/firefox-logo-nightly.svg new file mode 100644 index 0000000000..f1af370d87 --- /dev/null +++ b/tools/profiler/tests/browser/firefox-logo-nightly.svg @@ -0,0 +1 @@ +firefox-logo-nightly \ No newline at end of file diff --git a/tools/profiler/tests/browser/head.js b/tools/profiler/tests/browser/head.js new file mode 100644 index 0000000000..ef0e3128c0 --- /dev/null +++ b/tools/profiler/tests/browser/head.js @@ -0,0 +1,159 @@ +/* import-globals-from ../shared-head.js */ + +Services.scriptloader.loadSubScript( + "chrome://mochitests/content/browser/tools/profiler/tests/browser/shared-head.js", + this +); + +const BASE_URL = "http://example.com/browser/tools/profiler/tests/browser/"; +const BASE_URL_HTTPS = + "https://example.com/browser/tools/profiler/tests/browser/"; + +registerCleanupFunction(async () => { + if (Services.profiler.IsActive()) { + info( + "The profiler was found to still be running at the end of the test, which means that some error likely occured. Let's stop it to prevent issues with following tests!" + ); + await Services.profiler.StopProfiler(); + } +}); + +/** + * This is a helper function that will stop the profiler and returns the main + * threads for the parent process and the content process with PID contentPid. + * This happens immediately, without waiting for any sampling to happen or + * finish. Use waitSamplingAndStopProfilerAndGetThreads below instead to wait + * for samples before stopping. + * This returns also the full profile in case the caller wants more information. + * + * @param {number} contentPid + * @returns {Promise<{profile, parentThread, contentProcess, contentThread}>} + */ +async function stopProfilerNowAndGetThreads(contentPid) { + const profile = await stopNowAndGetProfile(); + + const parentThread = profile.threads[0]; + const contentProcess = profile.processes.find( + p => p.threads[0].pid == contentPid + ); + if (!contentProcess) { + throw new Error( + `Could not find the content process with given pid: ${contentPid}` + ); + } + + if (!parentThread) { + throw new Error("The parent thread was not found in the profile."); + } + + const contentThread = contentProcess.threads[0]; + if (!contentThread) { + throw new Error("The content thread was not found in the profile."); + } + + return { profile, parentThread, contentProcess, contentThread }; +} + +/** + * This is a helper function that will stop the profiler and returns the main + * threads for the parent process and the content process with PID contentPid. + * As opposed to stopProfilerNowAndGetThreads (with "Now") above, the profiler + * in that PID will not stop until there is at least one periodic sample taken. + * + * @param {number} contentPid + * @returns {Promise<{profile, parentThread, contentProcess, contentThread}>} + */ +async function waitSamplingAndStopProfilerAndGetThreads(contentPid) { + await Services.profiler.waitOnePeriodicSampling(); + + return stopProfilerNowAndGetThreads(contentPid); +} + +/** This tries to find the service worker thread by targeting a very specific + * UserTiming marker. Indeed we use performance.mark to add this marker from the + * service worker's events. + * Then from this thread we get its parent thread. Indeed the parent thread is + * where all network stuff happens, so this is useful for network marker tests. + * + * @param {Object} profile + * @returns {{ serviceWorkerThread: Object, serviceWorkerParentThread: Object }} the found threads + */ +function findServiceWorkerThreads(profile) { + const allThreads = [ + profile.threads, + ...profile.processes.map(process => process.threads), + ].flat(); + + const serviceWorkerThread = allThreads.find( + ({ processType, markers }) => + processType === "tab" && + markers.data.some(markerTuple => { + const data = markerTuple[markers.schema.data]; + return ( + data && + data.type === "UserTiming" && + data.name === "__serviceworker_event" + ); + }) + ); + + if (!serviceWorkerThread) { + info( + "We couldn't find a service worker thread. Here are all the threads in this profile:" + ); + allThreads.forEach(logInformationForThread.bind(null, "")); + return null; + } + + const serviceWorkerParentThread = allThreads.find( + ({ name, pid }) => pid === serviceWorkerThread.pid && name === "GeckoMain" + ); + + if (!serviceWorkerParentThread) { + info( + `We couldn't find a parent thread for the service worker thread (pid: ${serviceWorkerThread.pid}, tid: ${serviceWorkerThread.tid}).` + ); + info("Here are all the threads in this profile:"); + allThreads.forEach(logInformationForThread.bind(null, "")); + + // Let's write the profile on disk if MOZ_UPLOAD_DIR is present + const path = Services.env.get("MOZ_UPLOAD_DIR"); + if (path) { + const profileName = `profile_${Date.now()}.json`; + const profilePath = PathUtils.join(path, profileName); + info( + `We wrote down the profile on disk as an artifact, with name ${profileName}.` + ); + // This function returns a Promise, but we're not waiting on it because + // we're in a synchronous function. Hopefully writing will be finished + // when the process ends. + IOUtils.writeJSON(profilePath, profile).catch(err => + console.error("An error happened when writing the profile on disk", err) + ); + } + throw new Error( + "We couldn't find a parent thread for the service worker thread. Please read logs to find more information." + ); + } + + return { serviceWorkerThread, serviceWorkerParentThread }; +} + +/** + * This logs some basic information about the passed thread. + * + * @param {string} prefix + * @param {Object} thread + */ +function logInformationForThread(prefix, thread) { + if (!thread) { + info(prefix + ": thread is null or undefined."); + return; + } + + const { name, pid, tid, processName, processType } = thread; + info( + `${prefix}: ` + + `name(${name}) pid(${pid}) tid(${tid}) processName(${processName}) processType(${processType})` + ); +} diff --git a/tools/profiler/tests/browser/multi_frame.html b/tools/profiler/tests/browser/multi_frame.html new file mode 100644 index 0000000000..b2efcedd50 --- /dev/null +++ b/tools/profiler/tests/browser/multi_frame.html @@ -0,0 +1,11 @@ + + + + + Multi Frame + + + Multi Frame + + + diff --git a/tools/profiler/tests/browser/page_with_resources.html b/tools/profiler/tests/browser/page_with_resources.html new file mode 100644 index 0000000000..9d2bb8f218 --- /dev/null +++ b/tools/profiler/tests/browser/page_with_resources.html @@ -0,0 +1,11 @@ + + + + + + + Testing + + + + diff --git a/tools/profiler/tests/browser/redirect.sjs b/tools/profiler/tests/browser/redirect.sjs new file mode 100644 index 0000000000..2a325c3d0b --- /dev/null +++ b/tools/profiler/tests/browser/redirect.sjs @@ -0,0 +1,8 @@ +function handleRequest(request, response) { + response.setStatusLine(request.httpVersion, 301, "Moved Permanently"); + response.setHeader( + "Location", + decodeURIComponent(request.queryString), + false + ); +} diff --git a/tools/profiler/tests/browser/serviceworkers/firefox-logo-nightly.svg b/tools/profiler/tests/browser/serviceworkers/firefox-logo-nightly.svg new file mode 100644 index 0000000000..f1af370d87 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/firefox-logo-nightly.svg @@ -0,0 +1 @@ +firefox-logo-nightly \ No newline at end of file diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker-utils.js b/tools/profiler/tests/browser/serviceworkers/serviceworker-utils.js new file mode 100644 index 0000000000..16a9f0c91f --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker-utils.js @@ -0,0 +1,39 @@ +// Most of this file has been stolen from dom/serviceworkers/test/utils.js. + +function waitForState(worker, state) { + return new Promise((resolve, reject) => { + function onStateChange() { + if (worker.state === state) { + worker.removeEventListener("statechange", onStateChange); + resolve(); + } + if (worker.state === "redundant") { + worker.removeEventListener("statechange", onStateChange); + reject(new Error("The service worker failed to install.")); + } + } + + // First add an event listener, so we won't miss any change that happens + // before we check the current state. + worker.addEventListener("statechange", onStateChange); + + // Now check if the worker is already in the desired state. + onStateChange(); + }); +} + +async function registerServiceWorkerAndWait(serviceWorkerFile) { + if (!serviceWorkerFile) { + throw new Error( + "No service worker filename has been specified. Please specify a valid filename." + ); + } + + console.log(`...registering the serviceworker "${serviceWorkerFile}"`); + const reg = await navigator.serviceWorker.register(`./${serviceWorkerFile}`, { + scope: "./", + }); + console.log("...waiting for activation"); + await waitForState(reg.installing, "activated"); + console.log("...activated!"); +} diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_cache_first.js b/tools/profiler/tests/browser/serviceworkers/serviceworker_cache_first.js new file mode 100644 index 0000000000..baa07fd6d8 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_cache_first.js @@ -0,0 +1,34 @@ +const files = ["serviceworker_page.html", "firefox-logo-nightly.svg"]; +const cacheName = "v1"; + +self.addEventListener("install", event => { + performance.mark("__serviceworker_event"); + console.log("[SW]:", "Install event"); + + event.waitUntil(cacheAssets()); +}); + +async function cacheAssets() { + const cache = await caches.open(cacheName); + await cache.addAll(files); +} + +self.addEventListener("fetch", event => { + performance.mark("__serviceworker_event"); + console.log("Handling fetch event for", event.request.url); + event.respondWith(handleFetch(event.request)); +}); + +async function handleFetch(request) { + const cachedResponse = await caches.match(request); + if (cachedResponse) { + console.log("Found response in cache:", cachedResponse); + + return cachedResponse; + } + console.log("No response found in cache. About to fetch from network..."); + + const networkResponse = await fetch(request); + console.log("Response from network is:", networkResponse); + return networkResponse; +} diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_no_fetch_handler.js b/tools/profiler/tests/browser/serviceworkers/serviceworker_no_fetch_handler.js new file mode 100644 index 0000000000..f656665ca0 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_no_fetch_handler.js @@ -0,0 +1,4 @@ +self.addEventListener("install", event => { + performance.mark("__serviceworker_event"); + console.log("[SW]:", "Install event"); +}); diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_no_respondWith_in_fetch_handler.js b/tools/profiler/tests/browser/serviceworkers/serviceworker_no_respondWith_in_fetch_handler.js new file mode 100644 index 0000000000..255c8269a1 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_no_respondWith_in_fetch_handler.js @@ -0,0 +1,9 @@ +self.addEventListener("install", event => { + performance.mark("__serviceworker_event"); + console.log("[SW]:", "Install event"); +}); + +self.addEventListener("fetch", event => { + performance.mark("__serviceworker_event"); + console.log("Handling fetch event for", event.request.url); +}); diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_page.html b/tools/profiler/tests/browser/serviceworkers/serviceworker_page.html new file mode 100644 index 0000000000..1c2100a9d6 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_page.html @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_register.html b/tools/profiler/tests/browser/serviceworkers/serviceworker_register.html new file mode 100644 index 0000000000..86719787f4 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_register.html @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_simple.html b/tools/profiler/tests/browser/serviceworkers/serviceworker_simple.html new file mode 100644 index 0000000000..f7c32d02c3 --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_simple.html @@ -0,0 +1,9 @@ + + + + + + + Testing + + diff --git a/tools/profiler/tests/browser/serviceworkers/serviceworker_synthetized_response.js b/tools/profiler/tests/browser/serviceworkers/serviceworker_synthetized_response.js new file mode 100644 index 0000000000..891b679a5f --- /dev/null +++ b/tools/profiler/tests/browser/serviceworkers/serviceworker_synthetized_response.js @@ -0,0 +1,27 @@ +self.addEventListener("install", event => { + performance.mark("__serviceworker_event"); + dump("[SW]:", "Install event\n"); +}); + +self.addEventListener("fetch", event => { + performance.mark("__serviceworker_event"); + dump(`Handling fetch event for ${event.request.url}\n`); + event.respondWith(handleFetch(event.request)); +}); + +async function handleFetch(request) { + if (request.url.endsWith("-generated.svg")) { + dump( + "An icon file that should be generated was requested, let's answer directly.\n" + ); + return new Response( + `firefox-logo-nightly`, + { headers: { "content-type": "image/svg+xml" } } + ); + } + + dump( + `A normal URL ${request.url} has been requested, let's fetch it from the network.\n` + ); + return fetch(request); +} diff --git a/tools/profiler/tests/browser/simple.html b/tools/profiler/tests/browser/simple.html new file mode 100644 index 0000000000..f7c32d02c3 --- /dev/null +++ b/tools/profiler/tests/browser/simple.html @@ -0,0 +1,9 @@ + + + + + + + Testing + + diff --git a/tools/profiler/tests/browser/single_frame.html b/tools/profiler/tests/browser/single_frame.html new file mode 100644 index 0000000000..ebdfc41da2 --- /dev/null +++ b/tools/profiler/tests/browser/single_frame.html @@ -0,0 +1,10 @@ + + + + + Single Frame + + + Single Frame + + diff --git a/tools/profiler/tests/chrome/chrome.ini b/tools/profiler/tests/chrome/chrome.ini new file mode 100644 index 0000000000..7089b8fb8e --- /dev/null +++ b/tools/profiler/tests/chrome/chrome.ini @@ -0,0 +1,8 @@ +[DEFAULT] +skip-if = tsan # Bug 1804081 +support-files=profiler_test_utils.js + +[test_profile_worker_bug_1428076.html] +skip-if = os == 'android' && processor == 'arm' # Bug 1541291 +[test_profile_worker.html] +skip-if = os == 'android' && processor == 'arm' # Bug 1541291 diff --git a/tools/profiler/tests/chrome/profiler_test_utils.js b/tools/profiler/tests/chrome/profiler_test_utils.js new file mode 100644 index 0000000000..d2e4499b34 --- /dev/null +++ b/tools/profiler/tests/chrome/profiler_test_utils.js @@ -0,0 +1,66 @@ +"use strict"; + +(function () { + async function startProfiler(settings) { + let startPromise = Services.profiler.StartProfiler( + settings.entries, + settings.interval, + settings.features, + settings.threads, + 0, + settings.duration + ); + + info("Parent Profiler has started"); + + await startPromise; + + info("Child profilers have started"); + } + + function getProfile() { + const profile = Services.profiler.getProfileData(); + info( + "We got a profile, run the mochitest with `--keep-open true` to see the logged profile in the Web Console." + ); + + // Run the mochitest with `--keep-open true` to see the logged profile in the + // Web console. + console.log(profile); + + return profile; + } + + async function stopProfiler() { + let stopPromise = Services.profiler.StopProfiler(); + info("Parent profiler has stopped"); + await stopPromise; + info("Child profilers have stopped"); + } + + function end(error) { + if (error) { + ok(false, `We got an error: ${error}`); + } else { + ok(true, "We ran the whole process"); + } + SimpleTest.finish(); + } + + async function runTest(settings, workload) { + SimpleTest.waitForExplicitFinish(); + try { + await startProfiler(settings); + await workload(); + await getProfile(); + await stopProfiler(); + await end(); + } catch (e) { + // By catching and handling the error, we're being nice to mochitest + // runners: instead of waiting for the timeout, we fail right away. + await end(e); + } + } + + window.runTest = runTest; +})(); diff --git a/tools/profiler/tests/chrome/test_profile_worker.html b/tools/profiler/tests/chrome/test_profile_worker.html new file mode 100644 index 0000000000..8e2bae7fbd --- /dev/null +++ b/tools/profiler/tests/chrome/test_profile_worker.html @@ -0,0 +1,66 @@ + + + + + + Test for Bug 1428076 + + + + +Mozilla Bug 1428076 + + + + + + diff --git a/tools/profiler/tests/chrome/test_profile_worker_bug_1428076.html b/tools/profiler/tests/chrome/test_profile_worker_bug_1428076.html new file mode 100644 index 0000000000..abe0e5748a --- /dev/null +++ b/tools/profiler/tests/chrome/test_profile_worker_bug_1428076.html @@ -0,0 +1,58 @@ + + + + + + Test for Bug 1428076 + + + + +Mozilla Bug 1428076 + + + + + + diff --git a/tools/profiler/tests/gtest/GeckoProfiler.cpp b/tools/profiler/tests/gtest/GeckoProfiler.cpp new file mode 100644 index 0000000000..78456662f5 --- /dev/null +++ b/tools/profiler/tests/gtest/GeckoProfiler.cpp @@ -0,0 +1,5099 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This file tests a lot of the profiler_*() functions in GeckoProfiler.h. +// Most of the tests just check that nothing untoward (e.g. crashes, deadlocks) +// happens when calling these functions. They don't do much inspection of +// profiler internals. + +#include "mozilla/ProfilerThreadPlatformData.h" +#include "mozilla/ProfilerThreadRegistration.h" +#include "mozilla/ProfilerThreadRegistrationInfo.h" +#include "mozilla/ProfilerThreadRegistry.h" +#include "mozilla/ProfilerUtils.h" +#include "mozilla/ProgressLogger.h" +#include "mozilla/UniquePtrExtensions.h" + +#include "nsIThread.h" +#include "nsThreadUtils.h" +#include "prthread.h" + +#include "gtest/gtest.h" +#include "mozilla/gtest/MozAssertions.h" + +#include + +#if defined(_MSC_VER) || defined(__MINGW32__) +# include +# include +#elif defined(__APPLE__) +# include +#endif + +#ifdef XP_WIN +#include "mozilla/WindowsVersion.h" +#endif + +#ifdef MOZ_GECKO_PROFILER + +# include "GeckoProfiler.h" +# include "mozilla/ProfilerMarkerTypes.h" +# include "mozilla/ProfilerMarkers.h" +# include "NetworkMarker.h" +# include "platform.h" +# include "ProfileBuffer.h" +# include "ProfilerControl.h" + +# include "js/Initialization.h" +# include "js/Printf.h" +# include "jsapi.h" +# include "json/json.h" +# include "mozilla/Atomics.h" +# include "mozilla/BlocksRingBuffer.h" +# include "mozilla/DataMutex.h" +# include "mozilla/ProfileBufferEntrySerializationGeckoExtensions.h" +# include "mozilla/ProfileJSONWriter.h" +# include "mozilla/ScopeExit.h" +# include "mozilla/net/HttpBaseChannel.h" +# include "nsIChannelEventSink.h" +# include "nsIThread.h" +# include "nsThreadUtils.h" + +# include +# include + +#endif // MOZ_GECKO_PROFILER + +// Note: profiler_init() has already been called in XRE_main(), so we can't +// test it here. Likewise for profiler_shutdown(), and AutoProfilerInit +// (which is just an RAII wrapper for profiler_init() and profiler_shutdown()). + +using namespace mozilla; + +TEST(GeckoProfiler, ProfilerUtils) +{ + profiler_init_main_thread_id(); + + static_assert(std::is_same_v); + static_assert( + std::is_same_v); + ProfilerProcessId processId = profiler_current_process_id(); + EXPECT_TRUE(processId.IsSpecified()); + EXPECT_EQ(processId, baseprofiler::profiler_current_process_id()); + + static_assert( + std::is_same_v); + static_assert( + std::is_same_v); + EXPECT_EQ(profiler_current_thread_id(), + baseprofiler::profiler_current_thread_id()); + + ProfilerThreadId mainTestThreadId = profiler_current_thread_id(); + EXPECT_TRUE(mainTestThreadId.IsSpecified()); + + ProfilerThreadId mainThreadId = profiler_main_thread_id(); + EXPECT_TRUE(mainThreadId.IsSpecified()); + + EXPECT_EQ(mainThreadId, mainTestThreadId) + << "Test should run on the main thread"; + EXPECT_TRUE(profiler_is_main_thread()); + + std::thread testThread([&]() { + EXPECT_EQ(profiler_current_process_id(), processId); + + const ProfilerThreadId testThreadId = profiler_current_thread_id(); + EXPECT_TRUE(testThreadId.IsSpecified()); + EXPECT_NE(testThreadId, mainThreadId); + EXPECT_FALSE(profiler_is_main_thread()); + + EXPECT_EQ(baseprofiler::profiler_current_process_id(), processId); + EXPECT_EQ(baseprofiler::profiler_current_thread_id(), testThreadId); + EXPECT_EQ(baseprofiler::profiler_main_thread_id(), mainThreadId); + EXPECT_FALSE(baseprofiler::profiler_is_main_thread()); + }); + testThread.join(); +} + +TEST(GeckoProfiler, ThreadRegistrationInfo) +{ + profiler_init_main_thread_id(); + + TimeStamp ts = TimeStamp::Now(); + { + profiler::ThreadRegistrationInfo trInfo{ + "name", ProfilerThreadId::FromNumber(123), false, ts}; + EXPECT_STREQ(trInfo.Name(), "name"); + EXPECT_NE(trInfo.Name(), "name") + << "ThreadRegistrationInfo should keep its own copy of the name"; + EXPECT_EQ(trInfo.RegisterTime(), ts); + EXPECT_EQ(trInfo.ThreadId(), ProfilerThreadId::FromNumber(123)); + EXPECT_EQ(trInfo.IsMainThread(), false); + } + + // Make sure the next timestamp will be different from `ts`. + while (TimeStamp::Now() == ts) { + } + + { + profiler::ThreadRegistrationInfo trInfoHere{"Here"}; + EXPECT_STREQ(trInfoHere.Name(), "Here"); + EXPECT_NE(trInfoHere.Name(), "Here") + << "ThreadRegistrationInfo should keep its own copy of the name"; + TimeStamp baseRegistrationTime = + baseprofiler::detail::GetThreadRegistrationTime(); + if (baseRegistrationTime) { + EXPECT_EQ(trInfoHere.RegisterTime(), baseRegistrationTime); + } else { + EXPECT_GT(trInfoHere.RegisterTime(), ts); + } + EXPECT_EQ(trInfoHere.ThreadId(), profiler_current_thread_id()); + EXPECT_EQ(trInfoHere.ThreadId(), profiler_main_thread_id()) + << "Gtests are assumed to run on the main thread"; + EXPECT_EQ(trInfoHere.IsMainThread(), true) + << "Gtests are assumed to run on the main thread"; + } + + { + // Sub-thread test. + // These will receive sub-thread data (to test move at thread end). + TimeStamp tsThread; + ProfilerThreadId threadThreadId; + UniquePtr trInfoThreadPtr; + + std::thread testThread([&]() { + profiler::ThreadRegistrationInfo trInfoThread{"Thread"}; + EXPECT_STREQ(trInfoThread.Name(), "Thread"); + EXPECT_NE(trInfoThread.Name(), "Thread") + << "ThreadRegistrationInfo should keep its own copy of the name"; + EXPECT_GT(trInfoThread.RegisterTime(), ts); + EXPECT_EQ(trInfoThread.ThreadId(), profiler_current_thread_id()); + EXPECT_NE(trInfoThread.ThreadId(), profiler_main_thread_id()); + EXPECT_EQ(trInfoThread.IsMainThread(), false); + + tsThread = trInfoThread.RegisterTime(); + threadThreadId = trInfoThread.ThreadId(); + trInfoThreadPtr = + MakeUnique(std::move(trInfoThread)); + }); + testThread.join(); + + ASSERT_NE(trInfoThreadPtr, nullptr); + EXPECT_STREQ(trInfoThreadPtr->Name(), "Thread"); + EXPECT_EQ(trInfoThreadPtr->RegisterTime(), tsThread); + EXPECT_EQ(trInfoThreadPtr->ThreadId(), threadThreadId); + EXPECT_EQ(trInfoThreadPtr->IsMainThread(), false) + << "Gtests are assumed to run on the main thread"; + } +} + +static constexpr ThreadProfilingFeatures scEachAndAnyThreadProfilingFeatures[] = + {ThreadProfilingFeatures::CPUUtilization, ThreadProfilingFeatures::Sampling, + ThreadProfilingFeatures::Markers, ThreadProfilingFeatures::Any}; + +TEST(GeckoProfiler, ThreadProfilingFeaturesType) +{ + ASSERT_EQ(static_cast(ThreadProfilingFeatures::Any), 1u + 2u + 4u) + << "This test assumes that there are 3 binary choices 1+2+4; " + "Is this test up to date?"; + + EXPECT_EQ(Combine(ThreadProfilingFeatures::CPUUtilization, + ThreadProfilingFeatures::Sampling, + ThreadProfilingFeatures::Markers), + ThreadProfilingFeatures::Any); + + constexpr ThreadProfilingFeatures allThreadProfilingFeatures[] = { + ThreadProfilingFeatures::NotProfiled, + ThreadProfilingFeatures::CPUUtilization, + ThreadProfilingFeatures::Sampling, ThreadProfilingFeatures::Markers, + ThreadProfilingFeatures::Any}; + + for (ThreadProfilingFeatures f1 : allThreadProfilingFeatures) { + // Combine and Intersect are commutative. + for (ThreadProfilingFeatures f2 : allThreadProfilingFeatures) { + EXPECT_EQ(Combine(f1, f2), Combine(f2, f1)); + EXPECT_EQ(Intersect(f1, f2), Intersect(f2, f1)); + } + + // Combine works like OR. + EXPECT_EQ(Combine(f1, f1), f1); + EXPECT_EQ(Combine(f1, f1, f1), f1); + + // 'OR NotProfiled' doesn't change anything. + EXPECT_EQ(Combine(f1, ThreadProfilingFeatures::NotProfiled), f1); + + // 'OR Any' makes Any. + EXPECT_EQ(Combine(f1, ThreadProfilingFeatures::Any), + ThreadProfilingFeatures::Any); + + // Intersect works like AND. + EXPECT_EQ(Intersect(f1, f1), f1); + EXPECT_EQ(Intersect(f1, f1, f1), f1); + + // 'AND NotProfiled' erases anything. + EXPECT_EQ(Intersect(f1, ThreadProfilingFeatures::NotProfiled), + ThreadProfilingFeatures::NotProfiled); + + // 'AND Any' doesn't change anything. + EXPECT_EQ(Intersect(f1, ThreadProfilingFeatures::Any), f1); + } + + for (ThreadProfilingFeatures f1 : scEachAndAnyThreadProfilingFeatures) { + EXPECT_TRUE(DoFeaturesIntersect(f1, f1)); + + // NotProfiled doesn't intersect with any feature. + EXPECT_FALSE(DoFeaturesIntersect(f1, ThreadProfilingFeatures::NotProfiled)); + + // Any intersects with any feature. + EXPECT_TRUE(DoFeaturesIntersect(f1, ThreadProfilingFeatures::Any)); + } +} + +static void TestConstUnlockedConstReader( + const profiler::ThreadRegistration::UnlockedConstReader& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + EXPECT_STREQ(aData.Info().Name(), "Test thread"); + EXPECT_GE(aData.Info().RegisterTime(), aBeforeRegistration); + EXPECT_LE(aData.Info().RegisterTime(), aAfterRegistration); + EXPECT_EQ(aData.Info().ThreadId(), aThreadId); + EXPECT_FALSE(aData.Info().IsMainThread()); + +#if (defined(_MSC_VER) || defined(__MINGW32__)) && defined(MOZ_GECKO_PROFILER) + HANDLE threadHandle = aData.PlatformDataCRef().ProfiledThread(); + EXPECT_NE(threadHandle, nullptr); + EXPECT_EQ(ProfilerThreadId::FromNumber(::GetThreadId(threadHandle)), + aThreadId); + // Test calling QueryThreadCycleTime, we cannot assume that it will always + // work, but at least it shouldn't crash. + ULONG64 cycles; + (void)QueryThreadCycleTime(threadHandle, &cycles); +#elif defined(__APPLE__) && defined(MOZ_GECKO_PROFILER) + // Test calling thread_info, we cannot assume that it will always work, but at + // least it shouldn't crash. + thread_basic_info_data_t threadBasicInfo; + mach_msg_type_number_t basicCount = THREAD_BASIC_INFO_COUNT; + (void)thread_info( + aData.PlatformDataCRef().ProfiledThread(), THREAD_BASIC_INFO, + reinterpret_cast(&threadBasicInfo), &basicCount); +#elif (defined(__linux__) || defined(__ANDROID__) || defined(__FreeBSD__)) && \ + defined(MOZ_GECKO_PROFILER) + // Test calling GetClockId, we cannot assume that it will always work, but at + // least it shouldn't crash. + Maybe maybeClockId = aData.PlatformDataCRef().GetClockId(); + if (maybeClockId) { + // Test calling clock_gettime, we cannot assume that it will always work, + // but at least it shouldn't crash. + timespec ts; + (void)clock_gettime(*maybeClockId, &ts); + } +#else + (void)aData.PlatformDataCRef(); +#endif + + EXPECT_GE(aData.StackTop(), aOnStackObject) + << "StackTop should be at &onStackChar, or higher on some " + "platforms"; +}; + +static void TestConstUnlockedConstReaderAndAtomicRW( + const profiler::ThreadRegistration::UnlockedConstReaderAndAtomicRW& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedConstReader(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + + (void)aData.ProfilingStackCRef(); + + EXPECT_EQ(aData.ProfilingFeatures(), ThreadProfilingFeatures::NotProfiled); + + EXPECT_FALSE(aData.IsSleeping()); +}; + +static void TestUnlockedConstReaderAndAtomicRW( + profiler::ThreadRegistration::UnlockedConstReaderAndAtomicRW& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedConstReaderAndAtomicRW(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + (void)aData.ProfilingStackRef(); + + EXPECT_FALSE(aData.IsSleeping()); + aData.SetSleeping(); + EXPECT_TRUE(aData.IsSleeping()); + aData.SetAwake(); + EXPECT_FALSE(aData.IsSleeping()); + + aData.ReinitializeOnResume(); + + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + aData.SetSleeping(); + // After sleeping, the 2nd+ calls can duplicate. + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_TRUE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_TRUE(aData.CanDuplicateLastSampleDueToSleep()); + aData.ReinitializeOnResume(); + // After reinit (and sleeping), the 2nd+ calls can duplicate. + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_TRUE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_TRUE(aData.CanDuplicateLastSampleDueToSleep()); + aData.SetAwake(); + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); + EXPECT_FALSE(aData.CanDuplicateLastSampleDueToSleep()); +}; + +static void TestConstUnlockedRWForLockedProfiler( + const profiler::ThreadRegistration::UnlockedRWForLockedProfiler& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedConstReaderAndAtomicRW(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + // We can't create a PSAutoLock here, so just verify that the call would + // compile and return the expected type. + static_assert(std::is_same_v())), + const ProfiledThreadData*>); +}; + +static void TestConstUnlockedReaderAndAtomicRWOnThread( + const profiler::ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& + aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedRWForLockedProfiler(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + EXPECT_EQ(aData.GetJSContext(), nullptr); +}; + +static void TestUnlockedRWForLockedProfiler( + profiler::ThreadRegistration::UnlockedRWForLockedProfiler& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedRWForLockedProfiler(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + TestUnlockedConstReaderAndAtomicRW(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + // No functions to test here. +}; + +static void TestUnlockedReaderAndAtomicRWOnThread( + profiler::ThreadRegistration::UnlockedReaderAndAtomicRWOnThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedReaderAndAtomicRWOnThread(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + TestUnlockedRWForLockedProfiler(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + // No functions to test here. +}; + +static void TestConstLockedRWFromAnyThread( + const profiler::ThreadRegistration::LockedRWFromAnyThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstUnlockedReaderAndAtomicRWOnThread(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + EXPECT_EQ(aData.GetJsFrameBuffer(), nullptr); + EXPECT_EQ(aData.GetEventTarget(), nullptr); +}; + +static void TestLockedRWFromAnyThread( + profiler::ThreadRegistration::LockedRWFromAnyThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstLockedRWFromAnyThread(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + TestUnlockedReaderAndAtomicRWOnThread(aData, aBeforeRegistration, + aAfterRegistration, aOnStackObject, + aThreadId); + + // We can't create a ProfiledThreadData nor PSAutoLock here, so just verify + // that the call would compile and return the expected type. + static_assert(std::is_same_v(), + std::declval(), + std::declval())), + void>); + + aData.ResetMainThread(nullptr); + + TimeDuration delay = TimeDuration::FromSeconds(1); + TimeDuration running = TimeDuration::FromSeconds(1); + aData.GetRunningEventDelay(TimeStamp::Now(), delay, running); + EXPECT_TRUE(delay.IsZero()); + EXPECT_TRUE(running.IsZero()); + + aData.StartJSSampling(123u); + aData.StopJSSampling(); +}; + +static void TestConstLockedRWOnThread( + const profiler::ThreadRegistration::LockedRWOnThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstLockedRWFromAnyThread(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + + // No functions to test here. +}; + +static void TestLockedRWOnThread( + profiler::ThreadRegistration::LockedRWOnThread& aData, + const TimeStamp& aBeforeRegistration, const TimeStamp& aAfterRegistration, + const void* aOnStackObject, + ProfilerThreadId aThreadId = profiler_current_thread_id()) { + TestConstLockedRWOnThread(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + TestLockedRWFromAnyThread(aData, aBeforeRegistration, aAfterRegistration, + aOnStackObject, aThreadId); + + // We don't want to really call SetJSContext here, so just verify that + // the call would compile and return the expected type. + static_assert( + std::is_same_v())), + void>); + aData.ClearJSContext(); + aData.PollJSSampling(); +}; + +TEST(GeckoProfiler, ThreadRegistration_DataAccess) +{ + using TR = profiler::ThreadRegistration; + + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + // Note that the main thread could already be registered, so we work in a new + // thread to test an actual registration that we control. + + std::thread testThread([&]() { + ASSERT_FALSE(TR::IsRegistered()) + << "A new std::thread should not start registered"; + EXPECT_FALSE(TR::GetOnThreadPtr()); + EXPECT_FALSE(TR::WithOnThreadRefOr([&](auto) { return true; }, false)); + + char onStackChar; + + TimeStamp beforeRegistration = TimeStamp::Now(); + TR tr{"Test thread", &onStackChar}; + TimeStamp afterRegistration = TimeStamp::Now(); + + ASSERT_TRUE(TR::IsRegistered()); + + // Note: This test will mostly be about checking the correct access to + // thread data, depending on how it's obtained. Not all the functionality + // related to that data is tested (e.g., because it involves JS or other + // external dependencies that would be difficult to control here.) + + auto TestOnThreadRef = [&](TR::OnThreadRef aOnThreadRef) { + // To test const-qualified member functions. + const TR::OnThreadRef& onThreadCRef = aOnThreadRef; + + // const UnlockedConstReader (always const) + + TestConstUnlockedConstReader(onThreadCRef.UnlockedConstReaderCRef(), + beforeRegistration, afterRegistration, + &onStackChar); + onThreadCRef.WithUnlockedConstReader( + [&](const TR::UnlockedConstReader& aData) { + TestConstUnlockedConstReader(aData, beforeRegistration, + afterRegistration, &onStackChar); + }); + + // const UnlockedConstReaderAndAtomicRW + + TestConstUnlockedConstReaderAndAtomicRW( + onThreadCRef.UnlockedConstReaderAndAtomicRWCRef(), beforeRegistration, + afterRegistration, &onStackChar); + onThreadCRef.WithUnlockedConstReaderAndAtomicRW( + [&](const TR::UnlockedConstReaderAndAtomicRW& aData) { + TestConstUnlockedConstReaderAndAtomicRW( + aData, beforeRegistration, afterRegistration, &onStackChar); + }); + + // non-const UnlockedConstReaderAndAtomicRW + + TestUnlockedConstReaderAndAtomicRW( + aOnThreadRef.UnlockedConstReaderAndAtomicRWRef(), beforeRegistration, + afterRegistration, &onStackChar); + aOnThreadRef.WithUnlockedConstReaderAndAtomicRW( + [&](TR::UnlockedConstReaderAndAtomicRW& aData) { + TestUnlockedConstReaderAndAtomicRW(aData, beforeRegistration, + afterRegistration, &onStackChar); + }); + + // const UnlockedRWForLockedProfiler + + TestConstUnlockedRWForLockedProfiler( + onThreadCRef.UnlockedRWForLockedProfilerCRef(), beforeRegistration, + afterRegistration, &onStackChar); + onThreadCRef.WithUnlockedRWForLockedProfiler( + [&](const TR::UnlockedRWForLockedProfiler& aData) { + TestConstUnlockedRWForLockedProfiler( + aData, beforeRegistration, afterRegistration, &onStackChar); + }); + + // non-const UnlockedRWForLockedProfiler + + TestUnlockedRWForLockedProfiler( + aOnThreadRef.UnlockedRWForLockedProfilerRef(), beforeRegistration, + afterRegistration, &onStackChar); + aOnThreadRef.WithUnlockedRWForLockedProfiler( + [&](TR::UnlockedRWForLockedProfiler& aData) { + TestUnlockedRWForLockedProfiler(aData, beforeRegistration, + afterRegistration, &onStackChar); + }); + + // const UnlockedReaderAndAtomicRWOnThread + + TestConstUnlockedReaderAndAtomicRWOnThread( + onThreadCRef.UnlockedReaderAndAtomicRWOnThreadCRef(), + beforeRegistration, afterRegistration, &onStackChar); + onThreadCRef.WithUnlockedReaderAndAtomicRWOnThread( + [&](const TR::UnlockedReaderAndAtomicRWOnThread& aData) { + TestConstUnlockedReaderAndAtomicRWOnThread( + aData, beforeRegistration, afterRegistration, &onStackChar); + }); + + // non-const UnlockedReaderAndAtomicRWOnThread + + TestUnlockedReaderAndAtomicRWOnThread( + aOnThreadRef.UnlockedReaderAndAtomicRWOnThreadRef(), + beforeRegistration, afterRegistration, &onStackChar); + aOnThreadRef.WithUnlockedReaderAndAtomicRWOnThread( + [&](TR::UnlockedReaderAndAtomicRWOnThread& aData) { + TestUnlockedReaderAndAtomicRWOnThread( + aData, beforeRegistration, afterRegistration, &onStackChar); + }); + + // LockedRWFromAnyThread + // Note: It cannot directly be accessed on the thread, this will be + // tested through LockedRWOnThread. + + // const LockedRWOnThread + + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + { + TR::OnThreadRef::ConstRWOnThreadWithLock constRWOnThreadWithLock = + onThreadCRef.ConstLockedRWOnThread(); + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + TestConstLockedRWOnThread(constRWOnThreadWithLock.DataCRef(), + beforeRegistration, afterRegistration, + &onStackChar); + } + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + onThreadCRef.WithConstLockedRWOnThread( + [&](const TR::LockedRWOnThread& aData) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + TestConstLockedRWOnThread(aData, beforeRegistration, + afterRegistration, &onStackChar); + }); + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + + // non-const LockedRWOnThread + + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + { + TR::OnThreadRef::RWOnThreadWithLock rwOnThreadWithLock = + aOnThreadRef.GetLockedRWOnThread(); + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + TestConstLockedRWOnThread(rwOnThreadWithLock.DataCRef(), + beforeRegistration, afterRegistration, + &onStackChar); + TestLockedRWOnThread(rwOnThreadWithLock.DataRef(), beforeRegistration, + afterRegistration, &onStackChar); + } + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + aOnThreadRef.WithLockedRWOnThread([&](TR::LockedRWOnThread& aData) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + TestLockedRWOnThread(aData, beforeRegistration, afterRegistration, + &onStackChar); + }); + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + }; + + TR::OnThreadPtr onThreadPtr = TR::GetOnThreadPtr(); + ASSERT_TRUE(onThreadPtr); + TestOnThreadRef(*onThreadPtr); + + TR::WithOnThreadRef( + [&](TR::OnThreadRef aOnThreadRef) { TestOnThreadRef(aOnThreadRef); }); + + EXPECT_TRUE(TR::WithOnThreadRefOr( + [&](TR::OnThreadRef aOnThreadRef) { + TestOnThreadRef(aOnThreadRef); + return true; + }, + false)); + }); + testThread.join(); +} + +// Thread name if registered, nullptr otherwise. +static const char* GetThreadName() { + return profiler::ThreadRegistration::WithOnThreadRefOr( + [](profiler::ThreadRegistration::OnThreadRef onThreadRef) { + return onThreadRef.WithUnlockedConstReader( + [](const profiler::ThreadRegistration::UnlockedConstReader& aData) { + return aData.Info().Name(); + }); + }, + nullptr); +} + +// Get the thread name, as registered in the PRThread, nullptr on failure. +static const char* GetPRThreadName() { + nsIThread* nsThread = NS_GetCurrentThread(); + if (!nsThread) { + return nullptr; + } + PRThread* prThread = nullptr; + if (NS_FAILED(nsThread->GetPRThread(&prThread))) { + return nullptr; + } + if (!prThread) { + return nullptr; + } + return PR_GetThreadName(prThread); +} + +TEST(GeckoProfiler, ThreadRegistration_MainThreadName) +{ + EXPECT_TRUE(profiler::ThreadRegistration::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "GeckoMain"); + + // Check that the real thread name (outside the profiler) is *not* GeckoMain. + EXPECT_STRNE(GetPRThreadName(), "GeckoMain"); +} + +TEST(GeckoProfiler, ThreadRegistration_NestedRegistrations) +{ + using TR = profiler::ThreadRegistration; + + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + // Note that the main thread could already be registered, so we work in a new + // thread to test actual registrations that we control. + + std::thread testThread([&]() { + ASSERT_FALSE(TR::IsRegistered()) + << "A new std::thread should not start registered"; + + char onStackChar; + + // Blocks {} are mostly for clarity, but some control on-stack registration + // lifetimes. + + // On-stack registration. + { + TR rt{"Test thread #1", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #1"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #1"); + } + ASSERT_FALSE(TR::IsRegistered()); + + // Off-stack registration. + { + TR::RegisterThread("Test thread #2", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #2"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #2"); + + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()); + } + + // Extra un-registration should be ignored. + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()); + + // Nested on-stack. + { + TR rt2{"Test thread #3", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #3"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #3"); + + { + TR rt3{"Test thread #4", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #3") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #3") + << "Nested registration shouldn't change the PRThread name"; + } + ASSERT_TRUE(TR::IsRegistered()) + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #3") + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #3"); + } + ASSERT_FALSE(TR::IsRegistered()); + + // Nested off-stack. + { + TR::RegisterThread("Test thread #5", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #5"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #5"); + + { + TR::RegisterThread("Test thread #6", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #5") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #5") + << "Nested registration shouldn't change the PRThread name"; + + TR::UnregisterThread(); + ASSERT_TRUE(TR::IsRegistered()) + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #5") + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #5"); + } + + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()); + } + + // Nested on- and off-stack. + { + TR rt2{"Test thread #7", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #7"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #7"); + + { + TR::RegisterThread("Test thread #8", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #7") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #7") + << "Nested registration shouldn't change the PRThread name"; + + TR::UnregisterThread(); + ASSERT_TRUE(TR::IsRegistered()) + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #7") + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #7"); + } + } + ASSERT_FALSE(TR::IsRegistered()); + + // Nested off- and on-stack. + { + TR::RegisterThread("Test thread #9", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #9"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #9"); + + { + TR rt3{"Test thread #10", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #9") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #9") + << "Nested registration shouldn't change the PRThread name"; + } + ASSERT_TRUE(TR::IsRegistered()) + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #9") + << "Thread should still be registered after nested un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #9"); + + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()); + } + + // Excess UnregisterThread with on-stack TR. + { + TR rt2{"Test thread #11", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #11"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #11"); + + TR::UnregisterThread(); + ASSERT_TRUE(TR::IsRegistered()) + << "On-stack thread should still be registered after off-stack " + "un-registration"; + EXPECT_STREQ(GetThreadName(), "Test thread #11") + << "On-stack thread should still be registered after off-stack " + "un-registration"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #11"); + } + ASSERT_FALSE(TR::IsRegistered()); + + // Excess on-thread TR destruction with already-unregistered root off-thread + // registration. + { + TR::RegisterThread("Test thread #12", &onStackChar); + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #12"); + EXPECT_STREQ(GetPRThreadName(), "Test thread #12"); + + { + TR rt3{"Test thread #13", &onStackChar}; + ASSERT_TRUE(TR::IsRegistered()); + EXPECT_STREQ(GetThreadName(), "Test thread #12") + << "Nested registration shouldn't change the name"; + EXPECT_STREQ(GetPRThreadName(), "Test thread #12") + << "Nested registration shouldn't change the PRThread name"; + + // Note that we unregister the root registration, while nested `rt3` is + // still alive. + TR::UnregisterThread(); + ASSERT_FALSE(TR::IsRegistered()) + << "UnregisterThread() of the root RegisterThread() should always work"; + + // At this end of this block, `rt3` will be destroyed, but nothing + // should happen. + } + ASSERT_FALSE(TR::IsRegistered()); + } + + ASSERT_FALSE(TR::IsRegistered()); + }); + testThread.join(); +} + +TEST(GeckoProfiler, ThreadRegistry_DataAccess) +{ + using TR = profiler::ThreadRegistration; + using TRy = profiler::ThreadRegistry; + + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + // Note that the main thread could already be registered, so we work in a new + // thread to test an actual registration that we control. + + std::thread testThread([&]() { + ASSERT_FALSE(TR::IsRegistered()) + << "A new std::thread should not start registered"; + EXPECT_FALSE(TR::GetOnThreadPtr()); + EXPECT_FALSE(TR::WithOnThreadRefOr([&](auto) { return true; }, false)); + + char onStackChar; + + TimeStamp beforeRegistration = TimeStamp::Now(); + TR tr{"Test thread", &onStackChar}; + TimeStamp afterRegistration = TimeStamp::Now(); + + ASSERT_TRUE(TR::IsRegistered()); + + // Note: This test will mostly be about checking the correct access to + // thread data, depending on how it's obtained. Not all the functionality + // related to that data is tested (e.g., because it involves JS or other + // external dependencies that would be difficult to control here.) + + const ProfilerThreadId testThreadId = profiler_current_thread_id(); + + auto testThroughRegistry = [&]() { + auto TestOffThreadRef = [&](TRy::OffThreadRef aOffThreadRef) { + // To test const-qualified member functions. + const TRy::OffThreadRef& offThreadCRef = aOffThreadRef; + + // const UnlockedConstReader (always const) + + TestConstUnlockedConstReader(offThreadCRef.UnlockedConstReaderCRef(), + beforeRegistration, afterRegistration, + &onStackChar, testThreadId); + offThreadCRef.WithUnlockedConstReader( + [&](const TR::UnlockedConstReader& aData) { + TestConstUnlockedConstReader(aData, beforeRegistration, + afterRegistration, &onStackChar, + testThreadId); + }); + + // const UnlockedConstReaderAndAtomicRW + + TestConstUnlockedConstReaderAndAtomicRW( + offThreadCRef.UnlockedConstReaderAndAtomicRWCRef(), + beforeRegistration, afterRegistration, &onStackChar, testThreadId); + offThreadCRef.WithUnlockedConstReaderAndAtomicRW( + [&](const TR::UnlockedConstReaderAndAtomicRW& aData) { + TestConstUnlockedConstReaderAndAtomicRW( + aData, beforeRegistration, afterRegistration, &onStackChar, + testThreadId); + }); + + // non-const UnlockedConstReaderAndAtomicRW + + TestUnlockedConstReaderAndAtomicRW( + aOffThreadRef.UnlockedConstReaderAndAtomicRWRef(), + beforeRegistration, afterRegistration, &onStackChar, testThreadId); + aOffThreadRef.WithUnlockedConstReaderAndAtomicRW( + [&](TR::UnlockedConstReaderAndAtomicRW& aData) { + TestUnlockedConstReaderAndAtomicRW(aData, beforeRegistration, + afterRegistration, + &onStackChar, testThreadId); + }); + + // const UnlockedRWForLockedProfiler + + TestConstUnlockedRWForLockedProfiler( + offThreadCRef.UnlockedRWForLockedProfilerCRef(), beforeRegistration, + afterRegistration, &onStackChar, testThreadId); + offThreadCRef.WithUnlockedRWForLockedProfiler( + [&](const TR::UnlockedRWForLockedProfiler& aData) { + TestConstUnlockedRWForLockedProfiler(aData, beforeRegistration, + afterRegistration, + &onStackChar, testThreadId); + }); + + // non-const UnlockedRWForLockedProfiler + + TestUnlockedRWForLockedProfiler( + aOffThreadRef.UnlockedRWForLockedProfilerRef(), beforeRegistration, + afterRegistration, &onStackChar, testThreadId); + aOffThreadRef.WithUnlockedRWForLockedProfiler( + [&](TR::UnlockedRWForLockedProfiler& aData) { + TestUnlockedRWForLockedProfiler(aData, beforeRegistration, + afterRegistration, &onStackChar, + testThreadId); + }); + + // UnlockedReaderAndAtomicRWOnThread + // Note: It cannot directly be accessed off the thread, this will be + // tested through LockedRWFromAnyThread. + + // const LockedRWFromAnyThread + + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + { + TRy::OffThreadRef::ConstRWFromAnyThreadWithLock + constRWFromAnyThreadWithLock = + offThreadCRef.ConstLockedRWFromAnyThread(); + if (profiler_current_thread_id() == testThreadId) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + } + TestConstLockedRWFromAnyThread( + constRWFromAnyThreadWithLock.DataCRef(), beforeRegistration, + afterRegistration, &onStackChar, testThreadId); + } + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + offThreadCRef.WithConstLockedRWFromAnyThread( + [&](const TR::LockedRWFromAnyThread& aData) { + if (profiler_current_thread_id() == testThreadId) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + } + TestConstLockedRWFromAnyThread(aData, beforeRegistration, + afterRegistration, &onStackChar, + testThreadId); + }); + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + + // non-const LockedRWFromAnyThread + + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + { + TRy::OffThreadRef::RWFromAnyThreadWithLock rwFromAnyThreadWithLock = + aOffThreadRef.GetLockedRWFromAnyThread(); + if (profiler_current_thread_id() == testThreadId) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + } + TestLockedRWFromAnyThread(rwFromAnyThreadWithLock.DataRef(), + beforeRegistration, afterRegistration, + &onStackChar, testThreadId); + } + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + aOffThreadRef.WithLockedRWFromAnyThread( + [&](TR::LockedRWFromAnyThread& aData) { + if (profiler_current_thread_id() == testThreadId) { + EXPECT_TRUE(TR::IsDataMutexLockedOnCurrentThread()); + } + TestLockedRWFromAnyThread(aData, beforeRegistration, + afterRegistration, &onStackChar, + testThreadId); + }); + EXPECT_FALSE(TR::IsDataMutexLockedOnCurrentThread()); + + // LockedRWOnThread + // Note: It can never be accessed off the thread. + }; + + int ranTest = 0; + TRy::WithOffThreadRef(testThreadId, [&](TRy::OffThreadRef aOffThreadRef) { + TestOffThreadRef(aOffThreadRef); + ++ranTest; + }); + EXPECT_EQ(ranTest, 1); + + EXPECT_TRUE(TRy::WithOffThreadRefOr( + testThreadId, + [&](TRy::OffThreadRef aOffThreadRef) { + TestOffThreadRef(aOffThreadRef); + return true; + }, + false)); + + ranTest = 0; + EXPECT_FALSE(TRy::IsRegistryMutexLockedOnCurrentThread()); + for (TRy::OffThreadRef offThreadRef : TRy::LockedRegistry{}) { + EXPECT_TRUE(TRy::IsRegistryMutexLockedOnCurrentThread() || + !TR::IsRegistered()); + if (offThreadRef.UnlockedConstReaderCRef().Info().ThreadId() == + testThreadId) { + TestOffThreadRef(offThreadRef); + ++ranTest; + } + } + EXPECT_EQ(ranTest, 1); + EXPECT_FALSE(TRy::IsRegistryMutexLockedOnCurrentThread()); + + { + ranTest = 0; + EXPECT_FALSE(TRy::IsRegistryMutexLockedOnCurrentThread()); + TRy::LockedRegistry lockedRegistry{}; + EXPECT_TRUE(TRy::IsRegistryMutexLockedOnCurrentThread() || + !TR::IsRegistered()); + for (TRy::OffThreadRef offThreadRef : lockedRegistry) { + if (offThreadRef.UnlockedConstReaderCRef().Info().ThreadId() == + testThreadId) { + TestOffThreadRef(offThreadRef); + ++ranTest; + } + } + EXPECT_EQ(ranTest, 1); + } + EXPECT_FALSE(TRy::IsRegistryMutexLockedOnCurrentThread()); + }; + + // Test on the current thread. + testThroughRegistry(); + + // Test from another thread. + std::thread otherThread([&]() { + ASSERT_NE(profiler_current_thread_id(), testThreadId); + testThroughRegistry(); + + // Test that this unregistered thread is really not registered. + int ranTest = 0; + TRy::WithOffThreadRef( + profiler_current_thread_id(), + [&](TRy::OffThreadRef aOffThreadRef) { ++ranTest; }); + EXPECT_EQ(ranTest, 0); + + EXPECT_FALSE(TRy::WithOffThreadRefOr( + profiler_current_thread_id(), + [&](TRy::OffThreadRef aOffThreadRef) { + ++ranTest; + return true; + }, + false)); + EXPECT_EQ(ranTest, 0); + }); + otherThread.join(); + }); + testThread.join(); +} + +TEST(GeckoProfiler, ThreadRegistration_RegistrationEdgeCases) +{ + using TR = profiler::ThreadRegistration; + using TRy = profiler::ThreadRegistry; + + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + // Note that the main thread could already be registered, so we work in a new + // thread to test an actual registration that we control. + + int registrationCount = 0; + int otherThreadLoops = 0; + int otherThreadReads = 0; + + // This thread will register and unregister in a loop, with some pauses. + // Another thread will attempty to access the test thread, and lock its data. + // The main goal is to check edges cases around (un)registrations. + std::thread testThread([&]() { + const ProfilerThreadId testThreadId = profiler_current_thread_id(); + + const TimeStamp endTestAt = TimeStamp::Now() + TimeDuration::FromSeconds(1); + + std::thread otherThread([&]() { + // Initial sleep so that testThread can start its loop. + PR_Sleep(PR_MillisecondsToInterval(1)); + + while (TimeStamp::Now() < endTestAt) { + ++otherThreadLoops; + + TRy::WithOffThreadRef(testThreadId, [&](TRy::OffThreadRef + aOffThreadRef) { + if (otherThreadLoops % 1000 == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + TRy::OffThreadRef::RWFromAnyThreadWithLock rwFromAnyThreadWithLock = + aOffThreadRef.GetLockedRWFromAnyThread(); + ++otherThreadReads; + if (otherThreadReads % 1000 == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + }); + } + }); + + while (TimeStamp::Now() < endTestAt) { + ASSERT_FALSE(TR::IsRegistered()) + << "A new std::thread should not start registered"; + EXPECT_FALSE(TR::GetOnThreadPtr()); + EXPECT_FALSE(TR::WithOnThreadRefOr([&](auto) { return true; }, false)); + + char onStackChar; + + TR tr{"Test thread", &onStackChar}; + ++registrationCount; + + ASSERT_TRUE(TR::IsRegistered()); + + int ranTest = 0; + TRy::WithOffThreadRef(testThreadId, [&](TRy::OffThreadRef aOffThreadRef) { + if (registrationCount % 2000 == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + ++ranTest; + }); + EXPECT_EQ(ranTest, 1); + + if (registrationCount % 1000 == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + } + + otherThread.join(); + }); + + testThread.join(); + + // It's difficult to guess what these numbers should be, but they definitely + // should be non-zero. The main goal was to test that nothing goes wrong. + EXPECT_GT(registrationCount, 0); + EXPECT_GT(otherThreadLoops, 0); + EXPECT_GT(otherThreadReads, 0); +} + +#ifdef MOZ_GECKO_PROFILER + +TEST(BaseProfiler, BlocksRingBuffer) +{ + constexpr uint32_t MBSize = 256; + uint8_t buffer[MBSize * 3]; + for (size_t i = 0; i < MBSize * 3; ++i) { + buffer[i] = uint8_t('A' + i); + } + BlocksRingBuffer rb(BlocksRingBuffer::ThreadSafety::WithMutex, + &buffer[MBSize], MakePowerOfTwo32()); + + { + nsCString cs("nsCString"_ns); + nsString s(u"nsString"_ns); + nsAutoCString acs("nsAutoCString"_ns); + nsAutoString as(u"nsAutoString"_ns); + nsAutoCStringN<8> acs8("nsAutoCStringN"_ns); + nsAutoStringN<8> as8(u"nsAutoStringN"_ns); + JS::UniqueChars jsuc = JS_smprintf("%s", "JS::UniqueChars"); + + rb.PutObjects(cs, s, acs, as, acs8, as8, jsuc); + } + + rb.ReadEach([](ProfileBufferEntryReader& aER) { + ASSERT_EQ(aER.ReadObject(), "nsCString"_ns); + ASSERT_EQ(aER.ReadObject(), u"nsString"_ns); + ASSERT_EQ(aER.ReadObject(), "nsAutoCString"_ns); + ASSERT_EQ(aER.ReadObject(), u"nsAutoString"_ns); + ASSERT_EQ(aER.ReadObject>(), "nsAutoCStringN"_ns); + ASSERT_EQ(aER.ReadObject>(), u"nsAutoStringN"_ns); + auto jsuc2 = aER.ReadObject(); + ASSERT_TRUE(!!jsuc2); + ASSERT_TRUE(strcmp(jsuc2.get(), "JS::UniqueChars") == 0); + }); + + // Everything around the sub-buffer should be unchanged. + for (size_t i = 0; i < MBSize; ++i) { + ASSERT_EQ(buffer[i], uint8_t('A' + i)); + } + for (size_t i = MBSize * 2; i < MBSize * 3; ++i) { + ASSERT_EQ(buffer[i], uint8_t('A' + i)); + } +} + +// Common JSON checks. + +// Check that the given JSON string include no JSON whitespace characters +// (excluding those in property names and strings). +void JSONWhitespaceCheck(const char* aOutput) { + ASSERT_NE(aOutput, nullptr); + + enum class State { Data, String, StringEscaped }; + State state = State::Data; + size_t length = 0; + size_t whitespaces = 0; + for (const char* p = aOutput; *p != '\0'; ++p) { + ++length; + const char c = *p; + + switch (state) { + case State::Data: + if (c == '\n' || c == '\r' || c == ' ' || c == '\t') { + ++whitespaces; + } else if (c == '"') { + state = State::String; + } + break; + + case State::String: + if (c == '"') { + state = State::Data; + } else if (c == '\\') { + state = State::StringEscaped; + } + break; + + case State::StringEscaped: + state = State::String; + break; + } + } + + EXPECT_EQ(whitespaces, 0u); + EXPECT_GT(length, 0u); +} + +// Does the GETTER return a non-null TYPE? (Non-critical) +# define EXPECT_HAS_JSON(GETTER, TYPE) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).is##TYPE()) { \ + EXPECT_TRUE((GETTER).is##TYPE()) \ + << #GETTER " didn't return type " #TYPE; \ + } \ + } while (false) + +// Does the GETTER return a non-null TYPE? (Critical) +# define ASSERT_HAS_JSON(GETTER, TYPE) \ + do { \ + ASSERT_FALSE((GETTER).isNull()); \ + ASSERT_TRUE((GETTER).is##TYPE()); \ + } while (false) + +// Does the GETTER return a non-null TYPE? (Critical) +// If yes, store the reference to Json::Value into VARIABLE. +# define GET_JSON(VARIABLE, GETTER, TYPE) \ + ASSERT_HAS_JSON(GETTER, TYPE); \ + const Json::Value& VARIABLE = (GETTER) + +// Does the GETTER return a non-null TYPE? (Critical) +// If yes, store the value as `const TYPE` into VARIABLE. +# define GET_JSON_VALUE(VARIABLE, GETTER, TYPE) \ + ASSERT_HAS_JSON(GETTER, TYPE); \ + const auto VARIABLE = (GETTER).as##TYPE() + +// Non-const GET_JSON_VALUE. +# define GET_JSON_MUTABLE_VALUE(VARIABLE, GETTER, TYPE) \ + ASSERT_HAS_JSON(GETTER, TYPE); \ + auto VARIABLE = (GETTER).as##TYPE() + +// Checks that the GETTER's value is present, is of the expected TYPE, and has +// the expected VALUE. (Non-critical) +# define EXPECT_EQ_JSON(GETTER, TYPE, VALUE) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).is##TYPE()) { \ + EXPECT_TRUE((GETTER).is##TYPE()) \ + << #GETTER " didn't return type " #TYPE; \ + } else { \ + EXPECT_EQ((GETTER).as##TYPE(), (VALUE)); \ + } \ + } while (false) + +// Checks that the GETTER's value is present, and is a valid index into the +// STRINGTABLE array, pointing at the expected STRING. +# define EXPECT_EQ_STRINGTABLE(GETTER, STRINGTABLE, STRING) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).isUInt()) { \ + EXPECT_TRUE((GETTER).isUInt()) << #GETTER " didn't return an index"; \ + } else { \ + EXPECT_LT((GETTER).asUInt(), (STRINGTABLE).size()); \ + EXPECT_EQ_JSON((STRINGTABLE)[(GETTER).asUInt()], String, (STRING)); \ + } \ + } while (false) + +# define EXPECT_JSON_ARRAY_CONTAINS(GETTER, TYPE, VALUE) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).isArray()) { \ + EXPECT_TRUE((GETTER).is##TYPE()) << #GETTER " is not an array"; \ + } else if (const Json::ArrayIndex size = (GETTER).size(); size == 0u) { \ + EXPECT_NE(size, 0u) << #GETTER " is an empty array"; \ + } else { \ + bool found = false; \ + for (Json::ArrayIndex i = 0; i < size; ++i) { \ + if (!(GETTER)[i].is##TYPE()) { \ + EXPECT_TRUE((GETTER)[i].is##TYPE()) \ + << #GETTER "[" << i << "] is not " #TYPE; \ + break; \ + } \ + if ((GETTER)[i].as##TYPE() == (VALUE)) { \ + found = true; \ + break; \ + } \ + } \ + EXPECT_TRUE(found) << #GETTER " doesn't contain " #VALUE; \ + } \ + } while (false) + +# define EXPECT_JSON_ARRAY_EXCLUDES(GETTER, TYPE, VALUE) \ + do { \ + if ((GETTER).isNull()) { \ + EXPECT_FALSE((GETTER).isNull()) \ + << #GETTER " doesn't exist or is null"; \ + } else if (!(GETTER).isArray()) { \ + EXPECT_TRUE((GETTER).is##TYPE()) << #GETTER " is not an array"; \ + } else { \ + const Json::ArrayIndex size = (GETTER).size(); \ + for (Json::ArrayIndex i = 0; i < size; ++i) { \ + if (!(GETTER)[i].is##TYPE()) { \ + EXPECT_TRUE((GETTER)[i].is##TYPE()) \ + << #GETTER "[" << i << "] is not " #TYPE; \ + break; \ + } \ + if ((GETTER)[i].as##TYPE() == (VALUE)) { \ + EXPECT_TRUE((GETTER)[i].as##TYPE() != (VALUE)) \ + << #GETTER " contains " #VALUE; \ + break; \ + } \ + } \ + } \ + } while (false) + +// Check that the given process root contains all the expected properties. +static void JSONRootCheck(const Json::Value& aRoot, + bool aWithMainThread = true) { + ASSERT_TRUE(aRoot.isObject()); + + EXPECT_HAS_JSON(aRoot["libs"], Array); + + GET_JSON(meta, aRoot["meta"], Object); + EXPECT_HAS_JSON(meta["version"], UInt); + EXPECT_HAS_JSON(meta["startTime"], Double); + EXPECT_HAS_JSON(meta["profilingStartTime"], Double); + EXPECT_HAS_JSON(meta["contentEarliestTime"], Double); + EXPECT_HAS_JSON(meta["profilingEndTime"], Double); + + EXPECT_HAS_JSON(aRoot["pages"], Array); + + EXPECT_HAS_JSON(aRoot["profilerOverhead"], Object); + + // "counters" is only present if there is any data to report. + // Test that expect "counters" should test for its presence first. + if (aRoot.isMember("counters")) { + // We have "counters", test their overall validity. + GET_JSON(counters, aRoot["counters"], Array); + for (const Json::Value& counter : counters) { + ASSERT_TRUE(counter.isObject()); + EXPECT_HAS_JSON(counter["name"], String); + EXPECT_HAS_JSON(counter["category"], String); + EXPECT_HAS_JSON(counter["description"], String); + GET_JSON(sampleGroups, counter["sample_groups"], Array); + for (const Json::Value& sampleGroup : sampleGroups) { + ASSERT_TRUE(sampleGroup.isObject()); + EXPECT_HAS_JSON(sampleGroup["id"], UInt); + + GET_JSON(samples, sampleGroup["samples"], Object); + GET_JSON(samplesSchema, samples["schema"], Object); + EXPECT_GE(samplesSchema.size(), 3u); + GET_JSON_VALUE(samplesTime, samplesSchema["time"], UInt); + GET_JSON_VALUE(samplesNumber, samplesSchema["number"], UInt); + GET_JSON_VALUE(samplesCount, samplesSchema["count"], UInt); + GET_JSON(samplesData, samples["data"], Array); + double previousTime = 0.0; + for (const Json::Value& sample : samplesData) { + ASSERT_TRUE(sample.isArray()); + GET_JSON_VALUE(time, sample[samplesTime], Double); + EXPECT_GE(time, previousTime); + previousTime = time; + if (sample.isValidIndex(samplesNumber)) { + EXPECT_HAS_JSON(sample[samplesNumber], UInt64); + } + if (sample.isValidIndex(samplesCount)) { + EXPECT_HAS_JSON(sample[samplesCount], Int64); + } + } + } + } + } + + GET_JSON(threads, aRoot["threads"], Array); + const Json::ArrayIndex threadCount = threads.size(); + for (Json::ArrayIndex i = 0; i < threadCount; ++i) { + GET_JSON(thread, threads[i], Object); + EXPECT_HAS_JSON(thread["processType"], String); + EXPECT_HAS_JSON(thread["name"], String); + EXPECT_HAS_JSON(thread["registerTime"], Double); + GET_JSON(samples, thread["samples"], Object); + EXPECT_HAS_JSON(thread["markers"], Object); + EXPECT_HAS_JSON(thread["pid"], Int64); + EXPECT_HAS_JSON(thread["tid"], Int64); + GET_JSON(stackTable, thread["stackTable"], Object); + GET_JSON(frameTable, thread["frameTable"], Object); + GET_JSON(stringTable, thread["stringTable"], Array); + + GET_JSON(stackTableSchema, stackTable["schema"], Object); + EXPECT_GE(stackTableSchema.size(), 2u); + GET_JSON_VALUE(stackTablePrefix, stackTableSchema["prefix"], UInt); + GET_JSON_VALUE(stackTableFrame, stackTableSchema["frame"], UInt); + GET_JSON(stackTableData, stackTable["data"], Array); + + GET_JSON(frameTableSchema, frameTable["schema"], Object); + EXPECT_GE(frameTableSchema.size(), 1u); + GET_JSON_VALUE(frameTableLocation, frameTableSchema["location"], UInt); + GET_JSON(frameTableData, frameTable["data"], Array); + + GET_JSON(samplesSchema, samples["schema"], Object); + GET_JSON_VALUE(sampleStackIndex, samplesSchema["stack"], UInt); + GET_JSON(samplesData, samples["data"], Array); + for (const Json::Value& sample : samplesData) { + ASSERT_TRUE(sample.isArray()); + if (sample.isValidIndex(sampleStackIndex)) { + if (!sample[sampleStackIndex].isNull()) { + GET_JSON_MUTABLE_VALUE(stack, sample[sampleStackIndex], UInt); + EXPECT_TRUE(stackTableData.isValidIndex(stack)); + for (;;) { + // `stack` (from the sample, or from the callee frame's "prefix" in + // the previous loop) points into the stackTable. + GET_JSON(stackTableEntry, stackTableData[stack], Array); + GET_JSON_VALUE(frame, stackTableEntry[stackTableFrame], UInt); + + // The stackTable entry's "frame" points into the frameTable. + EXPECT_TRUE(frameTableData.isValidIndex(frame)); + GET_JSON(frameTableEntry, frameTableData[frame], Array); + GET_JSON_VALUE(location, frameTableEntry[frameTableLocation], UInt); + + // The frameTable entry's "location" points at a string. + EXPECT_TRUE(stringTable.isValidIndex(location)); + + // The stackTable entry's "prefix" is null for the root frame. + if (stackTableEntry[stackTablePrefix].isNull()) { + break; + } + // Otherwise it recursively points at the caller in the stackTable. + GET_JSON_VALUE(prefix, stackTableEntry[stackTablePrefix], UInt); + EXPECT_TRUE(stackTableData.isValidIndex(prefix)); + stack = prefix; + } + } + } + } + } + + if (aWithMainThread) { + ASSERT_GT(threadCount, 0u); + GET_JSON(thread0, threads[0], Object); + EXPECT_EQ_JSON(thread0["name"], String, "GeckoMain"); + } + + EXPECT_HAS_JSON(aRoot["pausedRanges"], Array); + + const Json::Value& processes = aRoot["processes"]; + if (!processes.isNull()) { + ASSERT_TRUE(processes.isArray()); + const Json::ArrayIndex processCount = processes.size(); + for (Json::ArrayIndex i = 0; i < processCount; ++i) { + GET_JSON(process, processes[i], Object); + JSONRootCheck(process, aWithMainThread); + } + } + + GET_JSON(profilingLog, aRoot["profilingLog"], Object); + EXPECT_EQ(profilingLog.size(), 1u); + for (auto it = profilingLog.begin(); it != profilingLog.end(); ++it) { + // The key should be a pid. + const auto key = it.name(); + for (const auto letter : key) { + EXPECT_GE(letter, '0'); + EXPECT_LE(letter, '9'); + } + // And the value should be an object. + GET_JSON(logForPid, profilingLog[key], Object); + // Its content is not defined, but we expect at least these: + EXPECT_HAS_JSON(logForPid["profilingLogBegin_TSms"], Double); + EXPECT_HAS_JSON(logForPid["profilingLogEnd_TSms"], Double); + } +} + +// Check that various expected top properties are in the JSON, and then call the +// provided `aJSONCheckFunction` with the JSON root object. +template +void JSONOutputCheck(const char* aOutput, + JSONCheckFunction&& aJSONCheckFunction) { + ASSERT_NE(aOutput, nullptr); + + JSONWhitespaceCheck(aOutput); + + // Extract JSON. + Json::Value parsedRoot; + Json::CharReaderBuilder builder; + const std::unique_ptr reader(builder.newCharReader()); + ASSERT_TRUE( + reader->parse(aOutput, strchr(aOutput, '\0'), &parsedRoot, nullptr)); + + JSONRootCheck(parsedRoot); + + std::forward(aJSONCheckFunction)(parsedRoot); +} + +// Returns `static_cast(-1)` if callback could not be installed. +static SamplingState WaitForSamplingState() { + Atomic samplingState{-1}; + + if (!profiler_callback_after_sampling([&](SamplingState aSamplingState) { + samplingState = static_cast(aSamplingState); + })) { + return static_cast(-1); + } + + while (samplingState == -1) { + } + + return static_cast(static_cast(samplingState)); +} + +typedef Vector StrVec; + +static void InactiveFeaturesAndParamsCheck() { + int entries; + Maybe duration; + double interval; + uint32_t features; + StrVec filters; + uint64_t activeTabID; + + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::NativeAllocations)); + + profiler_get_start_params(&entries, &duration, &interval, &features, &filters, + &activeTabID); + + ASSERT_TRUE(entries == 0); + ASSERT_TRUE(duration == Nothing()); + ASSERT_TRUE(interval == 0); + ASSERT_TRUE(features == 0); + ASSERT_TRUE(filters.empty()); + ASSERT_TRUE(activeTabID == 0); +} + +static void ActiveParamsCheck(int aEntries, double aInterval, + uint32_t aFeatures, const char** aFilters, + size_t aFiltersLen, uint64_t aActiveTabID, + const Maybe& aDuration = Nothing()) { + int entries; + Maybe duration; + double interval; + uint32_t features; + StrVec filters; + uint64_t activeTabID; + + profiler_get_start_params(&entries, &duration, &interval, &features, &filters, + &activeTabID); + + ASSERT_TRUE(entries == aEntries); + ASSERT_TRUE(duration == aDuration); + ASSERT_TRUE(interval == aInterval); + ASSERT_TRUE(features == aFeatures); + ASSERT_TRUE(filters.length() == aFiltersLen); + ASSERT_TRUE(activeTabID == aActiveTabID); + for (size_t i = 0; i < aFiltersLen; i++) { + ASSERT_TRUE(strcmp(filters[i], aFilters[i]) == 0); + } +} + +TEST(GeckoProfiler, FeaturesAndParams) +{ + InactiveFeaturesAndParamsCheck(); + + // Try a couple of features and filters. + { + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + +# define PROFILER_DEFAULT_DURATION 20 /* seconds, for tests only */ + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 100, + Some(PROFILER_DEFAULT_DURATION)); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 100, + Some(PROFILER_DEFAULT_DURATION)); + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } + + // Try some different features and filters. + { + uint32_t features = + ProfilerFeature::MainThreadIO | ProfilerFeature::IPCMessages; + const char* filters[] = {"GeckoMain", "Foo", "Bar"}; + + // Testing with some arbitrary buffer size (as could be provided by + // external code), which we convert to the appropriate power of 2. + profiler_start(PowerOfTwo32(999999), 3, features, filters, + MOZ_ARRAY_LENGTH(filters), 123, Some(25.0)); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(int(PowerOfTwo32(999999).Value()), 3, features, filters, + MOZ_ARRAY_LENGTH(filters), 123, Some(25.0)); + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } + + // Try with no duration + { + uint32_t features = + ProfilerFeature::MainThreadIO | ProfilerFeature::IPCMessages; + const char* filters[] = {"GeckoMain", "Foo", "Bar"}; + + profiler_start(PowerOfTwo32(999999), 3, features, filters, + MOZ_ARRAY_LENGTH(filters), 0, Nothing()); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(int(PowerOfTwo32(999999).Value()), 3, features, filters, + MOZ_ARRAY_LENGTH(filters), 0, Nothing()); + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } + + // Try all supported features, and filters that match all threads. + { + uint32_t availableFeatures = profiler_get_available_features(); + const char* filters[] = {""}; + + profiler_start(PowerOfTwo32(88888), 10, availableFeatures, filters, + MOZ_ARRAY_LENGTH(filters), 0, Some(15.0)); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(PowerOfTwo32(88888).Value(), 10, availableFeatures, + filters, MOZ_ARRAY_LENGTH(filters), 0, Some(15.0)); + + // Don't call profiler_stop() here. + } + + // Try no features, and filters that match no threads. + { + uint32_t features = 0; + const char* filters[] = {"NoThreadWillMatchThis"}; + + // Second profiler_start() call in a row without an intervening + // profiler_stop(); this will do an implicit profiler_stop() and restart. + profiler_start(PowerOfTwo32(0), 0, features, filters, + MOZ_ARRAY_LENGTH(filters), 0, Some(0.0)); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::IPCMessages)); + + // Entries and intervals go to defaults if 0 is specified. + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 0, Nothing()); + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + + // These calls are no-ops. + profiler_stop(); + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } +} + +TEST(GeckoProfiler, EnsureStarted) +{ + InactiveFeaturesAndParamsCheck(); + + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + { + // Inactive -> Active + profiler_ensure_started(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0, + Some(PROFILER_DEFAULT_DURATION)); + + ActiveParamsCheck( + PROFILER_DEFAULT_ENTRIES.Value(), PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0, Some(PROFILER_DEFAULT_DURATION)); + } + + { + // Active -> Active with same settings + + Maybe info0 = profiler_get_buffer_info(); + ASSERT_TRUE(info0->mRangeEnd > 0); + + // First, write some samples into the buffer. + PR_Sleep(PR_MillisecondsToInterval(500)); + + Maybe info1 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd > info0->mRangeEnd); + + // Call profiler_ensure_started with the same settings as before. + // This operation must not clear our buffer! + profiler_ensure_started(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0, + Some(PROFILER_DEFAULT_DURATION)); + + ActiveParamsCheck( + PROFILER_DEFAULT_ENTRIES.Value(), PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0, Some(PROFILER_DEFAULT_DURATION)); + + // Check that our position in the buffer stayed the same or advanced, but + // not by much, and the range-start after profiler_ensure_started shouldn't + // have passed the range-end before. + Maybe info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info2->mRangeEnd >= info1->mRangeEnd); + ASSERT_TRUE(info2->mRangeEnd - info1->mRangeEnd < + info1->mRangeEnd - info0->mRangeEnd); + ASSERT_TRUE(info2->mRangeStart < info1->mRangeEnd); + } + + { + // Active -> Active with *different* settings + + Maybe info1 = profiler_get_buffer_info(); + + // Call profiler_ensure_started with a different feature set than the one + // it's currently running with. This is supposed to stop and restart the + // profiler, thereby discarding the buffer contents. + uint32_t differentFeatures = features | ProfilerFeature::CPUUtilization; + profiler_ensure_started(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + differentFeatures, filters, + MOZ_ARRAY_LENGTH(filters), 0); + + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, differentFeatures, filters, + MOZ_ARRAY_LENGTH(filters), 0); + + // Check the the buffer was cleared, so its range-start should be at/after + // its range-end before. + Maybe info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info2->mRangeStart >= info1->mRangeEnd); + } + + { + // Active -> Inactive + + profiler_stop(); + + InactiveFeaturesAndParamsCheck(); + } +} + +TEST(GeckoProfiler, MultiRegistration) +{ + // This whole test only checks that function calls don't crash, they don't + // actually verify that threads get profiled or not. + + { + std::thread thread([]() { + char top; + profiler_register_thread("thread, no unreg", &top); + }); + thread.join(); + } + + { + std::thread thread([]() { profiler_unregister_thread(); }); + thread.join(); + } + + { + std::thread thread([]() { + char top; + profiler_register_thread("thread 1st", &top); + profiler_unregister_thread(); + profiler_register_thread("thread 2nd", &top); + profiler_unregister_thread(); + }); + thread.join(); + } + + { + std::thread thread([]() { + char top; + profiler_register_thread("thread once", &top); + profiler_register_thread("thread again", &top); + profiler_unregister_thread(); + }); + thread.join(); + } + + { + std::thread thread([]() { + char top; + profiler_register_thread("thread to unreg twice", &top); + profiler_unregister_thread(); + profiler_unregister_thread(); + }); + thread.join(); + } +} + +TEST(GeckoProfiler, DifferentThreads) +{ + InactiveFeaturesAndParamsCheck(); + + nsCOMPtr thread; + nsresult rv = NS_NewNamedThread("GeckoProfGTest", getter_AddRefs(thread)); + ASSERT_NS_SUCCEEDED(rv); + + // Control the profiler on a background thread and verify flags on the + // main thread. + { + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_DifferentThreads_Test::TestBody"_ns, thread, + NS_NewRunnableFunction( + "GeckoProfiler_DifferentThreads_Test::TestBody", [&]() { + profiler_start(PROFILER_DEFAULT_ENTRIES, + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 0); + })); + + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE(!profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 0); + + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_DifferentThreads_Test::TestBody"_ns, thread, + NS_NewRunnableFunction("GeckoProfiler_DifferentThreads_Test::TestBody", + [&]() { profiler_stop(); })); + + InactiveFeaturesAndParamsCheck(); + } + + // Control the profiler on the main thread and verify flags on a + // background thread. + { + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_DifferentThreads_Test::TestBody"_ns, thread, + NS_NewRunnableFunction( + "GeckoProfiler_DifferentThreads_Test::TestBody", [&]() { + ASSERT_TRUE(profiler_is_active()); + ASSERT_TRUE( + !profiler_feature_active(ProfilerFeature::MainThreadIO)); + ASSERT_TRUE( + !profiler_feature_active(ProfilerFeature::IPCMessages)); + + ActiveParamsCheck(PROFILER_DEFAULT_ENTRIES.Value(), + PROFILER_DEFAULT_INTERVAL, features, filters, + MOZ_ARRAY_LENGTH(filters), 0); + })); + + profiler_stop(); + + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_DifferentThreads_Test::TestBody"_ns, thread, + NS_NewRunnableFunction("GeckoProfiler_DifferentThreads_Test::TestBody", + [&]() { InactiveFeaturesAndParamsCheck(); })); + } + + thread->Shutdown(); +} + +TEST(GeckoProfiler, GetBacktrace) +{ + ASSERT_TRUE(!profiler_get_backtrace()); + + { + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + // These will be destroyed while the profiler is active. + static const int N = 100; + { + UniqueProfilerBacktrace u[N]; + for (int i = 0; i < N; i++) { + u[i] = profiler_get_backtrace(); + ASSERT_TRUE(u[i]); + } + } + + // These will be destroyed after the profiler stops. + UniqueProfilerBacktrace u[N]; + for (int i = 0; i < N; i++) { + u[i] = profiler_get_backtrace(); + ASSERT_TRUE(u[i]); + } + + profiler_stop(); + } + + ASSERT_TRUE(!profiler_get_backtrace()); +} + +TEST(GeckoProfiler, Pause) +{ + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test must run on the main thread"; + + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain", "Profiled GeckoProfiler.Pause"}; + + ASSERT_TRUE(!profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Ignored GeckoProfiler.Pause - before start"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - before start"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + }}.join(); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(!profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + } + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Ignored GeckoProfiler.Pause - after start"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - after start"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + }}.join(); + + // Check that we are writing samples while not paused. + Maybe info1 = profiler_get_buffer_info(); + PR_Sleep(PR_MillisecondsToInterval(500)); + Maybe info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd != info2->mRangeEnd); + + // Check that we are writing markers while not paused. + ASSERT_TRUE(profiler_thread_is_being_profiled_for_markers()); + ASSERT_TRUE( + profiler_thread_is_being_profiled_for_markers(ProfilerThreadId{})); + ASSERT_TRUE(profiler_thread_is_being_profiled_for_markers( + profiler_current_thread_id())); + ASSERT_TRUE( + profiler_thread_is_being_profiled_for_markers(profiler_main_thread_id())); + info1 = profiler_get_buffer_info(); + PROFILER_MARKER_UNTYPED("Not paused", OTHER, {}); + info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd != info2->mRangeEnd); + + profiler_pause(); + + ASSERT_TRUE(profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + } + ASSERT_TRUE(!profiler_thread_is_being_profiled_for_markers()); + ASSERT_TRUE( + !profiler_thread_is_being_profiled_for_markers(ProfilerThreadId{})); + ASSERT_TRUE(!profiler_thread_is_being_profiled_for_markers( + profiler_current_thread_id())); + ASSERT_TRUE(!profiler_thread_is_being_profiled_for_markers( + profiler_main_thread_id())); + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Ignored GeckoProfiler.Pause - after pause"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - after pause"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + }}.join(); + + // Check that we are not writing samples while paused. + info1 = profiler_get_buffer_info(); + PR_Sleep(PR_MillisecondsToInterval(500)); + info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd == info2->mRangeEnd); + + // Check that we are now writing markers while paused. + info1 = profiler_get_buffer_info(); + PROFILER_MARKER_UNTYPED("Paused", OTHER, {}); + info2 = profiler_get_buffer_info(); + ASSERT_TRUE(info1->mRangeEnd == info2->mRangeEnd); + PROFILER_MARKER_UNTYPED("Paused v2", OTHER, {}); + Maybe info3 = profiler_get_buffer_info(); + ASSERT_TRUE(info2->mRangeEnd == info3->mRangeEnd); + + profiler_resume(); + + ASSERT_TRUE(!profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + } + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Ignored GeckoProfiler.Pause - after resume"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - after resume"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(profiler_thread_is_being_profiled(profiler_main_thread_id(), + features)); + } + } + }}.join(); + + profiler_stop(); + + ASSERT_TRUE(!profiler_is_paused()); + for (ThreadProfilingFeatures features : scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled(profiler_current_thread_id(), + features)); + } + + std::thread{[&]() { + { + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD("Ignored GeckoProfiler.Pause - after stop"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + { + AUTO_PROFILER_REGISTER_THREAD( + "Profiled GeckoProfiler.Pause - after stop"); + for (ThreadProfilingFeatures features : + scEachAndAnyThreadProfilingFeatures) { + ASSERT_TRUE(!profiler_thread_is_being_profiled(features)); + ASSERT_TRUE( + !profiler_thread_is_being_profiled(ProfilerThreadId{}, features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_current_thread_id(), features)); + ASSERT_TRUE(!profiler_thread_is_being_profiled( + profiler_main_thread_id(), features)); + } + } + }}.join(); +} + +TEST(GeckoProfiler, Markers) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + PROFILER_MARKER("tracing event", OTHER, {}, Tracing, "A"); + PROFILER_MARKER("tracing start", OTHER, MarkerTiming::IntervalStart(), + Tracing, "A"); + PROFILER_MARKER("tracing end", OTHER, MarkerTiming::IntervalEnd(), Tracing, + "A"); + + auto bt = profiler_capture_backtrace(); + PROFILER_MARKER("tracing event with stack", OTHER, + MarkerStack::TakeBacktrace(std::move(bt)), Tracing, "B"); + + { AUTO_PROFILER_TRACING_MARKER("C", "auto tracing", OTHER); } + + PROFILER_MARKER_UNTYPED("M1", OTHER, {}); + PROFILER_MARKER_UNTYPED("M3", OTHER, {}); + + // Create three strings: two that are the maximum allowed length, and one that + // is one char longer. + static const size_t kMax = ProfileBuffer::kMaxFrameKeyLength; + UniquePtr okstr1 = MakeUnique(kMax); + UniquePtr okstr2 = MakeUnique(kMax); + UniquePtr longstr = MakeUnique(kMax + 1); + UniquePtr longstrCut = MakeUnique(kMax + 1); + for (size_t i = 0; i < kMax; i++) { + okstr1[i] = 'a'; + okstr2[i] = 'b'; + longstr[i] = 'c'; + longstrCut[i] = 'c'; + } + okstr1[kMax - 1] = '\0'; + okstr2[kMax - 1] = '\0'; + longstr[kMax] = '\0'; + longstrCut[kMax] = '\0'; + // Should be output as-is. + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("", LAYOUT, ""); + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("", LAYOUT, okstr1.get()); + // Should be output as label + space + okstr2. + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("okstr2", LAYOUT, okstr2.get()); + // Should be output with kMax length, ending with "...\0". + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("", LAYOUT, longstr.get()); + ASSERT_EQ(longstrCut[kMax - 4], 'c'); + longstrCut[kMax - 4] = '.'; + ASSERT_EQ(longstrCut[kMax - 3], 'c'); + longstrCut[kMax - 3] = '.'; + ASSERT_EQ(longstrCut[kMax - 2], 'c'); + longstrCut[kMax - 2] = '.'; + ASSERT_EQ(longstrCut[kMax - 1], 'c'); + longstrCut[kMax - 1] = '\0'; + + // Test basic markers 2.0. + EXPECT_TRUE( + profiler_add_marker("default-templated markers 2.0 with empty options", + geckoprofiler::category::OTHER, {})); + + PROFILER_MARKER_UNTYPED( + "default-templated markers 2.0 with option", OTHER, + MarkerStack::TakeBacktrace(profiler_capture_backtrace())); + + PROFILER_MARKER("explicitly-default-templated markers 2.0 with empty options", + OTHER, {}, NoPayload); + + EXPECT_TRUE(profiler_add_marker( + "explicitly-default-templated markers 2.0 with option", + geckoprofiler::category::OTHER, {}, + ::geckoprofiler::markers::NoPayload{})); + + // Used in markers below. + TimeStamp ts1 = TimeStamp::Now(); + + // Sleep briefly to ensure a sample is taken and the pending markers are + // processed. + PR_Sleep(PR_MillisecondsToInterval(500)); + + // Used in markers below. + TimeStamp ts2 = TimeStamp::Now(); + // ts1 and ts2 should be different thanks to the sleep. + EXPECT_NE(ts1, ts2); + + // Test most marker payloads. + + // Keep this one first! (It's used to record `ts1` and `ts2`, to compare + // to serialized numbers in other markers.) + EXPECT_TRUE(profiler_add_marker("FirstMarker", geckoprofiler::category::OTHER, + MarkerTiming::Interval(ts1, ts2), + geckoprofiler::markers::TextMarker{}, + "First Marker")); + + // User-defined marker type with different properties, and fake schema. + struct GtestMarker { + static constexpr Span MarkerTypeName() { + return MakeStringSpan("markers-gtest"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter, int aInt, + double aDouble, const mozilla::ProfilerString8View& aText, + const mozilla::ProfilerString8View& aUniqueText, + const mozilla::TimeStamp& aTime) { + aWriter.NullProperty("null"); + aWriter.BoolProperty("bool-false", false); + aWriter.BoolProperty("bool-true", true); + aWriter.IntProperty("int", aInt); + aWriter.DoubleProperty("double", aDouble); + aWriter.StringProperty("text", aText); + aWriter.UniqueStringProperty("unique text", aUniqueText); + aWriter.UniqueStringProperty("unique text again", aUniqueText); + aWriter.TimeProperty("time", aTime); + } + static mozilla::MarkerSchema MarkerTypeDisplay() { + // Note: This is an test function that is not intended to actually output + // that correctly matches StreamJSONMarkerData data above! Instead we only + // test that it outputs the expected JSON at the end. + using MS = mozilla::MarkerSchema; + MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable, + MS::Location::TimelineOverview, MS::Location::TimelineMemory, + MS::Location::TimelineIPC, MS::Location::TimelineFileIO, + MS::Location::StackChart}; + // All label functions. + schema.SetChartLabel("chart label"); + schema.SetTooltipLabel("tooltip label"); + schema.SetTableLabel("table label"); + // All data functions, all formats, all "searchable" values. + schema.AddKeyFormat("key with url", MS::Format::Url); + schema.AddKeyLabelFormat("key with label filePath", "label filePath", + MS::Format::FilePath); + schema.AddKeyFormatSearchable("key with string not-searchable", + MS::Format::String, + MS::Searchable::NotSearchable); + schema.AddKeyLabelFormatSearchable("key with label duration searchable", + "label duration", MS::Format::Duration, + MS::Searchable::Searchable); + schema.AddKeyFormat("key with time", MS::Format::Time); + schema.AddKeyFormat("key with seconds", MS::Format::Seconds); + schema.AddKeyFormat("key with milliseconds", MS::Format::Milliseconds); + schema.AddKeyFormat("key with microseconds", MS::Format::Microseconds); + schema.AddKeyFormat("key with nanoseconds", MS::Format::Nanoseconds); + schema.AddKeyFormat("key with bytes", MS::Format::Bytes); + schema.AddKeyFormat("key with percentage", MS::Format::Percentage); + schema.AddKeyFormat("key with integer", MS::Format::Integer); + schema.AddKeyFormat("key with decimal", MS::Format::Decimal); + schema.AddStaticLabelValue("static label", "static value"); + return schema; + } + }; + EXPECT_TRUE( + profiler_add_marker("Gtest custom marker", geckoprofiler::category::OTHER, + MarkerTiming::Interval(ts1, ts2), GtestMarker{}, 42, + 43.0, "gtest text", "gtest unique text", ts1)); + + // User-defined marker type with no data, special frontend schema. + struct GtestSpecialMarker { + static constexpr Span MarkerTypeName() { + return MakeStringSpan("markers-gtest-special"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter) {} + static mozilla::MarkerSchema MarkerTypeDisplay() { + return mozilla::MarkerSchema::SpecialFrontendLocation{}; + } + }; + EXPECT_TRUE(profiler_add_marker("Gtest special marker", + geckoprofiler::category::OTHER, {}, + GtestSpecialMarker{})); + + // User-defined marker type that is never used, so it shouldn't appear in the + // output. + struct GtestUnusedMarker { + static constexpr Span MarkerTypeName() { + return MakeStringSpan("markers-gtest-unused"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter) {} + static mozilla::MarkerSchema MarkerTypeDisplay() { + return mozilla::MarkerSchema::SpecialFrontendLocation{}; + } + }; + + // Make sure the compiler doesn't complain about this unused struct. + mozilla::Unused << GtestUnusedMarker{}; + + // Other markers in alphabetical order of payload class names. + + nsCOMPtr uri; + ASSERT_TRUE( + NS_SUCCEEDED(NS_NewURI(getter_AddRefs(uri), "http://mozilla.org/"_ns))); + // The marker name will be "Load : ". + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 1, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_START, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheHit, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false + /* const mozilla::net::TimingStruct* aTimings = nullptr */ + /* mozilla::UniquePtr aSource = + nullptr */ + /* const mozilla::Maybe& aContentType = + mozilla::Nothing() */ + /* nsIURI* aRedirectURI = nullptr */ + /* uint64_t aRedirectChannelId = 0 */ + ); + + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 2, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_STOP, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe& aContentType = + mozilla::Nothing() */ + Some(nsDependentCString("text/html")), + /* nsIURI* aRedirectURI = nullptr */ nullptr, + /* uint64_t aRedirectChannelId = 0 */ 0); + + nsCOMPtr redirectURI; + ASSERT_TRUE(NS_SUCCEEDED( + NS_NewURI(getter_AddRefs(redirectURI), "http://example.com/"_ns))); + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 3, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_REDIRECT, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe& aContentType = + mozilla::Nothing() */ + mozilla::Nothing(), + /* nsIURI* aRedirectURI = nullptr */ redirectURI, + /* uint32_t aRedirectFlags = 0 */ + nsIChannelEventSink::REDIRECT_TEMPORARY, + /* uint64_t aRedirectChannelId = 0 */ 103); + + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 4, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_REDIRECT, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe& aContentType = + mozilla::Nothing() */ + mozilla::Nothing(), + /* nsIURI* aRedirectURI = nullptr */ redirectURI, + /* uint32_t aRedirectFlags = 0 */ + nsIChannelEventSink::REDIRECT_PERMANENT, + /* uint64_t aRedirectChannelId = 0 */ 104); + + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 5, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_REDIRECT, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe& aContentType = + mozilla::Nothing() */ + mozilla::Nothing(), + /* nsIURI* aRedirectURI = nullptr */ redirectURI, + /* uint32_t aRedirectFlags = 0 */ nsIChannelEventSink::REDIRECT_INTERNAL, + /* uint64_t aRedirectChannelId = 0 */ 105); + + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 6, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_REDIRECT, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ false, + /* const mozilla::net::TimingStruct* aTimings = nullptr */ nullptr, + /* mozilla::UniquePtr aSource = + nullptr */ + nullptr, + /* const mozilla::Maybe& aContentType = + mozilla::Nothing() */ + mozilla::Nothing(), + /* nsIURI* aRedirectURI = nullptr */ redirectURI, + /* uint32_t aRedirectFlags = 0 */ nsIChannelEventSink::REDIRECT_INTERNAL | + nsIChannelEventSink::REDIRECT_STS_UPGRADE, + /* uint64_t aRedirectChannelId = 0 */ 106); + profiler_add_network_marker( + /* nsIURI* aURI */ uri, + /* const nsACString& aRequestMethod */ "GET"_ns, + /* int32_t aPriority */ 34, + /* uint64_t aChannelId */ 7, + /* NetworkLoadType aType */ net::NetworkLoadType::LOAD_START, + /* mozilla::TimeStamp aStart */ ts1, + /* mozilla::TimeStamp aEnd */ ts2, + /* int64_t aCount */ 56, + /* mozilla::net::CacheDisposition aCacheDisposition */ + net::kCacheUnresolved, + /* uint64_t aInnerWindowID */ 78, + /* bool aIsPrivateBrowsing */ true + /* const mozilla::net::TimingStruct* aTimings = nullptr */ + /* mozilla::UniquePtr aSource = + nullptr */ + /* const mozilla::Maybe& aContentType = + mozilla::Nothing() */ + /* nsIURI* aRedirectURI = nullptr */ + /* uint64_t aRedirectChannelId = 0 */ + ); + + EXPECT_TRUE(profiler_add_marker( + "Text in main thread with stack", geckoprofiler::category::OTHER, + {MarkerStack::Capture(), MarkerTiming::Interval(ts1, ts2)}, + geckoprofiler::markers::TextMarker{}, "")); + EXPECT_TRUE(profiler_add_marker( + "Text from main thread with stack", geckoprofiler::category::OTHER, + MarkerOptions(MarkerThreadId::MainThread(), MarkerStack::Capture()), + geckoprofiler::markers::TextMarker{}, "")); + + std::thread registeredThread([]() { + AUTO_PROFILER_REGISTER_THREAD("Marker test sub-thread"); + // Marker in non-profiled thread won't be stored. + EXPECT_FALSE(profiler_add_marker( + "Text in registered thread with stack", geckoprofiler::category::OTHER, + MarkerStack::Capture(), geckoprofiler::markers::TextMarker{}, "")); + // Marker will be stored in main thread, with stack from registered thread. + EXPECT_TRUE(profiler_add_marker( + "Text from registered thread with stack", + geckoprofiler::category::OTHER, + MarkerOptions(MarkerThreadId::MainThread(), MarkerStack::Capture()), + geckoprofiler::markers::TextMarker{}, "")); + }); + registeredThread.join(); + + std::thread unregisteredThread([]() { + // Marker in unregistered thread won't be stored. + EXPECT_FALSE(profiler_add_marker("Text in unregistered thread with stack", + geckoprofiler::category::OTHER, + MarkerStack::Capture(), + geckoprofiler::markers::TextMarker{}, "")); + // Marker will be stored in main thread, but stack cannot be captured in an + // unregistered thread. + EXPECT_TRUE(profiler_add_marker( + "Text from unregistered thread with stack", + geckoprofiler::category::OTHER, + MarkerOptions(MarkerThreadId::MainThread(), MarkerStack::Capture()), + geckoprofiler::markers::TextMarker{}, "")); + }); + unregisteredThread.join(); + + EXPECT_TRUE(profiler_add_marker("Tracing", geckoprofiler::category::OTHER, {}, + geckoprofiler::markers::Tracing{}, + "category")); + + EXPECT_TRUE(profiler_add_marker("Text", geckoprofiler::category::OTHER, {}, + geckoprofiler::markers::TextMarker{}, + "Text text")); + + // Ensure that we evaluate to false for markers with very long texts by + // testing against a ~3mb string. A string of this size should exceed the + // available buffer chunks (max: 2) that are available and be discarded. + EXPECT_FALSE(profiler_add_marker("Text", geckoprofiler::category::OTHER, {}, + geckoprofiler::markers::TextMarker{}, + std::string(3 * 1024 * 1024, 'x'))); + + EXPECT_TRUE(profiler_add_marker( + "MediaSample", geckoprofiler::category::OTHER, {}, + geckoprofiler::markers::MediaSampleMarker{}, 123, 456, 789)); + + SpliceableChunkedJSONWriter w{FailureLatchInfallibleSource::Singleton()}; + w.Start(); + EXPECT_TRUE(::profiler_stream_json_for_this_process(w).isOk()); + w.End(); + + EXPECT_FALSE(w.Failed()); + + UniquePtr profile = w.ChunkedWriteFunc().CopyData(); + ASSERT_TRUE(!!profile.get()); + + // Expected markers, in order. + enum State { + S_tracing_event, + S_tracing_start, + S_tracing_end, + S_tracing_event_with_stack, + S_tracing_auto_tracing_start, + S_tracing_auto_tracing_end, + S_M1, + S_M3, + S_Markers2DefaultEmptyOptions, + S_Markers2DefaultWithOptions, + S_Markers2ExplicitDefaultEmptyOptions, + S_Markers2ExplicitDefaultWithOptions, + S_FirstMarker, + S_CustomMarker, + S_SpecialMarker, + S_NetworkMarkerPayload_start, + S_NetworkMarkerPayload_stop, + S_NetworkMarkerPayload_redirect_temporary, + S_NetworkMarkerPayload_redirect_permanent, + S_NetworkMarkerPayload_redirect_internal, + S_NetworkMarkerPayload_redirect_internal_sts, + S_NetworkMarkerPayload_private_browsing, + + S_TextWithStack, + S_TextToMTWithStack, + S_RegThread_TextToMTWithStack, + S_UnregThread_TextToMTWithStack, + + S_LAST, + } state = State(0); + + // These will be set when first read from S_FirstMarker, then + // compared in following markers. + // TODO: Compute these values from the timestamps. + double ts1Double = 0.0; + double ts2Double = 0.0; + + JSONOutputCheck(profile.get(), [&](const Json::Value& root) { + { + GET_JSON(threads, root["threads"], Array); + ASSERT_EQ(threads.size(), 1u); + + { + GET_JSON(thread0, threads[0], Object); + + // Keep a reference to the string table in this block, it will be used + // below. + GET_JSON(stringTable, thread0["stringTable"], Array); + ASSERT_TRUE(stringTable.isArray()); + + // Test the expected labels in the string table. + bool foundEmpty = false; + bool foundOkstr1 = false; + bool foundOkstr2 = false; + const std::string okstr2Label = std::string("okstr2 ") + okstr2.get(); + bool foundTooLong = false; + for (const auto& s : stringTable) { + ASSERT_TRUE(s.isString()); + std::string sString = s.asString(); + if (sString.empty()) { + EXPECT_FALSE(foundEmpty); + foundEmpty = true; + } else if (sString == okstr1.get()) { + EXPECT_FALSE(foundOkstr1); + foundOkstr1 = true; + } else if (sString == okstr2Label) { + EXPECT_FALSE(foundOkstr2); + foundOkstr2 = true; + } else if (sString == longstrCut.get()) { + EXPECT_FALSE(foundTooLong); + foundTooLong = true; + } else { + EXPECT_NE(sString, longstr.get()); + } + } + EXPECT_TRUE(foundEmpty); + EXPECT_TRUE(foundOkstr1); + EXPECT_TRUE(foundOkstr2); + EXPECT_TRUE(foundTooLong); + + { + GET_JSON(markers, thread0["markers"], Object); + + { + GET_JSON(data, markers["data"], Array); + + for (const Json::Value& marker : data) { + // Name the indexes into the marker tuple: + // [name, startTime, endTime, phase, category, payload] + const unsigned int NAME = 0u; + const unsigned int START_TIME = 1u; + const unsigned int END_TIME = 2u; + const unsigned int PHASE = 3u; + const unsigned int CATEGORY = 4u; + const unsigned int PAYLOAD = 5u; + + const unsigned int PHASE_INSTANT = 0; + const unsigned int PHASE_INTERVAL = 1; + const unsigned int PHASE_START = 2; + const unsigned int PHASE_END = 3; + + const unsigned int SIZE_WITHOUT_PAYLOAD = 5u; + const unsigned int SIZE_WITH_PAYLOAD = 6u; + + ASSERT_TRUE(marker.isArray()); + // The payload is optional. + ASSERT_GE(marker.size(), SIZE_WITHOUT_PAYLOAD); + ASSERT_LE(marker.size(), SIZE_WITH_PAYLOAD); + + // root.threads[0].markers.data[i] is an array with 5 or 6 + // elements. + + ASSERT_TRUE(marker[NAME].isUInt()); // name id + GET_JSON(name, stringTable[marker[NAME].asUInt()], String); + std::string nameString = name.asString(); + + EXPECT_TRUE(marker[START_TIME].isNumeric()); + EXPECT_TRUE(marker[END_TIME].isNumeric()); + EXPECT_TRUE(marker[PHASE].isUInt()); + EXPECT_TRUE(marker[PHASE].asUInt() < 4); + EXPECT_TRUE(marker[CATEGORY].isUInt()); + +# define EXPECT_TIMING_INSTANT \ + EXPECT_NE(marker[START_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_INSTANT); +# define EXPECT_TIMING_INTERVAL \ + EXPECT_NE(marker[START_TIME].asDouble(), 0); \ + EXPECT_NE(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_INTERVAL); +# define EXPECT_TIMING_START \ + EXPECT_NE(marker[START_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_START); +# define EXPECT_TIMING_END \ + EXPECT_EQ(marker[START_TIME].asDouble(), 0); \ + EXPECT_NE(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_END); + +# define EXPECT_TIMING_INSTANT_AT(t) \ + EXPECT_EQ(marker[START_TIME].asDouble(), t); \ + EXPECT_EQ(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_INSTANT); +# define EXPECT_TIMING_INTERVAL_AT(start, end) \ + EXPECT_EQ(marker[START_TIME].asDouble(), start); \ + EXPECT_EQ(marker[END_TIME].asDouble(), end); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_INTERVAL); +# define EXPECT_TIMING_START_AT(start) \ + EXPECT_EQ(marker[START_TIME].asDouble(), start); \ + EXPECT_EQ(marker[END_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_START); +# define EXPECT_TIMING_END_AT(end) \ + EXPECT_EQ(marker[START_TIME].asDouble(), 0); \ + EXPECT_EQ(marker[END_TIME].asDouble(), end); \ + EXPECT_EQ(marker[PHASE].asUInt(), PHASE_END); + + if (marker.size() == SIZE_WITHOUT_PAYLOAD) { + // root.threads[0].markers.data[i] is an array with 5 elements, + // so there is no payload. + if (nameString == "M1") { + ASSERT_EQ(state, S_M1); + state = State(state + 1); + } else if (nameString == "M3") { + ASSERT_EQ(state, S_M3); + state = State(state + 1); + } else if (nameString == + "default-templated markers 2.0 with empty options") { + EXPECT_EQ(state, S_Markers2DefaultEmptyOptions); + state = State(S_Markers2DefaultEmptyOptions + 1); +// TODO: Re-enable this when bug 1646714 lands, and check for stack. +# if 0 + } else if (nameString == + "default-templated markers 2.0 with option") { + EXPECT_EQ(state, S_Markers2DefaultWithOptions); + state = State(S_Markers2DefaultWithOptions + 1); +# endif + } else if (nameString == + "explicitly-default-templated markers 2.0 with " + "empty " + "options") { + EXPECT_EQ(state, S_Markers2ExplicitDefaultEmptyOptions); + state = State(S_Markers2ExplicitDefaultEmptyOptions + 1); + } else if (nameString == + "explicitly-default-templated markers 2.0 with " + "option") { + EXPECT_EQ(state, S_Markers2ExplicitDefaultWithOptions); + state = State(S_Markers2ExplicitDefaultWithOptions + 1); + } + } else { + // root.threads[0].markers.data[i] is an array with 6 elements, + // so there is a payload. + GET_JSON(payload, marker[PAYLOAD], Object); + + // root.threads[0].markers.data[i][PAYLOAD] is an object + // (payload). + + // It should at least have a "type" string. + GET_JSON(type, payload["type"], String); + std::string typeString = type.asString(); + + if (nameString == "tracing event") { + EXPECT_EQ(state, S_tracing_event); + state = State(S_tracing_event + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_INSTANT; + EXPECT_EQ_JSON(payload["category"], String, "A"); + EXPECT_TRUE(payload["stack"].isNull()); + + } else if (nameString == "tracing start") { + EXPECT_EQ(state, S_tracing_start); + state = State(S_tracing_start + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_START; + EXPECT_EQ_JSON(payload["category"], String, "A"); + EXPECT_TRUE(payload["stack"].isNull()); + + } else if (nameString == "tracing end") { + EXPECT_EQ(state, S_tracing_end); + state = State(S_tracing_end + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_END; + EXPECT_EQ_JSON(payload["category"], String, "A"); + EXPECT_TRUE(payload["stack"].isNull()); + + } else if (nameString == "tracing event with stack") { + EXPECT_EQ(state, S_tracing_event_with_stack); + state = State(S_tracing_event_with_stack + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_INSTANT; + EXPECT_EQ_JSON(payload["category"], String, "B"); + EXPECT_TRUE(payload["stack"].isObject()); + + } else if (nameString == "auto tracing") { + switch (state) { + case S_tracing_auto_tracing_start: + state = State(S_tracing_auto_tracing_start + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_START; + EXPECT_EQ_JSON(payload["category"], String, "C"); + EXPECT_TRUE(payload["stack"].isNull()); + break; + case S_tracing_auto_tracing_end: + state = State(S_tracing_auto_tracing_end + 1); + EXPECT_EQ(typeString, "tracing"); + EXPECT_TIMING_END; + EXPECT_EQ_JSON(payload["category"], String, "C"); + ASSERT_TRUE(payload["stack"].isNull()); + break; + default: + EXPECT_TRUE(state == S_tracing_auto_tracing_start || + state == S_tracing_auto_tracing_end); + break; + } + + } else if (nameString == + "default-templated markers 2.0 with option") { + // TODO: Remove this when bug 1646714 lands. + EXPECT_EQ(state, S_Markers2DefaultWithOptions); + state = State(S_Markers2DefaultWithOptions + 1); + EXPECT_EQ(typeString, "NoPayloadUserData"); + EXPECT_FALSE(payload["stack"].isNull()); + + } else if (nameString == "FirstMarker") { + // Record start and end times, to compare with timestamps in + // following markers. + EXPECT_EQ(state, S_FirstMarker); + ts1Double = marker[START_TIME].asDouble(); + ts2Double = marker[END_TIME].asDouble(); + state = State(S_FirstMarker + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_EQ_JSON(payload["name"], String, "First Marker"); + + } else if (nameString == "Gtest custom marker") { + EXPECT_EQ(state, S_CustomMarker); + state = State(S_CustomMarker + 1); + EXPECT_EQ(typeString, "markers-gtest"); + EXPECT_EQ(payload.size(), 1u + 9u); + EXPECT_TRUE(payload["null"].isNull()); + EXPECT_EQ_JSON(payload["bool-false"], Bool, false); + EXPECT_EQ_JSON(payload["bool-true"], Bool, true); + EXPECT_EQ_JSON(payload["int"], Int64, 42); + EXPECT_EQ_JSON(payload["double"], Double, 43.0); + EXPECT_EQ_JSON(payload["text"], String, "gtest text"); + // Unique strings can be fetched from the string table. + ASSERT_TRUE(payload["unique text"].isUInt()); + auto textIndex = payload["unique text"].asUInt(); + GET_JSON(uniqueText, stringTable[textIndex], String); + ASSERT_TRUE(uniqueText.isString()); + ASSERT_EQ(uniqueText.asString(), "gtest unique text"); + // The duplicate unique text should have the exact same index. + EXPECT_EQ_JSON(payload["unique text again"], UInt, textIndex); + EXPECT_EQ_JSON(payload["time"], Double, ts1Double); + + } else if (nameString == "Gtest special marker") { + EXPECT_EQ(state, S_SpecialMarker); + state = State(S_SpecialMarker + 1); + EXPECT_EQ(typeString, "markers-gtest-special"); + EXPECT_EQ(payload.size(), 1u) << "Only 'type' in the payload"; + + } else if (nameString == "Load 1: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_start); + state = State(S_NetworkMarkerPayload_start + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 1); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Hit"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_TRUE(payload["RedirectURI"].isNull()); + EXPECT_TRUE(payload["redirectType"].isNull()); + EXPECT_TRUE(payload["isHttpToHttpsRedirect"].isNull()); + EXPECT_TRUE(payload["redirectId"].isNull()); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 2: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_stop); + state = State(S_NetworkMarkerPayload_stop + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 2); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_TRUE(payload["RedirectURI"].isNull()); + EXPECT_TRUE(payload["redirectType"].isNull()); + EXPECT_TRUE(payload["isHttpToHttpsRedirect"].isNull()); + EXPECT_TRUE(payload["redirectId"].isNull()); + EXPECT_EQ_JSON(payload["contentType"], String, "text/html"); + + } else if (nameString == "Load 3: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_redirect_temporary); + state = State(S_NetworkMarkerPayload_redirect_temporary + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 3); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_EQ_JSON(payload["RedirectURI"], String, + "http://example.com/"); + EXPECT_EQ_JSON(payload["redirectType"], String, "Temporary"); + EXPECT_EQ_JSON(payload["isHttpToHttpsRedirect"], Bool, false); + EXPECT_EQ_JSON(payload["redirectId"], Int64, 103); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 4: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_redirect_permanent); + state = State(S_NetworkMarkerPayload_redirect_permanent + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 4); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_EQ_JSON(payload["RedirectURI"], String, + "http://example.com/"); + EXPECT_EQ_JSON(payload["redirectType"], String, "Permanent"); + EXPECT_EQ_JSON(payload["isHttpToHttpsRedirect"], Bool, false); + EXPECT_EQ_JSON(payload["redirectId"], Int64, 104); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 5: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_redirect_internal); + state = State(S_NetworkMarkerPayload_redirect_internal + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 5); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_EQ_JSON(payload["RedirectURI"], String, + "http://example.com/"); + EXPECT_EQ_JSON(payload["redirectType"], String, "Internal"); + EXPECT_EQ_JSON(payload["isHttpToHttpsRedirect"], Bool, false); + EXPECT_EQ_JSON(payload["redirectId"], Int64, 105); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 6: http://mozilla.org/") { + EXPECT_EQ(state, + S_NetworkMarkerPayload_redirect_internal_sts); + state = + State(S_NetworkMarkerPayload_redirect_internal_sts + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 6); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_TRUE(payload["isPrivateBrowsing"].isNull()); + EXPECT_EQ_JSON(payload["RedirectURI"], String, + "http://example.com/"); + EXPECT_EQ_JSON(payload["redirectType"], String, "Internal"); + EXPECT_EQ_JSON(payload["isHttpToHttpsRedirect"], Bool, true); + EXPECT_EQ_JSON(payload["redirectId"], Int64, 106); + EXPECT_TRUE(payload["contentType"].isNull()); + + } else if (nameString == "Load 7: http://mozilla.org/") { + EXPECT_EQ(state, S_NetworkMarkerPayload_private_browsing); + state = State(S_NetworkMarkerPayload_private_browsing + 1); + EXPECT_EQ(typeString, "Network"); + EXPECT_EQ_JSON(payload["startTime"], Double, ts1Double); + EXPECT_EQ_JSON(payload["endTime"], Double, ts2Double); + EXPECT_EQ_JSON(payload["id"], Int64, 7); + EXPECT_EQ_JSON(payload["URI"], String, "http://mozilla.org/"); + EXPECT_EQ_JSON(payload["requestMethod"], String, "GET"); + EXPECT_EQ_JSON(payload["pri"], Int64, 34); + EXPECT_EQ_JSON(payload["count"], Int64, 56); + EXPECT_EQ_JSON(payload["cache"], String, "Unresolved"); + EXPECT_EQ_JSON(payload["isPrivateBrowsing"], Bool, true); + EXPECT_TRUE(payload["RedirectURI"].isNull()); + EXPECT_TRUE(payload["redirectType"].isNull()); + EXPECT_TRUE(payload["isHttpToHttpsRedirect"].isNull()); + EXPECT_TRUE(payload["redirectId"].isNull()); + EXPECT_TRUE(payload["contentType"].isNull()); + } else if (nameString == "Text in main thread with stack") { + EXPECT_EQ(state, S_TextWithStack); + state = State(S_TextWithStack + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_FALSE(payload["stack"].isNull()); + EXPECT_TIMING_INTERVAL_AT(ts1Double, ts2Double); + EXPECT_EQ_JSON(payload["name"], String, ""); + + } else if (nameString == "Text from main thread with stack") { + EXPECT_EQ(state, S_TextToMTWithStack); + state = State(S_TextToMTWithStack + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_FALSE(payload["stack"].isNull()); + EXPECT_EQ_JSON(payload["name"], String, ""); + + } else if (nameString == + "Text in registered thread with stack") { + ADD_FAILURE() + << "Unexpected 'Text in registered thread with stack'"; + + } else if (nameString == + "Text from registered thread with stack") { + EXPECT_EQ(state, S_RegThread_TextToMTWithStack); + state = State(S_RegThread_TextToMTWithStack + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_FALSE(payload["stack"].isNull()); + EXPECT_EQ_JSON(payload["name"], String, ""); + + } else if (nameString == + "Text in unregistered thread with stack") { + ADD_FAILURE() + << "Unexpected 'Text in unregistered thread with stack'"; + + } else if (nameString == + "Text from unregistered thread with stack") { + EXPECT_EQ(state, S_UnregThread_TextToMTWithStack); + state = State(S_UnregThread_TextToMTWithStack + 1); + EXPECT_EQ(typeString, "Text"); + EXPECT_TRUE(payload["stack"].isNull()); + EXPECT_EQ_JSON(payload["name"], String, ""); + } + } // marker with payload + } // for (marker : data) + } // markers.data + } // markers + } // thread0 + } // threads + // We should have read all expected markers. + EXPECT_EQ(state, S_LAST); + + { + GET_JSON(meta, root["meta"], Object); + + { + GET_JSON(markerSchema, meta["markerSchema"], Array); + + std::set testedSchemaNames; + + for (const Json::Value& schema : markerSchema) { + GET_JSON(name, schema["name"], String); + const std::string nameString = name.asString(); + + GET_JSON(display, schema["display"], Array); + + GET_JSON(data, schema["data"], Array); + + EXPECT_TRUE( + testedSchemaNames + .insert(std::string(nameString.data(), nameString.size())) + .second) + << "Each schema name should be unique (inserted once in the set)"; + + if (nameString == "Text") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 1u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "name"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Details"); + EXPECT_EQ_JSON(data[0u]["format"], String, "string"); + + } else if (nameString == "NoPayloadUserData") { + // TODO: Remove this when bug 1646714 lands. + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 0u); + + } else if (nameString == "FileIO") { + // These are defined in ProfilerIOInterposeObserver.cpp + + } else if (nameString == "tracing") { + EXPECT_EQ(display.size(), 3u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + EXPECT_EQ(display[2u].asString(), "timeline-overview"); + + ASSERT_EQ(data.size(), 1u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "category"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Type"); + EXPECT_EQ_JSON(data[0u]["format"], String, "string"); + + } else if (nameString == "BHR-detected hang") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 0u); + + } else if (nameString == "MainThreadLongTask") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 1u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "category"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Type"); + EXPECT_EQ_JSON(data[0u]["format"], String, "string"); + + } else if (nameString == "Log") { + EXPECT_EQ(display.size(), 1u); + EXPECT_EQ(display[0u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 2u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "module"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Module"); + EXPECT_EQ_JSON(data[0u]["format"], String, "string"); + + ASSERT_TRUE(data[1u].isObject()); + EXPECT_EQ_JSON(data[1u]["key"], String, "name"); + EXPECT_EQ_JSON(data[1u]["label"], String, "Name"); + EXPECT_EQ_JSON(data[1u]["format"], String, "string"); + + } else if (nameString == "MediaSample") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 3u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "sampleStartTimeUs"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Sample start time"); + EXPECT_EQ_JSON(data[0u]["format"], String, "microseconds"); + + ASSERT_TRUE(data[1u].isObject()); + EXPECT_EQ_JSON(data[1u]["key"], String, "sampleEndTimeUs"); + EXPECT_EQ_JSON(data[1u]["label"], String, "Sample end time"); + EXPECT_EQ_JSON(data[1u]["format"], String, "microseconds"); + + ASSERT_TRUE(data[2u].isObject()); + EXPECT_EQ_JSON(data[2u]["key"], String, "queueLength"); + EXPECT_EQ_JSON(data[2u]["label"], String, "Queue length"); + EXPECT_EQ_JSON(data[2u]["format"], String, "integer"); + + } else if (nameString == "VideoFallingBehind") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 2u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "videoFrameStartTimeUs"); + EXPECT_EQ_JSON(data[0u]["label"], String, "Video frame start time"); + EXPECT_EQ_JSON(data[0u]["format"], String, "microseconds"); + + ASSERT_TRUE(data[1u].isObject()); + EXPECT_EQ_JSON(data[1u]["key"], String, "mediaCurrentTimeUs"); + EXPECT_EQ_JSON(data[1u]["label"], String, "Media current time"); + EXPECT_EQ_JSON(data[1u]["format"], String, "microseconds"); + + } else if (nameString == "Budget") { + EXPECT_EQ(display.size(), 2u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + + ASSERT_EQ(data.size(), 0u); + + } else if (nameString == "markers-gtest") { + EXPECT_EQ(display.size(), 7u); + EXPECT_EQ(display[0u].asString(), "marker-chart"); + EXPECT_EQ(display[1u].asString(), "marker-table"); + EXPECT_EQ(display[2u].asString(), "timeline-overview"); + EXPECT_EQ(display[3u].asString(), "timeline-memory"); + EXPECT_EQ(display[4u].asString(), "timeline-ipc"); + EXPECT_EQ(display[5u].asString(), "timeline-fileio"); + EXPECT_EQ(display[6u].asString(), "stack-chart"); + + EXPECT_EQ_JSON(schema["chartLabel"], String, "chart label"); + EXPECT_EQ_JSON(schema["tooltipLabel"], String, "tooltip label"); + EXPECT_EQ_JSON(schema["tableLabel"], String, "table label"); + + ASSERT_EQ(data.size(), 14u); + + ASSERT_TRUE(data[0u].isObject()); + EXPECT_EQ_JSON(data[0u]["key"], String, "key with url"); + EXPECT_TRUE(data[0u]["label"].isNull()); + EXPECT_EQ_JSON(data[0u]["format"], String, "url"); + EXPECT_TRUE(data[0u]["searchable"].isNull()); + + ASSERT_TRUE(data[1u].isObject()); + EXPECT_EQ_JSON(data[1u]["key"], String, "key with label filePath"); + EXPECT_EQ_JSON(data[1u]["label"], String, "label filePath"); + EXPECT_EQ_JSON(data[1u]["format"], String, "file-path"); + EXPECT_TRUE(data[1u]["searchable"].isNull()); + + ASSERT_TRUE(data[2u].isObject()); + EXPECT_EQ_JSON(data[2u]["key"], String, + "key with string not-searchable"); + EXPECT_TRUE(data[2u]["label"].isNull()); + EXPECT_EQ_JSON(data[2u]["format"], String, "string"); + EXPECT_EQ_JSON(data[2u]["searchable"], Bool, false); + + ASSERT_TRUE(data[3u].isObject()); + EXPECT_EQ_JSON(data[3u]["key"], String, + "key with label duration searchable"); + EXPECT_TRUE(data[3u]["label duration"].isNull()); + EXPECT_EQ_JSON(data[3u]["format"], String, "duration"); + EXPECT_EQ_JSON(data[3u]["searchable"], Bool, true); + + ASSERT_TRUE(data[4u].isObject()); + EXPECT_EQ_JSON(data[4u]["key"], String, "key with time"); + EXPECT_TRUE(data[4u]["label"].isNull()); + EXPECT_EQ_JSON(data[4u]["format"], String, "time"); + EXPECT_TRUE(data[4u]["searchable"].isNull()); + + ASSERT_TRUE(data[5u].isObject()); + EXPECT_EQ_JSON(data[5u]["key"], String, "key with seconds"); + EXPECT_TRUE(data[5u]["label"].isNull()); + EXPECT_EQ_JSON(data[5u]["format"], String, "seconds"); + EXPECT_TRUE(data[5u]["searchable"].isNull()); + + ASSERT_TRUE(data[6u].isObject()); + EXPECT_EQ_JSON(data[6u]["key"], String, "key with milliseconds"); + EXPECT_TRUE(data[6u]["label"].isNull()); + EXPECT_EQ_JSON(data[6u]["format"], String, "milliseconds"); + EXPECT_TRUE(data[6u]["searchable"].isNull()); + + ASSERT_TRUE(data[7u].isObject()); + EXPECT_EQ_JSON(data[7u]["key"], String, "key with microseconds"); + EXPECT_TRUE(data[7u]["label"].isNull()); + EXPECT_EQ_JSON(data[7u]["format"], String, "microseconds"); + EXPECT_TRUE(data[7u]["searchable"].isNull()); + + ASSERT_TRUE(data[8u].isObject()); + EXPECT_EQ_JSON(data[8u]["key"], String, "key with nanoseconds"); + EXPECT_TRUE(data[8u]["label"].isNull()); + EXPECT_EQ_JSON(data[8u]["format"], String, "nanoseconds"); + EXPECT_TRUE(data[8u]["searchable"].isNull()); + + ASSERT_TRUE(data[9u].isObject()); + EXPECT_EQ_JSON(data[9u]["key"], String, "key with bytes"); + EXPECT_TRUE(data[9u]["label"].isNull()); + EXPECT_EQ_JSON(data[9u]["format"], String, "bytes"); + EXPECT_TRUE(data[9u]["searchable"].isNull()); + + ASSERT_TRUE(data[10u].isObject()); + EXPECT_EQ_JSON(data[10u]["key"], String, "key with percentage"); + EXPECT_TRUE(data[10u]["label"].isNull()); + EXPECT_EQ_JSON(data[10u]["format"], String, "percentage"); + EXPECT_TRUE(data[10u]["searchable"].isNull()); + + ASSERT_TRUE(data[11u].isObject()); + EXPECT_EQ_JSON(data[11u]["key"], String, "key with integer"); + EXPECT_TRUE(data[11u]["label"].isNull()); + EXPECT_EQ_JSON(data[11u]["format"], String, "integer"); + EXPECT_TRUE(data[11u]["searchable"].isNull()); + + ASSERT_TRUE(data[12u].isObject()); + EXPECT_EQ_JSON(data[12u]["key"], String, "key with decimal"); + EXPECT_TRUE(data[12u]["label"].isNull()); + EXPECT_EQ_JSON(data[12u]["format"], String, "decimal"); + EXPECT_TRUE(data[12u]["searchable"].isNull()); + + ASSERT_TRUE(data[13u].isObject()); + EXPECT_EQ_JSON(data[13u]["label"], String, "static label"); + EXPECT_EQ_JSON(data[13u]["value"], String, "static value"); + + } else if (nameString == "markers-gtest-special") { + EXPECT_EQ(display.size(), 0u); + ASSERT_EQ(data.size(), 0u); + + } else if (nameString == "markers-gtest-unused") { + ADD_FAILURE() << "Schema for GtestUnusedMarker should not be here"; + + } else { + printf("FYI: Unknown marker schema '%s'\n", nameString.c_str()); + } + } + + // Check that we've got all expected schema. + EXPECT_TRUE(testedSchemaNames.find("Text") != testedSchemaNames.end()); + EXPECT_TRUE(testedSchemaNames.find("tracing") != + testedSchemaNames.end()); + EXPECT_TRUE(testedSchemaNames.find("MediaSample") != + testedSchemaNames.end()); + } // markerSchema + } // meta + }); + + Maybe info = profiler_get_buffer_info(); + EXPECT_TRUE(info.isSome()); + printf("Profiler buffer range: %llu .. %llu (%llu bytes)\n", + static_cast(info->mRangeStart), + static_cast(info->mRangeEnd), + // sizeof(ProfileBufferEntry) == 9 + (static_cast(info->mRangeEnd) - + static_cast(info->mRangeStart)) * + 9); + printf("Stats: min(us) .. mean(us) .. max(us) [count]\n"); + printf("- Intervals: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mIntervalsUs.min, info->mIntervalsUs.sum / info->mIntervalsUs.n, + info->mIntervalsUs.max, info->mIntervalsUs.n); + printf("- Overheads: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mOverheadsUs.min, info->mOverheadsUs.sum / info->mOverheadsUs.n, + info->mOverheadsUs.max, info->mOverheadsUs.n); + printf(" - Locking: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mLockingsUs.min, info->mLockingsUs.sum / info->mLockingsUs.n, + info->mLockingsUs.max, info->mLockingsUs.n); + printf(" - Clearning: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mCleaningsUs.min, info->mCleaningsUs.sum / info->mCleaningsUs.n, + info->mCleaningsUs.max, info->mCleaningsUs.n); + printf(" - Counters: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mCountersUs.min, info->mCountersUs.sum / info->mCountersUs.n, + info->mCountersUs.max, info->mCountersUs.n); + printf(" - Threads: %7.1f .. %7.1f .. %7.1f [%u]\n", + info->mThreadsUs.min, info->mThreadsUs.sum / info->mThreadsUs.n, + info->mThreadsUs.max, info->mThreadsUs.n); + + profiler_stop(); + + // Try to add markers while the profiler is stopped. + PROFILER_MARKER_UNTYPED("marker after profiler_stop", OTHER); + + // Warning: this could be racy + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + // This last marker shouldn't get streamed. + SpliceableChunkedJSONWriter w2{FailureLatchInfallibleSource::Singleton()}; + w2.Start(); + EXPECT_TRUE(::profiler_stream_json_for_this_process(w2).isOk()); + w2.End(); + EXPECT_FALSE(w2.Failed()); + UniquePtr profile2 = w2.ChunkedWriteFunc().CopyData(); + ASSERT_TRUE(!!profile2.get()); + EXPECT_TRUE( + std::string_view(profile2.get()).find("marker after profiler_stop") == + std::string_view::npos); + + profiler_stop(); +} + +# define COUNTER_NAME "TestCounter" +# define COUNTER_DESCRIPTION "Test of counters in profiles" +# define COUNTER_NAME2 "Counter2" +# define COUNTER_DESCRIPTION2 "Second Test of counters in profiles" + +PROFILER_DEFINE_COUNT_TOTAL(TestCounter, COUNTER_NAME, COUNTER_DESCRIPTION); +PROFILER_DEFINE_COUNT_TOTAL(TestCounter2, COUNTER_NAME2, COUNTER_DESCRIPTION2); + +TEST(GeckoProfiler, Counters) +{ + uint32_t features = 0; + const char* filters[] = {"GeckoMain"}; + + // We will record some counter values, and check that they're present (and no + // other) when expected. + + struct NumberAndCount { + uint64_t mNumber; + int64_t mCount; + }; + + int64_t testCounters[] = {10, 7, -17}; + NumberAndCount expectedTestCounters[] = {{1u, 10}, {0u, 0}, {1u, 7}, + {0u, 0}, {0u, 0}, {1u, -17}, + {0u, 0}, {0u, 0}}; + constexpr size_t expectedTestCountersCount = + MOZ_ARRAY_LENGTH(expectedTestCounters); + + bool expectCounter2 = false; + int64_t testCounters2[] = {10}; + NumberAndCount expectedTestCounters2[] = {{1u, 10}, {0u, 0}}; + constexpr size_t expectedTestCounters2Count = + MOZ_ARRAY_LENGTH(expectedTestCounters2); + + auto checkCountersInJSON = [&](const Json::Value& aRoot) { + size_t nextExpectedTestCounter = 0u; + size_t nextExpectedTestCounter2 = 0u; + + GET_JSON(counters, aRoot["counters"], Array); + for (const Json::Value& counter : counters) { + ASSERT_TRUE(counter.isObject()); + GET_JSON_VALUE(name, counter["name"], String); + if (name == "TestCounter") { + EXPECT_EQ_JSON(counter["category"], String, COUNTER_NAME); + EXPECT_EQ_JSON(counter["description"], String, COUNTER_DESCRIPTION); + GET_JSON(sampleGroups, counter["sample_groups"], Array); + for (const Json::Value& sampleGroup : sampleGroups) { + ASSERT_TRUE(sampleGroup.isObject()); + EXPECT_EQ_JSON(sampleGroup["id"], UInt, 0u); + + GET_JSON(samples, sampleGroup["samples"], Object); + GET_JSON(samplesSchema, samples["schema"], Object); + EXPECT_GE(samplesSchema.size(), 3u); + GET_JSON_VALUE(samplesNumber, samplesSchema["number"], UInt); + GET_JSON_VALUE(samplesCount, samplesSchema["count"], UInt); + GET_JSON(samplesData, samples["data"], Array); + for (const Json::Value& sample : samplesData) { + ASSERT_TRUE(sample.isArray()); + ASSERT_LT(nextExpectedTestCounter, expectedTestCountersCount); + EXPECT_EQ_JSON( + sample[samplesNumber], UInt64, + expectedTestCounters[nextExpectedTestCounter].mNumber); + EXPECT_EQ_JSON( + sample[samplesCount], Int64, + expectedTestCounters[nextExpectedTestCounter].mCount); + ++nextExpectedTestCounter; + } + } + } else if (name == "TestCounter2") { + EXPECT_TRUE(expectCounter2); + + EXPECT_EQ_JSON(counter["category"], String, COUNTER_NAME2); + EXPECT_EQ_JSON(counter["description"], String, COUNTER_DESCRIPTION2); + GET_JSON(sampleGroups, counter["sample_groups"], Array); + for (const Json::Value& sampleGroup : sampleGroups) { + ASSERT_TRUE(sampleGroup.isObject()); + EXPECT_EQ_JSON(sampleGroup["id"], UInt, 0u); + + GET_JSON(samples, sampleGroup["samples"], Object); + GET_JSON(samplesSchema, samples["schema"], Object); + EXPECT_GE(samplesSchema.size(), 3u); + GET_JSON_VALUE(samplesNumber, samplesSchema["number"], UInt); + GET_JSON_VALUE(samplesCount, samplesSchema["count"], UInt); + GET_JSON(samplesData, samples["data"], Array); + for (const Json::Value& sample : samplesData) { + ASSERT_TRUE(sample.isArray()); + ASSERT_LT(nextExpectedTestCounter2, expectedTestCounters2Count); + EXPECT_EQ_JSON( + sample[samplesNumber], UInt64, + expectedTestCounters2[nextExpectedTestCounter2].mNumber); + EXPECT_EQ_JSON( + sample[samplesCount], Int64, + expectedTestCounters2[nextExpectedTestCounter2].mCount); + ++nextExpectedTestCounter2; + } + } + } + } + + EXPECT_EQ(nextExpectedTestCounter, expectedTestCountersCount); + if (expectCounter2) { + EXPECT_EQ(nextExpectedTestCounter2, expectedTestCounters2Count); + } + }; + + // Inactive -> Active + profiler_ensure_started(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + // Output all "TestCounter"s, with increasing delays (to test different + // number of counter samplings). + int samplingWaits = 2; + for (int64_t counter : testCounters) { + AUTO_PROFILER_COUNT_TOTAL(TestCounter, counter); + for (int i = 0; i < samplingWaits; ++i) { + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + ++samplingWaits; + } + + // Verify we got "TestCounter" in the output, but not "TestCounter2" yet. + UniquePtr profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), checkCountersInJSON); + + // Now introduce TestCounter2. + expectCounter2 = true; + for (int64_t counter2 : testCounters2) { + AUTO_PROFILER_COUNT_TOTAL(TestCounter2, counter2); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + + // Verify we got both "TestCounter" and "TestCounter2" in the output. + profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), checkCountersInJSON); + + profiler_stop(); +} + +TEST(GeckoProfiler, Time) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + double t1 = profiler_time(); + double t2 = profiler_time(); + ASSERT_TRUE(t1 <= t2); + + // profiler_start() restarts the timer used by profiler_time(). + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + double t3 = profiler_time(); + double t4 = profiler_time(); + ASSERT_TRUE(t3 <= t4); + + profiler_stop(); + + double t5 = profiler_time(); + double t6 = profiler_time(); + ASSERT_TRUE(t4 <= t5 && t1 <= t6); +} + +TEST(GeckoProfiler, GetProfile) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!profiler_get_profile()); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + mozilla::Maybe activeFeatures = profiler_features_if_active(); + ASSERT_TRUE(activeFeatures.isSome()); + // Not all platforms support stack-walking. + const bool hasStackWalk = ProfilerFeature::HasStackWalk(*activeFeatures); + + UniquePtr profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), [&](const Json::Value& aRoot) { + GET_JSON(meta, aRoot["meta"], Object); + { + GET_JSON(configuration, meta["configuration"], Object); + { + GET_JSON(features, configuration["features"], Array); + { + EXPECT_EQ(features.size(), (hasStackWalk ? 1u : 0u)); + if (hasStackWalk) { + EXPECT_JSON_ARRAY_CONTAINS(features, String, "stackwalk"); + } + } + GET_JSON(threads, configuration["threads"], Array); + { + EXPECT_EQ(threads.size(), 1u); + EXPECT_JSON_ARRAY_CONTAINS(threads, String, "GeckoMain"); + } + } + } + }); + + profiler_stop(); + + ASSERT_TRUE(!profiler_get_profile()); +} + +TEST(GeckoProfiler, StreamJSONForThisProcess) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + SpliceableChunkedJSONWriter w{FailureLatchInfallibleSource::Singleton()}; + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Fallible()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(&w.ChunkedWriteFunc().SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT( + &std::as_const(w.ChunkedWriteFunc()).SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT(!w.Fallible()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + MOZ_RELEASE_ASSERT(&w.SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT(&std::as_const(w).SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isErr()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + w.Start(); + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isOk()); + w.End(); + + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + + UniquePtr profile = w.ChunkedWriteFunc().CopyData(); + + JSONOutputCheck(profile.get(), [](const Json::Value&) {}); + + profiler_stop(); + + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isErr()); +} + +// Internal version of profiler_stream_json_for_this_process, which allows being +// called from a non-main thread of the parent process, at the risk of getting +// an incomplete profile. +ProfilerResult +do_profiler_stream_json_for_this_process( + SpliceableJSONWriter& aWriter, double aSinceTime, bool aIsShuttingDown, + ProfilerCodeAddressService* aService, + mozilla::ProgressLogger aProgressLogger); + +TEST(GeckoProfiler, StreamJSONForThisProcessThreaded) +{ + // Same as the previous test, but calling some things on background threads. + nsCOMPtr thread; + nsresult rv = NS_NewNamedThread("GeckoProfGTest", getter_AddRefs(thread)); + ASSERT_NS_SUCCEEDED(rv); + + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + SpliceableChunkedJSONWriter w{FailureLatchInfallibleSource::Singleton()}; + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Fallible()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(&w.ChunkedWriteFunc().SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT( + &std::as_const(w.ChunkedWriteFunc()).SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT(!w.Fallible()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + MOZ_RELEASE_ASSERT(&w.SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + MOZ_RELEASE_ASSERT(&std::as_const(w).SourceFailureLatch() == + &mozilla::FailureLatchInfallibleSource::Singleton()); + + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isErr()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + + // Start the profiler on the main thread. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + // Call profiler_stream_json_for_this_process on a background thread. + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_StreamJSONForThisProcessThreaded_Test::TestBody"_ns, + thread, + NS_NewRunnableFunction( + "GeckoProfiler_StreamJSONForThisProcessThreaded_Test::TestBody", + [&]() { + w.Start(); + ASSERT_TRUE(::do_profiler_stream_json_for_this_process( + w, /* double aSinceTime */ 0.0, + /* bool aIsShuttingDown */ false, + /* ProfilerCodeAddressService* aService */ nullptr, + mozilla::ProgressLogger{}) + .isOk()); + w.End(); + })); + + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().Failed()); + MOZ_RELEASE_ASSERT(!w.ChunkedWriteFunc().GetFailure()); + MOZ_RELEASE_ASSERT(!w.Failed()); + MOZ_RELEASE_ASSERT(!w.GetFailure()); + + UniquePtr profile = w.ChunkedWriteFunc().CopyData(); + + JSONOutputCheck(profile.get(), [](const Json::Value&) {}); + + // Stop the profiler and call profiler_stream_json_for_this_process on a + // background thread. + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_StreamJSONForThisProcessThreaded_Test::TestBody"_ns, + thread, + NS_NewRunnableFunction( + "GeckoProfiler_StreamJSONForThisProcessThreaded_Test::TestBody", + [&]() { + profiler_stop(); + ASSERT_TRUE(::do_profiler_stream_json_for_this_process( + w, /* double aSinceTime */ 0.0, + /* bool aIsShuttingDown */ false, + /* ProfilerCodeAddressService* aService */ nullptr, + mozilla::ProgressLogger{}) + .isErr()); + })); + thread->Shutdown(); + + // Call profiler_stream_json_for_this_process on the main thread. + ASSERT_TRUE(::profiler_stream_json_for_this_process(w).isErr()); +} + +TEST(GeckoProfiler, ProfilingStack) +{ + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + + AUTO_PROFILER_LABEL("A::B", OTHER); + + UniqueFreePtr dynamic(strdup("dynamic")); + { + AUTO_PROFILER_LABEL_DYNAMIC_CSTR("A::C", JS, dynamic.get()); + AUTO_PROFILER_LABEL_DYNAMIC_NSCSTRING("A::C2", JS, + nsDependentCString(dynamic.get())); + AUTO_PROFILER_LABEL_DYNAMIC_LOSSY_NSSTRING( + "A::C3", JS, NS_ConvertUTF8toUTF16(dynamic.get())); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(profiler_get_backtrace()); + } + + AutoProfilerLabel label1("A", nullptr, JS::ProfilingCategoryPair::DOM); + AutoProfilerLabel label2("A", dynamic.get(), + JS::ProfilingCategoryPair::NETWORK); + ASSERT_TRUE(profiler_get_backtrace()); + + profiler_stop(); + + ASSERT_TRUE(!profiler_get_profile()); +} + +TEST(GeckoProfiler, Bug1355807) +{ + uint32_t features = ProfilerFeature::JS; + const char* manyThreadsFilter[] = {""}; + const char* fewThreadsFilter[] = {"GeckoMain"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + manyThreadsFilter, MOZ_ARRAY_LENGTH(manyThreadsFilter), 0); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + fewThreadsFilter, MOZ_ARRAY_LENGTH(fewThreadsFilter), 0); + + // In bug 1355807 this caused an assertion failure in StopJSSampling(). + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + fewThreadsFilter, MOZ_ARRAY_LENGTH(fewThreadsFilter), 0); + + profiler_stop(); +} + +class GTestStackCollector final : public ProfilerStackCollector { + public: + GTestStackCollector() : mSetIsMainThread(0), mFrames(0) {} + + virtual void SetIsMainThread() { mSetIsMainThread++; } + + virtual void CollectNativeLeafAddr(void* aAddr) { mFrames++; } + virtual void CollectJitReturnAddr(void* aAddr) { mFrames++; } + virtual void CollectWasmFrame(const char* aLabel) { mFrames++; } + virtual void CollectProfilingStackFrame( + const js::ProfilingStackFrame& aFrame) { + mFrames++; + } + + int mSetIsMainThread; + int mFrames; +}; + +void DoSuspendAndSample(ProfilerThreadId aTidToSample, + nsIThread* aSamplingThread) { + NS_DispatchAndSpinEventLoopUntilComplete( + "GeckoProfiler_SuspendAndSample_Test::TestBody"_ns, aSamplingThread, + NS_NewRunnableFunction( + "GeckoProfiler_SuspendAndSample_Test::TestBody", [&]() { + uint32_t features = ProfilerFeature::CPUUtilization; + GTestStackCollector collector; + profiler_suspend_and_sample_thread(aTidToSample, features, + collector, + /* sampleNative = */ true); + + ASSERT_TRUE(collector.mSetIsMainThread == + (aTidToSample == profiler_main_thread_id())); + ASSERT_TRUE(collector.mFrames > 0); + })); +} + +TEST(GeckoProfiler, SuspendAndSample) +{ + nsCOMPtr thread; + nsresult rv = NS_NewNamedThread("GeckoProfGTest", getter_AddRefs(thread)); + ASSERT_NS_SUCCEEDED(rv); + + ProfilerThreadId tid = profiler_current_thread_id(); + + ASSERT_TRUE(!profiler_is_active()); + + // Suspend and sample while the profiler is inactive. + DoSuspendAndSample(tid, thread); + + DoSuspendAndSample(ProfilerThreadId{}, thread); + + uint32_t features = ProfilerFeature::JS; + const char* filters[] = {"GeckoMain", "Compositor"}; + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(profiler_is_active()); + + // Suspend and sample while the profiler is active. + DoSuspendAndSample(tid, thread); + + DoSuspendAndSample(ProfilerThreadId{}, thread); + + profiler_stop(); + + ASSERT_TRUE(!profiler_is_active()); +} + +TEST(GeckoProfiler, PostSamplingCallback) +{ + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_callback_after_sampling( + [&](SamplingState) { ASSERT_TRUE(false); })); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk, filters, MOZ_ARRAY_LENGTH(filters), + 0); + { + // Stack sampling -> This label should appear at least once. + AUTO_PROFILER_LABEL("PostSamplingCallback completed", OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + UniquePtr profileCompleted = profiler_get_profile(); + JSONOutputCheck(profileCompleted.get(), [](const Json::Value& aRoot) { + GET_JSON(threads, aRoot["threads"], Array); + { + GET_JSON(thread0, threads[0], Object); + { + EXPECT_JSON_ARRAY_CONTAINS(thread0["stringTable"], String, + "PostSamplingCallback completed"); + } + } + }); + + profiler_pause(); + { + // Paused -> This label should not appear. + AUTO_PROFILER_LABEL("PostSamplingCallback paused", OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingPaused); + } + UniquePtr profilePaused = profiler_get_profile(); + JSONOutputCheck(profilePaused.get(), [](const Json::Value& aRoot) {}); + // This string shouldn't appear *anywhere* in the profile. + ASSERT_FALSE(strstr(profilePaused.get(), "PostSamplingCallback paused")); + + profiler_resume(); + { + // Stack sampling -> This label should appear at least once. + AUTO_PROFILER_LABEL("PostSamplingCallback resumed", OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + UniquePtr profileResumed = profiler_get_profile(); + JSONOutputCheck(profileResumed.get(), [](const Json::Value& aRoot) { + GET_JSON(threads, aRoot["threads"], Array); + { + GET_JSON(thread0, threads[0], Object); + { + EXPECT_JSON_ARRAY_CONTAINS(thread0["stringTable"], String, + "PostSamplingCallback resumed"); + } + } + }); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk | ProfilerFeature::NoStackSampling, + filters, MOZ_ARRAY_LENGTH(filters), 0); + { + // No stack sampling -> This label should not appear. + AUTO_PROFILER_LABEL("PostSamplingCallback completed (no stacks)", OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::NoStackSamplingCompleted); + } + UniquePtr profileNoStacks = profiler_get_profile(); + JSONOutputCheck(profileNoStacks.get(), [](const Json::Value& aRoot) {}); + // This string shouldn't appear *anywhere* in the profile. + ASSERT_FALSE(strstr(profileNoStacks.get(), + "PostSamplingCallback completed (no stacks)")); + + // Note: There is no non-racy way to test for SamplingState::JustStopped, as + // it would require coordination between `profiler_stop()` and another thread + // doing `profiler_callback_after_sampling()` at just the right moment. + + profiler_stop(); + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_callback_after_sampling( + [&](SamplingState) { ASSERT_TRUE(false); })); +} + +TEST(GeckoProfiler, ProfilingStateCallback) +{ + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!profiler_is_active()); + + struct ProfilingStateAndId { + ProfilingState mProfilingState; + int mId; + }; + DataMutex> states{"Profiling states"}; + auto CreateCallback = [&states](int id) { + return [id, &states](ProfilingState aProfilingState) { + auto lockedStates = states.Lock(); + ASSERT_TRUE( + lockedStates->append(ProfilingStateAndId{aProfilingState, id})); + }; + }; + auto CheckStatesIsEmpty = [&states]() { + auto lockedStates = states.Lock(); + EXPECT_TRUE(lockedStates->empty()); + }; + auto CheckStatesOnlyContains = [&states](ProfilingState aProfilingState, + int aId) { + auto lockedStates = states.Lock(); + EXPECT_EQ(lockedStates->length(), 1u); + if (lockedStates->length() >= 1u) { + EXPECT_EQ((*lockedStates)[0].mProfilingState, aProfilingState); + EXPECT_EQ((*lockedStates)[0].mId, aId); + } + lockedStates->clear(); + }; + + profiler_add_state_change_callback(AllProfilingStates(), CreateCallback(1), + 1); + // This is in case of error, and it also exercises the (allowed) removal of + // unknown callback ids. + auto cleanup1 = mozilla::MakeScopeExit( + []() { profiler_remove_state_change_callback(1); }); + CheckStatesIsEmpty(); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk, filters, MOZ_ARRAY_LENGTH(filters), + 0); + + CheckStatesOnlyContains(ProfilingState::Started, 1); + + profiler_add_state_change_callback(AllProfilingStates(), CreateCallback(2), + 2); + // This is in case of error, and it also exercises the (allowed) removal of + // unknown callback ids. + auto cleanup2 = mozilla::MakeScopeExit( + []() { profiler_remove_state_change_callback(2); }); + CheckStatesOnlyContains(ProfilingState::AlreadyActive, 2); + + profiler_remove_state_change_callback(2); + CheckStatesOnlyContains(ProfilingState::RemovingCallback, 2); + // Note: The actual removal is effectively tested below, by not seeing any + // more invocations of the 2nd callback. + + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + UniquePtr profileCompleted = profiler_get_profile(); + CheckStatesOnlyContains(ProfilingState::GeneratingProfile, 1); + JSONOutputCheck(profileCompleted.get(), [](const Json::Value& aRoot) {}); + + profiler_pause(); + CheckStatesOnlyContains(ProfilingState::Pausing, 1); + UniquePtr profilePaused = profiler_get_profile(); + CheckStatesOnlyContains(ProfilingState::GeneratingProfile, 1); + JSONOutputCheck(profilePaused.get(), [](const Json::Value& aRoot) {}); + + profiler_resume(); + CheckStatesOnlyContains(ProfilingState::Resumed, 1); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + UniquePtr profileResumed = profiler_get_profile(); + CheckStatesOnlyContains(ProfilingState::GeneratingProfile, 1); + JSONOutputCheck(profileResumed.get(), [](const Json::Value& aRoot) {}); + + // This effectively stops the profiler before restarting it, but + // ProfilingState::Stopping is not notified. See `profiler_start` for details. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk | ProfilerFeature::NoStackSampling, + filters, MOZ_ARRAY_LENGTH(filters), 0); + CheckStatesOnlyContains(ProfilingState::Started, 1); + ASSERT_EQ(WaitForSamplingState(), SamplingState::NoStackSamplingCompleted); + UniquePtr profileNoStacks = profiler_get_profile(); + CheckStatesOnlyContains(ProfilingState::GeneratingProfile, 1); + JSONOutputCheck(profileNoStacks.get(), [](const Json::Value& aRoot) {}); + + profiler_stop(); + CheckStatesOnlyContains(ProfilingState::Stopping, 1); + ASSERT_TRUE(!profiler_is_active()); + + profiler_remove_state_change_callback(1); + CheckStatesOnlyContains(ProfilingState::RemovingCallback, 1); + + // Note: ProfilingState::ShuttingDown cannot be tested here, and the profiler + // can only be shut down once per process. +} + +TEST(GeckoProfiler, BaseProfilerHandOff) +{ + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!baseprofiler::profiler_is_active()); + ASSERT_TRUE(!profiler_is_active()); + + BASE_PROFILER_MARKER_UNTYPED("Base marker before base profiler", OTHER, {}); + PROFILER_MARKER_UNTYPED("Gecko marker before base profiler", OTHER, {}); + + // Start the Base Profiler. + baseprofiler::profiler_start( + PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk, filters, MOZ_ARRAY_LENGTH(filters)); + + ASSERT_TRUE(baseprofiler::profiler_is_active()); + ASSERT_TRUE(!profiler_is_active()); + + // Add at least a marker, which should go straight into the buffer. + Maybe info0 = + baseprofiler::profiler_get_buffer_info(); + BASE_PROFILER_MARKER_UNTYPED("Base marker during base profiler", OTHER, {}); + Maybe info1 = + baseprofiler::profiler_get_buffer_info(); + ASSERT_GT(info1->mRangeEnd, info0->mRangeEnd); + + PROFILER_MARKER_UNTYPED("Gecko marker during base profiler", OTHER, {}); + + // Start the Gecko Profiler, which should grab the Base Profiler profile and + // stop it. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk, filters, MOZ_ARRAY_LENGTH(filters), + 0); + + ASSERT_TRUE(!baseprofiler::profiler_is_active()); + ASSERT_TRUE(profiler_is_active()); + + BASE_PROFILER_MARKER_UNTYPED("Base marker during gecko profiler", OTHER, {}); + PROFILER_MARKER_UNTYPED("Gecko marker during gecko profiler", OTHER, {}); + + // Write some Gecko Profiler samples. + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + + // Check that the Gecko Profiler profile contains at least the Base Profiler + // main thread samples. + UniquePtr profile = profiler_get_profile(); + + profiler_stop(); + ASSERT_TRUE(!profiler_is_active()); + + BASE_PROFILER_MARKER_UNTYPED("Base marker after gecko profiler", OTHER, {}); + PROFILER_MARKER_UNTYPED("Gecko marker after gecko profiler", OTHER, {}); + + JSONOutputCheck(profile.get(), [](const Json::Value& aRoot) { + GET_JSON(threads, aRoot["threads"], Array); + { + bool found = false; + for (const Json::Value& thread : threads) { + ASSERT_TRUE(thread.isObject()); + GET_JSON(name, thread["name"], String); + if (name.asString() == "GeckoMain") { + found = true; + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Base marker before base profiler"); + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Gecko marker before base profiler"); + EXPECT_JSON_ARRAY_CONTAINS(thread["stringTable"], String, + "Base marker during base profiler"); + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Gecko marker during base profiler"); + EXPECT_JSON_ARRAY_CONTAINS(thread["stringTable"], String, + "Base marker during gecko profiler"); + EXPECT_JSON_ARRAY_CONTAINS(thread["stringTable"], String, + "Gecko marker during gecko profiler"); + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Base marker after gecko profiler"); + EXPECT_JSON_ARRAY_EXCLUDES(thread["stringTable"], String, + "Gecko marker after gecko profiler"); + break; + } + } + EXPECT_TRUE(found); + } + }); +} + +static std::string_view GetFeatureName(uint32_t feature) { + switch (feature) { +# define FEATURE_NAME(n_, str_, Name_, desc_) \ + case ProfilerFeature::Name_: \ + return str_; + + PROFILER_FOR_EACH_FEATURE(FEATURE_NAME) + +# undef FEATURE_NAME + + default: + return "?"; + } +} + +TEST(GeckoProfiler, FeatureCombinations) +{ + // Bug 1845606 + #ifdef XP_WIN + if (!IsWin8OrLater()) { + return; + } + #endif + + const char* filters[] = {"*"}; + + // List of features to test. Every combination of up to 3 of them will be + // tested, so be careful not to add too many to keep the test run at a + // reasonable time. + uint32_t featureList[] = {ProfilerFeature::JS, + ProfilerFeature::Screenshots, + ProfilerFeature::StackWalk, + ProfilerFeature::NoStackSampling, + ProfilerFeature::NativeAllocations, + ProfilerFeature::CPUUtilization, + ProfilerFeature::CPUAllThreads, + ProfilerFeature::SamplingAllThreads, + ProfilerFeature::MarkersAllThreads, + ProfilerFeature::UnregisteredThreads}; + constexpr uint32_t featureCount = uint32_t(MOZ_ARRAY_LENGTH(featureList)); + + auto testFeatures = [&](uint32_t features, + const std::string& featuresString) { + SCOPED_TRACE(featuresString.c_str()); + + ASSERT_TRUE(!profiler_is_active()); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(profiler_is_active()); + + // Write some Gecko Profiler samples. + EXPECT_EQ(WaitForSamplingState(), + (((features & ProfilerFeature::NoStackSampling) != 0) && + ((features & (ProfilerFeature::CPUUtilization | + ProfilerFeature::CPUAllThreads)) == 0)) + ? SamplingState::NoStackSamplingCompleted + : SamplingState::SamplingCompleted); + + // Check that the profile looks valid. Note that we don't test feature- + // specific changes. + UniquePtr profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), [](const Json::Value& aRoot) {}); + + profiler_stop(); + ASSERT_TRUE(!profiler_is_active()); + }; + + testFeatures(0, "Features: (none)"); + + for (uint32_t f1 = 0u; f1 < featureCount; ++f1) { + const uint32_t features1 = featureList[f1]; + std::string features1String = "Features: "; + features1String += GetFeatureName(featureList[f1]); + + testFeatures(features1, features1String); + + for (uint32_t f2 = f1 + 1u; f2 < featureCount; ++f2) { + const uint32_t features12 = f1 | featureList[f2]; + std::string features12String = features1String + " "; + features12String += GetFeatureName(featureList[f2]); + + testFeatures(features12, features12String); + + for (uint32_t f3 = f2 + 1u; f3 < featureCount; ++f3) { + const uint32_t features123 = features12 | featureList[f3]; + std::string features123String = features12String + " "; + features123String += GetFeatureName(featureList[f3]); + + testFeatures(features123, features123String); + } + } + } +} + +static void CountCPUDeltas(const Json::Value& aThread, size_t& aOutSamplings, + uint64_t& aOutCPUDeltaSum) { + GET_JSON(samples, aThread["samples"], Object); + { + Json::ArrayIndex threadCPUDeltaIndex = 0; + GET_JSON(schema, samples["schema"], Object); + { + GET_JSON(jsonThreadCPUDeltaIndex, schema["threadCPUDelta"], UInt); + threadCPUDeltaIndex = jsonThreadCPUDeltaIndex.asUInt(); + } + + aOutSamplings = 0; + aOutCPUDeltaSum = 0; + GET_JSON(data, samples["data"], Array); + aOutSamplings = data.size(); + for (const Json::Value& sample : data) { + ASSERT_TRUE(sample.isArray()); + if (sample.isValidIndex(threadCPUDeltaIndex)) { + if (!sample[threadCPUDeltaIndex].isNull()) { + GET_JSON(cpuDelta, sample[threadCPUDeltaIndex], UInt64); + aOutCPUDeltaSum += uint64_t(cpuDelta.asUInt64()); + } + } + } + } +} + +TEST(GeckoProfiler, CPUUsage) +{ + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + const char* filters[] = {"GeckoMain", "Idle test", "Busy test"}; + + enum class TestThreadsState { + // Initial state, while constructing and starting the idle thread. + STARTING, + // Set by the idle thread just before running its main mostly-idle loop. + RUNNING1, + RUNNING2, + // Set by the main thread when it wants the idle thread to stop. + STOPPING + }; + Atomic testThreadsState{TestThreadsState::STARTING}; + + std::thread idle([&]() { + AUTO_PROFILER_REGISTER_THREAD("Idle test"); + // Add a label to ensure that we have a non-empty stack, even if native + // stack-walking is not available. + AUTO_PROFILER_LABEL("Idle test", PROFILER); + ASSERT_TRUE(testThreadsState.compareExchange(TestThreadsState::STARTING, + TestThreadsState::RUNNING1) || + testThreadsState.compareExchange(TestThreadsState::RUNNING1, + TestThreadsState::RUNNING2)); + + while (testThreadsState != TestThreadsState::STOPPING) { + // Sleep for multiple profiler intervals, so the profiler should have + // samples with zero CPU utilization. + PR_Sleep(PR_MillisecondsToInterval(PROFILER_DEFAULT_INTERVAL * 10)); + } + }); + + std::thread busy([&]() { + AUTO_PROFILER_REGISTER_THREAD("Busy test"); + // Add a label to ensure that we have a non-empty stack, even if native + // stack-walking is not available. + AUTO_PROFILER_LABEL("Busy test", PROFILER); + ASSERT_TRUE(testThreadsState.compareExchange(TestThreadsState::STARTING, + TestThreadsState::RUNNING1) || + testThreadsState.compareExchange(TestThreadsState::RUNNING1, + TestThreadsState::RUNNING2)); + + while (testThreadsState != TestThreadsState::STOPPING) { + // Stay busy! + } + }); + + // Wait for idle thread to start running its main loop. + while (testThreadsState != TestThreadsState::RUNNING2) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + + // We want to ensure that CPU usage numbers are present whether or not we are + // collecting stack samples. + static constexpr bool scTestsWithOrWithoutStackSampling[] = {false, true}; + for (const bool testWithNoStackSampling : scTestsWithOrWithoutStackSampling) { + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_callback_after_sampling( + [&](SamplingState) { ASSERT_TRUE(false); })); + + profiler_start( + PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + ProfilerFeature::StackWalk | ProfilerFeature::CPUUtilization | + (testWithNoStackSampling ? ProfilerFeature::NoStackSampling : 0), + filters, MOZ_ARRAY_LENGTH(filters), 0); + // Grab a few samples, each with a different label on the stack. +# define SAMPLE_LABEL_PREFIX "CPUUsage sample label " + static constexpr const char* scSampleLabels[] = { + SAMPLE_LABEL_PREFIX "0", SAMPLE_LABEL_PREFIX "1", + SAMPLE_LABEL_PREFIX "2", SAMPLE_LABEL_PREFIX "3", + SAMPLE_LABEL_PREFIX "4", SAMPLE_LABEL_PREFIX "5", + SAMPLE_LABEL_PREFIX "6", SAMPLE_LABEL_PREFIX "7", + SAMPLE_LABEL_PREFIX "8", SAMPLE_LABEL_PREFIX "9"}; + static constexpr size_t scSampleLabelCount = + (sizeof(scSampleLabels) / sizeof(scSampleLabels[0])); + // We'll do two samplings for each label. + static constexpr size_t scMinSamplings = scSampleLabelCount * 2; + + for (const char* sampleLabel : scSampleLabels) { + AUTO_PROFILER_LABEL(sampleLabel, OTHER); + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + // Note: There could have been a delay before this label above, where the + // profiler could have sampled the stack and missed the label. By forcing + // another sampling now, the label is guaranteed to be present. + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + + UniquePtr profile = profiler_get_profile(); + + if (testWithNoStackSampling) { + // If we are testing nostacksampling, we shouldn't find this label prefix + // in the profile. + EXPECT_FALSE(strstr(profile.get(), SAMPLE_LABEL_PREFIX)); + } else { + // In normal sampling mode, we should find all labels. + for (const char* sampleLabel : scSampleLabels) { + EXPECT_TRUE(strstr(profile.get(), sampleLabel)); + } + } + + JSONOutputCheck(profile.get(), [testWithNoStackSampling]( + const Json::Value& aRoot) { + // Check that the "cpu" feature is present. + GET_JSON(meta, aRoot["meta"], Object); + { + GET_JSON(configuration, meta["configuration"], Object); + { + GET_JSON(features, configuration["features"], Array); + { EXPECT_JSON_ARRAY_CONTAINS(features, String, "cpu"); } + } + } + + { + GET_JSON(sampleUnits, meta["sampleUnits"], Object); + { + EXPECT_EQ_JSON(sampleUnits["time"], String, "ms"); + EXPECT_EQ_JSON(sampleUnits["eventDelay"], String, "ms"); +# if defined(GP_OS_windows) || defined(GP_OS_darwin) || \ + defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) + // Note: The exact string is not important here. + EXPECT_TRUE(sampleUnits["threadCPUDelta"].isString()) + << "There should be a sampleUnits.threadCPUDelta on this " + "platform"; +# else + EXPECT_FALSE(sampleUnits.isMember("threadCPUDelta")) + << "Unexpected sampleUnits.threadCPUDelta on this platform";; +# endif + } + } + + bool foundMain = false; + bool foundIdle = false; + uint64_t idleThreadCPUDeltaSum = 0u; + bool foundBusy = false; + uint64_t busyThreadCPUDeltaSum = 0u; + + // Check that the sample schema contains "threadCPUDelta". + GET_JSON(threads, aRoot["threads"], Array); + for (const Json::Value& thread : threads) { + ASSERT_TRUE(thread.isObject()); + GET_JSON(name, thread["name"], String); + if (name.asString() == "GeckoMain") { + foundMain = true; + GET_JSON(samples, thread["samples"], Object); + { + Json::ArrayIndex stackIndex = 0; + Json::ArrayIndex threadCPUDeltaIndex = 0; + GET_JSON(schema, samples["schema"], Object); + { + GET_JSON(jsonStackIndex, schema["stack"], UInt); + stackIndex = jsonStackIndex.asUInt(); + GET_JSON(jsonThreadCPUDeltaIndex, schema["threadCPUDelta"], UInt); + threadCPUDeltaIndex = jsonThreadCPUDeltaIndex.asUInt(); + } + + std::set stackLeaves; // To count distinct leaves. + unsigned threadCPUDeltaCount = 0; + GET_JSON(data, samples["data"], Array); + if (testWithNoStackSampling) { + // When not sampling stacks, the first sampling loop will have no + // running times, so it won't output anything. + EXPECT_GE(data.size(), scMinSamplings - 1); + } else { + EXPECT_GE(data.size(), scMinSamplings); + } + for (const Json::Value& sample : data) { + ASSERT_TRUE(sample.isArray()); + if (sample.isValidIndex(stackIndex)) { + if (!sample[stackIndex].isNull()) { + GET_JSON(stack, sample[stackIndex], UInt64); + stackLeaves.insert(stack.asUInt64()); + } + } + if (sample.isValidIndex(threadCPUDeltaIndex)) { + if (!sample[threadCPUDeltaIndex].isNull()) { + EXPECT_TRUE(sample[threadCPUDeltaIndex].isUInt64()); + ++threadCPUDeltaCount; + } + } + } + + if (testWithNoStackSampling) { + // in nostacksampling mode, there should only be one kind of stack + // leaf (the root). + EXPECT_EQ(stackLeaves.size(), 1u); + } else { + // in normal sampling mode, there should be at least one kind of + // stack leaf for each distinct label. + EXPECT_GE(stackLeaves.size(), scSampleLabelCount); + } + +# if defined(GP_OS_windows) || defined(GP_OS_darwin) || \ + defined(GP_OS_linux) || defined(GP_OS_android) || defined(GP_OS_freebsd) + EXPECT_GE(threadCPUDeltaCount, data.size() - 1u) + << "There should be 'threadCPUDelta' values in all but 1 " + "samples"; +# else + // All "threadCPUDelta" data should be absent or null on unsupported + // platforms. + EXPECT_EQ(threadCPUDeltaCount, 0u); +# endif + } + } else if (name.asString() == "Idle test") { + foundIdle = true; + size_t samplings; + CountCPUDeltas(thread, samplings, idleThreadCPUDeltaSum); + if (testWithNoStackSampling) { + // When not sampling stacks, the first sampling loop will have no + // running times, so it won't output anything. + EXPECT_GE(samplings, scMinSamplings - 1); + } else { + EXPECT_GE(samplings, scMinSamplings); + } +# if !(defined(GP_OS_windows) || defined(GP_OS_darwin) || \ + defined(GP_OS_linux) || defined(GP_OS_android) || \ + defined(GP_OS_freebsd)) + // All "threadCPUDelta" data should be absent or null on unsupported + // platforms. + EXPECT_EQ(idleThreadCPUDeltaSum, 0u); +# endif + } else if (name.asString() == "Busy test") { + foundBusy = true; + size_t samplings; + CountCPUDeltas(thread, samplings, busyThreadCPUDeltaSum); + if (testWithNoStackSampling) { + // When not sampling stacks, the first sampling loop will have no + // running times, so it won't output anything. + EXPECT_GE(samplings, scMinSamplings - 1); + } else { + EXPECT_GE(samplings, scMinSamplings); + } +# if !(defined(GP_OS_windows) || defined(GP_OS_darwin) || \ + defined(GP_OS_linux) || defined(GP_OS_android) || \ + defined(GP_OS_freebsd)) + // All "threadCPUDelta" data should be absent or null on unsupported + // platforms. + EXPECT_EQ(busyThreadCPUDeltaSum, 0u); +# endif + } + } + + EXPECT_TRUE(foundMain); + EXPECT_TRUE(foundIdle); + EXPECT_TRUE(foundBusy); + EXPECT_LE(idleThreadCPUDeltaSum, busyThreadCPUDeltaSum); + }); + + // Note: There is no non-racy way to test for SamplingState::JustStopped, as + // it would require coordination between `profiler_stop()` and another + // thread doing `profiler_callback_after_sampling()` at just the right + // moment. + + profiler_stop(); + ASSERT_TRUE(!profiler_is_active()); + ASSERT_TRUE(!profiler_callback_after_sampling( + [&](SamplingState) { ASSERT_TRUE(false); })); + } + + testThreadsState = TestThreadsState::STOPPING; + busy.join(); + idle.join(); +} + +TEST(GeckoProfiler, AllThreads) +{ + // Bug 1845606 + #ifdef XP_WIN + if (!IsWin8OrLater()) { + return; + } + #endif + + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + ASSERT_EQ(static_cast(ThreadProfilingFeatures::Any), 1u + 2u + 4u) + << "This test assumes that there are 3 binary choices 1+2+4; " + "Is this test up to date?"; + + for (uint32_t threadFeaturesBinary = 0u; + threadFeaturesBinary <= + static_cast(ThreadProfilingFeatures::Any); + ++threadFeaturesBinary) { + ThreadProfilingFeatures threadFeatures = + static_cast(threadFeaturesBinary); + const bool threadCPU = DoFeaturesIntersect( + threadFeatures, ThreadProfilingFeatures::CPUUtilization); + const bool threadSampling = + DoFeaturesIntersect(threadFeatures, ThreadProfilingFeatures::Sampling); + const bool threadMarkers = + DoFeaturesIntersect(threadFeatures, ThreadProfilingFeatures::Markers); + + ASSERT_TRUE(!profiler_is_active()); + + uint32_t features = ProfilerFeature::StackWalk; + std::string featuresString = "Features: StackWalk Threads"; + if (threadCPU) { + features |= ProfilerFeature::CPUAllThreads; + featuresString += " CPUAllThreads"; + } + if (threadSampling) { + features |= ProfilerFeature::SamplingAllThreads; + featuresString += " SamplingAllThreads"; + } + if (threadMarkers) { + features |= ProfilerFeature::MarkersAllThreads; + featuresString += " MarkersAllThreads"; + } + + SCOPED_TRACE(featuresString.c_str()); + + const char* filters[] = {"GeckoMain", "Selected"}; + + EXPECT_FALSE(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization)); + EXPECT_FALSE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling)); + EXPECT_FALSE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers)); + EXPECT_FALSE(profiler_thread_is_being_profiled_for_markers()); + + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + features, filters, MOZ_ARRAY_LENGTH(filters), 0); + + EXPECT_TRUE(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization)); + EXPECT_TRUE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling)); + EXPECT_TRUE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers)); + EXPECT_TRUE(profiler_thread_is_being_profiled_for_markers()); + + // This will signal all threads to stop spinning. + Atomic stopThreads{false}; + + Atomic selectedThreadSpins{0}; + std::thread selectedThread([&]() { + AUTO_PROFILER_REGISTER_THREAD("Selected test thread"); + // Add a label to ensure that we have a non-empty stack, even if native + // stack-walking is not available. + AUTO_PROFILER_LABEL("Selected test thread", PROFILER); + EXPECT_TRUE(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization)); + EXPECT_TRUE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling)); + EXPECT_TRUE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers)); + EXPECT_TRUE(profiler_thread_is_being_profiled_for_markers()); + while (!stopThreads) { + PROFILER_MARKER_UNTYPED("Spinning Selected!", PROFILER); + ++selectedThreadSpins; + PR_Sleep(PR_MillisecondsToInterval(1)); + } + }); + + Atomic unselectedThreadSpins{0}; + std::thread unselectedThread([&]() { + AUTO_PROFILER_REGISTER_THREAD("Registered test thread"); + // Add a label to ensure that we have a non-empty stack, even if native + // stack-walking is not available. + AUTO_PROFILER_LABEL("Registered test thread", PROFILER); + // This thread is *not* selected for full profiling, but it may still be + // profiled depending on the -allthreads features. + EXPECT_EQ(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization), + threadCPU); + EXPECT_EQ( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling), + threadSampling); + EXPECT_EQ( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers), + threadMarkers); + EXPECT_EQ(profiler_thread_is_being_profiled_for_markers(), threadMarkers); + while (!stopThreads) { + PROFILER_MARKER_UNTYPED("Spinning Registered!", PROFILER); + ++unselectedThreadSpins; + PR_Sleep(PR_MillisecondsToInterval(1)); + } + }); + + Atomic unregisteredThreadSpins{0}; + std::thread unregisteredThread([&]() { + // No `AUTO_PROFILER_REGISTER_THREAD` here. + EXPECT_FALSE(profiler_thread_is_being_profiled( + ThreadProfilingFeatures::CPUUtilization)); + EXPECT_FALSE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Sampling)); + EXPECT_FALSE( + profiler_thread_is_being_profiled(ThreadProfilingFeatures::Markers)); + EXPECT_FALSE(profiler_thread_is_being_profiled_for_markers()); + while (!stopThreads) { + PROFILER_MARKER_UNTYPED("Spinning Unregistered!", PROFILER); + ++unregisteredThreadSpins; + PR_Sleep(PR_MillisecondsToInterval(1)); + } + }); + + // Wait for all threads to have started at least one spin. + while (selectedThreadSpins == 0 || unselectedThreadSpins == 0 || + unregisteredThreadSpins == 0) { + PR_Sleep(PR_MillisecondsToInterval(1)); + } + + // Wait until the sampler has done at least one loop. + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + + // Restart the spin counts, and ensure each threads will do at least one + // more spin each. Since spins are increased after PROFILER_MARKER calls, in + // the worst case, each thread will have attempted to record at least one + // marker. + selectedThreadSpins = 0; + unselectedThreadSpins = 0; + unregisteredThreadSpins = 0; + while (selectedThreadSpins < 1 && unselectedThreadSpins < 1 && + unregisteredThreadSpins < 1) { + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + } + + profiler_pause(); + UniquePtr profile = profiler_get_profile(); + + profiler_stop(); + stopThreads = true; + unregisteredThread.join(); + unselectedThread.join(); + selectedThread.join(); + + JSONOutputCheck(profile.get(), [&](const Json::Value& aRoot) { + GET_JSON(threads, aRoot["threads"], Array); + int foundMain = 0; + int foundSelected = 0; + int foundSelectedMarker = 0; + int foundUnselected = 0; + int foundUnselectedMarker = 0; + for (const Json::Value& thread : threads) { + ASSERT_TRUE(thread.isObject()); + GET_JSON(stringTable, thread["stringTable"], Array); + GET_JSON(name, thread["name"], String); + if (name.asString() == "GeckoMain") { + ++foundMain; + // Don't check the main thread further in this test. + + } else if (name.asString() == "Selected test thread") { + ++foundSelected; + + GET_JSON(samples, thread["samples"], Object); + GET_JSON(samplesData, samples["data"], Array); + EXPECT_GT(samplesData.size(), 0u); + + GET_JSON(markers, thread["markers"], Object); + GET_JSON(markersData, markers["data"], Array); + for (const Json::Value& marker : markersData) { + const unsigned int NAME = 0u; + ASSERT_TRUE(marker[NAME].isUInt()); // name id + GET_JSON(name, stringTable[marker[NAME].asUInt()], String); + if (name == "Spinning Selected!") { + ++foundSelectedMarker; + } + } + } else if (name.asString() == "Registered test thread") { + ++foundUnselected; + + GET_JSON(samples, thread["samples"], Object); + GET_JSON(samplesData, samples["data"], Array); + if (threadCPU || threadSampling) { + EXPECT_GT(samplesData.size(), 0u); + } else { + EXPECT_EQ(samplesData.size(), 0u); + } + + GET_JSON(markers, thread["markers"], Object); + GET_JSON(markersData, markers["data"], Array); + for (const Json::Value& marker : markersData) { + const unsigned int NAME = 0u; + ASSERT_TRUE(marker[NAME].isUInt()); // name id + GET_JSON(name, stringTable[marker[NAME].asUInt()], String); + if (name == "Spinning Registered!") { + ++foundUnselectedMarker; + } + } + + } else { + EXPECT_STRNE(name.asString().c_str(), + "Unregistered test thread label"); + } + } + EXPECT_EQ(foundMain, 1); + EXPECT_EQ(foundSelected, 1); + EXPECT_GT(foundSelectedMarker, 0); + EXPECT_EQ(foundUnselected, + (threadCPU || threadSampling || threadMarkers) ? 1 : 0) + << "Unselected thread should only be present if at least one of the " + "allthreads feature is on"; + if (threadMarkers) { + EXPECT_GT(foundUnselectedMarker, 0); + } else { + EXPECT_EQ(foundUnselectedMarker, 0); + } + }); + } +} + +TEST(GeckoProfiler, FailureHandling) +{ + profiler_init_main_thread_id(); + ASSERT_TRUE(profiler_is_main_thread()) + << "This test assumes it runs on the main thread"; + + uint32_t features = ProfilerFeature::StackWalk; + const char* filters[] = {"GeckoMain"}; + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + + // User-defined marker type that generates a failure when streaming JSON. + struct GtestFailingMarker { + static constexpr Span MarkerTypeName() { + return MakeStringSpan("markers-gtest-failing"); + } + static void StreamJSONMarkerData( + mozilla::baseprofiler::SpliceableJSONWriter& aWriter) { + aWriter.SetFailure("boom!"); + } + static mozilla::MarkerSchema MarkerTypeDisplay() { + return mozilla::MarkerSchema::SpecialFrontendLocation{}; + } + }; + EXPECT_TRUE(profiler_add_marker("Gtest failing marker", + geckoprofiler::category::OTHER, {}, + GtestFailingMarker{})); + + ASSERT_EQ(WaitForSamplingState(), SamplingState::SamplingCompleted); + profiler_pause(); + + FailureLatchSource failureLatch; + SpliceableChunkedJSONWriter w{failureLatch}; + EXPECT_FALSE(w.Failed()); + ASSERT_FALSE(w.GetFailure()); + + w.Start(); + EXPECT_FALSE(w.Failed()); + ASSERT_FALSE(w.GetFailure()); + + // The marker will cause a failure during this function call. + EXPECT_FALSE(::profiler_stream_json_for_this_process(w).isOk()); + EXPECT_TRUE(w.Failed()); + ASSERT_TRUE(w.GetFailure()); + EXPECT_EQ(strcmp(w.GetFailure(), "boom!"), 0); + + // Already failed, check that we don't crash or reset the failure. + EXPECT_FALSE(::profiler_stream_json_for_this_process(w).isOk()); + EXPECT_TRUE(w.Failed()); + ASSERT_TRUE(w.GetFailure()); + EXPECT_EQ(strcmp(w.GetFailure(), "boom!"), 0); + + w.End(); + + profiler_stop(); + + EXPECT_TRUE(w.Failed()); + ASSERT_TRUE(w.GetFailure()); + EXPECT_EQ(strcmp(w.GetFailure(), "boom!"), 0); + + UniquePtr profile = w.ChunkedWriteFunc().CopyData(); + ASSERT_EQ(profile.get(), nullptr); +} + +TEST(GeckoProfiler, NoMarkerStacks) +{ + uint32_t features = ProfilerFeature::NoMarkerStacks; + const char* filters[] = {"GeckoMain"}; + + ASSERT_TRUE(!profiler_get_profile()); + + // Make sure that profiler_capture_backtrace returns nullptr when the profiler + // is not active. + ASSERT_TRUE(!profiler_capture_backtrace()); + + { + // Start the profiler without the NoMarkerStacks feature and make sure we + // capture stacks. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, + /* features */ 0, filters, MOZ_ARRAY_LENGTH(filters), 0); + + ASSERT_TRUE(profiler_capture_backtrace()); + profiler_stop(); + } + + // Start the profiler without the NoMarkerStacks feature and make sure we + // don't capture stacks. + profiler_start(PROFILER_DEFAULT_ENTRIES, PROFILER_DEFAULT_INTERVAL, features, + filters, MOZ_ARRAY_LENGTH(filters), 0); + + // Make sure that the active features has the NoMarkerStacks feature. + mozilla::Maybe activeFeatures = profiler_features_if_active(); + ASSERT_TRUE(activeFeatures.isSome()); + ASSERT_TRUE(ProfilerFeature::HasNoMarkerStacks(*activeFeatures)); + + // Make sure we don't capture stacks. + ASSERT_TRUE(!profiler_capture_backtrace()); + + // Add a marker with a stack to test. + EXPECT_TRUE(profiler_add_marker( + "Text with stack", geckoprofiler::category::OTHER, MarkerStack::Capture(), + geckoprofiler::markers::TextMarker{}, "")); + + UniquePtr profile = profiler_get_profile(); + JSONOutputCheck(profile.get(), [&](const Json::Value& aRoot) { + // Check that the meta.configuration.features array contains + // "nomarkerstacks". + GET_JSON(meta, aRoot["meta"], Object); + { + GET_JSON(configuration, meta["configuration"], Object); + { + GET_JSON(features, configuration["features"], Array); + { + EXPECT_EQ(features.size(), 1u); + EXPECT_JSON_ARRAY_CONTAINS(features, String, "nomarkerstacks"); + } + } + } + + // Make sure that the marker we captured doesn't have a stack. + GET_JSON(threads, aRoot["threads"], Array); + { + ASSERT_EQ(threads.size(), 1u); + GET_JSON(thread0, threads[0], Object); + { + GET_JSON(markers, thread0["markers"], Object); + { + GET_JSON(data, markers["data"], Array); + { + const unsigned int NAME = 0u; + const unsigned int PAYLOAD = 5u; + bool foundMarker = false; + GET_JSON(stringTable, thread0["stringTable"], Array); + + for (const Json::Value& marker : data) { + // Even though we only added one marker, some markers like + // NotifyObservers are being added as well. Let's iterate over + // them and make sure that we have the one we added explicitly and + // check its stack doesn't exist. + GET_JSON(name, stringTable[marker[NAME].asUInt()], String); + std::string nameString = name.asString(); + + if (nameString == "Text with stack") { + // Make sure that the marker doesn't have a stack. + foundMarker = true; + EXPECT_FALSE(marker[PAYLOAD].isNull()); + EXPECT_TRUE(marker[PAYLOAD]["stack"].isNull()); + } + } + + EXPECT_TRUE(foundMarker); + } + } + } + } + }); + + profiler_stop(); + + ASSERT_TRUE(!profiler_get_profile()); +} + +#endif // MOZ_GECKO_PROFILER diff --git a/tools/profiler/tests/gtest/LulTest.cpp b/tools/profiler/tests/gtest/LulTest.cpp new file mode 100644 index 0000000000..159a366567 --- /dev/null +++ b/tools/profiler/tests/gtest/LulTest.cpp @@ -0,0 +1,51 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "mozilla/Atomics.h" +#include "LulMain.h" +#include "GeckoProfiler.h" // for TracingKind +#include "platform-linux-lul.h" // for read_procmaps + +// Set this to 0 to make LUL be completely silent during tests. +// Set it to 1 to get logging output from LUL, presumably for +// the purpose of debugging it. +#define DEBUG_LUL_TEST 0 + +// LUL needs a callback for its logging sink. +static void gtest_logging_sink_for_LulIntegration(const char* str) { + if (DEBUG_LUL_TEST == 0) { + return; + } + // Ignore any trailing \n, since LOG will add one anyway. + size_t n = strlen(str); + if (n > 0 && str[n - 1] == '\n') { + char* tmp = strdup(str); + tmp[n - 1] = 0; + fprintf(stderr, "LUL-in-gtest: %s\n", tmp); + free(tmp); + } else { + fprintf(stderr, "LUL-in-gtest: %s\n", str); + } +} + +TEST(LulIntegration, unwind_consistency) +{ + // Set up LUL and get it to read unwind info for libxul.so, which is + // all we care about here, plus (incidentally) practically every + // other object in the process too. + lul::LUL* lul = new lul::LUL(gtest_logging_sink_for_LulIntegration); + read_procmaps(lul); + + // Run unwind tests and receive information about how many there + // were and how many were successful. + lul->EnableUnwinding(); + int nTests = 0, nTestsPassed = 0; + RunLulUnitTests(&nTests, &nTestsPassed, lul); + EXPECT_TRUE(nTests == 6) << "Unexpected number of tests"; + EXPECT_EQ(nTestsPassed, nTests) << "Not all tests passed"; + + delete lul; +} diff --git a/tools/profiler/tests/gtest/LulTestDwarf.cpp b/tools/profiler/tests/gtest/LulTestDwarf.cpp new file mode 100644 index 0000000000..55373ec093 --- /dev/null +++ b/tools/profiler/tests/gtest/LulTestDwarf.cpp @@ -0,0 +1,2733 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "LulCommonExt.h" +#include "LulDwarfExt.h" +#include "LulDwarfInt.h" +#include "LulTestInfrastructure.h" + +using lul_test::CFISection; +using lul_test::test_assembler::kBigEndian; +using lul_test::test_assembler::kLittleEndian; +using lul_test::test_assembler::Label; +using testing::_; +using testing::InSequence; +using testing::Return; +using testing::Sequence; +using testing::Test; + +#define PERHAPS_WRITE_DEBUG_FRAME_FILE(name, section) /**/ +#define PERHAPS_WRITE_EH_FRAME_FILE(name, section) /**/ + +// Set this to 0 to make LUL be completely silent during tests. +// Set it to 1 to get logging output from LUL, presumably for +// the purpose of debugging it. +#define DEBUG_LUL_TEST_DWARF 0 + +// LUL needs a callback for its logging sink. +static void gtest_logging_sink_for_LulTestDwarf(const char* str) { + if (DEBUG_LUL_TEST_DWARF == 0) { + return; + } + // Ignore any trailing \n, since LOG will add one anyway. + size_t n = strlen(str); + if (n > 0 && str[n - 1] == '\n') { + char* tmp = strdup(str); + tmp[n - 1] = 0; + fprintf(stderr, "LUL-in-gtest: %s\n", tmp); + free(tmp); + } else { + fprintf(stderr, "LUL-in-gtest: %s\n", str); + } +} + +namespace lul { + +class MockCallFrameInfoHandler : public CallFrameInfo::Handler { + public: + MOCK_METHOD6(Entry, + bool(size_t offset, uint64 address, uint64 length, uint8 version, + const std::string& augmentation, unsigned return_address)); + MOCK_METHOD2(UndefinedRule, bool(uint64 address, int reg)); + MOCK_METHOD2(SameValueRule, bool(uint64 address, int reg)); + MOCK_METHOD4(OffsetRule, + bool(uint64 address, int reg, int base_register, long offset)); + MOCK_METHOD4(ValOffsetRule, + bool(uint64 address, int reg, int base_register, long offset)); + MOCK_METHOD3(RegisterRule, bool(uint64 address, int reg, int base_register)); + MOCK_METHOD3(ExpressionRule, + bool(uint64 address, int reg, const ImageSlice& expression)); + MOCK_METHOD3(ValExpressionRule, + bool(uint64 address, int reg, const ImageSlice& expression)); + MOCK_METHOD0(End, bool()); + MOCK_METHOD2(PersonalityRoutine, bool(uint64 address, bool indirect)); + MOCK_METHOD2(LanguageSpecificDataArea, bool(uint64 address, bool indirect)); + MOCK_METHOD0(SignalHandler, bool()); +}; + +class MockCallFrameErrorReporter : public CallFrameInfo::Reporter { + public: + MockCallFrameErrorReporter() + : Reporter(gtest_logging_sink_for_LulTestDwarf, "mock filename", + "mock section") {} + MOCK_METHOD2(Incomplete, void(uint64, CallFrameInfo::EntryKind)); + MOCK_METHOD1(EarlyEHTerminator, void(uint64)); + MOCK_METHOD2(CIEPointerOutOfRange, void(uint64, uint64)); + MOCK_METHOD2(BadCIEId, void(uint64, uint64)); + MOCK_METHOD2(UnrecognizedVersion, void(uint64, int version)); + MOCK_METHOD2(UnrecognizedAugmentation, void(uint64, const string&)); + MOCK_METHOD2(InvalidPointerEncoding, void(uint64, uint8)); + MOCK_METHOD2(UnusablePointerEncoding, void(uint64, uint8)); + MOCK_METHOD2(RestoreInCIE, void(uint64, uint64)); + MOCK_METHOD3(BadInstruction, void(uint64, CallFrameInfo::EntryKind, uint64)); + MOCK_METHOD3(NoCFARule, void(uint64, CallFrameInfo::EntryKind, uint64)); + MOCK_METHOD3(EmptyStateStack, void(uint64, CallFrameInfo::EntryKind, uint64)); + MOCK_METHOD3(ClearingCFARule, void(uint64, CallFrameInfo::EntryKind, uint64)); +}; + +struct CFIFixture { + enum { kCFARegister = CallFrameInfo::Handler::kCFARegister }; + + CFIFixture() { + // Default expectations for the data handler. + // + // - Leave Entry and End without expectations, as it's probably a + // good idea to set those explicitly in each test. + // + // - Expect the *Rule functions to not be called, + // so that each test can simply list the calls they expect. + // + // I gather I could use StrictMock for this, but the manual seems + // to suggest using that only as a last resort, and this isn't so + // bad. + EXPECT_CALL(handler, UndefinedRule(_, _)).Times(0); + EXPECT_CALL(handler, SameValueRule(_, _)).Times(0); + EXPECT_CALL(handler, OffsetRule(_, _, _, _)).Times(0); + EXPECT_CALL(handler, ValOffsetRule(_, _, _, _)).Times(0); + EXPECT_CALL(handler, RegisterRule(_, _, _)).Times(0); + EXPECT_CALL(handler, ExpressionRule(_, _, _)).Times(0); + EXPECT_CALL(handler, ValExpressionRule(_, _, _)).Times(0); + EXPECT_CALL(handler, PersonalityRoutine(_, _)).Times(0); + EXPECT_CALL(handler, LanguageSpecificDataArea(_, _)).Times(0); + EXPECT_CALL(handler, SignalHandler()).Times(0); + + // Default expectations for the error/warning reporer. + EXPECT_CALL(reporter, Incomplete(_, _)).Times(0); + EXPECT_CALL(reporter, EarlyEHTerminator(_)).Times(0); + EXPECT_CALL(reporter, CIEPointerOutOfRange(_, _)).Times(0); + EXPECT_CALL(reporter, BadCIEId(_, _)).Times(0); + EXPECT_CALL(reporter, UnrecognizedVersion(_, _)).Times(0); + EXPECT_CALL(reporter, UnrecognizedAugmentation(_, _)).Times(0); + EXPECT_CALL(reporter, InvalidPointerEncoding(_, _)).Times(0); + EXPECT_CALL(reporter, UnusablePointerEncoding(_, _)).Times(0); + EXPECT_CALL(reporter, RestoreInCIE(_, _)).Times(0); + EXPECT_CALL(reporter, BadInstruction(_, _, _)).Times(0); + EXPECT_CALL(reporter, NoCFARule(_, _, _)).Times(0); + EXPECT_CALL(reporter, EmptyStateStack(_, _, _)).Times(0); + EXPECT_CALL(reporter, ClearingCFARule(_, _, _)).Times(0); + } + + MockCallFrameInfoHandler handler; + MockCallFrameErrorReporter reporter; +}; + +class LulDwarfCFI : public CFIFixture, public Test {}; + +TEST_F(LulDwarfCFI, EmptyRegion) { + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + static const char data[1] = {42}; + + ByteReader reader(ENDIANNESS_BIG); + CallFrameInfo parser(data, 0, &reader, &handler, &reporter); + EXPECT_TRUE(parser.Start()); +} + +TEST_F(LulDwarfCFI, IncompleteLength32) { + CFISection section(kBigEndian, 8); + section + // Not even long enough for an initial length. + .D16(0xa0f) + // Padding to keep valgrind happy. We subtract these off when we + // construct the parser. + .D16(0); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + EXPECT_CALL(reporter, Incomplete(_, CallFrameInfo::kUnknown)) + .WillOnce(Return()); + + string contents; + ASSERT_TRUE(section.GetContents(&contents)); + + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(8); + CallFrameInfo parser(contents.data(), contents.size() - 2, &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +TEST_F(LulDwarfCFI, IncompleteLength64) { + CFISection section(kLittleEndian, 4); + section + // An incomplete 64-bit DWARF initial length. + .D32(0xffffffff) + .D32(0x71fbaec2) + // Padding to keep valgrind happy. We subtract these off when we + // construct the parser. + .D32(0); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + EXPECT_CALL(reporter, Incomplete(_, CallFrameInfo::kUnknown)) + .WillOnce(Return()); + + string contents; + ASSERT_TRUE(section.GetContents(&contents)); + + ByteReader reader(ENDIANNESS_LITTLE); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size() - 4, &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +TEST_F(LulDwarfCFI, IncompleteId32) { + CFISection section(kBigEndian, 8); + section + .D32(3) // Initial length, not long enough for id + .D8(0xd7) + .D8(0xe5) + .D8(0xf1) // incomplete id + .CIEHeader(8727, 3983, 8889, 3, "") + .FinishEntry(); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + EXPECT_CALL(reporter, Incomplete(_, CallFrameInfo::kUnknown)) + .WillOnce(Return()); + + string contents; + ASSERT_TRUE(section.GetContents(&contents)); + + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(8); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +TEST_F(LulDwarfCFI, BadId32) { + CFISection section(kBigEndian, 8); + section + .D32(0x100) // Initial length + .D32(0xe802fade) // bogus ID + .Append(0x100 - 4, 0x42); // make the length true + section.CIEHeader(1672, 9872, 8529, 3, "").FinishEntry(); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + EXPECT_CALL(reporter, CIEPointerOutOfRange(_, 0xe802fade)).WillOnce(Return()); + + string contents; + ASSERT_TRUE(section.GetContents(&contents)); + + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(8); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +// A lone CIE shouldn't cause any handler calls. +TEST_F(LulDwarfCFI, SingleCIE) { + CFISection section(kLittleEndian, 4); + section.CIEHeader(0xffe799a8, 0x3398dcdd, 0x6e9683de, 3, ""); + section.Append(10, lul::DW_CFA_nop); + section.FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("SingleCIE", section); + + EXPECT_CALL(handler, Entry(_, _, _, _, _, _)).Times(0); + EXPECT_CALL(handler, End()).Times(0); + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_LITTLE); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// One FDE, one CIE. +TEST_F(LulDwarfCFI, OneFDE) { + CFISection section(kBigEndian, 4); + Label cie; + section.Mark(&cie) + .CIEHeader(0x4be22f75, 0x2492236e, 0x6b6efb87, 3, "") + .FinishEntry() + .FDEHeader(cie, 0x7714740d, 0x3d5a10cd) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("OneFDE", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0x7714740d, 0x3d5a10cd, 3, "", 0x6b6efb87)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// Two FDEs share a CIE. +TEST_F(LulDwarfCFI, TwoFDEsOneCIE) { + CFISection section(kBigEndian, 4); + Label cie; + section + // First FDE. readelf complains about this one because it makes + // a forward reference to its CIE. + .FDEHeader(cie, 0xa42744df, 0xa3b42121) + .FinishEntry() + // CIE. + .Mark(&cie) + .CIEHeader(0x04f7dc7b, 0x3d00c05f, 0xbd43cb59, 3, "") + .FinishEntry() + // Second FDE. + .FDEHeader(cie, 0x6057d391, 0x700f608d) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("TwoFDEsOneCIE", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0xa42744df, 0xa3b42121, 3, "", 0xbd43cb59)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0x6057d391, 0x700f608d, 3, "", 0xbd43cb59)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// Two FDEs, two CIEs. +TEST_F(LulDwarfCFI, TwoFDEsTwoCIEs) { + CFISection section(kLittleEndian, 8); + Label cie1, cie2; + section + // First CIE. + .Mark(&cie1) + .CIEHeader(0x694d5d45, 0x4233221b, 0xbf45e65a, 3, "") + .FinishEntry() + // First FDE which cites second CIE. readelf complains about + // this one because it makes a forward reference to its CIE. + .FDEHeader(cie2, 0x778b27dfe5871f05ULL, 0x324ace3448070926ULL) + .FinishEntry() + // Second FDE, which cites first CIE. + .FDEHeader(cie1, 0xf6054ca18b10bf5fULL, 0x45fdb970d8bca342ULL) + .FinishEntry() + // Second CIE. + .Mark(&cie2) + .CIEHeader(0xfba3fad7, 0x6287e1fd, 0x61d2c581, 2, "") + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("TwoFDEsTwoCIEs", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0x778b27dfe5871f05ULL, 0x324ace3448070926ULL, + 2, "", 0x61d2c581)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0xf6054ca18b10bf5fULL, 0x45fdb970d8bca342ULL, + 3, "", 0xbf45e65a)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_LITTLE); + reader.SetAddressSize(8); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// An FDE whose CIE specifies a version we don't recognize. +TEST_F(LulDwarfCFI, BadVersion) { + CFISection section(kBigEndian, 4); + Label cie1, cie2; + section.Mark(&cie1) + .CIEHeader(0xca878cf0, 0x7698ec04, 0x7b616f54, 0x52, "") + .FinishEntry() + // We should skip this entry, as its CIE specifies a version we + // don't recognize. + .FDEHeader(cie1, 0x08852292, 0x2204004a) + .FinishEntry() + // Despite the above, we should visit this entry. + .Mark(&cie2) + .CIEHeader(0x7c3ae7c9, 0xb9b9a512, 0x96cb3264, 3, "") + .FinishEntry() + .FDEHeader(cie2, 0x2094735a, 0x6e875501) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("BadVersion", section); + + EXPECT_CALL(reporter, UnrecognizedVersion(_, 0x52)).WillOnce(Return()); + + { + InSequence s; + // We should see no mention of the first FDE, but we should get + // a call to Entry for the second. + EXPECT_CALL(handler, Entry(_, 0x2094735a, 0x6e875501, 3, "", 0x96cb3264)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +// An FDE whose CIE specifies an augmentation we don't recognize. +TEST_F(LulDwarfCFI, BadAugmentation) { + CFISection section(kBigEndian, 4); + Label cie1, cie2; + section.Mark(&cie1) + .CIEHeader(0x4be22f75, 0x2492236e, 0x6b6efb87, 3, "spaniels!") + .FinishEntry() + // We should skip this entry, as its CIE specifies an + // augmentation we don't recognize. + .FDEHeader(cie1, 0x7714740d, 0x3d5a10cd) + .FinishEntry() + // Despite the above, we should visit this entry. + .Mark(&cie2) + .CIEHeader(0xf8bc4399, 0x8cf09931, 0xf2f519b2, 3, "") + .FinishEntry() + .FDEHeader(cie2, 0x7bf0fda0, 0xcbcd28d8) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("BadAugmentation", section); + + EXPECT_CALL(reporter, UnrecognizedAugmentation(_, "spaniels!")) + .WillOnce(Return()); + + { + InSequence s; + // We should see no mention of the first FDE, but we should get + // a call to Entry for the second. + EXPECT_CALL(handler, Entry(_, 0x7bf0fda0, 0xcbcd28d8, 3, "", 0xf2f519b2)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_FALSE(parser.Start()); +} + +// The return address column field is a byte in CFI version 1 +// (DWARF2), but a ULEB128 value in version 3 (DWARF3). +TEST_F(LulDwarfCFI, CIEVersion1ReturnColumn) { + CFISection section(kBigEndian, 4); + Label cie; + section + // CIE, using the version 1 format: return column is a ubyte. + .Mark(&cie) + // Use a value for the return column that is parsed differently + // as a ubyte and as a ULEB128. + .CIEHeader(0xbcdea24f, 0x5be28286, 0x9f, 1, "") + .FinishEntry() + // FDE, citing that CIE. + .FDEHeader(cie, 0xb8d347b5, 0x825e55dc) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("CIEVersion1ReturnColumn", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0xb8d347b5, 0x825e55dc, 1, "", 0x9f)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +// The return address column field is a byte in CFI version 1 +// (DWARF2), but a ULEB128 value in version 3 (DWARF3). +TEST_F(LulDwarfCFI, CIEVersion3ReturnColumn) { + CFISection section(kBigEndian, 4); + Label cie; + section + // CIE, using the version 3 format: return column is a ULEB128. + .Mark(&cie) + // Use a value for the return column that is parsed differently + // as a ubyte and as a ULEB128. + .CIEHeader(0x0ab4758d, 0xc010fdf7, 0x89, 3, "") + .FinishEntry() + // FDE, citing that CIE. + .FDEHeader(cie, 0x86763f2b, 0x2a66dc23) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("CIEVersion3ReturnColumn", section); + + { + InSequence s; + EXPECT_CALL(handler, Entry(_, 0x86763f2b, 0x2a66dc23, 3, "", 0x89)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + string contents; + EXPECT_TRUE(section.GetContents(&contents)); + ByteReader reader(ENDIANNESS_BIG); + reader.SetAddressSize(4); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter); + EXPECT_TRUE(parser.Start()); +} + +struct CFIInsnFixture : public CFIFixture { + CFIInsnFixture() : CFIFixture() { + data_factor = 0xb6f; + return_register = 0x9be1ed9f; + version = 3; + cfa_base_register = 0x383a3aa; + cfa_offset = 0xf748; + } + + // Prepare SECTION to receive FDE instructions. + // + // - Append a stock CIE header that establishes the fixture's + // code_factor, data_factor, return_register, version, and + // augmentation values. + // - Have the CIE set up a CFA rule using cfa_base_register and + // cfa_offset. + // - Append a stock FDE header, referring to the above CIE, for the + // fde_size bytes at fde_start. Choose fde_start and fde_size + // appropriately for the section's address size. + // - Set appropriate expectations on handler in sequence s for the + // frame description entry and the CIE's CFA rule. + // + // On return, SECTION is ready to have FDE instructions appended to + // it, and its FinishEntry member called. + void StockCIEAndFDE(CFISection* section) { + // Choose appropriate constants for our address size. + if (section->AddressSize() == 4) { + fde_start = 0xc628ecfbU; + fde_size = 0x5dee04a2; + code_factor = 0x60b; + } else { + assert(section->AddressSize() == 8); + fde_start = 0x0005c57ce7806bd3ULL; + fde_size = 0x2699521b5e333100ULL; + code_factor = 0x01008e32855274a8ULL; + } + + // Create the CIE. + (*section) + .Mark(&cie_label) + .CIEHeader(code_factor, data_factor, return_register, version, "") + .D8(lul::DW_CFA_def_cfa) + .ULEB128(cfa_base_register) + .ULEB128(cfa_offset) + .FinishEntry(); + + // Create the FDE. + section->FDEHeader(cie_label, fde_start, fde_size); + + // Expect an Entry call for the FDE and a ValOffsetRule call for the + // CIE's CFA rule. + EXPECT_CALL(handler, + Entry(_, fde_start, fde_size, version, "", return_register)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, + cfa_base_register, cfa_offset)) + .InSequence(s) + .WillOnce(Return(true)); + } + + // Run the contents of SECTION through a CallFrameInfo parser, + // expecting parser.Start to return SUCCEEDS. Caller may optionally + // supply, via READER, its own ByteReader. If that's absent, a + // local one is used. + void ParseSection(CFISection* section, bool succeeds = true, + ByteReader* reader = nullptr) { + string contents; + EXPECT_TRUE(section->GetContents(&contents)); + lul::Endianness endianness; + if (section->endianness() == kBigEndian) + endianness = ENDIANNESS_BIG; + else { + assert(section->endianness() == kLittleEndian); + endianness = ENDIANNESS_LITTLE; + } + ByteReader local_reader(endianness); + ByteReader* reader_to_use = reader ? reader : &local_reader; + reader_to_use->SetAddressSize(section->AddressSize()); + CallFrameInfo parser(contents.data(), contents.size(), reader_to_use, + &handler, &reporter); + if (succeeds) + EXPECT_TRUE(parser.Start()); + else + EXPECT_FALSE(parser.Start()); + } + + Label cie_label; + Sequence s; + uint64 code_factor; + int data_factor; + unsigned return_register; + unsigned version; + unsigned cfa_base_register; + int cfa_offset; + uint64 fde_start, fde_size; +}; + +class LulDwarfCFIInsn : public CFIInsnFixture, public Test {}; + +TEST_F(LulDwarfCFIInsn, DW_CFA_set_loc) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_set_loc) + .D32(0xb1ee3e7a) + // Use DW_CFA_def_cfa to force a handler call that we can use to + // check the effect of the DW_CFA_set_loc. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x4defb431) + .ULEB128(0x6d17b0ee) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_set_loc", section); + + EXPECT_CALL(handler, + ValOffsetRule(0xb1ee3e7a, kCFARegister, 0x4defb431, 0x6d17b0ee)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_advance_loc) { + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + section + .D8(lul::DW_CFA_advance_loc | 0x2a) + // Use DW_CFA_def_cfa to force a handler call that we can use to + // check the effect of the DW_CFA_advance_loc. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x5bbb3715) + .ULEB128(0x0186c7bf) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc", section); + + EXPECT_CALL(handler, ValOffsetRule(fde_start + 0x2a * code_factor, + kCFARegister, 0x5bbb3715, 0x0186c7bf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_advance_loc1) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_advance_loc1) + .D8(0xd8) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x69d5696a) + .ULEB128(0x1eb7fc93) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc1", section); + + EXPECT_CALL(handler, ValOffsetRule((fde_start + 0xd8 * code_factor), + kCFARegister, 0x69d5696a, 0x1eb7fc93)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_advance_loc2) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_advance_loc2) + .D16(0x3adb) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x3a368bed) + .ULEB128(0x3194ee37) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc2", section); + + EXPECT_CALL(handler, ValOffsetRule((fde_start + 0x3adb * code_factor), + kCFARegister, 0x3a368bed, 0x3194ee37)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_advance_loc4) { + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_advance_loc4) + .D32(0x15813c88) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x135270c5) + .ULEB128(0x24bad7cb) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc4", section); + + EXPECT_CALL(handler, ValOffsetRule((fde_start + 0x15813c88ULL * code_factor), + kCFARegister, 0x135270c5, 0x24bad7cb)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_MIPS_advance_loc8) { + code_factor = 0x2d; + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_MIPS_advance_loc8) + .D64(0x3c4f3945b92c14ULL) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0xe17ed602) + .ULEB128(0x3d162e7f) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_advance_loc8", section); + + EXPECT_CALL(handler, + ValOffsetRule((fde_start + 0x3c4f3945b92c14ULL * code_factor), + kCFARegister, 0xe17ed602, 0x3d162e7f)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa) + .ULEB128(0x4e363a85) + .ULEB128(0x815f9aa7) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("DW_CFA_def_cfa", section); + + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x4e363a85, 0x815f9aa7)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_sf) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_sf) + .ULEB128(0x8ccb32b7) + .LEB128(0x9ea) + .D8(lul::DW_CFA_def_cfa_sf) + .ULEB128(0x9b40f5da) + .LEB128(-0x40a2) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, 0x8ccb32b7, + 0x9ea * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, 0x9b40f5da, + -0x40a2 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_register) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_register).ULEB128(0x3e7e9363).FinishEntry(); + + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x3e7e9363, cfa_offset)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +// DW_CFA_def_cfa_register should have no effect when applied to a +// non-base/offset rule. +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_registerBadRule) { + ByteReader reader(ENDIANNESS_BIG); + CFISection section(kBigEndian, 4); + ImageSlice expr("needle in a haystack"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_expression) + .Block(expr) + .D8(lul::DW_CFA_def_cfa_register) + .ULEB128(0xf1b49e49) + .FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, kCFARegister, expr)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_offset) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_offset).ULEB128(0x1e8e3b9b).FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, cfa_base_register, + 0x1e8e3b9b)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_offset_sf) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_offset_sf) + .LEB128(0x970) + .D8(lul::DW_CFA_def_cfa_offset_sf) + .LEB128(-0x2cd) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, cfa_base_register, + 0x970 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start, kCFARegister, cfa_base_register, + -0x2cd * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +// DW_CFA_def_cfa_offset should have no effect when applied to a +// non-base/offset rule. +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_offsetBadRule) { + ByteReader reader(ENDIANNESS_BIG); + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + ImageSlice expr("six ways to Sunday"); + section.D8(lul::DW_CFA_def_cfa_expression) + .Block(expr) + .D8(lul::DW_CFA_def_cfa_offset) + .ULEB128(0x1e8e3b9b) + .FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, kCFARegister, expr)) + .WillRepeatedly(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_def_cfa_expression) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 8); + ImageSlice expr("eating crow"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_def_cfa_expression).Block(expr).FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, kCFARegister, expr)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_undefined) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_undefined).ULEB128(0x300ce45d).FinishEntry(); + + EXPECT_CALL(handler, UndefinedRule(fde_start, 0x300ce45d)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_same_value) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_same_value).ULEB128(0x3865a760).FinishEntry(); + + EXPECT_CALL(handler, SameValueRule(fde_start, 0x3865a760)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_offset) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset | 0x2c).ULEB128(0x9f6).FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x2c, kCFARegister, 0x9f6 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_offset_extended) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset_extended) + .ULEB128(0x402b) + .ULEB128(0xb48) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x402b, kCFARegister, 0xb48 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_offset_extended_sf) { + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset_extended_sf) + .ULEB128(0x997c23ee) + .LEB128(0x2d00) + .D8(lul::DW_CFA_offset_extended_sf) + .ULEB128(0x9519eb82) + .LEB128(-0xa77) + .FinishEntry(); + + EXPECT_CALL(handler, OffsetRule(fde_start, 0x997c23ee, kCFARegister, + 0x2d00 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(fde_start, 0x9519eb82, kCFARegister, + -0xa77 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_val_offset) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset) + .ULEB128(0x623562fe) + .ULEB128(0x673) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x623562fe, kCFARegister, + 0x673 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_val_offset_sf) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset_sf) + .ULEB128(0x6f4f) + .LEB128(0xaab) + .D8(lul::DW_CFA_val_offset_sf) + .ULEB128(0x2483) + .LEB128(-0x8a2) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x6f4f, kCFARegister, + 0xaab * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x2483, kCFARegister, + -0x8a2 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_register) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0x278d18f9) + .ULEB128(0x1a684414) + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0x278d18f9, 0x1a684414)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_expression) { + ByteReader reader(ENDIANNESS_BIG); + CFISection section(kBigEndian, 8); + StockCIEAndFDE(§ion); + ImageSlice expr("plus ça change, plus c'est la même chose"); + section.D8(lul::DW_CFA_expression) + .ULEB128(0xa1619fb2) + .Block(expr) + .FinishEntry(); + + EXPECT_CALL(handler, ExpressionRule(fde_start, 0xa1619fb2, expr)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_val_expression) { + ByteReader reader(ENDIANNESS_BIG); + CFISection section(kBigEndian, 4); + ImageSlice expr("he who has the gold makes the rules"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_expression) + .ULEB128(0xc5e4a9e3) + .Block(expr) + .FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, 0xc5e4a9e3, expr)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_restore) { + CFISection section(kLittleEndian, 8); + code_factor = 0x01bd188a9b1fa083ULL; + data_factor = -0x1ac8; + return_register = 0x8c35b049; + version = 2; + fde_start = 0x2d70fe998298bbb1ULL; + fde_size = 0x46ccc2e63cf0b108ULL; + Label cie; + section.Mark(&cie) + .CIEHeader(code_factor, data_factor, return_register, version, "") + // Provide a CFA rule, because register rules require them. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x6ca1d50e) + .ULEB128(0x372e38e8) + // Provide an offset(N) rule for register 0x3c. + .D8(lul::DW_CFA_offset | 0x3c) + .ULEB128(0xb348) + .FinishEntry() + // In the FDE... + .FDEHeader(cie, fde_start, fde_size) + // At a second address, provide a new offset(N) rule for register 0x3c. + .D8(lul::DW_CFA_advance_loc | 0x13) + .D8(lul::DW_CFA_offset | 0x3c) + .ULEB128(0x9a50) + // At a third address, restore the original rule for register 0x3c. + .D8(lul::DW_CFA_advance_loc | 0x01) + .D8(lul::DW_CFA_restore | 0x3c) + .FinishEntry(); + + { + InSequence s; + EXPECT_CALL(handler, + Entry(_, fde_start, fde_size, version, "", return_register)) + .WillOnce(Return(true)); + // CIE's CFA rule. + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x6ca1d50e, 0x372e38e8)) + .WillOnce(Return(true)); + // CIE's rule for register 0x3c. + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x3c, kCFARegister, 0xb348 * data_factor)) + .WillOnce(Return(true)); + // FDE's rule for register 0x3c. + EXPECT_CALL(handler, OffsetRule(fde_start + 0x13 * code_factor, 0x3c, + kCFARegister, 0x9a50 * data_factor)) + .WillOnce(Return(true)); + // Restore CIE's rule for register 0x3c. + EXPECT_CALL(handler, OffsetRule(fde_start + (0x13 + 0x01) * code_factor, + 0x3c, kCFARegister, 0xb348 * data_factor)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_restoreNoRule) { + CFISection section(kBigEndian, 4); + code_factor = 0x005f78143c1c3b82ULL; + data_factor = 0x25d0; + return_register = 0xe8; + version = 1; + fde_start = 0x4062e30f; + fde_size = 0x5302a389; + Label cie; + section.Mark(&cie) + .CIEHeader(code_factor, data_factor, return_register, version, "") + // Provide a CFA rule, because register rules require them. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x470aa334) + .ULEB128(0x099ef127) + .FinishEntry() + // In the FDE... + .FDEHeader(cie, fde_start, fde_size) + // At a second address, provide an offset(N) rule for register 0x2c. + .D8(lul::DW_CFA_advance_loc | 0x7) + .D8(lul::DW_CFA_offset | 0x2c) + .ULEB128(0x1f47) + // At a third address, restore the (missing) CIE rule for register 0x2c. + .D8(lul::DW_CFA_advance_loc | 0xb) + .D8(lul::DW_CFA_restore | 0x2c) + .FinishEntry(); + + { + InSequence s; + EXPECT_CALL(handler, + Entry(_, fde_start, fde_size, version, "", return_register)) + .WillOnce(Return(true)); + // CIE's CFA rule. + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x470aa334, 0x099ef127)) + .WillOnce(Return(true)); + // FDE's rule for register 0x2c. + EXPECT_CALL(handler, OffsetRule(fde_start + 0x7 * code_factor, 0x2c, + kCFARegister, 0x1f47 * data_factor)) + .WillOnce(Return(true)); + // Restore CIE's (missing) rule for register 0x2c. + EXPECT_CALL(handler, + SameValueRule(fde_start + (0x7 + 0xb) * code_factor, 0x2c)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_restore_extended) { + CFISection section(kBigEndian, 4); + code_factor = 0x126e; + data_factor = -0xd8b; + return_register = 0x77711787; + version = 3; + fde_start = 0x01f55a45; + fde_size = 0x452adb80; + Label cie; + section.Mark(&cie) + .CIEHeader(code_factor, data_factor, return_register, version, "", + true /* dwarf64 */) + // Provide a CFA rule, because register rules require them. + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x56fa0edd) + .ULEB128(0x097f78a5) + // Provide an offset(N) rule for register 0x0f9b8a1c. + .D8(lul::DW_CFA_offset_extended) + .ULEB128(0x0f9b8a1c) + .ULEB128(0xc979) + .FinishEntry() + // In the FDE... + .FDEHeader(cie, fde_start, fde_size) + // At a second address, provide a new offset(N) rule for reg 0x0f9b8a1c. + .D8(lul::DW_CFA_advance_loc | 0x3) + .D8(lul::DW_CFA_offset_extended) + .ULEB128(0x0f9b8a1c) + .ULEB128(0x3b7b) + // At a third address, restore the original rule for register 0x0f9b8a1c. + .D8(lul::DW_CFA_advance_loc | 0x04) + .D8(lul::DW_CFA_restore_extended) + .ULEB128(0x0f9b8a1c) + .FinishEntry(); + + { + InSequence s; + EXPECT_CALL(handler, + Entry(_, fde_start, fde_size, version, "", return_register)) + .WillOnce(Return(true)); + // CIE's CFA rule. + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x56fa0edd, 0x097f78a5)) + .WillOnce(Return(true)); + // CIE's rule for register 0x0f9b8a1c. + EXPECT_CALL(handler, OffsetRule(fde_start, 0x0f9b8a1c, kCFARegister, + 0xc979 * data_factor)) + .WillOnce(Return(true)); + // FDE's rule for register 0x0f9b8a1c. + EXPECT_CALL(handler, OffsetRule(fde_start + 0x3 * code_factor, 0x0f9b8a1c, + kCFARegister, 0x3b7b * data_factor)) + .WillOnce(Return(true)); + // Restore CIE's rule for register 0x0f9b8a1c. + EXPECT_CALL(handler, + OffsetRule(fde_start + (0x3 + 0x4) * code_factor, 0x0f9b8a1c, + kCFARegister, 0xc979 * data_factor)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_remember_and_restore_state) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + + // We create a state, save it, modify it, and then restore. We + // refer to the state that is overridden the restore as the + // "outgoing" state, and the restored state the "incoming" state. + // + // Register outgoing incoming expect + // 1 offset(N) no rule new "same value" rule + // 2 register(R) offset(N) report changed rule + // 3 offset(N) offset(M) report changed offset + // 4 offset(N) offset(N) no report + // 5 offset(N) no rule new "same value" rule + section + // Create the "incoming" state, which we will save and later restore. + .D8(lul::DW_CFA_offset | 2) + .ULEB128(0x9806) + .D8(lul::DW_CFA_offset | 3) + .ULEB128(0x995d) + .D8(lul::DW_CFA_offset | 4) + .ULEB128(0x7055) + .D8(lul::DW_CFA_remember_state) + // Advance to a new instruction; an implementation could legitimately + // ignore all but the final rule for a given register at a given address. + .D8(lul::DW_CFA_advance_loc | 1) + // Create the "outgoing" state, which we will discard. + .D8(lul::DW_CFA_offset | 1) + .ULEB128(0xea1a) + .D8(lul::DW_CFA_register) + .ULEB128(2) + .ULEB128(0x1d2a3767) + .D8(lul::DW_CFA_offset | 3) + .ULEB128(0xdd29) + .D8(lul::DW_CFA_offset | 5) + .ULEB128(0xf1ce) + // At a third address, restore the incoming state. + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + uint64 addr = fde_start; + + // Expect the incoming rules to be reported. + EXPECT_CALL(handler, OffsetRule(addr, 2, kCFARegister, 0x9806 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 3, kCFARegister, 0x995d * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 4, kCFARegister, 0x7055 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + + addr += code_factor; + + // After the save, we establish the outgoing rule set. + EXPECT_CALL(handler, OffsetRule(addr, 1, kCFARegister, 0xea1a * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, RegisterRule(addr, 2, 0x1d2a3767)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 3, kCFARegister, 0xdd29 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 5, kCFARegister, 0xf1ce * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + + addr += code_factor; + + // Finally, after the restore, expect to see the differences from + // the outgoing to the incoming rules reported. + EXPECT_CALL(handler, SameValueRule(addr, 1)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 2, kCFARegister, 0x9806 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(addr, 3, kCFARegister, 0x995d * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SameValueRule(addr, 5)) + .InSequence(s) + .WillOnce(Return(true)); + + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +// Check that restoring a rule set reports changes to the CFA rule. +TEST_F(LulDwarfCFIInsn, DW_CFA_remember_and_restore_stateCFA) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + + section.D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_def_cfa_offset) + .ULEB128(0x90481102) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start + code_factor, kCFARegister, + cfa_base_register, 0x90481102)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start + code_factor * 2, kCFARegister, + cfa_base_register, cfa_offset)) + .InSequence(s) + .WillOnce(Return(true)); + + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_nop) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_nop) + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x3fb8d4f1) + .ULEB128(0x078dc67b) + .D8(lul::DW_CFA_nop) + .FinishEntry(); + + EXPECT_CALL(handler, + ValOffsetRule(fde_start, kCFARegister, 0x3fb8d4f1, 0x078dc67b)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_GNU_window_save) { + CFISection section(kBigEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_GNU_window_save).FinishEntry(); + + // Don't include all the rules in any particular sequence. + + // The caller's %o0-%o7 have become the callee's %i0-%i7. This is + // the GCC register numbering. + for (int i = 8; i < 16; i++) + EXPECT_CALL(handler, RegisterRule(fde_start, i, i + 16)) + .WillOnce(Return(true)); + // The caller's %l0-%l7 and %i0-%i7 have been saved at the top of + // its frame. + for (int i = 16; i < 32; i++) + EXPECT_CALL(handler, OffsetRule(fde_start, i, kCFARegister, (i - 16) * 4)) + .WillOnce(Return(true)); + + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_GNU_args_size) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_GNU_args_size) + .ULEB128(0xeddfa520) + // Verify that we see this, meaning we parsed the above properly. + .D8(lul::DW_CFA_offset | 0x23) + .ULEB128(0x269) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x23, kCFARegister, 0x269 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIInsn, DW_CFA_GNU_negative_offset_extended) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_GNU_negative_offset_extended) + .ULEB128(0x430cc87a) + .ULEB128(0x613) + .FinishEntry(); + + EXPECT_CALL(handler, OffsetRule(fde_start, 0x430cc87a, kCFARegister, + -0x613 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion); +} + +// Three FDEs: skip the second +TEST_F(LulDwarfCFIInsn, SkipFDE) { + CFISection section(kBigEndian, 4); + Label cie; + section + // CIE, used by all FDEs. + .Mark(&cie) + .CIEHeader(0x010269f2, 0x9177, 0xedca5849, 2, "") + .D8(lul::DW_CFA_def_cfa) + .ULEB128(0x42ed390b) + .ULEB128(0x98f43aad) + .FinishEntry() + // First FDE. + .FDEHeader(cie, 0xa870ebdd, 0x60f6aa4) + .D8(lul::DW_CFA_register) + .ULEB128(0x3a860351) + .ULEB128(0x6c9a6bcf) + .FinishEntry() + // Second FDE. + .FDEHeader(cie, 0xc534f7c0, 0xf6552e9, true /* dwarf64 */) + .D8(lul::DW_CFA_register) + .ULEB128(0x1b62c234) + .ULEB128(0x26586b18) + .FinishEntry() + // Third FDE. + .FDEHeader(cie, 0xf681cfc8, 0x7e4594e) + .D8(lul::DW_CFA_register) + .ULEB128(0x26c53934) + .ULEB128(0x18eeb8a4) + .FinishEntry(); + + { + InSequence s; + + // Process the first FDE. + EXPECT_CALL(handler, Entry(_, 0xa870ebdd, 0x60f6aa4, 2, "", 0xedca5849)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + ValOffsetRule(0xa870ebdd, kCFARegister, 0x42ed390b, 0x98f43aad)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, RegisterRule(0xa870ebdd, 0x3a860351, 0x6c9a6bcf)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + // Skip the second FDE. + EXPECT_CALL(handler, Entry(_, 0xc534f7c0, 0xf6552e9, 2, "", 0xedca5849)) + .WillOnce(Return(false)); + + // Process the third FDE. + EXPECT_CALL(handler, Entry(_, 0xf681cfc8, 0x7e4594e, 2, "", 0xedca5849)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + ValOffsetRule(0xf681cfc8, kCFARegister, 0x42ed390b, 0x98f43aad)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, RegisterRule(0xf681cfc8, 0x26c53934, 0x18eeb8a4)) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + } + + ParseSection(§ion); +} + +// Quit processing in the middle of an entry's instructions. +TEST_F(LulDwarfCFIInsn, QuitMidentry) { + CFISection section(kLittleEndian, 8); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0xe0cf850d) + .ULEB128(0x15aab431) + .D8(lul::DW_CFA_expression) + .ULEB128(0x46750aa5) + .Block("meat") + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0xe0cf850d, 0x15aab431)) + .InSequence(s) + .WillOnce(Return(false)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseSection(§ion, false); +} + +class LulDwarfCFIRestore : public CFIInsnFixture, public Test {}; + +TEST_F(LulDwarfCFIRestore, RestoreUndefinedRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_undefined) + .ULEB128(0x0bac878e) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, UndefinedRule(fde_start, 0x0bac878e)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreUndefinedRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_undefined) + .ULEB128(0x7dedff5f) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_same_value) + .ULEB128(0x7dedff5f) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, UndefinedRule(fde_start, 0x7dedff5f)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SameValueRule(fde_start + code_factor, 0x7dedff5f)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + 2 * code_factor, 0x7dedff5f)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreSameValueRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_same_value) + .ULEB128(0xadbc9b3a) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, SameValueRule(fde_start, 0xadbc9b3a)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreSameValueRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_same_value) + .ULEB128(0x3d90dcb5) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0x3d90dcb5) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, SameValueRule(fde_start, 0x3d90dcb5)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0x3d90dcb5)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SameValueRule(fde_start + 2 * code_factor, 0x3d90dcb5)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreOffsetRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset | 0x14) + .ULEB128(0xb6f) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x14, kCFARegister, 0xb6f * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreOffsetRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset | 0x21) + .ULEB128(0xeb7) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0x21) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x21, kCFARegister, 0xeb7 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0x21)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(fde_start + 2 * code_factor, 0x21, + kCFARegister, 0xeb7 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreOffsetRuleChangedOffset) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_offset | 0x21) + .ULEB128(0x134) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_offset | 0x21) + .ULEB128(0xf4f) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, + OffsetRule(fde_start, 0x21, kCFARegister, 0x134 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(fde_start + code_factor, 0x21, kCFARegister, + 0xf4f * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, OffsetRule(fde_start + 2 * code_factor, 0x21, + kCFARegister, 0x134 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreValOffsetRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset) + .ULEB128(0x829caee6) + .ULEB128(0xe4c) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x829caee6, kCFARegister, + 0xe4c * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreValOffsetRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset) + .ULEB128(0xf17c36d6) + .ULEB128(0xeb7) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0xf17c36d6) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0xf17c36d6, kCFARegister, + 0xeb7 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0xf17c36d6)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start + 2 * code_factor, 0xf17c36d6, + kCFARegister, 0xeb7 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreValOffsetRuleChangedValOffset) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_offset) + .ULEB128(0x2cf0ab1b) + .ULEB128(0x562) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_val_offset) + .ULEB128(0x2cf0ab1b) + .ULEB128(0xe88) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValOffsetRule(fde_start, 0x2cf0ab1b, kCFARegister, + 0x562 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start + code_factor, 0x2cf0ab1b, + kCFARegister, 0xe88 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(fde_start + 2 * code_factor, 0x2cf0ab1b, + kCFARegister, 0x562 * data_factor)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreRegisterRuleUnchanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0x77514acc) + .ULEB128(0x464de4ce) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0x77514acc, 0x464de4ce)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreRegisterRuleChanged) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0xe39acce5) + .ULEB128(0x095f1559) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0xe39acce5) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0xe39acce5, 0x095f1559)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0xe39acce5)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + RegisterRule(fde_start + 2 * code_factor, 0xe39acce5, 0x095f1559)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreRegisterRuleChangedRegister) { + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_register) + .ULEB128(0xd40e21b1) + .ULEB128(0x16607d6a) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_register) + .ULEB128(0xd40e21b1) + .ULEB128(0xbabb4742) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, RegisterRule(fde_start, 0xd40e21b1, 0x16607d6a)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + RegisterRule(fde_start + code_factor, 0xd40e21b1, 0xbabb4742)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + RegisterRule(fde_start + 2 * code_factor, 0xd40e21b1, 0x16607d6a)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion); +} + +TEST_F(LulDwarfCFIRestore, RestoreExpressionRuleUnchanged) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice dwarf("dwarf"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_expression) + .ULEB128(0x666ae152) + .Block("dwarf") + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ExpressionRule(fde_start, 0x666ae152, dwarf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreExpressionRuleChanged) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice elf("elf"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_expression) + .ULEB128(0xb5ca5c46) + .Block(elf) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0xb5ca5c46) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ExpressionRule(fde_start, 0xb5ca5c46, elf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0xb5ca5c46)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + ExpressionRule(fde_start + 2 * code_factor, 0xb5ca5c46, elf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreExpressionRuleChangedExpression) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + StockCIEAndFDE(§ion); + ImageSlice smurf("smurf"); + ImageSlice orc("orc"); + section.D8(lul::DW_CFA_expression) + .ULEB128(0x500f5739) + .Block(smurf) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_expression) + .ULEB128(0x500f5739) + .Block(orc) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ExpressionRule(fde_start, 0x500f5739, smurf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ExpressionRule(fde_start + code_factor, 0x500f5739, orc)) + .InSequence(s) + .WillOnce(Return(true)); + // Expectations are not wishes. + EXPECT_CALL(handler, + ExpressionRule(fde_start + 2 * code_factor, 0x500f5739, smurf)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreValExpressionRuleUnchanged) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice hideous("hideous"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_expression) + .ULEB128(0x666ae152) + .Block(hideous) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, 0x666ae152, hideous)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreValExpressionRuleChanged) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice revolting("revolting"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_expression) + .ULEB128(0xb5ca5c46) + .Block(revolting) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_undefined) + .ULEB128(0xb5ca5c46) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("RestoreValExpressionRuleChanged", section); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, 0xb5ca5c46, revolting)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(fde_start + code_factor, 0xb5ca5c46)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValExpressionRule(fde_start + 2 * code_factor, + 0xb5ca5c46, revolting)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +TEST_F(LulDwarfCFIRestore, RestoreValExpressionRuleChangedValExpression) { + ByteReader reader(ENDIANNESS_LITTLE); + CFISection section(kLittleEndian, 4); + ImageSlice repulsive("repulsive"); + ImageSlice nauseous("nauseous"); + StockCIEAndFDE(§ion); + section.D8(lul::DW_CFA_val_expression) + .ULEB128(0x500f5739) + .Block(repulsive) + .D8(lul::DW_CFA_remember_state) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_val_expression) + .ULEB128(0x500f5739) + .Block(nauseous) + .D8(lul::DW_CFA_advance_loc | 1) + .D8(lul::DW_CFA_restore_state) + .FinishEntry(); + + PERHAPS_WRITE_DEBUG_FRAME_FILE("RestoreValExpressionRuleChangedValExpression", + section); + + EXPECT_CALL(handler, ValExpressionRule(fde_start, 0x500f5739, repulsive)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, + ValExpressionRule(fde_start + code_factor, 0x500f5739, nauseous)) + .InSequence(s) + .WillOnce(Return(true)); + // Expectations are not wishes. + EXPECT_CALL(handler, ValExpressionRule(fde_start + 2 * code_factor, + 0x500f5739, repulsive)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).WillOnce(Return(true)); + + ParseSection(§ion, true, &reader); +} + +struct EHFrameFixture : public CFIInsnFixture { + EHFrameFixture() : CFIInsnFixture(), section(kBigEndian, 4, true) { + encoded_pointer_bases.cfi = 0x7f496cb2; + encoded_pointer_bases.text = 0x540f67b6; + encoded_pointer_bases.data = 0xe3eab768; + section.SetEncodedPointerBases(encoded_pointer_bases); + } + CFISection section; + CFISection::EncodedPointerBases encoded_pointer_bases; + + // Parse CFIInsnFixture::ParseSection, but parse the section as + // .eh_frame data, supplying stock base addresses. + void ParseEHFrameSection(CFISection* section, bool succeeds = true) { + EXPECT_TRUE(section->ContainsEHFrame()); + string contents; + EXPECT_TRUE(section->GetContents(&contents)); + lul::Endianness endianness; + if (section->endianness() == kBigEndian) + endianness = ENDIANNESS_BIG; + else { + assert(section->endianness() == kLittleEndian); + endianness = ENDIANNESS_LITTLE; + } + ByteReader reader(endianness); + reader.SetAddressSize(section->AddressSize()); + reader.SetCFIDataBase(encoded_pointer_bases.cfi, contents.data()); + reader.SetTextBase(encoded_pointer_bases.text); + reader.SetDataBase(encoded_pointer_bases.data); + CallFrameInfo parser(contents.data(), contents.size(), &reader, &handler, + &reporter, true); + if (succeeds) + EXPECT_TRUE(parser.Start()); + else + EXPECT_FALSE(parser.Start()); + } +}; + +class LulDwarfEHFrame : public EHFrameFixture, public Test {}; + +// A simple CIE, an FDE, and a terminator. +TEST_F(LulDwarfEHFrame, Terminator) { + Label cie; + section.Mark(&cie) + .CIEHeader(9968, 2466, 67, 1, "") + .D8(lul::DW_CFA_def_cfa) + .ULEB128(3772) + .ULEB128(1372) + .FinishEntry() + .FDEHeader(cie, 0x848037a1, 0x7b30475e) + .D8(lul::DW_CFA_set_loc) + .D32(0x17713850) + .D8(lul::DW_CFA_undefined) + .ULEB128(5721) + .FinishEntry() + .D32(0) // Terminate the sequence. + // This FDE should be ignored. + .FDEHeader(cie, 0xf19629fe, 0x439fb09b) + .FinishEntry(); + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.Terminator", section); + + EXPECT_CALL(handler, Entry(_, 0x848037a1, 0x7b30475e, 1, "", 67)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(0x848037a1, kCFARegister, 3772, 1372)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(0x17713850, 5721)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + EXPECT_CALL(reporter, EarlyEHTerminator(_)).InSequence(s).WillOnce(Return()); + + ParseEHFrameSection(§ion); +} + +// The parser should recognize the Linux Standards Base 'z' augmentations. +TEST_F(LulDwarfEHFrame, SimpleFDE) { + lul::DwarfPointerEncoding lsda_encoding = lul::DwarfPointerEncoding( + lul::DW_EH_PE_indirect | lul::DW_EH_PE_datarel | lul::DW_EH_PE_sdata2); + lul::DwarfPointerEncoding fde_encoding = + lul::DwarfPointerEncoding(lul::DW_EH_PE_textrel | lul::DW_EH_PE_udata2); + + section.SetPointerEncoding(fde_encoding); + section.SetEncodedPointerBases(encoded_pointer_bases); + Label cie; + section.Mark(&cie) + .CIEHeader(4873, 7012, 100, 1, "zSLPR") + .ULEB128(7) // Augmentation data length + .D8(lsda_encoding) // LSDA pointer format + .D8(lul::DW_EH_PE_pcrel) // personality pointer format + .EncodedPointer(0x97baa00, lul::DW_EH_PE_pcrel) // and value + .D8(fde_encoding) // FDE pointer format + .D8(lul::DW_CFA_def_cfa) + .ULEB128(6706) + .ULEB128(31) + .FinishEntry() + .FDEHeader(cie, 0x540f6b56, 0xf686) + .ULEB128(2) // Augmentation data length + .EncodedPointer(0xe3eab475, lsda_encoding) // LSDA pointer, signed + .D8(lul::DW_CFA_set_loc) + .EncodedPointer(0x540fa4ce, fde_encoding) + .D8(lul::DW_CFA_undefined) + .ULEB128(0x675e) + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.SimpleFDE", section); + + EXPECT_CALL(handler, Entry(_, 0x540f6b56, 0xf686, 1, "zSLPR", 100)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, PersonalityRoutine(0x97baa00, false)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, LanguageSpecificDataArea(0xe3eab475, true)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SignalHandler()).InSequence(s).WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(0x540f6b56, kCFARegister, 6706, 31)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(0x540fa4ce, 0x675e)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +// Check that we can handle an empty 'z' augmentation. +TEST_F(LulDwarfEHFrame, EmptyZ) { + Label cie; + section.Mark(&cie) + .CIEHeader(5955, 5805, 228, 1, "z") + .ULEB128(0) // Augmentation data length + .D8(lul::DW_CFA_def_cfa) + .ULEB128(3629) + .ULEB128(247) + .FinishEntry() + .FDEHeader(cie, 0xda007738, 0xfb55c641) + .ULEB128(0) // Augmentation data length + .D8(lul::DW_CFA_advance_loc1) + .D8(11) + .D8(lul::DW_CFA_undefined) + .ULEB128(3769) + .FinishEntry(); + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.EmptyZ", section); + + EXPECT_CALL(handler, Entry(_, 0xda007738, 0xfb55c641, 1, "z", 228)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, ValOffsetRule(0xda007738, kCFARegister, 3629, 247)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, UndefinedRule(0xda007738 + 11 * 5955, 3769)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +// Check that we recognize bad 'z' augmentation characters. +TEST_F(LulDwarfEHFrame, BadZ) { + Label cie; + section.Mark(&cie) + .CIEHeader(6937, 1045, 142, 1, "zQ") + .ULEB128(0) // Augmentation data length + .D8(lul::DW_CFA_def_cfa) + .ULEB128(9006) + .ULEB128(7725) + .FinishEntry() + .FDEHeader(cie, 0x1293efa8, 0x236f53f2) + .ULEB128(0) // Augmentation data length + .D8(lul::DW_CFA_advance_loc | 12) + .D8(lul::DW_CFA_register) + .ULEB128(5667) + .ULEB128(3462) + .FinishEntry(); + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.BadZ", section); + + EXPECT_CALL(reporter, UnrecognizedAugmentation(_, "zQ")).WillOnce(Return()); + + ParseEHFrameSection(§ion, false); +} + +TEST_F(LulDwarfEHFrame, zL) { + Label cie; + lul::DwarfPointerEncoding lsda_encoding = + lul::DwarfPointerEncoding(lul::DW_EH_PE_funcrel | lul::DW_EH_PE_udata2); + section.Mark(&cie) + .CIEHeader(9285, 9959, 54, 1, "zL") + .ULEB128(1) // Augmentation data length + .D8(lsda_encoding) // encoding for LSDA pointer in FDE + + .FinishEntry() + .FDEHeader(cie, 0xd40091aa, 0x9aa6e746) + .ULEB128(2) // Augmentation data length + .EncodedPointer(0xd40099cd, lsda_encoding) // LSDA pointer + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.zL", section); + + EXPECT_CALL(handler, Entry(_, 0xd40091aa, 0x9aa6e746, 1, "zL", 54)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, LanguageSpecificDataArea(0xd40099cd, false)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +TEST_F(LulDwarfEHFrame, zP) { + Label cie; + lul::DwarfPointerEncoding personality_encoding = + lul::DwarfPointerEncoding(lul::DW_EH_PE_datarel | lul::DW_EH_PE_udata2); + section.Mark(&cie) + .CIEHeader(1097, 6313, 17, 1, "zP") + .ULEB128(3) // Augmentation data length + .D8(personality_encoding) // encoding for personality routine + .EncodedPointer(0xe3eaccac, personality_encoding) // value + .FinishEntry() + .FDEHeader(cie, 0x0c8350c9, 0xbef11087) + .ULEB128(0) // Augmentation data length + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.zP", section); + + EXPECT_CALL(handler, Entry(_, 0x0c8350c9, 0xbef11087, 1, "zP", 17)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, PersonalityRoutine(0xe3eaccac, false)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +TEST_F(LulDwarfEHFrame, zR) { + Label cie; + lul::DwarfPointerEncoding pointer_encoding = + lul::DwarfPointerEncoding(lul::DW_EH_PE_textrel | lul::DW_EH_PE_sdata2); + section.SetPointerEncoding(pointer_encoding); + section.Mark(&cie) + .CIEHeader(8011, 5496, 75, 1, "zR") + .ULEB128(1) // Augmentation data length + .D8(pointer_encoding) // encoding for FDE addresses + .FinishEntry() + .FDEHeader(cie, 0x540f9431, 0xbd0) + .ULEB128(0) // Augmentation data length + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.zR", section); + + EXPECT_CALL(handler, Entry(_, 0x540f9431, 0xbd0, 1, "zR", 75)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +TEST_F(LulDwarfEHFrame, zS) { + Label cie; + section.Mark(&cie) + .CIEHeader(9217, 7694, 57, 1, "zS") + .ULEB128(0) // Augmentation data length + .FinishEntry() + .FDEHeader(cie, 0xd40091aa, 0x9aa6e746) + .ULEB128(0) // Augmentation data length + .FinishEntry() + .D32(0); // terminator + + PERHAPS_WRITE_EH_FRAME_FILE("EHFrame.zS", section); + + EXPECT_CALL(handler, Entry(_, 0xd40091aa, 0x9aa6e746, 1, "zS", 57)) + .InSequence(s) + .WillOnce(Return(true)); + EXPECT_CALL(handler, SignalHandler()).InSequence(s).WillOnce(Return(true)); + EXPECT_CALL(handler, End()).InSequence(s).WillOnce(Return(true)); + + ParseEHFrameSection(§ion); +} + +// These tests require manual inspection of the test output. +struct CFIReporterFixture { + CFIReporterFixture() + : reporter(gtest_logging_sink_for_LulTestDwarf, "test file name", + "test section name") {} + CallFrameInfo::Reporter reporter; +}; + +class LulDwarfCFIReporter : public CFIReporterFixture, public Test {}; + +TEST_F(LulDwarfCFIReporter, Incomplete) { + reporter.Incomplete(0x0102030405060708ULL, CallFrameInfo::kUnknown); +} + +TEST_F(LulDwarfCFIReporter, EarlyEHTerminator) { + reporter.EarlyEHTerminator(0x0102030405060708ULL); +} + +TEST_F(LulDwarfCFIReporter, CIEPointerOutOfRange) { + reporter.CIEPointerOutOfRange(0x0123456789abcdefULL, 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, BadCIEId) { + reporter.BadCIEId(0x0123456789abcdefULL, 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, UnrecognizedVersion) { + reporter.UnrecognizedVersion(0x0123456789abcdefULL, 43); +} + +TEST_F(LulDwarfCFIReporter, UnrecognizedAugmentation) { + reporter.UnrecognizedAugmentation(0x0123456789abcdefULL, "poodles"); +} + +TEST_F(LulDwarfCFIReporter, InvalidPointerEncoding) { + reporter.InvalidPointerEncoding(0x0123456789abcdefULL, 0x42); +} + +TEST_F(LulDwarfCFIReporter, UnusablePointerEncoding) { + reporter.UnusablePointerEncoding(0x0123456789abcdefULL, 0x42); +} + +TEST_F(LulDwarfCFIReporter, RestoreInCIE) { + reporter.RestoreInCIE(0x0123456789abcdefULL, 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, BadInstruction) { + reporter.BadInstruction(0x0123456789abcdefULL, CallFrameInfo::kFDE, + 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, NoCFARule) { + reporter.NoCFARule(0x0123456789abcdefULL, CallFrameInfo::kCIE, + 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, EmptyStateStack) { + reporter.EmptyStateStack(0x0123456789abcdefULL, CallFrameInfo::kTerminator, + 0xfedcba9876543210ULL); +} + +TEST_F(LulDwarfCFIReporter, ClearingCFARule) { + reporter.ClearingCFARule(0x0123456789abcdefULL, CallFrameInfo::kFDE, + 0xfedcba9876543210ULL); +} +class LulDwarfExpr : public Test {}; + +class MockSummariser : public Summariser { + public: + MockSummariser() : Summariser(nullptr, 0, nullptr) {} + MOCK_METHOD2(Entry, void(uintptr_t, uintptr_t)); + MOCK_METHOD0(End, void()); + MOCK_METHOD5(Rule, void(uintptr_t, int, LExprHow, int16_t, int64_t)); + MOCK_METHOD1(AddPfxInstr, uint32_t(PfxInstr)); +}; + +TEST_F(LulDwarfExpr, SimpleTransliteration) { + MockSummariser summ; + ByteReader reader(ENDIANNESS_LITTLE); + + CFISection section(kLittleEndian, 8); + section.D8(DW_OP_lit0) + .D8(DW_OP_lit31) + .D8(DW_OP_breg0 + 17) + .LEB128(-1234) + .D8(DW_OP_const4s) + .D32(0xFEDC9876) + .D8(DW_OP_deref) + .D8(DW_OP_and) + .D8(DW_OP_plus) + .D8(DW_OP_minus) + .D8(DW_OP_shl) + .D8(DW_OP_ge); + string expr; + bool ok = section.GetContents(&expr); + EXPECT_TRUE(ok); + + { + InSequence s; + // required start marker + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Start, 0))); + // DW_OP_lit0 + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_SImm32, 0))); + // DW_OP_lit31 + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_SImm32, 31))); + // DW_OP_breg17 -1234 + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_DwReg, 17))); + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_SImm32, -1234))); + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Add))); + // DW_OP_const4s 0xFEDC9876 + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_SImm32, 0xFEDC9876))); + // DW_OP_deref + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Deref))); + // DW_OP_and + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_And))); + // DW_OP_plus + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Add))); + // DW_OP_minus + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Sub))); + // DW_OP_shl + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Shl))); + // DW_OP_ge + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_CmpGES))); + // required end marker + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_End))); + } + + int32_t ix = + parseDwarfExpr(&summ, &reader, ImageSlice(expr), false, false, false); + EXPECT_TRUE(ix >= 0); +} + +TEST_F(LulDwarfExpr, UnknownOpcode) { + MockSummariser summ; + ByteReader reader(ENDIANNESS_LITTLE); + + CFISection section(kLittleEndian, 8); + section.D8(DW_OP_lo_user - 1); + string expr; + bool ok = section.GetContents(&expr); + EXPECT_TRUE(ok); + + { + InSequence s; + // required start marker + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Start, 0))); + } + + int32_t ix = + parseDwarfExpr(&summ, &reader, ImageSlice(expr), false, false, false); + EXPECT_TRUE(ix == -1); +} + +TEST_F(LulDwarfExpr, ExpressionOverrun) { + MockSummariser summ; + ByteReader reader(ENDIANNESS_LITTLE); + + CFISection section(kLittleEndian, 8); + section.D8(DW_OP_const4s).D8(0x12).D8(0x34).D8(0x56); + string expr; + bool ok = section.GetContents(&expr); + EXPECT_TRUE(ok); + + { + InSequence s; + // required start marker + EXPECT_CALL(summ, AddPfxInstr(PfxInstr(PX_Start, 0))); + // DW_OP_const4s followed by 3 (a.k.a. not enough) bytes + // We expect PfxInstr(PX_Simm32, not-known-for-sure-32-bit-immediate) + // Hence must use _ as the argument. + EXPECT_CALL(summ, AddPfxInstr(_)); + } + + int32_t ix = + parseDwarfExpr(&summ, &reader, ImageSlice(expr), false, false, false); + EXPECT_TRUE(ix == -1); +} + +// We'll need to mention specific Dwarf registers in the EvaluatePfxExpr tests, +// and those names are arch-specific, so a bit of macro magic is helpful. +#if defined(GP_ARCH_arm) +# define TESTED_REG_STRUCT_NAME r11 +# define TESTED_REG_DWARF_NAME DW_REG_ARM_R11 +#elif defined(GP_ARCH_arm64) +# define TESTED_REG_STRUCT_NAME x29 +# define TESTED_REG_DWARF_NAME DW_REG_AARCH64_X29 +#elif defined(GP_ARCH_amd64) || defined(GP_ARCH_x86) +# define TESTED_REG_STRUCT_NAME xbp +# define TESTED_REG_DWARF_NAME DW_REG_INTEL_XBP +#else +# error "Unknown plat" +#endif + +struct EvaluatePfxExprFixture { + // Creates: + // initial stack, AVMA 0x12345678, at offset 4 bytes = 0xdeadbeef + // initial regs, with XBP = 0x14141356 + // initial CFA = 0x5432ABCD + EvaluatePfxExprFixture() { + // The test stack. + si.mStartAvma = 0x12345678; + si.mLen = 0; +#define XX(_byte) \ + do { \ + si.mContents[si.mLen++] = (_byte); \ + } while (0) + XX(0x55); + XX(0x55); + XX(0x55); + XX(0x55); + if (sizeof(void*) == 8) { + // le64 + XX(0xEF); + XX(0xBE); + XX(0xAD); + XX(0xDE); + XX(0); + XX(0); + XX(0); + XX(0); + } else { + // le32 + XX(0xEF); + XX(0xBE); + XX(0xAD); + XX(0xDE); + } + XX(0xAA); + XX(0xAA); + XX(0xAA); + XX(0xAA); +#undef XX + // The initial CFA. + initialCFA = TaggedUWord(0x5432ABCD); + // The initial register state. + memset(®s, 0, sizeof(regs)); + regs.TESTED_REG_STRUCT_NAME = TaggedUWord(0x14141356); + } + + StackImage si; + TaggedUWord initialCFA; + UnwindRegs regs; +}; + +class LulDwarfEvaluatePfxExpr : public EvaluatePfxExprFixture, public Test {}; + +TEST_F(LulDwarfEvaluatePfxExpr, NormalEvaluation) { + vector instrs; + // Put some junk at the start of the insn sequence. + instrs.push_back(PfxInstr(PX_End)); + instrs.push_back(PfxInstr(PX_End)); + + // Now the real sequence + // stack is empty + instrs.push_back(PfxInstr(PX_Start, 1)); + // 0x5432ABCD + instrs.push_back(PfxInstr(PX_SImm32, 0x31415927)); + // 0x5432ABCD 0x31415927 + instrs.push_back(PfxInstr(PX_DwReg, TESTED_REG_DWARF_NAME)); + // 0x5432ABCD 0x31415927 0x14141356 + instrs.push_back(PfxInstr(PX_SImm32, 42)); + // 0x5432ABCD 0x31415927 0x14141356 42 + instrs.push_back(PfxInstr(PX_Sub)); + // 0x5432ABCD 0x31415927 0x1414132c + instrs.push_back(PfxInstr(PX_Add)); + // 0x5432ABCD 0x45556c53 + instrs.push_back(PfxInstr(PX_SImm32, si.mStartAvma + 4)); + // 0x5432ABCD 0x45556c53 0x1234567c + instrs.push_back(PfxInstr(PX_Deref)); + // 0x5432ABCD 0x45556c53 0xdeadbeef + instrs.push_back(PfxInstr(PX_SImm32, 0xFE01DC23)); + // 0x5432ABCD 0x45556c53 0xdeadbeef 0xFE01DC23 + instrs.push_back(PfxInstr(PX_And)); + // 0x5432ABCD 0x45556c53 0xde019c23 + instrs.push_back(PfxInstr(PX_SImm32, 7)); + // 0x5432ABCD 0x45556c53 0xde019c23 7 + instrs.push_back(PfxInstr(PX_Shl)); + // 0x5432ABCD 0x45556c53 0x6f00ce1180 + instrs.push_back(PfxInstr(PX_SImm32, 0x7fffffff)); + // 0x5432ABCD 0x45556c53 0x6f00ce1180 7fffffff + instrs.push_back(PfxInstr(PX_And)); + // 0x5432ABCD 0x45556c53 0x00ce1180 + instrs.push_back(PfxInstr(PX_Add)); + // 0x5432ABCD 0x46237dd3 + instrs.push_back(PfxInstr(PX_Sub)); + // 0xe0f2dfa + + instrs.push_back(PfxInstr(PX_End)); + + TaggedUWord res = EvaluatePfxExpr(2 /*offset of start insn*/, ®s, + initialCFA, &si, instrs); + EXPECT_TRUE(res.Valid()); + EXPECT_TRUE(res.Value() == 0xe0f2dfa); +} + +TEST_F(LulDwarfEvaluatePfxExpr, EmptySequence) { + vector instrs; + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, BogusStartPoint) { + vector instrs; + instrs.push_back(PfxInstr(PX_SImm32, 42)); + instrs.push_back(PfxInstr(PX_SImm32, 24)); + instrs.push_back(PfxInstr(PX_SImm32, 4224)); + TaggedUWord res = EvaluatePfxExpr(1, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, MissingEndMarker) { + vector instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + instrs.push_back(PfxInstr(PX_SImm32, 24)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, StackUnderflow) { + vector instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, StackNoUnderflow) { + vector instrs; + instrs.push_back(PfxInstr(PX_Start, 1 /*push the initial CFA*/)); + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_TRUE(res.Valid()); + EXPECT_TRUE(res == initialCFA); +} + +TEST_F(LulDwarfEvaluatePfxExpr, StackOverflow) { + vector instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + for (int i = 0; i < 10 + 1; i++) { + instrs.push_back(PfxInstr(PX_SImm32, i + 100)); + } + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_FALSE(res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, StackNoOverflow) { + vector instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + for (int i = 0; i < 10 + 0; i++) { + instrs.push_back(PfxInstr(PX_SImm32, i + 100)); + } + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_TRUE(res.Valid()); + EXPECT_TRUE(res == TaggedUWord(109)); +} + +TEST_F(LulDwarfEvaluatePfxExpr, OutOfRangeShl) { + vector instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + instrs.push_back(PfxInstr(PX_SImm32, 1234)); + instrs.push_back(PfxInstr(PX_SImm32, 5678)); + instrs.push_back(PfxInstr(PX_Shl)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_TRUE(!res.Valid()); +} + +TEST_F(LulDwarfEvaluatePfxExpr, TestCmpGES) { + const int32_t argsL[6] = {0, 0, 1, -2, -1, -2}; + const int32_t argsR[6] = {0, 1, 0, -2, -2, -1}; + // expecting: t f t t t f = 101110 = 0x2E + vector instrs; + instrs.push_back(PfxInstr(PX_Start, 0)); + // The "running total" + instrs.push_back(PfxInstr(PX_SImm32, 0)); + for (unsigned int i = 0; i < sizeof(argsL) / sizeof(argsL[0]); i++) { + // Shift the "running total" at the bottom of the stack left by one bit + instrs.push_back(PfxInstr(PX_SImm32, 1)); + instrs.push_back(PfxInstr(PX_Shl)); + // Push both test args and do the comparison + instrs.push_back(PfxInstr(PX_SImm32, argsL[i])); + instrs.push_back(PfxInstr(PX_SImm32, argsR[i])); + instrs.push_back(PfxInstr(PX_CmpGES)); + // Or the result into the running total + instrs.push_back(PfxInstr(PX_Or)); + } + instrs.push_back(PfxInstr(PX_End)); + TaggedUWord res = EvaluatePfxExpr(0, ®s, initialCFA, &si, instrs); + EXPECT_TRUE(res.Valid()); + EXPECT_TRUE(res == TaggedUWord(0x2E)); +} + +} // namespace lul diff --git a/tools/profiler/tests/gtest/LulTestInfrastructure.cpp b/tools/profiler/tests/gtest/LulTestInfrastructure.cpp new file mode 100644 index 0000000000..6d49557e9c --- /dev/null +++ b/tools/profiler/tests/gtest/LulTestInfrastructure.cpp @@ -0,0 +1,498 @@ +// Copyright (c) 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Original author: Jim Blandy + +// Derived from: +// test_assembler.cc: Implementation of google_breakpad::TestAssembler. +// See test_assembler.h for details. + +// Derived from: +// cfi_assembler.cc: Implementation of google_breakpad::CFISection class. +// See cfi_assembler.h for details. + +#include "LulTestInfrastructure.h" + +#include "LulDwarfInt.h" + +#include + +namespace lul_test { +namespace test_assembler { + +using std::back_insert_iterator; + +Label::Label() : value_(new Binding()) {} +Label::Label(uint64_t value) : value_(new Binding(value)) {} +Label::Label(const Label& label) { + value_ = label.value_; + value_->Acquire(); +} +Label::~Label() { + if (value_->Release()) delete value_; +} + +Label& Label::operator=(uint64_t value) { + value_->Set(NULL, value); + return *this; +} + +Label& Label::operator=(const Label& label) { + value_->Set(label.value_, 0); + return *this; +} + +Label Label::operator+(uint64_t addend) const { + Label l; + l.value_->Set(this->value_, addend); + return l; +} + +Label Label::operator-(uint64_t subtrahend) const { + Label l; + l.value_->Set(this->value_, -subtrahend); + return l; +} + +// When NDEBUG is #defined, assert doesn't evaluate its argument. This +// means you can't simply use assert to check the return value of a +// function with necessary side effects. +// +// ALWAYS_EVALUATE_AND_ASSERT(x) evaluates x regardless of whether +// NDEBUG is #defined; when NDEBUG is not #defined, it further asserts +// that x is true. +#ifdef NDEBUG +# define ALWAYS_EVALUATE_AND_ASSERT(x) x +#else +# define ALWAYS_EVALUATE_AND_ASSERT(x) assert(x) +#endif + +uint64_t Label::operator-(const Label& label) const { + uint64_t offset; + ALWAYS_EVALUATE_AND_ASSERT(IsKnownOffsetFrom(label, &offset)); + return offset; +} + +bool Label::IsKnownConstant(uint64_t* value_p) const { + Binding* base; + uint64_t addend; + value_->Get(&base, &addend); + if (base != NULL) return false; + if (value_p) *value_p = addend; + return true; +} + +bool Label::IsKnownOffsetFrom(const Label& label, uint64_t* offset_p) const { + Binding *label_base, *this_base; + uint64_t label_addend, this_addend; + label.value_->Get(&label_base, &label_addend); + value_->Get(&this_base, &this_addend); + // If this and label are related, Get will find their final + // common ancestor, regardless of how indirect the relation is. This + // comparison also handles the constant vs. constant case. + if (this_base != label_base) return false; + if (offset_p) *offset_p = this_addend - label_addend; + return true; +} + +Label::Binding::Binding() : base_(this), addend_(), reference_count_(1) {} + +Label::Binding::Binding(uint64_t addend) + : base_(NULL), addend_(addend), reference_count_(1) {} + +Label::Binding::~Binding() { + assert(reference_count_ == 0); + if (base_ && base_ != this && base_->Release()) delete base_; +} + +void Label::Binding::Set(Binding* binding, uint64_t addend) { + if (!base_ && !binding) { + // We're equating two constants. This could be okay. + assert(addend_ == addend); + } else if (!base_) { + // We are a known constant, but BINDING may not be, so turn the + // tables and try to set BINDING's value instead. + binding->Set(NULL, addend_ - addend); + } else { + if (binding) { + // Find binding's final value. Since the final value is always either + // completely unconstrained or a constant, never a reference to + // another variable (otherwise, it wouldn't be final), this + // guarantees we won't create cycles here, even for code like this: + // l = m, m = n, n = l; + uint64_t binding_addend; + binding->Get(&binding, &binding_addend); + addend += binding_addend; + } + + // It seems likely that setting a binding to itself is a bug + // (although I can imagine this might turn out to be helpful to + // permit). + assert(binding != this); + + if (base_ != this) { + // Set the other bindings on our chain as well. Note that this + // is sufficient even though binding relationships form trees: + // All binding operations traverse their chains to the end, and + // all bindings related to us share some tail of our chain, so + // they will see the changes we make here. + base_->Set(binding, addend - addend_); + // We're not going to use base_ any more. + if (base_->Release()) delete base_; + } + + // Adopt BINDING as our base. Note that it should be correct to + // acquire here, after the release above, even though the usual + // reference-counting rules call for acquiring first, and then + // releasing: the self-reference assertion above should have + // complained if BINDING were 'this' or anywhere along our chain, + // so we didn't release BINDING. + if (binding) binding->Acquire(); + base_ = binding; + addend_ = addend; + } +} + +void Label::Binding::Get(Binding** base, uint64_t* addend) { + if (base_ && base_ != this) { + // Recurse to find the end of our reference chain (the root of our + // tree), and then rewrite every binding along the chain to refer + // to it directly, adjusting addends appropriately. (This is why + // this member function isn't this-const.) + Binding* final_base; + uint64_t final_addend; + base_->Get(&final_base, &final_addend); + if (final_base) final_base->Acquire(); + if (base_->Release()) delete base_; + base_ = final_base; + addend_ += final_addend; + } + *base = base_; + *addend = addend_; +} + +template +static inline void InsertEndian(test_assembler::Endianness endianness, + size_t size, uint64_t number, Inserter dest) { + assert(size > 0); + if (endianness == kLittleEndian) { + for (size_t i = 0; i < size; i++) { + *dest++ = (char)(number & 0xff); + number >>= 8; + } + } else { + assert(endianness == kBigEndian); + // The loop condition is odd, but it's correct for size_t. + for (size_t i = size - 1; i < size; i--) + *dest++ = (char)((number >> (i * 8)) & 0xff); + } +} + +Section& Section::Append(Endianness endianness, size_t size, uint64_t number) { + InsertEndian(endianness, size, number, + back_insert_iterator(contents_)); + return *this; +} + +Section& Section::Append(Endianness endianness, size_t size, + const Label& label) { + // If this label's value is known, there's no reason to waste an + // entry in references_ on it. + uint64_t value; + if (label.IsKnownConstant(&value)) return Append(endianness, size, value); + + // This will get caught when the references are resolved, but it's + // nicer to find out earlier. + assert(endianness != kUnsetEndian); + + references_.push_back(Reference(contents_.size(), endianness, size, label)); + contents_.append(size, 0); + return *this; +} + +#define ENDIANNESS_L kLittleEndian +#define ENDIANNESS_B kBigEndian +#define ENDIANNESS(e) ENDIANNESS_##e + +#define DEFINE_SHORT_APPEND_NUMBER_ENDIAN(e, bits) \ + Section& Section::e##bits(uint##bits##_t v) { \ + InsertEndian(ENDIANNESS(e), bits / 8, v, \ + back_insert_iterator(contents_)); \ + return *this; \ + } + +#define DEFINE_SHORT_APPEND_LABEL_ENDIAN(e, bits) \ + Section& Section::e##bits(const Label& v) { \ + return Append(ENDIANNESS(e), bits / 8, v); \ + } + +// Define L16, B32, and friends. +#define DEFINE_SHORT_APPEND_ENDIAN(e, bits) \ + DEFINE_SHORT_APPEND_NUMBER_ENDIAN(e, bits) \ + DEFINE_SHORT_APPEND_LABEL_ENDIAN(e, bits) + +DEFINE_SHORT_APPEND_LABEL_ENDIAN(L, 8); +DEFINE_SHORT_APPEND_LABEL_ENDIAN(B, 8); +DEFINE_SHORT_APPEND_ENDIAN(L, 16); +DEFINE_SHORT_APPEND_ENDIAN(L, 32); +DEFINE_SHORT_APPEND_ENDIAN(L, 64); +DEFINE_SHORT_APPEND_ENDIAN(B, 16); +DEFINE_SHORT_APPEND_ENDIAN(B, 32); +DEFINE_SHORT_APPEND_ENDIAN(B, 64); + +#define DEFINE_SHORT_APPEND_NUMBER_DEFAULT(bits) \ + Section& Section::D##bits(uint##bits##_t v) { \ + InsertEndian(endianness_, bits / 8, v, \ + back_insert_iterator(contents_)); \ + return *this; \ + } +#define DEFINE_SHORT_APPEND_LABEL_DEFAULT(bits) \ + Section& Section::D##bits(const Label& v) { \ + return Append(endianness_, bits / 8, v); \ + } +#define DEFINE_SHORT_APPEND_DEFAULT(bits) \ + DEFINE_SHORT_APPEND_NUMBER_DEFAULT(bits) \ + DEFINE_SHORT_APPEND_LABEL_DEFAULT(bits) + +DEFINE_SHORT_APPEND_LABEL_DEFAULT(8) +DEFINE_SHORT_APPEND_DEFAULT(16); +DEFINE_SHORT_APPEND_DEFAULT(32); +DEFINE_SHORT_APPEND_DEFAULT(64); + +Section& Section::LEB128(long long value) { + while (value < -0x40 || 0x3f < value) { + contents_ += (value & 0x7f) | 0x80; + if (value < 0) + value = (value >> 7) | ~(((unsigned long long)-1) >> 7); + else + value = (value >> 7); + } + contents_ += value & 0x7f; + return *this; +} + +Section& Section::ULEB128(uint64_t value) { + while (value > 0x7f) { + contents_ += (value & 0x7f) | 0x80; + value = (value >> 7); + } + contents_ += value; + return *this; +} + +Section& Section::Align(size_t alignment, uint8_t pad_byte) { + // ALIGNMENT must be a power of two. + assert(((alignment - 1) & alignment) == 0); + size_t new_size = (contents_.size() + alignment - 1) & ~(alignment - 1); + contents_.append(new_size - contents_.size(), pad_byte); + assert((contents_.size() & (alignment - 1)) == 0); + return *this; +} + +bool Section::GetContents(string* contents) { + // For each label reference, find the label's value, and patch it into + // the section's contents. + for (size_t i = 0; i < references_.size(); i++) { + Reference& r = references_[i]; + uint64_t value; + if (!r.label.IsKnownConstant(&value)) { + fprintf(stderr, "Undefined label #%zu at offset 0x%zx\n", i, r.offset); + return false; + } + assert(r.offset < contents_.size()); + assert(contents_.size() - r.offset >= r.size); + InsertEndian(r.endianness, r.size, value, contents_.begin() + r.offset); + } + contents->clear(); + std::swap(contents_, *contents); + references_.clear(); + return true; +} + +} // namespace test_assembler +} // namespace lul_test + +namespace lul_test { + +CFISection& CFISection::CIEHeader(uint64_t code_alignment_factor, + int data_alignment_factor, + unsigned return_address_register, + uint8_t version, const string& augmentation, + bool dwarf64) { + assert(!entry_length_); + entry_length_ = new PendingLength(); + in_fde_ = false; + + if (dwarf64) { + D32(kDwarf64InitialLengthMarker); + D64(entry_length_->length); + entry_length_->start = Here(); + D64(eh_frame_ ? kEHFrame64CIEIdentifier : kDwarf64CIEIdentifier); + } else { + D32(entry_length_->length); + entry_length_->start = Here(); + D32(eh_frame_ ? kEHFrame32CIEIdentifier : kDwarf32CIEIdentifier); + } + D8(version); + AppendCString(augmentation); + ULEB128(code_alignment_factor); + LEB128(data_alignment_factor); + if (version == 1) + D8(return_address_register); + else + ULEB128(return_address_register); + return *this; +} + +CFISection& CFISection::FDEHeader(Label cie_pointer, uint64_t initial_location, + uint64_t address_range, bool dwarf64) { + assert(!entry_length_); + entry_length_ = new PendingLength(); + in_fde_ = true; + fde_start_address_ = initial_location; + + if (dwarf64) { + D32(0xffffffff); + D64(entry_length_->length); + entry_length_->start = Here(); + if (eh_frame_) + D64(Here() - cie_pointer); + else + D64(cie_pointer); + } else { + D32(entry_length_->length); + entry_length_->start = Here(); + if (eh_frame_) + D32(Here() - cie_pointer); + else + D32(cie_pointer); + } + EncodedPointer(initial_location); + // The FDE length in an .eh_frame section uses the same encoding as the + // initial location, but ignores the base address (selected by the upper + // nybble of the encoding), as it's a length, not an address that can be + // made relative. + EncodedPointer(address_range, DwarfPointerEncoding(pointer_encoding_ & 0x0f)); + return *this; +} + +CFISection& CFISection::FinishEntry() { + assert(entry_length_); + Align(address_size_, lul::DW_CFA_nop); + entry_length_->length = Here() - entry_length_->start; + delete entry_length_; + entry_length_ = NULL; + in_fde_ = false; + return *this; +} + +CFISection& CFISection::EncodedPointer(uint64_t address, + DwarfPointerEncoding encoding, + const EncodedPointerBases& bases) { + // Omitted data is extremely easy to emit. + if (encoding == lul::DW_EH_PE_omit) return *this; + + // If (encoding & lul::DW_EH_PE_indirect) != 0, then we assume + // that ADDRESS is the address at which the pointer is stored --- in + // other words, that bit has no effect on how we write the pointer. + encoding = DwarfPointerEncoding(encoding & ~lul::DW_EH_PE_indirect); + + // Find the base address to which this pointer is relative. The upper + // nybble of the encoding specifies this. + uint64_t base; + switch (encoding & 0xf0) { + case lul::DW_EH_PE_absptr: + base = 0; + break; + case lul::DW_EH_PE_pcrel: + base = bases.cfi + Size(); + break; + case lul::DW_EH_PE_textrel: + base = bases.text; + break; + case lul::DW_EH_PE_datarel: + base = bases.data; + break; + case lul::DW_EH_PE_funcrel: + base = fde_start_address_; + break; + case lul::DW_EH_PE_aligned: + base = 0; + break; + default: + abort(); + }; + + // Make ADDRESS relative. Yes, this is appropriate even for "absptr" + // values; see gcc/unwind-pe.h. + address -= base; + + // Align the pointer, if required. + if ((encoding & 0xf0) == lul::DW_EH_PE_aligned) Align(AddressSize()); + + // Append ADDRESS to this section in the appropriate form. For the + // fixed-width forms, we don't need to differentiate between signed and + // unsigned encodings, because ADDRESS has already been extended to 64 + // bits before it was passed to us. + switch (encoding & 0x0f) { + case lul::DW_EH_PE_absptr: + Address(address); + break; + + case lul::DW_EH_PE_uleb128: + ULEB128(address); + break; + + case lul::DW_EH_PE_sleb128: + LEB128(address); + break; + + case lul::DW_EH_PE_udata2: + case lul::DW_EH_PE_sdata2: + D16(address); + break; + + case lul::DW_EH_PE_udata4: + case lul::DW_EH_PE_sdata4: + D32(address); + break; + + case lul::DW_EH_PE_udata8: + case lul::DW_EH_PE_sdata8: + D64(address); + break; + + default: + abort(); + } + + return *this; +}; + +} // namespace lul_test diff --git a/tools/profiler/tests/gtest/LulTestInfrastructure.h b/tools/profiler/tests/gtest/LulTestInfrastructure.h new file mode 100644 index 0000000000..9faa7ca858 --- /dev/null +++ b/tools/profiler/tests/gtest/LulTestInfrastructure.h @@ -0,0 +1,736 @@ +// -*- mode: C++ -*- + +// Copyright (c) 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Original author: Jim Blandy + +// Derived from: +// cfi_assembler.h: Define CFISection, a class for creating properly +// (and improperly) formatted DWARF CFI data for unit tests. + +// Derived from: +// test-assembler.h: interface to class for building complex binary streams. + +// To test the Breakpad symbol dumper and processor thoroughly, for +// all combinations of host system and minidump processor +// architecture, we need to be able to easily generate complex test +// data like debugging information and minidump files. +// +// For example, if we want our unit tests to provide full code +// coverage for stack walking, it may be difficult to persuade the +// compiler to generate every possible sort of stack walking +// information that we want to support; there are probably DWARF CFI +// opcodes that GCC never emits. Similarly, if we want to test our +// error handling, we will need to generate damaged minidumps or +// debugging information that (we hope) the client or compiler will +// never produce on its own. +// +// google_breakpad::TestAssembler provides a predictable and +// (relatively) simple way to generate complex formatted data streams +// like minidumps and CFI. Furthermore, because TestAssembler is +// portable, developers without access to (say) Visual Studio or a +// SPARC assembler can still work on test data for those targets. + +#ifndef LUL_TEST_INFRASTRUCTURE_H +#define LUL_TEST_INFRASTRUCTURE_H + +#include "LulDwarfExt.h" + +#include +#include + +using std::string; +using std::vector; + +namespace lul_test { +namespace test_assembler { + +// A Label represents a value not yet known that we need to store in a +// section. As long as all the labels a section refers to are defined +// by the time we retrieve its contents as bytes, we can use undefined +// labels freely in that section's construction. +// +// A label can be in one of three states: +// - undefined, +// - defined as the sum of some other label and a constant, or +// - a constant. +// +// A label's value never changes, but it can accumulate constraints. +// Adding labels and integers is permitted, and yields a label. +// Subtracting a constant from a label is permitted, and also yields a +// label. Subtracting two labels that have some relationship to each +// other is permitted, and yields a constant. +// +// For example: +// +// Label a; // a's value is undefined +// Label b; // b's value is undefined +// { +// Label c = a + 4; // okay, even though a's value is unknown +// b = c + 4; // also okay; b is now a+8 +// } +// Label d = b - 2; // okay; d == a+6, even though c is gone +// d.Value(); // error: d's value is not yet known +// d - a; // is 6, even though their values are not known +// a = 12; // now b == 20, and d == 18 +// d.Value(); // 18: no longer an error +// b.Value(); // 20 +// d = 10; // error: d is already defined. +// +// Label objects' lifetimes are unconstrained: notice that, in the +// above example, even though a and b are only related through c, and +// c goes out of scope, the assignment to a sets b's value as well. In +// particular, it's not necessary to ensure that a Label lives beyond +// Sections that refer to it. +class Label { + public: + Label(); // An undefined label. + explicit Label(uint64_t value); // A label with a fixed value + Label(const Label& value); // A label equal to another. + ~Label(); + + Label& operator=(uint64_t value); + Label& operator=(const Label& value); + Label operator+(uint64_t addend) const; + Label operator-(uint64_t subtrahend) const; + uint64_t operator-(const Label& subtrahend) const; + + // We could also provide == and != that work on undefined, but + // related, labels. + + // Return true if this label's value is known. If VALUE_P is given, + // set *VALUE_P to the known value if returning true. + bool IsKnownConstant(uint64_t* value_p = NULL) const; + + // Return true if the offset from LABEL to this label is known. If + // OFFSET_P is given, set *OFFSET_P to the offset when returning true. + // + // You can think of l.KnownOffsetFrom(m, &d) as being like 'd = l-m', + // except that it also returns a value indicating whether the + // subtraction is possible given what we currently know of l and m. + // It can be possible even if we don't know l and m's values. For + // example: + // + // Label l, m; + // m = l + 10; + // l.IsKnownConstant(); // false + // m.IsKnownConstant(); // false + // uint64_t d; + // l.IsKnownOffsetFrom(m, &d); // true, and sets d to -10. + // l-m // -10 + // m-l // 10 + // m.Value() // error: m's value is not known + bool IsKnownOffsetFrom(const Label& label, uint64_t* offset_p = NULL) const; + + private: + // A label's value, or if that is not yet known, how the value is + // related to other labels' values. A binding may be: + // - a known constant, + // - constrained to be equal to some other binding plus a constant, or + // - unconstrained, and free to take on any value. + // + // Many labels may point to a single binding, and each binding may + // refer to another, so bindings and labels form trees whose leaves + // are labels, whose interior nodes (and roots) are bindings, and + // where links point from children to parents. Bindings are + // reference counted, allowing labels to be lightweight, copyable, + // assignable, placed in containers, and so on. + class Binding { + public: + Binding(); + explicit Binding(uint64_t addend); + ~Binding(); + + // Increment our reference count. + void Acquire() { reference_count_++; }; + // Decrement our reference count, and return true if it is zero. + bool Release() { return --reference_count_ == 0; } + + // Set this binding to be equal to BINDING + ADDEND. If BINDING is + // NULL, then set this binding to the known constant ADDEND. + // Update every binding on this binding's chain to point directly + // to BINDING, or to be a constant, with addends adjusted + // appropriately. + void Set(Binding* binding, uint64_t value); + + // Return what we know about the value of this binding. + // - If this binding's value is a known constant, set BASE to + // NULL, and set ADDEND to its value. + // - If this binding is not a known constant but related to other + // bindings, set BASE to the binding at the end of the relation + // chain (which will always be unconstrained), and set ADDEND to the + // value to add to that binding's value to get this binding's + // value. + // - If this binding is unconstrained, set BASE to this, and leave + // ADDEND unchanged. + void Get(Binding** base, uint64_t* addend); + + private: + // There are three cases: + // + // - A binding representing a known constant value has base_ NULL, + // and addend_ equal to the value. + // + // - A binding representing a completely unconstrained value has + // base_ pointing to this; addend_ is unused. + // + // - A binding whose value is related to some other binding's + // value has base_ pointing to that other binding, and addend_ + // set to the amount to add to that binding's value to get this + // binding's value. We only represent relationships of the form + // x = y+c. + // + // Thus, the bind_ links form a chain terminating in either a + // known constant value or a completely unconstrained value. Most + // operations on bindings do path compression: they change every + // binding on the chain to point directly to the final value, + // adjusting addends as appropriate. + Binding* base_; + uint64_t addend_; + + // The number of Labels and Bindings pointing to this binding. + // (When a binding points to itself, indicating a completely + // unconstrained binding, that doesn't count as a reference.) + int reference_count_; + }; + + // This label's value. + Binding* value_; +}; + +// Conventions for representing larger numbers as sequences of bytes. +enum Endianness { + kBigEndian, // Big-endian: the most significant byte comes first. + kLittleEndian, // Little-endian: the least significant byte comes first. + kUnsetEndian, // used internally +}; + +// A section is a sequence of bytes, constructed by appending bytes +// to the end. Sections have a convenient and flexible set of member +// functions for appending data in various formats: big-endian and +// little-endian signed and unsigned values of different sizes; +// LEB128 and ULEB128 values (see below), and raw blocks of bytes. +// +// If you need to append a value to a section that is not convenient +// to compute immediately, you can create a label, append the +// label's value to the section, and then set the label's value +// later, when it's convenient to do so. Once a label's value is +// known, the section class takes care of updating all previously +// appended references to it. +// +// Once all the labels to which a section refers have had their +// values determined, you can get a copy of the section's contents +// as a string. +// +// Note that there is no specified "start of section" label. This is +// because there are typically several different meanings for "the +// start of a section": the offset of the section within an object +// file, the address in memory at which the section's content appear, +// and so on. It's up to the code that uses the Section class to +// keep track of these explicitly, as they depend on the application. +class Section { + public: + explicit Section(Endianness endianness = kUnsetEndian) + : endianness_(endianness){}; + + // A base class destructor should be either public and virtual, + // or protected and nonvirtual. + virtual ~Section(){}; + + // Return the default endianness of this section. + Endianness endianness() const { return endianness_; } + + // Append the SIZE bytes at DATA to the end of this section. Return + // a reference to this section. + Section& Append(const string& data) { + contents_.append(data); + return *this; + }; + + // Append data from SLICE to the end of this section. Return + // a reference to this section. + Section& Append(const lul::ImageSlice& slice) { + for (size_t i = 0; i < slice.length_; i++) { + contents_.append(1, slice.start_[i]); + } + return *this; + } + + // Append data from CSTRING to the end of this section. The terminating + // zero is not included. Return a reference to this section. + Section& Append(const char* cstring) { + for (size_t i = 0; cstring[i] != '\0'; i++) { + contents_.append(1, cstring[i]); + } + return *this; + } + + // Append SIZE copies of BYTE to the end of this section. Return a + // reference to this section. + Section& Append(size_t size, uint8_t byte) { + contents_.append(size, (char)byte); + return *this; + } + + // Append NUMBER to this section. ENDIANNESS is the endianness to + // use to write the number. SIZE is the length of the number in + // bytes. Return a reference to this section. + Section& Append(Endianness endianness, size_t size, uint64_t number); + Section& Append(Endianness endianness, size_t size, const Label& label); + + // Append SECTION to the end of this section. The labels SECTION + // refers to need not be defined yet. + // + // Note that this has no effect on any Labels' values, or on + // SECTION. If placing SECTION within 'this' provides new + // constraints on existing labels' values, then it's up to the + // caller to fiddle with those labels as needed. + Section& Append(const Section& section); + + // Append the contents of DATA as a series of bytes terminated by + // a NULL character. + Section& AppendCString(const string& data) { + Append(data); + contents_ += '\0'; + return *this; + } + + // Append VALUE or LABEL to this section, with the given bit width and + // endianness. Return a reference to this section. + // + // The names of these functions have the form : + // is either 'L' (little-endian, least significant byte first), + // 'B' (big-endian, most significant byte first), or + // 'D' (default, the section's default endianness) + // is 8, 16, 32, or 64. + // + // Since endianness doesn't matter for a single byte, all the + // =8 functions are equivalent. + // + // These can be used to write both signed and unsigned values, as + // the compiler will properly sign-extend a signed value before + // passing it to the function, at which point the function's + // behavior is the same either way. + Section& L8(uint8_t value) { + contents_ += value; + return *this; + } + Section& B8(uint8_t value) { + contents_ += value; + return *this; + } + Section& D8(uint8_t value) { + contents_ += value; + return *this; + } + Section &L16(uint16_t), &L32(uint32_t), &L64(uint64_t), &B16(uint16_t), + &B32(uint32_t), &B64(uint64_t), &D16(uint16_t), &D32(uint32_t), + &D64(uint64_t); + Section &L8(const Label& label), &L16(const Label& label), + &L32(const Label& label), &L64(const Label& label), + &B8(const Label& label), &B16(const Label& label), + &B32(const Label& label), &B64(const Label& label), + &D8(const Label& label), &D16(const Label& label), + &D32(const Label& label), &D64(const Label& label); + + // Append VALUE in a signed LEB128 (Little-Endian Base 128) form. + // + // The signed LEB128 representation of an integer N is a variable + // number of bytes: + // + // - If N is between -0x40 and 0x3f, then its signed LEB128 + // representation is a single byte whose value is N. + // + // - Otherwise, its signed LEB128 representation is (N & 0x7f) | + // 0x80, followed by the signed LEB128 representation of N / 128, + // rounded towards negative infinity. + // + // In other words, we break VALUE into groups of seven bits, put + // them in little-endian order, and then write them as eight-bit + // bytes with the high bit on all but the last. + // + // Note that VALUE cannot be a Label (we would have to implement + // relaxation). + Section& LEB128(long long value); + + // Append VALUE in unsigned LEB128 (Little-Endian Base 128) form. + // + // The unsigned LEB128 representation of an integer N is a variable + // number of bytes: + // + // - If N is between 0 and 0x7f, then its unsigned LEB128 + // representation is a single byte whose value is N. + // + // - Otherwise, its unsigned LEB128 representation is (N & 0x7f) | + // 0x80, followed by the unsigned LEB128 representation of N / + // 128, rounded towards negative infinity. + // + // Note that VALUE cannot be a Label (we would have to implement + // relaxation). + Section& ULEB128(uint64_t value); + + // Jump to the next location aligned on an ALIGNMENT-byte boundary, + // relative to the start of the section. Fill the gap with PAD_BYTE. + // ALIGNMENT must be a power of two. Return a reference to this + // section. + Section& Align(size_t alignment, uint8_t pad_byte = 0); + + // Return the current size of the section. + size_t Size() const { return contents_.size(); } + + // Return a label representing the start of the section. + // + // It is up to the user whether this label represents the section's + // position in an object file, the section's address in memory, or + // what have you; some applications may need both, in which case + // this simple-minded interface won't be enough. This class only + // provides a single start label, for use with the Here and Mark + // member functions. + // + // Ideally, we'd provide this in a subclass that actually knows more + // about the application at hand and can provide an appropriate + // collection of start labels. But then the appending member + // functions like Append and D32 would return a reference to the + // base class, not the derived class, and the chaining won't work. + // Since the only value here is in pretty notation, that's a fatal + // flaw. + Label start() const { return start_; } + + // Return a label representing the point at which the next Appended + // item will appear in the section, relative to start(). + Label Here() const { return start_ + Size(); } + + // Set *LABEL to Here, and return a reference to this section. + Section& Mark(Label* label) { + *label = Here(); + return *this; + } + + // If there are no undefined label references left in this + // section, set CONTENTS to the contents of this section, as a + // string, and clear this section. Return true on success, or false + // if there were still undefined labels. + bool GetContents(string* contents); + + private: + // Used internally. A reference to a label's value. + struct Reference { + Reference(size_t set_offset, Endianness set_endianness, size_t set_size, + const Label& set_label) + : offset(set_offset), + endianness(set_endianness), + size(set_size), + label(set_label) {} + + // The offset of the reference within the section. + size_t offset; + + // The endianness of the reference. + Endianness endianness; + + // The size of the reference. + size_t size; + + // The label to which this is a reference. + Label label; + }; + + // The default endianness of this section. + Endianness endianness_; + + // The contents of the section. + string contents_; + + // References to labels within those contents. + vector references_; + + // A label referring to the beginning of the section. + Label start_; +}; + +} // namespace test_assembler +} // namespace lul_test + +namespace lul_test { + +using lul::DwarfPointerEncoding; +using lul_test::test_assembler::Endianness; +using lul_test::test_assembler::Label; +using lul_test::test_assembler::Section; + +class CFISection : public Section { + public: + // CFI augmentation strings beginning with 'z', defined by the + // Linux/IA-64 C++ ABI, can specify interesting encodings for + // addresses appearing in FDE headers and call frame instructions (and + // for additional fields whose presence the augmentation string + // specifies). In particular, pointers can be specified to be relative + // to various base address: the start of the .text section, the + // location holding the address itself, and so on. These allow the + // frame data to be position-independent even when they live in + // write-protected pages. These variants are specified at the + // following two URLs: + // + // http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/dwarfext.html + // http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html + // + // CFISection leaves the production of well-formed 'z'-augmented CIEs and + // FDEs to the user, but does provide EncodedPointer, to emit + // properly-encoded addresses for a given pointer encoding. + // EncodedPointer uses an instance of this structure to find the base + // addresses it should use; you can establish a default for all encoded + // pointers appended to this section with SetEncodedPointerBases. + struct EncodedPointerBases { + EncodedPointerBases() : cfi(), text(), data() {} + + // The starting address of this CFI section in memory, for + // DW_EH_PE_pcrel. DW_EH_PE_pcrel pointers may only be used in data + // that has is loaded into the program's address space. + uint64_t cfi; + + // The starting address of this file's .text section, for DW_EH_PE_textrel. + uint64_t text; + + // The starting address of this file's .got or .eh_frame_hdr section, + // for DW_EH_PE_datarel. + uint64_t data; + }; + + // Create a CFISection whose endianness is ENDIANNESS, and where + // machine addresses are ADDRESS_SIZE bytes long. If EH_FRAME is + // true, use the .eh_frame format, as described by the Linux + // Standards Base Core Specification, instead of the DWARF CFI + // format. + CFISection(Endianness endianness, size_t address_size, bool eh_frame = false) + : Section(endianness), + address_size_(address_size), + eh_frame_(eh_frame), + pointer_encoding_(lul::DW_EH_PE_absptr), + encoded_pointer_bases_(), + entry_length_(NULL), + in_fde_(false) { + // The 'start', 'Here', and 'Mark' members of a CFISection all refer + // to section offsets. + start() = 0; + } + + // Return this CFISection's address size. + size_t AddressSize() const { return address_size_; } + + // Return true if this CFISection uses the .eh_frame format, or + // false if it contains ordinary DWARF CFI data. + bool ContainsEHFrame() const { return eh_frame_; } + + // Use ENCODING for pointers in calls to FDEHeader and EncodedPointer. + void SetPointerEncoding(DwarfPointerEncoding encoding) { + pointer_encoding_ = encoding; + } + + // Use the addresses in BASES as the base addresses for encoded + // pointers in subsequent calls to FDEHeader or EncodedPointer. + // This function makes a copy of BASES. + void SetEncodedPointerBases(const EncodedPointerBases& bases) { + encoded_pointer_bases_ = bases; + } + + // Append a Common Information Entry header to this section with the + // given values. If dwarf64 is true, use the 64-bit DWARF initial + // length format for the CIE's initial length. Return a reference to + // this section. You should call FinishEntry after writing the last + // instruction for the CIE. + // + // Before calling this function, you will typically want to use Mark + // or Here to make a label to pass to FDEHeader that refers to this + // CIE's position in the section. + CFISection& CIEHeader(uint64_t code_alignment_factor, + int data_alignment_factor, + unsigned return_address_register, uint8_t version = 3, + const string& augmentation = "", bool dwarf64 = false); + + // Append a Frame Description Entry header to this section with the + // given values. If dwarf64 is true, use the 64-bit DWARF initial + // length format for the CIE's initial length. Return a reference to + // this section. You should call FinishEntry after writing the last + // instruction for the CIE. + // + // This function doesn't support entries that are longer than + // 0xffffff00 bytes. (The "initial length" is always a 32-bit + // value.) Nor does it support .debug_frame sections longer than + // 0xffffff00 bytes. + CFISection& FDEHeader(Label cie_pointer, uint64_t initial_location, + uint64_t address_range, bool dwarf64 = false); + + // Note the current position as the end of the last CIE or FDE we + // started, after padding with DW_CFA_nops for alignment. This + // defines the label representing the entry's length, cited in the + // entry's header. Return a reference to this section. + CFISection& FinishEntry(); + + // Append the contents of BLOCK as a DW_FORM_block value: an + // unsigned LEB128 length, followed by that many bytes of data. + CFISection& Block(const lul::ImageSlice& block) { + ULEB128(block.length_); + Append(block); + return *this; + } + + // Append data from CSTRING as a DW_FORM_block value: an unsigned LEB128 + // length, followed by that many bytes of data. The terminating zero is not + // included. + CFISection& Block(const char* cstring) { + ULEB128(strlen(cstring)); + Append(cstring); + return *this; + } + + // Append ADDRESS to this section, in the appropriate size and + // endianness. Return a reference to this section. + CFISection& Address(uint64_t address) { + Section::Append(endianness(), address_size_, address); + return *this; + } + + // Append ADDRESS to this section, using ENCODING and BASES. ENCODING + // defaults to this section's default encoding, established by + // SetPointerEncoding. BASES defaults to this section's bases, set by + // SetEncodedPointerBases. If the DW_EH_PE_indirect bit is set in the + // encoding, assume that ADDRESS is where the true address is stored. + // Return a reference to this section. + // + // (C++ doesn't let me use default arguments here, because I want to + // refer to members of *this in the default argument expression.) + CFISection& EncodedPointer(uint64_t address) { + return EncodedPointer(address, pointer_encoding_, encoded_pointer_bases_); + } + CFISection& EncodedPointer(uint64_t address, DwarfPointerEncoding encoding) { + return EncodedPointer(address, encoding, encoded_pointer_bases_); + } + CFISection& EncodedPointer(uint64_t address, DwarfPointerEncoding encoding, + const EncodedPointerBases& bases); + + // Restate some member functions, to keep chaining working nicely. + CFISection& Mark(Label* label) { + Section::Mark(label); + return *this; + } + CFISection& D8(uint8_t v) { + Section::D8(v); + return *this; + } + CFISection& D16(uint16_t v) { + Section::D16(v); + return *this; + } + CFISection& D16(Label v) { + Section::D16(v); + return *this; + } + CFISection& D32(uint32_t v) { + Section::D32(v); + return *this; + } + CFISection& D32(const Label& v) { + Section::D32(v); + return *this; + } + CFISection& D64(uint64_t v) { + Section::D64(v); + return *this; + } + CFISection& D64(const Label& v) { + Section::D64(v); + return *this; + } + CFISection& LEB128(long long v) { + Section::LEB128(v); + return *this; + } + CFISection& ULEB128(uint64_t v) { + Section::ULEB128(v); + return *this; + } + + private: + // A length value that we've appended to the section, but is not yet + // known. LENGTH is the appended value; START is a label referring + // to the start of the data whose length was cited. + struct PendingLength { + Label length; + Label start; + }; + + // Constants used in CFI/.eh_frame data: + + // If the first four bytes of an "initial length" are this constant, then + // the data uses the 64-bit DWARF format, and the length itself is the + // subsequent eight bytes. + static const uint32_t kDwarf64InitialLengthMarker = 0xffffffffU; + + // The CIE identifier for 32- and 64-bit DWARF CFI and .eh_frame data. + static const uint32_t kDwarf32CIEIdentifier = ~(uint32_t)0; + static const uint64_t kDwarf64CIEIdentifier = ~(uint64_t)0; + static const uint32_t kEHFrame32CIEIdentifier = 0; + static const uint64_t kEHFrame64CIEIdentifier = 0; + + // The size of a machine address for the data in this section. + size_t address_size_; + + // If true, we are generating a Linux .eh_frame section, instead of + // a standard DWARF .debug_frame section. + bool eh_frame_; + + // The encoding to use for FDE pointers. + DwarfPointerEncoding pointer_encoding_; + + // The base addresses to use when emitting encoded pointers. + EncodedPointerBases encoded_pointer_bases_; + + // The length value for the current entry. + // + // Oddly, this must be dynamically allocated. Labels never get new + // values; they only acquire constraints on the value they already + // have, or assert if you assign them something incompatible. So + // each header needs truly fresh Label objects to cite in their + // headers and track their positions. The alternative is explicit + // destructor invocation and a placement new. Ick. + PendingLength* entry_length_; + + // True if we are currently emitting an FDE --- that is, we have + // called FDEHeader but have not yet called FinishEntry. + bool in_fde_; + + // If in_fde_ is true, this is its starting address. We use this for + // emitting DW_EH_PE_funcrel pointers. + uint64_t fde_start_address_; +}; + +} // namespace lul_test + +#endif // LUL_TEST_INFRASTRUCTURE_H diff --git a/tools/profiler/tests/gtest/ThreadProfileTest.cpp b/tools/profiler/tests/gtest/ThreadProfileTest.cpp new file mode 100644 index 0000000000..b8a15c39b2 --- /dev/null +++ b/tools/profiler/tests/gtest/ThreadProfileTest.cpp @@ -0,0 +1,60 @@ + +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef MOZ_GECKO_PROFILER + +# include "ProfileBuffer.h" + +# include "mozilla/PowerOfTwo.h" +# include "mozilla/ProfileBufferChunkManagerWithLocalLimit.h" +# include "mozilla/ProfileChunkedBuffer.h" + +# include "gtest/gtest.h" + +// Make sure we can record one entry and read it +TEST(ThreadProfile, InsertOneEntry) +{ + mozilla::ProfileBufferChunkManagerWithLocalLimit chunkManager( + 2 * (1 + uint32_t(sizeof(ProfileBufferEntry))) * 4, + 2 * (1 + uint32_t(sizeof(ProfileBufferEntry)))); + mozilla::ProfileChunkedBuffer profileChunkedBuffer( + mozilla::ProfileChunkedBuffer::ThreadSafety::WithMutex, chunkManager); + auto pb = mozilla::MakeUnique(profileChunkedBuffer); + pb->AddEntry(ProfileBufferEntry::Time(123.1)); + ProfileBufferEntry entry = pb->GetEntry(pb->BufferRangeStart()); + ASSERT_TRUE(entry.IsTime()); + ASSERT_EQ(123.1, entry.GetDouble()); +} + +// See if we can insert some entries +TEST(ThreadProfile, InsertEntriesNoWrap) +{ + mozilla::ProfileBufferChunkManagerWithLocalLimit chunkManager( + 100 * (1 + uint32_t(sizeof(ProfileBufferEntry))), + 100 * (1 + uint32_t(sizeof(ProfileBufferEntry))) / 4); + mozilla::ProfileChunkedBuffer profileChunkedBuffer( + mozilla::ProfileChunkedBuffer::ThreadSafety::WithMutex, chunkManager); + auto pb = mozilla::MakeUnique(profileChunkedBuffer); + const int test_size = 50; + for (int i = 0; i < test_size; i++) { + pb->AddEntry(ProfileBufferEntry::Time(i)); + } + int times = 0; + uint64_t readPos = pb->BufferRangeStart(); + while (readPos != pb->BufferRangeEnd()) { + ProfileBufferEntry entry = pb->GetEntry(readPos); + readPos++; + if (entry.GetKind() == ProfileBufferEntry::Kind::INVALID) { + continue; + } + ASSERT_TRUE(entry.IsTime()); + ASSERT_EQ(times, entry.GetDouble()); + times++; + } + ASSERT_EQ(test_size, times); +} + +#endif // MOZ_GECKO_PROFILER diff --git a/tools/profiler/tests/gtest/moz.build b/tools/profiler/tests/gtest/moz.build new file mode 100644 index 0000000000..4eb1fef762 --- /dev/null +++ b/tools/profiler/tests/gtest/moz.build @@ -0,0 +1,45 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at http://mozilla.org/MPL/2.0/. + +if ( + CONFIG["MOZ_GECKO_PROFILER"] + and CONFIG["OS_TARGET"] in ("Android", "Linux") + and CONFIG["CPU_ARCH"] + in ( + "arm", + "aarch64", + "x86", + "x86_64", + ) +): + UNIFIED_SOURCES += [ + "LulTest.cpp", + "LulTestDwarf.cpp", + "LulTestInfrastructure.cpp", + ] + +LOCAL_INCLUDES += [ + "/netwerk/base", + "/netwerk/protocol/http", + "/toolkit/components/jsoncpp/include", + "/tools/profiler/core", + "/tools/profiler/gecko", + "/tools/profiler/lul", +] + +if CONFIG["OS_TARGET"] != "Android": + UNIFIED_SOURCES += [ + "GeckoProfiler.cpp", + "ThreadProfileTest.cpp", + ] + +USE_LIBS += [ + "jsoncpp", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul-gtest" diff --git a/tools/profiler/tests/shared-head.js b/tools/profiler/tests/shared-head.js new file mode 100644 index 0000000000..d1b2f6868a --- /dev/null +++ b/tools/profiler/tests/shared-head.js @@ -0,0 +1,591 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* globals Assert */ +/* globals info */ + +/** + * This file contains utilities that can be shared between xpcshell tests and mochitests. + */ + +// The marker phases. +const INSTANT = 0; +const INTERVAL = 1; +const INTERVAL_START = 2; +const INTERVAL_END = 3; + +// This Services declaration may shadow another from head.js, so define it as +// a var rather than a const. + +const defaultSettings = { + entries: 8 * 1024 * 1024, // 8M entries = 64MB + interval: 1, // ms + features: [], + threads: ["GeckoMain"], +}; + +// Effectively `async`: Start the profiler and return the `startProfiler` +// promise that will get resolved when all child process have started their own +// profiler. +async function startProfiler(callersSettings) { + if (Services.profiler.IsActive()) { + Assert.ok( + Services.env.exists("MOZ_PROFILER_STARTUP"), + "The profiler is active at the begining of the test, " + + "the MOZ_PROFILER_STARTUP environment variable should be set." + ); + if (Services.env.exists("MOZ_PROFILER_STARTUP")) { + // If the startup profiling environment variable exists, it is likely + // that tests are being profiled. + // Stop the profiler before starting profiler tests. + info( + "This test starts and stops the profiler and is not compatible " + + "with the use of MOZ_PROFILER_STARTUP. " + + "Stopping the profiler before starting the test." + ); + await Services.profiler.StopProfiler(); + } else { + throw new Error( + "The profiler must not be active before starting it in a test." + ); + } + } + const settings = Object.assign({}, defaultSettings, callersSettings); + return Services.profiler.StartProfiler( + settings.entries, + settings.interval, + settings.features, + settings.threads, + 0, + settings.duration + ); +} + +function startProfilerForMarkerTests() { + return startProfiler({ + features: ["nostacksampling", "js"], + threads: ["GeckoMain", "DOM Worker"], + }); +} + +/** + * This is a helper function be able to run `await wait(500)`. Unfortunately + * this is needed as the act of collecting functions relies on the periodic + * sampling of the threads. See: + * https://bugzilla.mozilla.org/show_bug.cgi?id=1529053 + * + * @param {number} time + * @returns {Promise} + */ +function wait(time) { + return new Promise(resolve => { + // eslint-disable-next-line mozilla/no-arbitrary-setTimeout + setTimeout(resolve, time); + }); +} + +/** + * Get the payloads of a type recursively, including from all subprocesses. + * + * @param {Object} profile The gecko profile. + * @param {string} type The marker payload type, e.g. "DiskIO". + * @param {Array} payloadTarget The recursive list of payloads. + * @return {Array} The final payloads. + */ +function getPayloadsOfTypeFromAllThreads(profile, type, payloadTarget = []) { + for (const { markers } of profile.threads) { + for (const markerTuple of markers.data) { + const payload = markerTuple[markers.schema.data]; + if (payload && payload.type === type) { + payloadTarget.push(payload); + } + } + } + + for (const subProcess of profile.processes) { + getPayloadsOfTypeFromAllThreads(subProcess, type, payloadTarget); + } + + return payloadTarget; +} + +/** + * Get the payloads of a type from a single thread. + * + * @param {Object} thread The thread from a profile. + * @param {string} type The marker payload type, e.g. "DiskIO". + * @return {Array} The payloads. + */ +function getPayloadsOfType(thread, type) { + const { markers } = thread; + const results = []; + for (const markerTuple of markers.data) { + const payload = markerTuple[markers.schema.data]; + if (payload && payload.type === type) { + results.push(payload); + } + } + return results; +} + +/** + * Applies the marker schema to create individual objects for each marker + * + * @param {Object} thread The thread from a profile. + * @return {InflatedMarker[]} The markers. + */ +function getInflatedMarkerData(thread) { + const { markers, stringTable } = thread; + return markers.data.map(markerTuple => { + const marker = {}; + for (const [key, tupleIndex] of Object.entries(markers.schema)) { + marker[key] = markerTuple[tupleIndex]; + if (key === "name") { + // Use the string from the string table. + marker[key] = stringTable[marker[key]]; + } + } + return marker; + }); +} + +/** + * Applies the marker schema to create individual objects for each marker, then + * keeps only the network markers that match the profiler tests. + * + * @param {Object} thread The thread from a profile. + * @return {InflatedMarker[]} The filtered network markers. + */ +function getInflatedNetworkMarkers(thread) { + const markers = getInflatedMarkerData(thread); + return markers.filter( + m => + m.data && + m.data.type === "Network" && + // We filter out network markers that aren't related to the test, to + // avoid intermittents. + m.data.URI.includes("/tools/profiler/") + ); +} + +/** + * From a list of network markers, this returns pairs of start/stop markers. + * If a stop marker can't be found for a start marker, this will return an array + * of only 1 element. + * + * @param {InflatedMarker[]} networkMarkers Network markers + * @return {InflatedMarker[][]} Pairs of network markers + */ +function getPairsOfNetworkMarkers(allNetworkMarkers) { + // For each 'start' marker we want to find the next 'stop' or 'redirect' + // marker with the same id. + const result = []; + const mapOfStartMarkers = new Map(); // marker id -> id in result array + for (const marker of allNetworkMarkers) { + const { data } = marker; + if (data.status === "STATUS_START") { + if (mapOfStartMarkers.has(data.id)) { + const previousMarker = result[mapOfStartMarkers.get(data.id)][0]; + Assert.ok( + false, + `We found 2 start markers with the same id ${data.id}, without end marker in-between.` + + `The first marker has URI ${previousMarker.data.URI}, the second marker has URI ${data.URI}.` + + ` This should not happen.` + ); + continue; + } + + mapOfStartMarkers.set(data.id, result.length); + result.push([marker]); + } else { + // STOP or REDIRECT + if (!mapOfStartMarkers.has(data.id)) { + Assert.ok( + false, + `We found an end marker without a start marker (id: ${data.id}, URI: ${data.URI}). This should not happen.` + ); + continue; + } + result[mapOfStartMarkers.get(data.id)].push(marker); + mapOfStartMarkers.delete(data.id); + } + } + + return result; +} + +/** + * It can be helpful to force the profiler to collect a JavaScript sample. This + * function spins on a while loop until at least one more sample is collected. + * + * @return {number} The index of the collected sample. + */ +function captureAtLeastOneJsSample() { + function getProfileSampleCount() { + const profile = Services.profiler.getProfileData(); + return profile.threads[0].samples.data.length; + } + + const sampleCount = getProfileSampleCount(); + // Create an infinite loop until a sample has been collected. + while (true) { + if (sampleCount < getProfileSampleCount()) { + return sampleCount; + } + } +} + +function isJSONWhitespace(c) { + return ["\n", "\r", " ", "\t"].includes(c); +} + +function verifyJSONStringIsCompact(s) { + const stateData = 0; + const stateString = 1; + const stateEscapedChar = 2; + let state = stateData; + for (let i = 0; i < s.length; ++i) { + let c = s[i]; + switch (state) { + case stateData: + if (isJSONWhitespace(c)) { + Assert.ok( + false, + `"Unexpected JSON whitespace at index ${i} in profile: <<<${s}>>>"` + ); + return; + } + if (c == '"') { + state = stateString; + } + break; + case stateString: + if (c == '"') { + state = stateData; + } else if (c == "\\") { + state = stateEscapedChar; + } + break; + case stateEscapedChar: + state = stateString; + break; + } + } +} + +/** + * This function pauses the profiler before getting the profile. Then after + * getting the data, the profiler is stopped, and all profiler data is removed. + * @returns {Promise} + */ +async function stopNowAndGetProfile() { + // Don't await the pause, because each process will handle it before it + // receives the following `getProfileDataAsArrayBuffer()`. + Services.profiler.Pause(); + + const profileArrayBuffer = + await Services.profiler.getProfileDataAsArrayBuffer(); + await Services.profiler.StopProfiler(); + + const profileUint8Array = new Uint8Array(profileArrayBuffer); + const textDecoder = new TextDecoder("utf-8", { fatal: true }); + const profileString = textDecoder.decode(profileUint8Array); + verifyJSONStringIsCompact(profileString); + + return JSON.parse(profileString); +} + +/** + * This function ensures there's at least one sample, then pauses the profiler + * before getting the profile. Then after getting the data, the profiler is + * stopped, and all profiler data is removed. + * @returns {Promise} + */ +async function waitSamplingAndStopAndGetProfile() { + await Services.profiler.waitOnePeriodicSampling(); + return stopNowAndGetProfile(); +} + +/** + * Verifies that a marker is an interval marker. + * + * @param {InflatedMarker} marker + * @returns {boolean} + */ +function isIntervalMarker(inflatedMarker) { + return ( + inflatedMarker.phase === 1 && + typeof inflatedMarker.startTime === "number" && + typeof inflatedMarker.endTime === "number" + ); +} + +/** + * @param {Profile} profile + * @returns {Thread[]} + */ +function getThreads(profile) { + const threads = []; + + function getThreadsRecursive(process) { + for (const thread of process.threads) { + threads.push(thread); + } + for (const subprocess of process.processes) { + getThreadsRecursive(subprocess); + } + } + + getThreadsRecursive(profile); + return threads; +} + +/** + * Find a specific marker schema from any process of a profile. + * + * @param {Profile} profile + * @param {string} name + * @returns {MarkerSchema} + */ +function getSchema(profile, name) { + { + const schema = profile.meta.markerSchema.find(s => s.name === name); + if (schema) { + return schema; + } + } + for (const subprocess of profile.processes) { + const schema = subprocess.meta.markerSchema.find(s => s.name === name); + if (schema) { + return schema; + } + } + console.error("Parent process schema", profile.meta.markerSchema); + for (const subprocess of profile.processes) { + console.error("Child process schema", subprocess.meta.markerSchema); + } + throw new Error(`Could not find a schema for "${name}".`); +} + +/** + * This escapes all characters that have a special meaning in RegExps. + * This was stolen from https://github.com/sindresorhus/escape-string-regexp and + * so it is licence MIT and: + * Copyright (c) Sindre Sorhus (https://sindresorhus.com). + * See the full license in https://raw.githubusercontent.com/sindresorhus/escape-string-regexp/main/license. + * @param {string} string The string to be escaped + * @returns {string} The result + */ +function escapeStringRegexp(string) { + if (typeof string !== "string") { + throw new TypeError("Expected a string"); + } + + // Escape characters with special meaning either inside or outside character + // sets. Use a simple backslash escape when it’s always valid, and a `\xnn` + // escape when the simpler form would be disallowed by Unicode patterns’ + // stricter grammar. + return string.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&").replace(/-/g, "\\x2d"); +} + +/** ------ Assertions helper ------ */ +/** + * This assert helper function makes it easy to check a lot of properties in an + * object. We augment Assert.sys.mjs to make it easier to use. + */ +Object.assign(Assert, { + /* + * It checks if the properties on the right are all present in the object on + * the left. Note that the object might still have other properties (see + * objectContainsOnly below if you want the stricter form). + * + * The basic form does basic equality on each expected property: + * + * Assert.objectContains(fixture, { + * foo: "foo", + * bar: 1, + * baz: true, + * }); + * + * But it also has a more powerful form with expectations. The available + * expectations are: + * - any(): this only checks for the existence of the property, not its value + * - number(), string(), boolean(), bigint(), function(), symbol(), object(): + * this checks if the value is of this type + * - objectContains(expected): this applies Assert.objectContains() + * recursively on this property. + * - stringContains(needle): this checks if the expected value is included in + * the property value. + * - stringMatches(regexp): this checks if the property value matches this + * regexp. The regexp can be passed as a string, to be dynamically built. + * + * example: + * + * Assert.objectContains(fixture, { + * name: Expect.stringMatches(`Load \\d+:.*${url}`), + * data: Expect.objectContains({ + * status: "STATUS_STOP", + * URI: Expect.stringContains("https://"), + * requestMethod: "GET", + * contentType: Expect.string(), + * startTime: Expect.number(), + * cached: Expect.boolean(), + * }), + * }); + * + * Each expectation will translate into one or more Assert call. Therefore if + * one expectation fails, this will be clearly visible in the test output. + * + * Expectations can also be normal functions, for example: + * + * Assert.objectContains(fixture, { + * number: value => Assert.greater(value, 5) + * }); + * + * Note that you'll need to use Assert inside this function. + */ + objectContains(object, expectedProperties) { + // Basic tests: we don't want to run other assertions if these tests fail. + if (typeof object !== "object") { + this.ok( + false, + `The first parameter should be an object, but found: ${object}.` + ); + return; + } + + if (typeof expectedProperties !== "object") { + this.ok( + false, + `The second parameter should be an object, but found: ${expectedProperties}.` + ); + return; + } + + for (const key of Object.keys(expectedProperties)) { + const expected = expectedProperties[key]; + if (!(key in object)) { + this.report( + true, + object, + expectedProperties, + `The object should contain the property "${key}", but it's missing.` + ); + continue; + } + + if (typeof expected === "function") { + // This is a function, so let's call it. + expected( + object[key], + `The object should contain the property "${key}" with an expected value and type.` + ); + } else { + // Otherwise, we check for equality. + this.equal( + object[key], + expectedProperties[key], + `The object should contain the property "${key}" with an expected value.` + ); + } + } + }, + + /** + * This is very similar to the previous `objectContains`, but this also looks + * at the number of the objects' properties. Thus this will fail if the + * objects don't have the same properties exactly. + */ + objectContainsOnly(object, expectedProperties) { + // Basic tests: we don't want to run other assertions if these tests fail. + if (typeof object !== "object") { + this.ok( + false, + `The first parameter should be an object but found: ${object}.` + ); + return; + } + + if (typeof expectedProperties !== "object") { + this.ok( + false, + `The second parameter should be an object but found: ${expectedProperties}.` + ); + return; + } + + // In objectContainsOnly, we specifically want to check if all properties + // from the fixture object are expected. + // We'll be failing a test only for the specific properties that weren't + // expected, and only fail with one message, so that the test outputs aren't + // spammed. + const extraProperties = []; + for (const fixtureKey of Object.keys(object)) { + if (!(fixtureKey in expectedProperties)) { + extraProperties.push(fixtureKey); + } + } + + if (extraProperties.length) { + // Some extra properties have been found. + this.report( + true, + object, + expectedProperties, + `These properties are present, but shouldn't: "${extraProperties.join( + '", "' + )}".` + ); + } + + // Now, let's carry on the rest of our work. + this.objectContains(object, expectedProperties); + }, +}); + +const Expect = { + any: + () => + actual => {} /* We don't check anything more than the presence of this property. */, +}; + +/* These functions are part of the Assert object, and we want to reuse them. */ +[ + "stringContains", + "stringMatches", + "objectContains", + "objectContainsOnly", +].forEach( + assertChecker => + (Expect[assertChecker] = + expected => + (actual, ...moreArgs) => + Assert[assertChecker](actual, expected, ...moreArgs)) +); + +/* These functions will only check for the type. */ +[ + "number", + "string", + "boolean", + "bigint", + "symbol", + "object", + "function", +].forEach(type => (Expect[type] = makeTypeChecker(type))); + +function makeTypeChecker(type) { + return (...unexpectedArgs) => { + if (unexpectedArgs.length) { + throw new Error( + "Type checkers expectations aren't expecting any argument." + ); + } + return (actual, message) => { + const isCorrect = typeof actual === type; + Assert.report(!isCorrect, actual, type, message, "has type"); + }; + }; +} +/* ------ End of assertion helper ------ */ diff --git a/tools/profiler/tests/xpcshell/head.js b/tools/profiler/tests/xpcshell/head.js new file mode 100644 index 0000000000..ce87b32fd5 --- /dev/null +++ b/tools/profiler/tests/xpcshell/head.js @@ -0,0 +1,244 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* import-globals-from ../shared-head.js */ + +// This Services declaration may shadow another from head.js, so define it as +// a var rather than a const. + +const { AppConstants } = ChromeUtils.importESModule( + "resource://gre/modules/AppConstants.sys.mjs" +); +const { setTimeout } = ChromeUtils.importESModule( + "resource://gre/modules/Timer.sys.mjs" +); + +// Load the shared head +const sharedHead = do_get_file("shared-head.js", false); +if (!sharedHead) { + throw new Error("Could not load the shared head."); +} +Services.scriptloader.loadSubScript( + Services.io.newFileURI(sharedHead).spec, + this +); + +/** + * This function takes a thread, and a sample tuple from the "data" array, and + * inflates the frame to be an array of strings. + * + * @param {Object} thread - The thread from the profile. + * @param {Array} sample - The tuple from the thread.samples.data array. + * @returns {Array} An array of function names. + */ +function getInflatedStackLocations(thread, sample) { + let stackTable = thread.stackTable; + let frameTable = thread.frameTable; + let stringTable = thread.stringTable; + let SAMPLE_STACK_SLOT = thread.samples.schema.stack; + let STACK_PREFIX_SLOT = stackTable.schema.prefix; + let STACK_FRAME_SLOT = stackTable.schema.frame; + let FRAME_LOCATION_SLOT = frameTable.schema.location; + + // Build the stack from the raw data and accumulate the locations in + // an array. + let stackIndex = sample[SAMPLE_STACK_SLOT]; + let locations = []; + while (stackIndex !== null) { + let stackEntry = stackTable.data[stackIndex]; + let frame = frameTable.data[stackEntry[STACK_FRAME_SLOT]]; + locations.push(stringTable[frame[FRAME_LOCATION_SLOT]]); + stackIndex = stackEntry[STACK_PREFIX_SLOT]; + } + + // The profiler tree is inverted, so reverse the array. + return locations.reverse(); +} + +/** + * This utility matches up stacks to see if they contain a certain sequence of + * stack frames. A correctly functioning profiler will have a certain sequence + * of stacks, but we can't always determine exactly which stacks will show up + * due to implementation changes, as well as memory addresses being arbitrary to + * that particular build. + * + * This function triggers a test failure with a nice debug message when it + * fails. + * + * @param {Array} actualStackFrames - As generated by + * inflatedStackFrames. + * @param {Array} expectedStackFrames - Matches a subset of + * actualStackFrames + */ +function expectStackToContain( + actualStackFrames, + expectedStackFrames, + message = "The actual stack and expected stack do not match." +) { + // Log the stacks that are being passed to this assertion, as it could be + // useful for when these tests fail. + console.log("Actual stack: ", actualStackFrames); + console.log( + "Expected to contain: ", + expectedStackFrames.map(s => s.toString()) + ); + + let actualIndex = 0; + + // Start walking the expected stack and look for matches. + for ( + let expectedIndex = 0; + expectedIndex < expectedStackFrames.length; + expectedIndex++ + ) { + const expectedStackFrame = expectedStackFrames[expectedIndex]; + + while (true) { + // Make sure that we haven't run out of actual stack frames. + if (actualIndex >= actualStackFrames.length) { + info(`Could not find a match for: "${expectedStackFrame.toString()}"`); + Assert.ok(false, message); + } + + const actualStackFrame = actualStackFrames[actualIndex]; + actualIndex++; + + const itMatches = + typeof expectedStackFrame === "string" + ? expectedStackFrame === actualStackFrame + : actualStackFrame.match(expectedStackFrame); + + if (itMatches) { + // We found a match, break out of this loop. + break; + } + // Keep on looping looking for a match. + } + } + + Assert.ok(true, message); +} + +/** + * @param {Thread} thread + * @param {string} filename - The filename used to trigger FileIO. + * @returns {InflatedMarkers[]} + */ +function getInflatedFileIOMarkers(thread, filename) { + const markers = getInflatedMarkerData(thread); + return markers.filter( + marker => + marker.data?.type === "FileIO" && + marker.data?.filename?.endsWith(filename) + ); +} + +/** + * Checks properties common to all FileIO markers. + * + * @param {InflatedMarkers[]} markers + * @param {string} filename + */ +function checkInflatedFileIOMarkers(markers, filename) { + greater(markers.length, 0, "Found some markers"); + + // See IOInterposeObserver::Observation::ObservedOperationString + const validOperations = new Set([ + "write", + "fsync", + "close", + "stat", + "create/open", + "read", + ]); + const validSources = new Set(["PoisonIOInterposer", "NSPRIOInterposer"]); + + for (const marker of markers) { + try { + ok( + marker.name.startsWith("FileIO"), + "Has a marker.name that starts with FileIO" + ); + equal(marker.data.type, "FileIO", "Has a marker.data.type"); + ok(isIntervalMarker(marker), "All FileIO markers are interval markers"); + ok( + validOperations.has(marker.data.operation), + `The markers have a known operation - "${marker.data.operation}"` + ); + ok( + validSources.has(marker.data.source), + `The FileIO marker has a known source "${marker.data.source}"` + ); + ok(marker.data.filename.endsWith(filename)); + ok(Boolean(marker.data.stack), "A stack was collected"); + } catch (error) { + console.error("Failing inflated FileIO marker:", marker); + throw error; + } + } +} + +/** + * Do deep equality checks for schema, but then surface nice errors for a user to know + * what to do if the check fails. + */ +function checkSchema(actual, expected) { + const schemaName = expected.name; + info(`Checking marker schema for "${schemaName}"`); + + try { + ok( + actual, + `Schema was found for "${schemaName}". See the test output for more information.` + ); + // Check individual properties to surface easier to debug errors. + deepEqual( + expected.display, + actual.display, + `The "display" property for ${schemaName} schema matches. See the test output for more information.` + ); + if (expected.data) { + ok(actual.data, `Schema was found for "${schemaName}"`); + for (const expectedDatum of expected.data) { + const actualDatum = actual.data.find(d => d.key === expectedDatum.key); + deepEqual( + expectedDatum, + actualDatum, + `The "${schemaName}" field "${expectedDatum.key}" matches expectations. See the test output for more information.` + ); + } + equal( + expected.data.length, + actual.data.length, + "The expected and actual data have the same number of items" + ); + } + + // Finally do a true deep equal. + deepEqual(expected, actual, "The entire schema is deepEqual"); + } catch (error) { + // The test results are not very human readable. This is a bit of a hacky + // solution to make it more readable. + dump("-----------------------------------------------------\n"); + dump("The expected marker schema:\n"); + dump("-----------------------------------------------------\n"); + dump(JSON.stringify(expected, null, 2)); + dump("\n"); + dump("-----------------------------------------------------\n"); + dump("The actual marker schema:\n"); + dump("-----------------------------------------------------\n"); + dump(JSON.stringify(actual, null, 2)); + dump("\n"); + dump("-----------------------------------------------------\n"); + dump("A marker schema was not equal to expectations. If you\n"); + dump("are modifying the schema, then please copy and paste\n"); + dump("the new schema into this test.\n"); + dump("-----------------------------------------------------\n"); + dump("Copy this: " + JSON.stringify(actual)); + dump("\n"); + dump("-----------------------------------------------------\n"); + + throw error; + } +} diff --git a/tools/profiler/tests/xpcshell/test_active_configuration.js b/tools/profiler/tests/xpcshell/test_active_configuration.js new file mode 100644 index 0000000000..c4336f3f32 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_active_configuration.js @@ -0,0 +1,115 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + info( + "Checking that the profiler can fetch the information about the active " + + "configuration that is being used to power the profiler." + ); + + equal( + Services.profiler.activeConfiguration, + null, + "When the profile is off, there is no active configuration." + ); + + { + info("Start the profiler."); + const entries = 10000; + const interval = 1; + const threads = ["GeckoMain"]; + const features = ["js"]; + const activeTabID = 123; + await Services.profiler.StartProfiler( + entries, + interval, + features, + threads, + activeTabID + ); + + info("Generate the activeConfiguration."); + const { activeConfiguration } = Services.profiler; + const expectedConfiguration = { + interval, + threads, + features, + activeTabID, + // The buffer is created as a power of two that can fit all of the entires + // into it. If the ratio of entries to buffer size ever changes, this setting + // will need to be updated. + capacity: Math.pow(2, 14), + }; + + deepEqual( + activeConfiguration, + expectedConfiguration, + "The active configuration matches configuration given." + ); + + info("Get the profile."); + const profile = Services.profiler.getProfileData(); + deepEqual( + profile.meta.configuration, + expectedConfiguration, + "The configuration also matches on the profile meta object." + ); + } + + { + const entries = 20000; + const interval = 0.5; + const threads = ["GeckoMain", "DOM Worker"]; + const features = []; + const activeTabID = 111; + const duration = 20; + + info("Restart the profiler with a new configuration."); + await Services.profiler.StartProfiler( + entries, + interval, + features, + threads, + activeTabID, + // Also start it with duration, this property is optional. + duration + ); + + info("Generate the activeConfiguration."); + const { activeConfiguration } = Services.profiler; + const expectedConfiguration = { + interval, + threads, + features, + activeTabID, + duration, + // The buffer is created as a power of two that can fit all of the entires + // into it. If the ratio of entries to buffer size ever changes, this setting + // will need to be updated. + capacity: Math.pow(2, 15), + }; + + deepEqual( + activeConfiguration, + expectedConfiguration, + "The active configuration matches the new configuration." + ); + + info("Get the profile."); + const profile = Services.profiler.getProfileData(); + deepEqual( + profile.meta.configuration, + expectedConfiguration, + "The configuration also matches on the profile meta object." + ); + } + + await Services.profiler.StopProfiler(); + + equal( + Services.profiler.activeConfiguration, + null, + "When the profile is off, there is no active configuration." + ); +}); diff --git a/tools/profiler/tests/xpcshell/test_addProfilerMarker.js b/tools/profiler/tests/xpcshell/test_addProfilerMarker.js new file mode 100644 index 0000000000..b11545a41c --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_addProfilerMarker.js @@ -0,0 +1,221 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that ChromeUtils.addProfilerMarker is working correctly. + */ + +const markerNamePrefix = "test_addProfilerMarker"; +const markerText = "Text payload"; +// The same startTime will be used for all markers with a duration, +// and we store this value globally so that expectDuration and +// expectNoDuration can access it. The value isn't set here as we +// want a start time after the profiler has started +var startTime; + +function expectNoDuration(marker) { + Assert.equal( + typeof marker.startTime, + "number", + "startTime should be a number" + ); + Assert.greater( + marker.startTime, + startTime, + "startTime should be after the begining of the test" + ); + Assert.equal(typeof marker.endTime, "number", "endTime should be a number"); + Assert.equal(marker.endTime, 0, "endTime should be 0"); +} + +function expectDuration(marker) { + Assert.equal( + typeof marker.startTime, + "number", + "startTime should be a number" + ); + // Floats can cause rounding issues. We've seen up to a 4.17e-5 difference in + // intermittent failures, so we are permissive and accept up to 5e-5. + Assert.less( + Math.abs(marker.startTime - startTime), + 5e-5, + "startTime should be the expected time" + ); + Assert.equal(typeof marker.endTime, "number", "endTime should be a number"); + Assert.greater( + marker.endTime, + startTime, + "endTime should be after startTime" + ); +} + +function expectNoData(marker) { + Assert.equal( + typeof marker.data, + "undefined", + "The data property should be undefined" + ); +} + +function expectText(marker) { + Assert.equal( + typeof marker.data, + "object", + "The data property should be an object" + ); + Assert.equal(marker.data.type, "Text", "Should be a Text marker"); + Assert.equal( + marker.data.name, + markerText, + "The payload should contain the expected text" + ); +} + +function expectNoStack(marker) { + Assert.ok(!marker.data || !marker.data.stack, "There should be no stack"); +} + +function expectStack(marker, thread) { + let stack = marker.data.stack; + Assert.ok(!!stack, "There should be a stack"); + + // Marker stacks are recorded as a profile of a thread with a single sample, + // get the stack id. + stack = stack.samples.data[0][stack.samples.schema.stack]; + + const stackPrefixCol = thread.stackTable.schema.prefix; + const stackFrameCol = thread.stackTable.schema.frame; + const frameLocationCol = thread.frameTable.schema.location; + + // Get the entire stack in an array for easier processing. + let result = []; + while (stack != null) { + let stackEntry = thread.stackTable.data[stack]; + let frame = thread.frameTable.data[stackEntry[stackFrameCol]]; + result.push(thread.stringTable[frame[frameLocationCol]]); + stack = stackEntry[stackPrefixCol]; + } + + Assert.greaterOrEqual( + result.length, + 1, + "There should be at least one frame in the stack" + ); + + Assert.ok( + result.some(frame => frame.includes("testMarker")), + "the 'testMarker' function should be visible in the stack" + ); + + Assert.ok( + !result.some(frame => frame.includes("ChromeUtils.addProfilerMarker")), + "the 'ChromeUtils.addProfilerMarker' label frame should not be visible in the stack" + ); +} + +add_task(async () => { + startProfilerForMarkerTests(); + startTime = Cu.now(); + while (Cu.now() < startTime + 1) { + // Busy wait for 1ms to ensure the intentionally set start time of markers + // will be significantly different from the time at which the marker is + // recorded. + } + info("startTime used for markers with durations: " + startTime); + + /* Each call to testMarker will record a marker with a unique name. + * The testFunctions and testCases objects contain respectively test + * functions to verify that the marker found in the captured profile + * matches expectations, and a string that can be printed to describe + * in which way ChromeUtils.addProfilerMarker was called. */ + let testFunctions = {}; + let testCases = {}; + let markerId = 0; + function testMarker(args, checks) { + let name = markerNamePrefix + markerId++; + ChromeUtils.addProfilerMarker(name, ...args); + testFunctions[name] = checks; + testCases[name] = `ChromeUtils.addProfilerMarker(${[name, ...args] + .toSource() + .slice(1, -1)})`; + } + + info("Record markers without options object."); + testMarker([], m => { + expectNoDuration(m); + expectNoData(m); + }); + testMarker([startTime], m => { + expectDuration(m); + expectNoData(m); + }); + testMarker([undefined, markerText], m => { + expectNoDuration(m); + expectText(m); + }); + testMarker([startTime, markerText], m => { + expectDuration(m); + expectText(m); + }); + + info("Record markers providing the duration as the startTime property."); + testMarker([{ startTime }], m => { + expectDuration(m); + expectNoData(m); + }); + testMarker([{}, markerText], m => { + expectNoDuration(m); + expectText(m); + }); + testMarker([{ startTime }, markerText], m => { + expectDuration(m); + expectText(m); + }); + + info("Record markers to test the captureStack property."); + const captureStack = true; + testMarker([], expectNoStack); + testMarker([startTime, markerText], expectNoStack); + testMarker([{ captureStack: false }], expectNoStack); + testMarker([{ captureStack }], expectStack); + testMarker([{ startTime, captureStack }], expectStack); + testMarker([{ captureStack }, markerText], expectStack); + testMarker([{ startTime, captureStack }, markerText], expectStack); + + info("Record markers to test the category property"); + function testCategory(args, expectedCategory) { + testMarker(args, marker => { + Assert.equal(marker.category, expectedCategory); + }); + } + testCategory([], "JavaScript"); + testCategory([{ category: "Test" }], "Test"); + testCategory([{ category: "Test" }, markerText], "Test"); + testCategory([{ category: "JavaScript" }], "JavaScript"); + testCategory([{ category: "Other" }], "Other"); + testCategory([{ category: "DOM" }], "DOM"); + testCategory([{ category: "does not exist" }], "Other"); + + info("Capture the profile"); + const profile = await stopNowAndGetProfile(); + const mainThread = profile.threads.find(({ name }) => name === "GeckoMain"); + const markers = getInflatedMarkerData(mainThread).filter(m => + m.name.startsWith(markerNamePrefix) + ); + Assert.equal( + markers.length, + Object.keys(testFunctions).length, + `Found ${markers.length} test markers in the captured profile` + ); + + for (let marker of markers) { + marker.category = profile.meta.categories[marker.category].name; + info(`${testCases[marker.name]} -> ${marker.toSource()}`); + + testFunctions[marker.name](marker, mainThread); + delete testFunctions[marker.name]; + } + + Assert.equal(0, Object.keys(testFunctions).length, "all markers were found"); +}); diff --git a/tools/profiler/tests/xpcshell/test_asm.js b/tools/profiler/tests/xpcshell/test_asm.js new file mode 100644 index 0000000000..ced36ce429 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_asm.js @@ -0,0 +1,76 @@ +// Check that asm.js code shows up on the stack. +add_task(async () => { + // This test assumes that it's starting on an empty profiler stack. + // (Note that the other profiler tests also assume the profiler + // isn't already started.) + Assert.ok(!Services.profiler.IsActive()); + + let jsFuns = Cu.getJSTestingFunctions(); + if (!jsFuns.isAsmJSCompilationAvailable()) { + return; + } + + const ms = 10; + await Services.profiler.StartProfiler(10000, ms, ["js"]); + + let stack = null; + function ffi_function() { + var delayMS = 5; + while (1) { + let then = Date.now(); + do { + // do nothing + } while (Date.now() - then < delayMS); + + var thread0 = Services.profiler.getProfileData().threads[0]; + + if (delayMS > 30000) { + return; + } + + delayMS *= 2; + + if (!thread0.samples.data.length) { + continue; + } + + var lastSample = thread0.samples.data[thread0.samples.data.length - 1]; + stack = String(getInflatedStackLocations(thread0, lastSample)); + if (stack.includes("trampoline")) { + return; + } + } + } + + function asmjs_module(global, ffis) { + "use asm"; + var ffi = ffis.ffi; + function asmjs_function() { + ffi(); + } + return asmjs_function; + } + + Assert.ok(jsFuns.isAsmJSModule(asmjs_module)); + + var asmjs_function = asmjs_module(null, { ffi: ffi_function }); + Assert.ok(jsFuns.isAsmJSFunction(asmjs_function)); + + asmjs_function(); + + Assert.notEqual(stack, null); + + var i1 = stack.indexOf("entry trampoline"); + Assert.ok(i1 !== -1); + var i2 = stack.indexOf("asmjs_function"); + Assert.ok(i2 !== -1); + var i3 = stack.indexOf("exit trampoline"); + Assert.ok(i3 !== -1); + var i4 = stack.indexOf("ffi_function"); + Assert.ok(i4 !== -1); + Assert.ok(i1 < i2); + Assert.ok(i2 < i3); + Assert.ok(i3 < i4); + + await Services.profiler.StopProfiler(); +}); diff --git a/tools/profiler/tests/xpcshell/test_assertion_helper.js b/tools/profiler/tests/xpcshell/test_assertion_helper.js new file mode 100644 index 0000000000..baa4c34818 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_assertion_helper.js @@ -0,0 +1,162 @@ +add_task(function setup() { + // With the default reporter, an assertion doesn't throw if it fails, it + // merely report the result to the reporter and then go on. But in this test + // we want that a failure really throws, so that we can actually assert that + // it throws in case of failures! + // That's why we disable the default repoter here. + // I noticed that this line needs to be in an add_task (or possibly run_test) + // function. If put outside this will crash the test. + Assert.setReporter(null); +}); + +add_task(function test_objectContains() { + const fixture = { + foo: "foo", + bar: "bar", + }; + + Assert.objectContains(fixture, { foo: "foo" }, "Matches one property value"); + Assert.objectContains( + fixture, + { foo: "foo", bar: "bar" }, + "Matches both properties" + ); + Assert.objectContainsOnly( + fixture, + { foo: "foo", bar: "bar" }, + "Matches both properties" + ); + Assert.throws( + () => Assert.objectContainsOnly(fixture, { foo: "foo" }), + /AssertionError/, + "Fails if some properties are missing" + ); + Assert.throws( + () => Assert.objectContains(fixture, { foo: "bar" }), + /AssertionError/, + "Fails if the value for a present property is wrong" + ); + Assert.throws( + () => Assert.objectContains(fixture, { hello: "world" }), + /AssertionError/, + "Fails if an expected property is missing" + ); + Assert.throws( + () => Assert.objectContains(fixture, { foo: "foo", hello: "world" }), + /AssertionError/, + "Fails if some properties are present but others are missing" + ); +}); + +add_task(function test_objectContains_expectations() { + const fixture = { + foo: "foo", + bar: "bar", + num: 42, + nested: { + nestedFoo: "nestedFoo", + nestedBar: "nestedBar", + }, + }; + + Assert.objectContains( + fixture, + { + foo: Expect.stringMatches(/^fo/), + bar: Expect.stringContains("ar"), + num: Expect.number(), + nested: Expect.objectContainsOnly({ + nestedFoo: Expect.stringMatches(/[Ff]oo/), + nestedBar: Expect.stringMatches(/[Bb]ar/), + }), + }, + "Supports expectations" + ); + Assert.objectContainsOnly( + fixture, + { + foo: Expect.stringMatches(/^fo/), + bar: Expect.stringContains("ar"), + num: Expect.number(), + nested: Expect.objectContains({ + nestedFoo: Expect.stringMatches(/[Ff]oo/), + }), + }, + "Supports expectations" + ); + + Assert.objectContains(fixture, { + num: val => Assert.greater(val, 40), + }); + + // Failed expectations + Assert.throws( + () => + Assert.objectContains(fixture, { + foo: Expect.stringMatches(/bar/), + }), + /AssertionError/, + "Expect.stringMatches shouldn't match when the value is unexpected" + ); + Assert.throws( + () => + Assert.objectContains(fixture, { + foo: Expect.stringContains("bar"), + }), + /AssertionError/, + "Expect.stringContains shouldn't match when the value is unexpected" + ); + Assert.throws( + () => + Assert.objectContains(fixture, { + foo: Expect.number(), + }), + /AssertionError/, + "Expect.number shouldn't match when the value isn't a number" + ); + Assert.throws( + () => + Assert.objectContains(fixture, { + nested: Expect.objectContains({ + nestedFoo: "bar", + }), + }), + /AssertionError/, + "Expect.objectContains should throw when the value is unexpected" + ); + + Assert.throws( + () => + Assert.objectContains(fixture, { + num: val => Assert.less(val, 40), + }), + /AssertionError/, + "Expect.objectContains should throw when a function assertion fails" + ); +}); + +add_task(function test_type_expectations() { + const fixture = { + any: "foo", + string: "foo", + number: 42, + boolean: true, + bigint: 42n, + symbol: Symbol("foo"), + object: { foo: "foo" }, + function1() {}, + function2: () => {}, + }; + + Assert.objectContains(fixture, { + any: Expect.any(), + string: Expect.string(), + number: Expect.number(), + boolean: Expect.boolean(), + bigint: Expect.bigint(), + symbol: Expect.symbol(), + object: Expect.object(), + function1: Expect.function(), + function2: Expect.function(), + }); +}); diff --git a/tools/profiler/tests/xpcshell/test_enterjit_osr.js b/tools/profiler/tests/xpcshell/test_enterjit_osr.js new file mode 100644 index 0000000000..86845ddc76 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_enterjit_osr.js @@ -0,0 +1,52 @@ +// Check that the EnterJIT frame, added by the JIT trampoline and +// usable by a native unwinder to resume unwinding after encountering +// JIT code, is pushed as expected. +function run_test() { + // This test assumes that it's starting on an empty profiler stack. + // (Note that the other profiler tests also assume the profiler + // isn't already started.) + Assert.ok(!Services.profiler.IsActive()); + + const ms = 5; + Services.profiler.StartProfiler(10000, ms, ["js"]); + + function has_arbitrary_name_in_stack() { + // A frame for |arbitrary_name| has been pushed. Do a sequence of + // increasingly long spins until we get a sample. + var delayMS = 5; + while (1) { + info("loop: ms = " + delayMS); + const then = Date.now(); + do { + let n = 10000; + // eslint-disable-next-line no-empty + while (--n) {} // OSR happens here + // Spin in the hope of getting a sample. + } while (Date.now() - then < delayMS); + let profile = Services.profiler.getProfileData().threads[0]; + + // Go through all of the stacks, and search for this function name. + for (const sample of profile.samples.data) { + const stack = getInflatedStackLocations(profile, sample); + info(`The following stack was found: ${stack}`); + for (var i = 0; i < stack.length; i++) { + if (stack[i].match(/arbitrary_name/)) { + // This JS sample was correctly found. + return true; + } + } + } + + // Continue running this function with an increasingly long delay. + delayMS *= 2; + if (delayMS > 30000) { + return false; + } + } + } + Assert.ok( + has_arbitrary_name_in_stack(), + "A JS frame was found before the test timeout." + ); + Services.profiler.StopProfiler(); +} diff --git a/tools/profiler/tests/xpcshell/test_enterjit_osr_disabling.js b/tools/profiler/tests/xpcshell/test_enterjit_osr_disabling.js new file mode 100644 index 0000000000..558c9b0c3b --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_enterjit_osr_disabling.js @@ -0,0 +1,14 @@ +function run_test() { + Assert.ok(!Services.profiler.IsActive()); + + Services.profiler.StartProfiler(100, 10, ["js"]); + // The function is entered with the profiler enabled + (function () { + Services.profiler.StopProfiler(); + let n = 10000; + // eslint-disable-next-line no-empty + while (--n) {} // OSR happens here with the profiler disabled. + // An assertion will fail when this function returns, if the + // profiler stack was misbalanced. + })(); +} diff --git a/tools/profiler/tests/xpcshell/test_enterjit_osr_enabling.js b/tools/profiler/tests/xpcshell/test_enterjit_osr_enabling.js new file mode 100644 index 0000000000..313d939caf --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_enterjit_osr_enabling.js @@ -0,0 +1,14 @@ +function run_test() { + Assert.ok(!Services.profiler.IsActive()); + + // The function is entered with the profiler disabled. + (function () { + Services.profiler.StartProfiler(100, 10, ["js"]); + let n = 10000; + // eslint-disable-next-line no-empty + while (--n) {} // OSR happens here with the profiler enabled. + // An assertion will fail when this function returns, if the + // profiler stack was misbalanced. + })(); + Services.profiler.StopProfiler(); +} diff --git a/tools/profiler/tests/xpcshell/test_feature_fileioall.js b/tools/profiler/tests/xpcshell/test_feature_fileioall.js new file mode 100644 index 0000000000..e5ac040b98 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_fileioall.js @@ -0,0 +1,159 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + info( + "Test that off-main thread fileio is captured for a profiled thread, " + + "and that it will be sent to the main thread." + ); + const filename = "test_marker_fileio"; + const profile = await startProfilerAndTriggerFileIO({ + features: ["fileioall"], + threadsFilter: ["GeckoMain", "BgIOThreadPool"], + filename, + }); + + const threads = getThreads(profile); + const mainThread = threads.find(thread => thread.name === "GeckoMain"); + const mainThreadFileIO = getInflatedFileIOMarkers(mainThread, filename); + let backgroundThread; + let backgroundThreadFileIO; + for (const thread of threads) { + // Check for FileIO in any of the background threads. + if (thread.name.startsWith("BgIOThreadPool")) { + const markers = getInflatedFileIOMarkers(thread, filename); + if (markers.length) { + backgroundThread = thread; + backgroundThreadFileIO = markers; + break; + } + } + } + + info("Check all of the main thread FileIO markers."); + checkInflatedFileIOMarkers(mainThreadFileIO, filename); + for (const { data, name } of mainThreadFileIO) { + equal( + name, + "FileIO (non-main thread)", + "The markers from off main thread are labeled as such." + ); + equal( + data.threadId, + backgroundThread.tid, + "The main thread FileIO markers were all sent from the background thread." + ); + } + + info("Check all of the background thread FileIO markers."); + checkInflatedFileIOMarkers(backgroundThreadFileIO, filename); + for (const { data, name } of backgroundThreadFileIO) { + equal( + name, + "FileIO", + "The markers on the thread where they were generated just say FileIO" + ); + equal( + data.threadId, + undefined, + "The background thread FileIO correctly excludes the threadId." + ); + } +}); + +add_task(async () => { + info( + "Test that off-main thread fileio is captured for a thread that is not profiled, " + + "and that it will be sent to the main thread." + ); + const filename = "test_marker_fileio"; + const profile = await startProfilerAndTriggerFileIO({ + features: ["fileioall"], + threadsFilter: ["GeckoMain"], + filename, + }); + + const threads = getThreads(profile); + const mainThread = threads.find(thread => thread.name === "GeckoMain"); + const mainThreadFileIO = getInflatedFileIOMarkers(mainThread, filename); + + info("Check all of the main thread FileIO markers."); + checkInflatedFileIOMarkers(mainThreadFileIO, filename); + for (const { data, name } of mainThreadFileIO) { + equal( + name, + "FileIO (non-profiled thread)", + "The markers from off main thread are labeled as such." + ); + equal(typeof data.threadId, "number", "A thread ID is captured."); + } +}); + +/** + * @typedef {Object} TestConfig + * @prop {Array} features The list of profiler features + * @prop {string[]} threadsFilter The list of threads to profile + * @prop {string} filename A filename to trigger a write operation + */ + +/** + * Start the profiler and get FileIO markers. + * @param {TestConfig} + * @returns {Profile} + */ +async function startProfilerAndTriggerFileIO({ + features, + threadsFilter, + filename, +}) { + const entries = 10000; + const interval = 10; + await Services.profiler.StartProfiler( + entries, + interval, + features, + threadsFilter + ); + + const path = PathUtils.join(PathUtils.tempDir, filename); + + info(`Using a temporary file to test FileIO: ${path}`); + + if (fileExists(path)) { + console.warn( + "This test is triggering FileIO by writing to a file. However, the test found an " + + "existing file at the location it was trying to write to. This could happen " + + "because a previous run of the test failed to clean up after itself. This test " + + " will now clean up that file before running the test again." + ); + await removeFile(path); + } + + info("Write to the file, but do so using a background thread."); + + // IOUtils handles file operations using a background thread. + await IOUtils.write(path, new TextEncoder().encode("Test data.")); + const exists = await fileExists(path); + ok(exists, `Created temporary file at: ${path}`); + + info("Remove the file"); + await removeFile(path); + + return stopNowAndGetProfile(); +} + +async function fileExists(file) { + try { + let { type } = await IOUtils.stat(file); + return type === "regular"; + } catch (_error) { + return false; + } +} + +async function removeFile(file) { + await IOUtils.remove(file); + const exists = await fileExists(file); + ok(!exists, `Removed temporary file: ${file}`); +} diff --git a/tools/profiler/tests/xpcshell/test_feature_java.js b/tools/profiler/tests/xpcshell/test_feature_java.js new file mode 100644 index 0000000000..e2f6879c2b --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_java.js @@ -0,0 +1,31 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that Java capturing works as expected. + */ +add_task(async () => { + info("Test that Android Java sampler works as expected."); + const entries = 10000; + const interval = 1; + const threads = []; + const features = ["java"]; + + Services.profiler.StartProfiler(entries, interval, features, threads); + Assert.ok(Services.profiler.IsActive()); + + await captureAtLeastOneJsSample(); + + info( + "Stop the profiler and check that we have successfully captured a profile" + + " with the AndroidUI thread." + ); + const profile = await stopNowAndGetProfile(); + Assert.notEqual(profile, null); + const androidUiThread = profile.threads.find( + thread => thread.name == "AndroidUI (JVM)" + ); + Assert.notEqual(androidUiThread, null); + Assert.ok(!Services.profiler.IsActive()); +}); diff --git a/tools/profiler/tests/xpcshell/test_feature_js.js b/tools/profiler/tests/xpcshell/test_feature_js.js new file mode 100644 index 0000000000..a5949e4a0c --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_js.js @@ -0,0 +1,63 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that JS capturing works as expected. + */ +add_task(async () => { + const entries = 10000; + const interval = 1; + const threads = []; + const features = ["js"]; + + await Services.profiler.StartProfiler(entries, interval, features, threads); + + // Call the following to get a nice stack in the profiler: + // functionA -> functionB -> functionC -> captureAtLeastOneJsSample + const sampleIndex = await functionA(); + + const profile = await stopNowAndGetProfile(); + + const [thread] = profile.threads; + const { samples } = thread; + + const inflatedStackFrames = getInflatedStackLocations( + thread, + samples.data[sampleIndex] + ); + + expectStackToContain( + inflatedStackFrames, + [ + "(root)", + "js::RunScript", + // The following regexes match a string similar to: + // + // "functionA (/gecko/obj/_tests/xpcshell/tools/profiler/tests/xpcshell/test_feature_js.js:47:0)" + // or + // "functionA (test_feature_js.js:47:0)" + // + // this matches the script location + // | match the line number + // | | match the column number + // v v v + /^functionA \(.*test_feature_js\.js:\d+:\d+\)$/, + /^functionB \(.*test_feature_js\.js:\d+:\d+\)$/, + /^functionC \(.*test_feature_js\.js:\d+:\d+\)$/, + ], + "The stack contains a few frame labels, as well as the JS functions that we called." + ); +}); + +function functionA() { + return functionB(); +} + +function functionB() { + return functionC(); +} + +async function functionC() { + return captureAtLeastOneJsSample(); +} diff --git a/tools/profiler/tests/xpcshell/test_feature_mainthreadio.js b/tools/profiler/tests/xpcshell/test_feature_mainthreadio.js new file mode 100644 index 0000000000..8ff5c9206d --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_mainthreadio.js @@ -0,0 +1,122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const { FileUtils } = ChromeUtils.importESModule( + "resource://gre/modules/FileUtils.sys.mjs" +); + +/** + * Test that the IOInterposer is working correctly to capture main thread IO. + * + * This test should not run on release or beta, as the IOInterposer is wrapped in + * an ifdef. + */ +add_task(async () => { + { + const filename = "profiler-mainthreadio-test-firstrun"; + const { markers, schema } = await runProfilerWithFileIO( + ["mainthreadio"], + filename + ); + info("Check the FileIO markers when using the mainthreadio feature"); + checkInflatedFileIOMarkers(markers, filename); + + checkSchema(schema, { + name: "FileIO", + display: ["marker-chart", "marker-table", "timeline-fileio"], + data: [ + { + key: "operation", + label: "Operation", + format: "string", + searchable: true, + }, + { key: "source", label: "Source", format: "string", searchable: true }, + { + key: "filename", + label: "Filename", + format: "file-path", + searchable: true, + }, + { + key: "threadId", + label: "Thread ID", + format: "string", + searchable: true, + }, + ], + }); + } + + { + const filename = "profiler-mainthreadio-test-no-instrumentation"; + const { markers } = await runProfilerWithFileIO([], filename); + equal( + markers.length, + 0, + "No FileIO markers are found when the mainthreadio feature is not turned on " + + "in the profiler." + ); + } + + { + const filename = "profiler-mainthreadio-test-secondrun"; + const { markers } = await runProfilerWithFileIO(["mainthreadio"], filename); + info("Check the FileIO markers when re-starting the mainthreadio feature"); + checkInflatedFileIOMarkers(markers, filename); + } +}); + +/** + * Start the profiler and get FileIO markers and schema. + * + * @param {Array} features The list of profiler features + * @param {string} filename A filename to trigger a write operation + * @returns {{ + * markers: InflatedMarkers[]; + * schema: MarkerSchema; + * }} + */ +async function runProfilerWithFileIO(features, filename) { + const entries = 10000; + const interval = 10; + const threads = []; + await Services.profiler.StartProfiler(entries, interval, features, threads); + + info("Get the file"); + const file = FileUtils.getFile("TmpD", [filename]); + if (file.exists()) { + console.warn( + "This test is triggering FileIO by writing to a file. However, the test found an " + + "existing file at the location it was trying to write to. This could happen " + + "because a previous run of the test failed to clean up after itself. This test " + + " will now clean up that file before running the test again." + ); + file.remove(false); + } + + info( + "Generate file IO on the main thread using FileUtils.openSafeFileOutputStream." + ); + const outputStream = FileUtils.openSafeFileOutputStream(file); + + const data = "Test data."; + info("Write to the file"); + outputStream.write(data, data.length); + + info("Close the file"); + FileUtils.closeSafeFileOutputStream(outputStream); + + info("Remove the file"); + file.remove(false); + + const profile = await stopNowAndGetProfile(); + const mainThread = profile.threads.find(({ name }) => name === "GeckoMain"); + + const schema = getSchema(profile, "FileIO"); + + const markers = getInflatedFileIOMarkers(mainThread, filename); + + return { schema, markers }; +} diff --git a/tools/profiler/tests/xpcshell/test_feature_nativeallocations.js b/tools/profiler/tests/xpcshell/test_feature_nativeallocations.js new file mode 100644 index 0000000000..64398d7ef9 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_nativeallocations.js @@ -0,0 +1,158 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + if (!Services.profiler.GetFeatures().includes("nativeallocations")) { + Assert.ok( + true, + "Native allocations are not supported by this build, " + + "skip run the rest of the test." + ); + return; + } + + Assert.ok( + !Services.profiler.IsActive(), + "The profiler is not currently active" + ); + + info( + "Test that the profiler can install memory hooks and collect native allocation " + + "information in the marker payloads." + ); + { + info("Start the profiler."); + await startProfiler({ + // Only instrument the main thread. + threads: ["GeckoMain"], + features: ["js", "nativeallocations"], + }); + + info( + "Do some JS work for a little bit. This will increase the amount of allocations " + + "that take place." + ); + doWork(); + + info("Get the profile data and analyze it."); + const profile = await waitSamplingAndStopAndGetProfile(); + + const { + allocationPayloads, + unmatchedAllocations, + logAllocationsAndDeallocations, + } = getAllocationInformation(profile); + + Assert.greater( + allocationPayloads.length, + 0, + "Native allocation payloads were recorded for the parent process' main thread when " + + "the Native Allocation feature was turned on." + ); + + if (unmatchedAllocations.length !== 0) { + info( + "There were unmatched allocations. Log all of the allocations and " + + "deallocations in order to aid debugging." + ); + logAllocationsAndDeallocations(); + ok( + false, + "Found a deallocation that did not have a matching allocation site. " + + "This could happen if balanced allocations is broken, or if the the " + + "buffer size of this test was too small, and some markers ended up " + + "rolling off." + ); + } + + ok(true, "All deallocation sites had matching allocations."); + } + + info("Restart the profiler, to ensure that we get no more allocations."); + { + await startProfiler({ features: ["js"] }); + info("Do some work again."); + doWork(); + info("Wait for the periodic sampling."); + const profile = await waitSamplingAndStopAndGetProfile(); + const allocationPayloads = getPayloadsOfType( + profile.threads[0], + "Native allocation" + ); + + Assert.equal( + allocationPayloads.length, + 0, + "No native allocations were collected when the feature was disabled." + ); + } +}); + +function doWork() { + this.n = 0; + for (let i = 0; i < 1e5; i++) { + this.n += Math.random(); + } +} + +/** + * Extract the allocation payloads, and find the unmatched allocations. + */ +function getAllocationInformation(profile) { + // Get all of the allocation payloads. + const allocationPayloads = getPayloadsOfType( + profile.threads[0], + "Native allocation" + ); + + // Decide what is an allocation and deallocation. + const allocations = allocationPayloads.filter( + payload => ensureIsNumber(payload.size) >= 0 + ); + const deallocations = allocationPayloads.filter( + payload => ensureIsNumber(payload.size) < 0 + ); + + // Now determine the unmatched allocations by building a set + const allocationSites = new Set( + allocations.map(({ memoryAddress }) => memoryAddress) + ); + + const unmatchedAllocations = deallocations.filter( + ({ memoryAddress }) => !allocationSites.has(memoryAddress) + ); + + // Provide a helper to log out the allocations and deallocations on failure. + function logAllocationsAndDeallocations() { + for (const { memoryAddress } of allocations) { + console.log("Allocations", formatHex(memoryAddress)); + allocationSites.add(memoryAddress); + } + + for (const { memoryAddress } of deallocations) { + console.log("Deallocations", formatHex(memoryAddress)); + } + + for (const { memoryAddress } of unmatchedAllocations) { + console.log("Deallocation with no allocation", formatHex(memoryAddress)); + } + } + + return { + allocationPayloads, + unmatchedAllocations, + logAllocationsAndDeallocations, + }; +} + +function ensureIsNumber(value) { + if (typeof value !== "number") { + throw new Error(`Expected a number: ${value}`); + } + return value; +} + +function formatHex(number) { + return `0x${number.toString(16)}`; +} diff --git a/tools/profiler/tests/xpcshell/test_feature_stackwalking.js b/tools/profiler/tests/xpcshell/test_feature_stackwalking.js new file mode 100644 index 0000000000..aa0bc86547 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_feature_stackwalking.js @@ -0,0 +1,48 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Do a basic test to see if native frames are being collected for stackwalking. This + * test is fairly naive, as it does not attempt to check that these are valid symbols, + * only that some kind of stack walking is happening. It does this by making sure at + * least two native frames are collected. + */ +add_task(async () => { + const entries = 10000; + const interval = 1; + const threads = []; + const features = ["stackwalk"]; + + await Services.profiler.StartProfiler(entries, interval, features, threads); + const sampleIndex = await captureAtLeastOneJsSample(); + + const profile = await stopNowAndGetProfile(); + const [thread] = profile.threads; + const { samples } = thread; + + const inflatedStackFrames = getInflatedStackLocations( + thread, + samples.data[sampleIndex] + ); + const nativeStack = /^0x[0-9a-f]+$/; + + expectStackToContain( + inflatedStackFrames, + [ + "(root)", + // There are probably more native stacks here. + nativeStack, + nativeStack, + // Since this is an xpcshell test we know that JavaScript will run: + "js::RunScript", + // There are probably more native stacks here. + nativeStack, + nativeStack, + ], + "Expected native stacks to be interleaved between some frame labels. There should" + + "be more than one native stack if stack walking is working correctly. There " + + "is no attempt here to determine if the memory addresses point to the correct " + + "symbols" + ); +}); diff --git a/tools/profiler/tests/xpcshell/test_get_features.js b/tools/profiler/tests/xpcshell/test_get_features.js new file mode 100644 index 0000000000..e9bf0047c8 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_get_features.js @@ -0,0 +1,8 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + var profilerFeatures = Services.profiler.GetFeatures(); + Assert.ok(profilerFeatures != null); +} diff --git a/tools/profiler/tests/xpcshell/test_merged_stacks.js b/tools/profiler/tests/xpcshell/test_merged_stacks.js new file mode 100644 index 0000000000..7f851e8de9 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_merged_stacks.js @@ -0,0 +1,74 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we correctly merge the three stack types, JS, native, and frame labels. + */ +add_task(async () => { + const entries = 10000; + const interval = 1; + const threads = []; + const features = ["js", "stackwalk"]; + + await Services.profiler.StartProfiler(entries, interval, features, threads); + + // Call the following to get a nice stack in the profiler: + // functionA -> functionB -> functionC + const sampleIndex = await functionA(); + + const profile = await stopNowAndGetProfile(); + const [thread] = profile.threads; + const { samples } = thread; + + const inflatedStackFrames = getInflatedStackLocations( + thread, + samples.data[sampleIndex] + ); + + const nativeStack = /^0x[0-9a-f]+$/; + + expectStackToContain( + inflatedStackFrames, + [ + "(root)", + nativeStack, + nativeStack, + // There are more native stacks and frame labels here, but we know some execute + // and then the "js::RunScript" frame label runs. + "js::RunScript", + nativeStack, + nativeStack, + // The following regexes match a string similar to: + // + // "functionA (/gecko/obj/_tests/xpcshell/tools/profiler/tests/xpcshell/test_merged_stacks.js:47:0)" + // or + // "functionA (test_merged_stacks.js:47:0)" + // + // this matches the script location + // | match the line number + // | | match the column number + // v v v + /^functionA \(.*test_merged_stacks\.js:\d+:\d+\)$/, + /^functionB \(.*test_merged_stacks\.js:\d+:\d+\)$/, + /^functionC \(.*test_merged_stacks\.js:\d+:\d+\)$/, + // After the JS frames, then there are a bunch of arbitrary native stack frames + // that run. + nativeStack, + nativeStack, + ], + "The stack contains a few frame labels, as well as the JS functions that we called." + ); +}); + +async function functionA() { + return functionB(); +} + +async function functionB() { + return functionC(); +} + +async function functionC() { + return captureAtLeastOneJsSample(); +} diff --git a/tools/profiler/tests/xpcshell/test_pause.js b/tools/profiler/tests/xpcshell/test_pause.js new file mode 100644 index 0000000000..0e621fb19f --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_pause.js @@ -0,0 +1,126 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + Assert.ok(!Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + + let startPromise = Services.profiler.StartProfiler(1000, 10, []); + + // Default: Active and not paused. + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + await startPromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + // Pause everything, implicitly pauses sampling. + let pausePromise = Services.profiler.Pause(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await pausePromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // While fully paused, pause and resume sampling only, no expected changes. + let pauseSamplingPromise = Services.profiler.PauseSampling(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await pauseSamplingPromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + let resumeSamplingPromise = Services.profiler.ResumeSampling(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await resumeSamplingPromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // Resume everything. + let resumePromise = Services.profiler.Resume(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + await resumePromise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + // Pause sampling only. + let pauseSampling2Promise = Services.profiler.PauseSampling(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await pauseSampling2Promise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // While sampling is paused, pause everything. + let pause2Promise = Services.profiler.Pause(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await pause2Promise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // Resume, but sampling is still paused separately. + let resume2promise = Services.profiler.Resume(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + await resume2promise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(Services.profiler.IsSamplingPaused()); + + // Resume sampling only. + let resumeSampling2Promise = Services.profiler.ResumeSampling(); + + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + await resumeSampling2Promise; + Assert.ok(Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + let stopPromise = Services.profiler.StopProfiler(); + Assert.ok(!Services.profiler.IsActive()); + // Stopping is not pausing. + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); + + await stopPromise; + Assert.ok(!Services.profiler.IsActive()); + Assert.ok(!Services.profiler.IsPaused()); + Assert.ok(!Services.profiler.IsSamplingPaused()); +}); diff --git a/tools/profiler/tests/xpcshell/test_responsiveness.js b/tools/profiler/tests/xpcshell/test_responsiveness.js new file mode 100644 index 0000000000..5f57173090 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_responsiveness.js @@ -0,0 +1,50 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Test that we can measure non-zero event delays + */ + +add_task(async () => { + const entries = 10000; + const interval = 1; + const threads = []; + const features = []; + + await Services.profiler.StartProfiler(entries, interval, features, threads); + + await functionA(); + + const profile = await stopNowAndGetProfile(); + const [thread] = profile.threads; + const { samples } = thread; + const message = "eventDelay > 0 not found."; + let SAMPLE_STACK_SLOT = thread.samples.schema.eventDelay; + + for (let i = 0; i < samples.data.length; i++) { + if (samples.data[i][SAMPLE_STACK_SLOT] > 0) { + Assert.ok(true, message); + return; + } + } + Assert.ok(false, message); +}); + +function doSyncWork(milliseconds) { + const start = Date.now(); + while (true) { + this.n = 0; + for (let i = 0; i < 1e5; i++) { + this.n += Math.random(); + } + if (Date.now() - start > milliseconds) { + return; + } + } +} + +async function functionA() { + doSyncWork(100); + return captureAtLeastOneJsSample(); +} diff --git a/tools/profiler/tests/xpcshell/test_run.js b/tools/profiler/tests/xpcshell/test_run.js new file mode 100644 index 0000000000..0e30edfd4e --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_run.js @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + Assert.ok(!Services.profiler.IsActive()); + + Services.profiler.StartProfiler(1000, 10, []); + + Assert.ok(Services.profiler.IsActive()); + + do_test_pending(); + + do_timeout(1000, function wait() { + // Check text profile format + var profileStr = Services.profiler.GetProfile(); + Assert.ok(profileStr.length > 10); + + // check json profile format + var profileObj = Services.profiler.getProfileData(); + Assert.notEqual(profileObj, null); + Assert.notEqual(profileObj.threads, null); + // We capture memory counters by default only when jemalloc is turned + // on (and it isn't for ASAN), so unless we can conditionalize for ASAN + // here we can't check that we're capturing memory counter data. + Assert.notEqual(profileObj.counters, null); + Assert.notEqual(profileObj.memory, null); + Assert.ok(profileObj.threads.length >= 1); + Assert.notEqual(profileObj.threads[0].samples, null); + // NOTE: The number of samples will be empty since we + // don't have any labels in the xpcshell code + + Services.profiler.StopProfiler(); + Assert.ok(!Services.profiler.IsActive()); + do_test_finished(); + }); +} diff --git a/tools/profiler/tests/xpcshell/test_shared_library.js b/tools/profiler/tests/xpcshell/test_shared_library.js new file mode 100644 index 0000000000..e211ca642b --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_shared_library.js @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + var libs = Services.profiler.sharedLibraries; + + Assert.equal(typeof libs, "object"); + Assert.ok(Array.isArray(libs)); + Assert.equal(typeof libs, "object"); + Assert.ok(libs.length >= 1); + Assert.equal(typeof libs[0], "object"); + Assert.equal(typeof libs[0].name, "string"); + Assert.equal(typeof libs[0].path, "string"); + Assert.equal(typeof libs[0].debugName, "string"); + Assert.equal(typeof libs[0].debugPath, "string"); + Assert.equal(typeof libs[0].arch, "string"); + Assert.equal(typeof libs[0].start, "number"); + Assert.equal(typeof libs[0].end, "number"); + Assert.ok(libs[0].start <= libs[0].end); +} diff --git a/tools/profiler/tests/xpcshell/test_start.js b/tools/profiler/tests/xpcshell/test_start.js new file mode 100644 index 0000000000..c9ae135eb8 --- /dev/null +++ b/tools/profiler/tests/xpcshell/test_start.js @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +add_task(async () => { + Assert.ok(!Services.profiler.IsActive()); + + let startPromise = Services.profiler.StartProfiler(10, 100, []); + + Assert.ok(Services.profiler.IsActive()); + + await startPromise; + Assert.ok(Services.profiler.IsActive()); + + let stopPromise = Services.profiler.StopProfiler(); + + Assert.ok(!Services.profiler.IsActive()); + + await stopPromise; + Assert.ok(!Services.profiler.IsActive()); +}); diff --git a/tools/profiler/tests/xpcshell/xpcshell.ini b/tools/profiler/tests/xpcshell/xpcshell.ini new file mode 100644 index 0000000000..a7c461b4ac --- /dev/null +++ b/tools/profiler/tests/xpcshell/xpcshell.ini @@ -0,0 +1,72 @@ +[DEFAULT] +head = head.js +support-files = + ../shared-head.js + +[test_active_configuration.js] +skip-if = tsan # Intermittent timeouts, bug 1781449 +[test_addProfilerMarker.js] +[test_start.js] +skip-if = true +[test_get_features.js] +[test_responsiveness.js] +skip-if = tsan # Times out on TSan, bug 1612707 +[test_shared_library.js] +[test_run.js] +skip-if = true +[test_pause.js] +[test_enterjit_osr.js] +[test_enterjit_osr_disabling.js] +skip-if = !debug +[test_enterjit_osr_enabling.js] +skip-if = !debug +[test_asm.js] +[test_feature_mainthreadio.js] +skip-if = + release_or_beta + os == "win" && socketprocess_networking +[test_feature_fileioall.js] +skip-if = + release_or_beta + +# The sanitizer checks appears to overwrite our own memory hooks in xpcshell tests, +# and no allocation markers are gathered. Skip this test in that configuration. +[test_feature_nativeallocations.js] +skip-if = + os == "android" && verify # bug 1757528 + asan + tsan + socketprocess_networking + +# Native stackwalking is somewhat unreliable depending on the platform. +# +# We don't have frame pointers on macOS release and beta, so stack walking does not +# work. See Bug 1571216 for more details. +# +# Linux can be very unreliable when native stackwalking through JavaScript code. +# See Bug 1434402 for more details. +# +# For sanitizer builds, there were many intermittents, and we're not getting much +# additional coverage there, so it's better to be a bit more reliable. +[test_feature_stackwalking.js] +skip-if = + os == "mac" && release_or_beta + os == "linux" && release_or_beta && !debug + asan + tsan + +[test_feature_js.js] +skip-if = tsan # Times out on TSan, bug 1612707 + +# See the comment on test_feature_stackwalking.js +[test_merged_stacks.js] +skip-if = + os == "mac" && release_or_beta + os == "linux" && release_or_beta && !debug + asan + tsan + +[test_assertion_helper.js] +[test_feature_java.js] +skip-if = + os != "android" diff --git a/tools/quitter/README.md b/tools/quitter/README.md new file mode 100644 index 0000000000..515a3c952f --- /dev/null +++ b/tools/quitter/README.md @@ -0,0 +1,6 @@ +All code for this is part of the quitter project on github: +https://github.com/mozilla-extensions/quitter + +This is within the mozilla-extensions organization so that we can sign the .xpi. + +To update code, please submit a PR to the github repo. Once that is merged in, a JIRA ticket (ex [ADDONSOPS-194](https://mozilla-hub.atlassian.net/browse/ADDONSOPS-194)) is needed to resign the .xpi. diff --git a/tools/quitter/quitter@mozilla.org.xpi b/tools/quitter/quitter@mozilla.org.xpi new file mode 100644 index 0000000000..5f25ca3dad Binary files /dev/null and b/tools/quitter/quitter@mozilla.org.xpi differ diff --git a/tools/rb/README b/tools/rb/README new file mode 100644 index 0000000000..30418602b1 --- /dev/null +++ b/tools/rb/README @@ -0,0 +1,6 @@ +This is the Refcount Balancer. See +https://firefox-source-docs.mozilla.org/performance/memory/refcount_tracing_and_balancing.html +for documentation. + +Note that the `fix_stacks.py` script is used in several other places in the +repository. diff --git a/tools/rb/filter-log.pl b/tools/rb/filter-log.pl new file mode 100755 index 0000000000..4a1f66741b --- /dev/null +++ b/tools/rb/filter-log.pl @@ -0,0 +1,44 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Filter a refcount log to show only the entries for a single object. +# Useful when manually examining refcount logs containing multiple +# objects. + +use 5.004; +use strict; +use Getopt::Long; + +GetOptions("object=s"); + +$::opt_object || + die qq{ +usage: filter-log-for.pl < logfile + --object The address of the object to examine (required) +}; + +warn "object $::opt_object\n"; + +LINE: while (<>) { + next LINE if (! /^) { + print; + last CALLSITE if (/^$/); + } +} diff --git a/tools/rb/find-comptr-leakers.pl b/tools/rb/find-comptr-leakers.pl new file mode 100755 index 0000000000..925119935c --- /dev/null +++ b/tools/rb/find-comptr-leakers.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Script loosely based on Chris Waterson's find-leakers.pl and make-tree.pl + +use 5.004; +use strict; +use Getopt::Long; + +# GetOption will create $opt_object, so ignore the +# warning that gets spit out about those vbls. +GetOptions("object=s", "list", "help"); + +# use $::opt_help twice to eliminate warning... +($::opt_help) && ($::opt_help) && die qq{ +usage: find-comptr-leakers.pl < logfile + --object Examine only object + --list Only list leaked objects + --help This message :-) +}; + +if ($::opt_object) { + warn "Examining only object $::opt_object (THIS IS BROKEN)\n"; +} else { + warn "Examining all objects\n"; +} + +my %allocs = ( ); +my %counter; +my $id = 0; + +my $accumulating = 0; +my $savedata = 0; +my $class; +my $obj; +my $sno; +my $op; +my $cnt; +my $ptr; +my $strace; + +sub save_data { + # save the data + if ($op eq 'nsCOMPtrAddRef') { + push @{ $allocs{$sno}->{$ptr} }, [ +1, $strace ]; + } + elsif ($op eq 'nsCOMPtrRelease') { + push @{ $allocs{$sno}->{$ptr} }, [ -1, $strace ]; + my $sum = 0; + my @ptrallocs = @{ $allocs{$sno}->{$ptr} }; + foreach my $alloc (@ptrallocs) { + $sum += @$alloc[0]; + } + if ( $sum == 0 ) { + delete($allocs{$sno}{$ptr}); + } + } +} + +LINE: while (<>) { + if (/^{$comptr} }; + foreach my $alloc (@ptrallocs) { + $sum += @$alloc[0]; + } + print "Object ", $serial, " held by ", $comptr, " is ", $sum, " out of balance.\n"; + unless ($::opt_list) { + print "\n"; + foreach my $alloc (@ptrallocs) { + if (@$alloc[0] == +1) { + print "Put into nsCOMPtr at:\n"; + } elsif (@$alloc[0] == -1) { + print "Released from nsCOMPtr at:\n"; + } + print @$alloc[1]; # the stack trace + print "\n"; + } + print "\n\n"; + } + } +} + diff --git a/tools/rb/find_leakers.py b/tools/rb/find_leakers.py new file mode 100755 index 0000000000..9e9a37ac69 --- /dev/null +++ b/tools/rb/find_leakers.py @@ -0,0 +1,110 @@ +#!/usr/bin/python +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script processes a `refcount' log, and finds out if any object leaked. +# It simply goes through the log, finds `AddRef' or `Ctor' lines, and then +# sees if they `Release' or `Dtor'. If not, it reports them as leaks. +# Please see README file in the same directory. + +import sys + +import six + + +def print_output(allocation, obj_to_class): + """Formats and prints output.""" + items = [] + for ( + obj, + count, + ) in six.iteritems(allocation): + # Adding items to a list, so we can sort them. + items.append((obj, count)) + # Sorting by count. + items.sort(key=lambda item: item[1]) + + for ( + obj, + count, + ) in items: + print( + "{obj} ({count}) @ {class_name}".format( + obj=obj, count=count, class_name=obj_to_class[obj] + ) + ) + + +def process_log(log_lines): + """Process through the log lines, and print out the result. + + @param log_lines: List of strings. + """ + allocation = {} + class_count = {} + obj_to_class = {} + + for log_line in log_lines: + if not log_line.startswith("<"): + continue + + (class_name, obj, ignore, operation, count,) = log_line.strip("\r\n").split( + " " + )[:5] + + # for AddRef/Release `count' is the refcount, + # for Ctor/Dtor it's the size. + + if (operation == "AddRef" and count == "1") or operation == "Ctor": + # Examples: + # 0x01AFD3B8 1 AddRef 1 + # 0x08880BD0 8 Ctor (20) + class_count[class_name] = class_count.setdefault(class_name, 0) + 1 + allocation[obj] = class_count[class_name] + obj_to_class[obj] = class_name + + elif (operation == "Release" and count == "0") or operation == "Dtor": + # Examples: + # 0x01AFD3B8 1 Release 0 + # 0x08880BD0 8 Dtor (20) + if obj not in allocation: + print( + "An object was released that wasn't allocated!", + ) + print(obj, "@", class_name) + else: + allocation.pop(obj) + obj_to_class.pop(obj) + + # Printing out the result. + print_output(allocation, obj_to_class) + + +def print_usage(): + print("") + print("Usage: find-leakers.py [log-file]") + print("") + print("If `log-file' provided, it will read that as the input log.") + print("Else, it will read the stdin as the input log.") + print("") + + +def main(): + """Main method of the script.""" + if len(sys.argv) == 1: + # Reading log from stdin. + process_log(sys.stdin.readlines()) + elif len(sys.argv) == 2: + # Reading log from file. + with open(sys.argv[1], "r") as log_file: + log_lines = log_file.readlines() + process_log(log_lines) + else: + print("ERROR: Invalid number of arguments") + print_usage() + + +if __name__ == "__main__": + main() diff --git a/tools/rb/fix_stacks.py b/tools/rb/fix_stacks.py new file mode 100755 index 0000000000..f30aa9944a --- /dev/null +++ b/tools/rb/fix_stacks.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# vim:sw=4:ts=4:et: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script uses `fix-stacks` to post-process the entries produced by +# MozFormatCodeAddress(). + +import atexit +import os +import platform +import re +import sys +from subprocess import PIPE, Popen + +# Matches lines produced by MozFormatCodeAddress(), e.g. +# `#01: ???[tests/example +0x43a0]`. +line_re = re.compile("#\d+: .+\[.+ \+0x[0-9A-Fa-f]+\]") + +fix_stacks = None + + +def autobootstrap(): + import buildconfig + from mozbuild.configure import ConfigureSandbox + + sandbox = ConfigureSandbox( + {}, + argv=[ + "configure", + "--help", + "--host={}".format(buildconfig.substs["HOST_ALIAS"]), + ], + ) + moz_configure = os.path.join(buildconfig.topsrcdir, "build", "moz.configure") + sandbox.include_file(os.path.join(moz_configure, "init.configure")) + # bootstrap_search_path_order has a dependency on developer_options, which + # is not defined in init.configure. Its value doesn't matter for us, though. + sandbox["developer_options"] = sandbox["always"] + sandbox.include_file(os.path.join(moz_configure, "bootstrap.configure")) + # Expand the `bootstrap_path` template for "fix-stacks", and execute the + # expanded function via `_value_for`, which will trigger autobootstrap. + sandbox._value_for(sandbox["bootstrap_path"]("fix-stacks")) + + +def initFixStacks(jsonMode, slowWarning, breakpadSymsDir, hide_errors): + # Look in MOZ_FETCHES_DIR (for automation), then in MOZBUILD_STATE_PATH + # (for a local build where the user has that set), then in ~/.mozbuild + # (for a local build with default settings). + base = os.environ.get( + "MOZ_FETCHES_DIR", + os.environ.get("MOZBUILD_STATE_PATH", os.path.expanduser("~/.mozbuild")), + ) + fix_stacks_exe = base + "/fix-stacks/fix-stacks" + if platform.system() == "Windows": + fix_stacks_exe = fix_stacks_exe + ".exe" + + if not (os.path.isfile(fix_stacks_exe) and os.access(fix_stacks_exe, os.X_OK)): + try: + autobootstrap() + except ImportError: + # We're out-of-tree (e.g. tests tasks on CI) and can't autobootstrap + # (we shouldn't anyways). + pass + + if not (os.path.isfile(fix_stacks_exe) and os.access(fix_stacks_exe, os.X_OK)): + raise Exception("cannot find `fix-stacks`; please run `./mach bootstrap`") + + args = [fix_stacks_exe] + if jsonMode: + args.append("-j") + if breakpadSymsDir: + args.append("-b") + args.append(breakpadSymsDir) + + # Sometimes we need to prevent errors from going to stderr. + stderr = open(os.devnull) if hide_errors else None + + global fix_stacks + fix_stacks = Popen( + args, stdin=PIPE, stdout=PIPE, stderr=stderr, universal_newlines=True + ) + + # Shut down the fix_stacks process on exit. We use `terminate()` + # because it is more forceful than `wait()`, and the Python docs warn + # about possible deadlocks with `wait()`. + def cleanup(fix_stacks): + for fn in [fix_stacks.stdin.close, fix_stacks.terminate]: + try: + fn() + except OSError: + pass + + atexit.register(cleanup, fix_stacks) + + if slowWarning: + print( + "Initializing stack-fixing for the first stack frame, this may take a while..." + ) + + +def fixSymbols( + line, jsonMode=False, slowWarning=False, breakpadSymsDir=None, hide_errors=False +): + is_bytes = isinstance(line, bytes) + line_str = line.decode("utf-8") if is_bytes else line + if line_re.search(line_str) is None: + return line + + if not fix_stacks: + initFixStacks(jsonMode, slowWarning, breakpadSymsDir, hide_errors) + + # Sometimes `line` is lacking a trailing newline. If we pass such a `line` + # to `fix-stacks` it will wait until it receives a newline, causing this + # script to hang. So we add a newline if one is missing and then remove it + # from the output. + is_missing_newline = not line_str.endswith("\n") + if is_missing_newline: + line_str = line_str + "\n" + fix_stacks.stdin.write(line_str) + fix_stacks.stdin.flush() + out = fix_stacks.stdout.readline() + if is_missing_newline: + out = out[:-1] + + if is_bytes and not isinstance(out, bytes): + out = out.encode("utf-8") + return out + + +if __name__ == "__main__": + bpsyms = os.environ.get("BREAKPAD_SYMBOLS_PATH", None) + for line in sys.stdin: + sys.stdout.write(fixSymbols(line, breakpadSymsDir=bpsyms)) diff --git a/tools/rb/make-tree.pl b/tools/rb/make-tree.pl new file mode 100755 index 0000000000..04f0d85341 --- /dev/null +++ b/tools/rb/make-tree.pl @@ -0,0 +1,303 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use 5.004; +use strict; +use Getopt::Long; + +$::opt_prune_depth = 0; +$::opt_subtree_size = 0; +$::opt_reverse = 0; + +# GetOption will create $opt_object & $opt_exclude, so ignore the +# warning that gets spit out about those vbls. +GetOptions("object=s", "exclude=s", "comptrs=s", "ignore-balanced", "subtree-size=i", "prune-depth=i", + "collapse-to-method", "collapse-to-class", "old-style", "reverse"); + +$::opt_object || + die qq{ +usage: leak.pl < logfile + --object The address of the object to examine (required) + --exclude Exclude routines listed in + --comptrs Subtract all the data in the balanced COMPtr log + --ignore-balanced Ignore balanced subtrees + --subtree-size Print subtrees with more than nodes separately + --prune-depth Prune the tree to + --collapse-to-method Aggregate data by method + --collapse-to-class Aggregate data by class (subsumes --collapse-to-method) + --reverse Reverse call stacks, showing leaves first + --old-style Old-style formatting +}; + +$::opt_prune_depth = 0 if $::opt_prune_depth < 0; +$::opt_subtree_size = 0 if $::opt_subtree_size < 0; + +warn "object $::opt_object\n"; +warn "ignoring balanced subtrees\n" if $::opt_ignore_balanced; +warn "prune depth $::opt_prune_depth\n" if $::opt_prune_depth; +warn "collapsing to class\n" if $::opt_collapse_to_class; +warn "collapsing to method\n" if $::opt_collapse_to_method && !$::opt_collapse_to_class; +warn "reversing call stacks\n" if $::opt_reverse; + + +# The 'excludes' are functions that, if detected in a particular call +# stack, will cause the _entire_ call stack to be ignored. You might, +# for example, explicitly exclude two functions that have a matching +# AddRef/Release pair. + +my %excludes; + +if ($::opt_exclude) { + open(EXCLUDE, "<".$::opt_exclude) + || die "unable to open $::opt_exclude"; + + while () { + chomp $_; + warn "excluding $_\n"; + $excludes{$_} = 1; + } +} + +# Each entry in the tree rooted by callGraphRoot contains the following: +# #name# This call's name+offset string +# #refcount# The net reference count of this call +# #label# The label used for this subtree; only defined for labeled nodes +# #children# List of children in alphabetical order +# zero or more children indexed by method name+offset strings. + +my $callGraphRoot; +$callGraphRoot = { '#name#' => '.root', '#refcount#' => 'n/a' }; + +# The 'imbalance' is a gross count of how balanced a particular +# callsite is. It is used to prune away callsites that are detected to +# be balanced; that is, that have matching AddRef/Release() pairs. + +my %imbalance; +$imbalance{'.root'} = 'n/a'; + +# The main read loop. + +sub read_data($$$) { + my ($INFILE, $plus, $minus) = @_; + + LINE: while (<$INFILE>) { + next LINE if (! /^) { + chomp; + last CALLSITE if (/^$/); + $_ =~ s/#\d+: /#00: /; # replace frame number with 0 + $stack[++$#stack] = $_; + } + + # Reverse the remaining fields to produce the call stack, with the + # oldest frame at the front of the array. + if (! $::opt_reverse) { + @stack = reverse(@stack); + } + + my $call; + + # If any of the functions in the stack are supposed to be excluded, + # march on to the next line. + foreach $call (@stack) { + next LINE if exists($excludes{$call}); + } + + + # Add the callstack as a path through the call graph, updating + # refcounts at each node. + + my $caller = $callGraphRoot; + + foreach $call (@stack) { + + # Chop the method offset if we're 'collapsing to method' or + # 'collapsing to class'. + $call =~ s/\+0x.*$//g if ($::opt_collapse_to_method || $::opt_collapse_to_class); + + # Chop the method name if we're 'collapsing to class'. + $call =~ s/::.*$//g if ($::opt_collapse_to_class); + + my $site = $caller->{$call}; + if (!$site) { + # This is the first time we've seen this callsite. Add a + # new entry to the call tree. + + $site = { '#name#' => $call, '#refcount#' => 0 }; + $caller->{$call} = $site; + } + + if ($op eq $plus) { + ++($site->{'#refcount#'}); + ++($imbalance{$call}); + } elsif ($op eq $minus) { + --($site->{'#refcount#'}); + --($imbalance{$call}); + } else { + die "Bad operation $op"; + } + + $caller = $site; + } + } +} + +read_data(*STDIN, "AddRef", "Release"); + +if ($::opt_comptrs) { + warn "Subtracting comptr log ". $::opt_comptrs . "\n"; + open(COMPTRS, "<".$::opt_comptrs) + || die "unable to open $::opt_comptrs"; + + # read backwards to subtract + read_data(*COMPTRS, "nsCOMPtrRelease", "nsCOMPtrAddRef"); +} + +sub num_alpha { + my ($aN, $aS, $bN, $bS); + ($aN, $aS) = ($1, $2) if $a =~ /^(\d+) (.+)$/; + ($bN, $bS) = ($1, $2) if $b =~ /^(\d+) (.+)$/; + return $a cmp $b unless defined $aN && defined $bN; + return $aN <=> $bN unless $aN == $bN; + return $aS cmp $bS; +} + +# Given a subtree and its nesting level, return true if that subtree should be pruned. +# If it shouldn't be pruned, destructively attempt to prune its children. +# Also compute the #children# properties of unpruned nodes. +sub prune($$) { + my ($site, $nest) = @_; + + # If they want us to prune the tree's depth, do so here. + return 1 if ($::opt_prune_depth && $nest >= $::opt_prune_depth); + + # If the subtree is balanced, ignore it. + return 1 if ($::opt_ignore_balanced && !$site->{'#refcount#'}); + + my $name = $site->{'#name#'}; + + # If the symbol isn't imbalanced, then prune here (and warn) + if ($::opt_ignore_balanced && !$imbalance{$name}) { + warn "discarding " . $name . "\n"; +# return 1; + } + + my @children; + foreach my $child (sort num_alpha keys(%$site)) { + if (substr($child, 0, 1) ne '#') { + if (prune($site->{$child}, $nest + 1)) { + delete $site->{$child}; + } else { + push @children, $site->{$child}; + } + } + } + $site->{'#children#'} = \@children; + return 0; +} + + +# Compute the #label# properties of this subtree. +# Return the subtree's number of nodes, not counting nodes reachable +# through a labeled node. +sub createLabels($) { + my ($site) = @_; + my @children = @{$site->{'#children#'}}; + my $nChildren = @children; + my $nDescendants = 0; + + foreach my $child (@children) { + my $childDescendants = createLabels($child); + if ($nChildren > 1 && $childDescendants > $::opt_subtree_size) { + die "Internal error" if defined($child->{'#label#'}); + $child->{'#label#'} = "__label__"; + $childDescendants = 1; + } + $nDescendants += $childDescendants; + } + return $nDescendants + 1; +} + + +my $nextLabel = 0; +my @labeledSubtrees; + +sub list($$$$$) { + my ($site, $nest, $nestStr, $childrenLeft, $root) = @_; + my $label = !$root && $site->{'#label#'}; + + # Assign a unique number to the label. + if ($label) { + die unless $label eq "__label__"; + $label = "__" . ++$nextLabel . "__"; + $site->{'#label#'} = $label; + push @labeledSubtrees, $site; + } + + print $nestStr; + if ($::opt_old_style) { + print $label, " " if $label; + print $site->{'#name#'}, ": bal=", $site->{'#refcount#'}, "\n"; + } else { + my $refcount = $site->{'#refcount#'}; + my $l = 8 - length $refcount; + $l = 1 if $l < 1; + print $refcount, " " x $l; + print $label, " " if $label; + print $site->{'#name#'}, "\n"; + } + + $nestStr .= $childrenLeft && !$::opt_old_style ? "| " : " "; + if (!$label) { + my @children = @{$site->{'#children#'}}; + $childrenLeft = @children; + foreach my $child (@children) { + $childrenLeft--; + list($child, $nest + 1, $nestStr, $childrenLeft); + } + } +} + + +if (!prune($callGraphRoot, 0)) { + createLabels $callGraphRoot if ($::opt_subtree_size); + list $callGraphRoot, 0, "", 0, 1; + while (@labeledSubtrees) { + my $labeledSubtree = shift @labeledSubtrees; + print "\n------------------------------\n", +$labeledSubtree->{'#label#'}, "\n"; + list $labeledSubtree, 0, "", 0, 1; + } + print "\n------------------------------\n" if @labeledSubtrees; +} + +print qq{ +Imbalance +--------- +}; + +foreach my $call (sort num_alpha keys(%imbalance)) { + print $call . " " . $imbalance{$call} . "\n"; +} + diff --git a/tools/rewriting/Generated.txt b/tools/rewriting/Generated.txt new file mode 100644 index 0000000000..ba036abcf1 --- /dev/null +++ b/tools/rewriting/Generated.txt @@ -0,0 +1,31 @@ +.gradle/ +browser/components/newtab/content-src/asrouter/schemas/BackgroundTaskMessagingExperiment.schema.json +browser/components/newtab/content-src/asrouter/schemas/MessagingExperiment.schema.json +browser/components/newtab/logs/ +browser/components/newtab/node_modules/ +browser/components/storybook/storybook-static/ +browser/locales/l10n-changesets.json +browser/locales/l10n-onchange-changesets.json +devtools/client/aboutdebugging/test/jest/node_modules/ +devtools/client/application/test/components/node_modules/ +devtools/client/debugger/node_modules/ +dom/tests/ajax/jquery/ +dom/tests/ajax/mochikit/ +intl/components/src/UnicodeScriptCodes.h +intl/unicharutil/util/nsSpecialCasingData.cpp +intl/unicharutil/util/nsUnicodePropertyData.cpp +mobile/locales/l10n-changesets.json +mobile/locales/l10n-onchange-changesets.json +node_modules/ +security/manager/tools/KnownRootHashes.json +security/manager/tools/PreloadedHPKPins.json +services/settings/dumps/ +testing/talos/talos/tests/devtools/addon/content/pages/custom/debugger/static/js/minified.js +toolkit/components/nimbus/schemas/NimbusExperiment.schema.json +toolkit/components/pdfjs/content/PdfJsDefaultPreferences.sys.mjs +toolkit/components/uniffi-js/UniFFIGeneratedScaffolding.cpp +toolkit/components/uniffi-js/UniFFIFixtureScaffolding.cpp +toolkit/components/uniffi-bindgen-gecko-js/fixtures/generated +tools/browsertime/package.json +tools/browsertime/package-lock.json +try_task_config.json diff --git a/tools/rewriting/ThirdPartyPaths.txt b/tools/rewriting/ThirdPartyPaths.txt new file mode 100644 index 0000000000..8f581671d7 --- /dev/null +++ b/tools/rewriting/ThirdPartyPaths.txt @@ -0,0 +1,195 @@ +browser/components/newtab/vendor/ +browser/components/pocket/content/panels/css/normalize.scss +browser/components/pocket/content/panels/js/vendor/ +browser/components/storybook/node_modules/ +browser/extensions/formautofill/content/third-party/ +browser/extensions/formautofill/test/fixtures/third_party/ +browser/extensions/screenshots/build/raven.js +devtools/client/inspector/markup/test/lib_* +devtools/client/jsonview/lib/require.js +devtools/client/shared/build/babel.js +devtools/client/shared/source-map/ +devtools/client/shared/sourceeditor/codemirror/ +devtools/client/shared/sourceeditor/test/cm_mode_ruby.js +devtools/client/shared/sourceeditor/test/codemirror/ +devtools/client/shared/vendor/ +devtools/client/inspector/markup/test/helper_diff.js +devtools/shared/acorn/ +devtools/shared/compatibility/dataset/css-properties.json +devtools/shared/heapsnapshot/CoreDump.pb.cc +devtools/shared/heapsnapshot/CoreDump.pb.h +devtools/shared/jsbeautify/ +devtools/shared/node-properties/ +devtools/shared/qrcode/decoder/ +devtools/shared/qrcode/encoder/ +devtools/shared/sprintfjs/ +devtools/shared/storage/vendor/ +dom/canvas/test/webgl-conf/checkout/ +dom/imptests/ +dom/media/gmp/rlz/ +dom/media/gmp/widevine-adapter/content_decryption_module_export.h +dom/media/gmp/widevine-adapter/content_decryption_module_ext.h +dom/media/gmp/widevine-adapter/content_decryption_module.h +dom/media/gmp/widevine-adapter/content_decryption_module_proxy.h +dom/media/platforms/ffmpeg/ffmpeg57/ +dom/media/platforms/ffmpeg/ffmpeg58/ +dom/media/platforms/ffmpeg/ffmpeg59/ +dom/media/platforms/ffmpeg/ffmpeg60/ +dom/media/platforms/ffmpeg/libav53/ +dom/media/platforms/ffmpeg/libav54/ +dom/media/platforms/ffmpeg/libav55/ +dom/media/webaudio/test/blink/ +dom/media/webrtc/tests/mochitests/helpers_from_wpt/sdp.js +dom/media/webrtc/transport/third_party/ +dom/media/webspeech/recognition/endpointer.cc +dom/media/webspeech/recognition/endpointer.h +dom/media/webspeech/recognition/energy_endpointer.cc +dom/media/webspeech/recognition/energy_endpointer.h +dom/media/webspeech/recognition/energy_endpointer_params.cc +dom/media/webspeech/recognition/energy_endpointer_params.h +dom/media/webvtt/vtt.sys.mjs +dom/tests/mochitest/ajax/ +dom/tests/mochitest/dom-level1-core/ +dom/tests/mochitest/dom-level2-core/ +dom/tests/mochitest/dom-level2-html/ +dom/u2f/tests/pkijs/ +dom/webauthn/cbor-cpp/ +dom/webauthn/tests/pkijs/ +dom/webgpu/tests/cts/checkout/ +editor/libeditor/tests/browserscope/lib/richtext/ +editor/libeditor/tests/browserscope/lib/richtext2/ +extensions/spellcheck/hunspell/src/ +function2/ +gfx/angle/checkout/ +gfx/cairo/ +gfx/graphite2/ +gfx/harfbuzz/ +gfx/ots/ +gfx/qcms/ +gfx/sfntly/ +gfx/skia/ +gfx/vr/service/openvr/ +gfx/vr/service/openvr/headers/openvr.h +gfx/vr/service/openvr/src/README +gfx/vr/service/openvr/src/dirtools_public.cpp +gfx/vr/service/openvr/src/dirtools_public.h +gfx/vr/service/openvr/src/envvartools_public.cpp +gfx/vr/service/openvr/src/envvartools_public.h +gfx/vr/service/openvr/src/hmderrors_public.cpp +gfx/vr/service/openvr/src/hmderrors_public.h +gfx/vr/service/openvr/src/ivrclientcore.h +gfx/vr/service/openvr/src/openvr_api_public.cpp +gfx/vr/service/openvr/src/pathtools_public.cpp +gfx/vr/service/openvr/src/pathtools_public.h +gfx/vr/service/openvr/src/sharedlibtools_public.cpp +gfx/vr/service/openvr/src/sharedlibtools_public.h +gfx/vr/service/openvr/src/strtools_public.cpp +gfx/vr/service/openvr/src/strtools_public.h +gfx/vr/service/openvr/src/vrpathregistry_public.cpp +gfx/vr/service/openvr/src/vrpathregistry_public.h +gfx/wr/ +gfx/ycbcr/ +intl/icu/ +ipc/chromium/src/third_party/ +js/src/ctypes/libffi/ +js/src/dtoa.c +js/src/editline/ +js/src/jit/arm64/vixl/ +js/src/octane/ +js/src/vtune/disable_warnings.h +js/src/vtune/ittnotify_config.h +js/src/vtune/ittnotify.h +js/src/vtune/ittnotify_static.c +js/src/vtune/ittnotify_static.h +js/src/vtune/ittnotify_types.h +js/src/vtune/jitprofiling.c +js/src/vtune/jitprofiling.h +js/src/vtune/legacy/ +js/src/zydis/ +layout/docs/css-gap-decorations/ +media/ffvpx/ +media/kiss_fft/ +media/libaom/ +media/libcubeb/ +media/libdav1d/ +media/libjpeg/ +media/libmkv/ +media/libnestegg/ +media/libogg/ +media/libopus/ +media/libpng/ +media/libsoundtouch/ +media/libspeex_resampler/ +media/libtheora/ +media/libtremor/ +media/libvorbis/ +media/libvpx/ +media/libwebp/ +media/libyuv/ +media/mozva/va +media/mp4parse-rust/ +media/openmax_dl/ +media/openmax_il/ +media/webrtc/signaling/gtest/MockCall.h +mfbt/double-conversion/double-conversion/ +mfbt/lz4/.* +mobile/android/exoplayer2/ +modules/brotli/ +modules/fdlibm/ +modules/freetype2/ +modules/woff2/ +modules/xz-embedded/ +modules/zlib/ +mozglue/misc/decimal/ +mozglue/tests/glibc_printf_tests/ +netwerk/dns/nsIDNKitInterface.h +netwerk/sctp/src/ +netwerk/srtp/src/ +nsprpub/ +other-licenses/ +parser/expat/ +remote/cdp/test/browser/chrome-remote-interface.js +remote/test/puppeteer/ +security/manager/tools/log_list.json +security/nss/ +security/sandbox/chromium/ +security/sandbox/chromium-shim/ +testing/gtest/gmock/ +testing/gtest/gtest/ +testing/mochitest/MochiKit/ +testing/mochitest/pywebsocket3/ +testing/mochitest/tests/MochiKit-1.4.2/ +testing/modules/sinon-7.2.7.js +testing/mozbase/mozproxy/mozproxy/backends/mitm/scripts/catapult/ +testing/talos/talos/tests/dromaeo/ +testing/talos/talos/tests/kraken/ +testing/talos/talos/tests/v8_7/ +testing/web-platform/tests/resources/webidl2/ +testing/web-platform/tests/tools/third_party/ +testing/web-platform/mozilla/tests/webgpu/ +testing/xpcshell/dns-packet/ +testing/xpcshell/node_ip/ +testing/xpcshell/node-http2/ +testing/xpcshell/node-ws/ +third_party/ +toolkit/components/certviewer/content/vendor/ +toolkit/components/jsoncpp/ +toolkit/components/normandy/vendor/ +toolkit/components/passwordmgr/PasswordRulesParser.sys.mjs +toolkit/components/protobuf/ +toolkit/components/translation/cld2/ +toolkit/components/translations/bergamot-translator +toolkit/components/translations/fasttext/fasttext.js +toolkit/components/translations/fasttext/fasttext_wasm.js +toolkit/components/url-classifier/chromium/ +toolkit/components/utils/mozjexl.js +toolkit/components/viaduct/fetch_msg_types.pb.cc +toolkit/components/viaduct/fetch_msg_types.pb.h +toolkit/content/widgets/vendor/lit.all.mjs +toolkit/crashreporter/breakpad-client/ +toolkit/crashreporter/google-breakpad/ +tools/fuzzing/libfuzzer/ +tools/profiler/core/vtune/ +xpcom/build/mach_override.c +xpcom/build/mach_override.h +xpcom/io/crc32c.c diff --git a/tools/rusttests/app.mozbuild b/tools/rusttests/app.mozbuild new file mode 100644 index 0000000000..9a4a225cf5 --- /dev/null +++ b/tools/rusttests/app.mozbuild @@ -0,0 +1,5 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +include('/toolkit/toolkit.mozbuild') diff --git a/tools/rusttests/config/mozconfigs/common b/tools/rusttests/config/mozconfigs/common new file mode 100644 index 0000000000..cac5b57ce9 --- /dev/null +++ b/tools/rusttests/config/mozconfigs/common @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file is included by all "tools/rusttests" mozconfigs + +MOZ_AUTOMATION_BUILD_SYMBOLS=0 +MOZ_AUTOMATION_PACKAGE=0 +MOZ_AUTOMATION_PACKAGE_GENERATED_SOURCES=0 +MOZ_AUTOMATION_UPLOAD=0 +MOZ_AUTOMATION_CHECK=0 +ac_add_options --enable-project=tools/rusttests +. "$topsrcdir/build/mozconfig.common" +. "$topsrcdir/build/mozconfig.common.override" + +unset ENABLE_CLANG_PLUGIN + +# Test geckodriver, which isn't built as part of the build jobs +ac_add_options --enable-geckodriver diff --git a/tools/rusttests/config/mozconfigs/linux32/rusttests b/tools/rusttests/config/mozconfigs/linux32/rusttests new file mode 100644 index 0000000000..fa52698f9e --- /dev/null +++ b/tools/rusttests/config/mozconfigs/linux32/rusttests @@ -0,0 +1,2 @@ +. "$topsrcdir/build/unix/mozconfig.linux32" +. "$topsrcdir/tools/rusttests/config/mozconfigs/common" diff --git a/tools/rusttests/config/mozconfigs/linux32/rusttests-debug b/tools/rusttests/config/mozconfigs/linux32/rusttests-debug new file mode 100644 index 0000000000..b207ac6b4b --- /dev/null +++ b/tools/rusttests/config/mozconfigs/linux32/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/linux32/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/linux64/rusttests b/tools/rusttests/config/mozconfigs/linux64/rusttests new file mode 100644 index 0000000000..64d502087b --- /dev/null +++ b/tools/rusttests/config/mozconfigs/linux64/rusttests @@ -0,0 +1,2 @@ +. "$topsrcdir/build/unix/mozconfig.linux" +. "$topsrcdir/tools/rusttests/config/mozconfigs/common" diff --git a/tools/rusttests/config/mozconfigs/linux64/rusttests-debug b/tools/rusttests/config/mozconfigs/linux64/rusttests-debug new file mode 100644 index 0000000000..ef7157e53b --- /dev/null +++ b/tools/rusttests/config/mozconfigs/linux64/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/linux64/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/macosx64/rusttests b/tools/rusttests/config/mozconfigs/macosx64/rusttests new file mode 100644 index 0000000000..f70026f5fd --- /dev/null +++ b/tools/rusttests/config/mozconfigs/macosx64/rusttests @@ -0,0 +1,2 @@ +. "$topsrcdir/build/macosx/mozconfig.common" +. "$topsrcdir/tools/rusttests/config/mozconfigs/common" diff --git a/tools/rusttests/config/mozconfigs/macosx64/rusttests-debug b/tools/rusttests/config/mozconfigs/macosx64/rusttests-debug new file mode 100644 index 0000000000..760d51266e --- /dev/null +++ b/tools/rusttests/config/mozconfigs/macosx64/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/macosx64/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/win32/rusttests b/tools/rusttests/config/mozconfigs/win32/rusttests new file mode 100644 index 0000000000..d953f9c0df --- /dev/null +++ b/tools/rusttests/config/mozconfigs/win32/rusttests @@ -0,0 +1,3 @@ +ac_add_options --target=i686-pc-windows-msvc +. "$topsrcdir/build/win32/mozconfig.vs-latest" +. "$topsrcdir/tools/rusttests/config/mozconfigs/windows-common" diff --git a/tools/rusttests/config/mozconfigs/win32/rusttests-debug b/tools/rusttests/config/mozconfigs/win32/rusttests-debug new file mode 100644 index 0000000000..046d2d0a1d --- /dev/null +++ b/tools/rusttests/config/mozconfigs/win32/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/win32/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/win64/rusttests b/tools/rusttests/config/mozconfigs/win64/rusttests new file mode 100644 index 0000000000..ed81e3b767 --- /dev/null +++ b/tools/rusttests/config/mozconfigs/win64/rusttests @@ -0,0 +1,3 @@ +ac_add_options --target=x86_64-pc-windows-msvc +. "$topsrcdir/build/win64/mozconfig.vs-latest" +. "$topsrcdir/tools/rusttests/config/mozconfigs/windows-common" diff --git a/tools/rusttests/config/mozconfigs/win64/rusttests-debug b/tools/rusttests/config/mozconfigs/win64/rusttests-debug new file mode 100644 index 0000000000..5f079b4d36 --- /dev/null +++ b/tools/rusttests/config/mozconfigs/win64/rusttests-debug @@ -0,0 +1,2 @@ +. "$topsrcdir/tools/rusttests/config/mozconfigs/win64/rusttests" +ac_add_options --enable-debug diff --git a/tools/rusttests/config/mozconfigs/windows-common b/tools/rusttests/config/mozconfigs/windows-common new file mode 100644 index 0000000000..afdb996920 --- /dev/null +++ b/tools/rusttests/config/mozconfigs/windows-common @@ -0,0 +1,2 @@ +. "$topsrcdir/build/mozconfig.win-common" +. "$topsrcdir/tools/rusttests/config/mozconfigs/common" diff --git a/tools/rusttests/moz.configure b/tools/rusttests/moz.configure new file mode 100644 index 0000000000..520989a8cf --- /dev/null +++ b/tools/rusttests/moz.configure @@ -0,0 +1,5 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +include("../../toolkit/moz.configure") diff --git a/tools/sanitizer/docs/asan.rst b/tools/sanitizer/docs/asan.rst new file mode 100644 index 0000000000..a241908ced --- /dev/null +++ b/tools/sanitizer/docs/asan.rst @@ -0,0 +1,378 @@ +Address Sanitizer +================= + +What is Address Sanitizer? +-------------------------- + +Address Sanitizer (ASan) is a fast memory error detector that detects +use-after-free and out-of-bound bugs in C/C++ programs. It uses a +compile-time instrumentation to check all reads and writes during the +execution. In addition, the runtime part replaces the ``malloc`` and +``free`` functions to check dynamically allocated memory. More +information on how ASan works can be found on `the Address Sanitizer +wiki `__. + +A `meta bug called asan-maintenance `__ +is maintained to keep track of all the bugs found with ASan. + +Downloading artifact builds +--------------------------- + +For Linux and Windows users, the easiest way to get Firefox builds with +Address Sanitizer is to download a continuous integration asan build of +mozilla-central (updated at least daily): + +- mozilla-central optimized builds: + `linux `__ + \| + `windows `__ + (recommended for testing) +- mozilla-central debug builds: + `linux `__ + \| + `windows `__ + (recommended for debugging if the optimized builds don't do the job) + +The fuzzing team also offers a tool called ``fuzzfetch`` to download these and many +other CI builds. It makes downloading and unpacking these builds much easier and +can be used not just for fuzzing but for all purposes that require a CI build download. + +You can install ``fuzzfetch`` from +`Github `__ or +`via pip `__. + +Afterwards, you can run e.g. + +:: + + $ python -m fuzzfetch --asan -n firefox-asan + +to get the optimized Linux ASan build mentioned above unpacked into a directory called ``firefox-asan``. +The ``--debug`` and ``--os`` switches can be used to get the other variants listed above. + +Creating Try builds +------------------- + +If for some reason you can't use the pre-built binaries mentioned in the +previous section (e.g. you want a non-Linux build or you need to test a +patch), you can either build Firefox yourself (see the following +section) or use the :ref:`try server ` to +create the customized build for you. Pushing to try requires L1 commit +access. If you don't have this access yet you can request access (see +`Becoming A Mozilla +Committer `__ +and `Mozilla Commit Access +Policy `__ +for the requirements). + +The tree contains `several mozconfig files for creating asan +builds `__ +(the "nightly-asan" files create release builds, whereas the +"debug-asan" files create debug+opt builds). For Linux builds, the +appropriate configuration file is used by the ``linux64-asan`` target. +If you want to create a macOS or Windows build, you'll need to copy the +appropriate configuration file over the regular debug configuration +before pushing to try. For example: + +:: + + cp browser/config/mozconfigs/macosx64/debug-asan browser/config/mozconfigs/macosx64/debug + +You can then `push to Try in the usual +way `__ +and, once the build is complete, download the appropriate build +artifact. + +Creating local builds on Windows +-------------------------------- + +On Windows, ASan is supported only in 64-bit builds. + +Run ``mach bootstrap`` to get an updated clang-cl in your +``~/.mozbuild`` directory, then use the following +:ref:`mozconfig `: + +:: + + ac_add_options --enable-address-sanitizer + ac_add_options --disable-jemalloc + + export LDFLAGS="clang_rt.asan_dynamic-x86_64.lib clang_rt.asan_dynamic_runtime_thunk-x86_64.lib" + CLANG_LIB_DIR="$(cd ~/.mozbuild/clang/lib/clang/*/lib/windows && pwd)" + export MOZ_CLANG_RT_ASAN_LIB_PATH="${CLANG_LIB_DIR}/clang_rt.asan_dynamic-x86_64.dll" + +If you launch an ASan build under WinDbg, you may see spurious +first-chance Access Violation exceptions. These come from ASan creating +shadow memory pages on demand, and can be ignored. Run ``sxi av`` to +ignore these exceptions. (You will still catch second-chance Access +Violation exceptions if you actually crash.) + +LeakSanitizer (LSan) is not supported on Windows. + +Creating local builds on Linux or Mac +------------------------------------- + +Build prerequisites +~~~~~~~~~~~~~~~~~~~ + +LLVM/Clang +^^^^^^^^^^ + +The ASan instrumentation is implemented as an LLVM pass and integrated +into Clang. Any clang version that is capable of compiling Firefox has +everything needed to do an ASAN build. + +Building Firefox +~~~~~~~~~~~~~~~~ + +Getting the source +^^^^^^^^^^^^^^^^^^ + +Using that or any later revision, all you need to do is to :ref:`get yourself +a clone of mozilla-central `. + +Adjusting the build configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the build configuration file ``mozconfig`` with the following +content in your mozilla-central directory: + +:: + + # Combined .mozconfig file for ASan on Linux+Mac + + mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/objdir-ff-asan + + # Enable ASan specific code and build workarounds + ac_add_options --enable-address-sanitizer + + # These three are required by ASan + ac_add_options --disable-jemalloc + ac_add_options --disable-crashreporter + ac_add_options --disable-elf-hack + + # Keep symbols to symbolize ASan traces later + export MOZ_DEBUG_SYMBOLS=1 + ac_add_options --enable-debug-symbols + ac_add_options --disable-install-strip + + # Settings for an opt build (preferred) + # The -gline-tables-only ensures that all the necessary debug information for ASan + # is present, but the rest is stripped so the resulting binaries are smaller. + ac_add_options --enable-optimize="-O2 -gline-tables-only" + ac_add_options --disable-debug + + # Settings for a debug+opt build + #ac_add_options --enable-optimize + #ac_add_options --enable-debug + + # MacOSX only: Uncomment and adjust this path to match your SDK + # ac_add_options --with-macos-sdk=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.8.sdk + +You may also need this, as seen in +``browser/config/mozconfigs/linux64/nightly-asan`` (the config file used +for Address Sanitizer builds used for automated testing): + +:: + + # ASan specific options on Linux + ac_add_options --enable-valgrind + +Starting the build process +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now you start the build process using the regular ``./mach build`` +command. + +Starting Firefox +^^^^^^^^^^^^^^^^ + +After the build has completed, ``./mach run`` with the usual options for +running in a debugger (``gdb``, ``lldb``, ``rr``, etc.) work fine, as do +the ``--disable-e10s`` and other options. + +Building only the JavaScript shell +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to build only the JavaScript shell instead of doing a full +Firefox build, the build script below will probably help you to do so. +Execute this script in the ``js/src/`` subdirectory and pass a directory +name as the first parameter. The build will then be created in a new +subdirectory with that name. + +:: + + #! /bin/sh + + if [ -z $1 ] ; then + echo "usage: $0 " + elif [ -d $1 ] ; then + echo "directory $1 already exists" + else + autoconf2.13 + mkdir $1 + cd $1 + CC="clang" \ + CXX="clang++" \ + CFLAGS="-fsanitize=address" \ + CXXFLAGS="-fsanitize=address" \ + LDFLAGS="-fsanitize=address" \ + ../configure --enable-debug --enable-optimize --enable-address-sanitizer --disable-jemalloc + fi + +Getting Symbols in Address Sanitizer Traces +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, ASan traces are unsymbolized and only print the +binary/library and a memory offset instead. In order to get more useful +traces, containing symbols, there are two approaches. + +Using the LLVM Symbolizer (recommended) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +LLVM ships with a symbolizer binary that ASan will readily use to +immediately output symbolized traces. To use it, just set the +environment variable ``ASAN_SYMBOLIZER_PATH`` to reflect the location of +your ``llvm-symbolizer`` binary, before running the process. This +program is usually included in an LLVM distribution. Stacks without +symbols can also be post-processed, see below. + +.. warning:: + + .. note:: + + **Warning:** On OS X, the content sandbox prevents the symbolizer + from running. To use llvm-symbolizer on ASan output from a + content process, the content sandbox must be disabled. This can be + done by setting ``MOZ_DISABLE_CONTENT_SANDBOX=1`` in your run + environment. Setting this in .mozconfig has no effect. + + +Post-Processing Traces with asan_symbolize.py +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Instead of using the llvm-symbolizer binary, you can also pipe the +output through the ``asan_symbolize.py`` script, shipped with LLVM +(``$LLVM_HOME/projects/compiler-rt/lib/asan/scripts/asan_symbolize.py``), +often included in LLVM distributions. The disadvantage is that the +script will need to use ``addr2line`` to get the symbols, which means +that every library will have to be loaded into memory +(including``libxul``, which takes a bit). + +However, in certain situations it makes sense to use this script. For +example, if you have/received an unsymbolized trace, then you can still +use the script to turn it into a symbolized trace, given that you can +get the original binaries that produced the unsymbolized trace. In order +for the script to work in such cases, you need to ensure that the paths +in the trace point to the actual binaries, or change the paths +accordingly. + +Since the output of the ``asan_symbolize.py`` script is still mangled, +you might want to pipe the output also through ``c++filt`` afterwards. + +Troubleshooting / Known problems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Cannot specify -o when generating multiple output files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you get the error +"``cannot specify -o when generating multiple output files"`` from +clang, disable ``elf-hack`` in your ``mozconfig`` to work around the +issue: + +:: + + ac_add_options --disable-elf-hack + +Optimized build +^^^^^^^^^^^^^^^ + +Since `an issue with -O2/-Os and +ASan `__ +has been resolved, the regular optimizations used by Firefox should work +without any problems. The optimized build has only a barely noticeable +speed penalty and seems to be even faster than regular debug builds. + +No "AddressSanitizer: **libc** interceptors initialized" shows after running ./mach run +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + $ ASAN_OPTIONS=verbosity=2 ./mach run + +Use the above command instead + +"An admin user name and password" is required to enter Developer Mode +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Please enable **Developer** **mode** by: + +:: + + $ /usr/sbin/DevToolsSecurity -enable + Developer mode is now enabled. + +Debugging issues that ASan finds +-------------------------------- + +When ASan discovers an issue it will simply print an error message and +exit the app. To stop the app in a debugger before ASan exits it, set a +breakpoint on ``__asan::ReportGenericError``. For more info on using +ASan and debugging issues that it uncovers, see the page `Address +sanitizer and a +debugger `__ +page on the upstream wiki. + +``__asan_describe_address(pointer)`` issued at the debugger prompt or +even directly in the code allows outputting lots of information about +this memory address (thread and stack of allocation, of deallocation, +whether or not it is a bit outside a known buffer, thread and stack of +allocation of this buffer, etc.). This can be useful to understand where +some buffer that is not aligned was allocated, when doing SIMD work, for +example. + +`rr `__ (Linux x86 only) works great with ASan +and combined, this combo allows doing some very powerful debugging +strategies. + +LeakSanitizer +------------- + +LeakSanitizer (LSan) is a special execution mode for regular ASan. It +takes advantage of how ASan tracks the set of live blocks at any given +point to print out the allocation stack of any block that is still alive +at shutdown, but is not reachable from the stack, according to a +conservative scan. This is very useful for detecting leaks of things +such as ``char*`` that do not participate in the usual Gecko shutdown +leak detection. LSan is supported on x86_64 Linux and OS X. + +LSan is enabled by default in ASan builds, as of more recent versions of +Clang. To make an ASan build not run LSan, set the environment variable +``ASAN_OPTIONS`` to ``detect_leaks=0`` (or add it as an entry to a +``:``-separated list if it is already set to something). If you want to +enable it when it is not for some reason, set it to 1 instead of 0. If +LSan is enabled and you are using a non-debug build, you will also want +to set the environment variable ``MOZ_CC_RUN_DURING_SHUTDOWN=1``, to +ensure that we run shutdown GCs and CCs to avoid spurious leaks. + +If an object that is reported by LSan is intentionally never freed, a +symbol can be added to ``build/sanitizers/lsan_suppressions.txt`` to get +LSan to ignore it. + +For some more information on LSan, see the `Leak Sanitizer wiki +page `__. + + +A `meta bug called lsan `__ +is maintained to keep track of all the bugs found with LSan. + + + +Frequently Asked Questions about ASan +------------------------------------- + +How does ASan work exactly? +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +More information on how ASan works can be found on `the Address Sanitizer wiki `__. diff --git a/tools/sanitizer/docs/asan_nightly.rst b/tools/sanitizer/docs/asan_nightly.rst new file mode 100644 index 0000000000..790def3931 --- /dev/null +++ b/tools/sanitizer/docs/asan_nightly.rst @@ -0,0 +1,204 @@ +ASan Nightly +============ + +The **ASan Nightly Project** involves building a Firefox Nightly browser +with the popular +`AddressSanitizer `__ +tool and enhancing it with remote crash reporting capabilities for any +errors detected. + +The purpose of the project is to find subtle memory corruptions +occurring during regular browsing that would either not crash at all or +crash in a way that we cannot figure out what the exact problem is just +from the crash dump. We have a lot of inactionable crash reports and +AddressSanitizer traces are usually a lot more actionable on their own +(especially use-after-free traces). Part of this project is to figure +out if and how many actionable crash reports ASan can give us just by +surfing around. The success of the project of course also depends on the +number of participants. + +You can download the latest build using one of the links below. The +builds are self-updating daily like regular nightly builds (like with +regular builds, you can go to *"Help"* → *"About Nightly"* to force an +update check or confirm that you run the latest version). + +.. note:: + + If you came here looking for regular ASan builds (e.g. for fuzzing or + as a developer to reproduce a crash), you should probably go to the + :ref:`Address Sanitizer` doc instead. + +.. _Requirements: + +Requirements +~~~~~~~~~~~~ + +Current requirements are: + +- Windows or Linux-based Operating System +- 16 GB of RAM recommended +- Special ASan Nightly Firefox Build + + - `Linux + Download `__ + - `Windows + Download `__ + +If you are already using regular Nightly, it should be safe to share the +profile with the regular Nightly instance. If you normally use a beta or +release build (and you would like to be able to switch back to these), +you should consider using a second profile. + +.. warning:: + + **Windows Users:** Please note that the Windows builds currently show + an error during setup (see "*Known Issues*" section below), but + installation works nonetheless. We are working on the problem. + +.. note:: + + If you run in an environment with any sorts of additional security + restrictions (e.g. custom process sandboxing), please make sure that + your /tmp directory is writable and the shipped ``llvm-symbolizer`` + binary is executable from within the Firefox process. + +Preferences +~~~~~~~~~~~ + +If you wish for your crash report to be identifiable, you can go to +``about:config`` and set the **``asanreporter.clientid``** to your +**valid email address**. This isn't mandatory, you can of course report +crash traces anonymously. If you decide to send reports with your email +address and you have a Bugzilla account, consider using the same email +as your Bugzilla account uses. We will then Cc you on any bugs filed +from your crash reports. If your email does not belong to a Bugzilla +account, then we will not publish it but only use it to resolve +questions about your crash reports. + +.. note:: + + Setting this preference helps us to get back to you in case we have + questions about your setup/OS. Please consider using it so we can get + back to you if necessary. + +Bug Bounty Program +~~~~~~~~~~~~~~~~~~ + +As a special reward for participating in the program, we decided to +treat all submitted reports as if they were filed directly in Bugzilla. +This means that reports that + +- indicate a security issue of critical or high rating +- **and** that can be fixed by our developers + +are eligible for a bug bounty according to our `client bug bounty +program +rules `__. As +the report will usually not include any steps to reproduce or a test +case, it will most likely receive a lower-end bounty. Like with regular +bug reports, we would typically reward the first (identifable) report of +an issue. + +.. warning:: + + If you would like to participate in the bounty program, make sure you + set your **``asanreporter.clientid``** preference as specified above. + We cannot reward any reports that are submitted with no email + address. + + +Known Issues +~~~~~~~~~~~~ + +This section lists all currently known limitations of the ASan Nightly +builds that are considered bugs. + +- [STRIKEOUT:Flash is currently not working] +- `Bug + 1477490 `__\ [STRIKEOUT:- + Windows: Stack instrumentation disabled due to false positives] +- `Bug + 1478096 `__ - + **Windows:** Error during install with maintenanceservice_tmp.exe +- It has been reported that ASan Nightly performance is particularly + bad if you run on a screen with 120hz refresh rate. Switching to 60hz + should improve performance drastically. + +Note that these bugs are **specific** to ASan Nightly as listed in the +`tracking bug dependency +list `__. +For the full list of bugs found by this project, see `this +list `__ +instead and note that some bugs might not be shown because they are +security bugs. + +If you encounter a bug not listed here, please file a bug at +`bugzilla.mozilla.org `__ or send an +email to +`choller@mozilla.com `__. +When filing a bug, it greatly helps if you Cc that email address and +make the bug block `bug +1386297 `__. + +FAQ +~~~ + +What additional data is collected? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The project only collects ASan traces and (if you set it in the +preferences) your email address. We don't collect any other browser +data, in particular not the sites you were visiting or page contents. It +is really just crash traces submitted to a remote location. + +.. note:: + + The ASan Nightly browser also still has all the data collection + capabilities of a regular Nightly browser. The answer above only + refers to what this project collects **in addition** to what the + regular Nightly browser can collect. + +What's the performance impact? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ASan Nightly build only comes with a slight slowdown at startup and +browsing, sometimes it is not even noticeable. The RAM consumption +however is much higher than with a regular build. Be prepared to restart +your browser sometimes, especially if you use a lot of tabs at once. +Also, the updates are larger than the regular ones, so download times +for updates will be higher, especially if you have a slower internet +connection. + +.. warning:: + + If you experience performance issues, see also the *"Known Issues"* + section above, in particular the problem about screen refresh rate + slowing down Firefox. + +What about stability? +^^^^^^^^^^^^^^^^^^^^^ + +The browser is as stable as a regular Nightly build. Various people have +been surfing around with it for their daily work for weeks now and we +have barely received any crash reports. + +How do I confirm that I'm running the correct build? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you open ``about:config`` and type *"asanreporter"* into the search +field, you should see an entry called ``asanreporter.apiurl`` associated +with a URL. Do not modify this value. + +.. warning:: + + Since Firefox 64, the *"ASan Crash Reporter"*  feature is no longer + listed in ``about:support`` + +Will there be support for Mac? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We are working on support for Mac, but it might take longer because we +have no ASan CI coverage on Mac due to hardware constraints. If you work +on Release Engineering and would like to help make e.g. Mac happen +earlier, feel free to `contact +me `__. diff --git a/tools/sanitizer/docs/index.rst b/tools/sanitizer/docs/index.rst new file mode 100644 index 0000000000..3a1dc0806e --- /dev/null +++ b/tools/sanitizer/docs/index.rst @@ -0,0 +1,33 @@ +Sanitizer +========= + +.. toctree:: + :maxdepth: 1 + :hidden: + :glob: + + * + +**Address Sanitizer** + +Address Sanitizer (ASan) is a fast memory error detector that detects use-after-free and out-of-bound bugs in C/C++ programs. It uses a compile-time instrumentation to check all reads and writes during the execution. In addition, the runtime part replaces the malloc and free functions to check dynamically allocated memory. + +:ref:`More information
` + +**Thread Sanitizer** + +Thread Sanitizer (TSan) is a fast data race detector for C/C++ programs. It uses a compile-time instrumentation to check all non-race-free memory access at runtime. Unlike other tools, it understands compiler-builtin atomics and synchronization and therefore provides very accurate results with no false positives (except if unsupported synchronization primitives like inline assembly or memory fences are used). + +:ref:`More information ` + +**Memory Sanitizer** + +Memory Sanitizer (MSan) is a fast detector used for uninitialized memory in C/C++ programs. It uses a compile-time instrumentation to ensure that all memory access at runtime uses only memory that has been initialized. + +:ref:`More information ` + +**ASan Nightly Project** + +The ASan Nightly Project involves building a Firefox Nightly browser with the popular AddressSanitizer tool and enhancing it with remote crash reporting capabilities for any errors detected. + +:ref:`More information ` diff --git a/tools/sanitizer/docs/memory_sanitizer.rst b/tools/sanitizer/docs/memory_sanitizer.rst new file mode 100644 index 0000000000..161826fbc2 --- /dev/null +++ b/tools/sanitizer/docs/memory_sanitizer.rst @@ -0,0 +1,173 @@ +Memory Sanitizer +================ + ++--------------------------------------------------------------------+ +| This page is an import from MDN and the contents might be outdated | ++--------------------------------------------------------------------+ + +What is Memory Sanitizer? +------------------------- + +Memory Sanitizer (MSan) is a fast detector used for uninitialized memory +in C/C++ programs. It uses a compile-time instrumentation to ensure that +all memory access at runtime uses only memory that has been initialized. +Unlike most other sanitizers, MSan can easily cause false positives if +not all libraries are instrumented. This happens because MSan is +not able to observe memory initialization in uninstrumented libraries. +More information on MSan can be found on `the Memory Sanitizer +wiki `__. + +Public Builds +------------- + +**Note:** No public builds are available at this time yet. + +Manual Build +------------ + +Build prerequisites +~~~~~~~~~~~~~~~~~~~ + +**Note:** MemorySanitizer requires **64-bit Linux** to work. Other +platforms/operating systems are not supported. + +LLVM/Clang +^^^^^^^^^^ + +The MSan instrumentation is implemented as an LLVM pass and integrated +into Clang. As MSan is one of the newer sanitizers, we recommend using a +recent Clang version, such as Clang 3.7+. + +You can find precompiled binaries for LLVM/Clang on `the LLVM releases +page `__. + +Building Firefox +~~~~~~~~~~~~~~~~ + +.. warning:: + + **Warning: Running Firefox with MemorySanitizer would require all + external dependencies to be built with MemorySanitizer as well. To + our knowledge, this has never been attempted yet, so the build + configuration provided here is untested and without an appropriately + instrumented userland, it will cause false positives.** + +Getting the source +^^^^^^^^^^^^^^^^^^ + +If you don't have a source code repository clone yet, you need to :ref:`get +yourself a clone of Mozilla-central `. + +Adjusting the build configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the build configuration file ``.mozconfig`` with the following +content in your Mozilla-central directory: + +.. code:: + + mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/objdir-ff-msan + + # Enable LLVM specific code and build workarounds + ac_add_options --enable-memory-sanitizer + # If clang is already in your $PATH, then these can simply be: + # export CC=clang + # export CXX=clang++ + export CC="/path/to/clang" + export CXX="/path/to/clang++" + + # llvm-symbolizer displays much more complete backtraces when data races are detected. + # If it's not already in your $PATH, then uncomment this next line: + #export LLVM_SYMBOLIZER="/path/to/llvm-symbolizer" + + # Add MSan to our compiler flags + export CFLAGS="-fsanitize=memory" + export CXXFLAGS="-fsanitize=memory" + + # Additionally, we need the MSan flag during linking. Normally, our C/CXXFLAGS would + # be used during linking as well but there is at least one place in our build where + # our CFLAGS are not added during linking. + # Note: The use of this flag causes Clang to automatically link the MSan runtime :) + export LDFLAGS="-fsanitize=memory" + + # These three are required by MSan + ac_add_options --disable-jemalloc + ac_add_options --disable-crashreporter + ac_add_options --disable-elf-hack + + # Keep symbols to symbolize MSan traces + export MOZ_DEBUG_SYMBOLS=1 + ac_add_options --enable-debug-symbols + ac_add_options --disable-install-strip + + # Settings for an opt build + ac_add_options --enable-optimize="-O2 -gline-tables-only" + ac_add_options --disable-debug + +Starting the build process +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now you start the build process using the regular ``make -f client.mk`` +command. + +Starting Firefox +^^^^^^^^^^^^^^^^ + +After the build has completed, you can start Firefox from the ``objdir`` +as usual. + +Building the JavaScript shell +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Note:** Unlike Firefox itself, the JavaScript shell does **not** +require an instrumented userland. Calls to external libraries like +zlib are handled with special annotations inside the engine. + +.. warning:: + + **Warning: Certain technologies used inside the JavaScript engine are + incompatible with MSan and must be disabled at runtime to prevent + false positives. This includes the JITs and asm.js. Therefore always + make sure to run with + ``--no-ion --no-baseline --no-asmjs --no-native-regexp``.** + +If you want to build only the JavaScript shell instead of doing a full +Firefox build, the build script below will probably help you to do so. +Before using it, you must, of course, adjust the path name for +``LLVM_ROOT`` to match your setup. Once you have adjusted everything, +execute this script in the ``js/src/`` subdirectory and pass a directory +name as the first parameter. The build will then be created in a new +subdirectory with that name. + +.. code:: + + #! /bin/sh + + if [ -z $1 ] ; then + echo "usage: $0 " + elif [ -d $1 ] ; then + echo "directory $1 already exists" + else + autoconf2.13 + mkdir $1 + cd $1 + LLVM_ROOT="/path/to/llvm" + CC="$LLVM_ROOT/build/bin/clang" \ + CXX="$LLVM_ROOT/build/bin/clang++" \ + CFLAGS="-fsanitize=memory" \ + CXXFLAGS="-fsanitize=memory" \ + LDFLAGS=""-fsanitize=memory" \ + ../configure --enable-debug --enable-optimize --enable-memory-sanitizer --disable-jemalloc --enable-posix-nspr-emulation + make -j 8 + fi + +Using LLVM Symbolizer for faster/better traces +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, MSan traces are not symbolized. + +LLVM ships with the symbolizer binary ``llvm-symbolize`` that MSan will +readily use to immediately output symbolized traces if the program is +found on the ``PATH``. If your ``llvm-symbolizer`` lives outside the +``PATH``, you can set the ``MSAN_SYMBOLIZER_PATH`` environment variable +to point to your symbolizer binary. diff --git a/tools/sanitizer/docs/tsan.rst b/tools/sanitizer/docs/tsan.rst new file mode 100644 index 0000000000..77fb6c89d7 --- /dev/null +++ b/tools/sanitizer/docs/tsan.rst @@ -0,0 +1,327 @@ +Thread Sanitizer +================ + +What is Thread Sanitizer? +-------------------------- + +Thread Sanitizer (TSan) is a fast data race detector for C/C++ and Rust +programs. It uses a compile-time instrumentation to check all non-race-free +memory access at runtime. Unlike other tools, it understands compiler-builtin +atomics and synchronization and therefore provides very accurate results +with no false positives (except if unsupported synchronization primitives +like inline assembly or memory fences are used). More information on how +TSan works can be found on `the Thread Sanitizer wiki `__. + +A `meta bug called tsan `__ +is maintained to keep track of all the bugs found with TSan. + +A `blog post on hacks.mozilla.org `__ describes this project. + +Note that unlike other sanitizers, TSan is currently **only supported on Linux**. + +Downloading artifact builds +--------------------------- + +The easiest way to get Firefox builds with Thread Sanitizer is to download a +continuous integration TSan build of mozilla-central (updated at least daily): + +- mozilla-central optimized builds: + `linux `__ + +The fuzzing team also offers a tool called ``fuzzfetch`` to download this and many +other CI builds. It makes downloading and unpacking these builds much easier and +can be used not just for fuzzing but for all purposes that require a CI build download. + +You can install ``fuzzfetch`` from +`Github `__ or +`via pip `__. + +Afterwards, you can run + +:: + + $ python -m fuzzfetch --tsan -n firefox-tsan + +to get the build mentioned above unpacked into a directory called ``firefox-tsan``. + +Creating Try builds +------------------- + +If for some reason you can't use the pre-built binaries mentioned in the +previous section (e.g. you need to test a patch), you can either build +Firefox yourself (see the following section) or use the :ref:`try server ` +to create the customized build for you. Pushing to try requires L1 commit +access. If you don't have this access yet you can request access (see +`Becoming A Mozilla +Committer `__ +and `Mozilla Commit Access +Policy `__ +for the requirements). + +Using ``mach try fuzzy --full`` you can select the ``build-linux64-tsan/opt`` job +and related tests (if required). + +Creating local builds on Linux +------------------------------ + +Build prerequisites +~~~~~~~~~~~~~~~~~~~ + +LLVM/Clang/Rust +^^^^^^^^^^^^^^^ + +The TSan instrumentation is implemented as an LLVM pass and integrated +into Clang. We strongly recommend that you use the Clang version supplied +as part of the ``mach bootstrap`` process, as we backported several required +fixes for TSan on Firefox. + +Sanitizer support in Rust is genuinely experimental, +so our build system only works with a specially patched version of Rust +that we build in our CI. To install that specific version (or update to a newer +version), run the following in the root of your mozilla-central checkout: + +:: + + ./mach artifact toolchain --from-build linux64-rust-dev + rm -rf ~/.mozbuild/rustc-sanitizers + mv rustc ~/.mozbuild/rustc-sanitizers + rustup toolchain link gecko-sanitizers ~/.mozbuild/rustc-sanitizers + rustup override set gecko-sanitizers + +``mach artifact`` will always download the ``linux64-rust-dev`` toolchain associated +with the current mozilla central commit you have checked out. The toolchain should +mostly behave like a normal rust nightly but we don't recommend using it for anything +other than building gecko, just in case. Also note that +``~/.mozbuild/rustc-sanitizers`` is just a reasonable default location -- feel +free to "install" the toolchain wherever you please. + +Building Firefox +~~~~~~~~~~~~~~~~ + +Getting the source +^^^^^^^^^^^^^^^^^^ + +Using that or any later revision, all you need to do is to :ref:`get yourself +a clone of mozilla-central `. + +Adjusting the build configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Create the build configuration file ``mozconfig`` with the following +content in your mozilla-central directory: + +:: + + # Combined .mozconfig file for TSan on Linux+Mac + + mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/objdir-ff-tsan + + # Enable ASan specific code and build workarounds + ac_add_options --enable-thread-sanitizer + + # This ensures that we also instrument Rust code. + export RUSTFLAGS="-Zsanitizer=thread" + + # rustfmt is currently missing in Rust nightly + unset RUSTFMT + + # Current Rust Nightly has warnings + ac_add_options --disable-warnings-as-errors + + # These are required by TSan + ac_add_options --disable-jemalloc + ac_add_options --disable-crashreporter + ac_add_options --disable-elf-hack + ac_add_options --disable-profiling + + # The Thread Sanitizer is not compatible with sandboxing + # (see bug 1182565) + ac_add_options --disable-sandbox + + # Keep symbols to symbolize TSan traces later + export MOZ_DEBUG_SYMBOLS=1 + ac_add_options --enable-debug-symbols + ac_add_options --disable-install-strip + + # Settings for an opt build (preferred) + # The -gline-tables-only ensures that all the necessary debug information for ASan + # is present, but the rest is stripped so the resulting binaries are smaller. + ac_add_options --enable-optimize="-O2 -gline-tables-only" + ac_add_options --disable-debug + + # Settings for a debug+opt build + #ac_add_options --enable-optimize + #ac_add_options --enable-debug + + +Starting the build process +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now you start the build process using the regular ``./mach build`` +command. + +Starting Firefox +^^^^^^^^^^^^^^^^ + +After the build has completed, ``./mach run`` with the usual options for +running in a debugger (``gdb``, ``lldb``, ``rr``, etc.) work fine, as do +the ``--disable-e10s`` and other options. + +Building only the JavaScript shell +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you want to build only the JavaScript shell instead of doing a full +Firefox build, the build script below will probably help you to do so. +Execute this script in the ``js/src/`` subdirectory and pass a directory +name as the first parameter. The build will then be created in a new +subdirectory with that name. + +:: + + #! /bin/sh + + if [ -z $1 ] ; then + echo "usage: $0 " + elif [ -d $1 ] ; then + echo "directory $1 already exists" + else + autoconf2.13 + mkdir $1 + cd $1 + CC="/path/to/mozbuild/clang" \ + CXX="/path/to/mozbuild/clang++" \ + ../configure --disable-debug --enable-optimize="-O2 -gline-tables-only" --enable-thread-sanitizer --disable-jemalloc + fi + +Thread Sanitizer and Symbols +---------------------------- + +Unlike Address Sanitizer, TSan requires in-process symbolizing to work +properly in the first place, as any kind of runtime suppressions will +otherwise not work. + +Hence, it is required that you have a copy of ``llvm-symbolizer`` either +in your ``PATH`` or pointed to by the ``TSAN_SYMBOLIZER_PATH`` environment +variable. This binary is included in your local mozbuild directory, obtained +by ``./mach bootstrap``. + + +Runtime Suppressions +-------------------- + +TSan has the ability to suppress race reports at runtime. This can be used to +silence a race while a fix is developed as well as to permanently silence a +(benign) race that cannot be fixed. + +.. warning:: + **Warning**: Many races *look* benign but are indeed not. Please read + the :ref:`FAQ section ` carefully + and think twice before attempting to suppress a race. + +The runtime Suppression list is directly baked into Firefox at compile-time and +located at `mozglue/build/TsanOptions.cpp `__. + +.. warning:: + **Important**: When adding a suppression, always make sure to include + the bug number. If the suppression is supposed to be permanent, please + add the string ``permanent`` in the same line as the bug number. + +.. warning:: + **Important**: When adding a suppression for a *data race*, always make + sure to include a stack frame from **each** of the two race stacks. + Adding only one suppression for one stack can cause intermittent failures + that are later on hard to track. One exception to this rule is when suppressing + races on global variables. In that case, a single race entry with the name of + the variable is sufficient. + +Troubleshooting / Known Problems +-------------------------------- + +Known Sources of False Positives +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +TSan has a number of things that can cause false positives, namely: + + * The use of memory fences (e.g. Rust Arc) + * The use of inline assembly for synchronization + * Uninstrumented code (e.g. external libraries) using compiler-builtins for synchronization + * A lock order inversion involving only a single thread can cause a false positive deadlock + report (see also https://github.com/google/sanitizers/issues/488). + +If none of these four items are involved, you should *never* assume that TSan is reporting +a false positive to you without consulting TSan peers. It is very easy to misjudge a race +to be a false positive because races can be highly complex and totally non-obvious due to +compiler optimizations and the nature of parallel code. + +Intermittent Broken Stacks +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you intermittently see race reports where one stack is missing with a ``failed to restore the stack`` +message, this can indicate that a suppression is partially covering the race you are seeing. + +Any race where only one of the two stacks is matched by a runtime suppression will show up +if that particular stack fails to symbolize for some reason. The usual solution is to search +the suppressions for potential candidates and disable them temporarily to check if your race +report now becomes mostly consistent. + +However, there are other reasons for broken TSan stacks, in particular if they are not intermittent. +See also the ``history_size`` parameter in the `TSan flags `__. + +Intermittent Race Reports +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unfortunately, the TSan algorithm does not guarantee, that a race is detected 100% of the +time. Intermittent failures with TSan are (to a certain degree) to be expected and the races +involved should be filed and fixed to solve the problem. + +.. _Frequently Asked Questions about TSan: + +Frequently Asked Questions about TSan +------------------------------------- + +Why fix data races? +~~~~~~~~~~~~~~~~~~~ + +Data races are undefined behavior and can cause crashes as well as correctness issues. +Compiler optimizations can cause racy code to have unpredictable and hard-to-reproduce behavior. + +At Mozilla, we have already seen several dangerous races, causing random +`use-after-free crashes `__, +`intermittent test failures `__, +`hangs `__, +`performance issues `__ and +`intermittent asserts `__. Such problems do +not only decrease the quality of our code and user experience, but they also waste countless hours +of developer time. + +Since it is very hard to judge if a particular race could cause such a situation, we +have decided to fix all data races wherever possible, since doing so is often cheaper +than analyzing a race. + +My race is benign, can we ignore it? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While it is possible to add a runtime suppression to ignore the race, we *strongly* encourage +you to not do so, for two reasons: + + 1. Each suppressed race decreases the overall performance of the TSan build, as the race + has to be symbolized each time when it occurs. Since TSan is already in itself a slow + build, we need to keep the amount of suppressed races as low as possible. + + 2. Deciding if a race is truly benign is surprisingly hard. We recommend to read + `this blog post `__ + and `this paper ` + on the effects of seemingly benign races. + +Valid reasons to suppress a confirmed benign race include performance problems arising from +fixing the race or cases where fixing the race would require an unreasonable amount of work. + +Note that the use of atomics usually does not have the bad performance impact that developers +tend to associate with it. If you assume that e.g. using atomics for synchronization will +cause performance regressions, we suggest to perform a benchmark to confirm this. In many +cases, the difference is not measurable. + +How does TSan work exactly? +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +More information on how TSan works can be found on `the Thread Sanitizer wiki `__. diff --git a/tools/tryselect/__init__.py b/tools/tryselect/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/cli.py b/tools/tryselect/cli.py new file mode 100644 index 0000000000..74c82c4bed --- /dev/null +++ b/tools/tryselect/cli.py @@ -0,0 +1,167 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import subprocess +import tempfile +from argparse import ArgumentParser + +from .task_config import all_task_configs + +COMMON_ARGUMENT_GROUPS = { + "push": [ + [ + ["-m", "--message"], + { + "const": "editor", + "default": "{msg}", + "nargs": "?", + "help": "Use the specified commit message, or create it in your " + "$EDITOR if blank. Defaults to computed message.", + }, + ], + [ + ["--closed-tree"], + { + "action": "store_true", + "default": False, + "help": "Push despite a closed try tree", + }, + ], + ], + "preset": [ + [ + ["--save"], + { + "default": None, + "help": "Save selection for future use with --preset.", + }, + ], + [ + ["--preset"], + { + "default": None, + "help": "Load a saved selection.", + }, + ], + [ + ["--list-presets"], + { + "action": "store_const", + "dest": "preset_action", + "const": "list", + "default": None, + "help": "List available preset selections.", + }, + ], + [ + ["--edit-presets"], + { + "action": "store_const", + "dest": "preset_action", + "const": "edit", + "default": None, + "help": "Edit the preset file.", + }, + ], + ], + "task": [ + [ + ["--full"], + { + "action": "store_true", + "default": False, + "help": "Use the full set of tasks as input to fzf (instead of " + "target tasks).", + }, + ], + [ + ["-p", "--parameters"], + { + "default": None, + "help": "Use the given parameters.yml to generate tasks, " + "defaults to a default set of parameters", + }, + ], + ], +} + +NO_PUSH_ARGUMENT_GROUP = [ + [ + ["--stage-changes"], + { + "dest": "stage_changes", + "action": "store_true", + "help": "Locally stage changes created by this command but do not " + "push to try.", + }, + ], + [ + ["--no-push"], + { + "dest": "dry_run", + "action": "store_true", + "help": "Do not push to try as a result of running this command (if " + "specified this command will only print calculated try " + "syntax and selection info and not change files).", + }, + ], +] + + +class BaseTryParser(ArgumentParser): + name = "try" + common_groups = ["push", "preset"] + arguments = [] + task_configs = [] + + def __init__(self, *args, **kwargs): + ArgumentParser.__init__(self, *args, **kwargs) + + group = self.add_argument_group("{} arguments".format(self.name)) + for cli, kwargs in self.arguments: + group.add_argument(*cli, **kwargs) + + for name in self.common_groups: + group = self.add_argument_group("{} arguments".format(name)) + arguments = COMMON_ARGUMENT_GROUPS[name] + + # Preset arguments are all mutually exclusive. + if name == "preset": + group = group.add_mutually_exclusive_group() + + for cli, kwargs in arguments: + group.add_argument(*cli, **kwargs) + + if name == "push": + group_no_push = group.add_mutually_exclusive_group() + arguments = NO_PUSH_ARGUMENT_GROUP + for cli, kwargs in arguments: + group_no_push.add_argument(*cli, **kwargs) + + group = self.add_argument_group("task configuration arguments") + self.task_configs = {c: all_task_configs[c]() for c in self.task_configs} + for cfg in self.task_configs.values(): + cfg.add_arguments(group) + + def validate(self, args): + if hasattr(args, "message"): + if args.message == "editor": + if "EDITOR" not in os.environ: + self.error( + "must set the $EDITOR environment variable to use blank --message" + ) + + with tempfile.NamedTemporaryFile(mode="r") as fh: + subprocess.call([os.environ["EDITOR"], fh.name]) + args.message = fh.read().strip() + + if "{msg}" not in args.message: + args.message = "{}\n\n{}".format(args.message, "{msg}") + + def parse_known_args(self, *args, **kwargs): + args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs) + self.validate(args) + return args, remainder diff --git a/tools/tryselect/docs/configuration.rst b/tools/tryselect/docs/configuration.rst new file mode 100644 index 0000000000..6743d5f385 --- /dev/null +++ b/tools/tryselect/docs/configuration.rst @@ -0,0 +1,30 @@ +Configuring Try +=============== + + +Getting Level 1 Commit Access +----------------------------- + +In order to push to try, `Level 1 Commit Access`_ is required. Please see `Becoming a Mozilla +Committer`_ for more information on how to obtain this. + + +Configuring Version Control +--------------------------- + +After you have level 1 access, you'll need to do a little bit of setup before you can push. Both +``hg`` and ``git`` are supported, move on to the appropriate section below. + + +Configuring Try with Mercurial / Git +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The recommended way to push to try is via the ``mach try`` command. This requires the +``push-to-try`` extension which can be installed by running: + +.. code-block:: shell + + $ mach vcs-setup + +.. _Level 1 Commit Access: https://www.mozilla.org/en-US/about/governance/policies/commit/access-policy/ +.. _Becoming a Mozilla Committer: https://www.mozilla.org/en-US/about/governance/policies/commit/ diff --git a/tools/tryselect/docs/img/add-new-jobs.png b/tools/tryselect/docs/img/add-new-jobs.png new file mode 100644 index 0000000000..513565cb14 Binary files /dev/null and b/tools/tryselect/docs/img/add-new-jobs.png differ diff --git a/tools/tryselect/docs/img/phab-treeherder-link.png b/tools/tryselect/docs/img/phab-treeherder-link.png new file mode 100644 index 0000000000..52b58b6231 Binary files /dev/null and b/tools/tryselect/docs/img/phab-treeherder-link.png differ diff --git a/tools/tryselect/docs/index.rst b/tools/tryselect/docs/index.rst new file mode 100644 index 0000000000..673b93a233 --- /dev/null +++ b/tools/tryselect/docs/index.rst @@ -0,0 +1,71 @@ +Pushing to Try +============== + +"Pushing to Try" allows developers to build and test their changes on Mozilla's automation servers +without requiring their code to be reviewed and landed. + +First, :doc:`ensure that you can push to Try `. +Try knows how to run tasks that are defined in-tree, +such as ``build-linux64/opt`` (build Firefox for Linux). To manually select some tasks for +Try to process, run the following command: + +.. code-block:: shell + + ./mach try fuzzy + +After submitting your requested tasks, you'll be given a link to your "push" in Treeherder. +It may take a few minutes for your push to appear in Treeherder! Be patient, and it will automatically +update when Try begins processing your work. + +Another very useful Try command is ``./mach try auto``, which will automatically select the tasks +that are mostly likely to be affected by your changes. +See the :doc:`selectors page ` to view all the other ways to select which tasks to push. + +Resolving " is damaged and can't be opened" error +------------------------------------------------------------ + +Apple automatically quarantines apps that are downloaded with a browser from an untrusted +location. This "quarantine status" can be cleared by doing ``xattr -c `` after +downloading. You can avoid this "quarantine status" by downloading the build from the command +line instead, such as by using ``curl``: + +.. code-block:: shell + + curl -L -o + +.. _attach-job-review: + +Adding Try jobs to a Phabricator patch +-------------------------------------- + +For every patch submitted for review in Phabricator, a new Try run is automatically created. +A link called ``Treeherder Jobs`` can be found in the ``Diff Detail`` section of the review in +Phabricator. + +.. image:: img/phab-treeherder-link.png + +This run is created for static analysis, linting and other tasks. Attaching new jobs to the run is +easy and doesn't require more actions from the developer. +Click on the down-arrow to access the actions menu, select the relevant jobs +and, click on ``Trigger X new jobs`` (located on the top of the job). + +.. image:: img/add-new-jobs.png + +Table of Contents +----------------- + +.. toctree:: + :maxdepth: 2 + + configuration + selectors/index + presets + tasks + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tools/tryselect/docs/presets.rst b/tools/tryselect/docs/presets.rst new file mode 100644 index 0000000000..6e1008a456 --- /dev/null +++ b/tools/tryselect/docs/presets.rst @@ -0,0 +1,85 @@ +Presets +======= + +Some selectors, such as ``fuzzy`` and ``syntax``, allow saving and loading presets from a file. This is a +good way to re-use a selection, either at a later date or by sharing with others. Look for a +'preset' section in ``mach --help`` to determine whether the selector supports this +functionality. + +Using Presets +------------- + +To save a preset, run: + +.. code-block:: shell + + $ mach try --save + +For example, to save a preset that selects all Windows mochitests: + +.. code-block:: shell + + $ mach try fuzzy --save all-windows-mochitests --query "'win 'mochitest" + preset saved, run with: --preset=all-windows-mochitests + +Then run that saved preset like this: + +.. code-block:: shell + + $ mach try --preset all-windows-mochitests + +To see a list of all available presets run: + +.. code-block:: shell + + $ mach try --list-presets + + +Editing and Sharing Presets +--------------------------- + +Presets can be defined in one of two places, in your home directory or in a file checked into +mozilla-central. + +Local Presets +~~~~~~~~~~~~~ + +These are defined in your ``$MOZBUILD_STATE_DIR``, typically ``~/.mozbuild/try_presets.yml``. +Presets defined here are your own personal collection of presets. You can modify them by running: + +.. code-block:: shell + + $ ./mach try --edit-presets + + +Shared Presets +~~~~~~~~~~~~~~ + +You can also check presets into mozilla-central in `tools/tryselect/try_presets.yml`_. These presets +will be available to all users of ``mach try``, so please be mindful when editing this file. Make +sure the name of the preset is scoped appropriately (i.e doesn't contain any team or module specific +terminology). It is good practice to prefix the preset name with the name of the team or module that +will get the most use out of it. + + +Preset Format +~~~~~~~~~~~~~ + +Presets are simple key/value objects, with the name as the key and a metadata object as the value. +For example, the preset saved above would look something like this in ``try_presets.yml``: + +.. code-block:: yaml + + all-windows-mochitests: + selector: fuzzy + description: >- + Runs all windows mochitests. + query: + - "'win 'mochitest" + +The ``selector`` key (required) allows ``mach try`` to determine which subcommand to dispatch to. +The ``description`` key (optional in user presets but required for shared presets) is a human +readable string describing what the preset selects and when to use it. All other values in the +preset are forwarded to the specified selector as is. + +.. _tools/tryselect/try_presets.yml: https://searchfox.org/mozilla-central/source/tools/tryselect/try_presets.yml diff --git a/tools/tryselect/docs/selectors/again.rst b/tools/tryselect/docs/selectors/again.rst new file mode 100644 index 0000000000..da592b7a34 --- /dev/null +++ b/tools/tryselect/docs/selectors/again.rst @@ -0,0 +1,36 @@ +Again Selector +============== + +When you push to try, the computed ``try_task_config.json`` is saved in a +history file under ``~/.mozbuild/srcdirs//history`` (note: the +``syntax`` selector does not use ``try_task_config.json`` yet so does not save +any history). You can then use the ``again`` selector to re-push any of your +previously generated task configs. + +In the simple case, you can re-run your last try push with: + +.. code-block:: shell + + $ mach try again + +If you want to re-push a task config a little further down the history stack, +first you need to figure out its index with: + +.. code-block:: shell + + $ mach try again --list + +Then run: + +.. code-block:: shell + + $ mach try again --index + +Note that index ``0`` is always the most recent ``try_task_config.json`` in the +history. You can clear your history with: + +.. code-block:: shell + + $ mach try again --purge + +Only the 10 most recent pushes will be saved in your history. diff --git a/tools/tryselect/docs/selectors/auto.rst b/tools/tryselect/docs/selectors/auto.rst new file mode 100644 index 0000000000..39ab5b7450 --- /dev/null +++ b/tools/tryselect/docs/selectors/auto.rst @@ -0,0 +1,24 @@ +Auto Selector +============= + +This selector automatically determines the most efficient set of tests and +tasks to run against your push. It accomplishes this via a combination of +machine learning and manual heuristics. The tasks selected here should match +pretty closely to what would be scheduled if your patch were pushed to +autoland. + +It is the officially recommended selector to use when you are unsure of which +tasks should run on your push. + +To use: + +.. code-block:: + + $ mach try auto + +Unlike other try selectors, tasks are not chosen locally. Rather they will be +computed by the decision task. + +Like most other selectors, ``mach try auto`` supports many of the standard +templates such as ``--artifact`` or ``--env``. See ``mach try auto --help`` for +the full list of supported templates. diff --git a/tools/tryselect/docs/selectors/chooser.rst b/tools/tryselect/docs/selectors/chooser.rst new file mode 100644 index 0000000000..6c59d54009 --- /dev/null +++ b/tools/tryselect/docs/selectors/chooser.rst @@ -0,0 +1,32 @@ +Chooser Selector +================ + +When pushing to try, there are a very large amount of builds and tests to choose from. Often too +many to remember, making it easy to forget a set of tasks which should otherwise have been run. + +This selector allows you to select tasks from a web interface that lists all the possible build and +test tasks and allows you to select them from a list. It is similar in concept to the old `try +syntax chooser`_ page, except that the values are dynamically generated using the :ref:`taskgraph` as an +input. This ensures that it will never be out of date. + +To use: + +.. code-block:: shell + + $ mach try chooser + +This will spin up a local web server (using Flask) which serves the chooser app. After making your +selection, simply press ``Push`` and the rest will be handled from there. No need to copy/paste any +syntax strings or the like. + +You can run: + +.. code-block:: shell + + $ mach try chooser --full + +To generate the interface using the full :ref:`taskgraph` instead. This will include tasks that don't run +on mozilla-central. + + +.. _try syntax chooser: https://mozilla-releng.net/trychooser diff --git a/tools/tryselect/docs/selectors/compare.rst b/tools/tryselect/docs/selectors/compare.rst new file mode 100644 index 0000000000..a87b263030 --- /dev/null +++ b/tools/tryselect/docs/selectors/compare.rst @@ -0,0 +1,17 @@ +Compare Selector +================ + +When this command runs it pushes two identical try jobs to treeherder. The first +job is on the current commit you are on, and the second one is a commit +specified in the command line arguments. This selector is aimed at helping +engineers test performance enhancements or resolve performance regressions. + +Currently the only way you can select jobs is through fuzzy but we are +planning on expanding to other choosing frameworks also. + +You pass the commit you want to compare against as a commit hash as either +``-cc`` or ``--compare-commit``, an example is show below + +.. code-block:: shell + + $ mach try compare --compare-commit diff --git a/tools/tryselect/docs/selectors/empty.rst b/tools/tryselect/docs/selectors/empty.rst new file mode 100644 index 0000000000..c3ea61b9ce --- /dev/null +++ b/tools/tryselect/docs/selectors/empty.rst @@ -0,0 +1,21 @@ +Empty Selector +============== + +The ``mach try empty`` subcommand is very simple, it won't schedule any additional tasks. You'll +still see lint tasks and python-unittest tasks if applicable, this is due to a configuration option +in taskcluster. + +Other than those, your try run will be empty. You can use treeherder's ``Add new jobs`` feature to +selectively add additional tasks after the fact. + +.. note:: + + To use ``Add new jobs`` you'll need to be logged in and have commit access level 1, just as if + you were pushing to try. + +To do this: + + 1. Click the drop-down arrow at the top right of your commit. + 2. Select ``Add new jobs`` (it may take a couple seconds to load). + 3. Choose your desired tasks by clicking them one at a time. + 4. At the top of your commit, select ``Trigger New Jobs``. diff --git a/tools/tryselect/docs/selectors/fuzzy.rst b/tools/tryselect/docs/selectors/fuzzy.rst new file mode 100644 index 0000000000..d50f801eb0 --- /dev/null +++ b/tools/tryselect/docs/selectors/fuzzy.rst @@ -0,0 +1,371 @@ +Fuzzy Selector +============== + +The fuzzy selector uses a tool called `fzf`_. It allows you to filter down all of the task labels +from a terminal based UI and an intelligent fuzzy finding algorithm. If the ``fzf`` binary is not +installed, you'll be prompted to bootstrap it on first run. + + +Understanding the Interface +--------------------------- + +When you run ``mach try fuzzy`` an interface similar to the one below will open. This is `fzf`_. + +.. image:: fzf.png + +There's a lot to unpack here, so let's examine each component a bit more closely. + + A. The set of tasks that match the currently typed-out query. In the above image only tasks that + match the query ``'linux64 mochibrochr`` are displayed. + + B. The set of selected tasks. These are the tasks that will be scheduled once you hit ``Enter``. + In other words, if the task you want does not appear here, *it won't be scheduled*. + + C. Count information of the form ``x/y (z)``, where ``x`` is the number of tasks that match the + current query, ``y`` is the total number of tasks and ``z`` is the number of tasks you have + selected. + + D. The input bar for entering queries. As you type you'll notice the list of tasks in ``A`` + starts to update immediately. In the image above, the query ``'linux64 mochibrochr`` is entered. + Correspondingly only tasks matching that query are displayed. + +In general terms, you first find tasks on the left. Then you move them over to the right by +selecting them. Once you are satisfied with your selection, press ``Enter`` to push to try. + + +Selecting Tasks +--------------- + +There are few ways you can select tasks. If you are feeling a bit overwhelmed, it might be best to +stick with the mouse to start: + + 1. Enter a query (e.g ``mochitest``) to reduce the task list a little. + 2. Scroll up and look for the task(s) you want. + 3. ``Right-Click`` as many tasks as desired to select them. + 4. Optionally delete your query, go back to step 1) and repeat. + 5. Press ``Enter`` to push (or ``Esc`` to cancel). + +.. note:: + + Dependencies are automatically filled in, so you can select a test task without needing + to select the build it depends on. + +As you ``Right-Click``, notice that a little arrow appears to the left of the task label. This +indicates that the task is selected and exists in the preview pane to the right. + +Once you are a bit more comfortable with the interface, using the keyboard is much better at quickly +selecting tasks. Here are the main shortcuts you'll need: + +.. code-block:: text + + Ctrl-K / Up => Move cursor up + Ctrl-J / Down => Move cursor down + Tab => Select task + move cursor down + Shift-Tab => Select task + move cursor up + Ctrl-A => Select all currently filtered tasks + Ctrl-T => Toggle select all currently filtered tasks + Ctrl-D => De-select all selected tasks (both filtered and not) + Alt-Bspace => Clear query from input bar + Enter => Accept selection and exit + Ctrl-C / Esc => Cancel selection and exit + ? => Toggle preview pane + + +The process for selecting tasks is otherwise the same as for a mouse. A particularly fast and +powerful way to select tasks is to: + +.. code-block:: text + + Write a precise query => Ctrl-A => Alt-Bspace => Repeat + +As before, when you are satisfied with your selection press ``Enter`` and all the tasks in the +preview pane will be pushed to try. If you change your mind you can press ``Esc`` or ``Ctrl-C`` to +exit the interface without pushing anything. + +.. note:: + + Initially ``fzf`` will automatically select whichever task is under your cursor. This is a + convenience feature for the case where you are only selecting a single task. This feature will be + turned off as soon as you *lock in* a selection with ``Right-Click``, ``Tab`` or ``Ctrl-A``. + + +Writing Queries +--------------- + +Queries are built from a series of terms, each separated by a space. Terms are logically joined by +the AND operator. For example: + +.. code-block:: text + + 'windows 'mochitest + +This query has two terms, and is the equivalent of saying: Give me all the tasks that match both the +term ``'windows'`` and the term ``'mochitest'``. In other words, this query matches all Windows +mochitest tasks. + +The single quote prefix before each term tells ``fzf`` to use exact substring matches, so only tasks +that contain both the literal string ``windows`` AND the literal string ``mochitest`` will be +matched. + +Another thing to note is that the order of the terms makes no difference, so ``'windows 'mochitest`` +and ``'mochitest 'windows`` are equivalent. + + +Fuzzy terms +~~~~~~~~~~~ + +If a term is *not* prefixed with a single quote, that makes it a fuzzy term. This means the +characters in the term need to show up in order, but not in sequence. E.g the fuzzy term ``max`` +would match the string ``mozilla firefox`` (as first there is an ``m``, then an ``a`` and finally an +``x``), but not the string ``firefox by mozilla`` (since the ``x`` is now out of order). Here's a +less contrived example: + +.. code-block:: text + + wndws mchtst + +Like the query above, this one would also select all Windows mochitest tasks. But it will +additionally select: + +.. code-block:: text + + test-macosx1014-64-shippable/opt-talos-sessionrestore-many-windows-e10s + +This is because both sequences of letters (``wndws`` and ``mchtst``) independently appear in order +somewhere in this string (remember the order of the terms makes no difference). + +At first fuzzy terms may not seem very useful, but they are actually extremely powerful! Let's use +the term from the interface image above, ``'linux64 mochibrochr``, as an example. First, just notice +how in the image ``fzf`` highlights the characters that constitute the match in green. Next, notice +how typing ``mochibrochr`` can quickly get us all mochitest browser-chrome tasks. The power of fuzzy +terms is that you don't need to memorize the exact task labels you are looking for. Just start +typing something you think is vaguely correct and chances are you'll see the task you're looking for. + + +Term Modifiers +~~~~~~~~~~~~~~ + +The following modifiers can be applied to a search term: + +.. code-block:: text + + 'word => exact match (line must contain the literal string "word") + ^word => exact prefix match (line must start with literal "word") + word$ => exact suffix match (line must end with literal "word") + !word => exact negation match (line must not contain literal "word") + 'a | 'b => OR operator (joins two exact match operators together) + +For example: + +.. code-block:: text + + ^start 'exact | 'other !ignore fuzzy end$ + +would match the string: + +.. code-block:: text + + starting to bake isn't exactly fun, but pizza is yummy in the end + +.. note:: + + The best way to learn how to write queries is to run ``mach try fuzzy --no-push`` and play + around with all of these modifiers! + + +Specifying Queries on the Command Line +-------------------------------------- + +Sometimes it's more convenient to skip the interactive interface and specify a query on the command +line with ``-q/--query``. This is equivalent to opening the interface then typing: +````. + +For example: + +.. code-block:: shell + + # selects all mochitest tasks + $ mach try fuzzy --query "mochitest" + +You can pass in multiple queries at once and the results of each will be joined together: + +.. code-block:: shell + + # selects all mochitest and reftest tasks + $ mach try fuzzy -q "mochitest" -q "reftest" + +If instead you want the intersection of queries, you can pass in ``-x/--and``: + +.. code-block:: shell + + # selects all windows mochitest tasks + $ mach try fuzzy --and -q "mochitest" -q "windows" + + +Modifying Presets +~~~~~~~~~~~~~~~~~ + +:doc:`Presets <../presets>` make it easy to run a pre-determined set of tasks. But sometimes you +might not want to run that set exactly as is, you may only want to use the preset as a starting +point then add or remove tasks as needed. This can be accomplished with ``-q/--query`` or +``-i/--interactive``. + +Here are some examples of adding tasks to a preset: + +.. code-block:: shell + + # selects all perf tasks plus all mochitest-chrome tasks + $ mach try fuzzy --preset perf -q "mochitest-chrome" + + # adds tasks to the perf preset interactively + $ mach try fuzzy --preset perf -i + +Similarly, ``-x/--and`` can be used to filter down a preset by taking the intersection of the two +sets: + +.. code-block:: shell + + # limits perf tasks to windows only + $ mach try fuzzy --preset perf -xq "windows" + + # limits perf tasks interactively + $ mach try fuzzy --preset perf -xi + + +Shell Conflicts +~~~~~~~~~~~~~~~ + +Unfortunately ``fzf``'s query language uses some characters (namely ``'``, ``!`` and ``$``) that can +interfere with your shell when using ``-q/--query``. Below are some tips for how to type out a query +on the command line. + +The ``!`` character is typically used for history expansion. If you don't use this feature, the +easiest way to specify queries on the command line is to disable it: + +.. code-block:: shell + + # bash + $ set +H + $ ./mach try fuzzy -q "'foo !bar" + + # zsh + $ setopt no_banghist + $ ./mach try fuzzy -q "'foo !bar" + +If using ``bash``, add ``set +H`` to your ``~/.bashrc``, ``~/.bash_profile`` or equivalent. If using +``zsh``, add ``setopt no_banghist`` to your ``~/.zshrc`` or equivalent. + +If you don't want to disable history expansion, you can escape your queries like this: + +.. code-block:: shell + + # bash + $ ./mach try fuzzy -q $'\'foo !bar' + + # zsh + $ ./mach try fuzzy -q "'foo \!bar" + + +The third option is to use ``-e/--exact`` which reverses the behaviour of the ``'`` character (see +:ref:`additional-arguments` for more details). Using this flag means you won't need to escape the +``'`` character as often and allows you to run your queries like this: + +.. code-block:: shell + + # bash and zsh + $ ./mach try fuzzy -eq 'foo !bar' + +This method is only useful if you find you almost always prefix terms with ``'`` (and rarely use +fuzzy terms). Otherwise as soon as you want to use a fuzzy match you'll run into the same problem as +before. + +.. note:: All the examples in these three approaches will select the same set of tasks. + +If you use ``fish`` shell, you won't need to escape ``!``, however you will need to escape ``$``: + +.. code-block:: shell + + # fish + $ ./mach try fuzzy -q "'foo !bar baz\$" + + +Test Paths +---------- + +One or more paths to a file or directory may be specified as positional arguments. When +specifying paths, the list of available tasks to choose from is filtered down such that +only suites that have tests in a specified path can be selected. Notably, only the first +chunk of each suite/platform appears. When the tasks are scheduled, only tests that live +under one of the specified paths will be run. + +.. note:: + + When using paths, be aware that all tests under the specified paths will run in the + same chunk. This might produce a different ordering from what gets run on production + branches, and may yield different results. + + For suites that restart the browser between each manifest (like mochitest), this + shouldn't be as big of a concern. + +Paths can be used with the interactive ``fzf`` window, or using the ``-q/--query`` argument. +For example, running: + +.. code-block:: shell + + $ mach try fuzzy layout/reftests/reftest-sanity -q "!pgo !cov !asan 'linux64" + +Would produce the following ``try_task_config.json``: + +.. code-block:: json + + { + "env":{ + "MOZHARNESS_TEST_PATHS":"{\"reftest\":\"layout/reftests/reftest-sanity\"}" + }, + "tasks":[ + "test-linux64-qr/debug-reftest-e10s-1", + "test-linux64-qr/opt-reftest-e10s-1", + "test-linux64/debug-reftest-e10s-1", + "test-linux64/debug-reftest-no-accel-e10s-1", + "test-linux64/opt-reftest-e10s-1", + "test-linux64/opt-reftest-no-accel-e10s-1", + ] + } + +Inside of these tasks, the reftest harness will only run tests that live under +``layout/reftests/reftest-sanity``. + + +.. _additional-arguments: + +Additional Arguments +-------------------- + +There are a few additional command line arguments you may wish to use: + +``-e/--exact`` +By default, ``fzf`` treats terms as a fuzzy match and prefixing a term with ``'`` turns it into an exact +match. If passing in ``--exact``, this behaviour is reversed. Non-prefixed terms become exact, and a +``'`` prefix makes a term fuzzy. + +``--full`` +By default, only target tasks (e.g tasks that would normally run on mozilla-central) +are generated. Passing in ``--full`` allows you to select from all tasks. This is useful for +things like nightly or release tasks. + +``-u/--update`` +Update the bootstrapped ``fzf`` binary to the latest version. + +For a full list of command line arguments, run: + +.. code-block:: shell + + $ mach try fuzzy --help + +For more information on using ``fzf``, run: + +.. code-block:: shell + + $ man fzf + +.. _fzf: https://github.com/junegunn/fzf diff --git a/tools/tryselect/docs/selectors/fzf.png b/tools/tryselect/docs/selectors/fzf.png new file mode 100644 index 0000000000..a64f4b04f3 Binary files /dev/null and b/tools/tryselect/docs/selectors/fzf.png differ diff --git a/tools/tryselect/docs/selectors/index.rst b/tools/tryselect/docs/selectors/index.rst new file mode 100644 index 0000000000..be618202d6 --- /dev/null +++ b/tools/tryselect/docs/selectors/index.rst @@ -0,0 +1,44 @@ +Selectors +========= + +These are the currently implemented try selectors: + +* :doc:`auto `: Have tasks chosen for you automatically. +* :doc:`fuzzy `: Select tasks using a fuzzy finding algorithm and + a terminal interface. +* :doc:`chooser `: Select tasks using a web interface. +* :doc:`again `: Re-run a previous ``try_task_config.json`` based + push. +* :doc:`empty `: Don't select any tasks. Taskcluster will still run + some tasks automatically (like lint and python unittest tasks). Further tasks + can be chosen with treeherder's ``Add New Jobs`` feature. +* :doc:`syntax `: Select tasks using classic try syntax. +* :doc:`release `: Prepare a tree for doing a staging release. +* :doc:`scriptworker `: Run scriptworker tasks against a recent release. +* :doc:`compare `: Push two identical try jobs, one on your current commit and another of your choice + +You can run them with: + +.. code-block:: shell + + $ mach try + +See selector specific options by running: + +.. code-block:: shell + + $ mach try --help + +.. toctree:: + :caption: Available Selectors + :maxdepth: 1 + :hidden: + + Auto + Fuzzy + Chooser + Again + Empty + Syntax + Release + Scriptworker diff --git a/tools/tryselect/docs/selectors/release.rst b/tools/tryselect/docs/selectors/release.rst new file mode 100644 index 0000000000..946266249b --- /dev/null +++ b/tools/tryselect/docs/selectors/release.rst @@ -0,0 +1,31 @@ +Release Selector +================ + +This command configures the tree in preparation for doing a staging release, +and pushes the result to try. The changes that that are made include: + +- Updating the version number. +- Applying the migrations that are done as part of merge day. +- Disabling repacking most locales. (This can be disabled by passing ``--no-limit-locales``). + +For staging a beta release, run the following (with an appropriate version number): + +.. code-block:: shell + + $ mach try release --version 64.0b5 --migration central-to-beta + +For staging a final release (rc or patch), run the following (with an appropriate version number) + +.. code-block:: shell + + $ mach try release --version 64.0 --migration central-to-beta --migration beta-to-release + +Once the decision task is on the push is complete, you can start the release +through `staging ship-it instance `_\ [#shipit]_. + +.. note:: + + If pushing from beta or release, the corresponding migration should not be + passed, as they have already been applied. + +.. [#shipit] This is only available to release engineering and release management (as of 2018-10-15). diff --git a/tools/tryselect/docs/selectors/scriptworker.rst b/tools/tryselect/docs/selectors/scriptworker.rst new file mode 100644 index 0000000000..a3cba08cbe --- /dev/null +++ b/tools/tryselect/docs/selectors/scriptworker.rst @@ -0,0 +1,31 @@ +Scriptworker Selector +===================== + +This command runs a selection of scriptworker tasks against builds from a +recent release. This is aimed at release engineering, to test changes to +scriptworker implementations. It currently requires being connected to +Mozilla's internal datacenter VPN with access to shipit\ [#shipit]_. + +There are a number of preset groups of tasks to run. To run a particular +set of tasks, pass the name of the set to ``mach try scriptworker``: + +.. code-block:: shell + + $ mach try scriptworker linux-signing + +To get the list of task sets, along with the list of tasks they will run: + +.. code-block:: shell + + $ mach try scriptworker list + +The selector defaults to using tasks from the most recent beta, to use tasks +from a different release, pass ``--release-type ``: + +.. code-block:: shell + + $ mach try scriptworker --release-type release linux-signing + + +.. [#shipit] The shipit API is not currently publicly available, and is used + to find the release graph to pull previous tasks from. diff --git a/tools/tryselect/docs/selectors/syntax.rst b/tools/tryselect/docs/selectors/syntax.rst new file mode 100644 index 0000000000..b64bb65ab7 --- /dev/null +++ b/tools/tryselect/docs/selectors/syntax.rst @@ -0,0 +1,41 @@ +Syntax Selector +=============== + +.. warning:: + + Try syntax is antiquated and hard to understand. If you aren't already + familiar with try syntax, you might want to use the ``fuzzy`` selector + instead. + +Try syntax is a command line string that goes into the commit message. Using +``mach try syntax`` will automatically create a temporary commit with your +chosen syntax and then delete it again after pushing to try. + +Try syntax can contain all kinds of different options parsed by various +places in various repos, but the majority are parsed by `try_option_syntax.py`_. +The most common arguments include: + + * ``-b/--build`` - One of ``d``, ``o`` or ``do``. This is the build type, + either opt, debug or both (required). + * ``-p/--platform`` - The platforms you want to build and/or run tests on + (required). + * ``-u/--unittests`` - The test tasks you want to run (optional). + * ``-t/--talos`` - The talos tasks you want to run (optional). + +Here are some examples: + +.. code-block:: shell + + $ mach try syntax -b do -p linux,macosx64 -u mochitest-e10s-1,crashtest -t none + $ mach try syntax -b d -p win64 -u all + $ mach try syntax -b o -p linux64 + +Unfortunately, knowing the magic strings that make it work can be a bit of a +guessing game. If you are unsure of what string will get your task to run, try +using :doc:`mach try fuzzy ` instead. + +While using ``mach try syntax -b do -p all -u all -t all`` will work, heavy use +of ``all`` is discouraged as it consumes a lot of unnecessary resources (some of +which are hardware constrained). + +.. _try_option_syntax.py: https://searchfox.org/mozilla-central/source/taskcluster/gecko_taskgraph/try_option_syntax.py diff --git a/tools/tryselect/docs/tasks.rst b/tools/tryselect/docs/tasks.rst new file mode 100644 index 0000000000..61de9ec9ed --- /dev/null +++ b/tools/tryselect/docs/tasks.rst @@ -0,0 +1,152 @@ +Task Generation +=============== + +Many selectors (including ``chooser``, ``coverage`` and ``fuzzy``) source their available tasks +directly from the :ref:`taskgraph ` module by building the taskgraph +locally. This means that the list of available tasks will never be stale. While this is very +powerful, it comes with a large enough performance cost to get annoying (around twenty seconds). + +The result of the taskgraph generation will be cached, so this penalty will only be incurred +whenever a file in the ``/taskcluster`` directory is modified. Unfortunately this directory changes +pretty frequently, so developers can expect to rebuild the cache each time they pull in +``mozilla-central``. Developers who regularly work on ``/taskcluster`` can expect to rebuild even +more frequently. + + +Configuring Watchman +-------------------- + +It's possible to bypass this penalty completely by using the file watching service `watchman`_. If +you use the ``fsmonitor`` mercurial extension, you already have ``watchman`` installed. + +.. note:: + + If you aren't using `fsmonitor`_ but end up installng watchman anyway, you + might as well enable it for a faster Mercurial experience. + +Otherwise, `install watchman`_. If using Linux you'll likely run into the `inotify limits`_ outlined +on that page due to the size of ``mozilla-central``. You can `read this page`_ for more information +on how to bump the limits permanently. + +Next run the following commands: + +.. code-block:: shell + + $ cd path/to/mozilla-central + $ watchman watch . + $ watchman -j < tools/tryselect/watchman.json + +You should see output like: + +.. code-block:: json + + { + "triggerid": "rebuild-taskgraph-cache", + "disposition": "created", + "version": "20200920.192359.0" + } + +That's it. Now anytime a file under ``/taskcluster`` is modified (either by your editor, or by +updating version control), the taskgraph cache will be rebuilt in the background, allowing you to +skip the wait the next time you run ``mach try``. + +.. note:: + + Watchman triggers are persistent and don't need to be added more than once. + See `Managing Triggers`_ for how to remove a trigger. + +You can test that everything is working by running these commands: + +.. code-block:: shell + + $ statedir=`mach python -c "from mach.util import get_state_dir; print(get_state_dir(specific_to_topsrcdir=True))"` + $ rm -rf $statedir/cache/taskgraph + $ touch taskcluster/mach_commands.py + # wait a minute for generation to trigger and finish + $ ls $statedir/cache/taskgraph + +If the ``target_task_set`` file exists, you are good to go. If not you can look at the ``watchman`` +log to see if there were any errors. This typically lives somewhere like +``/usr/local/var/run/watchman/$USER-state/log``. In this case please file a bug under ``Firefox +Build System :: Try`` and include the relevant portion of the log. + + +Running Watchman on Startup +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Watchman is both a client and a service all in one. When running a ``watchman`` command, the client +binary will start the service in the background if it isn't running. This means on reboot the +service won't be running and you'll need to start the service each time by invoking the client +binary (e.g by running ``watchman version``). + +If you'd like this to happen automatically, you can use your favourite platform specific way of +running commands at startup (``crontab``, ``rc.local``, etc.). Watchman stores separate state for +each user, so be sure you run the command as the user that set up the triggers. + +Setting up a systemd Service +++++++++++++++++++++++++++++ + +If ``systemd`` is an option you can create a service: + +.. code-block:: ini + + [Unit] + Description=Watchman for %i + After=network.target + + [Service] + Type=simple + User=%i + ExecStart=/usr/local/bin/watchman --log-level 1 watch-list -f + ExecStop=/usr/local/bin/watchman shutdown-server + + [Install] + WantedBy=multi-user.target + +Save this to a file called ``/etc/systemd/system/watchman@.service``. Then run: + +.. code-block:: shell + + $ sudo systemctl enable watchman@$USER.service + $ sudo systemctl start watchman@$USER.service + +The next time you reboot, the watchman service should start automatically. + + +Managing Triggers +~~~~~~~~~~~~~~~~~ + +When adding a trigger watchman writes it to disk. Typically it'll be a path similar to +``/usr/local/var/run/watchman/$USER-state/state``. While editing that file by hand would work, the +watchman binary provides an interface for managing your triggers. + +To see all directories you are currently watching: + +.. code-block:: shell + + $ watchman watch-list + +To view triggers that are active in a specified watch: + +.. code-block:: shell + + $ watchman trigger-list + +To delete a trigger from a specified watch: + +.. code-block:: shell + + $ watchman trigger-del + +In the above two examples, replace ```` with the path of the watch, presumably +``mozilla-central``. Using ``.`` works as well if that is already your working directory. For more +information on managing triggers and a reference of other commands, see the `official docs`_. + + +.. _watchman: https://facebook.github.io/watchman/ +.. _fsmonitor: https://www.mercurial-scm.org/wiki/FsMonitorExtension +.. _install watchman: https://facebook.github.io/watchman/docs/install.html +.. _inotify limits: https://facebook.github.io/watchman/docs/install.html#linux-inotify-limits +.. _read this page: https://github.com/guard/listen/wiki/Increasing-the-amount-of-inotify-watchers +.. _this hint: https://github.com/facebook/watchman/commit/2985377eaf8c8538b28fae9add061b67991a87c2 +.. _official docs: https://facebook.github.io/watchman/docs/cmd/trigger.html diff --git a/tools/tryselect/mach_commands.py b/tools/tryselect/mach_commands.py new file mode 100644 index 0000000000..26c15a0990 --- /dev/null +++ b/tools/tryselect/mach_commands.py @@ -0,0 +1,528 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import argparse +import importlib +import os +import sys + +from mach.decorators import Command, SettingsProvider, SubCommand +from mach.util import get_state_dir +from mozbuild.base import BuildEnvironmentNotFoundException +from mozbuild.util import memoize + +CONFIG_ENVIRONMENT_NOT_FOUND = """ +No config environment detected. This means we are unable to properly +detect test files in the specified paths or tags. Please run: + + $ mach configure + +and try again. +""".lstrip() + + +class get_parser: + def __init__(self, selector): + self.selector = selector + + def __call__(self): + mod = importlib.import_module("tryselect.selectors.{}".format(self.selector)) + return getattr(mod, "{}Parser".format(self.selector.capitalize()))() + + +def generic_parser(): + from tryselect.cli import BaseTryParser + + parser = BaseTryParser() + parser.add_argument("argv", nargs=argparse.REMAINDER) + return parser + + +@SettingsProvider +class TryConfig: + @classmethod + def config_settings(cls): + from mach.registrar import Registrar + + desc = ( + "The default selector to use when running `mach try` without a subcommand." + ) + choices = Registrar.command_handlers["try"].subcommand_handlers.keys() + + return [ + ("try.default", "string", desc, "auto", {"choices": choices}), + ( + "try.maxhistory", + "int", + "Maximum number of pushes to save in history.", + 10, + ), + ] + + +def init(command_context): + from tryselect import push + + push.MAX_HISTORY = command_context._mach_context.settings["try"]["maxhistory"] + + +@memoize +def presets(command_context): + from tryselect.preset import MergedHandler + + # Create our handler using both local and in-tree presets. The first + # path in this list will be treated as the 'user' file for the purposes + # of saving and editing. All subsequent paths are 'read-only'. We check + # an environment variable first for testing purposes. + if os.environ.get("MACH_TRY_PRESET_PATHS"): + preset_paths = os.environ["MACH_TRY_PRESET_PATHS"].split(os.pathsep) + else: + preset_paths = [ + os.path.join(get_state_dir(), "try_presets.yml"), + os.path.join( + command_context.topsrcdir, "tools", "tryselect", "try_presets.yml" + ), + ] + + return MergedHandler(*preset_paths) + + +def handle_presets( + command_context, preset_action=None, save=None, preset=None, **kwargs +): + """Handle preset related arguments. + + This logic lives here so that the underlying selectors don't need + special preset handling. They can all save and load presets the same + way. + """ + from tryselect.util.dicttools import merge + + user_presets = presets(command_context).handlers[0] + if preset_action == "list": + presets(command_context).list() + sys.exit() + + if preset_action == "edit": + user_presets.edit() + sys.exit() + + parser = command_context._mach_context.handler.parser + subcommand = command_context._mach_context.handler.subcommand + if "preset" not in parser.common_groups: + return kwargs + + default = parser.get_default + if save: + selector = ( + subcommand or command_context._mach_context.settings["try"]["default"] + ) + + # Only save non-default values for simplicity. + kwargs = {k: v for k, v in kwargs.items() if v != default(k)} + user_presets.save(save, selector=selector, **kwargs) + print("preset saved, run with: --preset={}".format(save)) + sys.exit() + + if preset: + if preset not in presets(command_context): + command_context._mach_context.parser.error( + "preset '{}' does not exist".format(preset) + ) + + name = preset + preset = presets(command_context)[name] + selector = preset.pop("selector") + preset.pop("description", None) # description isn't used by any selectors + + if not subcommand: + subcommand = selector + elif subcommand != selector: + print( + "error: preset '{}' exists for a different selector " + "(did you mean to run 'mach try {}' instead?)".format(name, selector) + ) + sys.exit(1) + + # Order of precedence is defaults -> presets -> cli. Configuration + # from the right overwrites configuration from the left. + defaults = {} + nondefaults = {} + for k, v in kwargs.items(): + if v == default(k): + defaults[k] = v + else: + nondefaults[k] = v + + kwargs = merge(defaults, preset, nondefaults) + + return kwargs + + +def handle_try_config(command_context, **kwargs): + from tryselect.util.dicttools import merge + + to_validate = [] + kwargs.setdefault("try_config", {}) + for cls in command_context._mach_context.handler.parser.task_configs.values(): + try_config = cls.try_config(**kwargs) + if try_config is not None: + to_validate.append(cls) + kwargs["try_config"] = merge(kwargs["try_config"], try_config) + + for name in cls.dests: + del kwargs[name] + + # Validate task_configs after they have all been parsed to avoid + # depending on the order they were processed. + for cls in to_validate: + cls.validate(**kwargs) + return kwargs + + +def run(command_context, **kwargs): + kwargs = handle_presets(command_context, **kwargs) + + if command_context._mach_context.handler.parser.task_configs: + kwargs = handle_try_config(command_context, **kwargs) + + mod = importlib.import_module( + "tryselect.selectors.{}".format( + command_context._mach_context.handler.subcommand + ) + ) + return mod.run(**kwargs) + + +@Command( + "try", + category="ci", + description="Push selected tasks to the try server", + parser=generic_parser, +) +def try_default(command_context, argv=None, **kwargs): + """Push selected tests to the try server. + + The |mach try| command is a frontend for scheduling tasks to + run on try server using selectors. A selector is a subcommand + that provides its own set of command line arguments and are + listed below. + + If no subcommand is specified, the `auto` selector is run by + default. Run |mach try auto --help| for more information on + scheduling with the `auto` selector. + """ + init(command_context) + subcommand = command_context._mach_context.handler.subcommand + # We do special handling of presets here so that `./mach try --preset foo` + # works no matter what subcommand 'foo' was saved with. + preset = kwargs["preset"] + if preset: + if preset not in presets(command_context): + command_context._mach_context.handler.parser.error( + "preset '{}' does not exist".format(preset) + ) + + subcommand = presets(command_context)[preset]["selector"] + + sub = subcommand or command_context._mach_context.settings["try"]["default"] + return command_context._mach_context.commands.dispatch( + "try", command_context._mach_context, subcommand=sub, argv=argv, **kwargs + ) + + +@SubCommand( + "try", + "fuzzy", + description="Select tasks on try using a fuzzy finder", + parser=get_parser("fuzzy"), +) +def try_fuzzy(command_context, **kwargs): + """Select which tasks to run with a fuzzy finding interface (fzf). + + When entering the fzf interface you'll be confronted by two panes. The + one on the left contains every possible task you can schedule, the one + on the right contains the list of selected tasks. In other words, the + tasks that will be scheduled once you you press . + + At first fzf will automatically select whichever task is under your + cursor, which simplifies the case when you are looking for a single + task. But normally you'll want to select many tasks. To accomplish + you'll generally start by typing a query in the search bar to filter + down the list of tasks (see Extended Search below). Then you'll either: + + A) Move the cursor to each task you want and press to select it. + Notice it now shows up in the pane to the right. + + OR + + B) Press to select every task that matches your filter. + + You can delete your query, type a new one and select further tasks as + many times as you like. Once you are happy with your selection, press + to push the selected tasks to try. + + All selected task labels and their dependencies will be scheduled. This + means you can select a test task and its build will automatically be + filled in. + + + Keyboard Shortcuts + ------------------ + + When in the fuzzy finder interface, start typing to filter down the + task list. Then use the following keyboard shortcuts to select tasks: + + Ctrl-K / Up => Move cursor up + Ctrl-J / Down => Move cursor down + Tab => Select task + move cursor down + Shift-Tab => Select task + move cursor up + Ctrl-A => Select all currently filtered tasks + Ctrl-D => De-select all currently filtered tasks + Ctrl-T => Toggle select all currently filtered tasks + Alt-Bspace => Clear query from input bar + Enter => Accept selection and exit + Ctrl-C / Esc => Cancel selection and exit + ? => Toggle preview pane + + There are many more shortcuts enabled by default, you can also define + your own shortcuts by setting `--bind` in the $FZF_DEFAULT_OPTS + environment variable. See `man fzf` for more info. + + + Extended Search + --------------- + + When typing in search terms, the following modifiers can be applied: + + 'word: exact match (line must contain the literal string "word") + ^word: exact prefix match (line must start with literal "word") + word$: exact suffix match (line must end with literal "word") + !word: exact negation match (line must not contain literal "word") + 'a | 'b: OR operator (joins two exact match operators together) + + For example: + + ^start 'exact | !ignore fuzzy end$ + + + Documentation + ------------- + + For more detailed documentation, please see: + https://firefox-source-docs.mozilla.org/tools/try/selectors/fuzzy.html + """ + init(command_context) + if kwargs.pop("interactive"): + kwargs["query"].append("INTERACTIVE") + + if kwargs.pop("intersection"): + kwargs["intersect_query"] = kwargs["query"] + del kwargs["query"] + + if kwargs.get("save") and not kwargs.get("query"): + # If saving preset without -q/--query, allow user to use the + # interface to build the query. + kwargs_copy = kwargs.copy() + kwargs_copy["dry_run"] = True + kwargs_copy["save"] = None + kwargs["query"] = run(command_context, save_query=True, **kwargs_copy) + if not kwargs["query"]: + return + + if kwargs.get("paths"): + kwargs["test_paths"] = kwargs["paths"] + + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "chooser", + description="Schedule tasks by selecting them from a web interface.", + parser=get_parser("chooser"), +) +def try_chooser(command_context, **kwargs): + """Push tasks selected from a web interface to try. + + This selector will build the taskgraph and spin up a dynamically + created 'trychooser-like' web-page on the localhost. After a selection + has been made, pressing the 'Push' button will automatically push the + selection to try. + """ + init(command_context) + command_context.activate_virtualenv() + path = os.path.join( + "tools", "tryselect", "selectors", "chooser", "requirements.txt" + ) + command_context.virtualenv_manager.install_pip_requirements(path, quiet=True) + + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "auto", + description="Automatically determine which tasks to run. This runs the same " + "set of tasks that would be run on autoland. This " + "selector is EXPERIMENTAL.", + parser=get_parser("auto"), +) +def try_auto(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "again", + description="Schedule a previously generated (non try syntax) push again.", + parser=get_parser("again"), +) +def try_again(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "empty", + description="Push to try without scheduling any tasks.", + parser=get_parser("empty"), +) +def try_empty(command_context, **kwargs): + """Push to try, running no builds or tests + + This selector does not prompt you to run anything, it just pushes + your patches to try, running no builds or tests by default. After + the push finishes, you can manually add desired jobs to your push + via Treeherder's Add New Jobs feature, located in the per-push + menu. + """ + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "syntax", + description="Select tasks on try using try syntax", + parser=get_parser("syntax"), +) +def try_syntax(command_context, **kwargs): + """Push the current tree to try, with the specified syntax. + + Build options, platforms and regression tests may be selected + using the usual try options (-b, -p and -u respectively). In + addition, tests in a given directory may be automatically + selected by passing that directory as a positional argument to the + command. For example: + + mach try -b d -p linux64 dom testing/web-platform/tests/dom + + would schedule a try run for linux64 debug consisting of all + tests under dom/ and testing/web-platform/tests/dom. + + Test selection using positional arguments is available for + mochitests, reftests, xpcshell tests and web-platform-tests. + + Tests may be also filtered by passing --tag to the command, + which will run only tests marked as having the specified + tags e.g. + + mach try -b d -p win64 --tag media + + would run all tests tagged 'media' on Windows 64. + + If both positional arguments or tags and -u are supplied, the + suites in -u will be run in full. Where tests are selected by + positional argument they will be run in a single chunk. + + If no build option is selected, both debug and opt will be + scheduled. If no platform is selected a default is taken from + the AUTOTRY_PLATFORM_HINT environment variable, if set. + + The command requires either its own mercurial extension ("push-to-try", + installable from mach vcs-setup) or a git repo using git-cinnabar + (installable from mach vcs-setup). + + """ + init(command_context) + try: + if command_context.substs.get("MOZ_ARTIFACT_BUILDS"): + kwargs["local_artifact_build"] = True + except BuildEnvironmentNotFoundException: + # If we don't have a build locally, we can't tell whether + # an artifact build is desired, but we still want the + # command to succeed, if possible. + pass + + config_status = os.path.join(command_context.topobjdir, "config.status") + if (kwargs["paths"] or kwargs["tags"]) and not config_status: + print(CONFIG_ENVIRONMENT_NOT_FOUND) + sys.exit(1) + + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "coverage", + description="Select tasks on try using coverage data", + parser=get_parser("coverage"), +) +def try_coverage(command_context, **kwargs): + """Select which tasks to use using coverage data.""" + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "release", + description="Push the current tree to try, configured for a staging release.", + parser=get_parser("release"), +) +def try_release(command_context, **kwargs): + """Push the current tree to try, configured for a staging release.""" + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "scriptworker", + description="Run scriptworker tasks against a recent release.", + parser=get_parser("scriptworker"), +) +def try_scriptworker(command_context, **kwargs): + """Run scriptworker tasks against a recent release. + + Requires VPN and shipit access. + """ + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "compare", + description="Push two try jobs, one on your current commit and another on the one you specify", + parser=get_parser("compare"), +) +def try_compare(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) + + +@SubCommand( + "try", + "perf", + description="Try selector for running performance tests.", + parser=get_parser("perf"), +) +def try_perf(command_context, **kwargs): + init(command_context) + return run(command_context, **kwargs) diff --git a/tools/tryselect/preset.py b/tools/tryselect/preset.py new file mode 100644 index 0000000000..dc8cba5c57 --- /dev/null +++ b/tools/tryselect/preset.py @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import shlex +import subprocess + +import yaml + + +# Ensure strings with ' like fzf query strings get double-quoted: +def represent_str(self, data): + if "'" in data: + return self.represent_scalar("tag:yaml.org,2002:str", data, style='"') + return self.represent_scalar("tag:yaml.org,2002:str", data) + + +yaml.SafeDumper.add_representer(str, represent_str) + + +class PresetHandler: + def __init__(self, path): + self.path = path + self._presets = {} + + @property + def presets(self): + if not self._presets and os.path.isfile(self.path): + with open(self.path) as fh: + self._presets = yaml.safe_load(fh) or {} + + return self._presets + + def __contains__(self, name): + return name in self.presets + + def __getitem__(self, name): + return self.presets[name] + + def __len__(self): + return len(self.presets) + + def __str__(self): + if not self.presets: + return "" + return yaml.safe_dump(self.presets, default_flow_style=False) + + def list(self): + if not self.presets: + print("no presets found") + else: + print(self) + + def edit(self): + if "EDITOR" not in os.environ: + print( + "error: must set the $EDITOR environment variable to use --edit-presets" + ) + return + + subprocess.call(shlex.split(os.environ["EDITOR"]) + [self.path]) + + def save(self, name, **data): + self.presets[name] = data + + with open(self.path, "w") as fh: + fh.write(str(self)) + + +class MergedHandler: + def __init__(self, *paths): + """Helper class for dealing with multiple preset files.""" + self.handlers = [PresetHandler(p) for p in paths] + + def __contains__(self, name): + return any(name in handler for handler in self.handlers) + + def __getitem__(self, name): + for handler in self.handlers: + if name in handler: + return handler[name] + raise KeyError(name) + + def __len__(self): + return sum(len(h) for h in self.handlers) + + def __str__(self): + all_presets = { + k: v for handler in self.handlers for k, v in handler.presets.items() + } + return yaml.safe_dump(all_presets, default_flow_style=False) + + def list(self): + if len(self) == 0: + print("no presets found") + return + + for handler in self.handlers: + val = str(handler) + if val: + val = "\n ".join( + [""] + val.splitlines() + [""] + ) # indent all lines by 2 spaces + print("Presets from {}:".format(handler.path)) + print(val) diff --git a/tools/tryselect/push.py b/tools/tryselect/push.py new file mode 100644 index 0000000000..cade05db9b --- /dev/null +++ b/tools/tryselect/push.py @@ -0,0 +1,239 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import sys +import traceback + +import six +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozversioncontrol import MissingVCSExtension, get_repository_object + +from .util.estimates import duration_summary +from .util.manage_estimates import ( + download_task_history_data, + make_trimmed_taskgraph_cache, +) + +GIT_CINNABAR_NOT_FOUND = """ +Could not detect `git-cinnabar`. + +The `mach try` command requires git-cinnabar to be installed when +pushing from git. Please install it by running: + + $ ./mach vcs-setup +""".lstrip() + +HG_PUSH_TO_TRY_NOT_FOUND = """ +Could not detect `push-to-try`. + +The `mach try` command requires the push-to-try extension enabled +when pushing from hg. Please install it by running: + + $ ./mach vcs-setup +""".lstrip() + +VCS_NOT_FOUND = """ +Could not detect version control. Only `hg` or `git` are supported. +""".strip() + +UNCOMMITTED_CHANGES = """ +ERROR please commit changes before continuing +""".strip() + +MAX_HISTORY = 10 + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) +vcs = get_repository_object(build.topsrcdir) + +history_path = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "history", "try_task_configs.json" +) + + +def write_task_config(try_task_config): + config_path = os.path.join(vcs.path, "try_task_config.json") + with open(config_path, "w") as fh: + json.dump(try_task_config, fh, indent=4, separators=(",", ": "), sort_keys=True) + fh.write("\n") + return config_path + + +def write_task_config_history(msg, try_task_config): + if not os.path.isfile(history_path): + if not os.path.isdir(os.path.dirname(history_path)): + os.makedirs(os.path.dirname(history_path)) + history = [] + else: + with open(history_path) as fh: + history = fh.read().strip().splitlines() + + history.insert(0, json.dumps([msg, try_task_config])) + history = history[:MAX_HISTORY] + with open(history_path, "w") as fh: + fh.write("\n".join(history)) + + +def check_working_directory(push=True): + if not push: + return + + if not vcs.working_directory_clean(): + print(UNCOMMITTED_CHANGES) + sys.exit(1) + + +def generate_try_task_config(method, labels, try_config=None, routes=None): + try_task_config = try_config or {} + try_task_config.setdefault("env", {})["TRY_SELECTOR"] = method + try_task_config.update( + { + "version": 1, + "tasks": sorted(labels), + } + ) + if routes: + try_task_config["routes"] = routes + + return try_task_config + + +def task_labels_from_try_config(try_task_config): + if try_task_config["version"] == 2: + parameters = try_task_config.get("parameters", {}) + if parameters.get("try_mode") == "try_task_config": + return parameters["try_task_config"]["tasks"] + else: + return None + elif try_task_config["version"] == 1: + return try_task_config.get("tasks", list()) + else: + return None + + +def display_push_estimates(try_task_config): + task_labels = task_labels_from_try_config(try_task_config) + if task_labels is None: + return + + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + + graph_cache = None + dep_cache = None + target_file = None + for graph_cache_file in ["target_task_graph", "full_task_graph"]: + graph_cache = os.path.join(cache_dir, graph_cache_file) + if os.path.isfile(graph_cache): + dep_cache = graph_cache.replace("task_graph", "task_dependencies") + target_file = graph_cache.replace("task_graph", "task_set") + break + + if not dep_cache: + return + + download_task_history_data(cache_dir=cache_dir) + make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_file) + + durations = duration_summary(dep_cache, task_labels, cache_dir) + + print( + "estimates: Runs {} tasks ({} selected, {} dependencies)".format( + durations["dependency_count"] + durations["selected_count"], + durations["selected_count"], + durations["dependency_count"], + ) + ) + print( + "estimates: Total task duration {}".format( + durations["dependency_duration"] + durations["selected_duration"] + ) + ) + if "percentile" in durations: + print( + "estimates: In the top {}% of durations".format( + 100 - durations["percentile"] + ) + ) + print( + "estimates: Should take about {} (Finished around {})".format( + durations["wall_duration_seconds"], + durations["eta_datetime"].strftime("%Y-%m-%d %H:%M"), + ) + ) + + +def push_to_try( + method, + msg, + try_task_config=None, + stage_changes=False, + dry_run=False, + closed_tree=False, + files_to_change=None, + allow_log_capture=False, +): + push = not stage_changes and not dry_run + check_working_directory(push) + + if try_task_config and method not in ("auto", "empty"): + try: + display_push_estimates(try_task_config) + except Exception: + traceback.print_exc() + print("warning: unable to display push estimates") + + # Format the commit message + closed_tree_string = " ON A CLOSED TREE" if closed_tree else "" + commit_message = "{}{}\n\nPushed via `mach try {}`".format( + msg, + closed_tree_string, + method, + ) + + config_path = None + changed_files = [] + if try_task_config: + if push and method not in ("again", "auto", "empty"): + write_task_config_history(msg, try_task_config) + config_path = write_task_config(try_task_config) + changed_files.append(config_path) + + if (push or stage_changes) and files_to_change: + for path, content in files_to_change.items(): + path = os.path.join(vcs.path, path) + with open(path, "wb") as fh: + fh.write(six.ensure_binary(content)) + changed_files.append(path) + + try: + if not push: + print("Commit message:") + print(commit_message) + if config_path: + print("Calculated try_task_config.json:") + with open(config_path) as fh: + print(fh.read()) + return + + vcs.add_remove_files(*changed_files) + + try: + vcs.push_to_try(commit_message, allow_log_capture=allow_log_capture) + except MissingVCSExtension as e: + if e.ext == "push-to-try": + print(HG_PUSH_TO_TRY_NOT_FOUND) + elif e.ext == "cinnabar": + print(GIT_CINNABAR_NOT_FOUND) + else: + raise + sys.exit(1) + finally: + if config_path and os.path.isfile(config_path): + os.remove(config_path) diff --git a/tools/tryselect/selectors/__init__.py b/tools/tryselect/selectors/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/selectors/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/selectors/again.py b/tools/tryselect/selectors/again.py new file mode 100644 index 0000000000..434aed7cc1 --- /dev/null +++ b/tools/tryselect/selectors/again.py @@ -0,0 +1,151 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os + +from ..cli import BaseTryParser +from ..push import history_path, push_to_try + + +class AgainParser(BaseTryParser): + name = "again" + arguments = [ + [ + ["--index"], + { + "default": 0, + "const": "list", + "nargs": "?", + "help": "Index of entry in the history to re-push, " + "where '0' is the most recent (default 0). " + "Use --index without a value to display indices.", + }, + ], + [ + ["--list"], + { + "default": False, + "action": "store_true", + "dest": "list_configs", + "help": "Display history and exit", + }, + ], + [ + ["--list-tasks"], + { + "default": 0, + "action": "count", + "dest": "list_tasks", + "help": "Like --list, but display selected tasks " + "for each history entry, up to 10. Repeat " + "to display all selected tasks.", + }, + ], + [ + ["--purge"], + { + "default": False, + "action": "store_true", + "help": "Remove all history and exit", + }, + ], + ] + common_groups = ["push"] + + +def run( + index=0, purge=False, list_configs=False, list_tasks=0, message="{msg}", **pushargs +): + if index == "list": + list_configs = True + else: + try: + index = int(index) + except ValueError: + print("error: '--index' must be an integer") + return 1 + + if purge: + os.remove(history_path) + return + + if not os.path.isfile(history_path): + print("error: history file not found: {}".format(history_path)) + return 1 + + with open(history_path) as fh: + history = fh.readlines() + + if list_configs or list_tasks > 0: + for i, data in enumerate(history): + msg, config = json.loads(data) + version = config.get("version", "1") + settings = {} + if version == 1: + tasks = config["tasks"] + settings = config + elif version == 2: + try_config = config.get("parameters", {}).get("try_task_config", {}) + tasks = try_config.get("tasks") + else: + tasks = None + + if tasks is not None: + # Select only the things that are of interest to display. + settings = settings.copy() + env = settings.pop("env", {}).copy() + env.pop("TRY_SELECTOR", None) + for name in ("tasks", "version"): + settings.pop(name, None) + + def pluralize(n, noun): + return "{n} {noun}{s}".format( + n=n, noun=noun, s="" if n == 1 else "s" + ) + + out = str(i) + ". (" + pluralize(len(tasks), "task") + if env: + out += ", " + pluralize(len(env), "env var") + if settings: + out += ", " + pluralize(len(settings), "setting") + out += ") " + msg + print(out) + + if list_tasks > 0: + indent = " " * 4 + if list_tasks > 1: + shown_tasks = tasks + else: + shown_tasks = tasks[:10] + print(indent + ("\n" + indent).join(shown_tasks)) + + num_hidden_tasks = len(tasks) - len(shown_tasks) + if num_hidden_tasks > 0: + print("{}... and {} more".format(indent, num_hidden_tasks)) + + if list_tasks and env: + for line in ("env: " + json.dumps(env, indent=2)).splitlines(): + print(" " + line) + + if list_tasks and settings: + for line in ( + "settings: " + json.dumps(settings, indent=2) + ).splitlines(): + print(" " + line) + else: + print( + "{index}. {msg}".format( + index=i, + msg=msg, + ) + ) + + return + + msg, try_task_config = json.loads(history[index]) + return push_to_try( + "again", message.format(msg=msg), try_task_config=try_task_config, **pushargs + ) diff --git a/tools/tryselect/selectors/auto.py b/tools/tryselect/selectors/auto.py new file mode 100644 index 0000000000..7b27b33616 --- /dev/null +++ b/tools/tryselect/selectors/auto.py @@ -0,0 +1,115 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.util.python_path import find_object + +from ..cli import BaseTryParser +from ..push import push_to_try + +TRY_AUTO_PARAMETERS = { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", # noqa + "optimize_target_tasks": True, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {}, +} + + +class AutoParser(BaseTryParser): + name = "auto" + common_groups = ["push"] + task_configs = [ + "artifact", + "env", + "chemspill-prio", + "disable-pgo", + "worker-overrides", + ] + arguments = [ + [ + ["--strategy"], + { + "default": None, + "help": "Override the default optimization strategy. Valid values " + "are the experimental strategies defined at the bottom of " + "`taskcluster/gecko_taskgraph/optimize/__init__.py`.", + }, + ], + [ + ["--tasks-regex"], + { + "default": [], + "action": "append", + "help": "Apply a regex filter to the tasks selected. Specifying " + "multiple times schedules the union of computed tasks.", + }, + ], + [ + ["--tasks-regex-exclude"], + { + "default": [], + "action": "append", + "help": "Apply a regex filter to the tasks selected. Specifying " + "multiple times excludes computed tasks matching any regex.", + }, + ], + ] + + def validate(self, args): + super().validate(args) + + if args.strategy: + if ":" not in args.strategy: + args.strategy = "gecko_taskgraph.optimize:tryselect.{}".format( + args.strategy + ) + + try: + obj = find_object(args.strategy) + except (ImportError, AttributeError): + self.error("invalid module path '{}'".format(args.strategy)) + + if not isinstance(obj, dict): + self.error("object at '{}' must be a dict".format(args.strategy)) + + +def run( + message="{msg}", + stage_changes=False, + dry_run=False, + closed_tree=False, + strategy=None, + tasks_regex=None, + tasks_regex_exclude=None, + try_config=None, + **ignored +): + msg = message.format(msg="Tasks automatically selected.") + + params = TRY_AUTO_PARAMETERS.copy() + if try_config: + params["try_task_config"] = try_config + + if strategy: + params["optimize_strategies"] = strategy + + if tasks_regex or tasks_regex_exclude: + params.setdefault("try_task_config", {})["tasks-regex"] = {} + params["try_task_config"]["tasks-regex"]["include"] = tasks_regex + params["try_task_config"]["tasks-regex"]["exclude"] = tasks_regex_exclude + + task_config = { + "version": 2, + "parameters": params, + } + return push_to_try( + "auto", + msg, + try_task_config=task_config, + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/chooser/.eslintrc.js b/tools/tryselect/selectors/chooser/.eslintrc.js new file mode 100644 index 0000000000..861d6bafc2 --- /dev/null +++ b/tools/tryselect/selectors/chooser/.eslintrc.js @@ -0,0 +1,16 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +module.exports = { + env: { + jquery: true, + }, + globals: { + apply: true, + applyChunks: true, + tasks: true, + }, +}; diff --git a/tools/tryselect/selectors/chooser/__init__.py b/tools/tryselect/selectors/chooser/__init__.py new file mode 100644 index 0000000000..b71cf801ea --- /dev/null +++ b/tools/tryselect/selectors/chooser/__init__.py @@ -0,0 +1,97 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import webbrowser +from threading import Timer + +from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks + +from tryselect.cli import BaseTryParser +from tryselect.push import ( + check_working_directory, + generate_try_task_config, + push_to_try, +) +from tryselect.tasks import generate_tasks + +here = os.path.abspath(os.path.dirname(__file__)) + + +class ChooserParser(BaseTryParser): + name = "chooser" + arguments = [] + common_groups = ["push", "task"] + task_configs = [ + "artifact", + "browsertime", + "chemspill-prio", + "disable-pgo", + "env", + "gecko-profile", + "path", + "pernosco", + "rebuild", + "worker-overrides", + ] + + +def run( + update=False, + query=None, + try_config=None, + full=False, + parameters=None, + save=False, + preset=None, + mod_presets=False, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, +): + from .app import create_application + + push = not stage_changes and not dry_run + check_working_directory(push) + + tg = generate_tasks(parameters, full) + + # Remove tasks that are not to be shown unless `--full` is specified. + if not full: + blacklisted_tasks = [ + label + for label in tg.tasks.keys() + if not filter_by_uncommon_try_tasks(label) + ] + for task in blacklisted_tasks: + tg.tasks.pop(task) + + app = create_application(tg) + + if os.environ.get("WERKZEUG_RUN_MAIN") == "true": + # we are in the reloader process, don't open the browser or do any try stuff + app.run() + return + + # give app a second to start before opening the browser + url = "http://127.0.0.1:5000" + Timer(1, lambda: webbrowser.open(url)).start() + print("Starting trychooser on {}".format(url)) + app.run() + + selected = app.tasks + if not selected: + print("no tasks selected") + return + + msg = "Try Chooser Enhanced ({} tasks selected)".format(len(selected)) + return push_to_try( + "chooser", + message.format(msg=msg), + try_task_config=generate_try_task_config("chooser", selected, try_config), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/chooser/app.py b/tools/tryselect/selectors/chooser/app.py new file mode 100644 index 0000000000..adbf6f33dd --- /dev/null +++ b/tools/tryselect/selectors/chooser/app.py @@ -0,0 +1,177 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from abc import ABCMeta, abstractproperty +from collections import defaultdict + +from flask import Flask, render_template, request + +SECTIONS = [] +SUPPORTED_KINDS = set() + + +def register_section(cls): + assert issubclass(cls, Section) + instance = cls() + SECTIONS.append(instance) + SUPPORTED_KINDS.update(instance.kind.split(",")) + + +class Section(object): + __metaclass__ = ABCMeta + + @abstractproperty + def name(self): + pass + + @abstractproperty + def kind(self): + pass + + @abstractproperty + def title(self): + pass + + @abstractproperty + def attrs(self): + pass + + def contains(self, task): + return task.kind in self.kind.split(",") + + def get_context(self, tasks): + labels = defaultdict(lambda: {"max_chunk": 0, "attrs": defaultdict(list)}) + + for task in tasks.values(): + if not self.contains(task): + continue + + task = task.attributes + label = labels[self.labelfn(task)] + for attr in self.attrs: + if attr in task and task[attr] not in label["attrs"][attr]: + label["attrs"][attr].append(task[attr]) + + if "test_chunk" in task: + label["max_chunk"] = max( + label["max_chunk"], int(task["test_chunk"]) + ) + + return { + "name": self.name, + "kind": self.kind, + "title": self.title, + "labels": labels, + } + + +@register_section +class Platform(Section): + name = "platform" + kind = "build" + title = "Platforms" + attrs = ["build_platform"] + + def labelfn(self, task): + return task["build_platform"] + + def contains(self, task): + if not Section.contains(self, task): + return False + + # android-stuff tasks aren't actual platforms + return task.task["tags"].get("android-stuff", False) != "true" + + +@register_section +class Test(Section): + name = "test" + kind = "test" + title = "Test Suites" + attrs = ["unittest_suite"] + + def labelfn(self, task): + suite = task["unittest_suite"].replace(" ", "-") + + if suite.endswith("-chunked"): + suite = suite[: -len("-chunked")] + + return suite + + def contains(self, task): + if not Section.contains(self, task): + return False + return task.attributes["unittest_suite"] not in ("raptor", "talos") + + +@register_section +class Perf(Section): + name = "perf" + kind = "test" + title = "Performance" + attrs = ["unittest_suite", "raptor_try_name", "talos_try_name"] + + def labelfn(self, task): + suite = task["unittest_suite"] + label = task["{}_try_name".format(suite)] + + if not label.startswith(suite): + label = "{}-{}".format(suite, label) + + if label.endswith("-e10s"): + label = label[: -len("-e10s")] + + return label + + def contains(self, task): + if not Section.contains(self, task): + return False + return task.attributes["unittest_suite"] in ("raptor", "talos") + + +@register_section +class Analysis(Section): + name = "analysis" + kind = "build,static-analysis-autotest" + title = "Analysis" + attrs = ["build_platform"] + + def labelfn(self, task): + return task["build_platform"] + + def contains(self, task): + if not Section.contains(self, task): + return False + if task.kind == "build": + return task.task["tags"].get("android-stuff", False) == "true" + return True + + +def create_application(tg): + tasks = {l: t for l, t in tg.tasks.items() if t.kind in SUPPORTED_KINDS} + sections = [s.get_context(tasks) for s in SECTIONS] + context = { + "tasks": {l: t.attributes for l, t in tasks.items()}, + "sections": sections, + } + + app = Flask(__name__) + app.env = "development" + app.tasks = [] + + @app.route("/", methods=["GET", "POST"]) + def chooser(): + if request.method == "GET": + return render_template("chooser.html", **context) + + if request.form["action"] == "Push": + labels = request.form["selected-tasks"].splitlines() + app.tasks.extend(labels) + + shutdown = request.environ.get("werkzeug.server.shutdown") + if shutdown: + shutdown() + return render_template("close.html") + + return app diff --git a/tools/tryselect/selectors/chooser/requirements.txt b/tools/tryselect/selectors/chooser/requirements.txt new file mode 100644 index 0000000000..966eae636f --- /dev/null +++ b/tools/tryselect/selectors/chooser/requirements.txt @@ -0,0 +1,44 @@ +Flask==1.1.4 \ + --hash=sha256:0fbeb6180d383a9186d0d6ed954e0042ad9f18e0e8de088b2b419d526927d196 \ + --hash=sha256:c34f04500f2cbbea882b1acb02002ad6fe6b7ffa64a6164577995657f50aed22 +click==7.1.2 \ + --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \ + --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc +Werkzeug==1.0.1 \ + --hash=sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43 \ + --hash=sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c +itsdangerous==1.1.0 \ + --hash=sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19 \ + --hash=sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749 +Jinja2==2.11.3 \ + --hash=sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419 \ + --hash=sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6 +MarkupSafe==1.1.1 \ + --hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \ + --hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \ + --hash=sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235 \ + --hash=sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5 \ + --hash=sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff \ + --hash=sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b \ + --hash=sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1 \ + --hash=sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e \ + --hash=sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183 \ + --hash=sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66 \ + --hash=sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1 \ + --hash=sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1 \ + --hash=sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e \ + --hash=sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b \ + --hash=sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905 \ + --hash=sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735 \ + --hash=sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d \ + --hash=sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e \ + --hash=sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d \ + --hash=sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c \ + --hash=sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21 \ + --hash=sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2 \ + --hash=sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5 \ + --hash=sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b \ + --hash=sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6 \ + --hash=sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f \ + --hash=sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f \ + --hash=sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7 diff --git a/tools/tryselect/selectors/chooser/static/filter.js b/tools/tryselect/selectors/chooser/static/filter.js new file mode 100644 index 0000000000..2d8731e61f --- /dev/null +++ b/tools/tryselect/selectors/chooser/static/filter.js @@ -0,0 +1,116 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const selection = $("#selection")[0]; +const count = $("#selection-count")[0]; +const pluralize = (count, noun, suffix = "s") => + `${count} ${noun}${count !== 1 ? suffix : ""}`; + +var selected = []; + +var updateLabels = () => { + $(".tab-pane.active > .filter-label").each(function (index) { + let box = $("#" + this.htmlFor)[0]; + let method = box.checked ? "add" : "remove"; + $(this)[method + "Class"]("is-checked"); + }); +}; + +var apply = () => { + let filters = {}; + let kinds = []; + + $(".filter:checked").each(function (index) { + for (let kind of this.name.split(",")) { + if (!kinds.includes(kind)) { + kinds.push(kind); + } + } + + // Checkbox element values are generated by Section.get_context() in app.py + let attrs = JSON.parse(this.value); + for (let attr in attrs) { + if (!(attr in filters)) { + filters[attr] = []; + } + + let values = attrs[attr]; + filters[attr] = filters[attr].concat(values); + } + }); + updateLabels(); + + if ( + !Object.keys(filters).length || + (Object.keys(filters).length == 1 && "build_type" in filters) + ) { + selection.value = ""; + count.innerHTML = "0 tasks selected"; + return; + } + + var taskMatches = label => { + let task = tasks[label]; + + // If no box for the given kind has been checked, this task is + // automatically not selected. + if (!kinds.includes(task.kind)) { + return false; + } + + for (let attr in filters) { + let values = filters[attr]; + if (!(attr in task) || values.includes(task[attr])) { + continue; + } + return false; + } + return true; + }; + + selected = Object.keys(tasks).filter(taskMatches); + applyChunks(); +}; + +var applyChunks = () => { + // For tasks that have a chunk filter applied, we handle that here. + let filters = {}; + $(".filter:text").each(function (index) { + let value = $(this).val(); + if (value === "") { + return; + } + + let attrs = JSON.parse(this.name); + let key = `${attrs.unittest_suite}-${attrs.unittest_flavor}`; + if (!(key in filters)) { + filters[key] = []; + } + + // Parse the chunk strings. These are formatted like printer page setups, e.g: "1,4-6,9" + for (let item of value.split(",")) { + if (!item.includes("-")) { + filters[key].push(parseInt(item)); + continue; + } + + let [start, end] = item.split("-"); + for (let i = parseInt(start); i <= parseInt(end); ++i) { + filters[key].push(i); + } + } + }); + + let chunked = selected.filter(function (label) { + let task = tasks[label]; + let key = task.unittest_suite + "-" + task.unittest_flavor; + if (key in filters && !filters[key].includes(parseInt(task.test_chunk))) { + return false; + } + return true; + }); + + selection.value = chunked.join("\n"); + count.innerText = pluralize(chunked.length, "task") + " selected"; +}; diff --git a/tools/tryselect/selectors/chooser/static/select.js b/tools/tryselect/selectors/chooser/static/select.js new file mode 100644 index 0000000000..8a315c0a52 --- /dev/null +++ b/tools/tryselect/selectors/chooser/static/select.js @@ -0,0 +1,46 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const labels = $("label.multiselect"); +const boxes = $("label.multiselect input:checkbox"); +var lastChecked = {}; + +// implements shift+click +labels.click(function (e) { + if (e.target.tagName === "INPUT") { + return; + } + + let box = $("#" + this.htmlFor)[0]; + let activeSection = $("div.tab-pane.active")[0].id; + + if (activeSection in lastChecked) { + // Bug 559506 - In Firefox shift/ctrl/alt+clicking a label doesn't check the box. + let isFirefox = navigator.userAgent.toLowerCase().indexOf("firefox") > -1; + + if (e.shiftKey) { + if (isFirefox) { + box.checked = !box.checked; + } + + let start = boxes.index(box); + let end = boxes.index(lastChecked[activeSection]); + + boxes + .slice(Math.min(start, end), Math.max(start, end) + 1) + .prop("checked", box.checked); + apply(); + } + } + + lastChecked[activeSection] = box; +}); + +function selectAll(btn) { + let checked = !!btn.value; + $("div.active label.filter-label").each(function (index) { + $(this).find("input:checkbox")[0].checked = checked; + }); + apply(); +} diff --git a/tools/tryselect/selectors/chooser/static/style.css b/tools/tryselect/selectors/chooser/static/style.css new file mode 100644 index 0000000000..6b2f96935b --- /dev/null +++ b/tools/tryselect/selectors/chooser/static/style.css @@ -0,0 +1,107 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +body { + padding-top: 70px; +} + +/* Tabs */ + +#tabbar .nav-link { + color: #009570; + font-size: 18px; + padding-bottom: 15px; + padding-top: 15px; +} + +#tabbar .nav-link.active { + color: #212529; +} + +#tabbar .nav-link:hover { + color: #0f5a3a; +} + +/* Sections */ + +.tab-content button { + font-size: 14px; + margin-bottom: 5px; + margin-top: 10px; +} + +.filter-label { + display: block; + font-size: 16px; + position: relative; + padding-left: 15px; + padding-right: 15px; + padding-top: 10px; + padding-bottom: 10px; + margin-bottom: 0; + user-select: none; + vertical-align: middle; +} + +.filter-label span { + display: flex; + min-height: 34px; + align-items: center; + justify-content: space-between; +} + +.filter-label input[type="checkbox"] { + position: absolute; + opacity: 0; + height: 0; + width: 0; +} + +.filter-label input[type="text"] { + width: 50px; +} + +.filter-label:hover { + background-color: #91a0b0; +} + +.filter-label.is-checked:hover { + background-color: #91a0b0; +} + +.filter-label.is-checked { + background-color: #404c59; + color: white; +} + +/* Preview pane */ + +#preview { + position: fixed; + height: 100vh; + margin-left: 66%; + width: 100%; +} + +#submit-tasks { + display: flex; + flex-direction: column; + height: 80%; +} + +#buttons { + display: flex; + justify-content: space-between; +} + +#push { + background-color: #00e9b7; + margin-left: 5px; + width: 100%; +} + +#selection { + height: 100%; + width: 100%; +} diff --git a/tools/tryselect/selectors/chooser/templates/chooser.html b/tools/tryselect/selectors/chooser/templates/chooser.html new file mode 100644 index 0000000000..d89870ac77 --- /dev/null +++ b/tools/tryselect/selectors/chooser/templates/chooser.html @@ -0,0 +1,78 @@ + + +{% extends 'layout.html' %} +{% block content %} +
+
+
+
+ Build Type +
+ + +
+ {% for type in ["opt", "debug"] %} +
+ + +
+ {% endfor %} +
+ +
+ + + {% for section in sections %} + {% if loop.first %} +
+ {% else %} +
+ {% endif %} + {% for label, meta in section.labels|dictsort %} + + {% endfor %} +
+ {% endfor %} +
+
+
+
+ + 0 tasks selected
+ + + + +
+
+
+
+{% endblock %} + +{% block scripts %} + + + +{% endblock %} diff --git a/tools/tryselect/selectors/chooser/templates/close.html b/tools/tryselect/selectors/chooser/templates/close.html new file mode 100644 index 0000000000..9dc0a161f3 --- /dev/null +++ b/tools/tryselect/selectors/chooser/templates/close.html @@ -0,0 +1,11 @@ + + +{% extends 'layout.html' %} {% block content %} +
+ +
+{% endblock %} diff --git a/tools/tryselect/selectors/chooser/templates/layout.html b/tools/tryselect/selectors/chooser/templates/layout.html new file mode 100644 index 0000000000..8553ae94df --- /dev/null +++ b/tools/tryselect/selectors/chooser/templates/layout.html @@ -0,0 +1,71 @@ + + + + + + Try Chooser Enhanced + + + + + + {% block content %}{% endblock %} + + + + {% block scripts %}{% endblock %} + + diff --git a/tools/tryselect/selectors/compare.py b/tools/tryselect/selectors/compare.py new file mode 100644 index 0000000000..ac468e0974 --- /dev/null +++ b/tools/tryselect/selectors/compare.py @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from mozbuild.base import MozbuildObject +from mozversioncontrol import get_repository_object + +from tryselect.cli import BaseTryParser + +from .again import run as again_run +from .fuzzy import run as fuzzy_run + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + + +class CompareParser(BaseTryParser): + name = "compare" + arguments = [ + [ + ["-cc", "--compare-commit"], + { + "default": None, + "help": "The commit that you want to compare your current revision with", + }, + ], + ] + common_groups = ["task"] + task_configs = [ + "rebuild", + ] + + def get_revisions_to_run(vcs, compare_commit): + if compare_commit is None: + compare_commit = vcs.base_ref + if vcs.branch: + current_revision_ref = vcs.branch + else: + current_revision_ref = vcs.head_ref + + return compare_commit, current_revision_ref + + +def run(compare_commit=None, **kwargs): + vcs = get_repository_object(build.topsrcdir) + compare_commit, current_revision_ref = CompareParser.get_revisions_to_run( + vcs, compare_commit + ) + print("********************************************") + print("* 2 commits are created with this command *") + print("********************************************") + + try: + fuzzy_run(**kwargs) + print("********************************************") + print("* The base commit can be found above *") + print("********************************************") + vcs.update(compare_commit) + again_run() + print("*****************************************") + print("* The compare commit can be found above *") + print("*****************************************") + finally: + vcs.update(current_revision_ref) diff --git a/tools/tryselect/selectors/coverage.py b/tools/tryselect/selectors/coverage.py new file mode 100644 index 0000000000..a871f35a11 --- /dev/null +++ b/tools/tryselect/selectors/coverage.py @@ -0,0 +1,447 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import collections +import datetime +import hashlib +import json +import os +import shutil +import sqlite3 +import subprocess + +import requests +import six +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozpack.files import FileFinder +from moztest.resolve import TestResolver +from mozversioncontrol import get_repository_object + +from ..cli import BaseTryParser +from ..push import generate_try_task_config, push_to_try +from ..tasks import filter_tasks_by_paths, generate_tasks, resolve_tests_by_suite + +here = os.path.abspath(os.path.dirname(__file__)) +build = None +vcs = None +CHUNK_MAPPING_FILE = None +CHUNK_MAPPING_TAG_FILE = None + + +def setup_globals(): + # Avoid incurring expensive computation on import. + global build, vcs, CHUNK_MAPPING_TAG_FILE, CHUNK_MAPPING_FILE + build = MozbuildObject.from_environment(cwd=here) + vcs = get_repository_object(build.topsrcdir) + + root_hash = hashlib.sha256( + six.ensure_binary(os.path.abspath(build.topsrcdir)) + ).hexdigest() + cache_dir = os.path.join(get_state_dir(), "cache", root_hash, "chunk_mapping") + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + CHUNK_MAPPING_FILE = os.path.join(cache_dir, "chunk_mapping.sqlite") + CHUNK_MAPPING_TAG_FILE = os.path.join(cache_dir, "chunk_mapping_tag.json") + + +# Maps from platform names in the chunk_mapping sqlite database to respective +# substrings in task names. +PLATFORM_MAP = { + "linux": "test-linux64/opt", + "windows": "test-windows10-64/opt", +} + +# List of platform/build type combinations that are included in pushes by |mach try coverage|. +OPT_TASK_PATTERNS = [ + "macosx64/opt", + "windows10-64/opt", + "windows7-32/opt", + "linux64/opt", +] + + +class CoverageParser(BaseTryParser): + name = "coverage" + arguments = [] + common_groups = ["push", "task"] + task_configs = [ + "artifact", + "env", + "rebuild", + "chemspill-prio", + "disable-pgo", + "worker-overrides", + ] + + +def read_test_manifests(): + """Uses TestResolver to read all test manifests in the tree. + + Returns a (tests, support_files_map) tuple that describes the tests in the tree: + tests - a set of test file paths + support_files_map - a dict that maps from each support file to a list with + test files that require them it + """ + test_resolver = TestResolver.from_environment(cwd=here) + file_finder = FileFinder(build.topsrcdir) + support_files_map = collections.defaultdict(list) + tests = set() + + for test in test_resolver.resolve_tests(build.topsrcdir): + tests.add(test["srcdir_relpath"]) + if "support-files" not in test: + continue + + for support_file_pattern in test["support-files"].split(): + # Get the pattern relative to topsrcdir. + if support_file_pattern.startswith("!/"): + support_file_pattern = support_file_pattern[2:] + elif support_file_pattern.startswith("/"): + support_file_pattern = support_file_pattern[1:] + else: + support_file_pattern = os.path.normpath( + os.path.join(test["dir_relpath"], support_file_pattern) + ) + + # If it doesn't have a glob, then it's a single file. + if "*" not in support_file_pattern: + # Simple case: single support file, just add it here. + support_files_map[support_file_pattern].append(test["srcdir_relpath"]) + continue + + for support_file, _ in file_finder.find(support_file_pattern): + support_files_map[support_file].append(test["srcdir_relpath"]) + + return tests, support_files_map + + +# TODO cache the output of this function +all_tests, all_support_files = read_test_manifests() + + +def download_coverage_mapping(base_revision): + try: + with open(CHUNK_MAPPING_TAG_FILE) as f: + tags = json.load(f) + if tags["target_revision"] == base_revision: + return + else: + print("Base revision changed.") + except (OSError, ValueError): + print("Chunk mapping file not found.") + + CHUNK_MAPPING_URL_TEMPLATE = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.relman.code-coverage.production.cron.{}/artifacts/public/chunk_mapping.tar.xz" # noqa + JSON_PUSHES_URL_TEMPLATE = "https://hg.mozilla.org/mozilla-central/json-pushes?version=2&tipsonly=1&startdate={}" # noqa + + # Get pushes from at most one month ago. + PUSH_HISTORY_DAYS = 30 + delta = datetime.timedelta(days=PUSH_HISTORY_DAYS) + start_time = (datetime.datetime.now() - delta).strftime("%Y-%m-%d") + pushes_url = JSON_PUSHES_URL_TEMPLATE.format(start_time) + pushes_data = requests.get(pushes_url + "&tochange={}".format(base_revision)).json() + if "error" in pushes_data: + if "unknown revision" in pushes_data["error"]: + print( + "unknown revision {}, trying with latest mozilla-central".format( + base_revision + ) + ) + pushes_data = requests.get(pushes_url).json() + + if "error" in pushes_data: + raise Exception(pushes_data["error"]) + + pushes = pushes_data["pushes"] + + print("Looking for coverage data. This might take a minute or two.") + print("Base revision:", base_revision) + for push_id in sorted(pushes.keys())[::-1]: + rev = pushes[push_id]["changesets"][0] + url = CHUNK_MAPPING_URL_TEMPLATE.format(rev) + print("push id: {},\trevision: {}".format(push_id, rev)) + + r = requests.head(url) + if not r.ok: + continue + + print("Chunk mapping found, downloading...") + r = requests.get(url, stream=True) + + CHUNK_MAPPING_ARCHIVE = os.path.join(build.topsrcdir, "chunk_mapping.tar.xz") + with open(CHUNK_MAPPING_ARCHIVE, "wb") as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + + subprocess.check_call( + [ + "tar", + "-xJf", + CHUNK_MAPPING_ARCHIVE, + "-C", + os.path.dirname(CHUNK_MAPPING_FILE), + ] + ) + os.remove(CHUNK_MAPPING_ARCHIVE) + assert os.path.isfile(CHUNK_MAPPING_FILE) + with open(CHUNK_MAPPING_TAG_FILE, "w") as f: + json.dump( + { + "target_revision": base_revision, + "chunk_mapping_revision": rev, + "download_date": start_time, + }, + f, + ) + return + raise Exception("Could not find suitable coverage data.") + + +def is_a_test(cursor, path): + """Checks the all_tests global and the chunk mapping database to see if a + given file is a test file. + """ + if path in all_tests: + return True + + cursor.execute("SELECT COUNT(*) from chunk_to_test WHERE path=?", (path,)) + if cursor.fetchone()[0]: + return True + + cursor.execute("SELECT COUNT(*) from file_to_test WHERE test=?", (path,)) + if cursor.fetchone()[0]: + return True + + return False + + +def tests_covering_file(cursor, path): + """Returns a set of tests that cover a given source file.""" + cursor.execute("SELECT test FROM file_to_test WHERE source=?", (path,)) + return {e[0] for e in cursor.fetchall()} + + +def tests_in_chunk(cursor, platform, chunk): + """Returns a set of tests that are contained in a given chunk.""" + cursor.execute( + "SELECT path FROM chunk_to_test WHERE platform=? AND chunk=?", (platform, chunk) + ) + # Because of bug 1480103, some entries in this table contain both a file name and a test name, + # separated by a space. With the split, only the file name is kept. + return {e[0].split(" ")[0] for e in cursor.fetchall()} + + +def chunks_covering_file(cursor, path): + """Returns a set of (platform, chunk) tuples with the chunks that cover a given source file.""" + cursor.execute("SELECT platform, chunk FROM file_to_chunk WHERE path=?", (path,)) + return set(cursor.fetchall()) + + +def tests_supported_by_file(path): + """Returns a set of tests that are using the given file as a support-file.""" + return set(all_support_files[path]) + + +def find_tests(changed_files): + """Finds both individual tests and test chunks that should be run to test code changes. + Argument: a list of file paths relative to the source checkout. + + Returns: a (test_files, test_chunks) tuple with two sets. + test_files - contains tests that should be run to verify changes to changed_files. + test_chunks - contains (platform, chunk) tuples with chunks that should be + run. These chunnks do not support running a subset of the tests (like + cppunit or gtest), so the whole chunk must be run. + """ + test_files = set() + test_chunks = set() + files_no_coverage = set() + + with sqlite3.connect(CHUNK_MAPPING_FILE) as conn: + c = conn.cursor() + for path in changed_files: + # If path is a test, add it to the list and continue. + if is_a_test(c, path): + test_files.add(path) + continue + + # Look at the chunk mapping and add all tests that cover this file. + tests = tests_covering_file(c, path) + chunks = chunks_covering_file(c, path) + # If we found tests covering this, then it's not a support-file, so + # save these and continue. + if tests or chunks: + test_files |= tests + test_chunks |= chunks + continue + + # Check if the path is a support-file for any test, by querying test manifests. + tests = tests_supported_by_file(path) + if tests: + test_files |= tests + continue + + # There is no coverage information for this file. + files_no_coverage.add(path) + + files_covered = set(changed_files) - files_no_coverage + test_files = {s.replace("\\", "/") for s in test_files} + + _print_found_tests(files_covered, files_no_coverage, test_files, test_chunks) + + remaining_test_chunks = set() + # For all test_chunks, try to find the tests contained by them in the + # chunk_to_test mapping. + for platform, chunk in test_chunks: + tests = tests_in_chunk(c, platform, chunk) + if tests: + for test in tests: + test_files.add(test.replace("\\", "/")) + else: + remaining_test_chunks.add((platform, chunk)) + + return test_files, remaining_test_chunks + + +def _print_found_tests(files_covered, files_no_coverage, test_files, test_chunks): + """Print a summary of what will be run to the user's terminal.""" + files_covered = sorted(files_covered) + files_no_coverage = sorted(files_no_coverage) + test_files = sorted(test_files) + test_chunks = sorted(test_chunks) + + if files_covered: + print( + "Found {} modified source files with test coverage:".format( + len(files_covered) + ) + ) + for covered in files_covered: + print("\t", covered) + + if files_no_coverage: + print( + "Found {} modified source files with no coverage:".format( + len(files_no_coverage) + ) + ) + for f in files_no_coverage: + print("\t", f) + + if not files_covered: + print("No modified source files are covered by tests.") + elif not files_no_coverage: + print("All modified source files are covered by tests.") + + if test_files: + print("Running {} individual test files.".format(len(test_files))) + else: + print("Could not find any individual tests to run.") + + if test_chunks: + print("Running {} test chunks.".format(len(test_chunks))) + for platform, chunk in test_chunks: + print("\t", platform, chunk) + else: + print("Could not find any test chunks to run.") + + +def filter_tasks_by_chunks(tasks, chunks): + """Find all tasks that will run the given chunks.""" + selected_tasks = set() + for platform, chunk in chunks: + platform = PLATFORM_MAP[platform] + + selected_task = None + for task in tasks: + if not task.startswith(platform): + continue + + if not any( + task[len(platform) + 1 :].endswith(c) for c in [chunk, chunk + "-e10s"] + ): + continue + + assert ( + selected_task is None + ), "Only one task should be selected for a given platform-chunk couple ({} - {}), {} and {} were selected".format( # noqa + platform, chunk, selected_task, task + ) + selected_task = task + + if selected_task is None: + print("Warning: no task found for chunk", platform, chunk) + else: + selected_tasks.add(selected_task) + + return list(selected_tasks) + + +def is_opt_task(task): + """True if the task runs on a supported platform and build type combination. + This is used to remove -ccov/asan/pgo tasks, along with all /debug tasks. + """ + return any(platform in task for platform in OPT_TASK_PATTERNS) + + +def run( + try_config={}, + full=False, + parameters=None, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, +): + setup_globals() + download_coverage_mapping(vcs.base_ref) + + changed_sources = vcs.get_outgoing_files() + test_files, test_chunks = find_tests(changed_sources) + if not test_files and not test_chunks: + print("ERROR Could not find any tests or chunks to run.") + return 1 + + tg = generate_tasks(parameters, full) + all_tasks = tg.tasks.keys() + + tasks_by_chunks = filter_tasks_by_chunks(all_tasks, test_chunks) + tasks_by_path = filter_tasks_by_paths(all_tasks, test_files) + tasks = filter(is_opt_task, set(tasks_by_path) | set(tasks_by_chunks)) + tasks = list(tasks) + + if not tasks: + print("ERROR Did not find any matching tasks after filtering.") + return 1 + test_count_message = ( + "{test_count} test file{test_plural} that " + + "cover{test_singular} these changes " + + "({task_count} task{task_plural} to be scheduled)" + ).format( + test_count=len(test_files), + test_plural="" if len(test_files) == 1 else "s", + test_singular="s" if len(test_files) == 1 else "", + task_count=len(tasks), + task_plural="" if len(tasks) == 1 else "s", + ) + print("Found " + test_count_message) + + # Set the test paths to be run by setting MOZHARNESS_TEST_PATHS. + path_env = { + "MOZHARNESS_TEST_PATHS": six.ensure_text( + json.dumps(resolve_tests_by_suite(test_files)) + ) + } + try_config.setdefault("env", {}).update(path_env) + + # Build commit message. + msg = "try coverage - " + test_count_message + return push_to_try( + "coverage", + message.format(msg=msg), + try_task_config=generate_try_task_config("coverage", tasks, try_config), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/empty.py b/tools/tryselect/selectors/empty.py new file mode 100644 index 0000000000..2465999c90 --- /dev/null +++ b/tools/tryselect/selectors/empty.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from ..cli import BaseTryParser +from ..push import generate_try_task_config, push_to_try + + +class EmptyParser(BaseTryParser): + name = "empty" + common_groups = ["push"] + + +def run(message="{msg}", stage_changes=False, dry_run=False, closed_tree=False): + msg = 'No try selector specified, use "Add New Jobs" to select tasks.' + return push_to_try( + "empty", + message.format(msg=msg), + try_task_config=generate_try_task_config("empty", []), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/fuzzy.py b/tools/tryselect/selectors/fuzzy.py new file mode 100644 index 0000000000..03fc0531d4 --- /dev/null +++ b/tools/tryselect/selectors/fuzzy.py @@ -0,0 +1,255 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import sys +from pathlib import PurePath + +from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks +from mach.util import get_state_dir + +from ..cli import BaseTryParser +from ..push import check_working_directory, generate_try_task_config, push_to_try +from ..tasks import filter_tasks_by_paths, generate_tasks +from ..util.fzf import ( + FZF_NOT_FOUND, + PREVIEW_SCRIPT, + format_header, + fzf_bootstrap, + fzf_shortcuts, + run_fzf, +) +from ..util.manage_estimates import ( + download_task_history_data, + make_trimmed_taskgraph_cache, +) + + +class FuzzyParser(BaseTryParser): + name = "fuzzy" + arguments = [ + [ + ["-q", "--query"], + { + "metavar": "STR", + "action": "append", + "default": [], + "help": "Use the given query instead of entering the selection " + "interface. Equivalent to typing " + "from the interface. Specifying multiple times schedules " + "the union of computed tasks.", + }, + ], + [ + ["-i", "--interactive"], + { + "action": "store_true", + "default": False, + "help": "Force running fzf interactively even when using presets or " + "queries with -q/--query.", + }, + ], + [ + ["-x", "--and"], + { + "dest": "intersection", + "action": "store_true", + "default": False, + "help": "When specifying queries on the command line with -q/--query, " + "use the intersection of tasks rather than the union. This is " + "especially useful for post filtering presets.", + }, + ], + [ + ["-e", "--exact"], + { + "action": "store_true", + "default": False, + "help": "Enable exact match mode. Terms will use an exact match " + "by default, and terms prefixed with ' will become fuzzy.", + }, + ], + [ + ["-u", "--update"], + { + "action": "store_true", + "default": False, + "help": "Update fzf before running.", + }, + ], + [ + ["-s", "--show-estimates"], + { + "action": "store_true", + "default": False, + "help": "Show task duration estimates.", + }, + ], + [ + ["--disable-target-task-filter"], + { + "action": "store_true", + "default": False, + "help": "Some tasks run on mozilla-central but are filtered out " + "of the default list due to resource constraints. This flag " + "disables this filtering.", + }, + ], + ] + common_groups = ["push", "task", "preset"] + task_configs = [ + "artifact", + "browsertime", + "chemspill-prio", + "disable-pgo", + "env", + "gecko-profile", + "path", + "pernosco", + "rebuild", + "routes", + "worker-overrides", + ] + + +def run( + update=False, + query=None, + intersect_query=None, + try_config=None, + full=False, + parameters=None, + save_query=False, + stage_changes=False, + dry_run=False, + message="{msg}", + test_paths=None, + exact=False, + closed_tree=False, + show_estimates=False, + disable_target_task_filter=False, +): + fzf = fzf_bootstrap(update) + + if not fzf: + print(FZF_NOT_FOUND) + return 1 + + push = not stage_changes and not dry_run + check_working_directory(push) + tg = generate_tasks( + parameters, full=full, disable_target_task_filter=disable_target_task_filter + ) + all_tasks = sorted(tg.tasks.keys()) + + # graph_Cache created by generate_tasks, recreate the path to that file. + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + if full: + graph_cache = os.path.join(cache_dir, "full_task_graph") + dep_cache = os.path.join(cache_dir, "full_task_dependencies") + target_set = os.path.join(cache_dir, "full_task_set") + else: + graph_cache = os.path.join(cache_dir, "target_task_graph") + dep_cache = os.path.join(cache_dir, "target_task_dependencies") + target_set = os.path.join(cache_dir, "target_task_set") + + if show_estimates: + download_task_history_data(cache_dir=cache_dir) + make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set) + + if not full and not disable_target_task_filter: + # Put all_tasks into a list because it's used multiple times, and "filter()" + # returns a consumable iterator. + all_tasks = list(filter(filter_by_uncommon_try_tasks, all_tasks)) + + if test_paths: + all_tasks = filter_tasks_by_paths(all_tasks, test_paths) + if not all_tasks: + return 1 + + key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()] + base_cmd = [ + fzf, + "-m", + "--bind", + ",".join(key_shortcuts), + "--header", + format_header(), + "--preview-window=right:30%", + "--print-query", + ] + + if show_estimates: + base_cmd.extend( + [ + "--preview", + '{} {} -g {} -s -c {} -t "{{+f}}"'.format( + str(PurePath(sys.executable)), PREVIEW_SCRIPT, dep_cache, cache_dir + ), + ] + ) + else: + base_cmd.extend( + [ + "--preview", + '{} {} -t "{{+f}}"'.format( + str(PurePath(sys.executable)), PREVIEW_SCRIPT + ), + ] + ) + + if exact: + base_cmd.append("--exact") + + selected = set() + queries = [] + + def get_tasks(query_arg=None, candidate_tasks=all_tasks): + cmd = base_cmd[:] + if query_arg and query_arg != "INTERACTIVE": + cmd.extend(["-f", query_arg]) + + query_str, tasks = run_fzf(cmd, sorted(candidate_tasks)) + queries.append(query_str) + return set(tasks) + + for q in query or []: + selected |= get_tasks(q) + + for q in intersect_query or []: + if not selected: + tasks = get_tasks(q) + selected |= tasks + else: + tasks = get_tasks(q, selected) + selected &= tasks + + if not queries: + selected = get_tasks() + + if not selected: + print("no tasks selected") + return + + if save_query: + return queries + + # build commit message + msg = "Fuzzy" + args = ["query={}".format(q) for q in queries] + if test_paths: + args.append("paths={}".format(":".join(test_paths))) + if args: + msg = "{} {}".format(msg, "&".join(args)) + return push_to_try( + "fuzzy", + message.format(msg=msg), + try_task_config=generate_try_task_config("fuzzy", selected, try_config), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + ) diff --git a/tools/tryselect/selectors/perf.py b/tools/tryselect/selectors/perf.py new file mode 100644 index 0000000000..b570c8d63b --- /dev/null +++ b/tools/tryselect/selectors/perf.py @@ -0,0 +1,1335 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import copy +import itertools +import json +import os +import pathlib +import shutil +import subprocess +from contextlib import redirect_stdout +from datetime import datetime, timedelta + +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozversioncontrol import get_repository_object + +from ..push import generate_try_task_config, push_to_try +from ..util.fzf import ( + FZF_NOT_FOUND, + build_base_cmd, + fzf_bootstrap, + run_fzf, + setup_tasks_for_fzf, +) +from .compare import CompareParser +from .perfselector.classification import ( + Apps, + ClassificationProvider, + Platforms, + Suites, + Variants, +) +from .perfselector.perfcomparators import get_comparator +from .perfselector.utils import LogProcessor + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) +cache_file = pathlib.Path(get_state_dir(), "try_perf_revision_cache.json") + +PERFHERDER_BASE_URL = ( + "https://treeherder.mozilla.org/perfherder/" + "compare?originalProject=try&originalRevision=%s&newProject=try&newRevision=%s" +) +TREEHERDER_TRY_BASE_URL = "https://treeherder.mozilla.org/jobs?repo=try&revision=%s" + +# Prevent users from running more than 300 tests at once. It's possible, but +# it's more likely that a query is broken and is selecting far too much. +MAX_PERF_TASKS = 300 + +# Name of the base category with no variants applied to it +BASE_CATEGORY_NAME = "base" + +# Add environment variable for firefox-android integration. +# This will let us find the APK to upload automatically. However, +# the following option will need to be supplied: +# --browsertime-upload-apk firefox-android +# OR --mozperftest-upload-apk firefox-android +MOZ_FIREFOX_ANDROID_APK_OUTPUT = os.getenv("MOZ_FIREFOX_ANDROID_APK_OUTPUT", None) + + +class InvalidCategoryException(Exception): + """Thrown when a category is found to be invalid. + + See the `PerfParser.run_category_checks()` method for more info. + """ + + pass + + +class APKNotFound(Exception): + """Raised when a user-supplied path to an APK is invalid.""" + + pass + + +class InvalidRegressionDetectorQuery(Exception): + """Thrown when the detector query produces anything other than 1 task.""" + + pass + + +class PerfParser(CompareParser): + name = "perf" + common_groups = ["push", "task"] + task_configs = [ + "artifact", + "browsertime", + "disable-pgo", + "env", + "gecko-profile", + "path", + "rebuild", + ] + + provider = ClassificationProvider() + platforms = provider.platforms + apps = provider.apps + variants = provider.variants + suites = provider.suites + categories = provider.categories + + arguments = [ + [ + ["--show-all"], + { + "action": "store_true", + "default": False, + "help": "Show all available tasks.", + }, + ], + [ + ["--android"], + { + "action": "store_true", + "default": False, + "help": "Show android test categories (disabled by default).", + }, + ], + [ + ["--chrome"], + { + "action": "store_true", + "default": False, + "help": "Show tests available for Chrome-based browsers " + "(disabled by default).", + }, + ], + [ + ["--custom-car"], + { + "action": "store_true", + "default": False, + "help": "Show tests available for Custom Chromium-as-Release (disabled by default).", + }, + ], + [ + ["--safari"], + { + "action": "store_true", + "default": False, + "help": "Show tests available for Safari (disabled by default).", + }, + ], + [ + ["--live-sites"], + { + "action": "store_true", + "default": False, + "help": "Run tasks with live sites (if possible). " + "You can also use the `live-sites` variant.", + }, + ], + [ + ["--profile"], + { + "action": "store_true", + "default": False, + "help": "Run tasks with profiling (if possible). " + "You can also use the `profiling` variant.", + }, + ], + [ + ["--single-run"], + { + "action": "store_true", + "default": False, + "help": "Run tasks without a comparison", + }, + ], + [ + ["-q", "--query"], + { + "type": str, + "default": None, + "help": "Query to run in either the perf-category selector, " + "or the fuzzy selector if --show-all is provided.", + }, + ], + [ + ["--browsertime-upload-apk"], + { + "type": str, + "default": None, + "help": "Path to an APK to upload. Note that this " + "will replace the APK installed in all Android Performance " + "tests. If the Activity, Binary Path, or Intents required " + "change at all relative to the existing GeckoView, and Fenix " + "tasks, then you will need to make fixes in the associated " + "taskcluster files (e.g. taskcluster/ci/test/browsertime-mobile.yml). " + "Alternatively, set MOZ_FIREFOX_ANDROID_APK_OUTPUT to a path to " + "an APK, and then run the command with --browsertime-upload-apk " + "firefox-android. This option will only copy the APK for browsertime, see " + "--mozperftest-upload-apk to upload APKs for startup tests.", + }, + ], + [ + ["--mozperftest-upload-apk"], + { + "type": str, + "default": None, + "help": "See --browsertime-upload-apk. This option does the same " + "thing except it's for mozperftest tests such as the startup ones. " + "Note that those tests only exist through --show-all, as they " + "aren't contained in any existing categories.", + }, + ], + [ + ["--detect-changes"], + { + "action": "store_true", + "default": False, + "help": "Adds a task that detects performance changes using MWU.", + }, + ], + [ + ["--comparator"], + { + "type": str, + "default": "BasePerfComparator", + "help": "Either a path to a file to setup a custom comparison, " + "or a builtin name. See the Firefox source docs for mach try perf for " + "examples of how to build your own, along with the interface.", + }, + ], + [ + ["--comparator-args"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "comparator_args", + "help": "Arguments provided to the base, and new revision setup stages " + "of the comparator.", + "metavar": "ARG=VALUE", + }, + ], + [ + ["--variants"], + { + "nargs": "*", + "type": str, + "default": [BASE_CATEGORY_NAME], + "dest": "requested_variants", + "choices": list(variants.keys()), + "help": "Select variants to display in the selector from: " + + ", ".join(list(variants.keys())), + "metavar": "", + }, + ], + [ + ["--platforms"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "requested_platforms", + "choices": list(platforms.keys()), + "help": "Select specific platforms to target. Android only " + "available with --android. Available platforms: " + + ", ".join(list(platforms.keys())), + "metavar": "", + }, + ], + [ + ["--apps"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "requested_apps", + "choices": list(apps.keys()), + "help": "Select specific applications to target from: " + + ", ".join(list(apps.keys())), + "metavar": "", + }, + ], + [ + ["--extra-args"], + { + "nargs": "*", + "type": str, + "default": [], + "dest": "extra_args", + "help": "Set the extra args " + "(e.x, --extra-args verbose post-startup-delay=1)", + "metavar": "", + }, + ], + ] + + def get_tasks(base_cmd, queries, query_arg=None, candidate_tasks=None): + cmd = base_cmd[:] + if query_arg: + cmd.extend(["-f", query_arg]) + + query_str, tasks = run_fzf(cmd, sorted(candidate_tasks)) + queries.append(query_str) + return set(tasks) + + def get_perf_tasks(base_cmd, all_tg_tasks, perf_categories, query=None): + # Convert the categories to tasks + selected_tasks = set() + queries = [] + + selected_categories = PerfParser.get_tasks( + base_cmd, queries, query, perf_categories + ) + + for category, category_info in perf_categories.items(): + if category not in selected_categories: + continue + print("Gathering tasks for %s category" % category) + + category_tasks = set() + for suite in PerfParser.suites: + # Either perform a query to get the tasks (recommended), or + # use a hardcoded task list + suite_queries = category_info["queries"].get(suite) + + category_suite_tasks = set() + if suite_queries: + print( + "Executing %s queries: %s" % (suite, ", ".join(suite_queries)) + ) + + for perf_query in suite_queries: + if not category_suite_tasks: + # Get all tasks selected with the first query + category_suite_tasks |= PerfParser.get_tasks( + base_cmd, queries, perf_query, all_tg_tasks + ) + else: + # Keep only those tasks that matched in all previous queries + category_suite_tasks &= PerfParser.get_tasks( + base_cmd, queries, perf_query, category_suite_tasks + ) + + if len(category_suite_tasks) == 0: + print("Failed to find any tasks for query: %s" % perf_query) + break + + if category_suite_tasks: + category_tasks |= category_suite_tasks + + if category_info["tasks"]: + category_tasks = set(category_info["tasks"]) & all_tg_tasks + if category_tasks != set(category_info["tasks"]): + print( + "Some expected tasks could not be found: %s" + % ", ".join(category_info["tasks"] - category_tasks) + ) + + if not category_tasks: + print("Could not find any tasks for category %s" % category) + else: + # Add the new tasks to the currently selected ones + selected_tasks |= category_tasks + + return selected_tasks, selected_categories, queries + + def _check_app(app, target): + """Checks if the app exists in the target.""" + if app.value in target: + return True + return False + + def _check_platform(platform, target): + """Checks if the platform, or it's type exists in the target.""" + if ( + platform.value in target + or PerfParser.platforms[platform.value]["platform"] in target + ): + return True + return False + + def _build_initial_decision_matrix(): + # Build first stage of matrix APPS X PLATFORMS + initial_decision_matrix = [] + for platform in Platforms: + platform_row = [] + for app in Apps: + if PerfParser._check_platform( + platform, PerfParser.apps[app.value]["platforms"] + ): + # This app can run on this platform + platform_row.append(True) + else: + platform_row.append(False) + initial_decision_matrix.append(platform_row) + return initial_decision_matrix + + def _build_intermediate_decision_matrix(): + # Second stage of matrix building applies the 2D matrix found above + # to each suite + initial_decision_matrix = PerfParser._build_initial_decision_matrix() + + intermediate_decision_matrix = [] + for suite in Suites: + suite_matrix = copy.deepcopy(initial_decision_matrix) + suite_info = PerfParser.suites[suite.value] + + # Restric the platforms for this suite now + for platform in Platforms: + for app in Apps: + runnable = False + if PerfParser._check_app( + app, suite_info["apps"] + ) and PerfParser._check_platform(platform, suite_info["platforms"]): + runnable = True + suite_matrix[platform][app] = ( + runnable and suite_matrix[platform][app] + ) + + intermediate_decision_matrix.append(suite_matrix) + return intermediate_decision_matrix + + def _build_variants_matrix(): + # Third stage is expanding the intermediate matrix + # across all the variants (non-expanded). Start with the + # intermediate matrix in the list since it provides our + # base case with no variants + intermediate_decision_matrix = PerfParser._build_intermediate_decision_matrix() + + variants_matrix = [] + for variant in Variants: + variant_matrix = copy.deepcopy(intermediate_decision_matrix) + + for suite in Suites: + if variant.value in PerfParser.suites[suite.value]["variants"]: + # Allow the variant through and set it's platforms and apps + # based on how it sets it -> only restrict, don't make allowances + # here + for platform in Platforms: + for app in Apps: + if not ( + PerfParser._check_platform( + platform, + PerfParser.variants[variant.value]["platforms"], + ) + and PerfParser._check_app( + app, PerfParser.variants[variant.value]["apps"] + ) + ): + variant_matrix[suite][platform][app] = False + else: + # This variant matrix needs to be completely False + variant_matrix[suite] = [ + [False] * len(platform_row) + for platform_row in variant_matrix[suite] + ] + + variants_matrix.append(variant_matrix) + + return variants_matrix, intermediate_decision_matrix + + def _build_decision_matrix(): + """Build the decision matrix. + + This method builds the decision matrix that is used + to determine what categories will be shown to the user. + This matrix has the following form (as lists): + - Variants + - Suites + - Platforms + - Apps + + Each element in the 4D Matrix is either True or False and tells us + whether the particular combination is "runnable" according to + the given specifications. This does not mean that the combination + exists, just that it's fully configured in this selector. + + The ("base",) variant combination found in the matrix has + no variants applied to it. At this stage, it's a catch-all for those + categories. The query it uses is reduced further in later stages. + """ + # Get the variants matrix (see methods above) and the intermediate decision + # matrix to act as the base category + ( + variants_matrix, + intermediate_decision_matrix, + ) = PerfParser._build_variants_matrix() + + # Get all possible combinations of the variants + expanded_variants = [ + variant_combination + for set_size in range(len(Variants) + 1) + for variant_combination in itertools.combinations(list(Variants), set_size) + ] + + # Final stage combines the intermediate matrix with the + # expanded variants and leaves a "base" category which + # doesn't have any variant specifications (it catches them all) + decision_matrix = {(BASE_CATEGORY_NAME,): intermediate_decision_matrix} + for variant_combination in expanded_variants: + expanded_variant_matrix = [] + + # Perform an AND operation on the combination of variants + # to determine where this particular combination can run + for suite in Suites: + suite_matrix = [] + suite_variants = PerfParser.suites[suite.value]["variants"] + + # Disable the variant combination if none of them + # are found in the suite + disable_variant = not any( + [variant.value in suite_variants for variant in variant_combination] + ) + + for platform in Platforms: + if disable_variant: + platform_row = [False for _ in Apps] + else: + platform_row = [ + all( + variants_matrix[variant][suite][platform][app] + for variant in variant_combination + if variant.value in suite_variants + ) + for app in Apps + ] + suite_matrix.append(platform_row) + + expanded_variant_matrix.append(suite_matrix) + decision_matrix[variant_combination] = expanded_variant_matrix + + return decision_matrix + + def _skip_with_restrictions(value, restrictions, requested=[]): + """Determines if we should skip an app, platform, or variant. + + We add base here since it's the base category variant that + would always be displayed and it won't affect the app, or + platform selections. + """ + if restrictions is not None and value not in restrictions + [ + BASE_CATEGORY_NAME + ]: + return True + if requested and value not in requested + [BASE_CATEGORY_NAME]: + return True + return False + + def build_category_matrix( + requested_variants=[BASE_CATEGORY_NAME], + requested_platforms=[], + requested_apps=[], + **kwargs, + ): + """Build a decision matrix for all the categories. + + It will have the form: + - Category + - Variants + - ... + """ + # Build the base decision matrix + decision_matrix = PerfParser._build_decision_matrix() + + # Here, the variants are further restricted by the category settings + # using the `_skip_with_restrictions` method. This part also handles + # explicitly requested platforms, apps, and variants. + category_decision_matrix = {} + for category, category_info in PerfParser.categories.items(): + category_matrix = copy.deepcopy(decision_matrix) + + for variant_combination, variant_matrix in decision_matrix.items(): + variant_runnable = True + if BASE_CATEGORY_NAME not in variant_combination: + # Make sure that all portions of the variant combination + # target at least one of the suites in the category + tmp_variant_combination = set( + [v.value for v in variant_combination] + ) + for suite in Suites: + if suite.value not in category_info["suites"]: + continue + tmp_variant_combination = tmp_variant_combination - set( + [ + variant.value + for variant in variant_combination + if variant.value + in PerfParser.suites[suite.value]["variants"] + ] + ) + if tmp_variant_combination: + # If it's not empty, then some variants + # are non-existent + variant_runnable = False + + for suite, platform, app in itertools.product(Suites, Platforms, Apps): + runnable = variant_runnable + + # Disable this combination if there are any variant + # restrictions for this suite, or if the user didn't request it + # (and did request some variants). The same is done below with + # the apps, and platforms. + if any( + PerfParser._skip_with_restrictions( + variant.value if not isinstance(variant, str) else variant, + category_info.get("variant-restrictions", {}).get( + suite.value, None + ), + requested_variants, + ) + for variant in variant_combination + ): + runnable = False + + if PerfParser._skip_with_restrictions( + platform.value, + category_info.get("platform-restrictions", None), + requested_platforms, + ): + runnable = False + + # If the platform is restricted, check if the appropriate + # flags were provided (or appropriate conditions hit). We do + # the same thing for apps below. + if ( + PerfParser.platforms[platform.value].get("restriction", None) + is not None + ): + runnable = runnable and PerfParser.platforms[platform.value][ + "restriction" + ](**kwargs) + + if PerfParser._skip_with_restrictions( + app.value, + category_info.get("app-restrictions", {}).get( + suite.value, None + ), + requested_apps, + ): + runnable = False + if PerfParser.apps[app.value].get("restriction", None) is not None: + runnable = runnable and PerfParser.apps[app.value][ + "restriction" + ](**kwargs) + + category_matrix[variant_combination][suite][platform][app] = ( + runnable and variant_matrix[suite][platform][app] + ) + + category_decision_matrix[category] = category_matrix + + return category_decision_matrix + + def _enable_restriction(restriction, **kwargs): + """Used to simplify checking a restriction.""" + return restriction is not None and restriction(**kwargs) + + def _category_suites(category_info): + """Returns all the suite enum entries in this category.""" + return [suite for suite in Suites if suite.value in category_info["suites"]] + + def _add_variant_queries( + category_info, variant_matrix, variant_combination, platform, queries, app=None + ): + """Used to add the variant queries to various categories.""" + for variant in variant_combination: + for suite in PerfParser._category_suites(category_info): + if (app is not None and variant_matrix[suite][platform][app]) or ( + app is None and any(variant_matrix[suite][platform]) + ): + queries[suite.value].append( + PerfParser.variants[variant.value]["query"] + ) + + def _build_categories(category, category_info, category_matrix): + """Builds the categories to display.""" + categories = {} + + for variant_combination, variant_matrix in category_matrix.items(): + base_category = BASE_CATEGORY_NAME in variant_combination + + for platform in Platforms: + if not any( + any(variant_matrix[suite][platform]) + for suite in PerfParser._category_suites(category_info) + ): + # There are no apps available on this platform in either + # of the requested suites + continue + + # This code has the effect of restricting all suites to + # a platform. This means categories with mixed suites will + # be available even if some suites will no longer run + # given this platform constraint. The reasoning for this is that + # it's unexpected to receive desktop tests when you explcitly + # request android. + platform_queries = { + suite: ( + category_info["query"][suite] + + [PerfParser.platforms[platform.value]["query"]] + ) + for suite in category_info["suites"] + } + + platform_category_name = f"{category} {platform.value}" + platform_category_info = { + "queries": platform_queries, + "tasks": category_info["tasks"], + "platform": platform, + "app": None, + "suites": category_info["suites"], + "base-category": base_category, + "base-category-name": category, + } + for app in Apps: + if not any( + variant_matrix[suite][platform][app] + for suite in PerfParser._category_suites(category_info) + ): + # This app is not available on the given platform + # for any of the suites + continue + + # Add the queries for the app for any suites that need it and + # the variant queries if needed + app_queries = copy.deepcopy(platform_queries) + for suite in Suites: + if suite.value not in app_queries: + continue + app_queries[suite.value].append( + PerfParser.apps[app.value]["query"] + ) + if not base_category: + PerfParser._add_variant_queries( + category_info, + variant_matrix, + variant_combination, + platform, + app_queries, + app=app, + ) + + app_category_name = f"{platform_category_name} {app.value}" + if not base_category: + app_category_name = ( + f"{app_category_name} " + f"{'+'.join([v.value for v in variant_combination])}" + ) + categories[app_category_name] = { + "queries": app_queries, + "tasks": category_info["tasks"], + "platform": platform, + "app": app, + "suites": category_info["suites"], + "base-category": base_category, + } + + if not base_category: + platform_category_name = ( + f"{platform_category_name} " + f"{'+'.join([v.value for v in variant_combination])}" + ) + PerfParser._add_variant_queries( + category_info, + variant_matrix, + variant_combination, + platform, + platform_queries, + ) + categories[platform_category_name] = platform_category_info + + return categories + + def _handle_variant_negations(category, category_info, **kwargs): + """Handle variant negations. + + The reason why we're negating variants here instead of where we add + them to the queries is because we need to iterate over all of the variants + but when we add them, we only look at the variants in the combination. It's + possible to combine these, but that increases the complexity of the code + by quite a bit so it's best to do it separately. + """ + for variant in Variants: + if category_info["base-category"] and variant.value in kwargs.get( + "requested_variants", [BASE_CATEGORY_NAME] + ): + # When some particular variant(s) are requested, and we are at a + # base category, don't negate it. Otherwise, if the variant + # wasn't requested negate it + continue + if variant.value in category: + # If this variant is in the category name, skip negations + continue + if not PerfParser._check_platform( + category_info["platform"], + PerfParser.variants[variant.value]["platforms"], + ): + # Make sure the variant applies to the platform + continue + + for suite in category_info["suites"]: + if variant.value not in PerfParser.suites[suite]["variants"]: + continue + category_info["queries"][suite].append( + PerfParser.variants[variant.value]["negation"] + ) + + def _handle_app_negations(category, category_info, **kwargs): + """Handle app negations. + + This is where the global chrome/safari negations get added. We use kwargs + along with the app restriction method to make this decision. + """ + for app in Apps: + if PerfParser.apps[app.value].get("negation", None) is None: + continue + elif any( + PerfParser.apps[app.value]["negation"] + in category_info["queries"][suite] + for suite in category_info["suites"] + ): + # Already added the negations + continue + if category_info.get("app", None) is not None: + # We only need to handle this for categories that + # don't specify an app + continue + if PerfParser._enable_restriction( + PerfParser.apps[app.value].get("restriction", None), **kwargs + ): + continue + + for suite in category_info["suites"]: + if app.value not in PerfParser.suites[suite]["apps"]: + continue + category_info["queries"][suite].append( + PerfParser.apps[app.value]["negation"] + ) + + def _handle_negations(category, category_info, **kwargs): + """This method handles negations. + + This method should only include things that should be globally applied + to all the queries. The apps are included as chrome is negated if + --chrome isn't provided, and the variants are negated here too. + """ + PerfParser._handle_variant_negations(category, category_info, **kwargs) + PerfParser._handle_app_negations(category, category_info, **kwargs) + + def get_categories(**kwargs): + """Get the categories to be displayed. + + The categories are built using the decision matrices from `build_category_matrix`. + The methods above provide more detail on how this is done. Here, we use + this matrix to determine if we should show a category to a user. + + We also apply the negations for restricted apps/platforms and variants + at the end before displaying the categories. + """ + categories = {} + + # Setup the restrictions, and ease-of-use variants requested (if any) + for variant in Variants: + if PerfParser._enable_restriction( + PerfParser.variants[variant.value].get("restriction", None), **kwargs + ): + kwargs.setdefault("requested_variants", []).append(variant.value) + + category_decision_matrix = PerfParser.build_category_matrix(**kwargs) + + # Now produce the categories by finding all the entries that are True + for category, category_matrix in category_decision_matrix.items(): + categories.update( + PerfParser._build_categories( + category, PerfParser.categories[category], category_matrix + ) + ) + + # Handle the restricted app queries, and variant negations + for category, category_info in categories.items(): + PerfParser._handle_negations(category, category_info, **kwargs) + + return categories + + def inject_change_detector(base_cmd, all_tasks, selected_tasks): + query = "'perftest 'mwu 'detect" + mwu_task = PerfParser.get_tasks(base_cmd, [], query, all_tasks) + + if len(mwu_task) > 1 or len(mwu_task) == 0: + raise InvalidRegressionDetectorQuery( + f"Expected 1 task from change detector " + f"query, but found {len(mwu_task)}" + ) + + selected_tasks |= set(mwu_task) + + def check_cached_revision(selected_tasks, base_commit=None): + """ + If the base_commit parameter does not exist, remove expired cache data. + Cache data format: + { + base_commit[str]: [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-12", + "tasks": ["a-task"], + }, + { + "base_revision_treeherder": "999998888", + "date": "2023-03-12", + "tasks": ["b-task"], + }, + ] + } + + The list represents different pushes with different task selections. + + TODO: See if we can request additional tests on a given base revision. + + :param selected_tasks list: The list of tasks selected by the user + :param base_commit str: The base commit to search + :return: The base_revision_treeherder if found, else None + """ + today = datetime.now() + expired_date = (today - timedelta(weeks=2)).strftime("%Y-%m-%d") + today = today.strftime("%Y-%m-%d") + + if not cache_file.is_file(): + return + + with cache_file.open("r") as f: + cache_data = json.load(f) + + # Remove expired cache data + if base_commit is None: + for cached_base_commit in list(cache_data): + if not isinstance(cache_data[cached_base_commit], list): + # TODO: Remove in the future, this is for backwards-compatibility + # with the previous cache structure + cache_data.pop(cached_base_commit) + else: + # Go through the pushes, and expire any that are too old + new_pushes = [] + for push in cache_data[cached_base_commit]: + if push["date"] > expired_date: + new_pushes.append(push) + # If no pushes are left after expiration, expire the base commit + if new_pushes: + cache_data[cached_base_commit] = new_pushes + else: + cache_data.pop(cached_base_commit) + with cache_file.open("w") as f: + json.dump(cache_data, f, indent=4) + + cached_base_commit = cache_data.get(base_commit, None) + if cached_base_commit: + for push in cached_base_commit: + if set(selected_tasks) <= set(push["tasks"]): + return push["base_revision_treeherder"] + + def save_revision_treeherder(selected_tasks, base_commit, base_revision_treeherder): + """ + Save the base revision of treeherder to the cache. + See "check_cached_revision" for more information about the data structure. + + :param selected_tasks list: The list of tasks selected by the user + :param base_commit str: The base commit to save + :param base_revision_treeherder str: The base revision of treeherder to save + :return: None + """ + today = datetime.now().strftime("%Y-%m-%d") + new_revision = { + "base_revision_treeherder": base_revision_treeherder, + "date": today, + "tasks": list(selected_tasks), + } + cache_data = {} + + if cache_file.is_file(): + with cache_file.open("r") as f: + cache_data = json.load(f) + cache_data.setdefault(base_commit, []).append(new_revision) + else: + cache_data[base_commit] = [new_revision] + + with cache_file.open(mode="w") as f: + json.dump(cache_data, f, indent=4) + + def setup_try_config(try_config, extra_args, base_revision_treeherder=None): + if try_config is None: + try_config = {} + if extra_args: + args = " ".join(extra_args) + try_config.setdefault("env", {})["PERF_FLAGS"] = args + if base_revision_treeherder: + # Reset updated since we no longer need to worry + # about failing while we're on a base commit + try_config.setdefault("env", {})[ + "PERF_BASE_REVISION" + ] = base_revision_treeherder + + def perf_push_to_try( + selected_tasks, + selected_categories, + queries, + try_config, + dry_run, + single_run, + extra_args, + comparator, + comparator_args, + ): + """Perf-specific push to try method. + + This makes use of logic from the CompareParser to do something + very similar except with log redirection. We get the comparison + revisions, then use the repository object to update between revisions + and the LogProcessor for parsing out the revisions that are used + to build the Perfherder links. + """ + vcs = get_repository_object(build.topsrcdir) + compare_commit, current_revision_ref = PerfParser.get_revisions_to_run( + vcs, None + ) + + # Build commit message + msg = "Perf selections={} (queries={})".format( + ",".join(selected_categories), + "&".join([q for q in queries if q is not None and len(q) > 0]), + ) + + # Get the comparator to run + comparator_klass = get_comparator(comparator) + comparator_obj = comparator_klass( + vcs, compare_commit, current_revision_ref, comparator_args + ) + base_comparator = True + if comparator_klass.__name__ != "BasePerfComparator": + base_comparator = False + + new_revision_treeherder = "" + base_revision_treeherder = "" + try: + # redirect_stdout allows us to feed each line into + # a processor that we can use to catch the revision + # while providing real-time output + log_processor = LogProcessor() + + # Push the base revision first. This lets the new revision appear + # first in the Treeherder view, and it also lets us enhance the new + # revision with information about the base run. + base_revision_treeherder = None + if base_comparator: + # Don't cache the base revision when a custom comparison is being performed + # since the base revision is now unique and not general to all pushes + base_revision_treeherder = PerfParser.check_cached_revision( + selected_tasks, compare_commit + ) + + if not (dry_run or single_run or base_revision_treeherder): + # Setup the base revision, and try config. This lets us change the options + # we run the tests with through the PERF_FLAGS environment variable. + base_extra_args = list(extra_args) + base_try_config = copy.deepcopy(try_config) + comparator_obj.setup_base_revision(base_extra_args) + PerfParser.setup_try_config(base_try_config, base_extra_args) + + with redirect_stdout(log_processor): + # XXX Figure out if we can use the `again` selector in some way + # Right now we would need to modify it to be able to do this. + # XXX Fix up the again selector for the perf selector (if it makes sense to) + push_to_try( + "perf-again", + "{msg}".format(msg=msg), + try_task_config=generate_try_task_config( + "fuzzy", selected_tasks, base_try_config + ), + stage_changes=False, + dry_run=dry_run, + closed_tree=False, + allow_log_capture=True, + ) + + base_revision_treeherder = log_processor.revision + if base_comparator: + PerfParser.save_revision_treeherder( + selected_tasks, compare_commit, base_revision_treeherder + ) + + comparator_obj.teardown_base_revision() + + new_extra_args = list(extra_args) + comparator_obj.setup_new_revision(new_extra_args) + PerfParser.setup_try_config( + try_config, + new_extra_args, + base_revision_treeherder=base_revision_treeherder, + ) + + with redirect_stdout(log_processor): + push_to_try( + "perf", + "{msg}".format(msg=msg), + # XXX Figure out if changing `fuzzy` to `perf` will break something + try_task_config=generate_try_task_config( + "fuzzy", selected_tasks, try_config + ), + stage_changes=False, + dry_run=dry_run, + closed_tree=False, + allow_log_capture=True, + ) + + new_revision_treeherder = log_processor.revision + comparator_obj.teardown_new_revision() + + finally: + comparator_obj.teardown() + + return base_revision_treeherder, new_revision_treeherder + + def run( + update=False, + show_all=False, + parameters=None, + try_config=None, + dry_run=False, + single_run=False, + query=None, + detect_changes=False, + rebuild=1, + **kwargs, + ): + # Setup fzf + fzf = fzf_bootstrap(update) + + if not fzf: + print(FZF_NOT_FOUND) + return 1 + + all_tasks, dep_cache, cache_dir = setup_tasks_for_fzf( + not dry_run, + parameters, + full=True, + disable_target_task_filter=False, + ) + base_cmd = build_base_cmd(fzf, dep_cache, cache_dir, show_estimates=False) + + # Perform the selection, then push to try and return the revisions + queries = [] + selected_categories = [] + if not show_all: + # Expand the categories first + categories = PerfParser.get_categories(**kwargs) + selected_tasks, selected_categories, queries = PerfParser.get_perf_tasks( + base_cmd, all_tasks, categories, query=query + ) + else: + selected_tasks = PerfParser.get_tasks(base_cmd, queries, query, all_tasks) + + if len(selected_tasks) == 0: + print("No tasks selected") + return None + + if (len(selected_tasks) * rebuild) > MAX_PERF_TASKS: + print( + "That's a lot of tests selected (%s)!\n" + "These tests won't be triggered. If this was unexpected, " + "please file a bug in Testing :: Performance." % MAX_PERF_TASKS + ) + return None + + if detect_changes: + PerfParser.inject_change_detector(base_cmd, all_tasks, selected_tasks) + + return PerfParser.perf_push_to_try( + selected_tasks, + selected_categories, + queries, + try_config, + dry_run, + single_run, + kwargs.get("extra_args", []), + kwargs.get("comparator", "BasePerfComparator"), + kwargs.get("comparator_args", []), + ) + + def run_category_checks(): + # XXX: Add a jsonschema check for the category definition + # Make sure the queries don't specify variants in them + variant_queries = { + suite: [ + PerfParser.variants[variant]["query"] + for variant in suite_info.get( + "variants", list(PerfParser.variants.keys()) + ) + ] + + [ + PerfParser.variants[variant]["negation"] + for variant in suite_info.get( + "variants", list(PerfParser.variants.keys()) + ) + ] + for suite, suite_info in PerfParser.suites.items() + } + + for category, category_info in PerfParser.categories.items(): + for suite, query in category_info["query"].items(): + if len(variant_queries[suite]) == 0: + # This suite has no variants + continue + if any(any(v in q for q in query) for v in variant_queries[suite]): + raise InvalidCategoryException( + f"The '{category}' category suite query for '{suite}' " + f"uses a variant in it's query '{query}'." + "If you don't want a particular variant use the " + "`variant-restrictions` field in the category." + ) + + return True + + def setup_apk_upload(framework, apk_upload_path): + """Setup the APK for uploading to test on try. + + There are two ways of performing the upload: + (1) Passing a path to an APK with: + --browsertime-upload-apk + --mozperftest-upload-apk + (2) Setting MOZ_FIREFOX_ANDROID_APK_OUTPUT to a path that will + always point to an APK () that we can upload. + + The file is always copied to testing/raptor/raptor/user_upload.apk to + integrate with minimal changes for simpler cases when using raptor-browsertime. + + For mozperftest, the APK is always uploaded here for the same reasons: + python/mozperftest/mozperftest/user_upload.apk + """ + frameworks_to_locations = { + "browsertime": pathlib.Path( + build.topsrcdir, "testing", "raptor", "raptor", "user_upload.apk" + ), + "mozperftest": pathlib.Path( + build.topsrcdir, + "python", + "mozperftest", + "mozperftest", + "user_upload.apk", + ), + } + + print("Setting up custom APK upload") + if apk_upload_path in ("firefox-android"): + apk_upload_path = MOZ_FIREFOX_ANDROID_APK_OUTPUT + if apk_upload_path is None: + raise APKNotFound( + "MOZ_FIREFOX_ANDROID_APK_OUTPUT is not defined. It should " + "point to an APK to upload." + ) + apk_upload_path = pathlib.Path(apk_upload_path) + if not apk_upload_path.exists() or apk_upload_path.is_dir(): + raise APKNotFound( + "MOZ_FIREFOX_ANDROID_APK_OUTPUT needs to point to an APK." + ) + else: + apk_upload_path = pathlib.Path(apk_upload_path) + if not apk_upload_path.exists(): + raise APKNotFound(f"Path does not exist: {str(apk_upload_path)}") + + print("\nCopying file in-tree for upload...") + shutil.copyfile( + str(apk_upload_path), + frameworks_to_locations[framework], + ) + + hg_cmd = ["hg", "add", str(frameworks_to_locations[framework])] + print( + f"\nRunning the following hg command (RAM warnings are expected):\n" + f" {hg_cmd}" + ) + subprocess.check_output(hg_cmd) + print( + "\nAPK is setup for uploading. Please commit the changes, " + "and re-run this command. \nEnsure you supply the --android, " + "and select the correct tasks (fenix, geckoview) or use " + "--show-all for mozperftest task selection.\n" + ) + + +def run(**kwargs): + if ( + kwargs.get("browsertime_upload_apk") is not None + or kwargs.get("mozperftest_upload_apk") is not None + ): + framework = "browsertime" + upload_apk = kwargs.get("browsertime_upload_apk") + if upload_apk is None: + framework = "mozperftest" + upload_apk = kwargs.get("mozperftest_upload_apk") + + PerfParser.setup_apk_upload(framework, upload_apk) + return + + # Make sure the categories are following + # the rules we've setup + PerfParser.run_category_checks() + PerfParser.check_cached_revision([]) + + revisions = PerfParser.run( + profile=kwargs.get("try_config", {}).get("gecko-profile", False), + rebuild=kwargs.get("try_config", {}).get("rebuild", 1), + **kwargs, + ) + + if revisions is None: + return + + # Provide link to perfherder for comparisons now + if not kwargs.get("single_run", False): + perfcompare_url = PERFHERDER_BASE_URL % revisions + original_try_url = TREEHERDER_TRY_BASE_URL % revisions[0] + local_change_try_url = TREEHERDER_TRY_BASE_URL % revisions[1] + print( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison here " + "once the tests are complete (ensure you select the right " + "framework): %s\n" % perfcompare_url + ) + print("\n*******************************************************") + print("* 2 commits/try-runs are created... *") + print("*******************************************************") + print(f"Base revision's try run: {original_try_url}") + print(f"Local revision's try run: {local_change_try_url}\n") + print( + "If you need any help, you can find us in the #perf-help Matrix channel:\n" + "https://matrix.to/#/#perf-help:mozilla.org\n" + ) + print( + "For more information on the performance tests, see our PerfDocs here:\n" + "https://firefox-source-docs.mozilla.org/testing/perfdocs/" + ) diff --git a/tools/tryselect/selectors/perfselector/__init__.py b/tools/tryselect/selectors/perfselector/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/selectors/perfselector/classification.py b/tools/tryselect/selectors/perfselector/classification.py new file mode 100644 index 0000000000..4ed18ce3e5 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/classification.py @@ -0,0 +1,374 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import enum + + +class ClassificationEnum(enum.Enum): + """This class provides the ability to use Enums as array indices.""" + + @property + def value(self): + return self._value_["value"] + + def __index__(self): + return self._value_["index"] + + def __int__(self): + return self._value_["index"] + + +class Platforms(ClassificationEnum): + ANDROID_A51 = {"value": "android-a51", "index": 0} + ANDROID = {"value": "android", "index": 1} + WINDOWS = {"value": "windows", "index": 2} + LINUX = {"value": "linux", "index": 3} + MACOSX = {"value": "macosx", "index": 4} + DESKTOP = {"value": "desktop", "index": 5} + + +class Apps(ClassificationEnum): + FIREFOX = {"value": "firefox", "index": 0} + CHROME = {"value": "chrome", "index": 1} + CHROMIUM = {"value": "chromium", "index": 2} + GECKOVIEW = {"value": "geckoview", "index": 3} + FENIX = {"value": "fenix", "index": 4} + CHROME_M = {"value": "chrome-m", "index": 5} + SAFARI = {"value": "safari", "index": 6} + CHROMIUM_RELEASE = {"value": "custom-car", "index": 7} + + +class Suites(ClassificationEnum): + RAPTOR = {"value": "raptor", "index": 0} + TALOS = {"value": "talos", "index": 1} + AWSY = {"value": "awsy", "index": 2} + + +class Variants(ClassificationEnum): + NO_FISSION = {"value": "no-fission", "index": 0} + BYTECODE_CACHED = {"value": "bytecode-cached", "index": 1} + LIVE_SITES = {"value": "live-sites", "index": 2} + PROFILING = {"value": "profiling", "index": 3} + SWR = {"value": "swr", "index": 4} + + +""" +The following methods and constants are used for restricting +certain platforms and applications such as chrome, safari, and +android tests. These all require a flag such as --android to +enable (see build_category_matrix for more info). +""" + + +def check_for_android(android=False, **kwargs): + return android + + +def check_for_chrome(chrome=False, **kwargs): + return chrome + + +def check_for_custom_car(custom_car=False, **kwargs): + return custom_car + + +def check_for_safari(safari=False, **kwargs): + return safari + + +def check_for_live_sites(live_sites=False, **kwargs): + return live_sites + + +def check_for_profile(profile=False, **kwargs): + return profile + + +class ClassificationProvider: + @property + def platforms(self): + return { + Platforms.ANDROID_A51.value: { + "query": "'android 'a51 'shippable 'aarch64", + "restriction": check_for_android, + "platform": Platforms.ANDROID.value, + }, + Platforms.ANDROID.value: { + # The android, and android-a51 queries are expected to be the same, + # we don't want to run the tests on other mobile platforms. + "query": "'android 'a51 'shippable 'aarch64", + "restriction": check_for_android, + "platform": Platforms.ANDROID.value, + }, + Platforms.WINDOWS.value: { + "query": "!-32 'windows 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.LINUX.value: { + "query": "!clang 'linux 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.MACOSX.value: { + "query": "'osx 'shippable", + "platform": Platforms.DESKTOP.value, + }, + Platforms.DESKTOP.value: { + "query": "!android 'shippable !-32 !clang", + "platform": Platforms.DESKTOP.value, + }, + } + + @property + def apps(self): + return { + Apps.FIREFOX.value: { + "query": "!chrom !geckoview !fenix !safari !custom-car", + "platforms": [Platforms.DESKTOP.value], + }, + Apps.CHROME.value: { + "query": "'chrome", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.DESKTOP.value], + }, + Apps.CHROMIUM.value: { + "query": "'chromium", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.DESKTOP.value], + }, + Apps.GECKOVIEW.value: { + "query": "'geckoview", + "platforms": [Platforms.ANDROID.value], + }, + Apps.FENIX.value: { + "query": "'fenix", + "platforms": [Platforms.ANDROID.value], + }, + Apps.CHROME_M.value: { + "query": "'chrome-m", + "negation": "!chrom", + "restriction": check_for_chrome, + "platforms": [Platforms.ANDROID.value], + }, + Apps.SAFARI.value: { + "query": "'safari", + "negation": "!safari", + "restriction": check_for_safari, + "platforms": [Platforms.MACOSX.value], + }, + Apps.CHROMIUM_RELEASE.value: { + "query": "'custom-car", + "negation": "!custom-car", + "restriction": check_for_custom_car, + "platforms": [Platforms.LINUX.value, Platforms.WINDOWS.value], + }, + } + + @property + def variants(self): + return { + Variants.NO_FISSION.value: { + "query": "'nofis", + "negation": "!nofis", + "platforms": [Platforms.ANDROID.value], + "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value], + }, + Variants.BYTECODE_CACHED.value: { + "query": "'bytecode", + "negation": "!bytecode", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + Variants.LIVE_SITES.value: { + "query": "'live", + "negation": "!live", + "restriction": check_for_live_sites, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [ # XXX No live CaR tests + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + Apps.SAFARI.value, + ], + }, + Variants.PROFILING.value: { + "query": "'profil", + "negation": "!profil", + "restriction": check_for_profile, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value], + }, + Variants.SWR.value: { + "query": "'swr", + "negation": "!swr", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + } + + @property + def suites(self): + return { + Suites.RAPTOR.value: { + "apps": list(self.apps.keys()), + "platforms": list(self.platforms.keys()), + "variants": [ + Variants.NO_FISSION.value, + Variants.LIVE_SITES.value, + Variants.PROFILING.value, + Variants.BYTECODE_CACHED.value, + ], + }, + Suites.TALOS.value: { + "apps": [Apps.FIREFOX.value], + "platforms": [Platforms.DESKTOP.value], + "variants": [ + Variants.PROFILING.value, + Variants.SWR.value, + ], + }, + Suites.AWSY.value: { + "apps": [Apps.FIREFOX.value], + "platforms": [Platforms.DESKTOP.value], + "variants": [], + }, + } + + """ + Here you can find the base categories that are defined for the perf + selector. The following fields are available: + * query: Set the queries to use for each suite you need. + * suites: The suites that are needed for this category. + * tasks: A hard-coded list of tasks to select. + * platforms: The platforms that it can run on. + * app-restrictions: A list of apps that the category can run. + * variant-restrictions: A list of variants available for each suite. + + Note that setting the App/Variant-Restriction fields should be used to + restrict the available apps and variants, not expand them. + """ + + @property + def categories(self): + return { + "Pageload": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6"], + }, + "suites": [Suites.RAPTOR.value], + "tasks": [], + }, + "Pageload (essential)": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6 'essential"], + }, + "variant-restrictions": { + Suites.RAPTOR.value: [Variants.NO_FISSION.value] + }, + "suites": [Suites.RAPTOR.value], + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + Apps.CHROMIUM_RELEASE.value, + ], + }, + "tasks": [], + }, + "Speedometer 3": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'speedometer3"], + }, + "variant-restrictions": { + Suites.RAPTOR.value: [Variants.NO_FISSION.value] + }, + "suites": [Suites.RAPTOR.value], + "app-restrictions": {}, + "tasks": [], + }, + "Responsiveness": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'responsive"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + ], + }, + "tasks": [], + }, + "Benchmarks": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'benchmark"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + }, + "DAMP (Devtools)": { + "query": { + Suites.TALOS.value: ["'talos 'damp"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + }, + "Talos PerfTests": { + "query": { + Suites.TALOS.value: ["'talos"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + }, + "Resource Usage": { + "query": { + Suites.TALOS.value: ["'talos 'xperf | 'tp5"], + Suites.RAPTOR.value: ["'power 'osx"], + Suites.AWSY.value: ["'awsy"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value], + "platform-restrictions": [Platforms.DESKTOP.value], + "variant-restrictions": { + Suites.RAPTOR.value: [], + Suites.TALOS.value: [], + }, + "app-restrictions": { + Suites.RAPTOR.value: [Apps.FIREFOX.value], + Suites.TALOS.value: [Apps.FIREFOX.value], + }, + "tasks": [], + }, + "Graphics, & Media Playback": { + "query": { + # XXX This might not be an exhaustive list for talos atm + Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"], + Suites.RAPTOR.value: ["'browsertime 'youtube-playback"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value], + "variant-restrictions": { + Suites.RAPTOR.value: [Variants.NO_FISSION.value] + }, + "app-restrictions": { + Suites.RAPTOR.value: [ + Apps.FIREFOX.value, + Apps.CHROME.value, + Apps.CHROMIUM.value, + Apps.FENIX.value, + Apps.GECKOVIEW.value, + ], + }, + "tasks": [], + }, + } diff --git a/tools/tryselect/selectors/perfselector/perfcomparators.py b/tools/tryselect/selectors/perfselector/perfcomparators.py new file mode 100644 index 0000000000..fce35fe562 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/perfcomparators.py @@ -0,0 +1,258 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import importlib +import inspect +import pathlib + +BUILTIN_COMPARATORS = {} + + +class ComparatorNotFound(Exception): + """Raised when we can't find the specified comparator. + + Triggered when either the comparator name is incorrect for a builtin one, + or when a path to a specified comparator cannot be found. + """ + + pass + + +class GithubRequestFailure(Exception): + """Raised when we hit a failure during PR link parsing.""" + + pass + + +class BadComparatorArgs(Exception): + """Raised when the args given to the comparator are incorrect.""" + + pass + + +def comparator(comparator_klass): + BUILTIN_COMPARATORS[comparator_klass.__name__] = comparator_klass + return comparator_klass + + +@comparator +class BasePerfComparator: + def __init__(self, vcs, compare_commit, current_revision_ref, comparator_args): + """Initialize the standard/default settings for Comparators. + + :param vcs object: Used for updating the local repo. + :param compare_commit str: The base revision found for the local repo. + :param current_revision_ref str: The current revision of the local repo. + :param comparator_args list: List of comparator args in the format NAME=VALUE. + """ + self.vcs = vcs + self.compare_commit = compare_commit + self.current_revision_ref = current_revision_ref + self.comparator_args = comparator_args + + # Used to ensure that the local repo gets cleaned up appropriately on failures + self._updated = False + + def setup_base_revision(self, extra_args): + """Setup the base try run/revision. + + In this case, we update to the repo to the base revision and + push that to try. The extra_args can be used to set additional + arguments for Raptor (not available for other harnesses). + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + self.vcs.update(self.compare_commit) + self._updated = True + + def teardown_base_revision(self): + """Teardown the setup for the base revision.""" + if self._updated: + self.vcs.update(self.current_revision_ref) + self._updated = False + + def setup_new_revision(self, extra_args): + """Setup the new try run/revision. + + Note that the extra_args are reset between the base, and new revision runs. + + :param extra_args list: A list of extra arguments to pass to the try tasks. + """ + pass + + def teardown_new_revision(self): + """Teardown the new run/revision setup.""" + pass + + def teardown(self): + """Teardown for failures. + + This method can be used for ensuring that the repo is cleaned up + when a failure is hit at any point in the process of doing the + new/base revision setups, or the pushes to try. + """ + self.teardown_base_revision() + + +def get_github_pull_request_info(link): + """Returns information about a PR link. + + This method accepts a Github link in either of these formats: + https://github.com/mozilla-mobile/firefox-android/pull/1627, + https://github.com/mozilla-mobile/firefox-android/pull/1876/commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365 #noqa + + and returns the Github link, branch, and revision of the commit. + """ + from urllib.parse import urlparse + + import requests + + # Parse the url, and get all the necessary info + parsed_url = urlparse(link) + path_parts = parsed_url.path.strip("/").split("/") + owner, repo = path_parts[0], path_parts[1] + pr_number = path_parts[-1] + + if "/pull/" not in parsed_url.path: + raise GithubRequestFailure( + f"Link for Github PR is invalid (missing /pull/): {link}" + ) + + # Get the commit being targeted in the PR + pr_commit = None + if "/commits/" in parsed_url.path: + pr_commit = path_parts[-1] + pr_number = path_parts[-3] + + # Make the request, and get the PR info, otherwise, + # raise an exception if the response code is not 200 + api_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}" + response = requests.get(api_url) + if response.status_code == 200: + link_info = response.json() + return ( + link_info["head"]["repo"]["html_url"], + pr_commit if pr_commit else link_info["head"]["sha"], + link_info["head"]["ref"], + ) + + raise GithubRequestFailure( + f"The following url returned a non-200 status code: {api_url}" + ) + + +@comparator +class BenchmarkComparator(BasePerfComparator): + def _get_benchmark_info(self, arg_prefix): + # Get the flag from the comparator args + benchmark_info = {"repo": None, "branch": None, "revision": None, "link": None} + for arg in self.comparator_args: + if arg.startswith(arg_prefix): + _, settings = arg.split(arg_prefix) + setting, val = settings.split("=") + if setting not in benchmark_info: + raise BadComparatorArgs( + f"Unknown argument provided `{setting}`. Only the following " + f"are available (prefixed with `{arg_prefix}`): " + f"{list(benchmark_info.keys())}" + ) + benchmark_info[setting] = val + + # Parse the link for any required information + if benchmark_info.get("link", None) is not None: + ( + benchmark_info["repo"], + benchmark_info["revision"], + benchmark_info["branch"], + ) = get_github_pull_request_info(benchmark_info["link"]) + + return benchmark_info + + def _setup_benchmark_args(self, extra_args, benchmark_info): + # Setup the arguments for Raptor + extra_args.append(f"benchmark-repository={benchmark_info['repo']}") + extra_args.append(f"benchmark-revision={benchmark_info['revision']}") + + if benchmark_info.get("branch", None): + extra_args.append(f"benchmark-branch={benchmark_info['branch']}") + + def setup_base_revision(self, extra_args): + """Sets up the options for a base benchmark revision run. + + Checks for a `base-link` in the + command and adds the appropriate commands to the extra_args + which will be added to the PERF_FLAGS environment variable. + + If that isn't provided, then you must provide the repo, branch, + and revision directly through these (branch is optional): + + base-repo=https://github.com/mozilla-mobile/firefox-android + base-branch=main + base-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365 + + Otherwise, we'll use the default mach try perf + base behaviour. + + TODO: Get the information automatically from a commit link. Github + API doesn't provide the branch name from a link like that. + """ + base_info = self._get_benchmark_info("base-") + + # If no options were provided, use the default BasePerfComparator behaviour + if not any(v is not None for v in base_info.values()): + raise BadComparatorArgs( + f"Could not find the correct base-revision arguments in: {self.comparator_args}" + ) + + self._setup_benchmark_args(extra_args, base_info) + + def setup_new_revision(self, extra_args): + """Sets up the options for a new benchmark revision run. + + Same as `setup_base_revision`, except it uses + `new-` as the prefix instead of `base-`. + """ + new_info = self._get_benchmark_info("new-") + + # If no options were provided, use the default BasePerfComparator behaviour + if not any(v is not None for v in new_info.values()): + raise BadComparatorArgs( + f"Could not find the correct new-revision arguments in: {self.comparator_args}" + ) + + self._setup_benchmark_args(extra_args, new_info) + + +def get_comparator(comparator): + if comparator in BUILTIN_COMPARATORS: + return BUILTIN_COMPARATORS[comparator] + + file = pathlib.Path(comparator) + if not file.exists(): + raise ComparatorNotFound( + f"Expected either a path to a file containing a comparator, or a " + f"builtin comparator from this list: {BUILTIN_COMPARATORS.keys()}" + ) + + # Importing a source file directly + spec = importlib.util.spec_from_file_location(name=file.name, location=comparator) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + members = inspect.getmembers( + module, + lambda c: inspect.isclass(c) + and issubclass(c, BasePerfComparator) + and c != BasePerfComparator, + ) + + if not members: + raise ComparatorNotFound( + f"The path {comparator} was found but it was not a valid comparator. " + f"Ensure it is a subclass of BasePerfComparator and optionally contains the " + f"following methods: " + f"{', '.join(inspect.getmembers(BasePerfComparator, predicate=inspect.ismethod))}" + ) + + return members[0][-1] diff --git a/tools/tryselect/selectors/perfselector/utils.py b/tools/tryselect/selectors/perfselector/utils.py new file mode 100644 index 0000000000..105d003091 --- /dev/null +++ b/tools/tryselect/selectors/perfselector/utils.py @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +import sys + +REVISION_MATCHER = re.compile(r"remote:.*/try/rev/([\w]*)[ \t]*$") + + +class LogProcessor: + def __init__(self): + self.buf = "" + self.stdout = sys.__stdout__ + self._revision = None + + @property + def revision(self): + return self._revision + + def write(self, buf): + while buf: + try: + newline_index = buf.index("\n") + except ValueError: + # No newline, wait for next call + self.buf += buf + break + + # Get data up to next newline and combine with previously buffered data + data = self.buf + buf[: newline_index + 1] + buf = buf[newline_index + 1 :] + + # Reset buffer then output line + self.buf = "" + if data.strip() == "": + continue + self.stdout.write(data.strip("\n") + "\n") + + # Check if a temporary commit wa created + match = REVISION_MATCHER.match(data) + if match: + # Last line found is the revision we want + self._revision = match.group(1) diff --git a/tools/tryselect/selectors/preview.py b/tools/tryselect/selectors/preview.py new file mode 100644 index 0000000000..1d232af9e0 --- /dev/null +++ b/tools/tryselect/selectors/preview.py @@ -0,0 +1,102 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""This script is intended to be called through fzf as a preview formatter.""" + + +import argparse +import os +import sys + +here = os.path.abspath(os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(os.path.dirname(here), "util")) +from estimates import duration_summary + + +def process_args(): + """Process preview arguments.""" + argparser = argparse.ArgumentParser() + argparser.add_argument( + "-s", + "--show-estimates", + action="store_true", + help="Show task duration estimates (default: False)", + ) + argparser.add_argument( + "-g", + "--graph-cache", + type=str, + default=None, + help="Filename of task graph dependencies", + ) + argparser.add_argument( + "-c", + "--cache_dir", + type=str, + default=None, + help="Path to cache directory containing task durations", + ) + argparser.add_argument( + "-t", + "--tasklist", + type=str, + default=None, + help="Path to temporary file containing the selected tasks", + ) + return argparser.parse_args() + + +def plain_display(taskfile): + """Original preview window display.""" + with open(taskfile) as f: + tasklist = [line.strip() for line in f] + print("\n".join(sorted(tasklist))) + + +def duration_display(graph_cache_file, taskfile, cache_dir): + """Preview window display with task durations + metadata.""" + with open(taskfile) as f: + tasklist = [line.strip() for line in f] + + durations = duration_summary(graph_cache_file, tasklist, cache_dir) + output = "" + max_columns = int(os.environ["FZF_PREVIEW_COLUMNS"]) + + output += "\nSelected tasks take {}\n".format(durations["selected_duration"]) + output += "+{} dependencies, total {}\n".format( + durations["dependency_count"], + durations["selected_duration"] + durations["dependency_duration"], + ) + + if durations.get("percentile"): + output += "This is in the top {}% of requests\n".format( + 100 - durations["percentile"] + ) + + output += "Estimated finish in {} at {}".format( + durations["wall_duration_seconds"], durations["eta_datetime"].strftime("%H:%M") + ) + + duration_width = 5 # show five numbers at most. + output += "{:>{width}}\n".format("Duration", width=max_columns) + for task in tasklist: + duration = durations["task_durations"].get(task, 0.0) + output += "{:{align}{width}} {:{nalign}{nwidth}}s\n".format( + task, + duration, + align="<", + width=max_columns - (duration_width + 2), # 2: space and 's' + nalign=">", + nwidth=duration_width, + ) + + print(output) + + +if __name__ == "__main__": + args = process_args() + if args.show_estimates and os.path.isdir(args.cache_dir): + duration_display(args.graph_cache, args.tasklist, args.cache_dir) + else: + plain_display(args.tasklist) diff --git a/tools/tryselect/selectors/release.py b/tools/tryselect/selectors/release.py new file mode 100644 index 0000000000..a828b3d038 --- /dev/null +++ b/tools/tryselect/selectors/release.py @@ -0,0 +1,159 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os + +import attr +import yaml +from mozilla_version.gecko import FirefoxVersion + +from ..cli import BaseTryParser +from ..push import push_to_try, vcs + +TARGET_TASKS = { + "staging": "staging_release_builds", + "release-sim": "release_simulation", +} + + +def read_file(path): + with open(path) as fh: + return fh.read() + + +class ReleaseParser(BaseTryParser): + name = "release" + arguments = [ + [ + ["-v", "--version"], + { + "metavar": "STR", + "required": True, + "action": "store", + "type": FirefoxVersion.parse, + "help": "The version number to use for the staging release.", + }, + ], + [ + ["--migration"], + { + "metavar": "STR", + "action": "append", + "dest": "migrations", + "choices": [ + "central-to-beta", + "beta-to-release", + "early-to-late-beta", + "release-to-esr", + ], + "help": "Migration to run for the release (can be specified multiple times).", + }, + ], + [ + ["--no-limit-locales"], + { + "action": "store_false", + "dest": "limit_locales", + "help": "Don't build a limited number of locales in the staging release.", + }, + ], + [ + ["--tasks"], + { + "choices": TARGET_TASKS.keys(), + "default": "staging", + "help": "Which tasks to run on-push.", + }, + ], + ] + common_groups = ["push"] + task_configs = ["disable-pgo", "worker-overrides"] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.set_defaults(migrations=[]) + + +def run( + version, + migrations, + limit_locales, + tasks, + try_config=None, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, +): + app_version = attr.evolve(version, beta_number=None, is_esr=False) + + files_to_change = { + "browser/config/version.txt": "{}\n".format(app_version), + "browser/config/version_display.txt": "{}\n".format(version), + "config/milestone.txt": "{}\n".format(app_version), + } + with open("browser/config/version.txt") as f: + current_version = FirefoxVersion.parse(f.read()) + format_options = { + "current_major_version": current_version.major_number, + "next_major_version": version.major_number, + "current_weave_version": current_version.major_number + 2, + "next_weave_version": version.major_number + 2, + } + + if "beta-to-release" in migrations and "early-to-late-beta" not in migrations: + migrations.append("early-to-late-beta") + + release_type = version.version_type.name.lower() + if release_type not in ("beta", "release", "esr"): + raise Exception( + "Can't do staging release for version: {} type: {}".format( + version, version.version_type + ) + ) + elif release_type == "esr": + release_type += str(version.major_number) + task_config = { + "version": 2, + "parameters": { + "target_tasks_method": TARGET_TASKS[tasks], + "optimize_target_tasks": True, + "release_type": release_type, + }, + } + if try_config: + task_config["parameters"]["try_task_config"] = try_config + + with open(os.path.join(vcs.path, "taskcluster/ci/config.yml")) as f: + migration_configs = yaml.safe_load(f) + for migration in migrations: + migration_config = migration_configs["merge-automation"]["behaviors"][migration] + for (path, from_, to) in migration_config["replacements"]: + if path in files_to_change: + contents = files_to_change[path] + else: + contents = read_file(path) + from_ = from_.format(**format_options) + to = to.format(**format_options) + files_to_change[path] = contents.replace(from_, to) + + if limit_locales: + files_to_change["browser/locales/l10n-changesets.json"] = read_file( + os.path.join(vcs.path, "browser/locales/l10n-onchange-changesets.json") + ) + files_to_change["browser/locales/shipped-locales"] = "en-US\n" + read_file( + os.path.join(vcs.path, "browser/locales/onchange-locales") + ) + + msg = "staging release: {}".format(version) + return push_to_try( + "release", + message.format(msg=msg), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + try_task_config=task_config, + files_to_change=files_to_change, + ) diff --git a/tools/tryselect/selectors/scriptworker.py b/tools/tryselect/selectors/scriptworker.py new file mode 100644 index 0000000000..984ad26a88 --- /dev/null +++ b/tools/tryselect/selectors/scriptworker.py @@ -0,0 +1,177 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import sys + +import requests +from gecko_taskgraph.util.taskgraph import find_existing_tasks +from taskgraph.parameters import Parameters +from taskgraph.util.taskcluster import find_task_id, get_artifact, get_session + +from ..cli import BaseTryParser +from ..push import push_to_try + +TASK_TYPES = { + "linux-signing": [ + "build-signing-linux-shippable/opt", + "build-signing-linux64-shippable/opt", + "build-signing-win64-shippable/opt", + "build-signing-win32-shippable/opt", + "repackage-signing-win64-shippable/opt", + "repackage-signing-win32-shippable/opt", + "repackage-signing-msi-win32-shippable/opt", + "repackage-signing-msi-win64-shippable/opt", + "mar-signing-linux64-shippable/opt", + ], + "linux-signing-partial": ["partials-signing-linux64-shippable/opt"], + "mac-signing": ["build-signing-macosx64-shippable/opt"], + "beetmover-candidates": ["beetmover-repackage-linux64-shippable/opt"], + "bouncer-submit": ["release-bouncer-sub-firefox"], + "balrog-submit": [ + "release-balrog-submit-toplevel-firefox", + "balrog-linux64-shippable/opt", + ], + "tree": ["release-early-tagging-firefox", "release-version-bump-firefox"], +} + +RELEASE_TO_BRANCH = { + "beta": "releases/mozilla-beta", + "release": "releases/mozilla-release", +} + + +class ScriptworkerParser(BaseTryParser): + name = "scriptworker" + arguments = [ + [ + ["task_type"], + { + "choices": ["list"] + list(TASK_TYPES.keys()), + "metavar": "TASK-TYPE", + "help": "Scriptworker task types to run. (Use `list` to show possibilities)", + }, + ], + [ + ["--release-type"], + { + "choices": ["nightly"] + list(RELEASE_TO_BRANCH.keys()), + "default": "beta", + "help": "Release type to run", + }, + ], + ] + + common_groups = ["push"] + task_configs = ["worker-overrides", "routes"] + + +def get_releases(branch): + response = requests.get( + "https://shipitapi-public.services.mozilla.com/releases", + params={"product": "firefox", "branch": branch, "status": "shipped"}, + headers={"Accept": "application/json"}, + ) + response.raise_for_status() + return response.json() + + +def get_release_graph(release): + for phase in release["phases"]: + if phase["name"] in ("ship_firefox",): + return phase["actionTaskId"] + raise Exception("No ship phase.") + + +def get_nightly_graph(): + return find_task_id( + "gecko.v2.mozilla-central.latest.taskgraph.decision-nightly-desktop" + ) + + +def print_available_task_types(): + print("Available task types:") + for task_type, tasks in TASK_TYPES.items(): + print(" " * 4 + "{}:".format(task_type)) + for task in tasks: + print(" " * 8 + "- {}".format(task)) + + +def get_hg_file(parameters, path): + session = get_session() + response = session.get(parameters.file_url(path)) + response.raise_for_status() + return response.content + + +def run( + task_type, + release_type, + try_config=None, + stage_changes=False, + dry_run=False, + message="{msg}", + closed_tree=False, +): + if task_type == "list": + print_available_task_types() + sys.exit(0) + + if release_type == "nightly": + previous_graph = get_nightly_graph() + else: + release = get_releases(RELEASE_TO_BRANCH[release_type])[-1] + previous_graph = get_release_graph(release) + existing_tasks = find_existing_tasks([previous_graph]) + + previous_parameters = Parameters( + strict=False, **get_artifact(previous_graph, "public/parameters.yml") + ) + + # Copy L10n configuration from the commit the release we are using was + # based on. This *should* ensure that the chunking of L10n tasks is the + # same between graphs. + files_to_change = { + path: get_hg_file(previous_parameters, path) + for path in [ + "browser/locales/l10n-changesets.json", + "browser/locales/shipped-locales", + ] + } + + try_config = try_config or {} + task_config = { + "version": 2, + "parameters": { + "existing_tasks": existing_tasks, + "try_task_config": try_config, + "try_mode": "try_task_config", + }, + } + for param in ( + "app_version", + "build_number", + "next_version", + "release_history", + "release_product", + "release_type", + "version", + ): + task_config["parameters"][param] = previous_parameters[param] + + try_config["tasks"] = TASK_TYPES[task_type] + for label in try_config["tasks"]: + if label in existing_tasks: + del existing_tasks[label] + + msg = "scriptworker tests: {}".format(task_type) + return push_to_try( + "scriptworker", + message.format(msg=msg), + stage_changes=stage_changes, + dry_run=dry_run, + closed_tree=closed_tree, + try_task_config=task_config, + files_to_change=files_to_change, + ) diff --git a/tools/tryselect/selectors/syntax.py b/tools/tryselect/selectors/syntax.py new file mode 100644 index 0000000000..ad643f8314 --- /dev/null +++ b/tools/tryselect/selectors/syntax.py @@ -0,0 +1,708 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import re +import sys +from collections import defaultdict + +import mozpack.path as mozpath +from moztest.resolve import TestResolver + +from ..cli import BaseTryParser +from ..push import build, push_to_try + +here = os.path.abspath(os.path.dirname(__file__)) + + +class SyntaxParser(BaseTryParser): + name = "syntax" + arguments = [ + [ + ["paths"], + { + "nargs": "*", + "default": [], + "help": "Paths to search for tests to run on try.", + }, + ], + [ + ["-b", "--build"], + { + "dest": "builds", + "default": "do", + "help": "Build types to run (d for debug, o for optimized).", + }, + ], + [ + ["-p", "--platform"], + { + "dest": "platforms", + "action": "append", + "help": "Platforms to run (required if not found in the environment as " + "AUTOTRY_PLATFORM_HINT).", + }, + ], + [ + ["-u", "--unittests"], + { + "dest": "tests", + "action": "append", + "help": "Test suites to run in their entirety.", + }, + ], + [ + ["-t", "--talos"], + { + "action": "append", + "help": "Talos suites to run.", + }, + ], + [ + ["-j", "--jobs"], + { + "action": "append", + "help": "Job tasks to run.", + }, + ], + [ + ["--tag"], + { + "dest": "tags", + "action": "append", + "help": "Restrict tests to the given tag (may be specified multiple times).", + }, + ], + [ + ["--and"], + { + "action": "store_true", + "dest": "intersection", + "help": "When -u and paths are supplied run only the intersection of the " + "tests specified by the two arguments.", + }, + ], + [ + ["--no-artifact"], + { + "action": "store_true", + "help": "Disable artifact builds even if --enable-artifact-builds is set " + "in the mozconfig.", + }, + ], + [ + ["-v", "--verbose"], + { + "dest": "verbose", + "action": "store_true", + "default": False, + "help": "Print detailed information about the resulting test selection " + "and commands performed.", + }, + ], + ] + + # Arguments we will accept on the command line and pass through to try + # syntax with no further intervention. The set is taken from + # http://trychooser.pub.build.mozilla.org with a few additions. + # + # Note that the meaning of store_false and store_true arguments is + # not preserved here, as we're only using these to echo the literal + # arguments to another consumer. Specifying either store_false or + # store_true here will have an equivalent effect. + pass_through_arguments = { + "--rebuild": { + "action": "store", + "dest": "rebuild", + "help": "Re-trigger all test jobs (up to 20 times)", + }, + "--rebuild-talos": { + "action": "store", + "dest": "rebuild_talos", + "help": "Re-trigger all talos jobs", + }, + "--interactive": { + "action": "store_true", + "dest": "interactive", + "help": "Allow ssh-like access to running test containers", + }, + "--no-retry": { + "action": "store_true", + "dest": "no_retry", + "help": "Do not retrigger failed tests", + }, + "--setenv": { + "action": "append", + "dest": "setenv", + "help": "Set the corresponding variable in the test environment for " + "applicable harnesses.", + }, + "-f": { + "action": "store_true", + "dest": "failure_emails", + "help": "Request failure emails only", + }, + "--failure-emails": { + "action": "store_true", + "dest": "failure_emails", + "help": "Request failure emails only", + }, + "-e": { + "action": "store_true", + "dest": "all_emails", + "help": "Request all emails", + }, + "--all-emails": { + "action": "store_true", + "dest": "all_emails", + "help": "Request all emails", + }, + "--artifact": { + "action": "store_true", + "dest": "artifact", + "help": "Force artifact builds where possible.", + }, + "--upload-xdbs": { + "action": "store_true", + "dest": "upload_xdbs", + "help": "Upload XDB compilation db files generated by hazard build", + }, + } + task_configs = [] + + def __init__(self, *args, **kwargs): + BaseTryParser.__init__(self, *args, **kwargs) + + group = self.add_argument_group("pass-through arguments") + for arg, opts in self.pass_through_arguments.items(): + group.add_argument(arg, **opts) + + +class TryArgumentTokenizer: + symbols = [ + ("separator", ","), + ("list_start", r"\["), + ("list_end", r"\]"), + ("item", r"([^,\[\]\s][^,\[\]]+)"), + ("space", r"\s+"), + ] + token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols)) + + def tokenize(self, data): + for match in self.token_re.finditer(data): + symbol = match.lastgroup + data = match.group(symbol) + if symbol == "space": + pass + else: + yield symbol, data + + +class TryArgumentParser: + """Simple three-state parser for handling expressions + of the from "foo[sub item, another], bar,baz". This takes + input from the TryArgumentTokenizer and runs through a small + state machine, returning a dictionary of {top-level-item:[sub_items]} + i.e. the above would result in + {"foo":["sub item", "another"], "bar": [], "baz": []} + In the case of invalid input a ValueError is raised.""" + + EOF = object() + + def __init__(self): + self.reset() + + def reset(self): + self.tokens = None + self.current_item = None + self.data = {} + self.token = None + self.state = None + + def parse(self, tokens): + self.reset() + self.tokens = tokens + self.consume() + self.state = self.item_state + while self.token[0] != self.EOF: + self.state() + return self.data + + def consume(self): + try: + self.token = next(self.tokens) + except StopIteration: + self.token = (self.EOF, None) + + def expect(self, *types): + if self.token[0] not in types: + raise ValueError( + "Error parsing try string, unexpected %s" % (self.token[0]) + ) + + def item_state(self): + self.expect("item") + value = self.token[1].strip() + if value not in self.data: + self.data[value] = [] + self.current_item = value + self.consume() + if self.token[0] == "separator": + self.consume() + elif self.token[0] == "list_start": + self.consume() + self.state = self.subitem_state + elif self.token[0] == self.EOF: + pass + else: + raise ValueError + + def subitem_state(self): + self.expect("item") + value = self.token[1].strip() + self.data[self.current_item].append(value) + self.consume() + if self.token[0] == "separator": + self.consume() + elif self.token[0] == "list_end": + self.consume() + self.state = self.after_list_end_state + else: + raise ValueError + + def after_list_end_state(self): + self.expect("separator") + self.consume() + self.state = self.item_state + + +def parse_arg(arg): + tokenizer = TryArgumentTokenizer() + parser = TryArgumentParser() + return parser.parse(tokenizer.tokenize(arg)) + + +class AutoTry: + + # Maps from flavors to the job names needed to run that flavour + flavor_jobs = { + "mochitest": ["mochitest-1", "mochitest-e10s-1"], + "xpcshell": ["xpcshell"], + "chrome": ["mochitest-o"], + "browser-a11y": ["mochitest-ba"], + "browser-media": ["mochitest-bmda"], + "browser-chrome": [ + "mochitest-browser-chrome-1", + "mochitest-e10s-browser-chrome-1", + "mochitest-browser-chrome-e10s-1", + ], + "devtools-chrome": [ + "mochitest-devtools-chrome-1", + "mochitest-e10s-devtools-chrome-1", + "mochitest-devtools-chrome-e10s-1", + ], + "crashtest": ["crashtest", "crashtest-e10s"], + "reftest": ["reftest", "reftest-e10s"], + "remote": ["mochitest-remote"], + "web-platform-tests": ["web-platform-tests-1"], + } + + flavor_suites = { + "mochitest": "mochitests", + "xpcshell": "xpcshell", + "chrome": "mochitest-o", + "browser-chrome": "mochitest-bc", + "browser-a11y": "mochitest-ba", + "browser-media": "mochitest-bmda", + "devtools-chrome": "mochitest-dt", + "crashtest": "crashtest", + "reftest": "reftest", + "web-platform-tests": "web-platform-tests", + } + + compiled_suites = [ + "cppunit", + "gtest", + "jittest", + ] + + common_suites = [ + "cppunit", + "crashtest", + "firefox-ui-functional", + "geckoview", + "geckoview-junit", + "gtest", + "jittest", + "jsreftest", + "marionette", + "marionette-e10s", + "mochitests", + "reftest", + "robocop", + "web-platform-tests", + "xpcshell", + ] + + def __init__(self): + self.topsrcdir = build.topsrcdir + self._resolver = None + + @property + def resolver(self): + if self._resolver is None: + self._resolver = TestResolver.from_environment(cwd=here) + return self._resolver + + @classmethod + def split_try_string(cls, data): + return re.findall(r"(?:\[.*?\]|\S)+", data) + + def paths_by_flavor(self, paths=None, tags=None): + paths_by_flavor = defaultdict(set) + + if not (paths or tags): + return dict(paths_by_flavor) + + tests = list(self.resolver.resolve_tests(paths=paths, tags=tags)) + + for t in tests: + if t["flavor"] in self.flavor_suites: + flavor = t["flavor"] + if "subsuite" in t and t["subsuite"] == "devtools": + flavor = "devtools-chrome" + + if "subsuite" in t and t["subsuite"] == "a11y": + flavor = "browser-a11y" + + if "subsuite" in t and t["subsuite"] == "media-bc": + flavor = "browser-media" + + if flavor in ["crashtest", "reftest"]: + manifest_relpath = os.path.relpath(t["manifest"], self.topsrcdir) + paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath)) + elif "dir_relpath" in t: + paths_by_flavor[flavor].add(t["dir_relpath"]) + else: + file_relpath = os.path.relpath(t["path"], self.topsrcdir) + dir_relpath = os.path.dirname(file_relpath) + paths_by_flavor[flavor].add(dir_relpath) + + for flavor, path_set in paths_by_flavor.items(): + paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths) + + return dict(paths_by_flavor) + + def deduplicate_prefixes(self, path_set, input_paths): + # Removes paths redundant to test selection in the given path set. + # If a path was passed on the commandline that is the prefix of a + # path in our set, we only need to include the specified prefix to + # run the intended tests (every test in "layout/base" will run if + # "layout" is passed to the reftest harness). + removals = set() + additions = set() + + for path in path_set: + full_path = path + while path: + path, _ = os.path.split(path) + if path in input_paths: + removals.add(full_path) + additions.add(path) + + return additions | (path_set - removals) + + def remove_duplicates(self, paths_by_flavor, tests): + rv = {} + for item in paths_by_flavor: + if self.flavor_suites[item] not in tests: + rv[item] = paths_by_flavor[item].copy() + return rv + + def calc_try_syntax( + self, + platforms, + tests, + talos, + jobs, + builds, + paths_by_flavor, + tags, + extras, + intersection, + ): + parts = ["try:"] + + if platforms: + parts.extend(["-b", builds, "-p", ",".join(platforms)]) + + suites = tests if not intersection else {} + paths = set() + for flavor, flavor_tests in paths_by_flavor.items(): + suite = self.flavor_suites[flavor] + if suite not in suites and (not intersection or suite in tests): + for job_name in self.flavor_jobs[flavor]: + for test in flavor_tests: + paths.add("{}:{}".format(flavor, test)) + suites[job_name] = tests.get(suite, []) + + # intersection implies tests are expected + if intersection and not suites: + raise ValueError("No tests found matching filters") + + if extras.get("artifact") and any([p.endswith("-nightly") for p in platforms]): + print( + 'You asked for |--artifact| but "-nightly" platforms don\'t have artifacts. ' + "Running without |--artifact| instead." + ) + del extras["artifact"] + + if extras.get("artifact"): + rejected = [] + for suite in suites.keys(): + if any([suite.startswith(c) for c in self.compiled_suites]): + rejected.append(suite) + if rejected: + raise ValueError( + "You can't run {} with " + "--artifact option.".format(", ".join(rejected)) + ) + + if extras.get("artifact") and "all" in suites.keys(): + non_compiled_suites = set(self.common_suites) - set(self.compiled_suites) + message = ( + "You asked for |-u all| with |--artifact| but compiled-code tests ({tests})" + " can't run against an artifact build. Running (-u {non_compiled_suites}) " + "instead." + ) + string_format = { + "tests": ",".join(self.compiled_suites), + "non_compiled_suites": ",".join(non_compiled_suites), + } + print(message.format(**string_format)) + del suites["all"] + suites.update({suite_name: None for suite_name in non_compiled_suites}) + + if suites: + parts.append("-u") + parts.append( + ",".join( + "{}{}".format(k, "[%s]" % ",".join(v) if v else "") + for k, v in sorted(suites.items()) + ) + ) + + if talos: + parts.append("-t") + parts.append( + ",".join( + "{}{}".format(k, "[%s]" % ",".join(v) if v else "") + for k, v in sorted(talos.items()) + ) + ) + + if jobs: + parts.append("-j") + parts.append(",".join(jobs)) + + if tags: + parts.append(" ".join("--tag %s" % t for t in tags)) + + if paths: + parts.append("--try-test-paths %s" % " ".join(sorted(paths))) + + args_by_dest = { + v["dest"]: k for k, v in SyntaxParser.pass_through_arguments.items() + } + for dest, value in extras.items(): + assert dest in args_by_dest + arg = args_by_dest[dest] + action = SyntaxParser.pass_through_arguments[arg]["action"] + if action == "store": + parts.append(arg) + parts.append(value) + if action == "append": + for e in value: + parts.append(arg) + parts.append(e) + if action in ("store_true", "store_false"): + parts.append(arg) + + return " ".join(parts) + + def normalise_list(self, items, allow_subitems=False): + rv = defaultdict(list) + for item in items: + parsed = parse_arg(item) + for key, values in parsed.items(): + rv[key].extend(values) + + if not allow_subitems: + if not all(item == [] for item in rv.values()): + raise ValueError("Unexpected subitems in argument") + return rv.keys() + else: + return rv + + def validate_args(self, **kwargs): + tests_selected = kwargs["tests"] or kwargs["paths"] or kwargs["tags"] + if kwargs["platforms"] is None and (kwargs["jobs"] is None or tests_selected): + if "AUTOTRY_PLATFORM_HINT" in os.environ: + kwargs["platforms"] = [os.environ["AUTOTRY_PLATFORM_HINT"]] + elif tests_selected: + print("Must specify platform when selecting tests.") + sys.exit(1) + else: + print( + "Either platforms or jobs must be specified as an argument to autotry." + ) + sys.exit(1) + + try: + platforms = ( + self.normalise_list(kwargs["platforms"]) if kwargs["platforms"] else {} + ) + except ValueError as e: + print("Error parsing -p argument:\n%s" % e) + sys.exit(1) + + try: + tests = ( + self.normalise_list(kwargs["tests"], allow_subitems=True) + if kwargs["tests"] + else {} + ) + except ValueError as e: + print("Error parsing -u argument ({}):\n{}".format(kwargs["tests"], e)) + sys.exit(1) + + try: + talos = ( + self.normalise_list(kwargs["talos"], allow_subitems=True) + if kwargs["talos"] + else [] + ) + except ValueError as e: + print("Error parsing -t argument:\n%s" % e) + sys.exit(1) + + try: + jobs = self.normalise_list(kwargs["jobs"]) if kwargs["jobs"] else {} + except ValueError as e: + print("Error parsing -j argument:\n%s" % e) + sys.exit(1) + + paths = [] + for p in kwargs["paths"]: + p = mozpath.normpath(os.path.abspath(p)) + if not (os.path.isdir(p) and p.startswith(self.topsrcdir)): + print( + 'Specified path "%s" is not a directory under the srcdir,' + " unable to specify tests outside of the srcdir" % p + ) + sys.exit(1) + if len(p) <= len(self.topsrcdir): + print( + 'Specified path "%s" is at the top of the srcdir and would' + " select all tests." % p + ) + sys.exit(1) + paths.append(os.path.relpath(p, self.topsrcdir)) + + try: + tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else [] + except ValueError as e: + print("Error parsing --tags argument:\n%s" % e) + sys.exit(1) + + extra_values = {k["dest"] for k in SyntaxParser.pass_through_arguments.values()} + extra_args = {k: v for k, v in kwargs.items() if k in extra_values and v} + + return kwargs["builds"], platforms, tests, talos, jobs, paths, tags, extra_args + + def run(self, **kwargs): + if not any(kwargs[item] for item in ("paths", "tests", "tags")): + kwargs["paths"] = set() + kwargs["tags"] = set() + + builds, platforms, tests, talos, jobs, paths, tags, extra = self.validate_args( + **kwargs + ) + + if paths or tags: + paths = [ + os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir) + for item in paths + ] + paths_by_flavor = self.paths_by_flavor(paths=paths, tags=tags) + + if not paths_by_flavor and not tests: + print( + "No tests were found when attempting to resolve paths:\n\n\t%s" + % paths + ) + sys.exit(1) + + if not kwargs["intersection"]: + paths_by_flavor = self.remove_duplicates(paths_by_flavor, tests) + else: + paths_by_flavor = {} + + # No point in dealing with artifacts if we aren't running any builds + local_artifact_build = False + if platforms: + local_artifact_build = kwargs.get("local_artifact_build", False) + + # Add --artifact if --enable-artifact-builds is set ... + if local_artifact_build: + extra["artifact"] = True + # ... unless --no-artifact is explicitly given. + if kwargs["no_artifact"]: + if "artifact" in extra: + del extra["artifact"] + + try: + msg = self.calc_try_syntax( + platforms, + tests, + talos, + jobs, + builds, + paths_by_flavor, + tags, + extra, + kwargs["intersection"], + ) + except ValueError as e: + print(e) + sys.exit(1) + + if local_artifact_build and not kwargs["no_artifact"]: + print( + "mozconfig has --enable-artifact-builds; including " + "--artifact flag in try syntax (use --no-artifact " + "to override)" + ) + + if kwargs["verbose"] and paths_by_flavor: + print("The following tests will be selected: ") + for flavor, paths in paths_by_flavor.items(): + print("{}: {}".format(flavor, ",".join(paths))) + + if kwargs["verbose"]: + print("The following try syntax was calculated:\n%s" % msg) + + push_to_try( + "syntax", + kwargs["message"].format(msg=msg), + stage_changes=kwargs["stage_changes"], + dry_run=kwargs["dry_run"], + closed_tree=kwargs["closed_tree"], + ) + + +def run(**kwargs): + at = AutoTry() + return at.run(**kwargs) diff --git a/tools/tryselect/task_config.py b/tools/tryselect/task_config.py new file mode 100644 index 0000000000..07a39d39e9 --- /dev/null +++ b/tools/tryselect/task_config.py @@ -0,0 +1,530 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Templates provide a way of modifying the task definition of selected tasks. +They are added to 'try_task_config.json' and processed by the transforms. +""" + + +import json +import os +import subprocess +import sys +from abc import ABCMeta, abstractmethod, abstractproperty +from argparse import SUPPRESS, Action +from textwrap import dedent + +import mozpack.path as mozpath +import six +from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject + +from .tasks import resolve_tests_by_suite + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + + +class TryConfig: + __metaclass__ = ABCMeta + + def __init__(self): + self.dests = set() + + def add_arguments(self, parser): + for cli, kwargs in self.arguments: + action = parser.add_argument(*cli, **kwargs) + self.dests.add(action.dest) + + @abstractproperty + def arguments(self): + pass + + @abstractmethod + def try_config(self, **kwargs): + pass + + def validate(self, **kwargs): + pass + + +class Artifact(TryConfig): + + arguments = [ + [ + ["--artifact"], + {"action": "store_true", "help": "Force artifact builds where possible."}, + ], + [ + ["--no-artifact"], + { + "action": "store_true", + "help": "Disable artifact builds even if being used locally.", + }, + ], + ] + + def add_arguments(self, parser): + group = parser.add_mutually_exclusive_group() + return super().add_arguments(group) + + @classmethod + def is_artifact_build(cls): + try: + return build.substs.get("MOZ_ARTIFACT_BUILDS", False) + except BuildEnvironmentNotFoundException: + return False + + def try_config(self, artifact, no_artifact, **kwargs): + if artifact: + return {"use-artifact-builds": True, "disable-pgo": True} + + if no_artifact: + return + + if self.is_artifact_build(): + print("Artifact builds enabled, pass --no-artifact to disable") + return {"use-artifact-builds": True, "disable-pgo": True} + + +class Pernosco(TryConfig): + arguments = [ + [ + ["--pernosco"], + { + "action": "store_true", + "default": None, + "help": "Opt-in to analysis by the Pernosco debugging service.", + }, + ], + [ + ["--no-pernosco"], + { + "dest": "pernosco", + "action": "store_false", + "default": None, + "help": "Opt-out of the Pernosco debugging service (if you are on the whitelist).", + }, + ], + ] + + def add_arguments(self, parser): + group = parser.add_mutually_exclusive_group() + return super().add_arguments(group) + + def try_config(self, pernosco, **kwargs): + if pernosco is None: + return + + if pernosco: + try: + # The Pernosco service currently requires a Mozilla e-mail address to + # log in. Prevent people with non-Mozilla addresses from using this + # flag so they don't end up consuming time and resources only to + # realize they can't actually log in and see the reports. + cmd = ["ssh", "-G", "hg.mozilla.org"] + output = subprocess.check_output( + cmd, universal_newlines=True + ).splitlines() + address = [ + l.rsplit(" ", 1)[-1] for l in output if l.startswith("user") + ][0] + if not address.endswith("@mozilla.com"): + print( + dedent( + """\ + Pernosco requires a Mozilla e-mail address to view its reports. Please + push to try with an @mozilla.com address to use --pernosco. + + Current user: {} + """.format( + address + ) + ) + ) + sys.exit(1) + + except (subprocess.CalledProcessError, IndexError): + print("warning: failed to detect current user for 'hg.mozilla.org'") + print("Pernosco requires a Mozilla e-mail address to view its reports.") + while True: + answer = input( + "Do you have an @mozilla.com address? [Y/n]: " + ).lower() + if answer == "n": + sys.exit(1) + elif answer == "y": + break + + return { + "env": { + "PERNOSCO": str(int(pernosco)), + } + } + + def validate(self, **kwargs): + if kwargs["try_config"].get("use-artifact-builds"): + print( + "Pernosco does not support artifact builds at this time. " + "Please try again with '--no-artifact'." + ) + sys.exit(1) + + +class Path(TryConfig): + + arguments = [ + [ + ["paths"], + { + "nargs": "*", + "default": [], + "help": "Run tasks containing tests under the specified path(s).", + }, + ], + ] + + def try_config(self, paths, **kwargs): + if not paths: + return + + for p in paths: + if not os.path.exists(p): + print("error: '{}' is not a valid path.".format(p), file=sys.stderr) + sys.exit(1) + + paths = [ + mozpath.relpath(mozpath.join(os.getcwd(), p), build.topsrcdir) + for p in paths + ] + return { + "env": { + "MOZHARNESS_TEST_PATHS": six.ensure_text( + json.dumps(resolve_tests_by_suite(paths)) + ), + } + } + + +class Environment(TryConfig): + + arguments = [ + [ + ["--env"], + { + "action": "append", + "default": None, + "help": "Set an environment variable, of the form FOO=BAR. " + "Can be passed in multiple times.", + }, + ], + ] + + def try_config(self, env, **kwargs): + if not env: + return + return { + "env": dict(e.split("=", 1) for e in env), + } + + +class RangeAction(Action): + def __init__(self, min, max, *args, **kwargs): + self.min = min + self.max = max + kwargs["metavar"] = "[{}-{}]".format(self.min, self.max) + super().__init__(*args, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + name = option_string or self.dest + if values < self.min: + parser.error("{} can not be less than {}".format(name, self.min)) + if values > self.max: + parser.error("{} can not be more than {}".format(name, self.max)) + setattr(namespace, self.dest, values) + + +class Rebuild(TryConfig): + + arguments = [ + [ + ["--rebuild"], + { + "action": RangeAction, + "min": 2, + "max": 20, + "default": None, + "type": int, + "help": "Rebuild all selected tasks the specified number of times.", + }, + ], + ] + + def try_config(self, rebuild, **kwargs): + if not rebuild: + return + + if kwargs.get("full") and rebuild > 3: + print( + "warning: limiting --rebuild to 3 when using --full. " + "Use custom push actions to add more." + ) + rebuild = 3 + + return { + "rebuild": rebuild, + } + + +class Routes(TryConfig): + arguments = [ + [ + ["--route"], + { + "action": "append", + "dest": "routes", + "help": ( + "Additional route to add to the tasks " + "(note: these will not be added to the decision task)" + ), + }, + ], + ] + + def try_config(self, routes, **kwargs): + if routes: + return { + "routes": routes, + } + + +class ChemspillPrio(TryConfig): + + arguments = [ + [ + ["--chemspill-prio"], + { + "action": "store_true", + "help": "Run at a higher priority than most try jobs (chemspills only).", + }, + ], + ] + + def try_config(self, chemspill_prio, **kwargs): + if chemspill_prio: + return {"chemspill-prio": {}} + + +class GeckoProfile(TryConfig): + arguments = [ + [ + ["--gecko-profile"], + { + "dest": "profile", + "action": "store_true", + "default": False, + "help": "Create and upload a gecko profile during talos/raptor tasks.", + }, + ], + [ + ["--gecko-profile-interval"], + { + "dest": "gecko_profile_interval", + "type": float, + "help": "How frequently to take samples (ms)", + }, + ], + [ + ["--gecko-profile-entries"], + { + "dest": "gecko_profile_entries", + "type": int, + "help": "How many samples to take with the profiler", + }, + ], + [ + ["--gecko-profile-features"], + { + "dest": "gecko_profile_features", + "type": str, + "default": None, + "help": "Set the features enabled for the profiler.", + }, + ], + [ + ["--gecko-profile-threads"], + { + "dest": "gecko_profile_threads", + "type": str, + "help": "Comma-separated list of threads to sample.", + }, + ], + # For backwards compatibility + [ + ["--talos-profile"], + { + "dest": "profile", + "action": "store_true", + "default": False, + "help": SUPPRESS, + }, + ], + # This is added for consistency with the 'syntax' selector + [ + ["--geckoProfile"], + { + "dest": "profile", + "action": "store_true", + "default": False, + "help": SUPPRESS, + }, + ], + ] + + def try_config( + self, + profile, + gecko_profile_interval, + gecko_profile_entries, + gecko_profile_features, + gecko_profile_threads, + **kwargs + ): + if profile or not all( + s is None for s in (gecko_profile_features, gecko_profile_threads) + ): + cfg = { + "gecko-profile": True, + "gecko-profile-interval": gecko_profile_interval, + "gecko-profile-entries": gecko_profile_entries, + "gecko-profile-features": gecko_profile_features, + "gecko-profile-threads": gecko_profile_threads, + } + return {key: value for key, value in cfg.items() if value is not None} + + +class Browsertime(TryConfig): + arguments = [ + [ + ["--browsertime"], + { + "action": "store_true", + "help": "Use browsertime during Raptor tasks.", + }, + ], + ] + + def try_config(self, browsertime, **kwargs): + if browsertime: + return { + "browsertime": True, + } + + +class DisablePgo(TryConfig): + + arguments = [ + [ + ["--disable-pgo"], + { + "action": "store_true", + "help": "Don't run PGO builds", + }, + ], + ] + + def try_config(self, disable_pgo, **kwargs): + if disable_pgo: + return { + "disable-pgo": True, + } + + +class WorkerOverrides(TryConfig): + + arguments = [ + [ + ["--worker-override"], + { + "action": "append", + "dest": "worker_overrides", + "help": ( + "Override the worker pool used for a given taskgraph worker alias. " + "The argument should be `=`. " + "Can be specified multiple times." + ), + }, + ], + [ + ["--worker-suffix"], + { + "action": "append", + "dest": "worker_suffixes", + "help": ( + "Override the worker pool used for a given taskgraph worker alias, " + "by appending a suffix to the work-pool. " + "The argument should be `=`. " + "Can be specified multiple times." + ), + }, + ], + ] + + def try_config(self, worker_overrides, worker_suffixes, **kwargs): + from gecko_taskgraph.util.workertypes import get_worker_type + from taskgraph.config import load_graph_config + + overrides = {} + if worker_overrides: + for override in worker_overrides: + alias, worker_pool = override.split("=", 1) + if alias in overrides: + print( + "Can't override worker alias {alias} more than once. " + "Already set to use {previous}, but also asked to use {new}.".format( + alias=alias, previous=overrides[alias], new=worker_pool + ) + ) + sys.exit(1) + overrides[alias] = worker_pool + + if worker_suffixes: + root = build.topsrcdir + root = os.path.join(root, "taskcluster", "ci") + graph_config = load_graph_config(root) + for worker_suffix in worker_suffixes: + alias, suffix = worker_suffix.split("=", 1) + if alias in overrides: + print( + "Can't override worker alias {alias} more than once. " + "Already set to use {previous}, but also asked " + "to add suffix {suffix}.".format( + alias=alias, previous=overrides[alias], suffix=suffix + ) + ) + sys.exit(1) + provisioner, worker_type = get_worker_type( + graph_config, worker_type=alias, parameters={"level": "1"} + ) + overrides[alias] = "{provisioner}/{worker_type}{suffix}".format( + provisioner=provisioner, worker_type=worker_type, suffix=suffix + ) + + if overrides: + return {"worker-overrides": overrides} + + +all_task_configs = { + "artifact": Artifact, + "browsertime": Browsertime, + "chemspill-prio": ChemspillPrio, + "disable-pgo": DisablePgo, + "env": Environment, + "gecko-profile": GeckoProfile, + "path": Path, + "pernosco": Pernosco, + "rebuild": Rebuild, + "routes": Routes, + "worker-overrides": WorkerOverrides, +} diff --git a/tools/tryselect/tasks.py b/tools/tryselect/tasks.py new file mode 100644 index 0000000000..bca6b3de1b --- /dev/null +++ b/tools/tryselect/tasks.py @@ -0,0 +1,186 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +import re +import sys +from collections import defaultdict + +import mozpack.path as mozpath +import taskgraph +from mach.util import get_state_dir +from mozbuild.base import MozbuildObject +from mozpack.files import FileFinder +from moztest.resolve import TestManifestLoader, TestResolver, get_suite_definition +from taskgraph.generator import TaskGraphGenerator +from taskgraph.parameters import ParameterMismatch, parameters_loader +from taskgraph.taskgraph import TaskGraph + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + +PARAMETER_MISMATCH = """ +ERROR - The parameters being used to generate tasks differ from those expected +by your working copy: + + {} + +To fix this, either rebase onto the latest mozilla-central or pass in +-p/--parameters. For more information on how to define parameters, see: +https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters +""" + + +def invalidate(cache): + try: + cmod = os.path.getmtime(cache) + except OSError as e: + # File does not exist. We catch OSError rather than use `isfile` + # because the recommended watchman hook could possibly invalidate the + # cache in-between the check to `isfile` and the call to `getmtime` + # below. + if e.errno == 2: + return + raise + + tc_dir = os.path.join(build.topsrcdir, "taskcluster") + tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir)) + + if tmod > cmod: + os.remove(cache) + + +def cache_key(attr, params, disable_target_task_filter): + key = attr + if params and params["project"] not in ("autoland", "mozilla-central"): + key += f"-{params['project']}" + + if disable_target_task_filter and "full" not in attr: + key += "-uncommon" + return key + + +def generate_tasks(params=None, full=False, disable_target_task_filter=False): + attr = "full_task_set" if full else "target_task_set" + target_tasks_method = ( + "try_select_tasks" + if not disable_target_task_filter + else "try_select_tasks_uncommon" + ) + params = parameters_loader( + params, + strict=False, + overrides={ + "try_mode": "try_select", + "target_tasks_method": target_tasks_method, + }, + ) + root = os.path.join(build.topsrcdir, "taskcluster", "ci") + taskgraph.fast = True + generator = TaskGraphGenerator(root_dir=root, parameters=params) + + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + key = cache_key(attr, generator.parameters, disable_target_task_filter) + cache = os.path.join(cache_dir, key) + + invalidate(cache) + if os.path.isfile(cache): + with open(cache) as fh: + return TaskGraph.from_json(json.load(fh))[1] + + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + + print("Task configuration changed, generating {}".format(attr.replace("_", " "))) + + cwd = os.getcwd() + os.chdir(build.topsrcdir) + + def generate(attr): + try: + tg = getattr(generator, attr) + except ParameterMismatch as e: + print(PARAMETER_MISMATCH.format(e.args[0])) + sys.exit(1) + + # write cache + key = cache_key(attr, generator.parameters, disable_target_task_filter) + with open(os.path.join(cache_dir, key), "w") as fh: + json.dump(tg.to_json(), fh) + return tg + + # Cache both full_task_set and target_task_set regardless of whether or not + # --full was requested. Caching is cheap and can potentially save a lot of + # time. + tg_full = generate("full_task_set") + tg_target = generate("target_task_set") + + # discard results from these, we only need cache. + if full: + generate("full_task_graph") + generate("target_task_graph") + + os.chdir(cwd) + if full: + return tg_full + return tg_target + + +def filter_tasks_by_paths(tasks, paths): + resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) + run_suites, run_tests = resolver.resolve_metadata(paths) + flavors = {(t["flavor"], t.get("subsuite")) for t in run_tests} + + task_regexes = set() + for flavor, subsuite in flavors: + _, suite = get_suite_definition(flavor, subsuite, strict=True) + if "task_regex" not in suite: + print( + "warning: no tasks could be resolved from flavor '{}'{}".format( + flavor, " and subsuite '{}'".format(subsuite) if subsuite else "" + ) + ) + continue + + task_regexes.update(suite["task_regex"]) + + def match_task(task): + return any(re.search(pattern, task) for pattern in task_regexes) + + return filter(match_task, tasks) + + +def resolve_tests_by_suite(paths): + resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader) + _, run_tests = resolver.resolve_metadata(paths) + + suite_to_tests = defaultdict(list) + + # A dictionary containing all the input paths that we haven't yet + # assigned to a specific test flavor. + remaining_paths_by_suite = defaultdict(lambda: set(paths)) + + for test in run_tests: + key, _ = get_suite_definition(test["flavor"], test.get("subsuite"), strict=True) + + test_path = test.get("srcdir_relpath") + if test_path is None: + continue + found_path = None + manifest_relpath = None + if "manifest_relpath" in test: + manifest_relpath = mozpath.normpath(test["manifest_relpath"]) + for path in remaining_paths_by_suite[key]: + if test_path.startswith(path) or manifest_relpath == path: + found_path = path + break + if found_path: + suite_to_tests[key].append(found_path) + remaining_paths_by_suite[key].remove(found_path) + + return suite_to_tests diff --git a/tools/tryselect/test/conftest.py b/tools/tryselect/test/conftest.py new file mode 100644 index 0000000000..7ffffc4b9b --- /dev/null +++ b/tools/tryselect/test/conftest.py @@ -0,0 +1,101 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from unittest.mock import MagicMock + +import pytest +import yaml +from moztest.resolve import TestResolver +from taskgraph.graph import Graph +from taskgraph.task import Task +from taskgraph.taskgraph import TaskGraph +from tryselect import push + + +@pytest.fixture +def tg(request): + if not hasattr(request.module, "TASKS"): + pytest.fail( + "'tg' fixture used from a module that didn't define the TASKS variable" + ) + + tasks = request.module.TASKS + for task in tasks: + task.setdefault("task", {}) + task["task"].setdefault("tags", {}) + + tasks = {t["label"]: Task(**t) for t in tasks} + return TaskGraph(tasks, Graph(tasks.keys(), set())) + + +@pytest.fixture +def patch_resolver(monkeypatch): + def inner(suites, tests): + def fake_test_metadata(*args, **kwargs): + return suites, tests + + monkeypatch.setattr(TestResolver, "resolve_metadata", fake_test_metadata) + + return inner + + +@pytest.fixture(autouse=True) +def patch_vcs(monkeypatch): + attrs = { + "path": push.vcs.path, + } + mock = MagicMock() + mock.configure_mock(**attrs) + monkeypatch.setattr(push, "vcs", mock) + + +@pytest.fixture(scope="session") +def run_mach(): + import mach_initialize + from mach.config import ConfigSettings + from tryselect.tasks import build + + mach = mach_initialize.initialize(build.topsrcdir) + + def inner(args): + mach.settings = ConfigSettings() + return mach.run(args) + + return inner + + +def pytest_generate_tests(metafunc): + if all( + fixture in metafunc.fixturenames + for fixture in ("task_config", "args", "expected") + ): + + def load_tests(): + for task_config, tests in metafunc.module.TASK_CONFIG_TESTS.items(): + for args, expected in tests: + yield (task_config, args, expected) + + tests = list(load_tests()) + ids = ["{} {}".format(t[0], " ".join(t[1])).strip() for t in tests] + metafunc.parametrize("task_config,args,expected", tests, ids=ids) + + elif all( + fixture in metafunc.fixturenames for fixture in ("shared_name", "shared_preset") + ): + preset_path = os.path.join( + push.build.topsrcdir, "tools", "tryselect", "try_presets.yml" + ) + with open(preset_path, "r") as fh: + presets = list(yaml.safe_load(fh).items()) + + ids = [p[0] for p in presets] + + # Mark fuzzy presets on Windows xfail due to fzf not being installed. + if os.name == "nt": + for i, preset in enumerate(presets): + if preset[1]["selector"] == "fuzzy": + presets[i] = pytest.param(*preset, marks=pytest.mark.xfail) + + metafunc.parametrize("shared_name,shared_preset", presets, ids=ids) diff --git a/tools/tryselect/test/cram.ini b/tools/tryselect/test/cram.ini new file mode 100644 index 0000000000..b1f0d8dfec --- /dev/null +++ b/tools/tryselect/test/cram.ini @@ -0,0 +1,5 @@ +[test_auto.t] +[test_empty.t] +[test_fuzzy.t] +[test_message.t] +[test_preset.t] diff --git a/tools/tryselect/test/python.ini b/tools/tryselect/test/python.ini new file mode 100644 index 0000000000..4b75ea6ab5 --- /dev/null +++ b/tools/tryselect/test/python.ini @@ -0,0 +1,19 @@ +[DEFAULT] +subsuite=try + +[test_again.py] +[test_auto.py] +[test_chooser.py] +requirements = tools/tryselect/selectors/chooser/requirements.txt +[test_fuzzy.py] +[test_mozharness_integration.py] +[test_perf.py] +[test_perfcomparators.py] +[test_presets.py] +# Modifies "task_duration_history.json" in .mozbuild. Since other tests depend on this file, this test +# shouldn't be run in parallel with those other tests. +sequential = true +[test_release.py] +[test_scriptworker.py] +[test_tasks.py] +[test_task_configs.py] diff --git a/tools/tryselect/test/setup.sh b/tools/tryselect/test/setup.sh new file mode 100644 index 0000000000..d9f3b62a2e --- /dev/null +++ b/tools/tryselect/test/setup.sh @@ -0,0 +1,99 @@ +export topsrcdir=$TESTDIR/../../../ +export MOZBUILD_STATE_PATH=$TMP/mozbuild +export MACH_TRY_PRESET_PATHS=$MOZBUILD_STATE_PATH/try_presets.yml + +# This helps to find fzf when running these tests locally, since normally fzf +# would be found via MOZBUILD_STATE_PATH pointing to $HOME/.mozbuild +export PATH="$PATH:$HOME/.mozbuild/fzf/bin" + +export MACHRC=$TMP/machrc +cat > $MACHRC << EOF +[try] +default=syntax +EOF + +cmd="$topsrcdir/mach python -c 'from mach.util import get_state_dir; print(get_state_dir(specific_to_topsrcdir=True))'" +# First run local state dir generation so it doesn't affect test output. +eval $cmd > /dev/null 2>&1 +# Now run it again to get the actual directory. +cachedir=$(eval $cmd)/cache/taskgraph +mkdir -p $cachedir + +cat > $cachedir/target_task_set << EOF +{ + "test/foo-opt": { + "kind": "test", + "label": "test/foo-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/foo-debug": { + "kind": "test", + "label": "test/foo-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "build-baz": { + "kind": "build", + "label": "build-baz", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + } +} +EOF + +cat > $cachedir/full_task_set << EOF +{ + "test/foo-opt": { + "kind": "test", + "label": "test/foo-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/foo-debug": { + "kind": "test", + "label": "test/foo-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/bar-opt": { + "kind": "test", + "label": "test/bar-opt", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "test/bar-debug": { + "kind": "test", + "label": "test/bar-debug", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + }, + "build-baz": { + "kind": "build", + "label": "build-baz", + "attributes": {}, + "task": {}, + "optimization": {}, + "dependencies": {} + } +} +EOF + +# set mtime to the future so we don't re-generate tasks +find $cachedir -type f -exec touch -d "next day" {} + + +export testargs="--no-push --no-artifact" diff --git a/tools/tryselect/test/test_again.py b/tools/tryselect/test/test_again.py new file mode 100644 index 0000000000..0e5d9d6b6d --- /dev/null +++ b/tools/tryselect/test/test_again.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import mozunit +import pytest +from six.moves import reload_module as reload +from tryselect import push +from tryselect.selectors import again + + +@pytest.fixture(autouse=True) +def patch_history_path(tmpdir, monkeypatch): + monkeypatch.setattr(push, "history_path", tmpdir.join("history.json").strpath) + reload(again) + + +def test_try_again(monkeypatch): + push.push_to_try( + "fuzzy", + "Fuzzy message", + try_task_config=push.generate_try_task_config( + "fuzzy", + ["foo", "bar"], + {"use-artifact-builds": True}, + ), + ) + + assert os.path.isfile(push.history_path) + with open(push.history_path, "r") as fh: + assert len(fh.readlines()) == 1 + + def fake_push_to_try(*args, **kwargs): + return args, kwargs + + monkeypatch.setattr(push, "push_to_try", fake_push_to_try) + reload(again) + + args, kwargs = again.run() + + assert args[0] == "again" + assert args[1] == "Fuzzy message" + + try_task_config = kwargs.pop("try_task_config") + assert sorted(try_task_config.get("tasks")) == sorted(["foo", "bar"]) + assert try_task_config.get("env") == {"TRY_SELECTOR": "fuzzy"} + assert try_task_config.get("use-artifact-builds") + + with open(push.history_path, "r") as fh: + assert len(fh.readlines()) == 1 + + +def test_no_push_does_not_generate_history(tmpdir): + assert not os.path.isfile(push.history_path) + + push.push_to_try( + "fuzzy", + "Fuzzy", + try_task_config=push.generate_try_task_config( + "fuzzy", + ["foo", "bar"], + {"use-artifact-builds": True}, + ), + dry_run=True, + ) + assert not os.path.isfile(push.history_path) + assert again.run() == 1 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_auto.py b/tools/tryselect/test/test_auto.py new file mode 100644 index 0000000000..63f0fe6bd7 --- /dev/null +++ b/tools/tryselect/test/test_auto.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest +from tryselect.selectors.auto import AutoParser + + +def test_strategy_validation(): + parser = AutoParser() + args = parser.parse_args(["--strategy", "relevant_tests"]) + assert args.strategy == "gecko_taskgraph.optimize:tryselect.relevant_tests" + + args = parser.parse_args( + ["--strategy", "gecko_taskgraph.optimize:experimental.relevant_tests"] + ) + assert args.strategy == "gecko_taskgraph.optimize:experimental.relevant_tests" + + with pytest.raises(SystemExit): + parser.parse_args(["--strategy", "gecko_taskgraph.optimize:tryselect"]) + + with pytest.raises(SystemExit): + parser.parse_args(["--strategy", "foo"]) + + with pytest.raises(SystemExit): + parser.parse_args(["--strategy", "foo:bar"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_auto.t b/tools/tryselect/test/test_auto.t new file mode 100644 index 0000000000..c3fe797949 --- /dev/null +++ b/tools/tryselect/test/test_auto.t @@ -0,0 +1,61 @@ + + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test auto selector + + $ ./mach try auto $testargs + Commit message: + Tasks automatically selected. + + Pushed via `mach try auto` + Calculated try_task_config.json: + { + "parameters": { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", + "optimize_target_tasks": true, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {} + }, + "version": 2 + } + + + $ ./mach try auto $testargs --closed-tree + Commit message: + Tasks automatically selected. ON A CLOSED TREE + + Pushed via `mach try auto` + Calculated try_task_config.json: + { + "parameters": { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", + "optimize_target_tasks": true, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {} + }, + "version": 2 + } + + $ ./mach try auto $testargs --closed-tree -m "foo {msg} bar" + Commit message: + foo Tasks automatically selected. bar ON A CLOSED TREE + + Pushed via `mach try auto` + Calculated try_task_config.json: + { + "parameters": { + "optimize_strategies": "gecko_taskgraph.optimize:tryselect.bugbug_reduced_manifests_config_selection_medium", + "optimize_target_tasks": true, + "target_tasks_method": "try_auto", + "test_manifest_loader": "bugbug", + "try_mode": "try_auto", + "try_task_config": {} + }, + "version": 2 + } + diff --git a/tools/tryselect/test/test_chooser.py b/tools/tryselect/test/test_chooser.py new file mode 100644 index 0000000000..885ed41a78 --- /dev/null +++ b/tools/tryselect/test/test_chooser.py @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest +from tryselect.selectors.chooser.app import create_application + +TASKS = [ + { + "kind": "build", + "label": "build-windows", + "attributes": { + "build_platform": "windows", + }, + }, + { + "kind": "test", + "label": "test-windows-mochitest-e10s", + "attributes": { + "unittest_suite": "mochitest-browser-chrome", + "mochitest_try_name": "mochitest-browser-chrome", + }, + }, +] + + +@pytest.fixture +def app(tg): + app = create_application(tg) + app.config["TESTING"] = True + + ctx = app.app_context() + ctx.push() + yield app + ctx.pop() + + +def test_try_chooser(app): + client = app.test_client() + + response = client.get("/") + assert response.status_code == 200 + + expected_output = [ + b"""Try Chooser Enhanced""", + b"""""", # noqa + b"""""", # noqa + ] + + for expected in expected_output: + assert expected in response.data + + response = client.post("/", data={"action": "Cancel"}) + assert response.status_code == 200 + assert b"You may now close this page" in response.data + assert app.tasks == [] + + response = client.post("/", data={"action": "Push", "selected-tasks": ""}) + assert response.status_code == 200 + assert b"You may now close this page" in response.data + assert app.tasks == [] + + response = client.post( + "/", + data={ + "action": "Push", + "selected-tasks": "build-windows\ntest-windows-mochitest-e10s", + }, + ) + assert response.status_code == 200 + assert b"You may now close this page" in response.data + assert set(app.tasks) == set(["build-windows", "test-windows-mochitest-e10s"]) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_empty.t b/tools/tryselect/test/test_empty.t new file mode 100644 index 0000000000..f6dea366c4 --- /dev/null +++ b/tools/tryselect/test/test_empty.t @@ -0,0 +1,47 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test empty selector + + $ ./mach try empty --no-push + Commit message: + No try selector specified, use "Add New Jobs" to select tasks. + + Pushed via `mach try empty` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "empty" + }, + "tasks": [], + "version": 1 + } + + $ ./mach try empty --no-push --closed-tree + Commit message: + No try selector specified, use "Add New Jobs" to select tasks. ON A CLOSED TREE + + Pushed via `mach try empty` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "empty" + }, + "tasks": [], + "version": 1 + } + + $ ./mach try empty --no-push --closed-tree -m "foo {msg} bar" + Commit message: + foo No try selector specified, use "Add New Jobs" to select tasks. bar ON A CLOSED TREE + + Pushed via `mach try empty` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "empty" + }, + "tasks": [], + "version": 1 + } + diff --git a/tools/tryselect/test/test_fuzzy.py b/tools/tryselect/test/test_fuzzy.py new file mode 100644 index 0000000000..330ba99825 --- /dev/null +++ b/tools/tryselect/test/test_fuzzy.py @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import mozunit +import pytest + + +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_query_paths(run_mach, capfd): + cmd = [ + "try", + "fuzzy", + "--no-push", + "-q", + "^test-linux '64-qr/debug-mochitest-chrome-1proc-", + "caps/tests/mochitest/test_addonMayLoad.html", + ] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + # If there are more than one tasks here, it means that something went wrong + # with the path filtering. + expected = """ + "tasks": [ + "test-linux1804-64-qr/debug-mochitest-chrome-1proc-1" + ]""".lstrip() + + assert expected in output + + +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_query(run_mach, capfd): + cmd = ["try", "fuzzy", "--no-push", "-q", "'source-test-python-taskgraph-tests-py3"] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + # Should only ever mach one task exactly. + expected = """ + "tasks": [ + "source-test-python-taskgraph-tests-py3" + ]""".lstrip() + + assert expected in output + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_fuzzy.t b/tools/tryselect/test/test_fuzzy.t new file mode 100644 index 0000000000..5f53c07c97 --- /dev/null +++ b/tools/tryselect/test/test_fuzzy.t @@ -0,0 +1,201 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test fuzzy selector + + $ ./mach try fuzzy $testargs -q "'foo" + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ ./mach try fuzzy $testargs -q "'bar" + no tasks selected + $ ./mach try fuzzy $testargs --full -q "'bar" + Commit message: + Fuzzy query='bar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/bar-debug", + "test/bar-opt" + ], + "version": 1 + } + + +Test multiple selectors + + $ ./mach try fuzzy $testargs --full -q "'foo" -q "'bar" + Commit message: + Fuzzy query='foo&query='bar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/bar-debug", + "test/bar-opt", + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + +Test query intersection + + $ ./mach try fuzzy $testargs --and -q "'foo" -q "'opt" + Commit message: + Fuzzy query='foo&query='opt + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-opt" + ], + "version": 1 + } + + +Test intersection with preset containing multiple queries + + $ ./mach try fuzzy --save foo -q "'test" -q "'opt" + preset saved, run with: --preset=foo + + $ ./mach try fuzzy $testargs --preset foo -xq "'test" + Commit message: + Fuzzy query='test&query='opt&query='test + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ ./mach try $testargs --preset foo -xq "'test" + Commit message: + Fuzzy query='test&query='opt&query='test + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + +Test exact match + + $ ./mach try fuzzy $testargs --full -q "testfoo | 'testbar" + Commit message: + Fuzzy query=testfoo | 'testbar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ ./mach try fuzzy $testargs --full --exact -q "testfoo | 'testbar" + Commit message: + Fuzzy query=testfoo | 'testbar + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/bar-debug", + "test/bar-opt" + ], + "version": 1 + } + + + +Test task config + + $ ./mach try fuzzy --no-push --artifact -q "'foo" + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "disable-pgo": true, + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "use-artifact-builds": true, + "version": 1 + } + + $ ./mach try fuzzy $testargs --env FOO=1 --env BAR=baz -q "'foo" + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "BAR": "baz", + "FOO": "1", + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + diff --git a/tools/tryselect/test/test_message.t b/tools/tryselect/test/test_message.t new file mode 100644 index 0000000000..25a104833c --- /dev/null +++ b/tools/tryselect/test/test_message.t @@ -0,0 +1,63 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test custom commit messages with fuzzy selector + + $ ./mach try fuzzy $testargs -q foo --message "Foobar" + Commit message: + Foobar + + Fuzzy query=foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ ./mach try fuzzy $testargs -q foo -m "Foobar: {msg}" + Commit message: + Foobar: Fuzzy query=foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ unset EDITOR + $ ./mach try fuzzy $testargs -q foo -m > /dev/null 2>&1 + [2] + + +Test custom commit messages with syntax selector + + $ ./mach try syntax $testargs -p linux -u mochitests --message "Foobar" + Commit message: + Foobar + + try: -b do -p linux -u mochitests + + Pushed via `mach try syntax` + $ ./mach try syntax $testargs -p linux -u mochitests -m "Foobar: {msg}" + Commit message: + Foobar: try: -b do -p linux -u mochitests + + Pushed via `mach try syntax` + $ unset EDITOR + $ ./mach try syntax $testargs -p linux -u mochitests -m > /dev/null 2>&1 + [2] diff --git a/tools/tryselect/test/test_mozharness_integration.py b/tools/tryselect/test/test_mozharness_integration.py new file mode 100644 index 0000000000..984f8de25b --- /dev/null +++ b/tools/tryselect/test/test_mozharness_integration.py @@ -0,0 +1,143 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import imp +import json +import os + +import mozunit +import pytest +from tryselect.tasks import build, resolve_tests_by_suite + +MOZHARNESS_SCRIPTS = { + "android_emulator_unittest": { + "class_name": "AndroidEmulatorTest", + "configs": [ + "android/android_common.py", + ], + "xfail": [ + "cppunittest", + "crashtest-qr", + "gtest", + "geckoview-junit", + "jittest", + "jsreftest", + "reftest-qr", + ], + }, + "desktop_unittest": { + "class_name": "DesktopUnittest", + "configs": [ + "unittests/linux_unittest.py", + "unittests/mac_unittest.py", + "unittests/win_unittest.py", + ], + "xfail": [ + "cppunittest", + "gtest", + "jittest", + "jittest-chunked", + "jittest1", + "jittest2", + "jsreftest", + "mochitest-valgrind-plain", + "reftest-no-accel", + "reftest-snapshot", + "xpcshell-msix", + ], + }, +} +"""A suite being listed in a script's `xfail` list means it won't work +properly with MOZHARNESS_TEST_PATHS (the mechanism |mach try fuzzy | +uses). +""" + + +def get_mozharness_test_paths(name): + scriptdir = os.path.join(build.topsrcdir, "testing", "mozharness", "scripts") + + files = imp.find_module(name, [scriptdir]) + mod = imp.load_module("scripts.{}".format(name), *files) + + class_name = MOZHARNESS_SCRIPTS[name]["class_name"] + cls = getattr(mod, class_name) + return cls(require_config_file=False)._get_mozharness_test_paths + + +@pytest.fixture(scope="module") +def all_suites(): + from moztest.resolve import _test_flavors, _test_subsuites + + all_suites = [] + for flavor in _test_flavors: + all_suites.append({"flavor": flavor, "srcdir_relpath": "test"}) + + for flavor, subsuite in _test_subsuites: + all_suites.append( + {"flavor": flavor, "subsuite": subsuite, "srcdir_relpath": "test"} + ) + + return all_suites + + +def generate_suites_from_config(path): + configdir = os.path.join(build.topsrcdir, "testing", "mozharness", "configs") + + parent, name = os.path.split(path) + name = os.path.splitext(name)[0] + + files = imp.find_module("{}".format(name), [os.path.join(configdir, parent)]) + mod = imp.load_module("config.{}".format(name), *files) + config = mod.config + + for category in sorted(config["suite_definitions"]): + key = "all_{}_suites".format(category) + if key not in config: + yield category, + continue + + for suite in sorted(config["all_{}_suites".format(category)]): + yield category, suite + + +def generate_suites(): + for name, script in MOZHARNESS_SCRIPTS.items(): + seen = set() + + for path in script["configs"]: + for suite in generate_suites_from_config(path): + if suite in seen: + continue + seen.add(suite) + + item = (name, suite) + + if suite[-1] in script["xfail"]: + item = pytest.param(item, marks=pytest.mark.xfail) + + yield item + + +def idfn(item): + name, suite = item + return "{}/{}".format(name, suite[-1]) + + +@pytest.mark.parametrize("item", generate_suites(), ids=idfn) +def test_suites(item, patch_resolver, all_suites): + """An integration test to make sure the suites returned by + `tasks.resolve_tests_by_suite` match up with the names defined in + mozharness. + """ + patch_resolver([], all_suites) + suites = resolve_tests_by_suite(["test"]) + os.environ["MOZHARNESS_TEST_PATHS"] = json.dumps(suites) + + name, suite = item + func = get_mozharness_test_paths(name) + assert func(*suite) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_perf.py b/tools/tryselect/test/test_perf.py new file mode 100644 index 0000000000..28b8d56acb --- /dev/null +++ b/tools/tryselect/test/test_perf.py @@ -0,0 +1,1245 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import pathlib +import shutil +import tempfile +from unittest import mock + +import mozunit +import pytest +from tryselect.selectors.perf import ( + MAX_PERF_TASKS, + Apps, + InvalidCategoryException, + InvalidRegressionDetectorQuery, + PerfParser, + Platforms, + Suites, + Variants, + run, +) +from tryselect.selectors.perfselector.classification import ( + check_for_live_sites, + check_for_profile, +) + +TASKS = [ + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-animometer", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-optimizing", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-webaudio", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-speedometer", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-jetstream2", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-ares6", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-optimizing", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-sunspider", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-matrix-react-bench", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-baseline", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-twitch-animation", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-assorted-dom", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-stylebench", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-baseline", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-htmlsuite", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-unity-webgl", + "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot", +] + +# The TEST_VARIANTS, and TEST_CATEGORIES are used to force +# a particular set of categories to show up in testing. Otherwise, +# every time someone adds a category, or a variant, we'll need +# to redo all the category counts. The platforms, and apps are +# not forced because they change infrequently. +TEST_VARIANTS = { + Variants.NO_FISSION.value: { + "query": "'nofis", + "negation": "!nofis", + "platforms": [Platforms.ANDROID.value], + "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value], + }, + Variants.BYTECODE_CACHED.value: { + "query": "'bytecode", + "negation": "!bytecode", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, + Variants.LIVE_SITES.value: { + "query": "'live", + "negation": "!live", + "restriction": check_for_live_sites, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": list(PerfParser.apps.keys()), + }, + Variants.PROFILING.value: { + "query": "'profil", + "negation": "!profil", + "restriction": check_for_profile, + "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], + "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value], + }, + Variants.SWR.value: { + "query": "'swr", + "negation": "!swr", + "platforms": [Platforms.DESKTOP.value], + "apps": [Apps.FIREFOX.value], + }, +} + +TEST_CATEGORIES = { + "Pageload": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6"], + }, + "suites": [Suites.RAPTOR.value], + "tasks": [], + }, + "Pageload (essential)": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'tp6 'essential"], + }, + "variant-restrictions": {Suites.RAPTOR.value: [Variants.NO_FISSION.value]}, + "suites": [Suites.RAPTOR.value], + "tasks": [], + }, + "Responsiveness": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'responsive"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + }, + "Benchmarks": { + "query": { + Suites.RAPTOR.value: ["'browsertime 'benchmark"], + }, + "suites": [Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: []}, + "tasks": [], + }, + "DAMP (Devtools)": { + "query": { + Suites.TALOS.value: ["'talos 'damp"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + }, + "Talos PerfTests": { + "query": { + Suites.TALOS.value: ["'talos"], + }, + "suites": [Suites.TALOS.value], + "tasks": [], + }, + "Resource Usage": { + "query": { + Suites.TALOS.value: ["'talos 'xperf | 'tp5"], + Suites.RAPTOR.value: ["'power 'osx"], + Suites.AWSY.value: ["'awsy"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value], + "platform-restrictions": [Platforms.DESKTOP.value], + "variant-restrictions": { + Suites.RAPTOR.value: [], + Suites.TALOS.value: [], + }, + "app-restrictions": { + Suites.RAPTOR.value: [Apps.FIREFOX.value], + Suites.TALOS.value: [Apps.FIREFOX.value], + }, + "tasks": [], + }, + "Graphics, & Media Playback": { + "query": { + # XXX This might not be an exhaustive list for talos atm + Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"], + Suites.RAPTOR.value: ["'browsertime 'youtube-playback"], + }, + "suites": [Suites.TALOS.value, Suites.RAPTOR.value], + "variant-restrictions": {Suites.RAPTOR.value: [Variants.NO_FISSION.value]}, + "tasks": [], + }, +} + + +@pytest.mark.parametrize( + "category_options, expected_counts, unique_categories, missing", + [ + # Default should show the premade live category, but no chrome or android + # The benchmark desktop category should be visible in all configurations + # except for when there are requested apps/variants/platforms + ( + {}, + 58, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ] + }, + "Pageload macosx": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ] + }, + "Resource Usage desktop": { + "awsy": ["'awsy", "!android 'shippable !-32 !clang"], + "raptor": [ + "'power 'osx", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + "talos": [ + "'talos 'xperf | 'tp5", + "!android 'shippable !-32 !clang", + "!profil", + "!swr", + ], + }, + }, + [ + "Responsiveness android-p2 geckoview", + "Benchmarks desktop chromium", + ], + ), # Default settings + ( + {"live_sites": True}, + 66, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ] + }, + "Pageload macosx": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ] + }, + "Pageload macosx live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'live", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + }, + }, + [ + "Responsiveness android-p2 geckoview", + "Benchmarks desktop chromium", + "Benchmarks desktop firefox profiling", + "Talos desktop live-sites", + "Talos desktop profiling+swr", + "Benchmarks desktop firefox live-sites+profiling" + "Benchmarks desktop firefox live-sites", + ], + ), + ( + {"live_sites": True, "safari": True}, + 72, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!profil", + "!chrom", + "!custom-car", + ] + }, + "Pageload macosx safari": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'safari", + "!bytecode", + "!profil", + ] + }, + "Pageload macosx safari live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'safari", + "'live", + "!bytecode", + "!profil", + ], + }, + }, + [ + "Pageload linux safari", + "Pageload desktop safari", + ], + ), + ( + {"live_sites": True, "chrome": True}, + 114, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!profil", + "!safari", + "!custom-car", + ] + }, + "Pageload macosx live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'osx 'shippable", + "'live", + "!bytecode", + "!profil", + "!safari", + "!custom-car", + ], + }, + "Benchmarks desktop chromium": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "'chromium", + "!bytecode", + "!profil", + ], + }, + }, + [ + "Responsiveness android-p2 geckoview", + "Firefox Pageload linux chrome", + "Talos PerfTests desktop swr", + ], + ), + ( + {"android": True}, + 88, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + }, + "Responsiveness android-a51 geckoview": { + "raptor": [ + "'browsertime 'responsive", + "'android 'a51 'shippable 'aarch64", + "'geckoview", + "!nofis", + "!live", + "!profil", + ], + }, + }, + [ + "Responsiveness android-a51 chrome-m", + "Firefox Pageload android", + ], + ), + ( + {"android": True, "chrome": True}, + 138, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!profil", + "!safari", + "!custom-car", + ], + }, + "Responsiveness android-a51 chrome-m": { + "raptor": [ + "'browsertime 'responsive", + "'android 'a51 'shippable 'aarch64", + "'chrome-m", + "!nofis", + "!live", + "!profil", + ], + }, + }, + ["Responsiveness android-p2 chrome-m", "Resource Usage android"], + ), + ( + {"android": True, "chrome": True, "profile": True}, + 176, + { + "Benchmarks desktop": { + "raptor": [ + "'browsertime 'benchmark", + "!android 'shippable !-32 !clang", + "!bytecode", + "!live", + "!safari", + "!custom-car", + ] + }, + "Talos PerfTests desktop profiling": { + "talos": [ + "'talos", + "!android 'shippable !-32 !clang", + "'profil", + "!swr", + ] + }, + }, + [ + "Resource Usage desktop profiling", + "DAMP (Devtools) desktop chrome", + "Resource Usage android", + "Resource Usage windows chromium", + ], + ), + # Show all available windows tests, no other platform should exist + # including the desktop catgeory + ( + {"requested_platforms": ["windows"]}, + 14, + { + "Benchmarks windows firefox": { + "raptor": [ + "'browsertime 'benchmark", + "!-32 'windows 'shippable", + "!chrom !geckoview !fenix !safari !custom-car", + "!bytecode", + "!live", + "!profil", + ] + }, + }, + [ + "Resource Usage desktop", + "Benchmarks desktop", + "Benchmarks linux firefox bytecode-cached+profiling", + ], + ), + # Can't have fenix on the windows platform + ( + {"requested_platforms": ["windows"], "requested_apps": ["fenix"]}, + 0, + {}, + ["Benchmarks desktop"], + ), + # Android flag also needs to be supplied + ( + {"requested_platforms": ["android"], "requested_apps": ["fenix"]}, + 0, + {}, + ["Benchmarks desktop"], + ), + # There should be no global categories available, only fenix + ( + { + "requested_platforms": ["android"], + "requested_apps": ["fenix"], + "android": True, + }, + 10, + { + "Pageload android fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!nofis", + "!live", + "!profil", + ], + } + }, + ["Benchmarks desktop", "Pageload (live) android"], + ), + # Test with multiple apps + ( + { + "requested_platforms": ["android"], + "requested_apps": ["fenix", "geckoview"], + "android": True, + }, + 15, + { + "Benchmarks android geckoview": { + "raptor": [ + "'browsertime 'benchmark", + "'android 'a51 'shippable 'aarch64", + "'geckoview", + "!nofis", + "!live", + "!profil", + ], + }, + "Pageload android fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!nofis", + "!live", + "!profil", + ], + }, + }, + [ + "Benchmarks desktop", + "Pageload android no-fission", + "Pageload android fenix live-sites", + ], + ), + # Variants are inclusive, so we'll see the variant alongside the + # base here for fenix + ( + { + "requested_variants": ["no-fission"], + "requested_apps": ["fenix"], + "android": True, + }, + 32, + { + "Pageload android-a51 fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!live", + "!profil", + ], + }, + "Pageload android-a51 fenix no-fission": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + "Pageload (essential) android fenix no-fission": { + "raptor": [ + "'browsertime 'tp6 'essential", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + }, + [ + "Benchmarks desktop", + "Pageload (live) android", + "Pageload android-p2 fenix live-sites", + ], + ), + # With multiple variants, we'll see the base variant (with no combinations) + # for each of them + ( + { + "requested_variants": ["no-fission", "live-sites"], + "requested_apps": ["fenix"], + "android": True, + }, + 40, + { + "Pageload android-a51 fenix": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "!profil", + ], + }, + "Pageload android-a51 fenix no-fission": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + "Pageload android-a51 fenix live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'live", + "!nofis", + "!profil", + ], + }, + "Pageload (essential) android fenix no-fission": { + "raptor": [ + "'browsertime 'tp6 'essential", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "!live", + "!profil", + ], + }, + "Pageload android fenix no-fission+live-sites": { + "raptor": [ + "'browsertime 'tp6", + "'android 'a51 'shippable 'aarch64", + "'fenix", + "'nofis", + "'live", + "!profil", + ], + }, + }, + [ + "Benchmarks desktop", + "Pageload (live) android", + "Pageload android-p2 fenix live-sites", + "Pageload (essential) android fenix no-fission+live-sites", + ], + ), + # Make sure that no no-fission tasks are selected when a variant cannot + # run on a requested platform + ( + { + "requested_variants": ["no-fission"], + "requested_platforms": ["windows"], + }, + 14, + { + "Responsiveness windows firefox": { + "raptor": [ + "'browsertime 'responsive", + "!-32 'windows 'shippable", + "!chrom !geckoview !fenix !safari !custom-car", + "!bytecode", + "!live", + "!profil", + ], + }, + }, + ["Benchmarks desktop", "Responsiveness windows firefox no-fisson"], + ), + # We should only see the base and the live-site variants here for windows + ( + { + "requested_variants": ["no-fission", "live-sites"], + "requested_platforms": ["windows"], + "android": True, + }, + 16, + { + "Responsiveness windows firefox": { + "raptor": [ + "'browsertime 'responsive", + "!-32 'windows 'shippable", + "!chrom !geckoview !fenix !safari !custom-car", + "!bytecode", + "!profil", + ], + }, + "Pageload windows live-sites": { + "raptor": [ + "'browsertime 'tp6", + "!-32 'windows 'shippable", + "'live", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + }, + "Graphics, & Media Playback windows": { + "raptor": [ + "'browsertime 'youtube-playback", + "!-32 'windows 'shippable", + "!bytecode", + "!profil", + "!chrom", + "!safari", + "!custom-car", + ], + "talos": [ + "'talos 'svgr | 'bcv | 'webgl", + "!-32 'windows 'shippable", + "!profil", + "!swr", + ], + }, + }, + [ + "Benchmarks desktop", + "Responsiveness windows firefox no-fisson", + "Pageload (live) android", + "Talos desktop live-sites", + "Talos android", + "Graphics, & Media Playback windows live-sites", + "Graphics, & Media Playback android no-fission", + ], + ), + ], +) +def test_category_expansion( + category_options, expected_counts, unique_categories, missing +): + # Set the categories, and variants to expand + PerfParser.categories = TEST_CATEGORIES + PerfParser.variants = TEST_VARIANTS + + # Expand the categories, then either check if the unique_categories, + # exist or are missing from the categories + expanded_cats = PerfParser.get_categories(**category_options) + + assert len(expanded_cats) == expected_counts + assert not any([expanded_cats.get(ucat, None) is not None for ucat in missing]) + assert all( + [expanded_cats.get(ucat, None) is not None for ucat in unique_categories.keys()] + ) + + # Ensure that the queries are as expected + for cat_name, cat_query in unique_categories.items(): + # Don't use get here because these fields should always exist + assert cat_query == expanded_cats[cat_name]["queries"] + + +@pytest.mark.parametrize( + "options, call_counts, log_ind, expected_log_message", + [ + ( + {}, + [9, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"query": "'Pageload 'linux 'firefox"}, + [9, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"cached_revision": "cached_base_revision"}, + [9, 1, 1, 10, 2, 0], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=cached_base_revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"dry_run": True}, + [9, 1, 1, 10, 2, 0], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=&newProject=try&newRevision=revision\n" + ), + ), + ( + {"show_all": True}, + [1, 2, 2, 8, 2, 1], + 0, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"show_all": True, "query": "'shippable !32 speedometer 'firefox"}, + [1, 2, 2, 8, 2, 1], + 0, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ( + {"single_run": True}, + [9, 1, 1, 4, 2, 0], + 2, + ( + "If you need any help, you can find us in the #perf-help Matrix channel:\n" + "https://matrix.to/#/#perf-help:mozilla.org\n" + ), + ), + ( + {"detect_changes": True}, + [10, 2, 2, 10, 2, 1], + 2, + ( + "\n!!!NOTE!!!\n You'll be able to find a performance comparison " + "here once the tests are complete (ensure you select the right framework): " + "https://treeherder.mozilla.org/perfherder/compare?originalProject=try&original" + "Revision=revision&newProject=try&newRevision=revision\n" + ), + ), + ], +) +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_full_run(options, call_counts, log_ind, expected_log_message): + with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( + "tryselect.selectors.perf.run_fzf" + ) as fzf, mock.patch( + "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() + ), mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ) as logger, mock.patch( + "tryselect.selectors.perf.PerfParser.check_cached_revision", + ) as ccr, mock.patch( + "tryselect.selectors.perf.PerfParser.save_revision_treeherder" + ) as srt, mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print: + fzf.side_effect = [ + ["", ["Benchmarks linux"]], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", ["Perftest Change Detector"]], + ] + ccr.return_value = options.get("cached_revision", "") + + run(**options) + + assert fzf.call_count == call_counts[0] + assert ptt.call_count == call_counts[1] + assert logger.call_count == call_counts[2] + assert perf_print.call_count == call_counts[3] + assert ccr.call_count == call_counts[4] + assert srt.call_count == call_counts[5] + assert perf_print.call_args_list[log_ind][0][0] == expected_log_message + + +@pytest.mark.parametrize( + "options, call_counts, log_ind, expected_log_message, expected_failure", + [ + ( + {"detect_changes": True}, + [10, 0, 0, 2, 1], + 1, + ( + "Executing raptor queries: 'browsertime 'benchmark, !clang 'linux " + "'shippable, !bytecode, !live, !profil, !chrom, !safari, !custom-car" + ), + InvalidRegressionDetectorQuery, + ), + ], +) +@pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") +def test_change_detection_task_injection_failure( + options, + call_counts, + log_ind, + expected_log_message, + expected_failure, +): + with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( + "tryselect.selectors.perf.run_fzf" + ) as fzf, mock.patch( + "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() + ), mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ) as logger, mock.patch( + "tryselect.selectors.perf.PerfParser.check_cached_revision" + ) as ccr, mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print: + fzf.side_effect = [ + ["", ["Benchmarks linux"]], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ["", TASKS], + ] + + with pytest.raises(expected_failure): + run(**options) + + assert fzf.call_count == call_counts[0] + assert ptt.call_count == call_counts[1] + assert logger.call_count == call_counts[2] + assert perf_print.call_count == call_counts[3] + assert ccr.call_count == call_counts[4] + assert perf_print.call_args_list[log_ind][0][0] == expected_log_message + + +@pytest.mark.parametrize( + "query, should_fail", + [ + ( + { + "query": { + # Raptor has all variants available so it + # should fail on this category + "raptor": ["browsertime 'live 'no-fission"], + } + }, + True, + ), + ( + { + "query": { + # Awsy has no variants defined so it shouldn't fail + # on a query like this + "awsy": ["browsertime 'live 'no-fission"], + } + }, + False, + ), + ], +) +def test_category_rules(query, should_fail): + # Set the categories, and variants to expand + PerfParser.categories = {"test-live": query} + PerfParser.variants = TEST_VARIANTS + + if should_fail: + with pytest.raises(InvalidCategoryException): + PerfParser.run_category_checks() + else: + assert PerfParser.run_category_checks() + + # Reset the categories, and variants to expand + PerfParser.categories = TEST_CATEGORIES + PerfParser.variants = TEST_VARIANTS + + +@pytest.mark.parametrize( + "apk_name, apk_content, should_fail, failure_message", + [ + ( + "real-file", + "file-content", + False, + None, + ), + ("bad-file", None, True, "Path does not exist:"), + ], +) +def test_apk_upload(apk_name, apk_content, should_fail, failure_message): + with mock.patch("tryselect.selectors.perf.subprocess") as _, mock.patch( + "tryselect.selectors.perf.shutil" + ) as _: + temp_dir = None + try: + temp_dir = tempfile.mkdtemp() + sample_apk = pathlib.Path(temp_dir, apk_name) + if apk_content is not None: + with sample_apk.open("w") as f: + f.write(apk_content) + + if should_fail: + with pytest.raises(Exception) as exc_info: + PerfParser.setup_apk_upload("browsertime", str(sample_apk)) + assert failure_message in str(exc_info) + else: + PerfParser.setup_apk_upload("browsertime", str(sample_apk)) + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir) + + +@pytest.mark.parametrize( + "args, load_data, return_value, call_counts, exists_cache_file", + [ + ( + ( + [], + "base_commit", + ), + { + "base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": [], + }, + ], + }, + "2b04563b5", + [1, 0], + True, + ), + ( + ( + ["task-a"], + "subset_base_commit", + ), + { + "subset_base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": ["task-a", "task-b"], + }, + ], + }, + "2b04563b5", + [1, 0], + True, + ), + ( + ([], "not_exist_cached_base_commit"), + { + "base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": [], + }, + ], + }, + None, + [1, 0], + True, + ), + ( + ( + ["task-a", "task-b"], + "superset_base_commit", + ), + { + "superset_base_commit": [ + { + "base_revision_treeherder": "2b04563b5", + "date": "2023-03-31", + "tasks": ["task-a"], + }, + ], + }, + None, + [1, 0], + True, + ), + ( + ([], None), + {}, + None, + [1, 1], + True, + ), + ( + ([], None), + {}, + None, + [0, 0], + False, + ), + ], +) +def test_check_cached_revision( + args, load_data, return_value, call_counts, exists_cache_file +): + with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( + "tryselect.selectors.perf.json.dump" + ) as dump, mock.patch( + "tryselect.selectors.perf.pathlib.Path.is_file" + ) as is_file, mock.patch( + "tryselect.selectors.perf.pathlib.Path.open" + ): + load.return_value = load_data + is_file.return_value = exists_cache_file + result = PerfParser.check_cached_revision(*args) + + assert load.call_count == call_counts[0] + assert dump.call_count == call_counts[1] + assert result == return_value + + +@pytest.mark.parametrize( + "args, call_counts, exists_cache_file", + [ + ( + ["base_commit", "base_revision_treeherder"], + [0, 1], + False, + ), + ( + ["base_commit", "base_revision_treeherder"], + [1, 1], + True, + ), + ], +) +def test_save_revision_treeherder(args, call_counts, exists_cache_file): + with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( + "tryselect.selectors.perf.json.dump" + ) as dump, mock.patch( + "tryselect.selectors.perf.pathlib.Path.is_file" + ) as is_file, mock.patch( + "tryselect.selectors.perf.pathlib.Path.open" + ): + is_file.return_value = exists_cache_file + PerfParser.save_revision_treeherder(TASKS, args[0], args[1]) + + assert load.call_count == call_counts[0] + assert dump.call_count == call_counts[1] + + +@pytest.mark.parametrize( + "total_tasks, options, call_counts, expected_log_message, expected_failure", + [ + ( + MAX_PERF_TASKS + 1, + {}, + [1, 0, 0, 1], + ( + "That's a lot of tests selected (300)!\n" + "These tests won't be triggered. If this was unexpected, " + "please file a bug in Testing :: Performance." + ), + True, + ), + ( + MAX_PERF_TASKS, + {"show_all": True}, + [9, 0, 0, 8], + ( + "For more information on the performance tests, see our " + "PerfDocs here:\nhttps://firefox-source-docs.mozilla.org/testing/perfdocs/" + ), + False, + ), + ( + int((MAX_PERF_TASKS + 2) / 2), + {"show_all": True, "try_config": {"rebuild": 2}}, + [1, 0, 0, 1], + ( + "That's a lot of tests selected (300)!\n" + "These tests won't be triggered. If this was unexpected, " + "please file a bug in Testing :: Performance." + ), + True, + ), + (0, {}, [1, 0, 0, 1], ("No tasks selected"), True), + ], +) +def test_max_perf_tasks( + total_tasks, + options, + call_counts, + expected_log_message, + expected_failure, +): + # Set the categories, and variants to expand + PerfParser.categories = TEST_CATEGORIES + PerfParser.variants = TEST_VARIANTS + + with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( + "tryselect.selectors.perf.print", + ) as perf_print, mock.patch( + "tryselect.selectors.perf.LogProcessor.revision", + new_callable=mock.PropertyMock, + return_value="revision", + ), mock.patch( + "tryselect.selectors.perf.PerfParser.perf_push_to_try", + new_callable=mock.MagicMock, + return_value=("revision1", "revision2"), + ) as perf_push_to_try_mock, mock.patch( + "tryselect.selectors.perf.PerfParser.get_perf_tasks" + ) as get_perf_tasks_mock, mock.patch( + "tryselect.selectors.perf.PerfParser.get_tasks" + ) as get_tasks_mock, mock.patch( + "tryselect.selectors.perf.run_fzf" + ) as fzf, mock.patch( + "tryselect.selectors.perf.fzf_bootstrap", return_value=mock.MagicMock() + ): + tasks = ["a-task"] * total_tasks + get_tasks_mock.return_value = tasks + get_perf_tasks_mock.return_value = tasks, [], [] + + run(**options) + + assert perf_push_to_try_mock.call_count == 0 if expected_failure else 1 + assert ptt.call_count == call_counts[1] + assert perf_print.call_count == call_counts[3] + assert fzf.call_count == 0 + assert perf_print.call_args_list[-1][0][0] == expected_log_message + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_perfcomparators.py b/tools/tryselect/test/test_perfcomparators.py new file mode 100644 index 0000000000..51f0bdb287 --- /dev/null +++ b/tools/tryselect/test/test_perfcomparators.py @@ -0,0 +1,150 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import tempfile +from unittest import mock + +import mozunit +import pytest +from tryselect.selectors.perfselector.perfcomparators import ( + BadComparatorArgs, + BenchmarkComparator, + ComparatorNotFound, + get_comparator, +) + + +@pytest.mark.parametrize( + "test_link", + [ + "https://github.com/mozilla-mobile/firefox-android/pull/1627", + "https://github.com/mozilla-mobile/firefox-android/pull/1876/" + "commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365", + ], +) +def test_benchmark_comparator(test_link): + def _verify_extra_args(extra_args): + assert len(extra_args) == 3 + if "commit" in test_link: + assert ( + "benchmark-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365" + in extra_args + ) + else: + assert "benchmark-revision=sha-for-link" in extra_args + assert "benchmark-repository=url-for-link" in extra_args + assert "benchmark-branch=ref-for-link" in extra_args + + comparator = BenchmarkComparator( + None, None, None, [f"base-link={test_link}", f"new-link={test_link}"] + ) + + with mock.patch("requests.get") as mocked_get: + magic_get = mock.MagicMock() + magic_get.json.return_value = { + "head": { + "repo": { + "html_url": "url-for-link", + }, + "sha": "sha-for-link", + "ref": "ref-for-link", + } + } + magic_get.status_code = 200 + mocked_get.return_value = magic_get + + extra_args = [] + comparator.setup_base_revision(extra_args) + _verify_extra_args(extra_args) + + extra_args = [] + comparator.setup_new_revision(extra_args) + _verify_extra_args(extra_args) + + +def test_benchmark_comparator_no_pr_links(): + def _verify_extra_args(extra_args): + assert len(extra_args) == 3 + assert "benchmark-revision=rev" in extra_args + assert "benchmark-repository=link" in extra_args + assert "benchmark-branch=fake" in extra_args + + comparator = BenchmarkComparator( + None, + None, + None, + [ + "base-repo=link", + "base-branch=fake", + "base-revision=rev", + "new-repo=link", + "new-branch=fake", + "new-revision=rev", + ], + ) + + with mock.patch("requests.get") as mocked_get: + magic_get = mock.MagicMock() + magic_get.json.return_value = { + "head": { + "repo": { + "html_url": "url-for-link", + }, + "sha": "sha-for-link", + "ref": "ref-for-link", + } + } + magic_get.status_code = 200 + mocked_get.return_value = magic_get + + extra_args = [] + comparator.setup_base_revision(extra_args) + _verify_extra_args(extra_args) + + extra_args = [] + comparator.setup_new_revision(extra_args) + _verify_extra_args(extra_args) + + +def test_benchmark_comparator_bad_args(): + comparator = BenchmarkComparator( + None, + None, + None, + [ + "base-bad-args=val", + ], + ) + + with pytest.raises(BadComparatorArgs): + comparator.setup_base_revision([]) + + +def test_get_comparator_bad_name(): + with pytest.raises(ComparatorNotFound): + get_comparator("BadName") + + +def test_get_comparator_bad_script(): + with pytest.raises(ComparatorNotFound): + with tempfile.NamedTemporaryFile() as tmpf: + tmpf.close() + get_comparator(tmpf.name) + + +def test_get_comparator_benchmark_name(): + comparator_klass = get_comparator("BenchmarkComparator") + assert comparator_klass.__name__ == "BenchmarkComparator" + + +def test_get_comparator_benchmark_script(): + # If the get_comparator method is working for scripts, then + # it should find the first defined class in this file, or the + # first imported class that matches it + comparator_klass = get_comparator(__file__) + assert comparator_klass.__name__ == "BenchmarkComparator" + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_preset.t b/tools/tryselect/test/test_preset.t new file mode 100644 index 0000000000..d67a5d6341 --- /dev/null +++ b/tools/tryselect/test/test_preset.t @@ -0,0 +1,360 @@ + $ . $TESTDIR/setup.sh + $ cd $topsrcdir + +Test preset with no subcommand + + $ ./mach try $testargs --save foo -b do -p linux -u mochitests -t none --tag foo + preset saved, run with: --preset=foo + + $ ./mach try $testargs --preset foo + Commit message: + try: -b do -p linux -u mochitests -t none --tag foo + + Pushed via `mach try syntax` + + $ ./mach try syntax $testargs --preset foo + Commit message: + try: -b do -p linux -u mochitests -t none --tag foo + + Pushed via `mach try syntax` + + $ ./mach try $testargs --list-presets + Presets from */mozbuild/try_presets.yml: (glob) + + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + + $ unset EDITOR + $ ./mach try $testargs --edit-presets + error: must set the $EDITOR environment variable to use --edit-presets + $ export EDITOR=cat + $ ./mach try $testargs --edit-presets + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + +Test preset with syntax subcommand + + $ ./mach try syntax $testargs --save bar -b do -p win32 -u none -t all --tag bar + preset saved, run with: --preset=bar + + $ ./mach try syntax $testargs --preset bar + Commit message: + try: -b do -p win32 -u none -t all --tag bar + + Pushed via `mach try syntax` + + $ ./mach try $testargs --preset bar + Commit message: + try: -b do -p win32 -u none -t all --tag bar + + Pushed via `mach try syntax` + + $ ./mach try syntax $testargs --list-presets + Presets from */mozbuild/try_presets.yml: (glob) + + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + + $ ./mach try syntax $testargs --edit-presets + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + +Test preset with fuzzy subcommand + + $ ./mach try fuzzy $testargs --save baz -q "'foo" --rebuild 5 + preset saved, run with: --preset=baz + + $ ./mach try fuzzy $testargs --preset baz + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + + $ ./mach try $testargs --preset baz + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + +Queries can be appended to presets + + $ ./mach try fuzzy $testargs --preset baz -q "'build" + Commit message: + Fuzzy query='foo&query='build + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "build-baz", + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + + $ ./mach try $testargs --preset baz -xq "'opt" + Commit message: + Fuzzy query='foo&query='opt + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "rebuild": 5, + "tasks": [ + "test/foo-opt" + ], + "version": 1 + } + + + $ ./mach try fuzzy $testargs --list-presets + Presets from */mozbuild/try_presets.yml: (glob) + + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + baz: + dry_run: true + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + + $ ./mach try fuzzy $testargs --edit-presets + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + baz: + dry_run: true + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + +Test gecko-profile argument handling. Add in profiling to a preset. + + $ ./mach try fuzzy $testargs --preset baz --gecko-profile-features=nostacksampling,cpu + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "gecko-profile": true, + "gecko-profile-features": "nostacksampling,cpu", + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + +Check whether the gecko-profile flags can be used from a preset, and check +dashes vs underscores (presets save with underscores to match ArgumentParser +settings; everything else uses dashes.) + + $ ./mach try fuzzy $testargs --save profile -q "'foo" --rebuild 5 --gecko-profile-features=nostacksampling,cpu + preset saved, run with: --preset=profile + + $ ./mach try fuzzy $testargs --preset profile + Commit message: + Fuzzy query='foo + + Pushed via `mach try fuzzy` + Calculated try_task_config.json: + { + "env": { + "TRY_SELECTOR": "fuzzy" + }, + "gecko-profile": true, + "gecko-profile-features": "nostacksampling,cpu", + "rebuild": 5, + "tasks": [ + "test/foo-debug", + "test/foo-opt" + ], + "version": 1 + } + + $ EDITOR=cat ./mach try fuzzy $testargs --edit-preset profile + bar: + dry_run: true + no_artifact: true + platforms: + - win32 + selector: syntax + tags: + - bar + talos: + - all + tests: + - none + baz: + dry_run: true + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + foo: + no_artifact: true + platforms: + - linux + selector: syntax + tags: + - foo + talos: + - none + tests: + - mochitests + profile: + dry_run: true + gecko_profile_features: nostacksampling,cpu + no_artifact: true + query: + - "'foo" + rebuild: 5 + selector: fuzzy + + $ rm $MOZBUILD_STATE_PATH/try_presets.yml diff --git a/tools/tryselect/test/test_presets.py b/tools/tryselect/test/test_presets.py new file mode 100644 index 0000000000..89cc810808 --- /dev/null +++ b/tools/tryselect/test/test_presets.py @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import pytest + +TASKS = [ + { + "kind": "build", + "label": "build-windows", + "attributes": { + "build_platform": "windows", + }, + }, + { + "kind": "test", + "label": "test-windows-mochitest-e10s", + "attributes": { + "unittest_suite": "mochitest", + "unittest_flavor": "browser-chrome", + "mochitest_try_name": "mochitest", + }, + }, +] + + +@pytest.fixture(autouse=True) +def skip_taskgraph_generation(monkeypatch, tg): + def fake_generate_tasks(*args, **kwargs): + return tg + + from tryselect import tasks + + monkeypatch.setattr(tasks, "generate_tasks", fake_generate_tasks) + + +@pytest.mark.xfail( + strict=False, reason="Bug 1635204: " "test_shared_presets[sample-suites] is flaky" +) +def test_shared_presets(run_mach, shared_name, shared_preset): + """This test makes sure that we don't break any of the in-tree presets when + renaming/removing variables in any of the selectors. + """ + assert "description" in shared_preset + assert "selector" in shared_preset + + selector = shared_preset["selector"] + if selector == "fuzzy": + assert "query" in shared_preset + assert isinstance(shared_preset["query"], list) + + # Run the preset and assert there were no exceptions. + assert run_mach(["try", "--no-push", "--preset", shared_name]) == 0 + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_release.py b/tools/tryselect/test/test_release.py new file mode 100644 index 0000000000..a1a0d348b2 --- /dev/null +++ b/tools/tryselect/test/test_release.py @@ -0,0 +1,43 @@ +# Any copyright is dedicated to the Public Domain. +# https://creativecommons.org/publicdomain/zero/1.0/ + +from textwrap import dedent + +import mozunit + + +def test_release(run_mach, capfd): + cmd = [ + "try", + "release", + "--no-push", + "--version=97.0", + ] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + expected = dedent( + """ + Commit message: + staging release: 97.0 + + Pushed via `mach try release` + Calculated try_task_config.json: + { + "parameters": { + "optimize_target_tasks": true, + "release_type": "release", + "target_tasks_method": "staging_release_builds" + }, + "version": 2 + } + + """ + ).lstrip() + assert expected in output + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_scriptworker.py b/tools/tryselect/test/test_scriptworker.py new file mode 100644 index 0000000000..e25279ace4 --- /dev/null +++ b/tools/tryselect/test/test_scriptworker.py @@ -0,0 +1,39 @@ +# Any copyright is dedicated to the Public Domain. +# https://creativecommons.org/publicdomain/zero/1.0/ + +import re +from textwrap import dedent + +import mozunit + + +def test_release(run_mach, capfd): + cmd = [ + "try", + "scriptworker", + "--no-push", + "tree", + ] + assert run_mach(cmd) == 0 + + output = capfd.readouterr().out + print(output) + + expected = re.compile( + dedent( + r""" + Pushed via `mach try scriptworker` + Calculated try_task_config.json: + { + "parameters": { + "app_version": "\d+\.\d+", + "build_number": \d+, + """ + ).lstrip(), + re.MULTILINE, + ) + assert expected.search(output) + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_task_configs.py b/tools/tryselect/test/test_task_configs.py new file mode 100644 index 0000000000..4865a1bfca --- /dev/null +++ b/tools/tryselect/test/test_task_configs.py @@ -0,0 +1,144 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import inspect +import subprocess +from argparse import ArgumentParser +from textwrap import dedent + +import mozunit +import pytest +from tryselect.task_config import Pernosco, all_task_configs + +# task configs have a list of tests of the form (input, expected) +TASK_CONFIG_TESTS = { + "artifact": [ + (["--no-artifact"], None), + (["--artifact"], {"use-artifact-builds": True, "disable-pgo": True}), + ], + "chemspill-prio": [ + ([], None), + (["--chemspill-prio"], {"chemspill-prio": {}}), + ], + "env": [ + ([], None), + (["--env", "foo=bar", "--env", "num=10"], {"env": {"foo": "bar", "num": "10"}}), + ], + "path": [ + ([], None), + ( + ["dom/indexedDB"], + {"env": {"MOZHARNESS_TEST_PATHS": '{"xpcshell": ["dom/indexedDB"]}'}}, + ), + ( + ["dom/indexedDB", "testing"], + { + "env": { + "MOZHARNESS_TEST_PATHS": '{"xpcshell": ["dom/indexedDB", "testing"]}' + } + }, + ), + (["invalid/path"], SystemExit), + ], + "pernosco": [ + ([], None), + ], + "rebuild": [ + ([], None), + (["--rebuild", "10"], {"rebuild": 10}), + (["--rebuild", "1"], SystemExit), + (["--rebuild", "21"], SystemExit), + ], + "worker-overrides": [ + ([], None), + ( + ["--worker-override", "alias=worker/pool"], + {"worker-overrides": {"alias": "worker/pool"}}, + ), + ( + [ + "--worker-override", + "alias=worker/pool", + "--worker-override", + "alias=other/pool", + ], + SystemExit, + ), + ( + ["--worker-suffix", "b-linux=-dev"], + {"worker-overrides": {"b-linux": "gecko-1/b-linux-dev"}}, + ), + ( + [ + "--worker-override", + "b-linux=worker/pool" "--worker-suffix", + "b-linux=-dev", + ], + SystemExit, + ), + ], +} + + +@pytest.fixture +def config_patch_resolver(patch_resolver): + def inner(paths): + patch_resolver( + [], [{"flavor": "xpcshell", "srcdir_relpath": path} for path in paths] + ) + + return inner + + +def test_task_configs(config_patch_resolver, task_config, args, expected): + parser = ArgumentParser() + + cfg = all_task_configs[task_config]() + cfg.add_arguments(parser) + + if inspect.isclass(expected) and issubclass(expected, BaseException): + with pytest.raises(expected): + args = parser.parse_args(args) + if task_config == "path": + config_patch_resolver(**vars(args)) + + cfg.try_config(**vars(args)) + else: + args = parser.parse_args(args) + if task_config == "path": + config_patch_resolver(**vars(args)) + assert cfg.try_config(**vars(args)) == expected + + +@pytest.fixture +def patch_pernosco_email_check(monkeypatch): + def inner(val): + def fake_check_output(*args, **kwargs): + return val + + monkeypatch.setattr(subprocess, "check_output", fake_check_output) + + return inner + + +def test_pernosco(patch_pernosco_email_check): + patch_pernosco_email_check( + dedent( + """ + user foobar@mozilla.com + hostname hg.mozilla.com + """ + ) + ) + + parser = ArgumentParser() + + cfg = Pernosco() + cfg.add_arguments(parser) + args = parser.parse_args(["--pernosco"]) + assert cfg.try_config(**vars(args)) == {"env": {"PERNOSCO": "1"}} + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/test/test_tasks.py b/tools/tryselect/test/test_tasks.py new file mode 100644 index 0000000000..cdf8cf84c1 --- /dev/null +++ b/tools/tryselect/test/test_tasks.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +import mozunit +import pytest +from tryselect.tasks import cache_key, filter_tasks_by_paths, resolve_tests_by_suite + + +def test_filter_tasks_by_paths(patch_resolver): + tasks = ["foobar/xpcshell-1", "foobar/mochitest", "foobar/xpcshell"] + + patch_resolver(["xpcshell"], {}) + assert list(filter_tasks_by_paths(tasks, "dummy")) == [] + + patch_resolver([], [{"flavor": "xpcshell"}]) + assert list(filter_tasks_by_paths(tasks, "dummy")) == [ + "foobar/xpcshell-1", + "foobar/xpcshell", + ] + + +@pytest.mark.parametrize( + "input, tests, expected", + ( + pytest.param( + ["xpcshell.js"], + [{"flavor": "xpcshell", "srcdir_relpath": "xpcshell.js"}], + {"xpcshell": ["xpcshell.js"]}, + id="single test", + ), + pytest.param( + ["xpcshell.ini"], + [ + { + "flavor": "xpcshell", + "srcdir_relpath": "xpcshell.js", + "manifest_relpath": "xpcshell.ini", + }, + ], + {"xpcshell": ["xpcshell.ini"]}, + id="single manifest", + ), + pytest.param( + ["xpcshell.js", "mochitest.js"], + [ + {"flavor": "xpcshell", "srcdir_relpath": "xpcshell.js"}, + {"flavor": "mochitest", "srcdir_relpath": "mochitest.js"}, + ], + { + "xpcshell": ["xpcshell.js"], + "mochitest-plain": ["mochitest.js"], + }, + id="two tests", + ), + pytest.param( + ["test/xpcshell.ini"], + [ + { + "flavor": "xpcshell", + "srcdir_relpath": "test/xpcshell.js", + "manifest_relpath": os.path.join("test", "xpcshell.ini"), + }, + ], + {"xpcshell": ["test/xpcshell.ini"]}, + id="mismatched path separators", + ), + ), +) +def test_resolve_tests_by_suite(patch_resolver, input, tests, expected): + patch_resolver([], tests) + assert resolve_tests_by_suite(input) == expected + + +@pytest.mark.parametrize( + "attr,params,disable_target_task_filter,expected", + ( + ("target_task_set", None, False, "target_task_set"), + ("target_task_set", {"project": "autoland"}, False, "target_task_set"), + ("target_task_set", {"project": "mozilla-central"}, False, "target_task_set"), + ("target_task_set", None, True, "target_task_set-uncommon"), + ("full_task_set", {"project": "pine"}, False, "full_task_set-pine"), + ("full_task_set", None, True, "full_task_set"), + ), +) +def test_cache_key(attr, params, disable_target_task_filter, expected): + assert cache_key(attr, params, disable_target_task_filter) == expected + + +if __name__ == "__main__": + mozunit.main() diff --git a/tools/tryselect/try_presets.yml b/tools/tryselect/try_presets.yml new file mode 100644 index 0000000000..4b689ec0d5 --- /dev/null +++ b/tools/tryselect/try_presets.yml @@ -0,0 +1,266 @@ +--- +# Presets defined here will be available to all users. Run them with: +# $ mach try --preset +# +# If editing this file, make sure to run: +# $ mach python-test tools/tryselect/test/test_presets.py +# +# Descriptions are required. Please keep this in alphabetical order. + +# yamllint disable rule:line-length + +builds: + selector: fuzzy + description: >- + Run builds without any of the extras. + query: + - "^build- !fuzzing !notarization !reproduced !rusttests !signing !upload-symbols" + +builds-debug: + selector: fuzzy + description: >- + Run the bare minimum of debug build jobs to ensure builds work on + all tier-1 platforms. + query: + - "^build- 'debug !fuzzing !rusttests !signing !plain !asan !tsan !noopt !toolchain !upload-symbols" + +builds-debugopt: + selector: fuzzy + description: >- + Run the bare minimum of debug and opt build jobs to ensure builds work on + all tier-1 platforms. + query: + - "^build- !fuzzing !rusttests !signing !plain !asan !tsan !noopt !toolchain !upload-symbols" + +desktop-frontend: + description: >- + Run mochitest-browser, xpcshell, mochitest-chrome, mochitest-a11y, + marionette, firefox-ui-functional on all desktop platforms. + Excludes non-shipped/duplicate configurations like asan/tsan/msix + to reduce the runtime of the push as well as infra load. + Use with --artifact to speed up your trypush. + If this is green, you can be 99% sure that any frontend change will + stick on central. + selector: fuzzy + query: + # Runs 64-bit frontend-tests, plus win7. Tries to avoid running + # asan/tsan because they're not available as artifact builds, and + # rarely offer different results from debug/opt. It also avoids running + # msix/swr/a11y-checks/gpu/nofis/headless variants of otherwise + # identical tests, as again those are unlikely to show different + # results for frontend-only changes. + # This won't run 32-bit debug tests, which seems an acceptable + # trade-off for query complexity + runtime on infrastructure. + - "'browser-chrome 'windows7 | '64 !spi !asan !tsan !msix !a11y !swr | 'linux" + - "'mochitest-chrome 'windows7 | '64 !spi !asan !tsan !swr !gpu" + - "'xpcshell 'windows7 | '64 !spi !asan !tsan !msix !nofis !condprof" + - "'browser-a11y | 'mochitest-a11y 'windows7 | '64 !spi !asan !tsan !no-cache !swr" + - "'marionette 'windows7 | '64 !asan !source !headless !swr" + - "'firefox-ui-functional 'windows7 | '64 !asan !tsan" + +devtools: + selector: fuzzy + description: >- + Runs the tests relevant to the Firefox Devtools + query: + - "'node-debugger | 'node-devtools" + - "'mozlint-eslint" + # Windows: skip jobs on asan and 32 bits platforms + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'windows !asan !-32" + # macos: no extra platform to filter out + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'macosx" + # Linux is being named "linux1804" and may change over time, so use a more flexible search + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'linux '64-qr/ !swr" + - "'xpcshell 'linux !nofis '64-qr/" + +devtools-linux: + selector: fuzzy + description: >- + Runs the tests relevant to the Firefox Devtools, on Linux only. + query: + - "'node-debugger | 'node-devtools" + - "'mozlint-eslint" + - "'mochitest-devtools-chrome | 'mochitest-chrome-1proc 'linux '64-qr/ !swr" + - "'xpcshell 'linux !nofis '64-qr/" + +fpush-linux-android: + selector: fuzzy + description: >- + Runs correctness test suites on Linux and Android emulator platforms, as + well as builds across main platforms. The resulting jobs on TreeHerder + used to end up looking like a "F" shape (not so much these days) and so + this is typically referred to as an F-push. This is useful to do as a + general sanity check on changes to cross-platform Gecko code where you + unsure of what tests might be affected. Linux and Android (emulator) + test coverage are relatively cheap to run and cover a lot of the + codebase, while the builds on other platforms catch compilation problems + that might result from only building locally on one platform. + query: + - "'test-linux1804 'debug- !-shippable !-asan" + - "'test-android-em 'debug" + - "^build !-shippable !-signing !-asan !-fuzzing !-rusttests !-base-toolchain !-aar-" + +geckodriver: + selector: fuzzy + description: >- + Runs the tests relevant to geckodriver, which implements the WebDriver + specification. This preset can be filtered down further to limit it to + a specific platform or other tasks only. For example: + |mach try --preset geckodriver -xq "'linux"| + query: + - "'rusttests" + - "'platform 'wdspec 'debug 'nofis" + - "'browsertime 'amazon 'shippable 'firefox 'nofis" + +media-full: + selector: fuzzy + description: >- + Runs tests that exercise media playback and WebRTC code. + query: + - "mochitest-media !dfpi !nofis" + - "mochitest-media android !spi !swr !lite" + - "mochitest-browser-chrome !dfpi !nofis !a11y" + - "mochitest-browser-media" + - "web-platform-tests !dfpi !nofis !shippable" + - "web-platform-tests android !wdspec !spi !swr !lite" + - "crashtest !wdspec !nofis" + - "crashtest android !wdspec !spi !swr !lite" + - "'gtest" + +mochitest-bc: + description: >- + Runs mochitest-browser-chrome on all Desktop platforms in both opt + and debug. Excludes jobs that require non-artifact builds (asan, + tsan, msix, etc.) and some non-default configurations. For frontend + only changes, use this with --artifact to speed up your trypushes. + query: + - "'browser-chrome 'windows7 | '64 !tsan !asan !msix !spi !a11y !swr | 'linux" + selector: fuzzy + +perf: + selector: fuzzy + description: >- + Runs all performance (raptor and talos) tasks across all platforms. + This preset can be filtered down further (e.g to limit it to a specific + platform) via |mach try --preset perf -xq "'windows"|. + + Android hardware platforms are excluded due to resource limitations. + query: + - "^test- !android-hw 'raptor | 'talos" + rebuild: 5 + +perf-chrome: + description: >- + Runs the talos tests most likely to change when making a change to + the browser chrome. This skips a number of talos jobs that are unlikely + to be affected in order to conserve resources. + query: + - "opt-talos- 'chrome | 'svg | 'session | 'tabswitch | 'other | 'g5" + rebuild: 6 + selector: fuzzy + +remote-protocol: + selector: fuzzy + description: >- + Runs the tests relevant to the Remote protocol, which underpins + many test harnesses as well as our CDP and WebDriver implementations. + This preset can be filtered down further to limit it to a specific + platform or to opt/debug tasks only. For example: + |mach try --preset remote-protocol -xq "'linux 'opt"| + query: + - "'awsy-base" + - "'firefox-ui" + - "'marionette !swr | harness" + - "'mochitest-browser !spi !swr !nofis '-1$" + - "'mochitest-remote !spi !swr" + - "'platform 'reftest !swr !nofis | 'android !-lite -1$" + - "'platform 'wdspec !swr" + - "'platform !reftest !wdspec !swr !nofis | 'android !-lite -1$" + - "'puppeteer" + - "'reftest !platform !gpu !swr !no-accel !nofis | 'android !-lite -1$" + - "'xpcshell !spi !tsan !-lite" + +sample-suites: + selector: fuzzy + description: >- + Runs one chunk of every test suite plus all suites that aren't chunked. + It is useful for testing infrastructure changes that can affect the + harnesses themselves but are unlikely to break specific tests. + query: + - ^test- -1$ + # Only run a single talos + raptor suite per platform + - ^test- !1$ !2$ !3$ !4$ !5$ !6$ !7$ !8$ !9$ !0$ !raptor !talos + - ^test- 'raptor-speedometer | 'talos-g1 + +sm-shell-all: + selector: fuzzy + description: <- + Runs a set of tests aimed to give a reasonable level of confidence for + basic SpiderMonkey changes (shell only), all platforms + query: + - "'spidermonkey | 'shell-haz" + - "!shippable !android 'jittest" # macosx64 jittests + +sm-shell: + selector: fuzzy + description: <- + Runs a set of tests aimed to give a reasonable level of confidence for + basic SpiderMonkey changes (shell only) (linux only) + query: + - "!win !osx 'spidermonkey | 'shell-haz" + + +sm-all: + selector: fuzzy + description: <- + Runs a set of tests aimed to give a reasonable level of confidence for + basic SpiderMonkey changes, including those that would require a + browser build. + query: + - "'spidermonkey | 'hazard" + - "!android !asan !shippable 'xpcshell" + - "!android !asan !shippable 'jsreftest" + - "!shippable !android 'jittest" # macosx64 jittests + +webgpu: + selector: fuzzy + description: >- + Runs the tests relevant to WebGPU. + query: + - "'webgpu" + - "source-test-mozlint-updatebot" + - "source-test-vendor-rust" + +webrender: + selector: fuzzy + description: >- + Runs the conformance tests relevant to WebRender. + query: + - "!talos !raptor !shippable !asan '-qr" + - "^webrender-" + +webrender-reftests: + selector: fuzzy + description: >- + Runs the reftests relevant to WebRender. + query: + - "!talos !raptor !shippable !asan !nofis 'reftest" + +webrender-reftests-linux: + selector: fuzzy + description: >- + Runs the reftests relevant to WebRender on linux only. + query: + - "!talos !raptor !shippable !asan !nofis 'linux 'reftest" + +webrender-perf: + selector: fuzzy + description: >- + Runs the performance tests relevant to WebRender. + query: + - "'-qr 'svgr" + - "'-qr 'g1" + - "'-qr 'g4" + - "'-qr 'tp5" + - "'-qr 'talos-webgl" + - "'-qr 'motionmark-animometer" diff --git a/tools/tryselect/util/__init__.py b/tools/tryselect/util/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/tryselect/util/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/tryselect/util/dicttools.py b/tools/tryselect/util/dicttools.py new file mode 100644 index 0000000000..465e4a43de --- /dev/null +++ b/tools/tryselect/util/dicttools.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import copy + + +def merge_to(source, dest): + """ + Merge dict and arrays (override scalar values) + + Keys from source override keys from dest, and elements from lists in source + are appended to lists in dest. + + :param dict source: to copy from + :param dict dest: to copy to (modified in place) + """ + + for key, value in source.items(): + # Override mismatching or empty types + if type(value) != type(dest.get(key)): # noqa + dest[key] = source[key] + continue + + # Merge dict + if isinstance(value, dict): + merge_to(value, dest[key]) + continue + + if isinstance(value, list): + dest[key] = dest[key] + source[key] + continue + + dest[key] = source[key] + + return dest + + +def merge(*objects): + """ + Merge the given objects, using the semantics described for merge_to, with + objects later in the list taking precedence. From an inheritance + perspective, "parents" should be listed before "children". + + Returns the result without modifying any arguments. + """ + if len(objects) == 1: + return copy.deepcopy(objects[0]) + return merge_to(objects[-1], merge(*objects[:-1])) diff --git a/tools/tryselect/util/estimates.py b/tools/tryselect/util/estimates.py new file mode 100644 index 0000000000..634b1de2a2 --- /dev/null +++ b/tools/tryselect/util/estimates.py @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +from datetime import datetime, timedelta + +TASK_DURATION_CACHE = "task_duration_history.json" +GRAPH_QUANTILE_CACHE = "graph_quantile_cache.csv" +TASK_DURATION_TAG_FILE = "task_duration_tag.json" + + +def find_all_dependencies(graph, tasklist): + all_dependencies = dict() + + def find_dependencies(task): + dependencies = set() + if task in all_dependencies: + return all_dependencies[task] + if task not in graph: + # Don't add tasks (and so durations) for + # things optimised out. + return dependencies + dependencies.add(task) + for dep in graph.get(task, list()): + all_dependencies[dep] = find_dependencies(dep) + dependencies.update(all_dependencies[dep]) + return dependencies + + full_deps = set() + for task in tasklist: + full_deps.update(find_dependencies(task)) + + # Since these have been asked for, they're not inherited dependencies. + return sorted(full_deps - set(tasklist)) + + +def find_longest_path(graph, tasklist, duration_data): + + dep_durations = dict() + + def find_dependency_durations(task): + if task in dep_durations: + return dep_durations[task] + + durations = [find_dependency_durations(dep) for dep in graph.get(task, list())] + durations.append(0.0) + md = max(durations) + duration_data.get(task, 0.0) + dep_durations[task] = md + return md + + longest_paths = [find_dependency_durations(task) for task in tasklist] + # Default in case there are no tasks + if longest_paths: + return max(longest_paths) + else: + return 0 + + +def determine_percentile(quantiles_file, duration): + + duration = duration.total_seconds() + + with open(quantiles_file) as f: + f.readline() # skip header + boundaries = [float(l.strip()) for l in f.readlines()] + + boundaries.sort() + for i, v in enumerate(boundaries): + if duration < v: + break + # Estimate percentile from len(boundaries)-quantile + return int(100 * i / len(boundaries)) + + +def task_duration_data(cache_dir): + with open(os.path.join(cache_dir, TASK_DURATION_CACHE)) as f: + return json.load(f) + + +def duration_summary(graph_cache_file, tasklist, cache_dir): + durations = task_duration_data(cache_dir) + + graph = dict() + if graph_cache_file: + with open(graph_cache_file) as f: + graph = json.load(f) + dependencies = find_all_dependencies(graph, tasklist) + longest_path = find_longest_path(graph, tasklist, durations) + dependency_duration = 0.0 + for task in dependencies: + dependency_duration += int(durations.get(task, 0.0)) + + total_requested_duration = 0.0 + for task in tasklist: + duration = int(durations.get(task, 0.0)) + total_requested_duration += duration + output = dict() + + total_requested_duration = timedelta(seconds=total_requested_duration) + total_dependency_duration = timedelta(seconds=dependency_duration) + + output["selected_duration"] = total_requested_duration + output["dependency_duration"] = total_dependency_duration + output["dependency_count"] = len(dependencies) + output["selected_count"] = len(tasklist) + + percentile = None + graph_quantile_cache = os.path.join(cache_dir, GRAPH_QUANTILE_CACHE) + if os.path.isfile(graph_quantile_cache): + percentile = determine_percentile( + graph_quantile_cache, total_dependency_duration + total_requested_duration + ) + if percentile: + output["percentile"] = percentile + + output["wall_duration_seconds"] = timedelta(seconds=int(longest_path)) + output["eta_datetime"] = datetime.now() + timedelta(seconds=longest_path) + + output["task_durations"] = { + task: int(durations.get(task, 0.0)) for task in tasklist + } + + return output diff --git a/tools/tryselect/util/fzf.py b/tools/tryselect/util/fzf.py new file mode 100644 index 0000000000..66894ffb67 --- /dev/null +++ b/tools/tryselect/util/fzf.py @@ -0,0 +1,423 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import platform +import shutil +import subprocess +import sys +from distutils.spawn import find_executable + +import mozfile +import six +from gecko_taskgraph.target_tasks import filter_by_uncommon_try_tasks +from mach.util import get_state_dir +from mozboot.util import http_download_and_save +from mozbuild.base import MozbuildObject +from mozterm import Terminal +from packaging.version import Version + +from ..push import check_working_directory +from ..tasks import generate_tasks +from ..util.manage_estimates import ( + download_task_history_data, + make_trimmed_taskgraph_cache, +) + +terminal = Terminal() + +here = os.path.abspath(os.path.dirname(__file__)) +build = MozbuildObject.from_environment(cwd=here) + +PREVIEW_SCRIPT = os.path.join(build.topsrcdir, "tools/tryselect/selectors/preview.py") + +FZF_MIN_VERSION = "0.20.0" +FZF_CURRENT_VERSION = "0.29.0" + +# It would make more sense to have the full filename be the key; but that makes +# the line too long and ./mach lint and black can't agree about what to about that. +# You can get these from the github release, e.g. +# https://github.com/junegunn/fzf/releases/download/0.24.1/fzf_0.24.1_checksums.txt +# However the darwin releases may not be included, so double check you have everything +FZF_CHECKSUMS = { + "linux_armv5.tar.gz": "61d3c2aa77b977ba694836fd1134da9272bd97ee490ececaf87959b985820111", + "linux_armv6.tar.gz": "db6b30fcbbd99ac4cf7e3ff6c5db1d3c0afcbe37d10ec3961bdc43e8c4f2e4f9", + "linux_armv7.tar.gz": "ed86f0e91e41d2cea7960a78e3eb175dc2a5fc1510380c195d0c3559bfdc701c", + "linux_arm64.tar.gz": "47988d8b68905541cbc26587db3ed1cfa8bc3aa8da535120abb4229b988f259e", + "linux_amd64.tar.gz": "0106f458b933be65edb0e8f0edb9a16291a79167836fd26a77ff5496269b5e9a", + "windows_armv5.zip": "08eaac45b3600d82608d292c23e7312696e7e11b6278b292feba25e8eb91c712", + "windows_armv6.zip": "8b6618726a9d591a45120fddebc29f4164e01ce6639ed9aa8fc79ab03eefcfed", + "windows_armv7.zip": "c167117b4c08f4f098446291115871ce5f14a8a8b22f0ca70e1b4342452ab5d7", + "windows_arm64.zip": "0cda7bf68850a3e867224a05949612405e63a4421d52396c1a6c9427d4304d72", + "windows_amd64.zip": "f0797ceee089017108c80b09086c71b8eec43d4af11ce939b78b1d5cfd202540", + "darwin_arm64.zip": "2571b4d381f1fc691e7603bbc8113a67116da2404751ebb844818d512dd62b4b", + "darwin_amd64.zip": "bc541e8ae0feb94efa96424bfe0b944f746db04e22f5cccfe00709925839a57f", + "openbsd_amd64.tar.gz": "b62343827ff83949c09d5e2c8ca0c1198d05f733c9a779ec37edd840541ccdab", + "freebsd_amd64.tar.gz": "f0367f2321c070d103589c7c7eb6a771bc7520820337a6c2fbb75be37ff783a9", +} + +FZF_INSTALL_MANUALLY = """ +The `mach try fuzzy` command depends on fzf. Please install it following the +appropriate instructions for your platform: + + https://github.com/junegunn/fzf#installation + +Only the binary is required, if you do not wish to install the shell and +editor integrations, download the appropriate binary and put it on your $PATH: + + https://github.com/junegunn/fzf/releases +""".lstrip() + +FZF_COULD_NOT_DETERMINE_PLATFORM = ( + """ +Could not automatically obtain the `fzf` binary because we could not determine +your Operating System. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_COULD_NOT_DETERMINE_MACHINE = ( + """ +Could not automatically obtain the `fzf` binary because we could not determine +your machine type. It's reported as '%s' and we don't handle that case; but fzf +may still be available as a prebuilt binary. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_NOT_SUPPORTED_X86 = ( + """ +We don't believe that a prebuilt binary for `fzf` if available on %s, but we +could be wrong. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_NOT_FOUND = ( + """ +Could not find the `fzf` binary. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_VERSION_FAILED = ( + """ +Could not obtain the 'fzf' version; we require version > 0.20.0 for some of +the features. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_INSTALL_FAILED = ( + """ +Failed to install fzf. + +""".lstrip() + + FZF_INSTALL_MANUALLY +) + +FZF_HEADER = """ +For more shortcuts, see {t.italic_white}mach help try fuzzy{t.normal} and {t.italic_white}man fzf +{shortcuts} +""".strip() + +fzf_shortcuts = { + "ctrl-a": "select-all", + "ctrl-d": "deselect-all", + "ctrl-t": "toggle-all", + "alt-bspace": "beginning-of-line+kill-line", + "?": "toggle-preview", +} + +fzf_header_shortcuts = [ + ("select", "tab"), + ("accept", "enter"), + ("cancel", "ctrl-c"), + ("select-all", "ctrl-a"), + ("cursor-up", "up"), + ("cursor-down", "down"), +] + + +def get_fzf_platform(): + if platform.machine() in ["i386", "i686"]: + print(FZF_NOT_SUPPORTED_X86 % platform.machine()) + sys.exit(1) + + if platform.system().lower() == "windows": + if platform.machine().lower() in ["x86_64", "amd64"]: + return "windows_amd64.zip" + elif platform.machine().lower() == "arm64": + return "windows_arm64.zip" + else: + print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine()) + sys.exit(1) + elif platform.system().lower() == "darwin": + if platform.machine().lower() in ["x86_64", "amd64"]: + return "darwin_amd64.zip" + elif platform.machine().lower() == "arm64": + return "darwin_arm64.zip" + else: + print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine()) + sys.exit(1) + elif platform.system().lower() == "linux": + if platform.machine().lower() in ["x86_64", "amd64"]: + return "linux_amd64.tar.gz" + elif platform.machine().lower() == "arm64": + return "linux_arm64.tar.gz" + else: + print(FZF_COULD_NOT_DETERMINE_MACHINE % platform.machine()) + sys.exit(1) + else: + print(FZF_COULD_NOT_DETERMINE_PLATFORM) + sys.exit(1) + + +def get_fzf_state_dir(): + return os.path.join(get_state_dir(), "fzf") + + +def get_fzf_filename(): + return "fzf-%s-%s" % (FZF_CURRENT_VERSION, get_fzf_platform()) + + +def get_fzf_download_link(): + return "https://github.com/junegunn/fzf/releases/download/%s/%s" % ( + FZF_CURRENT_VERSION, + get_fzf_filename(), + ) + + +def clean_up_state_dir(): + """ + We used to have a checkout of fzf that we would update. + Now we only download the bin and cpin the hash; so if + we find the old git checkout, wipe it + """ + + fzf_path = os.path.join(get_state_dir(), "fzf") + git_path = os.path.join(fzf_path, ".git") + if os.path.isdir(git_path): + shutil.rmtree(fzf_path, ignore_errors=True) + + # Also delete any existing fzf binary + fzf_bin = find_executable("fzf", fzf_path) + if fzf_bin: + mozfile.remove(fzf_bin) + + # Make sure the state dir is present + if not os.path.isdir(fzf_path): + os.makedirs(fzf_path) + + +def download_and_install_fzf(): + clean_up_state_dir() + download_link = get_fzf_download_link() + download_file = get_fzf_filename() + download_destination_path = get_fzf_state_dir() + download_destination_file = os.path.join(download_destination_path, download_file) + http_download_and_save( + download_link, download_destination_file, FZF_CHECKSUMS[get_fzf_platform()] + ) + + mozfile.extract(download_destination_file, download_destination_path) + mozfile.remove(download_destination_file) + + +def get_fzf_version(fzf_bin): + cmd = [fzf_bin, "--version"] + try: + fzf_version = subprocess.check_output(cmd) + except subprocess.CalledProcessError: + print(FZF_VERSION_FAILED) + sys.exit(1) + + # Some fzf versions have extra, e.g 0.18.0 (ff95134) + fzf_version = six.ensure_text(fzf_version.split()[0]) + + return fzf_version + + +def should_force_fzf_update(fzf_bin): + fzf_version = get_fzf_version(fzf_bin) + + # 0.20.0 introduced passing selections through a temporary file, + # which is good for large ctrl-a actions. + if Version(fzf_version) < Version(FZF_MIN_VERSION): + print("fzf version is old, you must update to use ./mach try fuzzy.") + return True + return False + + +def fzf_bootstrap(update=False): + """ + Bootstrap fzf if necessary and return path to the executable. + + This function is a bit complicated. We fetch a new version of fzf if: + 1) an existing fzf is too outdated + 2) the user says --update and we are behind the recommended version + 3) no fzf can be found and + 3a) user passes --update + 3b) user agrees to a prompt + + """ + fzf_path = get_fzf_state_dir() + + fzf_bin = find_executable("fzf") + if not fzf_bin: + fzf_bin = find_executable("fzf", fzf_path) + + if fzf_bin and should_force_fzf_update(fzf_bin): # Case (1) + update = True + + if fzf_bin and not update: + return fzf_bin + + elif fzf_bin and update: + # Case 2 + fzf_version = get_fzf_version(fzf_bin) + if Version(fzf_version) < Version(FZF_CURRENT_VERSION) and update: + # Bug 1623197: We only want to run fzf's `install` if it's not in the $PATH + # Swap to os.path.commonpath when we're not on Py2 + if fzf_bin and update and not fzf_bin.startswith(fzf_path): + print( + "fzf installed somewhere other than {}, please update manually".format( + fzf_path + ) + ) + sys.exit(1) + + download_and_install_fzf() + print("Updated fzf to {}".format(FZF_CURRENT_VERSION)) + else: + print("fzf is the recommended version and does not need an update") + + else: # not fzf_bin: + if not update: + # Case 3b + install = input("Could not detect fzf, install it now? [y/n]: ") + if install.lower() != "y": + return + + # Case 3a and 3b-fall-through + download_and_install_fzf() + fzf_bin = find_executable("fzf", fzf_path) + print("Installed fzf to {}".format(fzf_path)) + + return fzf_bin + + +def format_header(): + shortcuts = [] + for action, key in fzf_header_shortcuts: + shortcuts.append( + "{t.white}{action}{t.normal}: {t.yellow}<{key}>{t.normal}".format( + t=terminal, action=action, key=key + ) + ) + return FZF_HEADER.format(shortcuts=", ".join(shortcuts), t=terminal) + + +def run_fzf(cmd, tasks): + env = dict(os.environ) + env.update( + {"PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p])} + ) + # Make sure fzf uses Windows' shell rather than MozillaBuild bash or + # whatever our caller uses, since it doesn't quote the arguments properly + # and thus windows paths like: C:\moz\foo end up as C:mozfoo... + if platform.system() == "Windows": + env["SHELL"] = env["COMSPEC"] + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + env=env, + universal_newlines=True, + ) + out = proc.communicate("\n".join(tasks))[0].splitlines() + + selected = [] + query = None + if out: + query = out[0] + selected = out[1:] + return query, selected + + +def setup_tasks_for_fzf( + push, + parameters, + full=False, + disable_target_task_filter=False, + show_estimates=True, +): + check_working_directory(push) + tg = generate_tasks( + parameters, full=full, disable_target_task_filter=disable_target_task_filter + ) + all_tasks = sorted(tg.tasks.keys()) + + # graph_Cache created by generate_tasks, recreate the path to that file. + cache_dir = os.path.join( + get_state_dir(specific_to_topsrcdir=True), "cache", "taskgraph" + ) + if full: + graph_cache = os.path.join(cache_dir, "full_task_graph") + dep_cache = os.path.join(cache_dir, "full_task_dependencies") + target_set = os.path.join(cache_dir, "full_task_set") + else: + graph_cache = os.path.join(cache_dir, "target_task_graph") + dep_cache = os.path.join(cache_dir, "target_task_dependencies") + target_set = os.path.join(cache_dir, "target_task_set") + + if show_estimates: + download_task_history_data(cache_dir=cache_dir) + make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=target_set) + + if not full and not disable_target_task_filter: + # Put all_tasks into a list because it's used multiple times, and "filter()" + # returns a consumable iterator. + all_tasks = list(filter(filter_by_uncommon_try_tasks, all_tasks)) + + return all_tasks, dep_cache, cache_dir + + +def build_base_cmd(fzf, dep_cache, cache_dir, show_estimates=True): + key_shortcuts = [k + ":" + v for k, v in fzf_shortcuts.items()] + base_cmd = [ + fzf, + "-m", + "--bind", + ",".join(key_shortcuts), + "--header", + format_header(), + "--preview-window=right:30%", + "--print-query", + ] + + if show_estimates: + base_cmd.extend( + [ + "--preview", + '{} {} -g {} -s -c {} -t "{{+f}}"'.format( + sys.executable, PREVIEW_SCRIPT, dep_cache, cache_dir + ), + ] + ) + else: + base_cmd.extend( + [ + "--preview", + '{} {} -t "{{+f}}"'.format(sys.executable, PREVIEW_SCRIPT), + ] + ) + + return base_cmd diff --git a/tools/tryselect/util/manage_estimates.py b/tools/tryselect/util/manage_estimates.py new file mode 100644 index 0000000000..23fa481228 --- /dev/null +++ b/tools/tryselect/util/manage_estimates.py @@ -0,0 +1,132 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import os +from datetime import datetime, timedelta + +import requests +import six + +TASK_DURATION_URL = ( + "https://storage.googleapis.com/mozilla-mach-data/task_duration_history.json" +) +GRAPH_QUANTILES_URL = ( + "https://storage.googleapis.com/mozilla-mach-data/machtry_quantiles.csv" +) +from .estimates import GRAPH_QUANTILE_CACHE, TASK_DURATION_CACHE, TASK_DURATION_TAG_FILE + + +def check_downloaded_history(tag_file, duration_cache, quantile_cache): + if not os.path.isfile(tag_file): + return False + + try: + with open(tag_file) as f: + duration_tags = json.load(f) + download_date = datetime.strptime( + duration_tags.get("download_date"), "%Y-%M-%d" + ) + if download_date < datetime.now() - timedelta(days=7): + return False + except (OSError, ValueError): + return False + + if not os.path.isfile(duration_cache): + return False + # Check for old format version of file. + with open(duration_cache) as f: + data = json.load(f) + if isinstance(data, list): + return False + if not os.path.isfile(quantile_cache): + return False + + return True + + +def download_task_history_data(cache_dir): + """Fetch task duration data exported from BigQuery.""" + task_duration_cache = os.path.join(cache_dir, TASK_DURATION_CACHE) + task_duration_tag_file = os.path.join(cache_dir, TASK_DURATION_TAG_FILE) + graph_quantile_cache = os.path.join(cache_dir, GRAPH_QUANTILE_CACHE) + + if check_downloaded_history( + task_duration_tag_file, task_duration_cache, graph_quantile_cache + ): + return + + try: + os.unlink(task_duration_tag_file) + os.unlink(task_duration_cache) + os.unlink(graph_quantile_cache) + except OSError: + print("No existing task history to clean up.") + + try: + r = requests.get(TASK_DURATION_URL, stream=True) + r.raise_for_status() + except requests.exceptions.RequestException as exc: + # This is fine, the durations just won't be in the preview window. + print( + "Error fetching task duration cache from {}: {}".format( + TASK_DURATION_URL, exc + ) + ) + return + + # The data retrieved from google storage is a newline-separated + # list of json entries, which Python's json module can't parse. + duration_data = list() + for line in r.text.splitlines(): + duration_data.append(json.loads(line)) + + # Reformat duration data to avoid list of dicts, as this is slow in the preview window + duration_data = {d["name"]: d["mean_duration_seconds"] for d in duration_data} + + with open(task_duration_cache, "w") as f: + json.dump(duration_data, f, indent=4) + + try: + r = requests.get(GRAPH_QUANTILES_URL, stream=True) + r.raise_for_status() + except requests.exceptions.RequestException as exc: + # This is fine, the percentile just won't be in the preview window. + print( + "Error fetching task group percentiles from {}: {}".format( + GRAPH_QUANTILES_URL, exc + ) + ) + return + + with open(graph_quantile_cache, "w") as f: + f.write(six.ensure_text(r.content)) + + with open(task_duration_tag_file, "w") as f: + json.dump({"download_date": datetime.now().strftime("%Y-%m-%d")}, f, indent=4) + + +def make_trimmed_taskgraph_cache(graph_cache, dep_cache, target_file=None): + """Trim the taskgraph cache used for dependencies. + + Speeds up the fzf preview window to less human-perceptible + ranges.""" + if not os.path.isfile(graph_cache): + return + + target_task_set = set() + if target_file and os.path.isfile(target_file): + with open(target_file) as f: + target_task_set = set(json.load(f).keys()) + + with open(graph_cache) as f: + graph = json.load(f) + graph = { + name: list(defn["dependencies"].values()) + for name, defn in graph.items() + if name in target_task_set + } + with open(dep_cache, "w") as f: + json.dump(graph, f, indent=4) diff --git a/tools/tryselect/watchman.json b/tools/tryselect/watchman.json new file mode 100644 index 0000000000..a41b1829d7 --- /dev/null +++ b/tools/tryselect/watchman.json @@ -0,0 +1,15 @@ +[ + "trigger", + ".", + { + "name": "rebuild-taskgraph-cache", + "expression": ["match", "taskcluster/**", "wholename"], + "command": [ + "./mach", + "python", + "-c", + "from tryselect.tasks import generate_tasks; generate_tasks()" + ], + "append_files": false + } +] diff --git a/tools/update-packaging/README b/tools/update-packaging/README new file mode 100644 index 0000000000..7029ffb8e5 --- /dev/null +++ b/tools/update-packaging/README @@ -0,0 +1,4 @@ +This directory contains a tool for generating update packages for +the update system described here: + + http://wiki.mozilla.org/Software_Update diff --git a/tools/update-packaging/app.mozbuild b/tools/update-packaging/app.mozbuild new file mode 100644 index 0000000000..9053ec17f1 --- /dev/null +++ b/tools/update-packaging/app.mozbuild @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + '/modules/libmar/src', + '/modules/libmar/tool', + '/other-licenses/bsdiff', +] diff --git a/tools/update-packaging/common.sh b/tools/update-packaging/common.sh new file mode 100755 index 0000000000..4b994f3016 --- /dev/null +++ b/tools/update-packaging/common.sh @@ -0,0 +1,201 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Code shared by update packaging scripts. +# Author: Darin Fisher +# + +# ----------------------------------------------------------------------------- +QUIET=0 + +# By default just assume that these tools exist on our path +MAR=${MAR:-mar} +MBSDIFF=${MBSDIFF:-mbsdiff} +XZ=${XZ:-xz} +$XZ --version > /dev/null 2>&1 +if [ $? -ne 0 ]; then + # If $XZ is not set and not found on the path then this is probably + # running on a windows buildbot. Some of the Windows build systems have + # xz.exe in topsrcdir/xz/. Look in the places this would be in both a + # mozilla-central and comm-central build. + XZ="$(dirname "$(dirname "$(dirname "$0")")")/xz/xz.exe" + $XZ --version > /dev/null 2>&1 + if [ $? -ne 0 ]; then + XZ="$(dirname "$(dirname "$(dirname "$(dirname "$0")")")")/xz/xz.exe" + $XZ --version > /dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "xz was not found on this system!" + echo "exiting" + exit 1 + fi + fi +fi +# Ensure that we're always using the right compression settings +export XZ_OPT="-T1 -7e" + +# ----------------------------------------------------------------------------- +# Helper routines + +notice() { + echo "$*" 1>&2 +} + +verbose_notice() { + if [ $QUIET -eq 0 ]; then + notice "$*" + fi +} + +get_file_size() { + info=($(ls -ln "$1")) + echo ${info[4]} +} + +copy_perm() { + reference="$1" + target="$2" + + if [ -x "$reference" ]; then + chmod 0755 "$target" + else + chmod 0644 "$target" + fi +} + +make_add_instruction() { + f="$1" + filev3="$2" + + # Used to log to the console + if [ $4 ]; then + forced=" (forced)" + else + forced= + fi + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + verbose_notice " add-if \"$testdir\" \"$f\"" + echo "add-if \"$testdir\" \"$f\"" >> "$filev3" + else + verbose_notice " add \"$f\"$forced" + echo "add \"$f\"" >> "$filev3" + fi +} + +check_for_add_if_not_update() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ + `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +make_add_if_not_instruction() { + f="$1" + filev3="$2" + + verbose_notice " add-if-not \"$f\" \"$f\"" + echo "add-if-not \"$f\" \"$f\"" >> "$filev3" +} + +make_patch_instruction() { + f="$1" + filev3="$2" + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + verbose_notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> "$filev3" + else + verbose_notice " patch \"$f.patch\" \"$f\"" + echo "patch \"$f.patch\" \"$f\"" >> "$filev3" + fi +} + +append_remove_instructions() { + dir="$1" + filev3="$2" + + if [ -f "$dir/removed-files" ]; then + listfile="$dir/removed-files" + elif [ -f "$dir/Contents/Resources/removed-files" ]; then + listfile="$dir/Contents/Resources/removed-files" + fi + if [ -n "$listfile" ]; then + # Map spaces to pipes so that we correctly handle filenames with spaces. + files=($(cat "$listfile" | tr " " "|" | sort -r)) + num_files=${#files[*]} + for ((i=0; $i<$num_files; i=$i+1)); do + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. + if [ -n "$f" ]; then + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + verbose_notice " rmdir \"$f\"" + echo "rmdir \"$f\"" >> "$filev3" + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + verbose_notice " rmrfdir \"$f\"" + echo "rmrfdir \"$f\"" >> "$filev3" + else + verbose_notice " remove \"$f\"" + echo "remove \"$f\"" >> "$filev3" + fi + fi + fi + done + fi +} + +# List all files in the current directory, stripping leading "./" +# Pass a variable name and it will be filled as an array. +list_files() { + count=0 + temp_filelist=$(mktemp) + find . -type f \ + ! -name "update.manifest" \ + ! -name "updatev2.manifest" \ + ! -name "updatev3.manifest" \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "${temp_filelist}" + while read file; do + eval "${1}[$count]=\"$file\"" + (( count++ )) + done < "${temp_filelist}" + rm "${temp_filelist}" +} + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + temp_dirlist=$(mktemp) + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "${temp_dirlist}" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "${temp_dirlist}" + rm "${temp_dirlist}" +} diff --git a/tools/update-packaging/make_full_update.sh b/tools/update-packaging/make_full_update.sh new file mode 100755 index 0000000000..db2c5898ef --- /dev/null +++ b/tools/update-packaging/make_full_update.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates full update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +if [ $1 = -h ]; then + print_usage + notice "" + notice "The contents of DIRECTORY will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice " -q be less verbose" + notice "" + exit 1 +fi + +if [ $1 = -q ]; then + QUIET=1 + export QUIET + shift +fi + +# ----------------------------------------------------------------------------- + +mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}" + +archive="$1" +targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi +workdir="$targetdir.work" +updatemanifestv3="$workdir/updatev3.manifest" +targetfiles="updatev3.manifest" + +mkdir -p "$workdir" + +# Generate a list of all files in the target directory. +pushd "$targetdir" +if test $? -ne 0 ; then + exit 1 +fi + +if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" + exit 1 + fi +fi + +list_files files + +popd + +# Add the type of update to the beginning of the update manifests. +> "$updatemanifestv3" +notice "" +notice "Adding type instruction to update manifests" +notice " type complete" +echo "type \"complete\"" >> "$updatemanifestv3" + +notice "" +notice "Adding file add instructions to update manifests" +num_files=${#files[*]} + +for ((i=0; $i<$num_files; i=$i+1)); do + f="${files[$i]}" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + else + make_add_instruction "$f" "$updatemanifestv3" + fi + + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" +done + +# Append remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv3" + +$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" + +mar_command="$mar_command -C \"$workdir\" -c output.mar" +eval "$mar_command $targetfiles" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/tools/update-packaging/make_incremental_update.sh b/tools/update-packaging/make_incremental_update.sh new file mode 100755 index 0000000000..24d6861673 --- /dev/null +++ b/tools/update-packaging/make_incremental_update.sh @@ -0,0 +1,313 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates incremental update packages for the update system. +# Author: Darin Fisher +# + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" + notice "" + notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice " -f clobber this file in the installation" + notice " Must be a path to a file to clobber in the partial update." + notice " -q be less verbose" + notice "" +} + +check_for_forced_update() { + force_list="$1" + forced_file_chk="$2" + + local f + + if [ "$forced_file_chk" = "precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/precomplete" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "$forced_file_chk" = "Contents/Resources/removed-files" ]; then + ## "true" *giggle* + return 0; + fi + + # notarization ticket + if [ "$forced_file_chk" = "Contents/CodeResources" ]; then + ## "true" *giggle* + return 0; + fi + + if [ "${forced_file_chk##*.}" = "chk" ]; then + ## "true" *giggle* + return 0; + fi + + for f in $force_list; do + #echo comparing $forced_file_chk to $f + if [ "$forced_file_chk" = "$f" ]; then + ## "true" *giggle* + return 0; + fi + done + ## 'false'... because this is bash. Oh yay! + return 1; +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +requested_forced_updates='Contents/MacOS/firefox' + +while getopts "hqf:" flag +do + case "$flag" in + h) print_usage; exit 0 + ;; + q) QUIET=1 + ;; + f) requested_forced_updates="$requested_forced_updates $OPTARG" + ;; + ?) print_usage; exit 1 + ;; + esac +done + +# ----------------------------------------------------------------------------- + +mar_command="$MAR -V ${MOZ_PRODUCT_VERSION:?} -H ${MAR_CHANNEL_ID:?}" + +let arg_start=$OPTIND-1 +shift $arg_start + +archive="$1" +olddir="$2" +newdir="$3" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then + # Remove the / + newdir=$(echo "$newdir" | sed -e 's:\/$::') +fi +workdir="$(mktemp -d)" +updatemanifestv3="$workdir/updatev3.manifest" +archivefiles="updatev3.manifest" + +mkdir -p "$workdir" + +# Generate a list of all files in the target directory. +pushd "$olddir" +if test $? -ne 0 ; then + exit 1 +fi + +list_files oldfiles +list_dirs olddirs + +popd + +pushd "$newdir" +if test $? -ne 0 ; then + exit 1 +fi + +if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" + exit 1 + fi +fi + +list_dirs newdirs +list_files newfiles + +popd + +# Add the type of update to the beginning of the update manifests. +notice "" +notice "Adding type instruction to update manifests" +> $updatemanifestv3 +notice " type partial" +echo "type \"partial\"" >> $updatemanifestv3 + +notice "" +notice "Adding file patch and add instructions to update manifests" + +num_oldfiles=${#oldfiles[*]} +remove_array= +num_removes=0 + +for ((i=0; $i<$num_oldfiles; i=$i+1)); do + f="${oldfiles[$i]}" + + # If this file exists in the new directory as well, then check if it differs. + if [ -f "$newdir/$f" ]; then + + if check_for_add_if_not_update "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p `dirname "$workdir/$f"` + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_if_not_instruction "$f" "$updatemanifestv3" + archivefiles="$archivefiles \"$f\"" + continue 1 + fi + + if check_for_forced_update "$requested_forced_updates" "$f"; then + # The full workdir may not exist yet, so create it if necessary. + mkdir -p `dirname "$workdir/$f"` + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + make_add_instruction "$f" "$updatemanifestv3" 1 + archivefiles="$archivefiles \"$f\"" + continue 1 + fi + + if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then + # Compute both the compressed binary diff and the compressed file, and + # compare the sizes. Then choose the smaller of the two to package. + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + verbose_notice "diffing \"$f\"" + # MBSDIFF_HOOK represents the communication interface with funsize and, + # if enabled, caches the intermediate patches for future use and + # compute avoidance + # + # An example of MBSDIFF_HOOK env variable could look like this: + # export MBSDIFF_HOOK="myscript.sh -A https://funsize/api -c /home/user" + # where myscript.sh has the following usage: + # myscript.sh -A SERVER-URL [-c LOCAL-CACHE-DIR-PATH] [-g] [-u] \ + # PATH-FROM-URL PATH-TO-URL PATH-PATCH SERVER-URL + # + # Note: patches are bzipped or xz stashed in funsize to gain more speed + + # if service is not enabled then default to old behavior + if [ -z "$MBSDIFF_HOOK" ]; then + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" + else + # if service enabled then check patch existence for retrieval + if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then + verbose_notice "file \"$f\" found in funsize, diffing skipped" + else + # if not found already - compute it and cache it for future use + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" + $XZ $XZ_OPT --compress --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch" + $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz" + fi + fi + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + patchfile="$workdir/$f.patch.xz" + patchsize=$(get_file_size "$patchfile") + fullsize=$(get_file_size "$workdir/$f") + + if [ $patchsize -lt $fullsize ]; then + make_patch_instruction "$f" "$updatemanifestv3" + mv -f "$patchfile" "$workdir/$f.patch" + rm -f "$workdir/$f" + archivefiles="$archivefiles \"$f.patch\"" + else + make_add_instruction "$f" "$updatemanifestv3" + rm -f "$patchfile" + archivefiles="$archivefiles \"$f\"" + fi + fi + else + # remove instructions are added after add / patch instructions for + # consistency with make_incremental_updates.py + remove_array[$num_removes]=$f + (( num_removes++ )) + fi +done + +# Newly added files +notice "" +notice "Adding file add instructions to update manifests" +num_newfiles=${#newfiles[*]} + +for ((i=0; $i<$num_newfiles; i=$i+1)); do + f="${newfiles[$i]}" + + # If we've already tested this file, then skip it + for ((j=0; $j<$num_oldfiles; j=$j+1)); do + if [ "$f" = "${oldfiles[j]}" ]; then + continue 2 + fi + done + + dir=$(dirname "$workdir/$f") + mkdir -p "$dir" + + $XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f" + copy_perm "$newdir/$f" "$workdir/$f" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + else + make_add_instruction "$f" "$updatemanifestv3" + fi + + + archivefiles="$archivefiles \"$f\"" +done + +notice "" +notice "Adding file remove instructions to update manifests" +for ((i=0; $i<$num_removes; i=$i+1)); do + f="${remove_array[$i]}" + verbose_notice " remove \"$f\"" + echo "remove \"$f\"" >> $updatemanifestv3 +done + +# Add remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$newdir" "$updatemanifestv3" + +notice "" +notice "Adding directory remove instructions for directories that no longer exist" +num_olddirs=${#olddirs[*]} + +for ((i=0; $i<$num_olddirs; i=$i+1)); do + f="${olddirs[$i]}" + # If this dir doesn't exist in the new directory remove it. + if [ ! -d "$newdir/$f" ]; then + verbose_notice " rmdir $f/" + echo "rmdir \"$f/\"" >> $updatemanifestv3 + fi +done + +$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" + +mar_command="$mar_command -C \"$workdir\" -c output.mar" +eval "$mar_command $archivefiles" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/tools/update-packaging/moz.build b/tools/update-packaging/moz.build new file mode 100644 index 0000000000..568f361a54 --- /dev/null +++ b/tools/update-packaging/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/update-packaging/moz.configure b/tools/update-packaging/moz.configure new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/update-packaging/test/buildrefmars.sh b/tools/update-packaging/test/buildrefmars.sh new file mode 100755 index 0000000000..fd2d5384d7 --- /dev/null +++ b/tools/update-packaging/test/buildrefmars.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Builds all the reference mars + +if [ -f "ref.mar" ]; then + rm "ref.mar" +fi +if [ -f "ref-mac.mar" ]; then + rm "ref-mac.mar" +fi + + ../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to + ../make_incremental_update.sh ref-mac.mar `pwd`/from-mac `pwd`/to-mac + +if [ -f "product-1.0.lang.platform.complete.mar" ]; then + rm "product-1.0.lang.platform.complete.mar" +fi +if [ -f "product-2.0.lang.platform.complete.mar" ]; then + rm "product-2.0.lang.platform.complete.mar" +fi +if [ -f "product-2.0.lang.mac.complete.mar" ]; then + rm "product-2.0.lang.mac.complete.mar" +fi + +./make_full_update.sh product-1.0.lang.platform.complete.mar "`pwd`/from" +./make_full_update.sh product-2.0.lang.platform.complete.mar "`pwd`/to" +./make_full_update.sh product-1.0.lang.mac.complete.mar "`pwd`/from-mac" +./make_full_update.sh product-2.0.lang.mac.complete.mar "`pwd`/to-mac" diff --git a/tools/update-packaging/test/common.sh b/tools/update-packaging/test/common.sh new file mode 100755 index 0000000000..ad88da2203 --- /dev/null +++ b/tools/update-packaging/test/common.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# Code shared by update packaging scripts. +# Author: Darin Fisher +# +# In here to use the local common.sh to allow the full mars to have unfiltered files + +# ----------------------------------------------------------------------------- +# By default just assume that these tools exist on our path +MAR=${MAR:-mar} +XZ=${XZ:-xz} +MBSDIFF=${MBSDIFF:-mbsdiff} + +# ----------------------------------------------------------------------------- +# Helper routines + +notice() { + echo "$*" 1>&2 +} + +get_file_size() { + info=($(ls -ln "$1")) + echo ${info[4]} +} + +copy_perm() { + reference="$1" + target="$2" + + if [ -x "$reference" ]; then + chmod 0755 "$target" + else + chmod 0644 "$target" + fi +} + +make_add_instruction() { + f="$1" + filev2="$2" + # The third param will be an empty string when a file add instruction is only + # needed in the version 2 manifest. This only happens when the file has an + # add-if-not instruction in the version 3 manifest. This is due to the + # precomplete file prior to the version 3 manifest having a remove instruction + # for this file so the file is removed before applying a complete update. + filev3="$3" + + # Used to log to the console + if [ $4 ]; then + forced=" (forced)" + else + forced= + fi + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " add-if \"$testdir\" \"$f\"" + echo "add-if \"$testdir\" \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add-if \"$testdir\" \"$f\"" >> $filev3 + fi + else + notice " add \"$f\"$forced" + echo "add \"$f\"" >> $filev2 + if [ ! $filev3 = "" ]; then + echo "add \"$f\"" >> $filev3 + fi + fi +} + +check_for_add_if_not_update() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "channel-prefs.js" -o \ + `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +check_for_add_to_manifestv2() { + add_if_not_file_chk="$1" + + if [ `basename $add_if_not_file_chk` = "update-settings.ini" ]; then + ## "true" *giggle* + return 0; + fi + ## 'false'... because this is bash. Oh yay! + return 1; +} + +make_add_if_not_instruction() { + f="$1" + filev3="$2" + + notice " add-if-not \"$f\" \"$f\"" + echo "add-if-not \"$f\" \"$f\"" >> $filev3 +} + +make_patch_instruction() { + f="$1" + filev2="$2" + filev3="$3" + + is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/') + if [ $is_extension = "1" ]; then + # Use the subdirectory of the extensions folder as the file to test + # before performing this add instruction. + testdir=$(echo "$f" | sed 's/\(.*distribution\/extensions\/[^\/]*\)\/.*/\1/') + notice " patch-if \"$testdir\" \"$f.patch\" \"$f\"" + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev2 + echo "patch-if \"$testdir\" \"$f.patch\" \"$f\"" >> $filev3 + else + notice " patch \"$f.patch\" \"$f\"" + echo "patch \"$f.patch\" \"$f\"" >> $filev2 + echo "patch \"$f.patch\" \"$f\"" >> $filev3 + fi +} + +append_remove_instructions() { + dir="$1" + filev2="$2" + filev3="$3" + + if [ -f "$dir/removed-files" ]; then + listfile="$dir/removed-files" + elif [ -f "$dir/Contents/Resources/removed-files" ]; then + listfile="$dir/Contents/Resources/removed-files" + fi + if [ -n "$listfile" ]; then + # Map spaces to pipes so that we correctly handle filenames with spaces. + files=($(cat "$listfile" | tr " " "|" | sort -r)) + num_files=${#files[*]} + for ((i=0; $i<$num_files; i=$i+1)); do + # Map pipes back to whitespace and remove carriage returns + f=$(echo ${files[$i]} | tr "|" " " | tr -d '\r') + # Trim whitespace + f=$(echo $f) + # Exclude blank lines. + if [ -n "$f" ]; then + # Exclude comments + if [ ! $(echo "$f" | grep -c '^#') = 1 ]; then + if [ $(echo "$f" | grep -c '\/$') = 1 ]; then + notice " rmdir \"$f\"" + echo "rmdir \"$f\"" >> $filev2 + echo "rmdir \"$f\"" >> $filev3 + elif [ $(echo "$f" | grep -c '\/\*$') = 1 ]; then + # Remove the * + f=$(echo "$f" | sed -e 's:\*$::') + notice " rmrfdir \"$f\"" + echo "rmrfdir \"$f\"" >> $filev2 + echo "rmrfdir \"$f\"" >> $filev3 + else + notice " remove \"$f\"" + echo "remove \"$f\"" >> $filev2 + echo "remove \"$f\"" >> $filev3 + fi + fi + fi + done + fi +} + +# List all files in the current directory, stripping leading "./" +# Pass a variable name and it will be filled as an array. +list_files() { + count=0 + + # Removed the exclusion cases here to allow for generation of testing mars + find . -type f \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "$workdir/temp-filelist" + while read file; do + eval "${1}[$count]=\"$file\"" + (( count++ )) + done < "$workdir/temp-filelist" + rm "$workdir/temp-filelist" +} + +# List all directories in the current directory, stripping leading "./" +list_dirs() { + count=0 + + find . -type d \ + ! -name "." \ + ! -name ".." \ + | sed 's/\.\/\(.*\)/\1/' \ + | sort -r > "$workdir/temp-dirlist" + while read dir; do + eval "${1}[$count]=\"$dir\"" + (( count++ )) + done < "$workdir/temp-dirlist" + rm "$workdir/temp-dirlist" +} diff --git a/tools/update-packaging/test/diffmar.sh b/tools/update-packaging/test/diffmar.sh new file mode 100755 index 0000000000..4b16ffb77a --- /dev/null +++ b/tools/update-packaging/test/diffmar.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Compares two mars + +marA="$1" +marB="$2" +testDir="$3" +workdir="/tmp/diffmar/$testDir" +fromdir="$workdir/0" +todir="$workdir/1" + +# On Windows, creation time can be off by a second or more between the files in +# the fromdir and todir due to them being extracted synchronously so use +# time-style and exclude seconds from the creation time. +lsargs="-algR" +unamestr=`uname` +if [ ! "$unamestr" = 'Darwin' ]; then + unamestr=`uname -o` + if [ "$unamestr" = 'Msys' -o "$unamestr" = "Cygwin" ]; then + lsargs="-algR --time-style=+%Y-%m-%d-%H:%M" + fi +fi + +rm -rf "$workdir" +mkdir -p "$fromdir" +mkdir -p "$todir" + +cp "$1" "$fromdir" +cp "$2" "$todir" + +cd "$fromdir" +mar -x "$1" +rm "$1" +rm -f updatev2.manifest # Older files may contain this +mv updatev3.manifest updatev3.manifest.xz +xz -d updatev3.manifest.xz +ls $lsargs > files.txt + +cd "$todir" +mar -x "$2" +rm "$2" +mv updatev3.manifest updatev3.manifest.xz +xz -d updatev3.manifest.xz +ls $lsargs > files.txt + +echo "diffing $fromdir and $todir" +echo "on linux shell sort and python sort return different results" +echo "which can cause differences in the manifest files" +diff -ru "$fromdir" "$todir" diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/force.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/same.bin b/tools/update-packaging/test/from-mac/Contents/MacOS/same.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/from-mac/Contents/MacOS/same.bin differ diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest b/tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest new file mode 100644 index 0000000000..d3e8ed851b --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt new file mode 100644 index 0000000000..d7c40c63cd --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/readme.txt @@ -0,0 +1 @@ +This from file should be ignored diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.bin differ diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest new file mode 100644 index 0000000000..d3e8ed851b --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/MacOS/{foodir/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/application.ini b/tools/update-packaging/test/from-mac/Contents/Resources/application.ini new file mode 100644 index 0000000000..942e91a16a --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=1 +BuildID=20120101010101 diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/chrome.manifest b/tools/update-packaging/test/from-mac/Contents/Resources/chrome.manifest new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/precomplete b/tools/update-packaging/test/from-mac/Contents/Resources/precomplete new file mode 100644 index 0000000000..2d9068d372 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/precomplete @@ -0,0 +1,26 @@ +remove "Contents/MacOS/{foodir/update.manifest" +remove "Contents/MacOS/{foodir/same.txt" +remove "Contents/MacOS/{foodir/same.bin" +remove "Contents/MacOS/{foodir/removed.txt" +remove "Contents/MacOS/{foodir/readme.txt" +remove "Contents/MacOS/{foodir/force.txt" +remove "Contents/MacOS/{foodir/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/update.manifest" +remove "Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/same.txt" +remove "Contents/MacOS/same.bin" +remove "Contents/MacOS/removed.txt" +remove "Contents/MacOS/readme.txt" +remove "Contents/MacOS/force.txt" +remove "Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/application.ini" +remove "Contents/Resources/precomplete" +rmdir "Contents/MacOS/{foodir/" +rmdir "Contents/MacOS/searchplugins/diff/" +rmdir "Contents/MacOS/searchplugins/" +rmdir "Contents/MacOS/extensions/diff/" +rmdir "Contents/MacOS/extensions/" +rmdir "Contents/MacOS/" +rmdir "Contents/Resources/" +rmdir "Contents/" diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/readme.txt b/tools/update-packaging/test/from-mac/Contents/Resources/readme.txt new file mode 100644 index 0000000000..b1a96f1fea --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/readme.txt @@ -0,0 +1,2 @@ +This from file should be ignored + diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/removed-files b/tools/update-packaging/test/from-mac/Contents/Resources/removed-files new file mode 100644 index 0000000000..5bbdac6f62 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/removed-files @@ -0,0 +1,8 @@ +Contents/Resources/removed1.txt +Contents/MacOS/removed2.bin +Contents/MacOS/recursivedir/meh/* +Contents/Resources/dir/ +Contents/MacOS/this file has spaces + + +Contents/MacOS/extra-spaces diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/removed.txt b/tools/update-packaging/test/from-mac/Contents/Resources/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/same.txt b/tools/update-packaging/test/from-mac/Contents/Resources/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini b/tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from-mac/Contents/Resources/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/application.ini b/tools/update-packaging/test/from/application.ini new file mode 100644 index 0000000000..942e91a16a --- /dev/null +++ b/tools/update-packaging/test/from/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=1 +BuildID=20120101010101 diff --git a/tools/update-packaging/test/from/chrome.manifest b/tools/update-packaging/test/from/chrome.manifest new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/update-packaging/test/from/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/force.txt b/tools/update-packaging/test/from/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/precomplete b/tools/update-packaging/test/from/precomplete new file mode 100644 index 0000000000..e27f4fc31c --- /dev/null +++ b/tools/update-packaging/test/from/precomplete @@ -0,0 +1,23 @@ +remove "{foodir/update.manifest" +remove "{foodir/same.txt" +remove "{foodir/same.bin" +remove "{foodir/removed.txt" +remove "{foodir/readme.txt" +remove "{foodir/force.txt" +remove "{foodir/diff-patch-larger-than-file.txt" +remove "update.manifest" +remove "searchplugins/diff/diff-patch-larger-than-file.txt" +remove "same.txt" +remove "same.bin" +remove "removed.txt" +remove "readme.txt" +remove "precomplete" +remove "force.txt" +remove "extensions/diff/diff-patch-larger-than-file.txt" +remove "diff-patch-larger-than-file.txt" +remove "application.ini" +rmdir "{foodir/" +rmdir "searchplugins/diff/" +rmdir "searchplugins/" +rmdir "extensions/diff/" +rmdir "extensions/" diff --git a/tools/update-packaging/test/from/readme.txt b/tools/update-packaging/test/from/readme.txt new file mode 100644 index 0000000000..b1a96f1fea --- /dev/null +++ b/tools/update-packaging/test/from/readme.txt @@ -0,0 +1,2 @@ +This from file should be ignored + diff --git a/tools/update-packaging/test/from/removed-files b/tools/update-packaging/test/from/removed-files new file mode 100644 index 0000000000..73b348d9c4 --- /dev/null +++ b/tools/update-packaging/test/from/removed-files @@ -0,0 +1,8 @@ +removed1.txt +removed2.bin +recursivedir/meh/* +dir/ +this file has spaces + + +extra-spaces diff --git a/tools/update-packaging/test/from/removed.txt b/tools/update-packaging/test/from/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from/same.bin b/tools/update-packaging/test/from/same.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/from/same.bin differ diff --git a/tools/update-packaging/test/from/same.txt b/tools/update-packaging/test/from/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/update-settings.ini b/tools/update-packaging/test/from/update-settings.ini new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/from/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from/update.manifest b/tools/update-packaging/test/from/update.manifest new file mode 100644 index 0000000000..d3e8ed851b --- /dev/null +++ b/tools/update-packaging/test/from/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/from/{foodir/channel-prefs.js b/tools/update-packaging/test/from/{foodir/channel-prefs.js new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..8098d25853 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +from file diff --git a/tools/update-packaging/test/from/{foodir/force.txt b/tools/update-packaging/test/from/{foodir/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/{foodir/readme.txt b/tools/update-packaging/test/from/{foodir/readme.txt new file mode 100644 index 0000000000..d7c40c63cd --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/readme.txt @@ -0,0 +1 @@ +This from file should be ignored diff --git a/tools/update-packaging/test/from/{foodir/removed.txt b/tools/update-packaging/test/from/{foodir/removed.txt new file mode 100644 index 0000000000..2c3f0b3406 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/removed.txt @@ -0,0 +1 @@ +removed diff --git a/tools/update-packaging/test/from/{foodir/same.bin b/tools/update-packaging/test/from/{foodir/same.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/from/{foodir/same.bin differ diff --git a/tools/update-packaging/test/from/{foodir/same.txt b/tools/update-packaging/test/from/{foodir/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/from/{foodir/update.manifest b/tools/update-packaging/test/from/{foodir/update.manifest new file mode 100644 index 0000000000..d3e8ed851b --- /dev/null +++ b/tools/update-packaging/test/from/{foodir/update.manifest @@ -0,0 +1 @@ +from file shouldn't go in update diff --git a/tools/update-packaging/test/make_full_update.sh b/tools/update-packaging/test/make_full_update.sh new file mode 100755 index 0000000000..cdcdae3c3b --- /dev/null +++ b/tools/update-packaging/test/make_full_update.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# +# This tool generates full update packages for the update system. +# Author: Darin Fisher +# +# In here to use the local common.sh to allow the full mars to have unfiltered files + +. $(dirname "$0")/common.sh + +# ----------------------------------------------------------------------------- + +print_usage() { + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY" +} + +if [ $# = 0 ]; then + print_usage + exit 1 +fi + +if [ $1 = -h ]; then + print_usage + notice "" + notice "The contents of DIRECTORY will be stored in ARCHIVE." + notice "" + notice "Options:" + notice " -h show this help text" + notice "" + exit 1 +fi + +# ----------------------------------------------------------------------------- + +archive="$1" +targetdir="$2" +# Prevent the workdir from being inside the targetdir so it isn't included in +# the update mar. +if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then + # Remove the / + targetdir=$(echo "$targetdir" | sed -e 's:\/$::') +fi +workdir="$targetdir.work" +updatemanifestv3="$workdir/updatev3.manifest" +targetfiles="updatev3.manifest" + +mkdir -p "$workdir" + +# Generate a list of all files in the target directory. +pushd "$targetdir" +if test $? -ne 0 ; then + exit 1 +fi + +if [ ! -f "precomplete" ]; then + if [ ! -f "Contents/Resources/precomplete" ]; then + notice "precomplete file is missing!" + exit 1 + fi +fi + +list_files files + +popd + +# Add the type of update to the beginning of the update manifests. +> $updatemanifestv3 +notice "" +notice "Adding type instruction to update manifests" +notice " type complete" +echo "type \"complete\"" >> $updatemanifestv3 + +notice "" +notice "Adding file add instructions to update manifests" +num_files=${#files[*]} + +for ((i=0; $i<$num_files; i=$i+1)); do + f="${files[$i]}" + + if check_for_add_if_not_update "$f"; then + make_add_if_not_instruction "$f" "$updatemanifestv3" + else + make_add_instruction "$f" "$updatemanifestv3" + fi + + dir=$(dirname "$f") + mkdir -p "$workdir/$dir" + $XZ $XZ_OPT --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f" + copy_perm "$targetdir/$f" "$workdir/$f" + + targetfiles="$targetfiles \"$f\"" +done + +# Append remove instructions for any dead files. +notice "" +notice "Adding file and directory remove instructions from file 'removed-files'" +append_remove_instructions "$targetdir" "$updatemanifestv3" + +$XZ $XZ_OPT --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3" + +eval "$MAR -C \"$workdir\" -c output.mar $targetfiles" +mv -f "$workdir/output.mar" "$archive" + +# cleanup +rm -fr "$workdir" + +notice "" +notice "Finished" +notice "" diff --git a/tools/update-packaging/test/runtests.sh b/tools/update-packaging/test/runtests.sh new file mode 100755 index 0000000000..c66e54eaa6 --- /dev/null +++ b/tools/update-packaging/test/runtests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +echo "testing make_incremental_updates.py" +python ../make_incremental_updates.py -f testpatchfile.txt + +echo "" +echo "diffing ref.mar and test.mar" +./diffmar.sh ref.mar test.mar test + +echo "" +echo "diffing ref-mac.mar and test-mac.mar" +./diffmar.sh ref-mac.mar test-mac.mar test-mac diff --git a/tools/update-packaging/test/testpatchfile.txt b/tools/update-packaging/test/testpatchfile.txt new file mode 100644 index 0000000000..a19c831eb4 --- /dev/null +++ b/tools/update-packaging/test/testpatchfile.txt @@ -0,0 +1,2 @@ +product-1.0.lang.platform.complete.mar,product-2.0.lang.platform.complete.mar,test.mar,"" +product-1.0.lang.mac.complete.mar,product-2.0.lang.mac.complete.mar,test-mac.mar,"" diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js b/tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js new file mode 100644 index 0000000000..3b2aed8e02 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/addFeedPrefs.js @@ -0,0 +1 @@ +this is a new file diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/added.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/added.txt new file mode 100644 index 0000000000..b242c36062 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.bin differ diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..a61ffbb5e0 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/force.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/same.bin b/tools/update-packaging/test/to-mac/Contents/MacOS/same.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/to-mac/Contents/MacOS/same.bin differ diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest b/tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest new file mode 100644 index 0000000000..73364fdca1 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt new file mode 100644 index 0000000000..b242c36062 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..a61ffbb5e0 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt new file mode 100644 index 0000000000..b5f7004cc9 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.bin differ diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest new file mode 100644 index 0000000000..73364fdca1 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/MacOS/{foodir/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/application.ini b/tools/update-packaging/test/to-mac/Contents/Resources/application.ini new file mode 100644 index 0000000000..7bdc78819f --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=2 +BuildID=20130101010101 diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/chrome.manifest b/tools/update-packaging/test/to-mac/Contents/Resources/chrome.manifest new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/precomplete b/tools/update-packaging/test/to-mac/Contents/Resources/precomplete new file mode 100644 index 0000000000..7af8bfd769 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/precomplete @@ -0,0 +1,33 @@ +remove "Contents/MacOS/{foodir/update.manifest" +remove "Contents/MacOS/{foodir/same.txt" +remove "Contents/MacOS/{foodir/same.bin" +remove "Contents/MacOS/{foodir/readme.txt" +remove "Contents/MacOS/{foodir/force.txt" +remove "Contents/MacOS/{foodir/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/{foodir/added.txt" +remove "Contents/MacOS/update.manifest" +remove "Contents/MacOS/searchplugins/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/searchplugins/added/file.txt" +remove "Contents/MacOS/same.txt" +remove "Contents/MacOS/same.bin" +remove "Contents/MacOS/removed-files" +remove "Contents/MacOS/readme.txt" +remove "Contents/MacOS/force.txt" +remove "Contents/MacOS/extensions/diff/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/extensions/added/file.txt" +remove "Contents/MacOS/diff-patch-larger-than-file.txt" +remove "Contents/MacOS/diff-patch-larger-than-file.bin" +remove "Contents/MacOS/application.ini" +remove "Contents/MacOS/added.txt" +remove "Contents/MacOS/addFeedPrefs.js" +remove "Contents/Resources/precomplete" +rmdir "Contents/MacOS/{foodir/" +rmdir "Contents/MacOS/searchplugins/diff/" +rmdir "Contents/MacOS/searchplugins/added/" +rmdir "Contents/MacOS/searchplugins/" +rmdir "Contents/MacOS/extensions/diff/" +rmdir "Contents/MacOS/extensions/added/" +rmdir "Contents/MacOS/extensions/" +rmdir "Contents/MacOS/" +rmdir "Contents/Resources/" +rmdir "Contents/" diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/readme.txt b/tools/update-packaging/test/to-mac/Contents/Resources/readme.txt new file mode 100644 index 0000000000..b5f7004cc9 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/removed-files b/tools/update-packaging/test/to-mac/Contents/Resources/removed-files new file mode 100644 index 0000000000..a756cc560b --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/removed-files @@ -0,0 +1,14 @@ +Contents/Resources/removed1.txt +Contents/MacOS/removed2.bin +Contents/MacOS/recursivedir/meh/* +Contents/MacOS/removed3-foo.txt +Contents/Resources/dir/ +Contents/MacOS/this file has spaces +Contents/MacOS/notherdir/ + + +Contents/Resources/extra-spaces + +Contents/MacOS/lastFile + + diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/same.txt b/tools/update-packaging/test/to-mac/Contents/Resources/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini b/tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js new file mode 100644 index 0000000000..d6ada4591d --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from partial file diff --git a/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to-mac/Contents/Resources/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/addFeedPrefs.js b/tools/update-packaging/test/to/addFeedPrefs.js new file mode 100644 index 0000000000..3b2aed8e02 --- /dev/null +++ b/tools/update-packaging/test/to/addFeedPrefs.js @@ -0,0 +1 @@ +this is a new file diff --git a/tools/update-packaging/test/to/added.txt b/tools/update-packaging/test/to/added.txt new file mode 100644 index 0000000000..b242c36062 --- /dev/null +++ b/tools/update-packaging/test/to/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to/application.ini b/tools/update-packaging/test/to/application.ini new file mode 100644 index 0000000000..7bdc78819f --- /dev/null +++ b/tools/update-packaging/test/to/application.ini @@ -0,0 +1,5 @@ +[App] +Vendor=Mozilla +Name=MarTest +Version=2 +BuildID=20130101010101 diff --git a/tools/update-packaging/test/to/chrome.manifest b/tools/update-packaging/test/to/chrome.manifest new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/update-packaging/test/to/diff-patch-larger-than-file.bin b/tools/update-packaging/test/to/diff-patch-larger-than-file.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/to/diff-patch-larger-than-file.bin differ diff --git a/tools/update-packaging/test/to/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..a61ffbb5e0 --- /dev/null +++ b/tools/update-packaging/test/to/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to/distribution/extensions/added/file.txt b/tools/update-packaging/test/to/distribution/extensions/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to/distribution/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to/distribution/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to/extensions/added/file.txt b/tools/update-packaging/test/to/extensions/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to/extensions/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to/force.txt b/tools/update-packaging/test/to/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/precomplete b/tools/update-packaging/test/to/precomplete new file mode 100644 index 0000000000..c2700dd972 --- /dev/null +++ b/tools/update-packaging/test/to/precomplete @@ -0,0 +1,30 @@ +remove "{foodir/update.manifest" +remove "{foodir/same.txt" +remove "{foodir/same.bin" +remove "{foodir/readme.txt" +remove "{foodir/force.txt" +remove "{foodir/diff-patch-larger-than-file.txt" +remove "{foodir/added.txt" +remove "update.manifest" +remove "searchplugins/diff/diff-patch-larger-than-file.txt" +remove "searchplugins/added/file.txt" +remove "same.txt" +remove "same.bin" +remove "removed-files" +remove "readme.txt" +remove "precomplete" +remove "force.txt" +remove "extensions/diff/diff-patch-larger-than-file.txt" +remove "extensions/added/file.txt" +remove "diff-patch-larger-than-file.txt" +remove "diff-patch-larger-than-file.bin" +remove "application.ini" +remove "added.txt" +remove "addFeedPrefs.js" +rmdir "{foodir/" +rmdir "searchplugins/diff/" +rmdir "searchplugins/added/" +rmdir "searchplugins/" +rmdir "extensions/diff/" +rmdir "extensions/added/" +rmdir "extensions/" diff --git a/tools/update-packaging/test/to/readme.txt b/tools/update-packaging/test/to/readme.txt new file mode 100644 index 0000000000..b5f7004cc9 --- /dev/null +++ b/tools/update-packaging/test/to/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to/removed-files b/tools/update-packaging/test/to/removed-files new file mode 100644 index 0000000000..4fdfff7fd5 --- /dev/null +++ b/tools/update-packaging/test/to/removed-files @@ -0,0 +1,14 @@ +removed1.txt +removed2.bin +recursivedir/meh/* +removed3-foo.txt +dir/ +this file has spaces +notherdir/ + + +extra-spaces + +lastFile + + diff --git a/tools/update-packaging/test/to/same.bin b/tools/update-packaging/test/to/same.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/to/same.bin differ diff --git a/tools/update-packaging/test/to/same.txt b/tools/update-packaging/test/to/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/searchplugins/added/file.txt b/tools/update-packaging/test/to/searchplugins/added/file.txt new file mode 100644 index 0000000000..4bbc6747ec --- /dev/null +++ b/tools/update-packaging/test/to/searchplugins/added/file.txt @@ -0,0 +1 @@ +extfile diff --git a/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..b779d9648a --- /dev/null +++ b/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +to file diff --git a/tools/update-packaging/test/to/update-settings.ini b/tools/update-packaging/test/to/update-settings.ini new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/to/update-settings.ini @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/to/update.manifest b/tools/update-packaging/test/to/update.manifest new file mode 100644 index 0000000000..73364fdca1 --- /dev/null +++ b/tools/update-packaging/test/to/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-packaging/test/to/{foodir/added.txt b/tools/update-packaging/test/to/{foodir/added.txt new file mode 100644 index 0000000000..b242c36062 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/added.txt @@ -0,0 +1 @@ +added file diff --git a/tools/update-packaging/test/to/{foodir/channel-prefs.js b/tools/update-packaging/test/to/{foodir/channel-prefs.js new file mode 100644 index 0000000000..5fa6a9909e --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/channel-prefs.js @@ -0,0 +1 @@ +add-if-not from complete file diff --git a/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt b/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt new file mode 100644 index 0000000000..a61ffbb5e0 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt @@ -0,0 +1 @@ +file to diff --git a/tools/update-packaging/test/to/{foodir/force.txt b/tools/update-packaging/test/to/{foodir/force.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/force.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/{foodir/readme.txt b/tools/update-packaging/test/to/{foodir/readme.txt new file mode 100644 index 0000000000..b5f7004cc9 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/readme.txt @@ -0,0 +1 @@ +This to file should be ignored diff --git a/tools/update-packaging/test/to/{foodir/same.bin b/tools/update-packaging/test/to/{foodir/same.bin new file mode 100644 index 0000000000..a9ee7258cc Binary files /dev/null and b/tools/update-packaging/test/to/{foodir/same.bin differ diff --git a/tools/update-packaging/test/to/{foodir/same.txt b/tools/update-packaging/test/to/{foodir/same.txt new file mode 100644 index 0000000000..0ed0d50124 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/same.txt @@ -0,0 +1 @@ +file is same diff --git a/tools/update-packaging/test/to/{foodir/update.manifest b/tools/update-packaging/test/to/{foodir/update.manifest new file mode 100644 index 0000000000..73364fdca1 --- /dev/null +++ b/tools/update-packaging/test/to/{foodir/update.manifest @@ -0,0 +1 @@ +to file shouldn't go in update diff --git a/tools/update-programs/README b/tools/update-programs/README new file mode 100644 index 0000000000..35de17eb8f --- /dev/null +++ b/tools/update-programs/README @@ -0,0 +1,19 @@ +This directory defines a build project for focused work on the "update +programs": programs owned or maintained by the Install/Update team +that are standalone binaries (i.e., not part of the Firefox binary +proper). + +To use this build project, prepare a minimal mozconfig with +``` +ac_add_options --enable-project=tools/update-programs +``` + +Depending on the mozconfig options and host and target OS, some of the +following will be built: + +1. the maintenance service (when `--enable-maintenance-service`); +2. the updater binary (when `MOZ_UPDATER=1`); +3. the Windows Default Browser Agent (when `--enable-default-browser-agent`); + +Packaging the installer and uninstaller is not yet supported: instead, +use an (artifact) build with `--enable-project=browser`. diff --git a/tools/update-programs/app.mozbuild b/tools/update-programs/app.mozbuild new file mode 100644 index 0000000000..330d9b11ec --- /dev/null +++ b/tools/update-programs/app.mozbuild @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +if CONFIG['MOZ_MAINTENANCE_SERVICE'] or \ + CONFIG['MOZ_UPDATE_AGENT'] or \ + CONFIG['MOZ_UPDATER']: + DIRS += [ + '/toolkit/mozapps/update/common', + ] + +if CONFIG['MOZ_DEFAULT_BROWSER_AGENT']: + DIRS += [ + '/toolkit/components/jsoncpp/src/lib_json', + '/toolkit/mozapps/defaultagent', + ] + +if CONFIG['MOZ_MAINTENANCE_SERVICE']: + DIRS += [ + '/toolkit/components/maintenanceservice' + ] + +if CONFIG['MOZ_UPDATER']: + # NSS (and NSPR). + DIRS += [ + '/modules/xz-embedded', + '/config/external/nspr', + '/config/external/sqlite', + '/config/external/zlib', + '/memory', + '/mfbt', + '/mozglue', + '/security', + ] + + # The signing related bits of libmar depend on NSS. + DIRS += [ + '/modules/libmar', + '/other-licenses/bsdiff', + '/toolkit/mozapps/update/updater/bspatch', + '/toolkit/mozapps/update/updater', + ] + +# Expose specific non-XPCOM headers when building standalone. +if not CONFIG['MOZ_UPDATER']: + # When building the updater, we build /mozglue, which includes this. + EXPORTS.mozilla += [ + '/mozglue/misc/DynamicallyLinkedFunctionPtr.h', + ] + +EXPORTS.mozilla += [ + '/toolkit/xre/CmdLineAndEnvUtils.h', + '/widget/windows/WinHeaderOnlyUtils.h', +] + +EXPORTS += [ + '/xpcom/base/nsAutoRef.h', + '/xpcom/base/nsWindowsHelpers.h', + '/xpcom/string/nsCharTraits.h', + '/xpcom/string/nsUTF8Utils.h', +] diff --git a/tools/update-programs/confvars.sh b/tools/update-programs/confvars.sh new file mode 100644 index 0000000000..66e2a80605 --- /dev/null +++ b/tools/update-programs/confvars.sh @@ -0,0 +1,12 @@ +#! /bin/sh +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +MOZ_APP_VENDOR=Mozilla + +MOZ_BRANDING_DIRECTORY=browser/branding/unofficial +MOZ_APP_ID={ec8030f7-c20a-464f-9b0e-13a3a9e97384} + +# Build the updater by default. Use --disable-updater to not. +MOZ_UPDATER=1 diff --git a/tools/update-programs/moz.configure b/tools/update-programs/moz.configure new file mode 100644 index 0000000000..2b50751905 --- /dev/null +++ b/tools/update-programs/moz.configure @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Spoof a stub of `js/moz.configure` for the included scripts. +@dependable +def js_standalone(): + return False + + +@depends(target) +def fold_libs(target): + return target.os in ("WINNT", "OSX", "Android") + + +set_config("MOZ_FOLD_LIBS", fold_libs) + + +include("../../build/moz.configure/rust.configure") +include("../../build/moz.configure/nspr.configure") +include("../../build/moz.configure/nss.configure") +include("../../build/moz.configure/update-programs.configure") diff --git a/tools/update-verify/README.md b/tools/update-verify/README.md new file mode 100644 index 0000000000..14eb2a5f9a --- /dev/null +++ b/tools/update-verify/README.md @@ -0,0 +1,118 @@ +Mozilla Build Verification Scripts +================================== + +Contents +-------- + +updates -> AUS and update verification + +l10n -> l10n vs. en-US verification + +common -> useful utility scripts + +Update Verification +------------------- + +`verify.sh` + +> Does a low-level check of all advertised MAR files. Expects to have a +> file named all-locales, but does not (yet) handle platform exceptions, so +> these should be removed from the locales file. +> +> Prints errors on both STDOUT and STDIN, the intention is to run the +> script with STDOUT redirected to an output log. If there is not output +> on the console and an exit code of 0 then all tests pass; otherwise one +> or more tests failed. +> +> Does the following: +> +> 1) download update.xml from AUS for a particular release +> 2) download the partial and full mar advertised +> 3) check that the partial and full match the advertised size and sha1sum +> 4) downloads the latest release, and an older release +> 5) applies MAR to the older release, and compares the two releases. +> +> Step 5 is repeated for both the complete and partial MAR. +> +> Expects to have an updates.cfg file, describing all releases to try updating +> from. + +Valid Platforms for AUS +----------------------- +- Linux_x86-gcc3 +- Darwin_Universal-gcc3 +- Linux_x86-gcc3 +- WINNT_x86-msvc +- Darwin_ppc-gcc3 + +--- +Running it locally +================== + +Requirements: +------------- + +- [Docker](https://docs.docker.com/get-docker/) +- [optional | Mac] zstd (`brew install zst`) + +Docker Image +------------ + +1. [Ship-it](https://shipit.mozilla-releng.net/recent) holds the latest builds. +1. Clicking on "Ship task" of latest build will open the task group in +Taskcluster. +1. On the "Name contains" lookup box, search for `release-update-verify-firefox` +and open a `update-verify` task +1. Make note of the `CHANNEL` under Payload. ie: `beta-localtest` +1. Click "See more" under Task Details and open the `docker-image-update-verify` +task. + +Download the image artifact from *docker-image-update-verify* task and load it +manually +``` +zstd -d image.tar.zst +docker image load -i image.tar +``` + +**OR** + +Load docker image using mach and a task +``` +# Replace TASK-ID with the ID of a docker-image-update-verify task +./mach taskcluster-load-image --task-id= +``` + +Update Verify Config +-------------------- + +1. Open Taskcluster Task Group +1. Search for `update-verify-config` and open the task +1. Under Artifacts, download `update-verify.cfg` file + +Run Docker +---------- + +To run the container interactively: +> Replace `` with gecko repository path on local host
+> Replace `` with path to `update-verify.cfg` file on local host. +ie.: `~/Downloads/update-verify.cfg` +> Replace `` with value from `update-verify` task (Docker steps) + +``` +docker run \ + -it \ + --rm \ + -e CHANNEL=beta-localtest \ + -e MOZ_FETCHES_DIR=/builds/worker/fetches \ + -e MOZBUILD_STATE_PATH=/builds/worker/.mozbuild \ + -v :/builds/worker/fetches/update-verify.cfg + -v :/builds/worker/checkouts/gecko \ + -w /builds/worker/checkouts/gecko \ + update-verify +``` +> Note that `MOZ_FETCHES_DIR` here is different from what is used in production. + +`total-chunks` and `this-chunk` refer to the number of lines in `update-verify.cfg` +``` +./tools/update-verify/scripts/chunked-verify.sh --total-chunks=228 --this-chunk=4 +``` diff --git a/tools/update-verify/python/util/__init__.py b/tools/update-verify/python/util/__init__.py new file mode 100644 index 0000000000..c580d191c1 --- /dev/null +++ b/tools/update-verify/python/util/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/tools/update-verify/python/util/commands.py b/tools/update-verify/python/util/commands.py new file mode 100644 index 0000000000..e53464e6f8 --- /dev/null +++ b/tools/update-verify/python/util/commands.py @@ -0,0 +1,57 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Functions for running commands""" + +import logging +import os +import subprocess +import time + +import six + +log = logging.getLogger(__name__) + + +# timeout message, used in TRANSIENT_HG_ERRORS and in tests. +TERMINATED_PROCESS_MSG = "timeout, process terminated" + + +def log_cmd(cmd, **kwargs): + # cwd is special in that we always want it printed, even if it's not + # explicitly chosen + kwargs = kwargs.copy() + if "cwd" not in kwargs: + kwargs["cwd"] = os.getcwd() + log.info("command: START") + log.info("command: %s" % subprocess.list2cmdline(cmd)) + for key, value in six.iteritems(kwargs): + log.info("command: %s: %s", key, str(value)) + + +def merge_env(env): + new_env = os.environ.copy() + new_env.update(env) + return new_env + + +def run_cmd(cmd, **kwargs): + """Run cmd (a list of arguments). Raise subprocess.CalledProcessError if + the command exits with non-zero. If the command returns successfully, + return 0.""" + log_cmd(cmd, **kwargs) + # We update this after logging because we don't want all of the inherited + # env vars muddling up the output + if "env" in kwargs: + kwargs["env"] = merge_env(kwargs["env"]) + try: + t = time.monotonic() + log.info("command: output:") + return subprocess.check_call(cmd, **kwargs) + except subprocess.CalledProcessError: + log.info("command: ERROR", exc_info=True) + raise + finally: + elapsed = time.monotonic() - t + log.info("command: END (%.2fs elapsed)\n", elapsed) diff --git a/tools/update-verify/release/common/cached_download.sh b/tools/update-verify/release/common/cached_download.sh new file mode 100644 index 0000000000..7cb3c42f8d --- /dev/null +++ b/tools/update-verify/release/common/cached_download.sh @@ -0,0 +1,40 @@ +# this library works like a wrapper around wget, to allow downloads to be cached +# so that if later the same url is retrieved, the entry from the cache will be +# returned. + +pushd `dirname $0` &>/dev/null +cache_dir="$(pwd)/cache" +popd &>/dev/null + +# Deletes all files in the cache directory +# We don't support folders or .dot(hidden) files +# By not deleting the cache directory, it allows us to use Docker tmpfs mounts, +# which are the only workaround to poor mount r/w performance on MacOS +# Reference: https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288 +clear_cache () { + rm -rf "${cache_dir}/*" +} + +# download method - you pass a filename to save the file under, and the url to call +cached_download () { + local output_file="${1}" + local url="${2}" + + if fgrep -x "${url}" "${cache_dir}/urls.list" >/dev/null; then + echo "Retrieving '${url}' from cache..." + local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')" + cp "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" "${output_file}" + else + echo "Downloading '${url}' and placing in cache..." + rm -f "${output_file}" + $retry wget -O "${output_file}" --progress=dot:giga --server-response "${url}" 2>&1 + local exit_code=$? + if [ "${exit_code}" == 0 ]; then + echo "${url}" >> "${cache_dir}/urls.list" + local line_number="$(fgrep -nx "${url}" "${cache_dir}/urls.list" | sed 's/:.*//')" + cp "${output_file}" "${cache_dir}/obj_$(printf "%05d\n" "${line_number}").cache" + else + return "${exit_code}" + fi + fi +} diff --git a/tools/update-verify/release/common/check_updates.sh b/tools/update-verify/release/common/check_updates.sh new file mode 100644 index 0000000000..6479f2f1f1 --- /dev/null +++ b/tools/update-verify/release/common/check_updates.sh @@ -0,0 +1,124 @@ +check_updates () { + # called with 10 args - platform, source package, target package, update package, old updater boolean, + # a path to the updater binary to use for the tests, a file to write diffs to, the update channel, + # update-settings.ini values, and a flag to indicate the target is dep-signed + update_platform=$1 + source_package=$2 + target_package=$3 + locale=$4 + use_old_updater=$5 + updater=$6 + diff_file=$7 + channel=$8 + mar_channel_IDs=$9 + update_to_dep=${10} + + # cleanup + rm -rf source/* + rm -rf target/* + + unpack_build $update_platform source "$source_package" $locale '' $mar_channel_IDs + if [ "$?" != "0" ]; then + echo "FAILED: cannot unpack_build $update_platform source $source_package" + return 1 + fi + unpack_build $update_platform target "$target_package" $locale + if [ "$?" != "0" ]; then + echo "FAILED: cannot unpack_build $update_platform target $target_package" + return 1 + fi + + case $update_platform in + Darwin_ppc-gcc | Darwin_Universal-gcc3 | Darwin_x86_64-gcc3 | Darwin_x86-gcc3-u-ppc-i386 | Darwin_x86-gcc3-u-i386-x86_64 | Darwin_x86_64-gcc3-u-i386-x86_64 | Darwin_aarch64-gcc3) + platform_dirname="*.app" + ;; + WINNT*) + platform_dirname="bin" + ;; + Linux_x86-gcc | Linux_x86-gcc3 | Linux_x86_64-gcc3) + platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'` + ;; + esac + + if [ -f update/update.status ]; then rm update/update.status; fi + if [ -f update/update.log ]; then rm update/update.log; fi + + if [ -d source/$platform_dirname ]; then + if [ `uname | cut -c-5` == "MINGW" ]; then + # windows + # change /c/path/to/pwd to c:\\path\\to\\pwd + four_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\\\,g') + two_backslash_pwd=$(echo $PWD | sed -e 's,^/\([a-zA-Z]\)/,\1:/,' | sed -e 's,/,\\,g') + cwd="$two_backslash_pwd\\source\\$platform_dirname" + update_abspath="$two_backslash_pwd\\update" + else + # not windows + # use ls here, because mac uses *.app, and we need to expand it + cwd=$(ls -d $PWD/source/$platform_dirname) + update_abspath="$PWD/update" + fi + + cd_dir=$(ls -d ${PWD}/source/${platform_dirname}) + cd "${cd_dir}" + set -x + "$updater" "$update_abspath" "$cwd" "$cwd" 0 + set +x + cd ../.. + else + echo "TEST-UNEXPECTED-FAIL: no dir in source/$platform_dirname" + return 1 + fi + + cat update/update.log + update_status=`cat update/update.status` + + if [ "$update_status" != "succeeded" ] + then + echo "TEST-UNEXPECTED-FAIL: update status was not successful: $update_status" + return 1 + fi + + # If we were testing an OS X mar on Linux, the unpack step copied the + # precomplete file from Contents/Resources to the root of the install + # to ensure the Linux updater binary could find it. However, only the + # precomplete file in Contents/Resources was updated, which means + # the copied version in the root of the install will usually have some + # differences between the source and target. To prevent this false + # positive from failing the tests, we simply remove it before diffing. + # The precomplete file in Contents/Resources is still diffed, so we + # don't lose any coverage by doing this. + cd `echo "source/$platform_dirname"` + if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]] + then + rm "precomplete" + fi + cd ../.. + cd `echo "target/$platform_dirname"` + if [[ -f "Contents/Resources/precomplete" && -f "precomplete" ]] + then + rm "precomplete" + fi + cd ../.. + + # If we are testing an OSX mar to update from a production-signed/notarized + # build to a dep-signed one, ignore Contents/CodeResources which won't be + # present in the target, to avoid spurious failures + if ${update_to_dep}; then + ignore_coderesources=--ignore-missing=Contents/CodeResources + else + ignore_coderesources= + fi + + ../compare-directories.py source/${platform_dirname} target/${platform_dirname} ${channel} ${ignore_coderesources} > "${diff_file}" + diffErr=$? + cat "${diff_file}" + if [ $diffErr == 2 ] + then + echo "TEST-UNEXPECTED-FAIL: differences found after update" + return 1 + elif [ $diffErr != 0 ] + then + echo "TEST-UNEXPECTED-FAIL: unknown error from diff: $diffErr" + return 3 + fi +} diff --git a/tools/update-verify/release/common/download_builds.sh b/tools/update-verify/release/common/download_builds.sh new file mode 100644 index 0000000000..e279c808db --- /dev/null +++ b/tools/update-verify/release/common/download_builds.sh @@ -0,0 +1,36 @@ +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3" + +download_builds() { + # cleanup + mkdir -p downloads/ + rm -rf downloads/* + + source_url="$1" + target_url="$2" + + if [ -z "$source_url" ] || [ -z "$target_url" ] + then + "download_builds usage: " + exit 1 + fi + + for url in "$source_url" "$target_url" + do + source_file=`basename "$url"` + if [ -f "$source_file" ]; then rm "$source_file"; fi + cd downloads + if [ -f "$source_file" ]; then rm "$source_file"; fi + cached_download "${source_file}" "${url}" + status=$? + if [ $status != 0 ]; then + echo "TEST-UNEXPECTED-FAIL: Could not download source $source_file from $url" + echo "skipping.." + cd ../ + return $status + fi + cd ../ + done +} diff --git a/tools/update-verify/release/common/download_mars.sh b/tools/update-verify/release/common/download_mars.sh new file mode 100644 index 0000000000..d2dab107d2 --- /dev/null +++ b/tools/update-verify/release/common/download_mars.sh @@ -0,0 +1,105 @@ +download_mars () { + update_url="$1" + only="$2" + test_only="$3" + to_build_id="$4" + to_app_version="$5" + to_display_version="$6" + + max_tries=5 + try=1 + # retrying until we get offered an update + while [ "$try" -le "$max_tries" ]; do + echo "Using $update_url" + # retrying until AUS gives us any response at all + cached_download update.xml "${update_url}" + + echo "Got this response:" + cat update.xml + # If the first line after is then we have an + # empty snippet. Otherwise we're done + if [ "$(grep -A1 '' update.xml | tail -1)" != "" ]; then + break; + fi + echo "Empty response, sleeping" + sleep 5 + try=$(($try+1)) + done + + echo; echo; # padding + + update_line=`fgrep " found for $update_url" + return 1 + fi + command=`echo $update_line | sed -e 's/^.*.*$::' -e 's:\&:\&:g'` + eval "export $command" + + if [ ! -z "$to_build_id" -a "$buildID" != "$to_build_id" ]; then + echo "TEST-UNEXPECTED-FAIL: expected buildID $to_build_id does not match actual $buildID" + return 1 + fi + + if [ ! -z "$to_display_version" -a "$displayVersion" != "$to_display_version" ]; then + echo "TEST-UNEXPECTED-FAIL: expected displayVersion $to_display_version does not match actual $displayVersion" + return 1 + fi + + if [ ! -z "$to_app_version" -a "$appVersion" != "$to_app_version" ]; then + echo "TEST-UNEXPECTED-FAIL: expected appVersion $to_app_version does not match actual $appVersion" + return 1 + fi + + mkdir -p update/ + if [ -z $only ]; then + only="partial complete" + fi + for patch_type in $only + do + line=`fgrep "patch type=\"$patch_type" update.xml` + grep_rv=$? + + if [ 0 -ne $grep_rv ]; then + echo "TEST-UNEXPECTED-FAIL: no $patch_type update found for $update_url" + return 1 + fi + + command=`echo $line | sed -e 's/^.*.*$::' -e 's:\&:\&:g'` + eval "export $command" + + if [ "$test_only" == "1" ] + then + echo "Testing $URL" + curl -s -I -L $URL + return + else + cached_download "update/${patch_type}.mar" "${URL}" + fi + if [ "$?" != 0 ]; then + echo "Could not download $patch_type!" + echo "from: $URL" + fi + actual_size=`perl -e "printf \"%d\n\", (stat(\"update/$patch_type.mar\"))[7]"` + actual_hash=`openssl dgst -$hashFunction update/$patch_type.mar | sed -e 's/^.*= //'` + + if [ $actual_size != $size ]; then + echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong size" + echo "TEST-UNEXPECTED-FAIL: update.xml size: $size" + echo "TEST-UNEXPECTED-FAIL: actual size: $actual_size" + return 1 + fi + + if [ $actual_hash != $hashValue ]; then + echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong hash" + echo "TEST-UNEXPECTED-FAIL: update.xml hash: $hashValue" + echo "TEST-UNEXPECTED-FAIL: actual hash: $actual_hash" + return 1 + fi + + cp update/$patch_type.mar update/update.mar + echo $actual_size > update/$patch_type.size + + done +} diff --git a/tools/update-verify/release/common/installdmg.ex b/tools/update-verify/release/common/installdmg.ex new file mode 100755 index 0000000000..08bcf9a201 --- /dev/null +++ b/tools/update-verify/release/common/installdmg.ex @@ -0,0 +1,45 @@ +#!/usr/bin/expect +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is Mozilla Corporation Code. +# +# The Initial Developer of the Original Code is +# Clint Talbert. +# Portions created by the Initial Developer are Copyright (C) 2007 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Armen Zambrano Gasparnian +# Axel Hecht +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +#send_user $argv +spawn hdiutil attach -readonly -mountroot /tmp -private -noautoopen [lindex $argv 0] +expect { +"byte" {send "G"; exp_continue} +"END" {send "\r"; exp_continue} +"Y/N?" {send "Y\r"; exp_continue} +} diff --git a/tools/update-verify/release/common/unpack-diskimage.sh b/tools/update-verify/release/common/unpack-diskimage.sh new file mode 100755 index 0000000000..b647a69c4d --- /dev/null +++ b/tools/update-verify/release/common/unpack-diskimage.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the installdmg.sh script from taols utilities +# +# The Initial Developer of the Original Code is +# Mozilla Corporation. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Chris AtLee +# Robert Kaiser +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +# Unpack a disk image to a specified target folder +# +# Usage: unpack-diskimage +# +# + +DMG_PATH=$1 +MOUNTPOINT=$2 +TARGETPATH=$3 +LOGFILE=unpack.output + +# How long to wait before giving up waiting for the mount to fininsh +TIMEOUT=90 + +# If the mount point already exists, then the previous run may not have cleaned +# up properly. We should try to umount and remove the its directory. +if [ -d $MOUNTPOINT ]; then + echo "$MOUNTPOINT already exists, trying to clean up" + hdiutil detach $MOUNTPOINT -force + rm -rdfv $MOUNTPOINT +fi + +# Install an on-exit handler that will unmount and remove the '$MOUNTPOINT' directory +trap "{ if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; rm -rdfv $MOUNTPOINT; fi; }" EXIT + +mkdir -p $MOUNTPOINT + +hdiutil attach -verbose -noautoopen -mountpoint $MOUNTPOINT "$DMG_PATH" &> $LOGFILE +# Wait for files to show up +# hdiutil uses a helper process, diskimages-helper, which isn't always done its +# work by the time hdiutil exits. So we wait until something shows up in the +# mount point directory. +i=0 +while [ "$(echo $MOUNTPOINT/*)" == "$MOUNTPOINT/*" ]; do + if [ $i -gt $TIMEOUT ]; then + echo "No files found, exiting" + exit 1 + fi + sleep 1 + i=$(expr $i + 1) +done +# Now we can copy everything out of the $MOUNTPOINT directory into the target directory +rsync -av $MOUNTPOINT/* $MOUNTPOINT/.DS_Store $MOUNTPOINT/.background $MOUNTPOINT/.VolumeIcon.icns $TARGETPATH/ > $LOGFILE +# sometimes hdiutil fails with "Resource busy" +hdiutil detach $MOUNTPOINT || { sleep 10; \ + if [ -d $MOUNTPOINT ]; then hdiutil detach $MOUNTPOINT -force; fi; } +i=0 +while [ "$(echo $MOUNTPOINT/*)" != "$MOUNTPOINT/*" ]; do + if [ $i -gt $TIMEOUT ]; then + echo "Cannot umount, exiting" + exit 1 + fi + sleep 1 + i=$(expr $i + 1) +done +rm -rdf $MOUNTPOINT diff --git a/tools/update-verify/release/common/unpack.sh b/tools/update-verify/release/common/unpack.sh new file mode 100755 index 0000000000..3249936493 --- /dev/null +++ b/tools/update-verify/release/common/unpack.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +function cleanup() { + hdiutil detach ${DEV_NAME} || + { sleep 5 && hdiutil detach ${DEV_NAME} -force; }; + return $1 && $?; +}; + +unpack_build () { + unpack_platform="$1" + dir_name="$2" + pkg_file="$3" + locale=$4 + unpack_jars=$5 + update_settings_string=$6 + + if [ ! -f "$pkg_file" ]; then + return 1 + fi + mkdir -p $dir_name + pushd $dir_name > /dev/null + case $unpack_platform in + # $unpack_platform is either + # - a balrog platform name (from testing/mozharness/scripts/release/update-verify-config-creator.py) + # - a simple platform name (from tools/update-verify/release/updates/verify.sh) + mac|Darwin_*) + os=`uname` + # How we unpack a dmg differs depending on which platform we're on. + if [[ "$os" == "Darwin" ]] + then + cd ../ + echo "installing $pkg_file" + ../common/unpack-diskimage.sh "$pkg_file" mnt $dir_name + else + 7z x ../"$pkg_file" > /dev/null + if [ `ls -1 | wc -l` -ne 1 ] + then + echo "Couldn't find .app package" + return 1 + fi + unpack_dir=$(ls -1) + unpack_dir=$(ls -d "${unpack_dir}") + mv "${unpack_dir}"/*.app . + rm -rf "${unpack_dir}" + appdir=$(ls -1) + appdir=$(ls -d *.app) + # The updater guesses the location of these files based on + # its own target architecture, not the mar. If we're not + # unpacking mac-on-mac, we need to copy them so it can find + # them. It's important to copy (and not move), because when + # we diff the installer vs updated build afterwards, the + # installer version will have them in their original place. + cp "${appdir}/Contents/Resources/update-settings.ini" "${appdir}/update-settings.ini" + cp "${appdir}/Contents/Resources/precomplete" "${appdir}/precomplete" + fi + update_settings_file="${appdir}/update-settings.ini" + ;; + win32|WINNT_*) + 7z x ../"$pkg_file" > /dev/null + if [ -d localized ] + then + mkdir bin/ + cp -rp nonlocalized/* bin/ + cp -rp localized/* bin/ + rm -rf nonlocalized + rm -rf localized + if [ $(find optional/ | wc -l) -gt 1 ] + then + cp -rp optional/* bin/ + rm -rf optional + fi + elif [ -d core ] + then + mkdir bin/ + cp -rp core/* bin/ + rm -rf core + else + for file in *.xpi + do + unzip -o $file > /dev/null + done + unzip -o ${locale}.xpi > /dev/null + fi + update_settings_file='bin/update-settings.ini' + ;; + linux|Linux_*) + if `echo $pkg_file | grep -q "tar.gz"` + then + tar xfz ../"$pkg_file" > /dev/null + elif `echo $pkg_file | grep -q "tar.bz2"` + then + tar xfj ../"$pkg_file" > /dev/null + else + echo "Unknown package type for file: $pkg_file" + exit 1 + fi + update_settings_file=`echo $product | tr '[A-Z]' '[a-z]'`'/update-settings.ini' + ;; + *) + echo "Unknown platform to unpack: $unpack_platform" + exit 1 + esac + + if [ ! -z $unpack_jars ]; then + for f in `find . -name '*.jar' -o -name '*.ja'`; do + unzip -o "$f" -d "$f.dir" > /dev/null + done + fi + + if [ ! -z $update_settings_string ]; then + echo "Modifying update-settings.ini" + cat "${update_settings_file}" | sed -e "s/^ACCEPTED_MAR_CHANNEL_IDS.*/ACCEPTED_MAR_CHANNEL_IDS=${update_settings_string}/" > "${update_settings_file}.new" + diff -u "${update_settings_file}" "${update_settings_file}.new" + echo " " + rm "${update_settings_file}" + mv "${update_settings_file}.new" "${update_settings_file}" + fi + + popd > /dev/null + +} diff --git a/tools/update-verify/release/compare-directories.py b/tools/update-verify/release/compare-directories.py new file mode 100755 index 0000000000..a45e78d62f --- /dev/null +++ b/tools/update-verify/release/compare-directories.py @@ -0,0 +1,273 @@ +#! /usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import difflib +import hashlib +import logging +import os +import sys + +""" Define the transformations needed to make source + update == target + +Required: +The files list describes the files which a transform may be used on. +The 'side' is one of ('source', 'target') and defines where each transform is applied +The 'channel_prefix' list controls which channels a transform may be used for, where a value of +'beta' means all of beta, beta-localtest, beta-cdntest, etc. + +One or more: +A 'deletion' specifies a start of line to match on, removing the whole line +A 'substitution' is a list of full string to match and its replacement +""" +TRANSFORMS = [ + # channel-prefs.js + { + # preprocessor comments, eg //@line 6 "/builds/worker/workspace/... + # this can be removed once each channel has a watershed above 59.0b2 (from bug 1431342) + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["aurora", "beta", "release", "esr"], + "side": "source", + "deletion": '//@line 6 "', + }, + { + # updates from a beta to an RC build, the latter specifies the release channel + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["beta"], + "side": "target", + "substitution": [ + 'pref("app.update.channel", "release");\n', + 'pref("app.update.channel", "beta");\n', + ], + }, + { + # updates from an RC to a beta build + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["beta"], + "side": "source", + "substitution": [ + 'pref("app.update.channel", "release");\n', + 'pref("app.update.channel", "beta");\n', + ], + }, + { + # Warning comments from bug 1576546 + # When updating from a pre-70.0 build to 70.0+ this removes the new comments in + # the target side. In the 70.0+ --> 70.0+ case with a RC we won't need this, and + # the channel munging above will make channel-prefs.js identical, allowing the code + # to break before applying this transform. + "files": [ + "defaults/pref/channel-prefs.js", + "Contents/Resources/defaults/pref/channel-prefs.js", + ], + "channel_prefix": ["aurora", "beta", "release", "esr"], + "side": "target", + "deletion": "//", + }, + # update-settings.ini + { + # updates from a beta to an RC build, the latter specifies the release channel + # on mac, we actually have both files. The second location is the real + # one but we copy to the first to run the linux64 updater + "files": ["update-settings.ini", "Contents/Resources/update-settings.ini"], + "channel_prefix": ["beta"], + "side": "target", + "substitution": [ + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n", + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n", + ], + }, + { + # updates from an RC to a beta build + # on mac, we only need to modify the legit file this time. unpack_build + # handles the copy for the updater in both source and target + "files": ["Contents/Resources/update-settings.ini"], + "channel_prefix": ["beta"], + "side": "source", + "substitution": [ + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release\n", + "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release\n", + ], + }, +] + + +def walk_dir(path): + all_files = [] + all_dirs = [] + + for root, dirs, files in os.walk(path): + all_dirs.extend([os.path.join(root, d) for d in dirs]) + all_files.extend([os.path.join(root, f) for f in files]) + + # trim off directory prefix for easier comparison + all_dirs = [d[len(path) + 1 :] for d in all_dirs] + all_files = [f[len(path) + 1 :] for f in all_files] + + return all_dirs, all_files + + +def compare_listings( + source_list, target_list, label, source_dir, target_dir, ignore_missing=None +): + obj1 = set(source_list) + obj2 = set(target_list) + difference_found = False + ignore_missing = ignore_missing or () + + left_diff = obj1 - obj2 + if left_diff: + if left_diff - set(ignore_missing): + _log = logging.error + difference_found = True + else: + _log = logging.warning + _log("Ignoring missing files due to ignore_missing") + + _log("{} only in {}:".format(label, source_dir)) + for d in sorted(left_diff): + _log(" {}".format(d)) + + right_diff = obj2 - obj1 + if right_diff: + logging.error("{} only in {}:".format(label, target_dir)) + for d in sorted(right_diff): + logging.error(" {}".format(d)) + difference_found = True + + return difference_found + + +def hash_file(filename): + h = hashlib.sha256() + with open(filename, "rb", buffering=0) as f: + for b in iter(lambda: f.read(128 * 1024), b""): + h.update(b) + return h.hexdigest() + + +def compare_common_files(files, channel, source_dir, target_dir): + difference_found = False + for filename in files: + source_file = os.path.join(source_dir, filename) + target_file = os.path.join(target_dir, filename) + + if os.stat(source_file).st_size != os.stat(target_file).st_size or hash_file( + source_file + ) != hash_file(target_file): + logging.info("Difference found in {}".format(filename)) + file_contents = { + "source": open(source_file).readlines(), + "target": open(target_file).readlines(), + } + + transforms = [ + t + for t in TRANSFORMS + if filename in t["files"] + and channel.startswith(tuple(t["channel_prefix"])) + ] + logging.debug( + "Got {} transform(s) to consider for {}".format( + len(transforms), filename + ) + ) + for transform in transforms: + side = transform["side"] + + if "deletion" in transform: + d = transform["deletion"] + logging.debug( + "Trying deleting lines starting {} from {}".format(d, side) + ) + file_contents[side] = [ + l for l in file_contents[side] if not l.startswith(d) + ] + + if "substitution" in transform: + r = transform["substitution"] + logging.debug("Trying replacement for {} in {}".format(r, side)) + file_contents[side] = [ + l.replace(r[0], r[1]) for l in file_contents[side] + ] + + if file_contents["source"] == file_contents["target"]: + logging.info("Transforms removed all differences") + break + + if file_contents["source"] != file_contents["target"]: + difference_found = True + logging.error( + "{} still differs after transforms, residual diff:".format(filename) + ) + for l in difflib.unified_diff( + file_contents["source"], file_contents["target"] + ): + logging.error(l.rstrip()) + + return difference_found + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + "Compare two directories recursively, with transformations for expected diffs" + ) + parser.add_argument("source", help="Directory containing updated Firefox") + parser.add_argument("target", help="Directory containing expected Firefox") + parser.add_argument("channel", help="Update channel used") + parser.add_argument( + "--verbose", "-v", action="store_true", help="Enable verbose logging" + ) + parser.add_argument( + "--ignore-missing", + action="append", + metavar="", + help="Ignore absence of in the target", + ) + + args = parser.parse_args() + level = logging.INFO + if args.verbose: + level = logging.DEBUG + logging.basicConfig(level=level, format="%(message)s", stream=sys.stdout) + + source = args.source + target = args.target + if not os.path.exists(source) or not os.path.exists(target): + logging.error("Source and/or target directory doesn't exist") + sys.exit(3) + + logging.info("Comparing {} with {}...".format(source, target)) + source_dirs, source_files = walk_dir(source) + target_dirs, target_files = walk_dir(target) + + dir_list_diff = compare_listings( + source_dirs, target_dirs, "Directories", source, target + ) + file_list_diff = compare_listings( + source_files, target_files, "Files", source, target, args.ignore_missing + ) + file_diff = compare_common_files( + set(source_files) & set(target_files), args.channel, source, target + ) + + if file_diff: + # Use status of 2 since python will use 1 if there is an error running the script + sys.exit(2) + elif dir_list_diff or file_list_diff: + # this has traditionally been a WARN, but we don't have files on one + # side anymore so lets FAIL + sys.exit(2) + else: + logging.info("No differences found") diff --git a/tools/update-verify/release/final-verification.sh b/tools/update-verify/release/final-verification.sh new file mode 100755 index 0000000000..879c64697f --- /dev/null +++ b/tools/update-verify/release/final-verification.sh @@ -0,0 +1,519 @@ +#!/bin/bash + +function usage { + log "In the updates subdirectory of the directory this script is in," + log "there are a bunch of config files. You should call this script," + log "passing the names of one or more of those files as parameters" + log "to this script." + log "" + log "This will validate that the update.xml files all exist for the" + log "given config file, and that they report the correct file sizes" + log "for the associated mar files, and that the associated mar files" + log "are available on the update servers." + log "" + log "This script will spawn multiple curl processes to query the" + log "snippets (update.xml file downloads) and the download urls in" + log "parallel. The number of parallel curl processes can be managed" + log "with the -p MAX_PROCS option." + log "" + log "Only the first three bytes of the mar files are downloaded" + log "using curl -r 0-2 option to save time. GET requests are issued" + log "rather than HEAD requests, since Akamai (one of our CDN" + log "partners) caches GET and HEAD requests separately - therefore" + log "they can be out-of-sync, and it is important that we validate" + log "that the GET requests return the expected results." + log "" + log "Please note this script can run on linux and OS X. It has not" + log "been tested on Windows, but may also work. It can be run" + log "locally, and does not require access to the mozilla vpn or" + log "any other special network, since the update servers are" + log "available over the internet. However, it does require an" + log "up-to-date checkout of the tools repository, as the updates/" + log "subfolder changes over time, and reflects the currently" + log "available updates. It makes no changes to the update servers" + log "so there is no harm in running it. It simply generates a" + log "report. However, please try to avoid hammering the update" + log "servers aggressively, e.g. with thousands of parallel" + log "processes. For example, feel free to run the examples below," + log "first making sure that your source code checkout is up-to-" + log "date on your own machine, to get the latest configs in the" + log "updates/ subdirectory." + log "" + log "Usage:" + log " $(basename "${0}") [-p MAX_PROCS] config1 [config2 config3 config4 ...]" + log " $(basename "${0}") -h" + log "" + log "Examples:" + log " 1. $(basename "${0}") -p 128 mozBeta-thunderbird-linux.cfg mozBeta-thunderbird-linux64.cfg" + log " 2. $(basename "${0}") mozBeta-thunderbird-linux64.cfg" +} + +function log { + echo "$(date): ${1}" +} + +# subprocesses don't log in real time, due to synchronisation +# issues which can cause log entries to overwrite each other. +# therefore this function outputs log entries written to +# temporary files on disk, and then deletes them. +function flush_logs { + ls -1rt "${TMPDIR}" | grep '^log\.' | while read LOG + do + cat "${TMPDIR}/${LOG}" + rm "${TMPDIR}/${LOG}" + done +} + +# this function takes an update.xml url as an argument +# and then logs a list of config files and their line +# numbers, that led to this update.xml url being tested +function show_cfg_file_entries { + local update_xml_url="${1}" + cat "${update_xml_urls}" | cut -f1 -d' ' | grep -Fn "${update_xml_url}" | sed 's/:.*//' | while read match_line_no + do + cfg_file="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f3 -d' ')" + cfg_line_no="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f4 -d' ')" + log " ${cfg_file} line ${cfg_line_no}: $(sed -n -e "${cfg_line_no}p" "${cfg_file}")" + done +} + +# this function takes a mar url as an argument and then +# logs information about which update.xml urls referenced +# this mar url, and which config files referenced those +# mar urls - so you have a full understanding of why this +# mar url was ever tested +function show_update_xml_entries { + local mar_url="${1}" + grep -Frl "${mar_url}" "${TMPDIR}" | grep '/update_xml_to_mar\.' | while read update_xml_to_mar + do + mar_size="$(cat "${update_xml_to_mar}" | cut -f2 -d' ')" + update_xml_url="$(cat "${update_xml_to_mar}" | cut -f3 -d' ')" + patch_type="$(cat "${update_xml_to_mar}" | cut -f4 -d' ')" + update_xml_actual_url="$(cat "${update_xml_to_mar}" | cut -f5 -d' ')" + log " ${update_xml_url}" + [ -n "${update_xml_actual_url}" ] && log " which redirected to: ${update_xml_actual_url}" + log " This contained an entry for:" + log " patch type: ${patch_type}" + log " mar size: ${mar_size}" + log " mar url: ${mar_url}" + log " The update.xml url above was retrieved because of the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" | sed 's/ / /' + done +} + +echo -n "$(date): Command called:" +for ((INDEX=0; INDEX<=$#; INDEX+=1)) +do + echo -n " '${!INDEX}'" +done +echo '' +log "From directory: '$(pwd)'" +log '' +log "Parsing arguments..." + +# Max procs lowered in bug 894368 to try to avoid spurious failures +MAX_PROCS=48 +BAD_ARG=0 +BAD_FILE=0 +while getopts p:h OPT +do + case "${OPT}" in + p) MAX_PROCS="${OPTARG}";; + h) usage + exit;; + *) BAD_ARG=1;; + esac +done +shift "$((OPTIND - 1))" + +# invalid option specified +[ "${BAD_ARG}" == 1 ] && exit 66 + +log "Checking one or more config files have been specified..." +if [ $# -lt 1 ] +then + usage + log "ERROR: You must specify one or more config files" + exit 64 +fi + +log "Checking whether MAX_PROCS is a number..." +if ! let x=MAX_PROCS 2>/dev/null +then + usage + log "ERROR: MAX_PROCS must be a number (-p option); you specified '${MAX_PROCS}' - this is not a number." + exit 65 +fi + +# config files are in updates subdirectory below this script +if ! cd "$(dirname "${0}")/updates" 2>/dev/null +then + log "ERROR: Cannot cd into '$(dirname "${0}")/updates' from '$(pwd)'" + exit 68 +fi + +log "Checking specified config files (and downloading them if necessary):" +log '' +configs=() +for file in "${@}" +do + if [[ ${file} == http* ]] + then + log " Downloading config file '${file}'" + cfg=$(mktemp) + curl -fL --retry 5 --compressed "${file}" > "$cfg" + if [ "$?" != 0 ]; then + log "Error downloading config file '${file}'" + BAD_FILE=1 + else + log " * '${file}' ok, downloaded to '${cfg}'" + configs+=($cfg) + fi + elif [ -f "${file}" ] + then + log " * '${file}' ok" + configs+=(${file}) + else + log " * '${file}' missing" + BAD_FILE=1 + fi +done +log '' + +# invalid config specified +if [ "${BAD_FILE}" == 1 ] +then + log "ERROR: Unable to download config file(s) or config files are missing from repo - see above" + exit 67 +fi + +log "All checks completed successfully." +log '' +log "Starting stopwatch..." +log '' +log "Please be aware output will now be buffered up, and only displayed after completion." +log "Therefore do not be alarmed if you see no output for several minutes." +log "See https://bugzilla.mozilla.org/show_bug.cgi?id=863602#c5 for details". +log '' + +START_TIME="$(date +%s)" + +# Create a temporary directory for all temp files, that can easily be +# deleted afterwards. See https://bugzilla.mozilla.org/show_bug.cgi?id=863602 +# to understand why we write everything in distinct temporary files rather +# than writing to standard error/standard out or files shared across +# processes. +# Need to unset TMPDIR first since it affects mktemp behaviour on next line +unset TMPDIR +export TMPDIR="$(mktemp -d -t final_verification.XXXXXXXXXX)" + +# this temporary file will list all update urls that need to be checked, in this format: +# +# e.g. +# https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zh-TW/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64-major.cfg 3 +# https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zu/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64.cfg 7 +# https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/ach/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64-major.cfg 11 +# https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/af/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64.cfg 17 +update_xml_urls="$(mktemp -t update_xml_urls.XXXXXXXXXX)" + +#################################################################################### +# And now a summary of all temp files that will get generated during this process... +# +# 1) mktemp -t failure.XXXXXXXXXX +# +# Each failure will generate a one line temp file, which is a space separated +# output of the error code, and the instance data for the failure. +# e.g. +# +# PATCH_TYPE_MISSING https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/update.xml?force=1 complete https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/default/default/default/update.xml?force=1 +# +# 2) mktemp -t update_xml_to_mar.XXXXXXXXXX +# +# For each mar url referenced in an update.xml file, a temp file will be created to store the +# association between update.xml url and mar url. This is later used (e.g. in function +# show_update_xml_entries) to trace back the update.xml url(s) that led to a mar url being +# tested. It is also used to keep a full list of mar urls to test. +# e.g. +# +# +# +# 3) mktemp -t log.XXXXXXXXXX +# +# For each log message logged by a subprocesses, we will create a temp log file with the +# contents of the log message, since we cannot safely output the log message from the subprocess +# and guarantee that it will be correctly output. By buffering log output in individual log files +# we guarantee that log messages will not interfere with each other. We then flush them when all +# forked subprocesses have completed. +# +# 4) mktemp -t mar_headers.XXXXXXXXXX +# +# We keep a copy of the mar url http headers retrieved in one file per mar url. +# +# 5) mktemp -t update.xml.headers.XXXXXXXXXX +# +# We keep a copy of the update.xml http headers retrieved in one file per update.xml url. +# +# 6) mktemp -t update.xml.XXXXXXXXXX +# +# We keep a copy of each update.xml file retrieved in individual files. +#################################################################################### + + +# generate full list of update.xml urls, followed by patch types, +# as defined in the specified config files - and write into "${update_xml_urls}" file +aus_server="https://aus5.mozilla.org" +for cfg_file in "${configs[@]}" +do + line_no=0 + sed -e 's/localtest/cdntest/' "${cfg_file}" | while read config_line + do + let line_no++ + # to avoid contamination between iterations, reset variables + # each loop in case they are not declared + # aus_server is not "cleared" each iteration - to be consistent with previous behaviour of old + # final-verification.sh script - might be worth reviewing if we really want this behaviour + release="" product="" platform="" build_id="" locales="" channel="" from="" patch_types="complete" + eval "${config_line}" + for locale in ${locales} + do + echo "${aus_server}/update/3/$product/$release/$build_id/$platform/$locale/$channel/default/default/default/update.xml?force=1" "${patch_types// /,}" "${cfg_file}" "${line_no}" + done + done +done > "${update_xml_urls}" + +# download update.xml files and grab the mar urls from downloaded file for each patch type required +cat "${update_xml_urls}" | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../get-update-xml.sh +if [ "$?" != 0 ]; then + flush_logs + log "Error generating update requests" + exit 70 +fi + +flush_logs + +# download http header for each mar url +find "${TMPDIR}" -name 'update_xml_to_mar.*' -type f | xargs cat | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../test-mar-url.sh +if [ "$?" != 0 ]; then + flush_logs + log "Error HEADing mar urls" + exit 71 +fi + +flush_logs + +log '' +log 'Stopping stopwatch...' +STOP_TIME="$(date +%s)" + +number_of_failures="$(find "${TMPDIR}" -name 'failure.*' -type f | wc -l | sed 's/ //g')" +number_of_update_xml_urls="$(cat "${update_xml_urls}" | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')" +number_of_mar_urls="$(find "${TMPDIR}" -name "update_xml_to_mar.*" | xargs cat | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')" + +if [ "${number_of_failures}" -eq 0 ] +then + log + log "All tests passed successfully." + log + exit_code=0 +else + log '' + log '====================================' + [ "${number_of_failures}" -gt 1 ] && log "${number_of_failures} FAILURES" || log '1 FAILURE' + failure=0 + ls -1tr "${TMPDIR}" | grep '^failure\.' | while read failure_file + do + while read failure_code entry1 entry2 entry3 entry4 entry5 entry6 entry7 + do + log '====================================' + log '' + case "${failure_code}" in + + UPDATE_XML_UNAVAILABLE) + update_xml_url="${entry1}" + update_xml="${entry2}" + update_xml_headers="${entry3}" + update_xml_debug="${entry4}" + update_xml_curl_exit_code="${entry5}" + log "FAILURE $((++failure)): Update xml file not available" + log "" + log " Download url: ${update_xml_url}" + log " Curl returned exit code: ${update_xml_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_headers}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_debug}" + log "" + log " The returned update.xml file was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml}" + log "" + log " This url was tested because of the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" + log "" + + ;; + + UPDATE_XML_REDIRECT_FAILED) + update_xml_url="${entry1}" + update_xml_actual_url="${entry2}" + update_xml="${entry3}" + update_xml_headers="${entry4}" + update_xml_debug="${entry5}" + update_xml_curl_exit_code="${entry6}" + log "FAILURE $((++failure)): Update xml file not available at *redirected* location" + log "" + log " Download url: ${update_xml_url}" + log " Redirected to: ${update_xml_actual_url}" + log " It could not be downloaded from this url." + log " Curl returned exit code: ${update_xml_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_headers}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_debug}" + log "" + log " The returned update.xml file was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml}" + log "" + log " This url was tested because of the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" + log "" + ;; + + PATCH_TYPE_MISSING) + update_xml_url="${entry1}" + patch_type="${entry2}" + update_xml="${entry3}" + update_xml_headers="${entry4}" + update_xml_debug="${entry5}" + update_xml_actual_url="${entry6}" + log "FAILURE $((++failure)): Patch type '${patch_type}' not present in the downloaded update.xml file." + log "" + log " Update xml file downloaded from: ${update_xml_url}" + [ -n "${update_xml_actual_url}" ] && log " This redirected to the download url: ${update_xml_actual_url}" + log " Curl returned exit code: 0 (success)" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_headers}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml_debug}" + log "" + log " The returned update.xml file was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${update_xml}" + log "" + log " This url and patch type combination was tested due to the following cfg file entries:" + show_cfg_file_entries "${update_xml_url}" + log "" + ;; + + NO_MAR_FILE) + mar_url="${entry1}" + mar_headers_file="${entry2}" + mar_headers_debug_file="${entry3}" + mar_file_curl_exit_code="${entry4}" + mar_actual_url="${entry5}" + log "FAILURE $((++failure)): Could not retrieve mar file" + log "" + log " Mar file url: ${mar_url}" + [ -n "${mar_actual_url}" ] && log " This redirected to: ${mar_actual_url}" + log " The mar file could not be downloaded from this location." + log " Curl returned exit code: ${mar_file_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_file}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_debug_file}" + log "" + log " The mar download was tested because it was referenced in the following update xml file(s):" + show_update_xml_entries "${mar_url}" + log "" + ;; + + MAR_FILE_WRONG_SIZE) + mar_url="${entry1}" + mar_required_size="${entry2}" + mar_actual_size="${entry3}" + mar_headers_file="${entry4}" + mar_headers_debug_file="${entry5}" + mar_file_curl_exit_code="${entry6}" + mar_actual_url="${entry7}" + log "FAILURE $((++failure)): Mar file is wrong size" + log "" + log " Mar file url: ${mar_url}" + [ -n "${mar_actual_url}" ] && log " This redirected to: ${mar_actual_url}" + log " The http header of the mar file url says that the mar file is ${mar_actual_size} bytes." + log " One or more of the following update.xml file(s) says that the file should be ${mar_required_size} bytes." + log "" + log " These are the update xml file(s) that referenced this mar:" + show_update_xml_entries "${mar_url}" + log "" + log " Curl returned exit code: ${mar_file_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_file}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_debug_file}" + log "" + ;; + + BAD_HTTP_RESPONSE_CODE_FOR_MAR) + mar_url="${entry1}" + mar_headers_file="${entry2}" + mar_headers_debug_file="${entry3}" + mar_file_curl_exit_code="${entry4}" + mar_actual_url="${entry5}" + http_response_code="$(sed -e "s/$(printf '\r')//" -n -e '/^HTTP\//p' "${mar_headers_file}" | tail -1)" + log "FAILURE $((++failure)): '${http_response_code}' for mar file" + log "" + log " Mar file url: ${mar_url}" + [ -n "${mar_actual_url}" ] && log " This redirected to: ${mar_actual_url}" + log "" + log " These are the update xml file(s) that referenced this mar:" + show_update_xml_entries "${mar_url}" + log "" + log " Curl returned exit code: ${mar_file_curl_exit_code}" + log "" + log " The HTTP headers were:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_file}" + log "" + log " The full curl debug output was:" + sed -e "s/$(printf '\r')//" -e "s/^/$(date): /" -e '$a\' "${mar_headers_debug_file}" + log "" + ;; + + *) + log "ERROR: Unknown failure code - '${failure_code}'" + log "ERROR: This is a serious bug in this script." + log "ERROR: Only known failure codes are: UPDATE_XML_UNAVAILABLE, UPDATE_XML_REDIRECT_FAILED, PATCH_TYPE_MISSING, NO_MAR_FILE, MAR_FILE_WRONG_SIZE, BAD_HTTP_RESPONSE_CODE_FOR_MAR" + log "" + log "FAILURE $((++failure)): Data from failure is: ${entry1} ${entry2} ${entry3} ${entry4} ${entry5} ${entry6}" + log "" + ;; + + esac + done < "${TMPDIR}/${failure_file}" + done + exit_code=1 +fi + + +log '' +log '====================================' +log 'KEY STATS' +log '====================================' +log '' +log "Config files scanned: ${#@}" +log "Update xml files downloaded and parsed: ${number_of_update_xml_urls}" +log "Unique mar urls found: ${number_of_mar_urls}" +log "Failures: ${number_of_failures}" +log "Parallel processes used (maximum limit): ${MAX_PROCS}" +log "Execution time: $((STOP_TIME-START_TIME)) seconds" +log '' + +rm -rf "${TMPDIR}" +exit ${exit_code} diff --git a/tools/update-verify/release/get-update-xml.sh b/tools/update-verify/release/get-update-xml.sh new file mode 100755 index 0000000000..4c1fa724a8 --- /dev/null +++ b/tools/update-verify/release/get-update-xml.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +update_xml_url="${1}" +patch_types="${2}" +update_xml="$(mktemp -t update.xml.XXXXXXXXXX)" +update_xml_headers="$(mktemp -t update.xml.headers.XXXXXXXXXX)" +update_xml_debug="$(mktemp -t update.xml.debug.XXXXXXXXXX)" +curl --retry 50 --retry-max-time 300 -s --show-error -D "${update_xml_headers}" -L -v -H "Cache-Control: max-stale=0" "${update_xml_url}" > "${update_xml}" 2>"${update_xml_debug}" +update_xml_curl_exit_code=$? +if [ "${update_xml_curl_exit_code}" == 0 ] +then + update_xml_actual_url="$(sed -e "s/$(printf '\r')//" -n -e 's/^Location: //p' "${update_xml_headers}" | tail -1)" + [ -n "${update_xml_actual_url}" ] && update_xml_url_with_redirects="${update_xml_url} => ${update_xml_actual_url}" || update_xml_url_with_redirects="${update_xml_url}" + echo "$(date): Downloaded update.xml file from ${update_xml_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)" + for patch_type in ${patch_types//,/ } + do + mar_url_and_size="$(sed -e 's/\&/\&/g' -n -e 's/.* "$(mktemp -t log.XXXXXXXXXX)" + echo "PATCH_TYPE_MISSING ${update_xml_url} ${patch_type} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" + else + echo "$(date): Mar url and file size for patch type '${patch_type}' extracted from ${update_xml_url_with_redirects} (${mar_url_and_size})" > "$(mktemp -t log.XXXXXXXXXX)" + echo "${mar_url_and_size} ${update_xml_url} ${patch_type} ${update_xml_actual_url}" > "$(mktemp -t update_xml_to_mar.XXXXXXXXXX)" + fi + done +else + if [ -z "${update_xml_actual_url}" ] + then + echo "$(date): FAILURE: Could not retrieve update.xml from ${update_xml_url} for patch type(s) '${patch_types}'" > "$(mktemp -t log.XXXXXXXXXX)" + echo "UPDATE_XML_UNAVAILABLE ${update_xml_url} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_curl_exit_code}" > "$(mktemp -t failure.XXXXXXXXXX)" + else + echo "$(date): FAILURE: update.xml from ${update_xml_url} redirected to ${update_xml_actual_url} but could not retrieve update.xml from here" > "$(mktemp -t log.XXXXXXXXXX)" + echo "UPDATE_XML_REDIRECT_FAILED ${update_xml_url} ${update_xml_actual_url} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_curl_exit_code}" > "$(mktemp -t failure.XXXXXXXXXX)" + fi +fi diff --git a/tools/update-verify/release/mar_certs/README b/tools/update-verify/release/mar_certs/README new file mode 100644 index 0000000000..dd931ef1d3 --- /dev/null +++ b/tools/update-verify/release/mar_certs/README @@ -0,0 +1,29 @@ +These certificates are imported from mozilla-central (https://hg.mozilla.org/mozilla-central/file/tip/toolkit/mozapps/update/updater) +and used to support staging update verify jobs. These jobs end up replacing the certificates within the binaries +(through a binary search and replace), and must all be the same length for this to work correctly. If we recreate +these certificates, and the resulting public certificates are not the same length anymore, the commonName may be +changed to line them up again. https://github.com/google/der-ascii is a useful tool for doing this. For example: + +To convert the certificate to ascii: +der2ascii -i dep1.der -o dep1.ascii + +Then use your favourite editor to change the commonName field. That block will look something like: + SEQUENCE { + SET { + SEQUENCE { + # commonName + OBJECT_IDENTIFIER { 2.5.4.3 } + PrintableString { "CI MAR signing key 1" } + } + } + } + +You can pad the PrintableString with spaces to increase the length of the cert (1 space = 1 byte). + +Then, convert back to der: +ascii2der -i dep1.ascii -o newdep1.der + +The certificats in the sha1 subdirectory are from +https://hg.mozilla.org/mozilla-central/file/0fcbe72581bc/toolkit/mozapps/update/updater +which are the SHA-1 certs from before they where updated in Bug 1105689. They only include the release +certs, since the nightly certs are different length, and we only care about updates from old ESRs. diff --git a/tools/update-verify/release/mar_certs/dep1.der b/tools/update-verify/release/mar_certs/dep1.der new file mode 100644 index 0000000000..5320f41dfa Binary files /dev/null and b/tools/update-verify/release/mar_certs/dep1.der differ diff --git a/tools/update-verify/release/mar_certs/dep2.der b/tools/update-verify/release/mar_certs/dep2.der new file mode 100644 index 0000000000..f3eb568425 Binary files /dev/null and b/tools/update-verify/release/mar_certs/dep2.der differ diff --git a/tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der b/tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der new file mode 100644 index 0000000000..44fd95dcff Binary files /dev/null and b/tools/update-verify/release/mar_certs/nightly_aurora_level3_primary.der differ diff --git a/tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der b/tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der new file mode 100644 index 0000000000..90f8e6e82c Binary files /dev/null and b/tools/update-verify/release/mar_certs/nightly_aurora_level3_secondary.der differ diff --git a/tools/update-verify/release/mar_certs/release_primary.der b/tools/update-verify/release/mar_certs/release_primary.der new file mode 100644 index 0000000000..1d94f88ad7 Binary files /dev/null and b/tools/update-verify/release/mar_certs/release_primary.der differ diff --git a/tools/update-verify/release/mar_certs/release_secondary.der b/tools/update-verify/release/mar_certs/release_secondary.der new file mode 100644 index 0000000000..474706c4b7 Binary files /dev/null and b/tools/update-verify/release/mar_certs/release_secondary.der differ diff --git a/tools/update-verify/release/mar_certs/sha1/dep1.der b/tools/update-verify/release/mar_certs/sha1/dep1.der new file mode 100644 index 0000000000..ec8ce6184d Binary files /dev/null and b/tools/update-verify/release/mar_certs/sha1/dep1.der differ diff --git a/tools/update-verify/release/mar_certs/sha1/dep2.der b/tools/update-verify/release/mar_certs/sha1/dep2.der new file mode 100644 index 0000000000..4d0f244df2 Binary files /dev/null and b/tools/update-verify/release/mar_certs/sha1/dep2.der differ diff --git a/tools/update-verify/release/mar_certs/sha1/release_primary.der b/tools/update-verify/release/mar_certs/sha1/release_primary.der new file mode 100644 index 0000000000..11417c35e7 Binary files /dev/null and b/tools/update-verify/release/mar_certs/sha1/release_primary.der differ diff --git a/tools/update-verify/release/mar_certs/sha1/release_secondary.der b/tools/update-verify/release/mar_certs/sha1/release_secondary.der new file mode 100644 index 0000000000..16a7ef6d91 Binary files /dev/null and b/tools/update-verify/release/mar_certs/sha1/release_secondary.der differ diff --git a/tools/update-verify/release/mar_certs/xpcshellCertificate.der b/tools/update-verify/release/mar_certs/xpcshellCertificate.der new file mode 100644 index 0000000000..ea1fd47faa Binary files /dev/null and b/tools/update-verify/release/mar_certs/xpcshellCertificate.der differ diff --git a/tools/update-verify/release/replace-updater-certs.py b/tools/update-verify/release/replace-updater-certs.py new file mode 100644 index 0000000000..9e981fbfe0 --- /dev/null +++ b/tools/update-verify/release/replace-updater-certs.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os.path +import sys + +cert_dir = sys.argv[1] +# Read twice, because strings cannot be copied +updater_data = open(sys.argv[2], "rb").read() +new_updater = open(sys.argv[2], "rb").read() +outfile = sys.argv[3] + +cert_pairs = sys.argv[4:] + +if (len(cert_pairs) % 2) != 0: + print("Certs must be provided in pairs") + sys.exit(1) + +for find_cert, replace_cert in zip(*[iter(cert_pairs)] * 2): + find = open(os.path.join(cert_dir, find_cert), "rb").read() + replace = open(os.path.join(cert_dir, replace_cert), "rb").read() + print("Looking for {}...".format(find_cert)) + if find in new_updater: + print("Replacing {} with {}".format(find_cert, replace_cert)) + new_updater = new_updater.replace(find, replace) + else: + print("Didn't find {}...".format(find_cert)) + +if len(updater_data) != len(new_updater): + print( + "WARNING: new updater is not the same length as the old one (old: {}, new: {})".format( + len(updater_data), len(new_updater) + ) + ) + +if updater_data == new_updater: + print("WARNING: updater is unchanged") + +with open(outfile, "wb+") as f: + f.write(new_updater) diff --git a/tools/update-verify/release/test-mar-url.sh b/tools/update-verify/release/test-mar-url.sh new file mode 100755 index 0000000000..217c03d72a --- /dev/null +++ b/tools/update-verify/release/test-mar-url.sh @@ -0,0 +1,46 @@ +#!/bin/bash +mar_url="${1}" +mar_required_size="${2}" + +mar_headers_file="$(mktemp -t mar_headers.XXXXXXXXXX)" +mar_headers_debug_file="$(mktemp -t mar_headers_debug.XXXXXXXXXX)" +curl --retry 50 --retry-max-time 300 -s -i -r 0-2 -L -v "${mar_url}" > "${mar_headers_file}" 2>"${mar_headers_debug_file}" +mar_file_curl_exit_code=$? + +# Bug 894368 - HTTP 408's are not handled by the "curl --retry" mechanism; in this case retry in bash +attempts=1 +while [ "$((++attempts))" -lt 50 ] && grep 'HTTP/1\.1 408 Request Timeout' "${mar_headers_file}" &>/dev/null +do + sleep 1 + curl --retry 50 --retry-max-time 300 -s -i -r 0-2 -L -v "${mar_url}" > "${mar_headers_file}" 2>"${mar_headers_debug_file}" + mar_file_curl_exit_code=$? +done + +# check file size matches what was written in update.xml +# strip out dos line returns from header if they occur +# note: below, using $(printf '\r') for Darwin compatibility, rather than simple '\r' +# (i.e. shell interprets '\r' rather than sed interpretting '\r') +mar_actual_size="$(sed -e "s/$(printf '\r')//" -n -e 's/^Content-Range: bytes 0-2\///ip' "${mar_headers_file}" | tail -1)" +mar_actual_url="$(sed -e "s/$(printf '\r')//" -n -e 's/^Location: //p' "${mar_headers_file}" | tail -1)" +# note: below, sed -n '/^HTTP\//p' acts as grep '^HTTP/', but requires less overhead as sed already running +http_response_code="$(sed -e "s/$(printf '\r')//" -n -e '/^HTTP\//p' "${mar_headers_file}" | tail -1)" + +[ -n "${mar_actual_url}" ] && mar_url_with_redirects="${mar_url} => ${mar_actual_url}" || mar_url_with_redirects="${mar_url}" + +if [ "${mar_actual_size}" == "${mar_required_size}" ] +then + echo "$(date): Mar file ${mar_url_with_redirects} available with correct size (${mar_actual_size} bytes)" > "$(mktemp -t log.XXXXXXXXXX)" +elif [ -z "${mar_actual_size}" ] +then + echo "$(date): FAILURE: Could not retrieve http header for mar file from ${mar_url}" > "$(mktemp -t log.XXXXXXXXXX)" + echo "NO_MAR_FILE ${mar_url} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" + # If we get a response code (i.e. not an empty string), it better contain "206 Partial Content" or we should report on it. + # If response code is empty, this should be caught by a different block to this one (e.g. "could not retrieve http header"). +elif [ -n "${http_response_code}" ] && [ "${http_response_code}" == "${http_response_code/206 Partial Content/}" ] +then + echo "$(date): FAILURE: received a '${http_response_code}' response for mar file from ${mar_url} (expected HTTP 206 Partial Content)" > "$(mktemp -t log.XXXXXXXXXX)" + echo "BAD_HTTP_RESPONSE_CODE_FOR_MAR ${mar_url} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" +else + echo "$(date): FAILURE: Mar file incorrect size - should be ${mar_required_size} bytes, but is ${mar_actual_size} bytes - ${mar_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)" + echo "MAR_FILE_WRONG_SIZE ${mar_url} ${mar_required_size} ${mar_actual_size} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)" +fi diff --git a/tools/update-verify/release/updates/verify.sh b/tools/update-verify/release/updates/verify.sh new file mode 100755 index 0000000000..3f8556b424 --- /dev/null +++ b/tools/update-verify/release/updates/verify.sh @@ -0,0 +1,292 @@ +#!/bin/bash +#set -x + +. ../common/cached_download.sh +. ../common/unpack.sh +. ../common/download_mars.sh +. ../common/download_builds.sh +. ../common/check_updates.sh + +# Cache init being handled by new async_download.py +# clear_cache +# create_cache + +ftp_server_to="http://stage.mozilla.org/pub/mozilla.org" +ftp_server_from="http://stage.mozilla.org/pub/mozilla.org" +aus_server="https://aus4.mozilla.org" +to="" +to_build_id="" +to_app_version="" +to_display_version="" +override_certs="" +diff_summary_log=${DIFF_SUMMARY_LOG:-"$PWD/diff-summary.log"} +if [ -e ${diff_summary_log} ]; then + rm ${diff_summary_log} +fi +touch ${diff_summary_log} + +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +retry="$MY_DIR/../../../../mach python -m redo.cmd -s 1 -a 3" +cert_replacer="$MY_DIR/../replace-updater-certs.py" + +dep_overrides="nightly_aurora_level3_primary.der dep1.der nightly_aurora_level3_secondary.der dep2.der release_primary.der dep1.der release_secondary.der dep2.der sha1/release_primary.der sha1/dep1.der sha1/release_secondary.der sha1/dep2.der" +nightly_overrides="dep1.der nightly_aurora_level3_primary.der dep2.der nightly_aurora_level3_secondary.der release_primary.der nightly_aurora_level3_primary.der release_secondary.der nightly_aurora_level3_secondary.der" +release_overrides="dep1.der release_primary.der dep2.der release_secondary.der nightly_aurora_level3_primary.der release_primary.der nightly_aurora_level3_secondary.der release_secondary.der" + +runmode=0 +config_file="updates.cfg" +UPDATE_ONLY=1 +TEST_ONLY=2 +MARS_ONLY=3 +COMPLETE=4 + +usage() +{ + echo "Usage: verify.sh [OPTION] [CONFIG_FILE]" + echo " -u, --update-only only download update.xml" + echo " -t, --test-only only test that MARs exist" + echo " -m, --mars-only only test MARs" + echo " -c, --complete complete upgrade test" +} + +if [ -z "$*" ] +then + usage + exit 0 +fi + +pass_arg_count=0 +while [ "$#" -gt "$pass_arg_count" ] +do + case "$1" in + -u | --update-only) + runmode=$UPDATE_ONLY + shift + ;; + -t | --test-only) + runmode=$TEST_ONLY + shift + ;; + -m | --mars-only) + runmode=$MARS_ONLY + shift + ;; + -c | --complete) + runmode=$COMPLETE + shift + ;; + *) + # Move the unrecognized arg to the end of the list + arg="$1" + shift + set -- "$@" "$arg" + pass_arg_count=`expr $pass_arg_count + 1` + esac +done + +if [ -n "$arg" ] +then + config_file=$arg + echo "Using config file $config_file" +else + echo "Using default config file $config_file" +fi + +if [ "$runmode" == "0" ] +then + usage + exit 0 +fi + +while read entry +do + # initialize all config variables + release="" + product="" + platform="" + build_id="" + locales="" + channel="" + from="" + patch_types="complete" + use_old_updater=0 + mar_channel_IDs="" + updater_package="" + eval $entry + + # the arguments for updater changed in Gecko 34/SeaMonkey 2.31 + major_version=`echo $release | cut -f1 -d.` + if [[ "$product" == "seamonkey" ]]; then + minor_version=`echo $release | cut -f2 -d.` + if [[ $major_version -le 2 && $minor_version -lt 31 ]]; then + use_old_updater=1 + fi + elif [[ $major_version -lt 34 ]]; then + use_old_updater=1 + fi + + # Note: cross platform tests seem to work for everything except Mac-on-Windows. + # We probably don't care about this use case. + if [[ "$updater_package" == "" ]]; then + updater_package="$from" + fi + + for locale in $locales + do + rm -f update/partial.size update/complete.size + for patch_type in $patch_types + do + update_path="${product}/${release}/${build_id}/${platform}/${locale}/${channel}/default/default/default" + if [ "$runmode" == "$MARS_ONLY" ] || [ "$runmode" == "$COMPLETE" ] || + [ "$runmode" == "$TEST_ONLY" ] + then + if [ "$runmode" == "$TEST_ONLY" ] + then + download_mars "${aus_server}/update/3/${update_path}/default/update.xml?force=1" ${patch_type} 1 \ + "${to_build_id}" "${to_app_version}" "${to_display_version}" + err=$? + else + download_mars "${aus_server}/update/3/${update_path}/update.xml?force=1" ${patch_type} 0 \ + "${to_build_id}" "${to_app_version}" "${to_display_version}" + err=$? + fi + if [ "$err" != "0" ]; then + echo "TEST-UNEXPECTED-FAIL: [${release} ${locale} ${patch_type}] download_mars returned non-zero exit code: ${err}" + continue + fi + else + mkdir -p updates/${update_path}/complete + mkdir -p updates/${update_path}/partial + $retry wget -q -O ${patch_type} updates/${update_path}/${patch_type}/update.xml "${aus_server}/update/3/${update_path}/update.xml?force=1" + + fi + if [ "$runmode" == "$COMPLETE" ] + then + if [ -z "$from" ] || [ -z "$to" ] + then + continue + fi + + updater_platform="" + updater_package_url=`echo "${ftp_server_from}${updater_package}" | sed "s/%locale%/${locale}/"` + updater_package_filename=`basename "$updater_package_url"` + case $updater_package_filename in + *dmg) + platform_dirname="*.app" + updater_bins="Contents/MacOS/updater.app/Contents/MacOS/updater Contents/MacOS/updater.app/Contents/MacOS/org.mozilla.updater" + updater_platform="mac" + ;; + *exe) + updater_package_url=`echo "${updater_package_url}" | sed "s/ja-JP-mac/ja/"` + platform_dirname="bin" + updater_bins="updater.exe" + updater_platform="win32" + ;; + *bz2) + updater_package_url=`echo "${updater_package_url}" | sed "s/ja-JP-mac/ja/"` + platform_dirname=`echo $product | tr '[A-Z]' '[a-z]'` + updater_bins="updater" + updater_platform="linux" + ;; + *) + echo "Couldn't detect updater platform" + exit 1 + ;; + esac + + rm -rf updater/* + cached_download "${updater_package_filename}" "${updater_package_url}" + unpack_build "$updater_platform" updater "$updater_package_filename" "$locale" + + # Even on Windows, we want Unix-style paths for the updater, because of MSYS. + cwd=$(\ls -d $PWD/updater/$platform_dirname) + # Bug 1209376. Linux updater linked against other libraries in the installation directory + export LD_LIBRARY_PATH=$cwd + updater="null" + for updater_bin in $updater_bins; do + if [ -e "$cwd/$updater_bin" ]; then + echo "Found updater at $updater_bin" + updater="$cwd/$updater_bin" + break + fi + done + + update_to_dep=false + if [ ! -z "$override_certs" ]; then + echo "Replacing certs in updater binary" + cp "${updater}" "${updater}.orig" + case ${override_certs} in + dep) + overrides=${dep_overrides} + update_to_dep=true + ;; + nightly) + overrides=${nightly_overrides} + ;; + release) + overrides=${release_overrides} + ;; + *) + echo "Unknown override cert - skipping" + ;; + esac + python3 "${cert_replacer}" "${MY_DIR}/../mar_certs" "${updater}.orig" "${updater}" ${overrides} + else + echo "override_certs is '${override_certs}', not replacing any certificates" + fi + + if [ "$updater" == "null" ]; then + echo "Couldn't find updater binary" + continue + fi + + from_path=`echo $from | sed "s/%locale%/${locale}/"` + to_path=`echo $to | sed "s/%locale%/${locale}/"` + download_builds "${ftp_server_from}${from_path}" "${ftp_server_to}${to_path}" + err=$? + if [ "$err" != "0" ]; then + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] download_builds returned non-zero exit code: $err" + continue + fi + source_file=`basename "$from_path"` + target_file=`basename "$to_path"` + diff_file="results.diff" + if [ -e ${diff_file} ]; then + rm ${diff_file} + fi + check_updates "${platform}" "downloads/${source_file}" "downloads/${target_file}" ${locale} ${use_old_updater} ${updater} ${diff_file} ${channel} "${mar_channel_IDs}" ${update_to_dep} + err=$? + if [ "$err" == "0" ]; then + continue + elif [ "$err" == "1" ]; then + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] check_updates returned failure for $platform downloads/$source_file vs. downloads/$target_file: $err" + elif [ "$err" == "2" ]; then + echo "WARN: [$release $locale $patch_type] check_updates returned warning for $platform downloads/$source_file vs. downloads/$target_file: $err" + else + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] check_updates returned unknown error for $platform downloads/$source_file vs. downloads/$target_file: $err" + fi + + if [ -s ${diff_file} ]; then + echo "Found diffs for ${patch_type} update from ${aus_server}/update/3/${update_path}/update.xml?force=1" >> ${diff_summary_log} + cat ${diff_file} >> ${diff_summary_log} + echo "" >> ${diff_summary_log} + fi + fi + done + if [ -f update/partial.size ] && [ -f update/complete.size ]; then + partial_size=`cat update/partial.size` + complete_size=`cat update/complete.size` + if [ $partial_size -gt $complete_size ]; then + echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] partial updates are larger than complete updates" + elif [ $partial_size -eq $complete_size ]; then + echo "WARN: [$release $locale $patch_type] partial updates are the same size as complete updates, this should only happen for major updates" + else + echo "SUCCESS: [$release $locale $patch_type] partial updates are smaller than complete updates, all is well in the universe" + fi + fi + done +done < $config_file + +clear_cache diff --git a/tools/update-verify/scripts/async_download.py b/tools/update-verify/scripts/async_download.py new file mode 100644 index 0000000000..efedc8295f --- /dev/null +++ b/tools/update-verify/scripts/async_download.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import asyncio +import glob +import logging +import os +import sys +import xml.etree.ElementTree as ET +from os import path + +import aiohttp + +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") +log = logging.getLogger(__name__) + +UV_CACHE_PATH = os.getenv( + "UV_CACHE_PATH", os.path.join(path.dirname(__file__), "../release/updates/cache/") +) +UV_PARALLEL_DOWNLOADS = os.getenv("UV_PARALLEL_DOWNLOADS", 20) + +FTP_SERVER_TO = os.getenv("ftp_server_to", "http://stage.mozilla.org/pub/mozilla.org") +FTP_SERVER_FROM = os.getenv( + "ftp_server_from", "http://stage.mozilla.org/pub/mozilla.org" +) +AUS_SERVER = os.getenv("aus_server", "https://aus5.mozilla.org") + + +def create_cache(): + if not os.path.isdir(UV_CACHE_PATH): + os.mkdir(UV_CACHE_PATH) + + +def remove_cache(): + """ + Removes all files in the cache folder + We don't support folders or .dot(hidden) files + By not deleting the cache directory, it allows us to use Docker tmpfs mounts, + which are the only workaround to poor mount r/w performance on MacOS + Bug Reference: + https://forums.docker.com/t/file-access-in-mounted-volumes-extremely-slow-cpu-bound/8076/288 + """ + files = glob.glob(f"{UV_CACHE_PATH}/*") + for f in files: + os.remove(f) + + +def _cachepath(i, ext): + # Helper function: given an index, return a cache file path + return path.join(UV_CACHE_PATH, f"obj_{i:0>5}.{ext}") + + +async def fetch_url(url, path, connector): + """ + Fetch/download a file to a specific path + + Parameters + ---------- + url : str + URL to be fetched + + path : str + Path to save binary + + Returns + ------- + dict + Request result. If error result['error'] is True + """ + + def _result(response, error=False): + data = { + "headers": dict(response.headers), + "status": response.status, + "reason": response.reason, + "_request_info": str(response._request_info), + "url": url, + "path": path, + "error": error, + } + return data + + # Set connection timeout to 15 minutes + timeout = aiohttp.ClientTimeout(total=900) + + try: + async with aiohttp.ClientSession( + connector=connector, connector_owner=False, timeout=timeout + ) as session: + log.info(f"Retrieving {url}") + async with session.get( + url, headers={"Cache-Control": "max-stale=0"} + ) as response: + # Any response code > 299 means something went wrong + if response.status > 299: + log.info(f"Failed to download {url} with status {response.status}") + return _result(response, True) + + with open(path, "wb") as fd: + while True: + chunk = await response.content.read() + if not chunk: + break + fd.write(chunk) + result = _result(response) + log.info(f'Finished downloading {url}\n{result["headers"]}') + return result + + except ( + UnicodeDecodeError, # Data parsing + asyncio.TimeoutError, # Async timeout + aiohttp.ClientError, # aiohttp error + ) as e: + log.error("=============") + log.error(f"Error downloading {url}") + log.error(e) + log.error("=============") + return {"path": path, "url": url, "error": True} + + +async def download_multi(targets, sourceFunc): + """ + Download list of targets + + Parameters + ---------- + targets : list + List of urls to download + + sourceFunc : str + Source function name (for filename) + + Returns + ------- + tuple + List of responses (Headers) + """ + + targets = set(targets) + amount = len(targets) + + connector = aiohttp.TCPConnector( + limit=UV_PARALLEL_DOWNLOADS, # Simultaneous connections, per host + ttl_dns_cache=600, # Cache DNS for 10 mins + ) + + log.info(f"\nDownloading {amount} files ({UV_PARALLEL_DOWNLOADS} async limit)") + + # Transform targets into {url, path} objects + payloads = [ + {"url": url, "path": _cachepath(i, sourceFunc)} + for (i, url) in enumerate(targets) + ] + + downloads = [] + + fetches = [fetch_url(t["url"], t["path"], connector) for t in payloads] + + downloads.extend(await asyncio.gather(*fetches)) + connector.close() + + results = [] + # Remove file if download failed + for fetch in downloads: + # If there's an error, try to remove the file, but keep going if file not present + if fetch["error"]: + try: + os.unlink(fetch.get("path", None)) + except (TypeError, FileNotFoundError) as e: + log.info(f"Unable to cleanup error file: {e} continuing...") + continue + + results.append(fetch) + + return results + + +async def download_builds(verifyConfig): + """ + Given UpdateVerifyConfig, download and cache all necessary updater files + Include "to" and "from"/"updater_pacakge" + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + Chunked config + + Returns + ------- + list : List of file paths and urls to each updater file + """ + + updaterUrls = set() + for release in verifyConfig.releases: + ftpServerFrom = release["ftp_server_from"] + ftpServerTo = release["ftp_server_to"] + + for locale in release["locales"]: + toUri = verifyConfig.to + if toUri is not None and ftpServerTo is not None: + toUri = toUri.replace("%locale%", locale) + updaterUrls.add(f"{ftpServerTo}{toUri}") + + for reference in ("updater_package", "from"): + uri = release.get(reference, None) + if uri is None: + continue + uri = uri.replace("%locale%", locale) + # /ja-JP-mac/ locale is replaced with /ja/ for updater packages + uri = uri.replace("ja-JP-mac", "ja") + updaterUrls.add(f"{ftpServerFrom}{uri}") + + log.info(f"About to download {len(updaterUrls)} updater packages") + + updaterResults = await download_multi(list(updaterUrls), "updater.async.cache") + return updaterResults + + +def get_mar_urls_from_update(path): + """ + Given an update.xml file, return MAR URLs + + If update.xml doesn't have URLs, returns empty list + + Parameters + ---------- + path : str + Path to update.xml file + + Returns + ------- + list : List of URLs + """ + + result = [] + root = ET.parse(path).getroot() + for patch in root.findall("update/patch"): + url = patch.get("URL") + if url: + result.append(url) + return result + + +async def download_mars(updatePaths): + """ + Given list of update.xml paths, download MARs for each + + Parameters + ---------- + update_paths : list + List of paths to update.xml files + """ + + patchUrls = set() + for updatePath in updatePaths: + for url in get_mar_urls_from_update(updatePath): + patchUrls.add(url) + + log.info(f"About to download {len(patchUrls)} MAR packages") + marResults = await download_multi(list(patchUrls), "mar.async.cache") + return marResults + + +async def download_update_xml(verifyConfig): + """ + Given UpdateVerifyConfig, download and cache all necessary update.xml files + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + Chunked config + + Returns + ------- + list : List of file paths and urls to each update.xml file + """ + + xmlUrls = set() + product = verifyConfig.product + urlTemplate = ( + "{server}/update/3/{product}/{release}/{build}/{platform}/" + "{locale}/{channel}/default/default/default/update.xml?force=1" + ) + + for release in verifyConfig.releases: + for locale in release["locales"]: + xmlUrls.add( + urlTemplate.format( + server=AUS_SERVER, + product=product, + release=release["release"], + build=release["build_id"], + platform=release["platform"], + locale=locale, + channel=verifyConfig.channel, + ) + ) + + log.info(f"About to download {len(xmlUrls)} update.xml files") + xmlResults = await download_multi(list(xmlUrls), "update.xml.async.cache") + return xmlResults + + +async def _download_from_config(verifyConfig): + """ + Given an UpdateVerifyConfig object, download all necessary files to cache + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + The config - already chunked + """ + remove_cache() + create_cache() + + downloadList = [] + ################## + # Download files # + ################## + xmlFiles = await download_update_xml(verifyConfig) + downloadList.extend(xmlFiles) + downloadList += await download_mars(x["path"] for x in xmlFiles) + downloadList += await download_builds(verifyConfig) + + ##################### + # Create cache.list # + ##################### + cacheLinks = [] + + # Rename files and add to cache_links + for download in downloadList: + cacheLinks.append(download["url"]) + fileIndex = len(cacheLinks) + os.rename(download["path"], _cachepath(fileIndex, "cache")) + + cacheIndexPath = path.join(UV_CACHE_PATH, "urls.list") + with open(cacheIndexPath, "w") as cache: + cache.writelines(f"{l}\n" for l in cacheLinks) + + # Log cache + log.info("Cache index urls.list contents:") + with open(cacheIndexPath, "r") as cache: + for ln, url in enumerate(cache.readlines()): + line = url.replace("\n", "") + log.info(f"Line {ln+1}: {line}") + + return None + + +def download_from_config(verifyConfig): + """ + Given an UpdateVerifyConfig object, download all necessary files to cache + (sync function that calls the async one) + + Parameters + ---------- + verifyConfig : UpdateVerifyConfig + The config - already chunked + """ + return asyncio.run(_download_from_config(verifyConfig)) diff --git a/tools/update-verify/scripts/chunked-verify.py b/tools/update-verify/scripts/chunked-verify.py new file mode 100644 index 0000000000..8c4320d4cc --- /dev/null +++ b/tools/update-verify/scripts/chunked-verify.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import os +import sys +from os import path +from tempfile import mkstemp + +sys.path.append(path.join(path.dirname(__file__), "../python")) +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s") +log = logging.getLogger(__name__) + +from async_download import download_from_config +from mozrelease.update_verify import UpdateVerifyConfig +from util.commands import run_cmd + +UPDATE_VERIFY_COMMAND = ["bash", "verify.sh", "-c"] +UPDATE_VERIFY_DIR = path.join(path.dirname(__file__), "../release/updates") + + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser("") + + parser.set_defaults( + chunks=None, + thisChunk=None, + ) + parser.add_argument("--verify-config", required=True, dest="verifyConfig") + parser.add_argument("--verify-channel", required=True, dest="verify_channel") + parser.add_argument("--chunks", required=True, dest="chunks", type=int) + parser.add_argument("--this-chunk", required=True, dest="thisChunk", type=int) + parser.add_argument("--diff-summary", required=True, type=str) + + options = parser.parse_args() + assert options.chunks and options.thisChunk, "chunks and this-chunk are required" + assert path.isfile(options.verifyConfig), "Update verify config must exist!" + verifyConfigFile = options.verifyConfig + + fd, configFile = mkstemp() + # Needs to be opened in "bytes" mode because we perform relative seeks on it + fh = os.fdopen(fd, "wb") + try: + verifyConfig = UpdateVerifyConfig() + verifyConfig.read(path.join(UPDATE_VERIFY_DIR, verifyConfigFile)) + myVerifyConfig = verifyConfig.getChunk(options.chunks, options.thisChunk) + # override the channel if explicitly set + if options.verify_channel: + myVerifyConfig.channel = options.verify_channel + myVerifyConfig.write(fh) + fh.close() + run_cmd(["cat", configFile]) + + # Before verifying, we want to download and cache all required files + download_from_config(myVerifyConfig) + + run_cmd( + UPDATE_VERIFY_COMMAND + [configFile], + cwd=UPDATE_VERIFY_DIR, + env={"DIFF_SUMMARY_LOG": path.abspath(options.diff_summary)}, + ) + finally: + if path.exists(configFile): + os.unlink(configFile) diff --git a/tools/update-verify/scripts/chunked-verify.sh b/tools/update-verify/scripts/chunked-verify.sh new file mode 100755 index 0000000000..ad6af19080 --- /dev/null +++ b/tools/update-verify/scripts/chunked-verify.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -ex +set -o pipefail +# This ugly hack is a cross-platform (Linux/Mac/Windows+MSYS) way to get the +# absolute path to the directory containing this script +pushd `dirname $0` &>/dev/null +MY_DIR=$(pwd) +popd &>/dev/null +SCRIPTS_DIR="$MY_DIR/.." +PYTHON='./mach python' +VERIFY_CONFIG="$MOZ_FETCHES_DIR/update-verify.cfg" + +while [ "$#" -gt 0 ]; do + case $1 in + # Parse total-chunks + --total-chunks=*) chunks="${1#*=}"; shift 1;; + --total-chunks) chunks="${2}"; shift 2;; + + # Parse this-chunk + --this-chunk=*) thisChunk="${1#*=}"; shift 1;; + --this-chunk) thisChunk="${2}"; shift 2;; + + # Stop if other parameters are sent + *) echo "Unknown parameter: ${1}"; exit 1;; + esac +done + +# Validate parameters +if [ -z "${chunks}" ]; then echo "Required parameter: --total-chunks"; exit 1; fi +if [ -z "${thisChunk}" ]; then echo "Required parameter: --this-chunk"; exit 1; fi + +# release promotion +if [ -n "$CHANNEL" ]; then + EXTRA_PARAMS="--verify-channel $CHANNEL" +else + EXTRA_PARAMS="" +fi +$PYTHON $MY_DIR/chunked-verify.py --chunks $chunks --this-chunk $thisChunk \ +--verify-config $VERIFY_CONFIG --diff-summary $PWD/diff-summary.log $EXTRA_PARAMS \ +2>&1 | tee $SCRIPTS_DIR/../verify_log.txt + +print_failed_msg() +{ + echo "-------------------------" + echo "This run has failed, see the above log" + echo + return 1 +} + +print_warning_msg() +{ + echo "-------------------------" + echo "This run has warnings, see the above log" + echo + return 2 +} + +set +x + +echo "Scanning log for failures and warnings" +echo "--------------------------------------" + +# Test for a failure, note we are set -e. +# Grep returns 0 on a match and 1 on no match +# Testing for failures first is important because it's OK to to mark as failed +# when there's failures+warnings, but not OK to mark as warnings in the same +# situation. +( ! grep 'TEST-UNEXPECTED-FAIL:' $SCRIPTS_DIR/../verify_log.txt ) || print_failed_msg +( ! grep 'WARN:' $SCRIPTS_DIR/../verify_log.txt ) || print_warning_msg + +echo "-------------------------" +echo "All is well" diff --git a/tools/vcs/mach_commands.py b/tools/vcs/mach_commands.py new file mode 100644 index 0000000000..4623a23634 --- /dev/null +++ b/tools/vcs/mach_commands.py @@ -0,0 +1,242 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import logging +import os +import re +import subprocess +import sys + +import mozpack.path as mozpath +from mach.decorators import Command, CommandArgument + +GITHUB_ROOT = "https://github.com/" +PR_REPOSITORIES = { + "webrender": { + "github": "servo/webrender", + "path": "gfx/wr", + "bugzilla_product": "Core", + "bugzilla_component": "Graphics: WebRender", + }, + "webgpu": { + "github": "gfx-rs/wgpu", + "path": "gfx/wgpu", + "bugzilla_product": "Core", + "bugzilla_component": "Graphics: WebGPU", + }, + "debugger": { + "github": "firefox-devtools/debugger", + "path": "devtools/client/debugger", + "bugzilla_product": "DevTools", + "bugzilla_component": "Debugger", + }, +} + + +@Command( + "import-pr", + category="misc", + description="Import a pull request from Github to the local repo.", +) +@CommandArgument("-b", "--bug-number", help="Bug number to use in the commit messages.") +@CommandArgument( + "-t", + "--bugzilla-token", + help="Bugzilla API token used to file a new bug if no bug number is provided.", +) +@CommandArgument("-r", "--reviewer", help="Reviewer nick to apply to commit messages.") +@CommandArgument( + "pull_request", + help="URL to the pull request to import (e.g. " + "https://github.com/servo/webrender/pull/3665).", +) +def import_pr( + command_context, + pull_request, + bug_number=None, + bugzilla_token=None, + reviewer=None, +): + import requests + + pr_number = None + repository = None + for r in PR_REPOSITORIES.values(): + if pull_request.startswith(GITHUB_ROOT + r["github"] + "/pull/"): + # sanitize URL, dropping anything after the PR number + pr_number = int(re.search("/pull/([0-9]+)", pull_request).group(1)) + pull_request = GITHUB_ROOT + r["github"] + "/pull/" + str(pr_number) + repository = r + break + + if repository is None: + command_context.log( + logging.ERROR, + "unrecognized_repo", + {}, + "The pull request URL was not recognized; add it to the list of " + "recognized repos in PR_REPOSITORIES in %s" % __file__, + ) + sys.exit(1) + + command_context.log( + logging.INFO, + "import_pr", + {"pr_url": pull_request}, + "Attempting to import {pr_url}", + ) + dirty = [ + f + for f in command_context.repository.get_changed_files(mode="all") + if f.startswith(repository["path"]) + ] + if dirty: + command_context.log( + logging.ERROR, + "dirty_tree", + repository, + "Local {path} tree is dirty; aborting!", + ) + sys.exit(1) + target_dir = mozpath.join( + command_context.topsrcdir, os.path.normpath(repository["path"]) + ) + + if bug_number is None: + if bugzilla_token is None: + command_context.log( + logging.WARNING, + "no_token", + {}, + "No bug number or bugzilla API token provided; bug number will not " + "be added to commit messages.", + ) + else: + bug_number = _file_bug( + command_context, bugzilla_token, repository, pr_number + ) + elif bugzilla_token is not None: + command_context.log( + logging.WARNING, + "too_much_bug", + {}, + "Providing a bugzilla token is unnecessary when a bug number is provided. " + "Using bug number; ignoring token.", + ) + + pr_patch = requests.get(pull_request + ".patch") + pr_patch.raise_for_status() + for patch in _split_patches(pr_patch.content, bug_number, pull_request, reviewer): + command_context.log( + logging.INFO, + "commit_msg", + patch, + "Processing commit [{commit_summary}] by [{author}] at [{date}]", + ) + patch_cmd = subprocess.Popen( + ["patch", "-p1", "-s"], stdin=subprocess.PIPE, cwd=target_dir + ) + patch_cmd.stdin.write(patch["diff"].encode("utf-8")) + patch_cmd.stdin.close() + patch_cmd.wait() + if patch_cmd.returncode != 0: + command_context.log( + logging.ERROR, + "commit_fail", + {}, + 'Error applying diff from commit via "patch -p1 -s". Aborting...', + ) + sys.exit(patch_cmd.returncode) + command_context.repository.commit( + patch["commit_msg"], patch["author"], patch["date"], [target_dir] + ) + command_context.log(logging.INFO, "commit_pass", {}, "Committed successfully.") + + +def _file_bug(command_context, token, repo, pr_number): + import requests + + bug = requests.post( + "https://bugzilla.mozilla.org/rest/bug?api_key=%s" % token, + json={ + "product": repo["bugzilla_product"], + "component": repo["bugzilla_component"], + "summary": "Land %s#%s in mozilla-central" % (repo["github"], pr_number), + "version": "unspecified", + }, + ) + bug.raise_for_status() + command_context.log(logging.DEBUG, "new_bug", {}, bug.content) + bugnumber = json.loads(bug.content)["id"] + command_context.log( + logging.INFO, "new_bug", {"bugnumber": bugnumber}, "Filed bug {bugnumber}" + ) + return bugnumber + + +def _split_patches(patchfile, bug_number, pull_request, reviewer): + INITIAL = 0 + HEADERS = 1 + STAT_AND_DIFF = 2 + + patch = b"" + state = INITIAL + for line in patchfile.splitlines(): + if state == INITIAL: + if line.startswith(b"From "): + state = HEADERS + elif state == HEADERS: + patch += line + b"\n" + if line == b"---": + state = STAT_AND_DIFF + elif state == STAT_AND_DIFF: + if line.startswith(b"From "): + yield _parse_patch(patch, bug_number, pull_request, reviewer) + patch = b"" + state = HEADERS + else: + patch += line + b"\n" + if len(patch) > 0: + yield _parse_patch(patch, bug_number, pull_request, reviewer) + return + + +def _parse_patch(patch, bug_number, pull_request, reviewer): + import email + from email import header, policy + + parse_policy = policy.compat32.clone(max_line_length=None) + parsed_mail = email.message_from_bytes(patch, policy=parse_policy) + + def header_as_unicode(key): + decoded = header.decode_header(parsed_mail[key]) + return str(header.make_header(decoded)) + + author = header_as_unicode("From") + date = header_as_unicode("Date") + commit_summary = header_as_unicode("Subject") + email_body = parsed_mail.get_payload(decode=True).decode("utf-8") + (commit_body, diff) = ("\n" + email_body).rsplit("\n---\n", 1) + + bug_prefix = "" + if bug_number is not None: + bug_prefix = "Bug %s - " % bug_number + commit_summary = re.sub(r"^\[PATCH[0-9 /]*\] ", bug_prefix, commit_summary) + if reviewer is not None: + commit_summary += " r=" + reviewer + + commit_msg = commit_summary + "\n" + if len(commit_body) > 0: + commit_msg += commit_body + "\n" + commit_msg += "\n[import_pr] From " + pull_request + "\n" + + patch_obj = { + "author": author, + "date": date, + "commit_summary": commit_summary, + "commit_msg": commit_msg, + "diff": diff, + } + return patch_obj -- cgit v1.2.3