diff options
Diffstat (limited to 'mobile/android/fenix/benchmark/src')
6 files changed, 231 insertions, 0 deletions
diff --git a/mobile/android/fenix/benchmark/src/main/AndroidManifest.xml b/mobile/android/fenix/benchmark/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..251d1c7b21 --- /dev/null +++ b/mobile/android/fenix/benchmark/src/main/AndroidManifest.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android"> + + <queries> + <package android:name="org.mozilla.fenix" /> + </queries> +</manifest> diff --git a/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/BaselineProfilesStartupBenchmark.kt b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/BaselineProfilesStartupBenchmark.kt new file mode 100644 index 0000000000..ccf228584c --- /dev/null +++ b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/BaselineProfilesStartupBenchmark.kt @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.benchmark + +import android.os.Build +import androidx.annotation.RequiresApi +import androidx.benchmark.macro.BaselineProfileMode +import androidx.benchmark.macro.CompilationMode +import androidx.benchmark.macro.StartupMode +import androidx.benchmark.macro.StartupTimingMetric +import androidx.benchmark.macro.junit4.MacrobenchmarkRule +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.mozilla.fenix.benchmark.utils.measureRepeatedDefault + +/** + * This test class benchmarks the speed of app startup. Run this benchmark to verify how effective + * a Baseline Profile is. It does this by comparing [CompilationMode.None], which represents the + * app with no Baseline Profiles optimizations, and [CompilationMode.Partial], which uses Baseline Profiles. + * + * Before running make sure `autosignReleaseWithDebugKey=true` is present in local.properties. + * + * Run this benchmark to see startup measurements and captured system traces for verifying + * the effectiveness of your Baseline Profiles. You can run it directly from Android + * Studio as an instrumentation test that logs the benchmark metrics with links to the Perfetto traces, + * + * or using the gradle command: + * + * ``` + * ./gradlew :benchmark:connectedBenchmarkAndroidTest -P android.testInstrumentationRunnerArguments.class=org.mozilla.fenix.benchmark.BaselineProfilesStartupBenchmark -P benchmarkTest + * ``` + * + * The metric results will be in `benchmark/build/outputs/connected_android_test_additional_output` folder. + * + * Run the benchmarks on a physical device, not an emulator because the emulator doesn't represent + * real world performance and shares system resources with its host. + * + * For more information, see the [Macrobenchmark documentation](https://d.android.com/macrobenchmark#create-macrobenchmark) + * and the [instrumentation arguments documentation](https://d.android.com/topic/performance/benchmarking/macrobenchmark-instrumentation-args). + **/ +@RunWith(AndroidJUnit4::class) +@RequiresApi(Build.VERSION_CODES.N) +class BaselineProfilesStartupBenchmark { + + @get:Rule + val benchmarkRule = MacrobenchmarkRule() + + @Test + fun startupNone() = startupBenchmark(CompilationMode.None()) + + @Test + fun startupPartialWithBaselineProfiles() = + startupBenchmark(CompilationMode.Partial(baselineProfileMode = BaselineProfileMode.Require)) + + private fun startupBenchmark(compilationMode: CompilationMode) = + benchmarkRule.measureRepeatedDefault( + metrics = listOf(StartupTimingMetric()), + startupMode = StartupMode.COLD, + compilationMode = compilationMode, + setupBlock = { + pressHome() + }, + ) { + startActivityAndWait() + } +} diff --git a/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/StartupBenchmark.kt b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/StartupBenchmark.kt new file mode 100644 index 0000000000..014df73e3a --- /dev/null +++ b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/StartupBenchmark.kt @@ -0,0 +1,49 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.benchmark + +import androidx.benchmark.macro.StartupMode +import androidx.benchmark.macro.StartupTimingMetric +import androidx.benchmark.macro.junit4.MacrobenchmarkRule +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.mozilla.fenix.benchmark.utils.measureRepeatedDefault + +/** + * This is a startup benchmark. + * It navigates to the device's home screen, and launches the default activity. + * + * Before running this benchmark, + * switch your app's active build variant in the Studio (affects Studio runs only) + * + * Run this benchmark from Studio to see startup measurements, and captured system traces + * for investigating your app's performance. + */ +@RunWith(AndroidJUnit4::class) +class StartupBenchmark { + @get:Rule + val benchmarkRule = MacrobenchmarkRule() + + @Test + fun startupCold() = startupBenchmark(StartupMode.COLD) + + @Test + fun startupWarm() = startupBenchmark(StartupMode.WARM) + + @Test + fun startupHot() = startupBenchmark(StartupMode.HOT) + + private fun startupBenchmark(startupMode: StartupMode) = benchmarkRule.measureRepeatedDefault( + metrics = listOf(StartupTimingMetric()), + startupMode = startupMode, + setupBlock = { + pressHome() + } + ) { + startActivityAndWait() + } +} diff --git a/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/baselineprofile/StartupOnlyBaselineProfileGenerator.kt b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/baselineprofile/StartupOnlyBaselineProfileGenerator.kt new file mode 100644 index 0000000000..f8e26847b5 --- /dev/null +++ b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/baselineprofile/StartupOnlyBaselineProfileGenerator.kt @@ -0,0 +1,60 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.benchmark.baselineprofile + +import android.os.Build +import androidx.annotation.RequiresApi +import androidx.benchmark.macro.junit4.BaselineProfileRule +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.mozilla.fenix.benchmark.utils.TARGET_PACKAGE + +/** + * This test class generates a basic startup baseline profile for the target package. + * + * Refer to the [baseline profile documentation](https://d.android.com/topic/performance/baselineprofiles) + * for more information. + * + * Make sure `autosignReleaseWithDebugKey=true` is present in local.properties. + * + * Generate the baseline profile using this gradle task: + * ``` + * ./gradlew :benchmark:pixel6Api34BenchmarkAndroidTest -P android.testInstrumentationRunnerArguments.class=org.mozilla.fenix.benchmark.baselineprofile.StartupOnlyBaselineProfileGenerator -P benchmarkTest -P disableOptimization + * ``` + * + * Check [documentation](https://d.android.com/topic/performance/benchmarking/macrobenchmark-instrumentation-args) + * for more information about available instrumentation arguments. + * + * As the experiment is only for nightly, run + * ``` + * ./gradlew copyBaselineProfile + * ``` + * This will copy the baseline profiles to the nightly folder. These are the profiles that will be compiled with the nightly build. + * + * Then, copy the profiles to app/src/benchmark/baselineProfiles to verify the improvements by running + * the [org.mozilla.fenix.benchmark.BaselineProfilesStartupBenchmark] benchmark. + * Notice that when we run the benchmark, we run the benchmark variant and not the nightly so copying the profiles here is important. + * These shouldn't be pushed to version control. + * + * When using this class to generate a baseline profile, only API 33+ or rooted API 28+ are supported. + **/ +@RequiresApi(Build.VERSION_CODES.P) +@RunWith(AndroidJUnit4::class) +class StartupOnlyBaselineProfileGenerator { + + @get:Rule + val rule = BaselineProfileRule() + + @Test + fun generateBaselineProfile() { + rule.collect( + packageName = TARGET_PACKAGE, + ) { + startActivityAndWait() + } + } +} diff --git a/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/Constants.kt b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/Constants.kt new file mode 100644 index 0000000000..14fe426118 --- /dev/null +++ b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/Constants.kt @@ -0,0 +1,8 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.benchmark.utils + +const val TARGET_PACKAGE = "org.mozilla.fenix" +const val DEFAULT_ITERATIONS = 5 diff --git a/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/MacroBenchmarkRule.kt b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/MacroBenchmarkRule.kt new file mode 100644 index 0000000000..d3fbd499cf --- /dev/null +++ b/mobile/android/fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/MacroBenchmarkRule.kt @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.benchmark.utils + +import androidx.annotation.IntRange +import androidx.benchmark.macro.CompilationMode +import androidx.benchmark.macro.MacrobenchmarkScope +import androidx.benchmark.macro.Metric +import androidx.benchmark.macro.StartupMode +import androidx.benchmark.macro.junit4.MacrobenchmarkRule + +/** + * Extension function that calls [MacrobenchmarkRule.measureRepeated] with + * defaults parameters set for [packageName] and [iterations]. + */ +fun MacrobenchmarkRule.measureRepeatedDefault( + packageName: String = TARGET_PACKAGE, + metrics: List<Metric>, + compilationMode: CompilationMode = CompilationMode.DEFAULT, + startupMode: StartupMode? = null, + @IntRange(from = 1) + iterations: Int = DEFAULT_ITERATIONS, + setupBlock: MacrobenchmarkScope.() -> Unit = {}, + measureBlock: MacrobenchmarkScope.() -> Unit, +) { + measureRepeated( + packageName = packageName, + metrics = metrics, + compilationMode = compilationMode, + startupMode = startupMode, + iterations = iterations, + setupBlock = setupBlock, + measureBlock = measureBlock, + ) +} |