diff options
Diffstat (limited to 'src/boost/libs/histogram/benchmark')
13 files changed, 1051 insertions, 0 deletions
diff --git a/src/boost/libs/histogram/benchmark/CMakeLists.txt b/src/boost/libs/histogram/benchmark/CMakeLists.txt new file mode 100644 index 000000000..1f2834cdd --- /dev/null +++ b/src/boost/libs/histogram/benchmark/CMakeLists.txt @@ -0,0 +1,60 @@ +# Copyright 2019 Hans Dembinski +# Distributed under the Boost Software License, Version 1.0. +# See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt + +include(BoostFetch) + +# setup for google benchmark +set(CMAKE_BUILD_TYPE Release) # ok, only set in local scope +option(BENCHMARK_ENABLE_TESTING "" OFF) +boost_fetch(google/benchmark) + +function(add_benchmark NAME) + + cmake_parse_arguments(PARSE_ARGV 1 _ "" "" ";INCLUDE_DIRECTORIES;LINK_LIBRARIES;COMPILE_OPTIONS") + + if(__UNPARSED_ARGUMENTS) + message(AUTHOR_WARNING "add_benchmark: extra arguments ignored: ${__UNPARSED_ARGUMENTS}") + endif() + + set(SOURCE ${NAME}) + set(NAME benchmark_${NAME}) + + if(DEFINED BUILD_TESTING AND NOT BUILD_TESTING) + return() + endif() + + add_executable(${NAME} ${SOURCE}) + target_include_directories(${NAME} PRIVATE ${__INCLUDE_DIRECTORIES}) + target_link_libraries(${NAME} PRIVATE Boost::histogram benchmark_main ${__LINK_LIBRARIES}) + target_compile_options(${NAME} PRIVATE -DNDEBUG -O3 -march=native -funsafe-math-optimizations ${__COMPILE_OPTIONS}) + +endfunction() + +add_benchmark(axis_index) +add_benchmark(histogram_filling) +add_benchmark(histogram_iteration) + +find_package(Threads) +if (Threads_FOUND) + add_benchmark(histogram_parallel_filling) +endif() + +find_package(GSL) +if (GSL_FOUND) + add_benchmark( + histogram_filling_gsl + INCLUDE_DIRECTORIES ${GSL_INCLUDE_DIRS} + LINK_LIBRARIES ${GSL_LIBRARIES}) +endif() + +find_package(ROOT QUIET) +if (ROOT_FOUND) + add_benchmark( + histogram_filling_root + INCLUDE_DIRECTORIES ${ROOT_INCLUDE_DIRS} + LINK_LIBRARIES ${ROOT_LIBRARIES} + COMPILE_OPTIONS -frtti -fexceptions + ) + # add ROOT linker flags? ${ROOT_EXE_LINKER_FLAGS} +endif() diff --git a/src/boost/libs/histogram/benchmark/axis_index.cpp b/src/boost/libs/histogram/benchmark/axis_index.cpp new file mode 100644 index 000000000..fb8e71f77 --- /dev/null +++ b/src/boost/libs/histogram/benchmark/axis_index.cpp @@ -0,0 +1,70 @@ +// Copyright 2018 Hans Dembinski +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt +// or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include <benchmark/benchmark.h> +#include <boost/histogram/axis.hpp> +#include <numeric> +#include "../test/throw_exception.hpp" +#include "generator.hpp" + +#include <boost/assert.hpp> +struct assert_check { + assert_check() { + BOOST_ASSERT(false); // don't run with asserts enabled + } +} _; + +using namespace boost::histogram; + +template <class Distribution> +static void regular(benchmark::State& state) { + auto a = axis::regular<>(100, 0.0, 1.0); + generator<Distribution> gen; + for (auto _ : state) benchmark::DoNotOptimize(a.index(gen())); +} + +template <class Distribution> +static void circular(benchmark::State& state) { + auto a = axis::circular<>(100, 0.0, 1.0); + generator<Distribution> gen; + for (auto _ : state) benchmark::DoNotOptimize(a.index(gen())); +} + +template <class T, class Distribution> +static void integer(benchmark::State& state) { + auto a = axis::integer<T>(0, 1); + generator<Distribution> gen; + for (auto _ : state) benchmark::DoNotOptimize(a.index(gen())); +} + +template <class Distribution> +static void variable(benchmark::State& state) { + std::vector<double> v; + for (double x = 0; x <= state.range(0); ++x) { v.push_back(x / state.range(0)); } + auto a = axis::variable<>(v); + generator<Distribution> gen; + for (auto _ : state) benchmark::DoNotOptimize(a.index(gen())); +} + +static void category(benchmark::State& state) { + std::vector<int> v(state.range(0)); + std::iota(v.begin(), v.end(), 0); + auto a = axis::category<int>(v); + generator<uniform_int> gen(static_cast<int>(state.range(0))); + for (auto _ : state) benchmark::DoNotOptimize(a.index(gen())); +} + +BENCHMARK_TEMPLATE(regular, uniform); +BENCHMARK_TEMPLATE(regular, normal); +BENCHMARK_TEMPLATE(circular, uniform); +BENCHMARK_TEMPLATE(circular, normal); +BENCHMARK_TEMPLATE(integer, int, uniform); +BENCHMARK_TEMPLATE(integer, int, normal); +BENCHMARK_TEMPLATE(integer, double, uniform); +BENCHMARK_TEMPLATE(integer, double, normal); +BENCHMARK_TEMPLATE(variable, uniform)->RangeMultiplier(10)->Range(10, 10000); +BENCHMARK_TEMPLATE(variable, normal)->RangeMultiplier(10)->Range(10, 10000); +BENCHMARK(category)->RangeMultiplier(10)->Range(10, 10000); diff --git a/src/boost/libs/histogram/benchmark/disable_cpu_scaling.sh b/src/boost/libs/histogram/benchmark/disable_cpu_scaling.sh new file mode 100755 index 000000000..f945e180e --- /dev/null +++ b/src/boost/libs/histogram/benchmark/disable_cpu_scaling.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Copyright Hans Dembinski 2019 +# Distributed under the Boost Software License, Version 1.0. +# See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt + +for cpu in /sys/devices/system/cpu/cpu? ; do echo performance > $cpu/cpufreq/scaling_governor; done diff --git a/src/boost/libs/histogram/benchmark/enable_cpu_scaling.sh b/src/boost/libs/histogram/benchmark/enable_cpu_scaling.sh new file mode 100755 index 000000000..d0a463eb0 --- /dev/null +++ b/src/boost/libs/histogram/benchmark/enable_cpu_scaling.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Copyright Hans Dembinski 2019 +# Distributed under the Boost Software License, Version 1.0. +# See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt + +for cpu in /sys/devices/system/cpu/cpu? ; do echo powersave > $cpu/cpufreq/scaling_governor; done diff --git a/src/boost/libs/histogram/benchmark/generator.hpp b/src/boost/libs/histogram/benchmark/generator.hpp new file mode 100644 index 000000000..f613ced35 --- /dev/null +++ b/src/boost/libs/histogram/benchmark/generator.hpp @@ -0,0 +1,50 @@ +// Copyright 2019 Hans Dembinski +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt +// or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include <array> +#include <random> + +using uniform = std::uniform_real_distribution<>; +using uniform_int = std::uniform_int_distribution<>; +using normal = std::normal_distribution<>; + +template <class Distribution, class... Ts> +Distribution init(Ts...); + +template <> +uniform init<uniform>() { + return uniform{0.0, 1.0}; +} + +template <> +normal init<normal>() { + return normal{0.5, 0.3}; +} + +template <> +uniform_int init<uniform_int, int>(int n) { + return uniform_int{0, n}; +} + +template <class Distribution, std::size_t N = 1 << 15> +struct generator : std::array<double, N> { + using base_t = std::array<double, N>; + + template <class... Ts> + generator(Ts... ts) { + std::default_random_engine rng(1); + auto dis = init<Distribution>(ts...); + std::generate(base_t::begin(), base_t::end(), [&] { return dis(rng); }); + } + + const double& operator()() { + ++ptr_; + if (ptr_ == base_t::data() + N) ptr_ = base_t::data(); + return *ptr_; + } + + const double* ptr_ = base_t::data() - 1; +}; diff --git a/src/boost/libs/histogram/benchmark/histogram_filling.cpp b/src/boost/libs/histogram/benchmark/histogram_filling.cpp new file mode 100644 index 000000000..0bc5164b8 --- /dev/null +++ b/src/boost/libs/histogram/benchmark/histogram_filling.cpp @@ -0,0 +1,176 @@ +// Copyright 2015-2019 Hans Dembinski +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt +// or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include <benchmark/benchmark.h> +#include <boost/histogram/axis/regular.hpp> +#include <boost/histogram/storage_adaptor.hpp> +#include <memory> +#include "../test/throw_exception.hpp" +#include "../test/utility_histogram.hpp" +#include "generator.hpp" + +#include <boost/assert.hpp> +struct assert_check { + assert_check() { + BOOST_ASSERT(false); // don't run with asserts enabled + } +} _; + +using SStore = std::vector<double>; + +// make benchmark compatible with older versions of the library +#if __has_include(<boost/histogram/unlimited_storage.hpp>) +#include <boost/histogram/unlimited_storage.hpp> +using DStore = boost::histogram::unlimited_storage<>; +#else +#include <boost/histogram/adaptive_storage.hpp> +using DStore = boost::histogram::adaptive_storage<>; +#endif + +using namespace boost::histogram; +using reg = axis::regular<>; + +template <class Distribution, class Tag, class Storage = SStore> +static void fill_1d(benchmark::State& state) { + auto h = make_s(Tag(), Storage(), reg(100, 0, 1)); + auto gen = generator<Distribution>(); + for (auto _ : state) benchmark::DoNotOptimize(h(gen())); + state.SetItemsProcessed(state.iterations()); +} + +template <class Distribution, class Tag, class Storage = SStore> +static void fill_n_1d(benchmark::State& state) { + auto h = make_s(Tag(), Storage(), reg(100, 0, 1)); + auto gen = generator<Distribution>(); + for (auto _ : state) h.fill(gen); + state.SetItemsProcessed(state.iterations() * gen.size()); +} + +template <class Distribution, class Tag, class Storage = SStore> +static void fill_2d(benchmark::State& state) { + auto h = make_s(Tag(), Storage(), reg(100, 0, 1), reg(100, 0, 1)); + auto gen = generator<Distribution>(); + for (auto _ : state) benchmark::DoNotOptimize(h(gen(), gen())); + state.SetItemsProcessed(state.iterations() * 2); +} + +template <class Distribution, class Tag, class Storage = SStore> +static void fill_n_2d(benchmark::State& state) { + auto h = make_s(Tag(), Storage(), reg(100, 0, 1), reg(100, 0, 1)); + auto gen = generator<Distribution>(); + auto v = {gen, gen}; + for (auto _ : state) h.fill(v); + state.SetItemsProcessed(state.iterations() * 2 * gen.size()); +} + +template <class Distribution, class Tag, class Storage = SStore> +static void fill_3d(benchmark::State& state) { + auto h = make_s(Tag(), Storage(), reg(100, 0, 1), reg(100, 0, 1), reg(100, 0, 1)); + auto gen = generator<Distribution>(); + for (auto _ : state) benchmark::DoNotOptimize(h(gen(), gen(), gen())); + state.SetItemsProcessed(state.iterations() * 3); +} + +template <class Distribution, class Tag, class Storage = SStore> +static void fill_n_3d(benchmark::State& state) { + auto h = make_s(Tag(), Storage(), reg(100, 0, 1), reg(100, 0, 1), reg(100, 0, 1)); + auto gen = generator<Distribution>(); + auto v = {gen, gen, gen}; + for (auto _ : state) h.fill(v); + state.SetItemsProcessed(state.iterations() * 3 * gen.size()); +} + +template <class Distribution, class Tag, class Storage = SStore> +static void fill_6d(benchmark::State& state) { + auto h = make_s(Tag(), Storage(), reg(10, 0, 1), reg(10, 0, 1), reg(10, 0, 1), + reg(10, 0, 1), reg(10, 0, 1), reg(10, 0, 1)); + auto gen = generator<Distribution>(); + for (auto _ : state) + benchmark::DoNotOptimize(h(gen(), gen(), gen(), gen(), gen(), gen())); + state.SetItemsProcessed(state.iterations() * 6); +} + +template <class Distribution, class Tag, class Storage = SStore> +static void fill_n_6d(benchmark::State& state) { + auto h = make_s(Tag(), Storage(), reg(10, 0, 1), reg(10, 0, 1), reg(10, 0, 1), + reg(10, 0, 1), reg(10, 0, 1), reg(10, 0, 1)); + auto gen = generator<Distribution>(); + auto v = {gen, gen, gen, gen, gen, gen}; + for (auto _ : state) h.fill(v); + state.SetItemsProcessed(state.iterations() * 6 * gen.size()); +} + +BENCHMARK_TEMPLATE(fill_1d, uniform, static_tag); +// BENCHMARK_TEMPLATE(fill_1d, uniform, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_1d, normal, static_tag); +// BENCHMARK_TEMPLATE(fill_1d, normal, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_1d, uniform, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_1d, uniform, dynamic_tag, DStore); +BENCHMARK_TEMPLATE(fill_1d, normal, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_1d, normal, dynamic_tag, DStore); + +BENCHMARK_TEMPLATE(fill_n_1d, uniform, static_tag); +// BENCHMARK_TEMPLATE(fill_n_1d, uniform, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_1d, normal, static_tag); +// BENCHMARK_TEMPLATE(fill_n_1d, normal, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_1d, uniform, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_n_1d, uniform, dynamic_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_1d, normal, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_n_1d, normal, dynamic_tag, DStore); + +BENCHMARK_TEMPLATE(fill_2d, uniform, static_tag); +// BENCHMARK_TEMPLATE(fill_2d, uniform, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_2d, normal, static_tag); +// BENCHMARK_TEMPLATE(fill_2d, normal, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_2d, uniform, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_2d, uniform, dynamic_tag, DStore); +BENCHMARK_TEMPLATE(fill_2d, normal, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_2d, normal, dynamic_tag, DStore); + +BENCHMARK_TEMPLATE(fill_n_2d, uniform, static_tag); +// BENCHMARK_TEMPLATE(fill_n_2d, uniform, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_2d, normal, static_tag); +// BENCHMARK_TEMPLATE(fill_n_2d, normal, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_2d, uniform, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_n_2d, uniform, dynamic_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_2d, normal, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_n_2d, normal, dynamic_tag, DStore); + +BENCHMARK_TEMPLATE(fill_3d, uniform, static_tag); +// BENCHMARK_TEMPLATE(fill_3d, uniform, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_3d, normal, static_tag); +// BENCHMARK_TEMPLATE(fill_3d, normal, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_3d, uniform, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_3d, uniform, dynamic_tag, DStore); +BENCHMARK_TEMPLATE(fill_3d, normal, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_3d, normal, dynamic_tag, DStore); + +BENCHMARK_TEMPLATE(fill_n_3d, uniform, static_tag); +// BENCHMARK_TEMPLATE(fill_n_3d, uniform, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_3d, normal, static_tag); +// BENCHMARK_TEMPLATE(fill_n_3d, normal, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_3d, uniform, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_n_3d, uniform, dynamic_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_3d, normal, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_n_3d, normal, dynamic_tag, DStore); + +BENCHMARK_TEMPLATE(fill_6d, uniform, static_tag); +// BENCHMARK_TEMPLATE(fill_6d, uniform, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_6d, normal, static_tag); +// BENCHMARK_TEMPLATE(fill_6d, normal, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_6d, uniform, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_6d, uniform, dynamic_tag, DStore); +BENCHMARK_TEMPLATE(fill_6d, normal, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_6d, normal, dynamic_tag, DStore); + +BENCHMARK_TEMPLATE(fill_n_6d, uniform, static_tag); +// BENCHMARK_TEMPLATE(fill_n_6d, uniform, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_6d, normal, static_tag); +// BENCHMARK_TEMPLATE(fill_n_6d, normal, static_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_6d, uniform, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_n_6d, uniform, dynamic_tag, DStore); +BENCHMARK_TEMPLATE(fill_n_6d, normal, dynamic_tag); +// BENCHMARK_TEMPLATE(fill_n_6d, normal, dynamic_tag, DStore); diff --git a/src/boost/libs/histogram/benchmark/histogram_filling_gsl.cpp b/src/boost/libs/histogram/benchmark/histogram_filling_gsl.cpp new file mode 100644 index 000000000..94d738966 --- /dev/null +++ b/src/boost/libs/histogram/benchmark/histogram_filling_gsl.cpp @@ -0,0 +1,45 @@ +// Copyright 2015-2019 Hans Dembinski +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt +// or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include <benchmark/benchmark.h> +#include <gsl/gsl_histogram.h> +#include <gsl/gsl_histogram2d.h> +#include "../test/throw_exception.hpp" +#include "generator.hpp" + +#include <boost/assert.hpp> +struct assert_check { + assert_check() { + BOOST_ASSERT(false); // don't run with asserts enabled + } +} _; + +template <class Distribution> +static void fill_1d(benchmark::State& state) { + gsl_histogram* h = gsl_histogram_alloc(100); + gsl_histogram_set_ranges_uniform(h, 0, 1); + generator<Distribution> gen; + for (auto _ : state) benchmark::DoNotOptimize(gsl_histogram_increment(h, gen())); + gsl_histogram_free(h); + state.SetItemsProcessed(state.iterations()); +} + +template <class Distribution> +static void fill_2d(benchmark::State& state) { + gsl_histogram2d* h = gsl_histogram2d_alloc(100, 100); + gsl_histogram2d_set_ranges_uniform(h, 0, 1, 0, 1); + generator<Distribution> gen; + for (auto _ : state) + benchmark::DoNotOptimize(gsl_histogram2d_increment(h, gen(), gen())); + gsl_histogram2d_free(h); + state.SetItemsProcessed(state.iterations() * 2); +} + +BENCHMARK_TEMPLATE(fill_1d, uniform); +BENCHMARK_TEMPLATE(fill_2d, uniform); + +BENCHMARK_TEMPLATE(fill_1d, normal); +BENCHMARK_TEMPLATE(fill_2d, normal); diff --git a/src/boost/libs/histogram/benchmark/histogram_filling_numpy.py b/src/boost/libs/histogram/benchmark/histogram_filling_numpy.py new file mode 100644 index 000000000..ce0fa2b73 --- /dev/null +++ b/src/boost/libs/histogram/benchmark/histogram_filling_numpy.py @@ -0,0 +1,14 @@ +from __future__ import print_function +import numpy as np +# pip install fast-histogram +from fast_histogram import histogram1d +import timeit + +x = np.random.rand(1 << 20) +nrepeat = 10 + +print(timeit.timeit("np.histogram(x, bins=100, range=(0, 1))", + "from __main__ import x, np", number=nrepeat) / (nrepeat * len(x)) / 1e-9) + +print(timeit.timeit("histogram1d(x, bins=100, range=(0, 1))", + "from __main__ import x, histogram1d", number=nrepeat) / (nrepeat * len(x)) / 1e-9)
\ No newline at end of file diff --git a/src/boost/libs/histogram/benchmark/histogram_filling_root.cpp b/src/boost/libs/histogram/benchmark/histogram_filling_root.cpp new file mode 100644 index 000000000..98d638517 --- /dev/null +++ b/src/boost/libs/histogram/benchmark/histogram_filling_root.cpp @@ -0,0 +1,67 @@ +// Copyright 2015-2019 Hans Dembinski +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt +// or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include <TH1I.h> +#include <TH2I.h> +#include <TH3I.h> +#include <THn.h> +#include <benchmark/benchmark.h> +#include "generator.hpp" + +#include <boost/assert.hpp> +struct assert_check { + assert_check() { + BOOST_ASSERT(false); // don't run with asserts enabled + } +} _; + +template <class Distribution> +static void fill_1d(benchmark::State& state) { + TH1I h("", "", 100, 0, 1); + generator<Distribution> gen; + for (auto _ : state) benchmark::DoNotOptimize(h.Fill(gen())); + state.SetItemsProcessed(state.iterations()); +} + +template <class Distribution> +static void fill_2d(benchmark::State& state) { + TH2I h("", "", 100, 0, 1, 100, 0, 1); + generator<Distribution> gen; + for (auto _ : state) benchmark::DoNotOptimize(h.Fill(gen(), gen())); + state.SetItemsProcessed(state.iterations() * 2); +} + +template <class Distribution> +static void fill_3d(benchmark::State& state) { + TH3I h("", "", 100, 0, 1, 100, 0, 1, 100, 0, 1); + generator<Distribution> gen; + for (auto _ : state) benchmark::DoNotOptimize(h.Fill(gen(), gen(), gen())); + state.SetItemsProcessed(state.iterations() * 3); +} + +template <class Distribution> +static void fill_6d(benchmark::State& state) { + int bin[] = {10, 10, 10, 10, 10, 10}; + double min[] = {0, 0, 0, 0, 0, 0}; + double max[] = {1, 1, 1, 1, 1, 1}; + THnI h("", "", 6, bin, min, max); + generator<Distribution> gen; + for (auto _ : state) { + const double buf[6] = {gen(), gen(), gen(), gen(), gen(), gen()}; + benchmark::DoNotOptimize(h.Fill(buf)); + } + state.SetItemsProcessed(state.iterations() * 6); +} + +BENCHMARK_TEMPLATE(fill_1d, uniform); +BENCHMARK_TEMPLATE(fill_2d, uniform); +BENCHMARK_TEMPLATE(fill_3d, uniform); +BENCHMARK_TEMPLATE(fill_6d, uniform); + +BENCHMARK_TEMPLATE(fill_1d, normal); +BENCHMARK_TEMPLATE(fill_2d, normal); +BENCHMARK_TEMPLATE(fill_3d, normal); +BENCHMARK_TEMPLATE(fill_6d, normal); diff --git a/src/boost/libs/histogram/benchmark/histogram_iteration.cpp b/src/boost/libs/histogram/benchmark/histogram_iteration.cpp new file mode 100644 index 000000000..bb18a0ba6 --- /dev/null +++ b/src/boost/libs/histogram/benchmark/histogram_iteration.cpp @@ -0,0 +1,178 @@ +// Copyright 2018 Hans Dembinski +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt +// or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include <benchmark/benchmark.h> +#include <boost/histogram/axis/integer.hpp> +#include <boost/histogram/axis/regular.hpp> +#include <boost/histogram/histogram.hpp> +#include <boost/histogram/indexed.hpp> +#include <boost/histogram/literals.hpp> +#include <boost/histogram/make_histogram.hpp> +#include <boost/mp11/integral.hpp> +#include <vector> +#include "../test/throw_exception.hpp" + +#include <boost/assert.hpp> +struct assert_check { + assert_check() { + BOOST_ASSERT(false); // don't run with asserts enabled + } +} _; + +using namespace boost::histogram; +using namespace boost::histogram::literals; + +struct tuple {}; +struct vector {}; +struct vector_of_variant {}; + +template <unsigned I> +using Dim_t = boost::mp11::mp_int<I>; + +using d1 = Dim_t<1>; +using d2 = Dim_t<2>; +using d3 = Dim_t<3>; + +auto make_histogram(tuple, d1, unsigned n) { + return make_histogram_with(std::vector<unsigned>(), axis::integer<>(0, n)); +} + +auto make_histogram(tuple, d2, unsigned n) { + return make_histogram_with(std::vector<unsigned>(), axis::integer<>(0, n), + axis::integer<>(0, n)); +} + +auto make_histogram(tuple, d3, unsigned n) { + return make_histogram_with(std::vector<unsigned>(), axis::integer<>(0, n), + axis::integer<>(0, n), axis::integer<>(0, n)); +} + +template <int Dim> +auto make_histogram(vector, boost::mp11::mp_int<Dim>, unsigned n) { + std::vector<axis::integer<>> axes; + for (unsigned d = 0; d < Dim; ++d) axes.emplace_back(axis::integer<>(0, n)); + return make_histogram_with(std::vector<unsigned>(), std::move(axes)); +} + +template <int Dim> +auto make_histogram(vector_of_variant, boost::mp11::mp_int<Dim>, unsigned n) { + std::vector<axis::variant<axis::integer<>>> axes; + for (unsigned d = 0; d < Dim; ++d) axes.emplace_back(axis::integer<>(0, n)); + return make_histogram_with(std::vector<unsigned>(), std::move(axes)); +} + +template <class Tag> +static void Naive(benchmark::State& state, Tag, d1, coverage cov) { + auto h = make_histogram(Tag(), d1(), state.range(0)); + const int d = cov == coverage::all; + for (auto _ : state) { + for (int i = -d; i < h.axis().size() + d; ++i) { + benchmark::DoNotOptimize(i); + benchmark::DoNotOptimize(h.at(i)); + } + } +} + +template <class Tag> +static void Naive(benchmark::State& state, Tag, d2, coverage cov) { + auto h = make_histogram(Tag(), d2(), state.range(0)); + const int d = cov == coverage::all; + for (auto _ : state) { + for (int i = -d; i < h.axis(0_c).size() + d; ++i) { + for (int j = -d; j < h.axis(1_c).size() + d; ++j) { + benchmark::DoNotOptimize(i); + benchmark::DoNotOptimize(j); + benchmark::DoNotOptimize(h.at(i, j)); + } + } + } +} + +template <class Tag> +static void Naive(benchmark::State& state, Tag, d3, coverage cov) { + auto h = make_histogram(Tag(), d3(), state.range(0)); + const int d = cov == coverage::all; + for (auto _ : state) { + for (int i = -d; i < h.axis(0_c).size() + d; ++i) { + for (int j = -d; j < h.axis(1_c).size() + d; ++j) { + for (int k = -d; k < h.axis(2_c).size() + d; ++k) { + benchmark::DoNotOptimize(i); + benchmark::DoNotOptimize(j); + benchmark::DoNotOptimize(k); + benchmark::DoNotOptimize(h.at(i, j, k)); + } + } + } + } +} + +template <class Tag> +static void Indexed(benchmark::State& state, Tag, d1, coverage cov) { + auto h = make_histogram(Tag(), d1(), state.range(0)); + for (auto _ : state) { + for (auto&& x : indexed(h, cov)) { + benchmark::DoNotOptimize(*x); + benchmark::DoNotOptimize(x.index()); + } + } +} + +template <class Tag> +static void Indexed(benchmark::State& state, Tag, d2, coverage cov) { + auto h = make_histogram(Tag(), d2(), state.range(0)); + for (auto _ : state) { + for (auto&& x : indexed(h, cov)) { + benchmark::DoNotOptimize(*x); + benchmark::DoNotOptimize(x.index(0)); + benchmark::DoNotOptimize(x.index(1)); + } + } +} + +template <class Tag> +static void Indexed(benchmark::State& state, Tag, d3, coverage cov) { + auto h = make_histogram(Tag(), d3(), state.range(0)); + for (auto _ : state) { + for (auto&& x : indexed(h, cov)) { + benchmark::DoNotOptimize(*x); + benchmark::DoNotOptimize(x.index(0)); + benchmark::DoNotOptimize(x.index(1)); + benchmark::DoNotOptimize(x.index(2)); + } + } +} + +#define BENCH(Type, Tag, Dim, Cov) \ + BENCHMARK_CAPTURE(Type, (Tag, Dim, Cov), Tag{}, Dim_t<Dim>{}, coverage::Cov) \ + ->RangeMultiplier(4) \ + ->Range(4, 256) + +BENCH(Naive, tuple, 1, inner); +BENCH(Indexed, tuple, 1, inner); + +BENCH(Naive, vector, 1, inner); +BENCH(Indexed, vector, 1, inner); + +BENCH(Naive, vector_of_variant, 1, inner); +BENCH(Indexed, vector_of_variant, 1, inner); + +BENCH(Naive, tuple, 2, inner); +BENCH(Indexed, tuple, 2, inner); + +BENCH(Naive, vector, 2, inner); +BENCH(Indexed, vector, 2, inner); + +BENCH(Naive, vector_of_variant, 2, inner); +BENCH(Indexed, vector_of_variant, 2, inner); + +BENCH(Naive, tuple, 3, inner); +BENCH(Indexed, tuple, 3, inner); + +BENCH(Naive, vector, 3, inner); +BENCH(Indexed, vector, 3, inner); + +BENCH(Naive, vector_of_variant, 3, inner); +BENCH(Indexed, vector_of_variant, 3, inner); diff --git a/src/boost/libs/histogram/benchmark/histogram_parallel_filling.cpp b/src/boost/libs/histogram/benchmark/histogram_parallel_filling.cpp new file mode 100644 index 000000000..0ca7eaada --- /dev/null +++ b/src/boost/libs/histogram/benchmark/histogram_parallel_filling.cpp @@ -0,0 +1,149 @@ +// Copyright 2015-2018 Hans Dembinski +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt +// or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include <benchmark/benchmark.h> +#include <boost/histogram/accumulators/thread_safe.hpp> +#include <boost/histogram/axis/regular.hpp> +#include <boost/histogram/histogram.hpp> +#include <boost/histogram/make_histogram.hpp> +#include <chrono> +#include <functional> +#include <mutex> +#include <numeric> +#include <random> +#include <thread> +#include <vector> +#include "../test/throw_exception.hpp" + +#include <boost/assert.hpp> +struct assert_check { + assert_check() { + BOOST_ASSERT(false); // don't run with asserts enabled + } +} _; + +using namespace boost::histogram; +using namespace std::chrono_literals; + +using DS = dense_storage<unsigned>; +using DSTS = dense_storage<accumulators::thread_safe<unsigned>>; + +static void NoThreads(benchmark::State& state) { + std::default_random_engine gen(1); + std::uniform_real_distribution<> dis(0, 1); + const unsigned nbins = state.range(0); + auto hist = make_histogram_with(DS(), axis::regular<>(nbins, 0, 1)); + for (auto _ : state) { + // simulate some work + for (volatile unsigned n = 0; n < state.range(1); ++n) + ; + + hist(dis(gen)); + } +} + +std::mutex init; +static auto hist = make_histogram_with(DSTS(), axis::regular<>()); + +static void AtomicStorage(benchmark::State& state) { + init.lock(); + if (state.thread_index == 0) { + const unsigned nbins = state.range(0); + hist = make_histogram_with(DSTS(), axis::regular<>(nbins, 0, 1)); + } + init.unlock(); + std::default_random_engine gen(state.thread_index); + std::uniform_real_distribution<> dis(0, 1); + for (auto _ : state) { + // simulate some work + for (volatile unsigned n = 0; n < state.range(1); ++n) + ; + hist(dis(gen)); + } +} + +BENCHMARK(NoThreads) + ->UseRealTime() + + ->Args({1 << 4, 0}) + ->Args({1 << 6, 0}) + ->Args({1 << 8, 0}) + ->Args({1 << 10, 0}) + ->Args({1 << 14, 0}) + ->Args({1 << 18, 0}) + + ->Args({1 << 4, 5}) + ->Args({1 << 6, 5}) + ->Args({1 << 8, 5}) + ->Args({1 << 10, 5}) + ->Args({1 << 14, 5}) + ->Args({1 << 18, 5}) + + ->Args({1 << 4, 10}) + ->Args({1 << 6, 10}) + ->Args({1 << 8, 10}) + ->Args({1 << 10, 10}) + ->Args({1 << 14, 10}) + ->Args({1 << 18, 10}) + + ->Args({1 << 4, 50}) + ->Args({1 << 6, 50}) + ->Args({1 << 8, 50}) + ->Args({1 << 10, 50}) + ->Args({1 << 14, 50}) + ->Args({1 << 18, 50}) + + ->Args({1 << 4, 100}) + ->Args({1 << 6, 100}) + ->Args({1 << 8, 100}) + ->Args({1 << 10, 100}) + ->Args({1 << 14, 100}) + ->Args({1 << 18, 100}) + + ; + +BENCHMARK(AtomicStorage) + ->UseRealTime() + ->Threads(1) + ->Threads(2) + ->Threads(4) + + ->Args({1 << 4, 0}) + ->Args({1 << 6, 0}) + ->Args({1 << 8, 0}) + ->Args({1 << 10, 0}) + ->Args({1 << 14, 0}) + ->Args({1 << 18, 0}) + + ->Args({1 << 4, 5}) + ->Args({1 << 6, 5}) + ->Args({1 << 8, 5}) + ->Args({1 << 10, 5}) + ->Args({1 << 14, 5}) + ->Args({1 << 18, 5}) + + ->Args({1 << 4, 10}) + ->Args({1 << 6, 10}) + ->Args({1 << 8, 10}) + ->Args({1 << 10, 10}) + ->Args({1 << 14, 10}) + ->Args({1 << 18, 10}) + + ->Args({1 << 4, 50}) + ->Args({1 << 6, 50}) + ->Args({1 << 8, 50}) + ->Args({1 << 10, 50}) + ->Args({1 << 14, 50}) + ->Args({1 << 18, 50}) + + ->Args({1 << 4, 100}) + ->Args({1 << 6, 100}) + ->Args({1 << 8, 100}) + ->Args({1 << 10, 100}) + ->Args({1 << 14, 100}) + ->Args({1 << 18, 100}) + + ; diff --git a/src/boost/libs/histogram/benchmark/plot_benchmarks.py b/src/boost/libs/histogram/benchmark/plot_benchmarks.py new file mode 100755 index 000000000..2661549fd --- /dev/null +++ b/src/boost/libs/histogram/benchmark/plot_benchmarks.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +# Copyright Hans Dembinski 2019 +# Distributed under the Boost Software License, Version 1.0. +# See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt + +from matplotlib import pyplot as plt, lines +import shelve +import json +import subprocess as subp +import sys +from collections import defaultdict +from run_benchmarks import get_commits, run +import numpy as np +import threading + +thread = None +current_index = 0 + +commits, comments = get_commits() + +def get_benchmarks(results): + benchmarks = defaultdict(lambda: []) + for hash in commits: + if hash in results and results[hash] is not None: + benchs = results[hash] + for b in benchs["benchmarks"]: + name = b["name"] + time = min(b["cpu_time"], b["real_time"]) + benchmarks[name].append((commits.index(hash), time)) + return benchmarks + +with shelve.open("benchmark_results") as results: + benchmarks = get_benchmarks(results) + +fig, ax = plt.subplots(4, 1, figsize=(10, 10), sharex=True) +plt.subplots_adjust(hspace=0, top=0.98, bottom=0.05, right=0.96) + +plt.sca(ax[0]) +for name, xy in benchmarks.items(): + if "uniform" in name: continue + if "_1d" in name: + x, y = np.transpose(xy) + plt.plot(x, y, ".-", label=name) +plt.legend(fontsize="xx-small") + +plt.sca(ax[1]) +for name, xy in benchmarks.items(): + if "uniform" in name: continue + if "_2d" in name: + x, y = np.transpose(xy) + plt.plot(x, y, ".-", label=name) +plt.legend(fontsize="xx-small") + +plt.sca(ax[2]) +for name, xy in benchmarks.items(): + if "uniform" in name: continue + if "_3d" in name: + x, y = np.transpose(xy) + plt.plot(x, y, ".-", label=name) +plt.legend(fontsize="xx-small") + +plt.sca(ax[3]) +for name, xy in benchmarks.items(): + if "uniform" in name: continue + if "_6d" in name: + x, y = np.transpose(xy) + plt.plot(x, y, ".-", label=name) +plt.legend(fontsize="xx-small") + +plt.figtext(0.01, 0.5, "time per loop / ns [smaller is better]", rotation=90, va="center") + +def format_coord(x, y): + global current_index + current_index = max(0, min(int(x + 0.5), len(commits) - 1)) + hash = commits[current_index] + comment = comments[hash] + return f"{hash} {comment}" + +for axi in ax.flatten(): + axi.format_coord = format_coord + +def on_key_press(event): + global thread + if thread and thread.is_alive(): return + + if event.key != "u": return + + hash = commits[current_index] + + def worker(fig, ax, hash): + with shelve.open("benchmark_results") as results: + run(results, comments, hash, True) + benchmarks = get_benchmarks(results) + + for name in benchmarks: + xy = benchmarks[name] + x, y = np.transpose(xy) + for axi in ax.flatten(): + for artist in axi.get_children(): + if isinstance(artist, lines.Line2D) and artist.get_label() == name: + artist.set_xdata(x) + artist.set_ydata(y) + + fig.canvas.draw() + + thread = threading.Thread(target=worker, args=(fig, ax, hash)) + thread.start() + +fig.canvas.mpl_connect('key_press_event', on_key_press) + +plt.show() diff --git a/src/boost/libs/histogram/benchmark/run_benchmarks.py b/src/boost/libs/histogram/benchmark/run_benchmarks.py new file mode 100755 index 000000000..db1055013 --- /dev/null +++ b/src/boost/libs/histogram/benchmark/run_benchmarks.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +# Copyright Hans Dembinski 2019 +# Distributed under the Boost Software License, Version 1.0. +# See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt + +""" +This script runs the benchmarks on previous versions of this library to track changes +in performance. + +Run this from a special build directory that uses the benchmark folder as root + + cd my_build_dir + cmake ../benchmark + ../run_benchmarks.py + +This creates a database, benchmark_results. Plot it: + + ../plot_benchmarks.py + +The script leaves the include folder in a modified state. To clean up, do: + + git checkout HEAD -- ../include + git clean -f -- ../include + +""" +import subprocess as subp +import tempfile +import os +import shelve +import json +import argparse + + +def get_commits(): + commits = [] + comments = {} + for line in subp.check_output(("git", "log", "--oneline")).decode("ascii").split("\n"): + if line: + ispace = line.index(" ") + hash = line[:ispace] + commits.append(hash) + comments[hash] = line[ispace+1:] + commits = commits[::-1] + return commits, comments + + +def recursion(results, commits, comments, ia, ib): + ic = int((ia + ib) / 2) + if ic == ia: + return + run(results, comments, commits[ic], False) + if all([results[commits[i]] is None for i in (ia, ib, ic)]): + return + recursion(results, commits, comments, ic, ib) + recursion(results, commits, comments, ia, ic) + + +def run(results, comments, hash, update): + if not update and hash in results: + return + print(hash, comments[hash]) + subp.call(("rm", "-rf", "../include")) + if subp.call(("git", "checkout", hash, "--", "../include")) != 0: + print("[Benchmark] Cannot checkout include folder\n") + return + print(hash, "make") + with tempfile.TemporaryFile() as out: + if subp.call(("make", "-j4", "histogram_filling"), stdout=out, stderr=out) != 0: + print("[Benchmark] Cannot make benchmarks\n") + out.seek(0) + print(out.read().decode("utf-8") + "\n") + return + print(hash, "run") + s = subp.check_output(("./histogram_filling", "--benchmark_format=json", "--benchmark_filter=normal")) + d = json.loads(s) + if update and hash in results and results[hash] is not None: + d2 = results[hash] + for i, (b, b2) in enumerate(zip(d["benchmarks"], d2["benchmarks"])): + d["benchmarks"][i] = b if b["cpu_time"] < b2["cpu_time"] else b2 + results[hash] = d + for benchmark in d["benchmarks"]: + print(benchmark["name"], min(benchmark["real_time"], benchmark["cpu_time"])) + + +def main(): + commits, comments = get_commits() + + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("first", type=str, default="begin", + help="first commit in range, special value `begin` is allowed") + parser.add_argument("last", type=str, default="end", + help="last commit in range, special value `end` is allowed") + parser.add_argument("-f", action="store_true", + help="override previous results") + + args = parser.parse_args() + + if args.first == "begin": + args.first = commits[0] + if args.last == "end": + args.last = commits[-1] + + with shelve.open("benchmark_results") as results: + a = commits.index(args.first) + b = commits.index(args.last) + if args.f: + for hash in commits[a:b+1]: + del results[hash] + run(results, comments, args.first, False) + run(results, comments, args.last, False) + recursion(results, commits, comments, a, b) + +if __name__ == "__main__": + main() |