summaryrefslogtreecommitdiffstats
path: root/contrib/ankerl
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-10 21:30:40 +0000
commit133a45c109da5310add55824db21af5239951f93 (patch)
treeba6ac4c0a950a0dda56451944315d66409923918 /contrib/ankerl
parentInitial commit. (diff)
downloadrspamd-upstream.tar.xz
rspamd-upstream.zip
Adding upstream version 3.8.1.upstream/3.8.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'contrib/ankerl')
-rw-r--r--contrib/ankerl/LICENSE21
-rw-r--r--contrib/ankerl/svector.h999
-rw-r--r--contrib/ankerl/unordered_dense.h2032
3 files changed, 3052 insertions, 0 deletions
diff --git a/contrib/ankerl/LICENSE b/contrib/ankerl/LICENSE
new file mode 100644
index 0000000..c4d1a0e
--- /dev/null
+++ b/contrib/ankerl/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Martin Leitner-Ankerl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/contrib/ankerl/svector.h b/contrib/ankerl/svector.h
new file mode 100644
index 0000000..dbd075b
--- /dev/null
+++ b/contrib/ankerl/svector.h
@@ -0,0 +1,999 @@
+// ┌─┐┬ ┬┌─┐┌─┐┌┬┐┌─┐┬─┐ Compact SVO optimized vector C++17 or higher
+// └─┐└┐┌┘├┤ │ │ │ │├┬┘ Version 1.0.2
+// └─┘ └┘ └─┘└─┘ ┴ └─┘┴└─ https://github.com/martinus/svector
+//
+// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#ifndef ANKERL_SVECTOR_H
+#define ANKERL_SVECTOR_H
+
+// see https://semver.org/spec/v2.0.0.html
+#define ANKERL_SVECTOR_VERSION_MAJOR 1 // incompatible API changes
+#define ANKERL_SVECTOR_VERSION_MINOR 0 // add functionality in a backwards compatible manner
+#define ANKERL_SVECTOR_VERSION_PATCH 2 // backwards compatible bug fixes
+
+// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/
+#define ANKERL_SVECTOR_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch
+#define ANKERL_SVECTOR_VERSION_CONCAT(major, minor, patch) ANKERL_SVECTOR_VERSION_CONCAT1(major, minor, patch)
+#define ANKERL_SVECTOR_NAMESPACE \
+ ANKERL_SVECTOR_VERSION_CONCAT(ANKERL_SVECTOR_VERSION_MAJOR, ANKERL_SVECTOR_VERSION_MINOR, ANKERL_SVECTOR_VERSION_PATCH)
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <initializer_list>
+#include <iterator>
+#include <limits>
+#include <memory>
+#include <new>
+#include <stdexcept>
+#include <type_traits>
+#include <utility>
+
+namespace ankerl {
+inline namespace ANKERL_SVECTOR_NAMESPACE {
+namespace detail {
+
+template <typename Condition, typename T = void>
+using enable_if_t = typename std::enable_if<Condition::value, T>::type;
+
+template <typename It>
+using is_input_iterator = std::is_base_of<std::input_iterator_tag, typename std::iterator_traits<It>::iterator_category>;
+
+constexpr auto round_up(size_t n, size_t multiple) -> size_t {
+ return ((n + (multiple - 1)) / multiple) * multiple;
+}
+
+template <typename T>
+constexpr auto cx_min(T a, T b) -> T {
+ return a < b ? a : b;
+}
+
+template <typename T>
+constexpr auto cx_max(T a, T b) -> T {
+ return a > b ? a : b;
+}
+
+template <typename T>
+constexpr auto alignment_of_svector() -> size_t {
+ return cx_max(sizeof(void*), std::alignment_of_v<T>);
+}
+
+/**
+ * @brief Calculates sizeof(svector<T, N>) for a given type and inline capacity
+ */
+template <typename T>
+constexpr auto size_of_svector(size_t min_inline_capacity) -> size_t {
+ // + 1 for one byte size in direct mode
+ return round_up(sizeof(T) * min_inline_capacity + 1, alignment_of_svector<T>());
+}
+
+/**
+ * @brief Calculates how many T we can actually store inside of an svector without increasing its sizeof().
+ *
+ * E.g. svector<char, 1> could store 7 bytes even though 1 is specified. This makes sure we don't waste any
+ * of the padding.
+ */
+template <typename T>
+constexpr auto automatic_capacity(size_t min_inline_capacity) -> size_t {
+ return cx_min((size_of_svector<T>(min_inline_capacity) - 1U) / sizeof(T), size_t{127});
+}
+
+/**
+ * Holds size & capacity, a glorified struct.
+ */
+class header {
+ size_t m_size{};
+ size_t const m_capacity;
+
+public:
+ inline explicit header(size_t capacity)
+ : m_capacity{capacity} {}
+
+ [[nodiscard]] inline auto size() const -> size_t {
+ return m_size;
+ }
+
+ [[nodiscard]] inline auto capacity() const -> size_t {
+ return m_capacity;
+ }
+
+ inline void size(size_t s) {
+ m_size = s;
+ }
+};
+
+/**
+ * @brief Holds header (size+capacity) plus an arbitrary number of T.
+ *
+ * To make storage compact, we don't actually store a pointer to T. We don't have to
+ * because we know exactly at which location it begins.
+ */
+template <typename T>
+struct storage : public header {
+ static constexpr auto alignment_of_t = std::alignment_of_v<T>;
+ static constexpr auto max_alignment = std::max(std::alignment_of_v<header>, std::alignment_of_v<T>);
+ static constexpr auto offset_to_data = detail::round_up(sizeof(header), alignment_of_t);
+ static_assert(max_alignment <= __STDCPP_DEFAULT_NEW_ALIGNMENT__);
+
+ explicit storage(size_t capacity)
+ : header(capacity) {}
+
+ auto data() -> T* {
+ auto ptr_to_data = reinterpret_cast<std::byte*>(this) + offset_to_data;
+ return std::launder(reinterpret_cast<T*>(ptr_to_data));
+ }
+
+ /**
+ * @brief Allocates space for storage plus capacity*T objects.
+ *
+ * Checks to make sure that allocation won't overflow.
+ *
+ * @param capacity Number of T to allocate.
+ * @return storage<T>*
+ */
+ static auto alloc(size_t capacity) -> storage<T>* {
+ // make sure we don't overflow!
+ auto mem = sizeof(T) * capacity;
+ if (mem < capacity) {
+ throw std::bad_alloc();
+ }
+ if (offset_to_data + mem < mem) {
+ throw std::bad_alloc();
+ }
+ mem += offset_to_data;
+ if (static_cast<uint64_t>(mem) > static_cast<uint64_t>(std::numeric_limits<std::ptrdiff_t>::max())) {
+ throw std::bad_alloc();
+ }
+
+ void* ptr = ::operator new(offset_to_data + sizeof(T) * capacity);
+ if (nullptr == ptr) {
+ throw std::bad_alloc();
+ }
+ // use void* to ensure we don't use an overload for T*
+ return new (ptr) storage<T>(capacity);
+ }
+};
+
+} // namespace detail
+
+template <typename T, size_t MinInlineCapacity>
+class svector {
+ static_assert(MinInlineCapacity <= 127, "sorry, can't have more than 127 direct elements");
+ static constexpr auto N = detail::automatic_capacity<T>(MinInlineCapacity);
+
+ enum class direction { direct, indirect };
+
+ /**
+ * A buffer to hold the data of the svector Depending on direct/indirect mode, the content it holds is like so:
+ *
+ * direct:
+ * m_data[0] & 1: lowest bit is 1 for direct mode.
+ * m_data[0] >> 1: size for direct mode
+ * Then 0-X bytes unused (padding), and then the actual inline T data.
+ * indirect:
+ * m_data[0] & 1: lowest bit is 0 for indirect mode
+ * m_data[0..7]: stores an uintptr_t, which points to the indirect data.
+ */
+ alignas(detail::alignment_of_svector<T>()) std::array<uint8_t, detail::size_of_svector<T>(MinInlineCapacity)> m_data;
+
+ // direct mode ///////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto is_direct() const -> bool {
+ return (m_data[0] & 1U) != 0U;
+ }
+
+ [[nodiscard]] auto direct_size() const -> size_t {
+ return m_data[0] >> 1U;
+ }
+
+ // sets size of direct mode and mode to direct too.
+ constexpr void set_direct_and_size(size_t s) {
+ m_data[0] = (s << 1U) | 1U;
+ }
+
+ [[nodiscard]] auto direct_data() -> T* {
+ return std::launder(reinterpret_cast<T*>(m_data.data() + std::alignment_of_v<T>));
+ }
+
+ // indirect mode /////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto indirect() -> detail::storage<T>* {
+ detail::storage<T>* ptr; // NOLINT(cppcoreguidelines-init-variables)
+ std::memcpy(&ptr, m_data.data(), sizeof(ptr));
+ return ptr;
+ }
+
+ [[nodiscard]] auto indirect() const -> detail::storage<T> const* {
+ return const_cast<svector*>(this)->indirect(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ void set_indirect(detail::storage<T>* ptr) {
+ std::memcpy(m_data.data(), &ptr, sizeof(ptr));
+
+ // safety check to guarantee the lowest bit is 0
+ if (is_direct()) {
+ throw std::bad_alloc(); // LCOV_EXCL_LINE
+ }
+ }
+
+ // helpers ///////////////////////////////////////////////////////////////
+
+ /**
+ * @brief Moves size objects from source_ptr to target_ptr, and destroys what remains in source_ptr.
+ *
+ * Assumes data is not overlapping
+ */
+ static void uninitialized_move_and_destroy(T* source_ptr, T* target_ptr, size_t size) {
+ if constexpr (std::is_trivially_copyable_v<T>) {
+ std::memcpy(target_ptr, source_ptr, size * sizeof(T));
+ } else {
+ std::uninitialized_move_n(source_ptr, size, target_ptr);
+ std::destroy_n(source_ptr, size);
+ }
+ }
+
+ /**
+ * @brief Reallocates all data when capacity changes.
+ *
+ * if new_capacity <= N chooses direct memory, otherwise indirect.
+ */
+ void realloc(size_t new_capacity) {
+ if (new_capacity <= N) {
+ // put everything into direct storage
+ if (is_direct()) {
+ // direct -> direct: nothing to do!
+ return;
+ }
+
+ // indirect -> direct
+ auto* storage = indirect();
+ uninitialized_move_and_destroy(storage->data(), direct_data(), storage->size());
+ set_direct_and_size(storage->size());
+ std::destroy_at(storage);
+ ::operator delete(storage);
+ } else {
+ // put everything into indirect storage
+ auto* storage = detail::storage<T>::alloc(new_capacity);
+ if (is_direct()) {
+ // direct -> indirect
+ uninitialized_move_and_destroy(data<direction::direct>(), storage->data(), size<direction::direct>());
+ storage->size(size<direction::direct>());
+ } else {
+ // indirect -> indirect
+ uninitialized_move_and_destroy(data<direction::indirect>(), storage->data(), size<direction::indirect>());
+ storage->size(size<direction::indirect>());
+ auto* storage = indirect();
+ std::destroy_at(storage);
+ ::operator delete(storage);
+ }
+ set_indirect(storage);
+ }
+ }
+
+ /**
+ * @brief Doubles starting_capacity until it is >= size_to_fit.
+ */
+ [[nodiscard]] static auto calculate_new_capacity(size_t size_to_fit, size_t starting_capacity) -> size_t {
+ if (size_to_fit > max_size()) {
+ // not enough space
+ throw std::bad_alloc();
+ }
+
+ if (size_to_fit == 0) {
+ // special handling for 0 so N==0 works
+ return starting_capacity;
+ }
+ // start with at least 1, so N==0 works
+ auto new_capacity = std::max<size_t>(1, starting_capacity);
+
+ // double capacity until its large enough, but make sure we don't overflow
+ while (new_capacity < size_to_fit && new_capacity * 2 > new_capacity) {
+ new_capacity *= 2;
+ }
+ if (new_capacity < size_to_fit) {
+ // got an overflow, set capacity to max
+ new_capacity = max_size();
+ }
+ return std::min(new_capacity, max_size());
+ }
+
+ template <direction D>
+ [[nodiscard]] auto capacity() const -> size_t {
+ if constexpr (D == direction::direct) {
+ return N;
+ } else {
+ return indirect()->capacity();
+ }
+ }
+
+ template <direction D>
+ [[nodiscard]] auto size() const -> size_t {
+ if constexpr (D == direction::direct) {
+ return direct_size();
+ } else {
+ return indirect()->size();
+ }
+ }
+
+ template <direction D>
+ void set_size(size_t s) {
+ if constexpr (D == direction::direct) {
+ set_direct_and_size(s);
+ } else {
+ indirect()->size(s);
+ }
+ }
+
+ void set_size(size_t s) {
+ if (is_direct()) {
+ set_size<direction::direct>(s);
+ } else {
+ set_size<direction::indirect>(s);
+ }
+ }
+
+ template <direction D>
+ [[nodiscard]] auto data() -> T* {
+ if constexpr (D == direction::direct) {
+ return direct_data();
+ } else {
+ return indirect()->data();
+ }
+ }
+
+ template <direction D>
+ [[nodiscard]] auto data() const -> T const* {
+ return const_cast<svector*>(this)->data<D>(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ template <direction D>
+ void pop_back() {
+ if constexpr (std::is_trivially_destructible_v<T>) {
+ set_size<D>(size<D>() - 1);
+ } else {
+ auto s = size<D>() - 1;
+ (data<D>() + s)->~T();
+ set_size<D>(s);
+ }
+ }
+
+ /**
+ * @brief We need variadic arguments so we can either use copy ctor or default ctor
+ */
+ template <direction D, class... Args>
+ void resize_after_reserve(size_t count, Args&&... args) {
+ auto current_size = size<D>();
+ if (current_size > count) {
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ auto* d = data<D>();
+ std::destroy(d + count, d + current_size);
+ }
+ } else {
+ auto* d = data<D>();
+ for (auto ptr = d + current_size, end = d + count; ptr != end; ++ptr) {
+ new (static_cast<void*>(ptr)) T(std::forward<Args>(args)...);
+ }
+ }
+ set_size<D>(count);
+ }
+
+ // Makes sure that to is not past the end iterator
+ template <direction D>
+ auto erase_checked_end(T const* cfrom, T const* to) -> T* {
+ auto* const erase_begin = const_cast<T*>(cfrom); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ auto* const container_end = data<D>() + size<D>();
+ auto* const erase_end = std::min(const_cast<T*>(to), container_end); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+
+ std::move(erase_end, container_end, erase_begin);
+ auto const num_erased = std::distance(erase_begin, erase_end);
+ std::destroy(container_end - num_erased, container_end);
+ set_size<D>(size<D>() - num_erased);
+ return erase_begin;
+ }
+
+ template <typename It>
+ void assign(It first, It last, std::input_iterator_tag /*unused*/) {
+ clear();
+
+ // TODO this can be made faster, e.g. by setting size only when finished.
+ while (first != last) {
+ push_back(*first);
+ ++first;
+ }
+ }
+
+ template <typename It>
+ void assign(It first, It last, std::forward_iterator_tag /*unused*/) {
+ clear();
+
+ auto s = std::distance(first, last);
+ reserve(s);
+ std::uninitialized_copy(first, last, data());
+ set_size(s);
+ }
+
+ // precondition: all uninitialized
+ void do_move_assign(svector&& other) {
+ if (!other.is_direct()) {
+ // take other's memory, even when empty
+ set_indirect(other.indirect());
+ } else {
+ auto* other_ptr = other.data<direction::direct>();
+ auto s = other.size<direction::direct>();
+ auto* other_end = other_ptr + s;
+
+ std::uninitialized_move(other_ptr, other_end, data<direction::direct>());
+ std::destroy(other_ptr, other_end);
+ set_size(s);
+ }
+ other.set_direct_and_size(0);
+ }
+
+ /**
+ * @brief Shifts data [source_begin, source_end( to the right, starting on target_begin.
+ *
+ * Preconditions:
+ * * contiguous memory
+ * * source_begin <= target_begin
+ * * source_end onwards is uninitialized memory
+ *
+ * Destroys then empty elements in [source_begin, source_end(
+ */
+ static void shift_right(T* source_begin, T* source_end, T* target_begin) {
+ // 1. uninitialized moves
+ auto const num_moves = std::distance(source_begin, source_end);
+ auto const target_end = target_begin + num_moves;
+ auto const num_uninitialized_move = std::min(num_moves, std::distance(source_end, target_end));
+ std::uninitialized_move(source_end - num_uninitialized_move, source_end, target_end - num_uninitialized_move);
+ std::move_backward(source_begin, source_end - num_uninitialized_move, target_end - num_uninitialized_move);
+ std::destroy(source_begin, std::min(source_end, target_begin));
+ }
+
+ template <direction D>
+ [[nodiscard]] auto make_uninitialized_space_new(size_t s, T* p, size_t count) -> T* {
+ auto target = svector();
+ // we know target is indirect because we're increasing capacity
+ target.reserve(s + count);
+
+ // move everything [begin, pos[
+ auto* target_pos = std::uninitialized_move(data<D>(), p, target.template data<direction::indirect>());
+
+ // move everything [pos, end]
+ std::uninitialized_move(p, data<D>() + s, target_pos + count);
+
+ target.template set_size<direction::indirect>(s + count);
+ *this = std::move(target);
+ return target_pos;
+ }
+
+ template <direction D>
+ [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* {
+ auto* const p = const_cast<T*>(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ auto s = size<D>();
+ if (s + count > capacity<D>()) {
+ return make_uninitialized_space_new<D>(s, p, count);
+ }
+
+ shift_right(p, data<D>() + s, p + count);
+ set_size<D>(s + count);
+ return p;
+ }
+
+ // makes space for uninitialized data of cout elements. Also updates size.
+ [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* {
+ if (is_direct()) {
+ return make_uninitialized_space<direction::direct>(pos, count);
+ }
+ return make_uninitialized_space<direction::indirect>(pos, count);
+ }
+
+ void destroy() {
+ auto const is_dir = is_direct();
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ T* ptr = nullptr;
+ size_t s = 0;
+ if (is_dir) {
+ ptr = data<direction::direct>();
+ s = size<direction::direct>();
+ } else {
+ ptr = data<direction::indirect>();
+ s = size<direction::indirect>();
+ }
+ std::destroy_n(ptr, s);
+ }
+ if (!is_dir) {
+ auto* storage = indirect();
+ std::destroy_at(storage);
+ ::operator delete(storage);
+ }
+ set_direct_and_size(0);
+ }
+
+ // performs a const_cast so we don't need this implementation twice
+ template <direction D>
+ auto at(size_t idx) -> T& {
+ if (idx >= size<D>()) {
+ throw std::out_of_range{"svector: idx out of range"};
+ }
+ auto* ptr = const_cast<T*>(data<D>() + idx); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ return *ptr;
+ } // LCOV_EXCL_LINE why is this single } marked as not covered? gcov bug?
+
+public:
+ using value_type = T;
+ using size_type = size_t;
+ using difference_type = std::ptrdiff_t;
+ using reference = value_type&;
+ using const_reference = value_type const&;
+ using pointer = T*;
+ using const_pointer = T const*;
+ using iterator = T*;
+ using const_iterator = T const*;
+ using reverse_iterator = std::reverse_iterator<iterator>;
+ using const_reverse_iterator = std::reverse_iterator<const_iterator>;
+
+ svector() {
+ set_direct_and_size(0);
+ }
+
+ svector(size_t count, T const& value)
+ : svector() {
+ resize(count, value);
+ }
+
+ explicit svector(size_t count)
+ : svector() {
+ reserve(count);
+ if (is_direct()) {
+ resize_after_reserve<direction::direct>(count);
+ } else {
+ resize_after_reserve<direction::indirect>(count);
+ }
+ }
+
+ template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>>
+ svector(InputIt first, InputIt last)
+ : svector() {
+ assign(first, last);
+ }
+
+ svector(svector const& other)
+ : svector() {
+ auto s = other.size();
+ reserve(s);
+ std::uninitialized_copy(other.begin(), other.end(), begin());
+ set_size(s);
+ }
+
+ svector(svector&& other) noexcept
+ : svector() {
+ do_move_assign(std::move(other));
+ }
+
+ svector(std::initializer_list<T> init)
+ : svector(init.begin(), init.end()) {}
+
+ ~svector() {
+ destroy();
+ }
+
+ void assign(size_t count, T const& value) {
+ clear();
+ resize(count, value);
+ }
+
+ template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>>
+ void assign(InputIt first, InputIt last) {
+ assign(first, last, typename std::iterator_traits<InputIt>::iterator_category());
+ }
+
+ void assign(std::initializer_list<T> l) {
+ assign(l.begin(), l.end());
+ }
+
+ auto operator=(svector const& other) -> svector& {
+ if (&other == this) {
+ return *this;
+ }
+
+ assign(other.begin(), other.end());
+ return *this;
+ }
+
+ auto operator=(svector&& other) noexcept -> svector& {
+ if (&other == this) {
+ // It doesn't seem to be required to do self-check, but let's do it anyways to be safe
+ return *this;
+ }
+ destroy();
+ do_move_assign(std::move(other));
+ return *this;
+ }
+
+ auto operator=(std::initializer_list<T> l) -> svector& {
+ assign(l.begin(), l.end());
+ return *this;
+ }
+
+ void resize(size_t count) {
+ if (count > capacity()) {
+ reserve(count);
+ }
+ if (is_direct()) {
+ resize_after_reserve<direction::direct>(count);
+ } else {
+ resize_after_reserve<direction::indirect>(count);
+ }
+ }
+
+ void resize(size_t count, T const& value) {
+ if (count > capacity()) {
+ reserve(count);
+ }
+ if (is_direct()) {
+ resize_after_reserve<direction::direct>(count, value);
+ } else {
+ resize_after_reserve<direction::indirect>(count, value);
+ }
+ }
+
+ void reserve(size_t s) {
+ auto old_capacity = capacity();
+ auto new_capacity = calculate_new_capacity(s, old_capacity);
+ if (new_capacity > old_capacity) {
+ realloc(new_capacity);
+ }
+ }
+
+ [[nodiscard]] auto capacity() const -> size_t {
+ if (is_direct()) {
+ return capacity<direction::direct>();
+ }
+ return capacity<direction::indirect>();
+ }
+
+ [[nodiscard]] auto size() const -> size_t {
+ if (is_direct()) {
+ return size<direction::direct>();
+ }
+ return size<direction::indirect>();
+ }
+
+ [[nodiscard]] auto data() -> T* {
+ if (is_direct()) {
+ return direct_data();
+ }
+ return indirect()->data();
+ }
+
+ [[nodiscard]] auto data() const -> T const* {
+ return const_cast<svector*>(this)->data(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ template <class... Args>
+ auto emplace_back(Args&&... args) -> T& {
+ size_t c; // NOLINT(cppcoreguidelines-init-variables)
+ size_t s; // NOLINT(cppcoreguidelines-init-variables)
+ bool is_dir = is_direct();
+ if (is_dir) {
+ c = capacity<direction::direct>();
+ s = size<direction::direct>();
+ } else {
+ c = capacity<direction::indirect>();
+ s = size<direction::indirect>();
+ }
+
+ if (s == c) {
+ auto new_capacity = calculate_new_capacity(s + 1, c);
+ realloc(new_capacity);
+ // reallocation happened, so we definitely are now in indirect mode
+ is_dir = false;
+ }
+
+ T* ptr; // NOLINT(cppcoreguidelines-init-variables)
+ if (is_dir) {
+ ptr = data<direction::direct>() + s;
+ set_size<direction::direct>(s + 1);
+ } else {
+ ptr = data<direction::indirect>() + s;
+ set_size<direction::indirect>(s + 1);
+ }
+ return *new (static_cast<void*>(ptr)) T(std::forward<Args>(args)...);
+ }
+
+ void push_back(T const& value) {
+ emplace_back(value);
+ }
+
+ void push_back(T&& value) {
+ emplace_back(std::move(value));
+ }
+
+ [[nodiscard]] auto operator[](size_t idx) const -> T const& {
+ return *(data() + idx);
+ }
+
+ [[nodiscard]] auto operator[](size_t idx) -> T& {
+ return *(data() + idx);
+ }
+
+ auto at(size_t idx) -> T& {
+ if (is_direct()) {
+ return at<direction::direct>(idx);
+ }
+ return at<direction::indirect>(idx);
+ }
+
+ auto at(size_t idx) const -> T const& {
+ return const_cast<svector*>(this)->at(idx); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ [[nodiscard]] auto begin() const -> T const* {
+ return data();
+ }
+
+ [[nodiscard]] auto cbegin() const -> T const* {
+ return begin();
+ }
+
+ [[nodiscard]] auto begin() -> T* {
+ return data();
+ }
+
+ [[nodiscard]] auto end() -> T* {
+ if (is_direct()) {
+ return data<direction::direct>() + size<direction::direct>();
+ }
+ return data<direction::indirect>() + size<direction::indirect>();
+ }
+
+ [[nodiscard]] auto end() const -> T const* {
+ return const_cast<svector*>(this)->end(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ [[nodiscard]] auto cend() const -> T const* {
+ return end();
+ }
+
+ [[nodiscard]] auto rbegin() -> reverse_iterator {
+ return reverse_iterator{end()};
+ }
+
+ [[nodiscard]] auto rbegin() const -> const_reverse_iterator {
+ return crbegin();
+ }
+
+ [[nodiscard]] auto crbegin() const -> const_reverse_iterator {
+ return const_reverse_iterator{end()};
+ }
+
+ [[nodiscard]] auto rend() -> reverse_iterator {
+ return reverse_iterator{begin()};
+ }
+
+ [[nodiscard]] auto rend() const -> const_reverse_iterator {
+ return crend();
+ }
+
+ [[nodiscard]] auto crend() const -> const_reverse_iterator {
+ return const_reverse_iterator{begin()};
+ }
+
+ [[nodiscard]] auto front() const -> T const& {
+ return *data();
+ }
+
+ [[nodiscard]] auto front() -> T& {
+ return *data();
+ }
+
+ [[nodiscard]] auto back() -> T& {
+ if (is_direct()) {
+ return *(data<direction::direct>() + size<direction::direct>() - 1);
+ }
+ return *(data<direction::indirect>() + size<direction::indirect>() - 1);
+ }
+
+ [[nodiscard]] auto back() const -> T const& {
+ return const_cast<svector*>(this)->back(); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ void clear() {
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ std::destroy(begin(), end());
+ }
+
+ if (is_direct()) {
+ set_size<direction::direct>(0);
+ } else {
+ set_size<direction::indirect>(0);
+ }
+ }
+
+ [[nodiscard]] auto empty() const -> bool {
+ return 0U == size();
+ }
+
+ void pop_back() {
+ if (is_direct()) {
+ pop_back<direction::direct>();
+ } else {
+ pop_back<direction::indirect>();
+ }
+ }
+
+ [[nodiscard]] static auto max_size() -> size_t {
+ return std::numeric_limits<std::ptrdiff_t>::max();
+ }
+
+ void swap(svector& other) {
+ // TODO we could try to do the minimum number of moves
+ std::swap(*this, other);
+ }
+
+ void shrink_to_fit() {
+ // per the standard we wouldn't need to do anything here. But since we are so nice,
+ // let's do the shrink.
+ auto const c = capacity();
+ auto const s = size();
+ if (s >= c) {
+ return;
+ }
+
+ auto new_capacity = calculate_new_capacity(s, N);
+ if (new_capacity == c) {
+ // nothing change!
+ return;
+ }
+
+ realloc(new_capacity);
+ }
+
+ template <class... Args>
+ auto emplace(const_iterator pos, Args&&... args) -> iterator {
+ auto* p = make_uninitialized_space(pos, 1);
+ return new (static_cast<void*>(p)) T(std::forward<Args>(args)...);
+ }
+
+ auto insert(const_iterator pos, T const& value) -> iterator {
+ return emplace(pos, value);
+ }
+
+ auto insert(const_iterator pos, T&& value) -> iterator {
+ return emplace(pos, std::move(value));
+ }
+
+ auto insert(const_iterator pos, size_t count, T const& value) -> iterator {
+ auto* p = make_uninitialized_space(pos, count);
+ std::uninitialized_fill_n(p, count, value);
+ return p;
+ }
+
+ template <typename It>
+ auto insert(const_iterator pos, It first, It last, std::input_iterator_tag /*unused*/) {
+ if (!(first != last)) {
+ return const_cast<T*>(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ // just input_iterator_tag makes this very slow. Let's do the same as the STL.
+ if (pos == end()) {
+ auto s = size();
+ while (first != last) {
+ emplace_back(*first);
+ ++first;
+ }
+ return begin() + s;
+ }
+
+ auto tmp = svector(first, last);
+ return insert(pos, std::make_move_iterator(tmp.begin()), std::make_move_iterator(tmp.end()));
+ }
+
+ template <typename It>
+ auto insert(const_iterator pos, It first, It last, std::forward_iterator_tag /*unused*/) -> iterator {
+ auto* p = make_uninitialized_space(pos, std::distance(first, last));
+ std::uninitialized_copy(first, last, p);
+ return p;
+ }
+
+ template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>>
+ auto insert(const_iterator pos, InputIt first, InputIt last) -> iterator {
+ return insert(pos, first, last, typename std::iterator_traits<InputIt>::iterator_category());
+ }
+
+ auto insert(const_iterator pos, std::initializer_list<T> l) -> iterator {
+ return insert(pos, l.begin(), l.end());
+ }
+
+ auto erase(const_iterator pos) -> iterator {
+ return erase(pos, pos + 1);
+ }
+
+ auto erase(const_iterator first, const_iterator last) -> iterator {
+ if (is_direct()) {
+ return erase_checked_end<direction::direct>(first, last);
+ }
+ return erase_checked_end<direction::indirect>(first, last);
+ }
+};
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator==(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return std::equal(a.begin(), a.end(), b.begin(), b.end());
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator!=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return !(a == b);
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator<(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end());
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator>=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return !(a < b);
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator>(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return std::lexicographical_compare(b.begin(), b.end(), a.begin(), a.end());
+}
+
+template <typename T, size_t NA, size_t NB>
+[[nodiscard]] auto operator<=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool {
+ return !(a > b);
+}
+
+} // namespace ANKERL_SVECTOR_NAMESPACE
+} // namespace ankerl
+
+// NOLINTNEXTLINE(cert-dcl58-cpp)
+namespace std {
+inline namespace ANKERL_SVECTOR_NAMESPACE {
+
+template <class T, size_t N, class U>
+constexpr auto erase(ankerl::svector<T, N>& sv, U const& value) -> typename ankerl::svector<T, N>::size_type {
+ auto* removed_begin = std::remove(sv.begin(), sv.end(), value);
+ auto num_removed = std::distance(removed_begin, sv.end());
+ sv.erase(removed_begin, sv.end());
+ return num_removed;
+}
+
+template <class T, size_t N, class Pred>
+constexpr auto erase_if(ankerl::svector<T, N>& sv, Pred pred) -> typename ankerl::svector<T, N>::size_type {
+ auto* removed_begin = std::remove_if(sv.begin(), sv.end(), pred);
+ auto num_removed = std::distance(removed_begin, sv.end());
+ sv.erase(removed_begin, sv.end());
+ return num_removed;
+}
+
+} // namespace ANKERL_SVECTOR_NAMESPACE
+} // namespace std
+
+#endif
diff --git a/contrib/ankerl/unordered_dense.h b/contrib/ankerl/unordered_dense.h
new file mode 100644
index 0000000..2aaacd6
--- /dev/null
+++ b/contrib/ankerl/unordered_dense.h
@@ -0,0 +1,2032 @@
+///////////////////////// ankerl::unordered_dense::{map, set} /////////////////////////
+
+// A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion.
+// Version 4.4.0
+// https://github.com/martinus/unordered_dense
+//
+// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2022-2023 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#ifndef ANKERL_UNORDERED_DENSE_H
+#define ANKERL_UNORDERED_DENSE_H
+
+// see https://semver.org/spec/v2.0.0.html
+#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes
+#define ANKERL_UNORDERED_DENSE_VERSION_MINOR 4 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality
+#define ANKERL_UNORDERED_DENSE_VERSION_PATCH 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes
+
+// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/
+
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch)
+#define ANKERL_UNORDERED_DENSE_NAMESPACE \
+ ANKERL_UNORDERED_DENSE_VERSION_CONCAT( \
+ ANKERL_UNORDERED_DENSE_VERSION_MAJOR, ANKERL_UNORDERED_DENSE_VERSION_MINOR, ANKERL_UNORDERED_DENSE_VERSION_PATCH)
+
+#if defined(_MSVC_LANG)
+# define ANKERL_UNORDERED_DENSE_CPP_VERSION _MSVC_LANG
+#else
+# define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus
+#endif
+
+#if defined(__GNUC__)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__))
+#elif defined(_MSC_VER)
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop))
+#endif
+
+// exceptions
+#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)
+# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 1 // NOLINT(cppcoreguidelines-macro-usage)
+#else
+# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 0 // NOLINT(cppcoreguidelines-macro-usage)
+#endif
+#ifdef _MSC_VER
+# define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline)
+#else
+# define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline))
+#endif
+
+// defined in unordered_dense.cpp
+#if !defined(ANKERL_UNORDERED_DENSE_EXPORT)
+# define ANKERL_UNORDERED_DENSE_EXPORT
+#endif
+
+#if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L
+# error ankerl::unordered_dense requires C++17 or higher
+#else
+# include <array> // for array
+# include <cstdint> // for uint64_t, uint32_t, uint8_t, UINT64_C
+# include <cstring> // for size_t, memcpy, memset
+# include <functional> // for equal_to, hash
+# include <initializer_list> // for initializer_list
+# include <iterator> // for pair, distance
+# include <limits> // for numeric_limits
+# include <memory> // for allocator, allocator_traits, shared_ptr
+# include <optional> // for optional
+# include <stdexcept> // for out_of_range
+# include <string> // for basic_string
+# include <string_view> // for basic_string_view, hash
+# include <tuple> // for forward_as_tuple
+# include <type_traits> // for enable_if_t, declval, conditional_t, ena...
+# include <utility> // for forward, exchange, pair, as_const, piece...
+# include <vector> // for vector
+# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() == 0
+# include <cstdlib> // for abort
+# endif
+
+# if defined(__has_include)
+# if __has_include(<memory_resource>)
+# define ANKERL_UNORDERED_DENSE_PMR std::pmr // NOLINT(cppcoreguidelines-macro-usage)
+# include <memory_resource> // for polymorphic_allocator
+# elif __has_include(<experimental/memory_resource>)
+# define ANKERL_UNORDERED_DENSE_PMR std::experimental::pmr // NOLINT(cppcoreguidelines-macro-usage)
+# include <experimental/memory_resource> // for polymorphic_allocator
+# endif
+# endif
+
+# if defined(_MSC_VER) && defined(_M_X64)
+# include <intrin.h>
+# pragma intrinsic(_umul128)
+# endif
+
+# if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
+# define ANKERL_UNORDERED_DENSE_LIKELY(x) __builtin_expect(x, 1) // NOLINT(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) __builtin_expect(x, 0) // NOLINT(cppcoreguidelines-macro-usage)
+# else
+# define ANKERL_UNORDERED_DENSE_LIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage)
+# endif
+
+namespace ankerl::unordered_dense {
+inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE {
+
+namespace detail {
+
+# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS()
+
+// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
+// inlinings more difficult. Throws are also generally the slow path.
+[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_not_found() {
+ throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found");
+}
+[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_bucket_overflow() {
+ throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size");
+}
+[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() {
+ throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements");
+}
+
+# else
+
+[[noreturn]] inline void on_error_key_not_found() {
+ abort();
+}
+[[noreturn]] inline void on_error_bucket_overflow() {
+ abort();
+}
+[[noreturn]] inline void on_error_too_many_elements() {
+ abort();
+}
+
+# endif
+
+} // namespace detail
+
+// hash ///////////////////////////////////////////////////////////////////////
+
+// This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash
+// No big-endian support (because different values on different machines don't matter),
+// hardcodes seed and the secret, reformats the code, and clang-tidy fixes.
+namespace detail::wyhash {
+
+inline void mum(uint64_t* a, uint64_t* b) {
+# if defined(__SIZEOF_INT128__)
+ __uint128_t r = *a;
+ r *= *b;
+ *a = static_cast<uint64_t>(r);
+ *b = static_cast<uint64_t>(r >> 64U);
+# elif defined(_MSC_VER) && defined(_M_X64)
+ *a = _umul128(*a, *b, b);
+# else
+ uint64_t ha = *a >> 32U;
+ uint64_t hb = *b >> 32U;
+ uint64_t la = static_cast<uint32_t>(*a);
+ uint64_t lb = static_cast<uint32_t>(*b);
+ uint64_t hi{};
+ uint64_t lo{};
+ uint64_t rh = ha * hb;
+ uint64_t rm0 = ha * lb;
+ uint64_t rm1 = hb * la;
+ uint64_t rl = la * lb;
+ uint64_t t = rl + (rm0 << 32U);
+ auto c = static_cast<uint64_t>(t < rl);
+ lo = t + (rm1 << 32U);
+ c += static_cast<uint64_t>(lo < t);
+ hi = rh + (rm0 >> 32U) + (rm1 >> 32U) + c;
+ *a = lo;
+ *b = hi;
+# endif
+}
+
+// multiply and xor mix function, aka MUM
+[[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t {
+ mum(&a, &b);
+ return a ^ b;
+}
+
+// read functions. WARNING: we don't care about endianness, so results are different on big endian!
+[[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t {
+ uint64_t v{};
+ std::memcpy(&v, p, 8U);
+ return v;
+}
+
+[[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t {
+ uint32_t v{};
+ std::memcpy(&v, p, 4);
+ return v;
+}
+
+// reads 1, 2, or 3 bytes
+[[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t {
+ return (static_cast<uint64_t>(p[0]) << 16U) | (static_cast<uint64_t>(p[k >> 1U]) << 8U) | p[k - 1];
+}
+
+[[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t {
+ static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f),
+ UINT64_C(0xe7037ed1a0b428db),
+ UINT64_C(0x8ebc6af09c88c6e3),
+ UINT64_C(0x589965cc75374cc3)};
+
+ auto const* p = static_cast<uint8_t const*>(key);
+ uint64_t seed = secret[0];
+ uint64_t a{};
+ uint64_t b{};
+ if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) {
+ if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) {
+ a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U));
+ b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U));
+ } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) {
+ a = r3(p, len);
+ b = 0;
+ } else {
+ a = 0;
+ b = 0;
+ }
+ } else {
+ size_t i = len;
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) {
+ uint64_t see1 = seed;
+ uint64_t see2 = seed;
+ do {
+ seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
+ see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1);
+ see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2);
+ p += 48;
+ i -= 48;
+ } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48));
+ seed ^= see1 ^ see2;
+ }
+ while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) {
+ seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
+ i -= 16;
+ p += 16;
+ }
+ a = r8(p + i - 16);
+ b = r8(p + i - 8);
+ }
+
+ return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed));
+}
+
+[[nodiscard]] inline auto hash(uint64_t x) -> uint64_t {
+ return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15));
+}
+
+} // namespace detail::wyhash
+
+ANKERL_UNORDERED_DENSE_EXPORT template <typename T, typename Enable = void>
+struct hash {
+ auto operator()(T const& obj) const noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>())))
+ -> uint64_t {
+ return std::hash<T>{}(obj);
+ }
+};
+
+template <typename CharT>
+struct hash<std::basic_string<CharT>> {
+ using is_avalanching = void;
+ auto operator()(std::basic_string<CharT> const& str) const noexcept -> uint64_t {
+ return detail::wyhash::hash(str.data(), sizeof(CharT) * str.size());
+ }
+};
+
+template <typename CharT>
+struct hash<std::basic_string_view<CharT>> {
+ using is_avalanching = void;
+ auto operator()(std::basic_string_view<CharT> const& sv) const noexcept -> uint64_t {
+ return detail::wyhash::hash(sv.data(), sizeof(CharT) * sv.size());
+ }
+};
+
+template <class T>
+struct hash<T*> {
+ using is_avalanching = void;
+ auto operator()(T* ptr) const noexcept -> uint64_t {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr));
+ }
+};
+
+template <class T>
+struct hash<std::unique_ptr<T>> {
+ using is_avalanching = void;
+ auto operator()(std::unique_ptr<T> const& ptr) const noexcept -> uint64_t {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
+ }
+};
+
+template <class T>
+struct hash<std::shared_ptr<T>> {
+ using is_avalanching = void;
+ auto operator()(std::shared_ptr<T> const& ptr) const noexcept -> uint64_t {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
+ }
+};
+
+template <typename Enum>
+struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
+ using is_avalanching = void;
+ auto operator()(Enum e) const noexcept -> uint64_t {
+ using underlying = typename std::underlying_type_t<Enum>;
+ return detail::wyhash::hash(static_cast<underlying>(e));
+ }
+};
+
+template <typename... Args>
+struct tuple_hash_helper {
+ // Converts the value into 64bit. If it is an integral type, just cast it. Mixing is doing the rest.
+ // If it isn't an integral we need to hash it.
+ template <typename Arg>
+ [[nodiscard]] constexpr static auto to64(Arg const& arg) -> uint64_t {
+ if constexpr (std::is_integral_v<Arg> || std::is_enum_v<Arg>) {
+ return static_cast<uint64_t>(arg);
+ } else {
+ return hash<Arg>{}(arg);
+ }
+ }
+
+ [[nodiscard]] static auto mix64(uint64_t state, uint64_t v) -> uint64_t {
+ return detail::wyhash::mix(state + v, uint64_t{0x9ddfea08eb382d69});
+ }
+
+ // Creates a buffer that holds all the data from each element of the tuple. If possible we memcpy the data directly. If
+ // not, we hash the object and use this for the array. Size of the array is known at compile time, and memcpy is optimized
+ // away, so filling the buffer is highly efficient. Finally, call wyhash with this buffer.
+ template <typename T, std::size_t... Idx>
+ [[nodiscard]] static auto calc_hash(T const& t, std::index_sequence<Idx...>) noexcept -> uint64_t {
+ auto h = uint64_t{};
+ ((h = mix64(h, to64(std::get<Idx>(t)))), ...);
+ return h;
+ }
+};
+
+template <typename... Args>
+struct hash<std::tuple<Args...>> : tuple_hash_helper<Args...> {
+ using is_avalanching = void;
+ auto operator()(std::tuple<Args...> const& t) const noexcept -> uint64_t {
+ return tuple_hash_helper<Args...>::calc_hash(t, std::index_sequence_for<Args...>{});
+ }
+};
+
+template <typename A, typename B>
+struct hash<std::pair<A, B>> : tuple_hash_helper<A, B> {
+ using is_avalanching = void;
+ auto operator()(std::pair<A, B> const& t) const noexcept -> uint64_t {
+ return tuple_hash_helper<A, B>::calc_hash(t, std::index_sequence_for<A, B>{});
+ }
+};
+
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+# define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T) \
+ template <> \
+ struct hash<T> { \
+ using is_avalanching = void; \
+ auto operator()(T const& obj) const noexcept -> uint64_t { \
+ return detail::wyhash::hash(static_cast<uint64_t>(obj)); \
+ } \
+ }
+
+# if defined(__GNUC__) && !defined(__clang__)
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wuseless-cast"
+# endif
+// see https://en.cppreference.com/w/cpp/utility/hash
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char);
+# if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L && defined(__cpp_char8_t)
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t);
+# endif
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char32_t);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(wchar_t);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(short);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned short);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(int);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned int);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long long);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long);
+ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long);
+
+# if defined(__GNUC__) && !defined(__clang__)
+# pragma GCC diagnostic pop
+# endif
+
+// bucket_type //////////////////////////////////////////////////////////
+
+namespace bucket_type {
+
+struct standard {
+ static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
+ static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
+
+ uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
+ uint32_t m_value_idx; // index into the m_values vector.
+};
+
+ANKERL_UNORDERED_DENSE_PACK(struct big {
+ static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint
+ static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint
+
+ uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
+ size_t m_value_idx; // index into the m_values vector.
+});
+
+} // namespace bucket_type
+
+namespace detail {
+
+struct nonesuch {};
+
+template <class Default, class AlwaysVoid, template <class...> class Op, class... Args>
+struct detector {
+ using value_t = std::false_type;
+ using type = Default;
+};
+
+template <class Default, template <class...> class Op, class... Args>
+struct detector<Default, std::void_t<Op<Args...>>, Op, Args...> {
+ using value_t = std::true_type;
+ using type = Op<Args...>;
+};
+
+template <template <class...> class Op, class... Args>
+using is_detected = typename detail::detector<detail::nonesuch, void, Op, Args...>::value_t;
+
+template <template <class...> class Op, class... Args>
+constexpr bool is_detected_v = is_detected<Op, Args...>::value;
+
+template <typename T>
+using detect_avalanching = typename T::is_avalanching;
+
+template <typename T>
+using detect_is_transparent = typename T::is_transparent;
+
+template <typename T>
+using detect_iterator = typename T::iterator;
+
+template <typename T>
+using detect_reserve = decltype(std::declval<T&>().reserve(size_t{}));
+
+// enable_if helpers
+
+template <typename Mapped>
+constexpr bool is_map_v = !std::is_void_v<Mapped>;
+
+// clang-format off
+template <typename Hash, typename KeyEqual>
+constexpr bool is_transparent_v = is_detected_v<detect_is_transparent, Hash> && is_detected_v<detect_is_transparent, KeyEqual>;
+// clang-format on
+
+template <typename From, typename To1, typename To2>
+constexpr bool is_neither_convertible_v = !std::is_convertible_v<From, To1> && !std::is_convertible_v<From, To2>;
+
+template <typename T>
+constexpr bool has_reserve = is_detected_v<detect_reserve, T>;
+
+// base type for map has mapped_type
+template <class T>
+struct base_table_type_map {
+ using mapped_type = T;
+};
+
+// base type for set doesn't have mapped_type
+struct base_table_type_set {};
+
+} // namespace detail
+
+// Very much like std::deque, but faster for indexing (in most cases). As of now this doesn't implement the full std::vector
+// API, but merely what's necessary to work as an underlying container for ankerl::unordered_dense::{map, set}.
+// It allocates blocks of equal size and puts them into the m_blocks vector. That means it can grow simply by adding a new
+// block to the back of m_blocks, and doesn't double its size like an std::vector. The disadvantage is that memory is not
+// linear and thus there is one more indirection necessary for indexing.
+template <typename T, typename Allocator = std::allocator<T>, size_t MaxSegmentSizeBytes = 4096>
+class segmented_vector {
+ template <bool IsConst>
+ class iter_t;
+
+public:
+ using allocator_type = Allocator;
+ using pointer = typename std::allocator_traits<allocator_type>::pointer;
+ using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer;
+ using difference_type = typename std::allocator_traits<allocator_type>::difference_type;
+ using value_type = T;
+ using size_type = std::size_t;
+ using reference = T&;
+ using const_reference = T const&;
+ using iterator = iter_t<false>;
+ using const_iterator = iter_t<true>;
+
+private:
+ using vec_alloc = typename std::allocator_traits<Allocator>::template rebind_alloc<pointer>;
+ std::vector<pointer, vec_alloc> m_blocks{};
+ size_t m_size{};
+
+ // Calculates the maximum number for x in (s << x) <= max_val
+ static constexpr auto num_bits_closest(size_t max_val, size_t s) -> size_t {
+ auto f = size_t{0};
+ while (s << (f + 1) <= max_val) {
+ ++f;
+ }
+ return f;
+ }
+
+ using self_t = segmented_vector<T, Allocator, MaxSegmentSizeBytes>;
+ static constexpr auto num_bits = num_bits_closest(MaxSegmentSizeBytes, sizeof(T));
+ static constexpr auto num_elements_in_block = 1U << num_bits;
+ static constexpr auto mask = num_elements_in_block - 1U;
+
+ /**
+ * Iterator class doubles as const_iterator and iterator
+ */
+ template <bool IsConst>
+ class iter_t {
+ using ptr_t = typename std::conditional_t<IsConst, segmented_vector::const_pointer const*, segmented_vector::pointer*>;
+ ptr_t m_data{};
+ size_t m_idx{};
+
+ template <bool B>
+ friend class iter_t;
+
+ public:
+ using difference_type = segmented_vector::difference_type;
+ using value_type = T;
+ using reference = typename std::conditional_t<IsConst, value_type const&, value_type&>;
+ using pointer = typename std::conditional_t<IsConst, segmented_vector::const_pointer, segmented_vector::pointer>;
+ using iterator_category = std::forward_iterator_tag;
+
+ iter_t() noexcept = default;
+
+ template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
+ // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
+ constexpr iter_t(iter_t<OtherIsConst> const& other) noexcept
+ : m_data(other.m_data)
+ , m_idx(other.m_idx) {}
+
+ constexpr iter_t(ptr_t data, size_t idx) noexcept
+ : m_data(data)
+ , m_idx(idx) {}
+
+ template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
+ constexpr auto operator=(iter_t<OtherIsConst> const& other) noexcept -> iter_t& {
+ m_data = other.m_data;
+ m_idx = other.m_idx;
+ return *this;
+ }
+
+ constexpr auto operator++() noexcept -> iter_t& {
+ ++m_idx;
+ return *this;
+ }
+
+ constexpr auto operator+(difference_type diff) noexcept -> iter_t {
+ return {m_data, static_cast<size_t>(static_cast<difference_type>(m_idx) + diff)};
+ }
+
+ template <bool OtherIsConst>
+ constexpr auto operator-(iter_t<OtherIsConst> const& other) noexcept -> difference_type {
+ return static_cast<difference_type>(m_idx) - static_cast<difference_type>(other.m_idx);
+ }
+
+ constexpr auto operator*() const noexcept -> reference {
+ return m_data[m_idx >> num_bits][m_idx & mask];
+ }
+
+ constexpr auto operator->() const noexcept -> pointer {
+ return &m_data[m_idx >> num_bits][m_idx & mask];
+ }
+
+ template <bool O>
+ constexpr auto operator==(iter_t<O> const& o) const noexcept -> bool {
+ return m_idx == o.m_idx;
+ }
+
+ template <bool O>
+ constexpr auto operator!=(iter_t<O> const& o) const noexcept -> bool {
+ return !(*this == o);
+ }
+ };
+
+ // slow path: need to allocate a new segment every once in a while
+ void increase_capacity() {
+ auto ba = Allocator(m_blocks.get_allocator());
+ pointer block = std::allocator_traits<Allocator>::allocate(ba, num_elements_in_block);
+ m_blocks.push_back(block);
+ }
+
+ // Moves everything from other
+ void append_everything_from(segmented_vector&& other) {
+ reserve(size() + other.size());
+ for (auto&& o : other) {
+ emplace_back(std::move(o));
+ }
+ }
+
+ // Copies everything from other
+ void append_everything_from(segmented_vector const& other) {
+ reserve(size() + other.size());
+ for (auto const& o : other) {
+ emplace_back(o);
+ }
+ }
+
+ void dealloc() {
+ auto ba = Allocator(m_blocks.get_allocator());
+ for (auto ptr : m_blocks) {
+ std::allocator_traits<Allocator>::deallocate(ba, ptr, num_elements_in_block);
+ }
+ }
+
+ [[nodiscard]] static constexpr auto calc_num_blocks_for_capacity(size_t capacity) {
+ return (capacity + num_elements_in_block - 1U) / num_elements_in_block;
+ }
+
+public:
+ segmented_vector() = default;
+
+ // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions)
+ segmented_vector(Allocator alloc)
+ : m_blocks(vec_alloc(alloc)) {}
+
+ segmented_vector(segmented_vector&& other, Allocator alloc)
+ : segmented_vector(alloc) {
+ *this = std::move(other);
+ }
+
+ segmented_vector(segmented_vector const& other, Allocator alloc)
+ : m_blocks(vec_alloc(alloc)) {
+ append_everything_from(other);
+ }
+
+ segmented_vector(segmented_vector&& other) noexcept
+ : segmented_vector(std::move(other), get_allocator()) {}
+
+ segmented_vector(segmented_vector const& other) {
+ append_everything_from(other);
+ }
+
+ auto operator=(segmented_vector const& other) -> segmented_vector& {
+ if (this == &other) {
+ return *this;
+ }
+ clear();
+ append_everything_from(other);
+ return *this;
+ }
+
+ auto operator=(segmented_vector&& other) noexcept -> segmented_vector& {
+ clear();
+ dealloc();
+ if (other.get_allocator() == get_allocator()) {
+ m_blocks = std::move(other.m_blocks);
+ m_size = std::exchange(other.m_size, {});
+ } else {
+ // make sure to construct with other's allocator!
+ m_blocks = std::vector<pointer, vec_alloc>(vec_alloc(other.get_allocator()));
+ append_everything_from(std::move(other));
+ }
+ return *this;
+ }
+
+ ~segmented_vector() {
+ clear();
+ dealloc();
+ }
+
+ [[nodiscard]] constexpr auto size() const -> size_t {
+ return m_size;
+ }
+
+ [[nodiscard]] constexpr auto capacity() const -> size_t {
+ return m_blocks.size() * num_elements_in_block;
+ }
+
+ // Indexing is highly performance critical
+ [[nodiscard]] constexpr auto operator[](size_t i) const noexcept -> T const& {
+ return m_blocks[i >> num_bits][i & mask];
+ }
+
+ [[nodiscard]] constexpr auto operator[](size_t i) noexcept -> T& {
+ return m_blocks[i >> num_bits][i & mask];
+ }
+
+ [[nodiscard]] constexpr auto begin() -> iterator {
+ return {m_blocks.data(), 0U};
+ }
+ [[nodiscard]] constexpr auto begin() const -> const_iterator {
+ return {m_blocks.data(), 0U};
+ }
+ [[nodiscard]] constexpr auto cbegin() const -> const_iterator {
+ return {m_blocks.data(), 0U};
+ }
+
+ [[nodiscard]] constexpr auto end() -> iterator {
+ return {m_blocks.data(), m_size};
+ }
+ [[nodiscard]] constexpr auto end() const -> const_iterator {
+ return {m_blocks.data(), m_size};
+ }
+ [[nodiscard]] constexpr auto cend() const -> const_iterator {
+ return {m_blocks.data(), m_size};
+ }
+
+ [[nodiscard]] constexpr auto back() -> reference {
+ return operator[](m_size - 1);
+ }
+ [[nodiscard]] constexpr auto back() const -> const_reference {
+ return operator[](m_size - 1);
+ }
+
+ void pop_back() {
+ back().~T();
+ --m_size;
+ }
+
+ [[nodiscard]] auto empty() const {
+ return 0 == m_size;
+ }
+
+ void reserve(size_t new_capacity) {
+ m_blocks.reserve(calc_num_blocks_for_capacity(new_capacity));
+ while (new_capacity > capacity()) {
+ increase_capacity();
+ }
+ }
+
+ [[nodiscard]] auto get_allocator() const -> allocator_type {
+ return allocator_type{m_blocks.get_allocator()};
+ }
+
+ template <class... Args>
+ auto emplace_back(Args&&... args) -> reference {
+ if (m_size == capacity()) {
+ increase_capacity();
+ }
+ auto* ptr = static_cast<void*>(&operator[](m_size));
+ auto& ref = *new (ptr) T(std::forward<Args>(args)...);
+ ++m_size;
+ return ref;
+ }
+
+ void clear() {
+ if constexpr (!std::is_trivially_destructible_v<T>) {
+ for (size_t i = 0, s = size(); i < s; ++i) {
+ operator[](i).~T();
+ }
+ }
+ m_size = 0;
+ }
+
+ void shrink_to_fit() {
+ auto ba = Allocator(m_blocks.get_allocator());
+ auto num_blocks_required = calc_num_blocks_for_capacity(m_size);
+ while (m_blocks.size() > num_blocks_required) {
+ std::allocator_traits<Allocator>::deallocate(ba, m_blocks.back(), num_elements_in_block);
+ m_blocks.pop_back();
+ }
+ m_blocks.shrink_to_fit();
+ }
+};
+
+namespace detail {
+
+// This is it, the table. Doubles as map and set, and uses `void` for T when its used as a set.
+template <class Key,
+ class T, // when void, treat it as a set.
+ class Hash,
+ class KeyEqual,
+ class AllocatorOrContainer,
+ class Bucket,
+ bool IsSegmented>
+class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, base_table_type_set> {
+ using underlying_value_type = typename std::conditional_t<is_map_v<T>, std::pair<Key, T>, Key>;
+ using underlying_container_type = std::conditional_t<IsSegmented,
+ segmented_vector<underlying_value_type, AllocatorOrContainer>,
+ std::vector<underlying_value_type, AllocatorOrContainer>>;
+
+public:
+ using value_container_type = std::
+ conditional_t<is_detected_v<detect_iterator, AllocatorOrContainer>, AllocatorOrContainer, underlying_container_type>;
+
+private:
+ using bucket_alloc =
+ typename std::allocator_traits<typename value_container_type::allocator_type>::template rebind_alloc<Bucket>;
+ using bucket_alloc_traits = std::allocator_traits<bucket_alloc>;
+
+ static constexpr uint8_t initial_shifts = 64 - 2; // 2^(64-m_shift) number of buckets
+ static constexpr float default_max_load_factor = 0.8F;
+
+public:
+ using key_type = Key;
+ using value_type = typename value_container_type::value_type;
+ using size_type = typename value_container_type::size_type;
+ using difference_type = typename value_container_type::difference_type;
+ using hasher = Hash;
+ using key_equal = KeyEqual;
+ using allocator_type = typename value_container_type::allocator_type;
+ using reference = typename value_container_type::reference;
+ using const_reference = typename value_container_type::const_reference;
+ using pointer = typename value_container_type::pointer;
+ using const_pointer = typename value_container_type::const_pointer;
+ using const_iterator = typename value_container_type::const_iterator;
+ using iterator = std::conditional_t<is_map_v<T>, typename value_container_type::iterator, const_iterator>;
+ using bucket_type = Bucket;
+
+private:
+ using value_idx_type = decltype(Bucket::m_value_idx);
+ using dist_and_fingerprint_type = decltype(Bucket::m_dist_and_fingerprint);
+
+ static_assert(std::is_trivially_destructible_v<Bucket>, "assert there's no need to call destructor / std::destroy");
+ static_assert(std::is_trivially_copyable_v<Bucket>, "assert we can just memset / memcpy");
+
+ value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes.
+ using bucket_pointer = typename std::allocator_traits<bucket_alloc>::pointer;
+ bucket_pointer m_buckets{};
+ size_t m_num_buckets = 0;
+ size_t m_max_bucket_capacity = 0;
+ float m_max_load_factor = default_max_load_factor;
+ Hash m_hash{};
+ KeyEqual m_equal{};
+ uint8_t m_shifts = initial_shifts;
+
+ [[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type {
+ return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == m_num_buckets)
+ ? 0
+ : static_cast<value_idx_type>(bucket_idx + 1U);
+ }
+
+ // Helper to access bucket through pointer types
+ [[nodiscard]] static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& {
+ return *(bucket_ptr + static_cast<typename std::allocator_traits<bucket_alloc>::difference_type>(offset));
+ }
+
+ // use the dist_inc and dist_dec functions so that uint16_t types work without warning
+ [[nodiscard]] static constexpr auto dist_inc(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
+ return static_cast<dist_and_fingerprint_type>(x + Bucket::dist_inc);
+ }
+
+ [[nodiscard]] static constexpr auto dist_dec(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
+ return static_cast<dist_and_fingerprint_type>(x - Bucket::dist_inc);
+ }
+
+ // The goal of mixed_hash is to always produce a high quality 64bit hash.
+ template <typename K>
+ [[nodiscard]] constexpr auto mixed_hash(K const& key) const -> uint64_t {
+ if constexpr (is_detected_v<detect_avalanching, Hash>) {
+ // we know that the hash is good because is_avalanching.
+ if constexpr (sizeof(decltype(m_hash(key))) < sizeof(uint64_t)) {
+ // 32bit hash and is_avalanching => multiply with a constant to avalanche bits upwards
+ return m_hash(key) * UINT64_C(0x9ddfea08eb382d69);
+ } else {
+ // 64bit and is_avalanching => only use the hash itself.
+ return m_hash(key);
+ }
+ } else {
+ // not is_avalanching => apply wyhash
+ return wyhash::hash(m_hash(key));
+ }
+ }
+
+ [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type {
+ return Bucket::dist_inc | (static_cast<dist_and_fingerprint_type>(hash) & Bucket::fingerprint_mask);
+ }
+
+ [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> value_idx_type {
+ return static_cast<value_idx_type>(hash >> m_shifts);
+ }
+
+ [[nodiscard]] static constexpr auto get_key(value_type const& vt) -> key_type const& {
+ if constexpr (is_map_v<T>) {
+ return vt.first;
+ } else {
+ return vt;
+ }
+ }
+
+ template <typename K>
+ [[nodiscard]] auto next_while_less(K const& key) const -> Bucket {
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (dist_and_fingerprint < at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+ return {dist_and_fingerprint, bucket_idx};
+ }
+
+ void place_and_shift_up(Bucket bucket, value_idx_type place) {
+ while (0 != at(m_buckets, place).m_dist_and_fingerprint) {
+ bucket = std::exchange(at(m_buckets, place), bucket);
+ bucket.m_dist_and_fingerprint = dist_inc(bucket.m_dist_and_fingerprint);
+ place = next(place);
+ }
+ at(m_buckets, place) = bucket;
+ }
+
+ [[nodiscard]] static constexpr auto calc_num_buckets(uint8_t shifts) -> size_t {
+ return (std::min)(max_bucket_count(), size_t{1} << (64U - shifts));
+ }
+
+ [[nodiscard]] constexpr auto calc_shifts_for_size(size_t s) const -> uint8_t {
+ auto shifts = initial_shifts;
+ while (shifts > 0 && static_cast<size_t>(static_cast<float>(calc_num_buckets(shifts)) * max_load_factor()) < s) {
+ --shifts;
+ }
+ return shifts;
+ }
+
+ // assumes m_values has data, m_buckets=m_buckets_end=nullptr, m_shifts is INITIAL_SHIFTS
+ void copy_buckets(table const& other) {
+ // assumes m_values has already the correct data copied over.
+ if (empty()) {
+ // when empty, at least allocate an initial buckets and clear them.
+ allocate_buckets_from_shift();
+ clear_buckets();
+ } else {
+ m_shifts = other.m_shifts;
+ allocate_buckets_from_shift();
+ std::memcpy(m_buckets, other.m_buckets, sizeof(Bucket) * bucket_count());
+ }
+ }
+
+ /**
+ * True when no element can be added any more without increasing the size
+ */
+ [[nodiscard]] auto is_full() const -> bool {
+ return size() > m_max_bucket_capacity;
+ }
+
+ void deallocate_buckets() {
+ auto ba = bucket_alloc(m_values.get_allocator());
+ if (nullptr != m_buckets) {
+ bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
+ m_buckets = nullptr;
+ }
+ m_num_buckets = 0;
+ m_max_bucket_capacity = 0;
+ }
+
+ void allocate_buckets_from_shift() {
+ auto ba = bucket_alloc(m_values.get_allocator());
+ m_num_buckets = calc_num_buckets(m_shifts);
+ m_buckets = bucket_alloc_traits::allocate(ba, m_num_buckets);
+ if (m_num_buckets == max_bucket_count()) {
+ // reached the maximum, make sure we can use each bucket
+ m_max_bucket_capacity = max_bucket_count();
+ } else {
+ m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(m_num_buckets) * max_load_factor());
+ }
+ }
+
+ void clear_buckets() {
+ if (m_buckets != nullptr) {
+ std::memset(&*m_buckets, 0, sizeof(Bucket) * bucket_count());
+ }
+ }
+
+ void clear_and_fill_buckets_from_values() {
+ clear_buckets();
+ for (value_idx_type value_idx = 0, end_idx = static_cast<value_idx_type>(m_values.size()); value_idx < end_idx;
+ ++value_idx) {
+ auto const& key = get_key(m_values[value_idx]);
+ auto [dist_and_fingerprint, bucket] = next_while_less(key);
+
+ // we know for certain that key has not yet been inserted, so no need to check it.
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket);
+ }
+ }
+
+ void increase_size() {
+ if (m_max_bucket_capacity == max_bucket_count()) {
+ // remove the value again, we can't add it!
+ m_values.pop_back();
+ on_error_bucket_overflow();
+ }
+ --m_shifts;
+ deallocate_buckets();
+ allocate_buckets_from_shift();
+ clear_and_fill_buckets_from_values();
+ }
+
+ template <typename Op>
+ void do_erase(value_idx_type bucket_idx, Op handle_erased_value) {
+ auto const value_idx_to_remove = at(m_buckets, bucket_idx).m_value_idx;
+
+ // shift down until either empty or an element with correct spot is found
+ auto next_bucket_idx = next(bucket_idx);
+ while (at(m_buckets, next_bucket_idx).m_dist_and_fingerprint >= Bucket::dist_inc * 2) {
+ at(m_buckets, bucket_idx) = {dist_dec(at(m_buckets, next_bucket_idx).m_dist_and_fingerprint),
+ at(m_buckets, next_bucket_idx).m_value_idx};
+ bucket_idx = std::exchange(next_bucket_idx, next(next_bucket_idx));
+ }
+ at(m_buckets, bucket_idx) = {};
+ handle_erased_value(std::move(m_values[value_idx_to_remove]));
+
+ // update m_values
+ if (value_idx_to_remove != m_values.size() - 1) {
+ // no luck, we'll have to replace the value with the last one and update the index accordingly
+ auto& val = m_values[value_idx_to_remove];
+ val = std::move(m_values.back());
+
+ // update the values_idx of the moved entry. No need to play the info game, just look until we find the values_idx
+ auto mh = mixed_hash(get_key(val));
+ bucket_idx = bucket_idx_from_hash(mh);
+
+ auto const values_idx_back = static_cast<value_idx_type>(m_values.size() - 1);
+ while (values_idx_back != at(m_buckets, bucket_idx).m_value_idx) {
+ bucket_idx = next(bucket_idx);
+ }
+ at(m_buckets, bucket_idx).m_value_idx = value_idx_to_remove;
+ }
+ m_values.pop_back();
+ }
+
+ template <typename K, typename Op>
+ auto do_erase_key(K&& key, Op handle_erased_value) -> size_t {
+ if (empty()) {
+ return 0;
+ }
+
+ auto [dist_and_fingerprint, bucket_idx] = next_while_less(key);
+
+ while (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
+ !m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ if (dist_and_fingerprint != at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ return 0;
+ }
+ do_erase(bucket_idx, handle_erased_value);
+ return 1;
+ }
+
+ template <class K, class M>
+ auto do_insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
+ auto it_isinserted = try_emplace(std::forward<K>(key), std::forward<M>(mapped));
+ if (!it_isinserted.second) {
+ it_isinserted.first->second = std::forward<M>(mapped);
+ }
+ return it_isinserted;
+ }
+
+ template <typename... Args>
+ auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, value_idx_type bucket_idx, Args&&... args)
+ -> std::pair<iterator, bool> {
+
+ // emplace the new value. If that throws an exception, no harm done; index is still in a valid state
+ m_values.emplace_back(std::forward<Args>(args)...);
+
+ auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
+ increase_size();
+ } else {
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
+ }
+
+ // place element and shift up until we find an empty spot
+ return {begin() + static_cast<difference_type>(value_idx), true};
+ }
+
+ template <typename K, typename... Args>
+ auto do_try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (true) {
+ auto* bucket = &at(m_buckets, bucket_idx);
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
+ if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return {begin() + static_cast<difference_type>(bucket->m_value_idx), false};
+ }
+ } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
+ return do_place_element(dist_and_fingerprint,
+ bucket_idx,
+ std::piecewise_construct,
+ std::forward_as_tuple(std::forward<K>(key)),
+ std::forward_as_tuple(std::forward<Args>(args)...));
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+ }
+
+ template <typename K>
+ auto do_find(K const& key) -> iterator {
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(empty())) {
+ return end();
+ }
+
+ auto mh = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(mh);
+ auto bucket_idx = bucket_idx_from_hash(mh);
+ auto* bucket = &at(m_buckets, bucket_idx);
+
+ // unrolled loop. *Always* check a few directly, then enter the loop. This is faster.
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ bucket = &at(m_buckets, bucket_idx);
+
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ bucket = &at(m_buckets, bucket_idx);
+
+ while (true) {
+ if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
+ if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
+ return begin() + static_cast<difference_type>(bucket->m_value_idx);
+ }
+ } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
+ return end();
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ bucket = &at(m_buckets, bucket_idx);
+ }
+ }
+
+ template <typename K>
+ auto do_find(K const& key) const -> const_iterator {
+ return const_cast<table*>(this)->do_find(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+ template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto do_at(K const& key) -> Q& {
+ if (auto it = find(key); ANKERL_UNORDERED_DENSE_LIKELY(end() != it)) {
+ return it->second;
+ }
+ on_error_key_not_found();
+ }
+
+ template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto do_at(K const& key) const -> Q const& {
+ return const_cast<table*>(this)->at(key); // NOLINT(cppcoreguidelines-pro-type-const-cast)
+ }
+
+public:
+ explicit table(size_t bucket_count,
+ Hash const& hash = Hash(),
+ KeyEqual const& equal = KeyEqual(),
+ allocator_type const& alloc_or_container = allocator_type())
+ : m_values(alloc_or_container)
+ , m_hash(hash)
+ , m_equal(equal) {
+ if (0 != bucket_count) {
+ reserve(bucket_count);
+ } else {
+ allocate_buckets_from_shift();
+ clear_buckets();
+ }
+ }
+
+ table()
+ : table(0) {}
+
+ table(size_t bucket_count, allocator_type const& alloc)
+ : table(bucket_count, Hash(), KeyEqual(), alloc) {}
+
+ table(size_t bucket_count, Hash const& hash, allocator_type const& alloc)
+ : table(bucket_count, hash, KeyEqual(), alloc) {}
+
+ explicit table(allocator_type const& alloc)
+ : table(0, Hash(), KeyEqual(), alloc) {}
+
+ template <class InputIt>
+ table(InputIt first,
+ InputIt last,
+ size_type bucket_count = 0,
+ Hash const& hash = Hash(),
+ KeyEqual const& equal = KeyEqual(),
+ allocator_type const& alloc = allocator_type())
+ : table(bucket_count, hash, equal, alloc) {
+ insert(first, last);
+ }
+
+ template <class InputIt>
+ table(InputIt first, InputIt last, size_type bucket_count, allocator_type const& alloc)
+ : table(first, last, bucket_count, Hash(), KeyEqual(), alloc) {}
+
+ template <class InputIt>
+ table(InputIt first, InputIt last, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
+ : table(first, last, bucket_count, hash, KeyEqual(), alloc) {}
+
+ table(table const& other)
+ : table(other, other.m_values.get_allocator()) {}
+
+ table(table const& other, allocator_type const& alloc)
+ : m_values(other.m_values, alloc)
+ , m_max_load_factor(other.m_max_load_factor)
+ , m_hash(other.m_hash)
+ , m_equal(other.m_equal) {
+ copy_buckets(other);
+ }
+
+ table(table&& other) noexcept
+ : table(std::move(other), other.m_values.get_allocator()) {}
+
+ table(table&& other, allocator_type const& alloc) noexcept
+ : m_values(alloc) {
+ *this = std::move(other);
+ }
+
+ table(std::initializer_list<value_type> ilist,
+ size_t bucket_count = 0,
+ Hash const& hash = Hash(),
+ KeyEqual const& equal = KeyEqual(),
+ allocator_type const& alloc = allocator_type())
+ : table(bucket_count, hash, equal, alloc) {
+ insert(ilist);
+ }
+
+ table(std::initializer_list<value_type> ilist, size_type bucket_count, allocator_type const& alloc)
+ : table(ilist, bucket_count, Hash(), KeyEqual(), alloc) {}
+
+ table(std::initializer_list<value_type> init, size_type bucket_count, Hash const& hash, allocator_type const& alloc)
+ : table(init, bucket_count, hash, KeyEqual(), alloc) {}
+
+ ~table() {
+ if (nullptr != m_buckets) {
+ auto ba = bucket_alloc(m_values.get_allocator());
+ bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
+ }
+ }
+
+ auto operator=(table const& other) -> table& {
+ if (&other != this) {
+ deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
+ m_values = other.m_values;
+ m_max_load_factor = other.m_max_load_factor;
+ m_hash = other.m_hash;
+ m_equal = other.m_equal;
+ m_shifts = initial_shifts;
+ copy_buckets(other);
+ }
+ return *this;
+ }
+
+ auto operator=(table&& other) noexcept(noexcept(std::is_nothrow_move_assignable_v<value_container_type> &&
+ std::is_nothrow_move_assignable_v<Hash> &&
+ std::is_nothrow_move_assignable_v<KeyEqual>)) -> table& {
+ if (&other != this) {
+ deallocate_buckets(); // deallocate before m_values is set (might have another allocator)
+ m_values = std::move(other.m_values);
+ other.m_values.clear();
+
+ // we can only reuse m_buckets when both maps have the same allocator!
+ if (get_allocator() == other.get_allocator()) {
+ m_buckets = std::exchange(other.m_buckets, nullptr);
+ m_num_buckets = std::exchange(other.m_num_buckets, 0);
+ m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0);
+ m_shifts = std::exchange(other.m_shifts, initial_shifts);
+ m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor);
+ m_hash = std::exchange(other.m_hash, {});
+ m_equal = std::exchange(other.m_equal, {});
+ other.allocate_buckets_from_shift();
+ other.clear_buckets();
+ } else {
+ // set max_load_factor *before* copying the other's buckets, so we have the same
+ // behavior
+ m_max_load_factor = other.m_max_load_factor;
+
+ // copy_buckets sets m_buckets, m_num_buckets, m_max_bucket_capacity, m_shifts
+ copy_buckets(other);
+ // clear's the other's buckets so other is now already usable.
+ other.clear_buckets();
+ m_hash = other.m_hash;
+ m_equal = other.m_equal;
+ }
+ // map "other" is now already usable, it's empty.
+ }
+ return *this;
+ }
+
+ auto operator=(std::initializer_list<value_type> ilist) -> table& {
+ clear();
+ insert(ilist);
+ return *this;
+ }
+
+ auto get_allocator() const noexcept -> allocator_type {
+ return m_values.get_allocator();
+ }
+
+ // iterators //////////////////////////////////////////////////////////////
+
+ auto begin() noexcept -> iterator {
+ return m_values.begin();
+ }
+
+ auto begin() const noexcept -> const_iterator {
+ return m_values.begin();
+ }
+
+ auto cbegin() const noexcept -> const_iterator {
+ return m_values.cbegin();
+ }
+
+ auto end() noexcept -> iterator {
+ return m_values.end();
+ }
+
+ auto cend() const noexcept -> const_iterator {
+ return m_values.cend();
+ }
+
+ auto end() const noexcept -> const_iterator {
+ return m_values.end();
+ }
+
+ // capacity ///////////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto empty() const noexcept -> bool {
+ return m_values.empty();
+ }
+
+ [[nodiscard]] auto size() const noexcept -> size_t {
+ return m_values.size();
+ }
+
+ [[nodiscard]] static constexpr auto max_size() noexcept -> size_t {
+ if constexpr ((std::numeric_limits<value_idx_type>::max)() == (std::numeric_limits<size_t>::max)()) {
+ return size_t{1} << (sizeof(value_idx_type) * 8 - 1);
+ } else {
+ return size_t{1} << (sizeof(value_idx_type) * 8);
+ }
+ }
+
+ // modifiers //////////////////////////////////////////////////////////////
+
+ void clear() {
+ m_values.clear();
+ clear_buckets();
+ }
+
+ auto insert(value_type const& value) -> std::pair<iterator, bool> {
+ return emplace(value);
+ }
+
+ auto insert(value_type&& value) -> std::pair<iterator, bool> {
+ return emplace(std::move(value));
+ }
+
+ template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
+ auto insert(P&& value) -> std::pair<iterator, bool> {
+ return emplace(std::forward<P>(value));
+ }
+
+ auto insert(const_iterator /*hint*/, value_type const& value) -> iterator {
+ return insert(value).first;
+ }
+
+ auto insert(const_iterator /*hint*/, value_type&& value) -> iterator {
+ return insert(std::move(value)).first;
+ }
+
+ template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true>
+ auto insert(const_iterator /*hint*/, P&& value) -> iterator {
+ return insert(std::forward<P>(value)).first;
+ }
+
+ template <class InputIt>
+ void insert(InputIt first, InputIt last) {
+ while (first != last) {
+ insert(*first);
+ ++first;
+ }
+ }
+
+ void insert(std::initializer_list<value_type> ilist) {
+ insert(ilist.begin(), ilist.end());
+ }
+
+ // nonstandard API: *this is emptied.
+ // Also see "A Standard flat_map" https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p0429r9.pdf
+ auto extract() && -> value_container_type {
+ return std::move(m_values);
+ }
+
+ // nonstandard API:
+ // Discards the internally held container and replaces it with the one passed. Erases non-unique elements.
+ auto replace(value_container_type&& container) {
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(container.size() > max_size())) {
+ on_error_too_many_elements();
+ }
+ auto shifts = calc_shifts_for_size(container.size());
+ if (0 == m_num_buckets || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
+ m_shifts = shifts;
+ deallocate_buckets();
+ allocate_buckets_from_shift();
+ }
+ clear_buckets();
+
+ m_values = std::move(container);
+
+ // can't use clear_and_fill_buckets_from_values() because container elements might not be unique
+ auto value_idx = value_idx_type{};
+
+ // loop until we reach the end of the container. duplicated entries will be replaced with back().
+ while (value_idx != static_cast<value_idx_type>(m_values.size())) {
+ auto const& key = get_key(m_values[value_idx]);
+
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ bool key_found = false;
+ while (true) {
+ auto const& bucket = at(m_buckets, bucket_idx);
+ if (dist_and_fingerprint > bucket.m_dist_and_fingerprint) {
+ break;
+ }
+ if (dist_and_fingerprint == bucket.m_dist_and_fingerprint &&
+ m_equal(key, get_key(m_values[bucket.m_value_idx]))) {
+ key_found = true;
+ break;
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ if (key_found) {
+ if (value_idx != static_cast<value_idx_type>(m_values.size() - 1)) {
+ m_values[value_idx] = std::move(m_values.back());
+ }
+ m_values.pop_back();
+ } else {
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
+ ++value_idx;
+ }
+ }
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(Key const& key, M&& mapped) -> std::pair<iterator, bool> {
+ return do_insert_or_assign(key, std::forward<M>(mapped));
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(Key&& key, M&& mapped) -> std::pair<iterator, bool> {
+ return do_insert_or_assign(std::move(key), std::forward<M>(mapped));
+ }
+
+ template <typename K,
+ typename M,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> {
+ return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped));
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(const_iterator /*hint*/, Key const& key, M&& mapped) -> iterator {
+ return do_insert_or_assign(key, std::forward<M>(mapped)).first;
+ }
+
+ template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto insert_or_assign(const_iterator /*hint*/, Key&& key, M&& mapped) -> iterator {
+ return do_insert_or_assign(std::move(key), std::forward<M>(mapped)).first;
+ }
+
+ template <typename K,
+ typename M,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto insert_or_assign(const_iterator /*hint*/, K&& key, M&& mapped) -> iterator {
+ return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped)).first;
+ }
+
+ // Single arguments for unordered_set can be used without having to construct the value_type
+ template <class K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<!is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto emplace(K&& key) -> std::pair<iterator, bool> {
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
+ m_equal(key, m_values[at(m_buckets, bucket_idx).m_value_idx])) {
+ // found it, return without ever actually creating anything
+ return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ // value is new, insert element first, so when exception happens we are in a valid state
+ return do_place_element(dist_and_fingerprint, bucket_idx, std::forward<K>(key));
+ }
+
+ template <class... Args>
+ auto emplace(Args&&... args) -> std::pair<iterator, bool> {
+ // we have to instantiate the value_type to be able to access the key.
+ // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop.
+ auto& key = get_key(m_values.emplace_back(std::forward<Args>(args)...));
+ auto hash = mixed_hash(key);
+ auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
+ if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
+ m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
+ m_values.pop_back(); // value was already there, so get rid of it
+ return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
+ }
+ dist_and_fingerprint = dist_inc(dist_and_fingerprint);
+ bucket_idx = next(bucket_idx);
+ }
+
+ // value is new, place the bucket and shift up until we find an empty spot
+ auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
+ if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
+ // increase_size just rehashes all the data we have in m_values
+ increase_size();
+ } else {
+ // place element and shift up until we find an empty spot
+ place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
+ }
+ return {begin() + static_cast<difference_type>(value_idx), true};
+ }
+
+ template <class... Args>
+ auto emplace_hint(const_iterator /*hint*/, Args&&... args) -> iterator {
+ return emplace(std::forward<Args>(args)...).first;
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(Key const& key, Args&&... args) -> std::pair<iterator, bool> {
+ return do_try_emplace(key, std::forward<Args>(args)...);
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(Key&& key, Args&&... args) -> std::pair<iterator, bool> {
+ return do_try_emplace(std::move(key), std::forward<Args>(args)...);
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(const_iterator /*hint*/, Key const& key, Args&&... args) -> iterator {
+ return do_try_emplace(key, std::forward<Args>(args)...).first;
+ }
+
+ template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto try_emplace(const_iterator /*hint*/, Key&& key, Args&&... args) -> iterator {
+ return do_try_emplace(std::move(key), std::forward<Args>(args)...).first;
+ }
+
+ template <
+ typename K,
+ typename... Args,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
+ bool> = true>
+ auto try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> {
+ return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...);
+ }
+
+ template <
+ typename K,
+ typename... Args,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>,
+ bool> = true>
+ auto try_emplace(const_iterator /*hint*/, K&& key, Args&&... args) -> iterator {
+ return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...).first;
+ }
+
+ auto erase(iterator it) -> iterator {
+ auto hash = mixed_hash(get_key(*it));
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
+ while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
+ bucket_idx = next(bucket_idx);
+ }
+
+ do_erase(bucket_idx, [](value_type&& /*unused*/) {
+ });
+ return begin() + static_cast<difference_type>(value_idx_to_remove);
+ }
+
+ auto extract(iterator it) -> value_type {
+ auto hash = mixed_hash(get_key(*it));
+ auto bucket_idx = bucket_idx_from_hash(hash);
+
+ auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
+ while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
+ bucket_idx = next(bucket_idx);
+ }
+
+ auto tmp = std::optional<value_type>{};
+ do_erase(bucket_idx, [&tmp](value_type&& val) {
+ tmp = std::move(val);
+ });
+ return std::move(tmp).value();
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto erase(const_iterator it) -> iterator {
+ return erase(begin() + (it - cbegin()));
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto extract(const_iterator it) -> value_type {
+ return extract(begin() + (it - cbegin()));
+ }
+
+ auto erase(const_iterator first, const_iterator last) -> iterator {
+ auto const idx_first = first - cbegin();
+ auto const idx_last = last - cbegin();
+ auto const first_to_last = std::distance(first, last);
+ auto const last_to_end = std::distance(last, cend());
+
+ // remove elements from left to right which moves elements from the end back
+ auto const mid = idx_first + (std::min)(first_to_last, last_to_end);
+ auto idx = idx_first;
+ while (idx != mid) {
+ erase(begin() + idx);
+ ++idx;
+ }
+
+ // all elements from the right are moved, now remove the last element until all done
+ idx = idx_last;
+ while (idx != mid) {
+ --idx;
+ erase(begin() + idx);
+ }
+
+ return begin() + idx_first;
+ }
+
+ auto erase(Key const& key) -> size_t {
+ return do_erase_key(key, [](value_type&& /*unused*/) {
+ });
+ }
+
+ auto extract(Key const& key) -> std::optional<value_type> {
+ auto tmp = std::optional<value_type>{};
+ do_erase_key(key, [&tmp](value_type&& val) {
+ tmp = std::move(val);
+ });
+ return tmp;
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto erase(K&& key) -> size_t {
+ return do_erase_key(std::forward<K>(key), [](value_type&& /*unused*/) {
+ });
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto extract(K&& key) -> std::optional<value_type> {
+ auto tmp = std::optional<value_type>{};
+ do_erase_key(std::forward<K>(key), [&tmp](value_type&& val) {
+ tmp = std::move(val);
+ });
+ return tmp;
+ }
+
+ void swap(table& other) noexcept(noexcept(std::is_nothrow_swappable_v<value_container_type> &&
+ std::is_nothrow_swappable_v<Hash> && std::is_nothrow_swappable_v<KeyEqual>)) {
+ using std::swap;
+ swap(other, *this);
+ }
+
+ // lookup /////////////////////////////////////////////////////////////////
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto at(key_type const& key) -> Q& {
+ return do_at(key);
+ }
+
+ template <typename K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto at(K const& key) -> Q& {
+ return do_at(key);
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto at(key_type const& key) const -> Q const& {
+ return do_at(key);
+ }
+
+ template <typename K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto at(K const& key) const -> Q const& {
+ return do_at(key);
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto operator[](Key const& key) -> Q& {
+ return try_emplace(key).first->second;
+ }
+
+ template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
+ auto operator[](Key&& key) -> Q& {
+ return try_emplace(std::move(key)).first->second;
+ }
+
+ template <typename K,
+ typename Q = T,
+ typename H = Hash,
+ typename KE = KeyEqual,
+ std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
+ auto operator[](K&& key) -> Q& {
+ return try_emplace(std::forward<K>(key)).first->second;
+ }
+
+ auto count(Key const& key) const -> size_t {
+ return find(key) == end() ? 0 : 1;
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto count(K const& key) const -> size_t {
+ return find(key) == end() ? 0 : 1;
+ }
+
+ auto find(Key const& key) -> iterator {
+ return do_find(key);
+ }
+
+ auto find(Key const& key) const -> const_iterator {
+ return do_find(key);
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto find(K const& key) -> iterator {
+ return do_find(key);
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto find(K const& key) const -> const_iterator {
+ return do_find(key);
+ }
+
+ auto contains(Key const& key) const -> bool {
+ return find(key) != end();
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto contains(K const& key) const -> bool {
+ return find(key) != end();
+ }
+
+ auto equal_range(Key const& key) -> std::pair<iterator, iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ auto equal_range(const Key& key) const -> std::pair<const_iterator, const_iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto equal_range(K const& key) -> std::pair<iterator, iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
+ auto equal_range(K const& key) const -> std::pair<const_iterator, const_iterator> {
+ auto it = do_find(key);
+ return {it, it == end() ? end() : it + 1};
+ }
+
+ // bucket interface ///////////////////////////////////////////////////////
+
+ auto bucket_count() const noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
+ return m_num_buckets;
+ }
+
+ static constexpr auto max_bucket_count() noexcept -> size_t { // NOLINT(modernize-use-nodiscard)
+ return max_size();
+ }
+
+ // hash policy ////////////////////////////////////////////////////////////
+
+ [[nodiscard]] auto load_factor() const -> float {
+ return bucket_count() ? static_cast<float>(size()) / static_cast<float>(bucket_count()) : 0.0F;
+ }
+
+ [[nodiscard]] auto max_load_factor() const -> float {
+ return m_max_load_factor;
+ }
+
+ void max_load_factor(float ml) {
+ m_max_load_factor = ml;
+ if (m_num_buckets != max_bucket_count()) {
+ m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(bucket_count()) * max_load_factor());
+ }
+ }
+
+ void rehash(size_t count) {
+ count = (std::min)(count, max_size());
+ auto shifts = calc_shifts_for_size((std::max)(count, size()));
+ if (shifts != m_shifts) {
+ m_shifts = shifts;
+ deallocate_buckets();
+ m_values.shrink_to_fit();
+ allocate_buckets_from_shift();
+ clear_and_fill_buckets_from_values();
+ }
+ }
+
+ void reserve(size_t capa) {
+ capa = (std::min)(capa, max_size());
+ if constexpr (has_reserve<value_container_type>) {
+ // std::deque doesn't have reserve(). Make sure we only call when available
+ m_values.reserve(capa);
+ }
+ auto shifts = calc_shifts_for_size((std::max)(capa, size()));
+ if (0 == m_num_buckets || shifts < m_shifts) {
+ m_shifts = shifts;
+ deallocate_buckets();
+ allocate_buckets_from_shift();
+ clear_and_fill_buckets_from_values();
+ }
+ }
+
+ // observers //////////////////////////////////////////////////////////////
+
+ auto hash_function() const -> hasher {
+ return m_hash;
+ }
+
+ auto key_eq() const -> key_equal {
+ return m_equal;
+ }
+
+ // nonstandard API: expose the underlying values container
+ [[nodiscard]] auto values() const noexcept -> value_container_type const& {
+ return m_values;
+ }
+
+ // non-member functions ///////////////////////////////////////////////////
+
+ friend auto operator==(table const& a, table const& b) -> bool {
+ if (&a == &b) {
+ return true;
+ }
+ if (a.size() != b.size()) {
+ return false;
+ }
+ for (auto const& b_entry : b) {
+ auto it = a.find(get_key(b_entry));
+ if constexpr (is_map_v<T>) {
+ // map: check that key is here, then also check that value is the same
+ if (a.end() == it || !(b_entry.second == it->second)) {
+ return false;
+ }
+ } else {
+ // set: only check that the key is here
+ if (a.end() == it) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ friend auto operator!=(table const& a, table const& b) -> bool {
+ return !(a == b);
+ }
+};
+
+} // namespace detail
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
+ class Bucket = bucket_type::standard>
+using map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
+ class Bucket = bucket_type::standard>
+using segmented_map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<Key>,
+ class Bucket = bucket_type::standard>
+using set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class AllocatorOrContainer = std::allocator<Key>,
+ class Bucket = bucket_type::standard>
+using segmented_set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>;
+
+# if defined(ANKERL_UNORDERED_DENSE_PMR)
+
+namespace pmr {
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using map =
+ detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using segmented_map =
+ detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, true>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using set = detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, false>;
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class Hash = hash<Key>,
+ class KeyEqual = std::equal_to<Key>,
+ class Bucket = bucket_type::standard>
+using segmented_set =
+ detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, true>;
+
+} // namespace pmr
+
+# endif
+
+// deduction guides ///////////////////////////////////////////////////////////
+
+// deduction guides for alias templates are only possible since C++20
+// see https://en.cppreference.com/w/cpp/language/class_template_argument_deduction
+
+} // namespace ANKERL_UNORDERED_DENSE_NAMESPACE
+} // namespace ankerl::unordered_dense
+
+// std extensions /////////////////////////////////////////////////////////////
+
+namespace std { // NOLINT(cert-dcl58-cpp)
+
+ANKERL_UNORDERED_DENSE_EXPORT template <class Key,
+ class T,
+ class Hash,
+ class KeyEqual,
+ class AllocatorOrContainer,
+ class Bucket,
+ class Pred,
+ bool IsSegmented>
+// NOLINTNEXTLINE(cert-dcl58-cpp)
+auto erase_if(ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>& map,
+ Pred pred) -> size_t {
+ using map_t = ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>;
+
+ // going back to front because erase() invalidates the end iterator
+ auto const old_size = map.size();
+ auto idx = old_size;
+ while (idx) {
+ --idx;
+ auto it = map.begin() + static_cast<typename map_t::difference_type>(idx);
+ if (pred(*it)) {
+ map.erase(it);
+ }
+ }
+
+ return old_size - map.size();
+}
+
+} // namespace std
+
+#endif
+#endif