diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-10 21:30:40 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-10 21:30:40 +0000 |
commit | 133a45c109da5310add55824db21af5239951f93 (patch) | |
tree | ba6ac4c0a950a0dda56451944315d66409923918 /contrib/ankerl | |
parent | Initial commit. (diff) | |
download | rspamd-upstream.tar.xz rspamd-upstream.zip |
Adding upstream version 3.8.1.upstream/3.8.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'contrib/ankerl')
-rw-r--r-- | contrib/ankerl/LICENSE | 21 | ||||
-rw-r--r-- | contrib/ankerl/svector.h | 999 | ||||
-rw-r--r-- | contrib/ankerl/unordered_dense.h | 2032 |
3 files changed, 3052 insertions, 0 deletions
diff --git a/contrib/ankerl/LICENSE b/contrib/ankerl/LICENSE new file mode 100644 index 0000000..c4d1a0e --- /dev/null +++ b/contrib/ankerl/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Martin Leitner-Ankerl + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/contrib/ankerl/svector.h b/contrib/ankerl/svector.h new file mode 100644 index 0000000..dbd075b --- /dev/null +++ b/contrib/ankerl/svector.h @@ -0,0 +1,999 @@ +// ┌─┐┬ ┬┌─┐┌─┐┌┬┐┌─┐┬─┐ Compact SVO optimized vector C++17 or higher +// └─┐└┐┌┘├┤ │ │ │ │├┬┘ Version 1.0.2 +// └─┘ └┘ └─┘└─┘ ┴ └─┘┴└─ https://github.com/martinus/svector +// +// Licensed under the MIT License <http://opensource.org/licenses/MIT>. +// SPDX-License-Identifier: MIT +// Copyright (c) 2022 Martin Leitner-Ankerl <martin.ankerl@gmail.com> +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ANKERL_SVECTOR_H +#define ANKERL_SVECTOR_H + +// see https://semver.org/spec/v2.0.0.html +#define ANKERL_SVECTOR_VERSION_MAJOR 1 // incompatible API changes +#define ANKERL_SVECTOR_VERSION_MINOR 0 // add functionality in a backwards compatible manner +#define ANKERL_SVECTOR_VERSION_PATCH 2 // backwards compatible bug fixes + +// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/ +#define ANKERL_SVECTOR_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch +#define ANKERL_SVECTOR_VERSION_CONCAT(major, minor, patch) ANKERL_SVECTOR_VERSION_CONCAT1(major, minor, patch) +#define ANKERL_SVECTOR_NAMESPACE \ + ANKERL_SVECTOR_VERSION_CONCAT(ANKERL_SVECTOR_VERSION_MAJOR, ANKERL_SVECTOR_VERSION_MINOR, ANKERL_SVECTOR_VERSION_PATCH) + +#include <algorithm> +#include <array> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <initializer_list> +#include <iterator> +#include <limits> +#include <memory> +#include <new> +#include <stdexcept> +#include <type_traits> +#include <utility> + +namespace ankerl { +inline namespace ANKERL_SVECTOR_NAMESPACE { +namespace detail { + +template <typename Condition, typename T = void> +using enable_if_t = typename std::enable_if<Condition::value, T>::type; + +template <typename It> +using is_input_iterator = std::is_base_of<std::input_iterator_tag, typename std::iterator_traits<It>::iterator_category>; + +constexpr auto round_up(size_t n, size_t multiple) -> size_t { + return ((n + (multiple - 1)) / multiple) * multiple; +} + +template <typename T> +constexpr auto cx_min(T a, T b) -> T { + return a < b ? a : b; +} + +template <typename T> +constexpr auto cx_max(T a, T b) -> T { + return a > b ? a : b; +} + +template <typename T> +constexpr auto alignment_of_svector() -> size_t { + return cx_max(sizeof(void*), std::alignment_of_v<T>); +} + +/** + * @brief Calculates sizeof(svector<T, N>) for a given type and inline capacity + */ +template <typename T> +constexpr auto size_of_svector(size_t min_inline_capacity) -> size_t { + // + 1 for one byte size in direct mode + return round_up(sizeof(T) * min_inline_capacity + 1, alignment_of_svector<T>()); +} + +/** + * @brief Calculates how many T we can actually store inside of an svector without increasing its sizeof(). + * + * E.g. svector<char, 1> could store 7 bytes even though 1 is specified. This makes sure we don't waste any + * of the padding. + */ +template <typename T> +constexpr auto automatic_capacity(size_t min_inline_capacity) -> size_t { + return cx_min((size_of_svector<T>(min_inline_capacity) - 1U) / sizeof(T), size_t{127}); +} + +/** + * Holds size & capacity, a glorified struct. + */ +class header { + size_t m_size{}; + size_t const m_capacity; + +public: + inline explicit header(size_t capacity) + : m_capacity{capacity} {} + + [[nodiscard]] inline auto size() const -> size_t { + return m_size; + } + + [[nodiscard]] inline auto capacity() const -> size_t { + return m_capacity; + } + + inline void size(size_t s) { + m_size = s; + } +}; + +/** + * @brief Holds header (size+capacity) plus an arbitrary number of T. + * + * To make storage compact, we don't actually store a pointer to T. We don't have to + * because we know exactly at which location it begins. + */ +template <typename T> +struct storage : public header { + static constexpr auto alignment_of_t = std::alignment_of_v<T>; + static constexpr auto max_alignment = std::max(std::alignment_of_v<header>, std::alignment_of_v<T>); + static constexpr auto offset_to_data = detail::round_up(sizeof(header), alignment_of_t); + static_assert(max_alignment <= __STDCPP_DEFAULT_NEW_ALIGNMENT__); + + explicit storage(size_t capacity) + : header(capacity) {} + + auto data() -> T* { + auto ptr_to_data = reinterpret_cast<std::byte*>(this) + offset_to_data; + return std::launder(reinterpret_cast<T*>(ptr_to_data)); + } + + /** + * @brief Allocates space for storage plus capacity*T objects. + * + * Checks to make sure that allocation won't overflow. + * + * @param capacity Number of T to allocate. + * @return storage<T>* + */ + static auto alloc(size_t capacity) -> storage<T>* { + // make sure we don't overflow! + auto mem = sizeof(T) * capacity; + if (mem < capacity) { + throw std::bad_alloc(); + } + if (offset_to_data + mem < mem) { + throw std::bad_alloc(); + } + mem += offset_to_data; + if (static_cast<uint64_t>(mem) > static_cast<uint64_t>(std::numeric_limits<std::ptrdiff_t>::max())) { + throw std::bad_alloc(); + } + + void* ptr = ::operator new(offset_to_data + sizeof(T) * capacity); + if (nullptr == ptr) { + throw std::bad_alloc(); + } + // use void* to ensure we don't use an overload for T* + return new (ptr) storage<T>(capacity); + } +}; + +} // namespace detail + +template <typename T, size_t MinInlineCapacity> +class svector { + static_assert(MinInlineCapacity <= 127, "sorry, can't have more than 127 direct elements"); + static constexpr auto N = detail::automatic_capacity<T>(MinInlineCapacity); + + enum class direction { direct, indirect }; + + /** + * A buffer to hold the data of the svector Depending on direct/indirect mode, the content it holds is like so: + * + * direct: + * m_data[0] & 1: lowest bit is 1 for direct mode. + * m_data[0] >> 1: size for direct mode + * Then 0-X bytes unused (padding), and then the actual inline T data. + * indirect: + * m_data[0] & 1: lowest bit is 0 for indirect mode + * m_data[0..7]: stores an uintptr_t, which points to the indirect data. + */ + alignas(detail::alignment_of_svector<T>()) std::array<uint8_t, detail::size_of_svector<T>(MinInlineCapacity)> m_data; + + // direct mode /////////////////////////////////////////////////////////// + + [[nodiscard]] auto is_direct() const -> bool { + return (m_data[0] & 1U) != 0U; + } + + [[nodiscard]] auto direct_size() const -> size_t { + return m_data[0] >> 1U; + } + + // sets size of direct mode and mode to direct too. + constexpr void set_direct_and_size(size_t s) { + m_data[0] = (s << 1U) | 1U; + } + + [[nodiscard]] auto direct_data() -> T* { + return std::launder(reinterpret_cast<T*>(m_data.data() + std::alignment_of_v<T>)); + } + + // indirect mode ///////////////////////////////////////////////////////// + + [[nodiscard]] auto indirect() -> detail::storage<T>* { + detail::storage<T>* ptr; // NOLINT(cppcoreguidelines-init-variables) + std::memcpy(&ptr, m_data.data(), sizeof(ptr)); + return ptr; + } + + [[nodiscard]] auto indirect() const -> detail::storage<T> const* { + return const_cast<svector*>(this)->indirect(); // NOLINT(cppcoreguidelines-pro-type-const-cast) + } + + void set_indirect(detail::storage<T>* ptr) { + std::memcpy(m_data.data(), &ptr, sizeof(ptr)); + + // safety check to guarantee the lowest bit is 0 + if (is_direct()) { + throw std::bad_alloc(); // LCOV_EXCL_LINE + } + } + + // helpers /////////////////////////////////////////////////////////////// + + /** + * @brief Moves size objects from source_ptr to target_ptr, and destroys what remains in source_ptr. + * + * Assumes data is not overlapping + */ + static void uninitialized_move_and_destroy(T* source_ptr, T* target_ptr, size_t size) { + if constexpr (std::is_trivially_copyable_v<T>) { + std::memcpy(target_ptr, source_ptr, size * sizeof(T)); + } else { + std::uninitialized_move_n(source_ptr, size, target_ptr); + std::destroy_n(source_ptr, size); + } + } + + /** + * @brief Reallocates all data when capacity changes. + * + * if new_capacity <= N chooses direct memory, otherwise indirect. + */ + void realloc(size_t new_capacity) { + if (new_capacity <= N) { + // put everything into direct storage + if (is_direct()) { + // direct -> direct: nothing to do! + return; + } + + // indirect -> direct + auto* storage = indirect(); + uninitialized_move_and_destroy(storage->data(), direct_data(), storage->size()); + set_direct_and_size(storage->size()); + std::destroy_at(storage); + ::operator delete(storage); + } else { + // put everything into indirect storage + auto* storage = detail::storage<T>::alloc(new_capacity); + if (is_direct()) { + // direct -> indirect + uninitialized_move_and_destroy(data<direction::direct>(), storage->data(), size<direction::direct>()); + storage->size(size<direction::direct>()); + } else { + // indirect -> indirect + uninitialized_move_and_destroy(data<direction::indirect>(), storage->data(), size<direction::indirect>()); + storage->size(size<direction::indirect>()); + auto* storage = indirect(); + std::destroy_at(storage); + ::operator delete(storage); + } + set_indirect(storage); + } + } + + /** + * @brief Doubles starting_capacity until it is >= size_to_fit. + */ + [[nodiscard]] static auto calculate_new_capacity(size_t size_to_fit, size_t starting_capacity) -> size_t { + if (size_to_fit > max_size()) { + // not enough space + throw std::bad_alloc(); + } + + if (size_to_fit == 0) { + // special handling for 0 so N==0 works + return starting_capacity; + } + // start with at least 1, so N==0 works + auto new_capacity = std::max<size_t>(1, starting_capacity); + + // double capacity until its large enough, but make sure we don't overflow + while (new_capacity < size_to_fit && new_capacity * 2 > new_capacity) { + new_capacity *= 2; + } + if (new_capacity < size_to_fit) { + // got an overflow, set capacity to max + new_capacity = max_size(); + } + return std::min(new_capacity, max_size()); + } + + template <direction D> + [[nodiscard]] auto capacity() const -> size_t { + if constexpr (D == direction::direct) { + return N; + } else { + return indirect()->capacity(); + } + } + + template <direction D> + [[nodiscard]] auto size() const -> size_t { + if constexpr (D == direction::direct) { + return direct_size(); + } else { + return indirect()->size(); + } + } + + template <direction D> + void set_size(size_t s) { + if constexpr (D == direction::direct) { + set_direct_and_size(s); + } else { + indirect()->size(s); + } + } + + void set_size(size_t s) { + if (is_direct()) { + set_size<direction::direct>(s); + } else { + set_size<direction::indirect>(s); + } + } + + template <direction D> + [[nodiscard]] auto data() -> T* { + if constexpr (D == direction::direct) { + return direct_data(); + } else { + return indirect()->data(); + } + } + + template <direction D> + [[nodiscard]] auto data() const -> T const* { + return const_cast<svector*>(this)->data<D>(); // NOLINT(cppcoreguidelines-pro-type-const-cast) + } + + template <direction D> + void pop_back() { + if constexpr (std::is_trivially_destructible_v<T>) { + set_size<D>(size<D>() - 1); + } else { + auto s = size<D>() - 1; + (data<D>() + s)->~T(); + set_size<D>(s); + } + } + + /** + * @brief We need variadic arguments so we can either use copy ctor or default ctor + */ + template <direction D, class... Args> + void resize_after_reserve(size_t count, Args&&... args) { + auto current_size = size<D>(); + if (current_size > count) { + if constexpr (!std::is_trivially_destructible_v<T>) { + auto* d = data<D>(); + std::destroy(d + count, d + current_size); + } + } else { + auto* d = data<D>(); + for (auto ptr = d + current_size, end = d + count; ptr != end; ++ptr) { + new (static_cast<void*>(ptr)) T(std::forward<Args>(args)...); + } + } + set_size<D>(count); + } + + // Makes sure that to is not past the end iterator + template <direction D> + auto erase_checked_end(T const* cfrom, T const* to) -> T* { + auto* const erase_begin = const_cast<T*>(cfrom); // NOLINT(cppcoreguidelines-pro-type-const-cast) + auto* const container_end = data<D>() + size<D>(); + auto* const erase_end = std::min(const_cast<T*>(to), container_end); // NOLINT(cppcoreguidelines-pro-type-const-cast) + + std::move(erase_end, container_end, erase_begin); + auto const num_erased = std::distance(erase_begin, erase_end); + std::destroy(container_end - num_erased, container_end); + set_size<D>(size<D>() - num_erased); + return erase_begin; + } + + template <typename It> + void assign(It first, It last, std::input_iterator_tag /*unused*/) { + clear(); + + // TODO this can be made faster, e.g. by setting size only when finished. + while (first != last) { + push_back(*first); + ++first; + } + } + + template <typename It> + void assign(It first, It last, std::forward_iterator_tag /*unused*/) { + clear(); + + auto s = std::distance(first, last); + reserve(s); + std::uninitialized_copy(first, last, data()); + set_size(s); + } + + // precondition: all uninitialized + void do_move_assign(svector&& other) { + if (!other.is_direct()) { + // take other's memory, even when empty + set_indirect(other.indirect()); + } else { + auto* other_ptr = other.data<direction::direct>(); + auto s = other.size<direction::direct>(); + auto* other_end = other_ptr + s; + + std::uninitialized_move(other_ptr, other_end, data<direction::direct>()); + std::destroy(other_ptr, other_end); + set_size(s); + } + other.set_direct_and_size(0); + } + + /** + * @brief Shifts data [source_begin, source_end( to the right, starting on target_begin. + * + * Preconditions: + * * contiguous memory + * * source_begin <= target_begin + * * source_end onwards is uninitialized memory + * + * Destroys then empty elements in [source_begin, source_end( + */ + static void shift_right(T* source_begin, T* source_end, T* target_begin) { + // 1. uninitialized moves + auto const num_moves = std::distance(source_begin, source_end); + auto const target_end = target_begin + num_moves; + auto const num_uninitialized_move = std::min(num_moves, std::distance(source_end, target_end)); + std::uninitialized_move(source_end - num_uninitialized_move, source_end, target_end - num_uninitialized_move); + std::move_backward(source_begin, source_end - num_uninitialized_move, target_end - num_uninitialized_move); + std::destroy(source_begin, std::min(source_end, target_begin)); + } + + template <direction D> + [[nodiscard]] auto make_uninitialized_space_new(size_t s, T* p, size_t count) -> T* { + auto target = svector(); + // we know target is indirect because we're increasing capacity + target.reserve(s + count); + + // move everything [begin, pos[ + auto* target_pos = std::uninitialized_move(data<D>(), p, target.template data<direction::indirect>()); + + // move everything [pos, end] + std::uninitialized_move(p, data<D>() + s, target_pos + count); + + target.template set_size<direction::indirect>(s + count); + *this = std::move(target); + return target_pos; + } + + template <direction D> + [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* { + auto* const p = const_cast<T*>(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast) + auto s = size<D>(); + if (s + count > capacity<D>()) { + return make_uninitialized_space_new<D>(s, p, count); + } + + shift_right(p, data<D>() + s, p + count); + set_size<D>(s + count); + return p; + } + + // makes space for uninitialized data of cout elements. Also updates size. + [[nodiscard]] auto make_uninitialized_space(T const* pos, size_t count) -> T* { + if (is_direct()) { + return make_uninitialized_space<direction::direct>(pos, count); + } + return make_uninitialized_space<direction::indirect>(pos, count); + } + + void destroy() { + auto const is_dir = is_direct(); + if constexpr (!std::is_trivially_destructible_v<T>) { + T* ptr = nullptr; + size_t s = 0; + if (is_dir) { + ptr = data<direction::direct>(); + s = size<direction::direct>(); + } else { + ptr = data<direction::indirect>(); + s = size<direction::indirect>(); + } + std::destroy_n(ptr, s); + } + if (!is_dir) { + auto* storage = indirect(); + std::destroy_at(storage); + ::operator delete(storage); + } + set_direct_and_size(0); + } + + // performs a const_cast so we don't need this implementation twice + template <direction D> + auto at(size_t idx) -> T& { + if (idx >= size<D>()) { + throw std::out_of_range{"svector: idx out of range"}; + } + auto* ptr = const_cast<T*>(data<D>() + idx); // NOLINT(cppcoreguidelines-pro-type-const-cast) + return *ptr; + } // LCOV_EXCL_LINE why is this single } marked as not covered? gcov bug? + +public: + using value_type = T; + using size_type = size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type&; + using const_reference = value_type const&; + using pointer = T*; + using const_pointer = T const*; + using iterator = T*; + using const_iterator = T const*; + using reverse_iterator = std::reverse_iterator<iterator>; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; + + svector() { + set_direct_and_size(0); + } + + svector(size_t count, T const& value) + : svector() { + resize(count, value); + } + + explicit svector(size_t count) + : svector() { + reserve(count); + if (is_direct()) { + resize_after_reserve<direction::direct>(count); + } else { + resize_after_reserve<direction::indirect>(count); + } + } + + template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>> + svector(InputIt first, InputIt last) + : svector() { + assign(first, last); + } + + svector(svector const& other) + : svector() { + auto s = other.size(); + reserve(s); + std::uninitialized_copy(other.begin(), other.end(), begin()); + set_size(s); + } + + svector(svector&& other) noexcept + : svector() { + do_move_assign(std::move(other)); + } + + svector(std::initializer_list<T> init) + : svector(init.begin(), init.end()) {} + + ~svector() { + destroy(); + } + + void assign(size_t count, T const& value) { + clear(); + resize(count, value); + } + + template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>> + void assign(InputIt first, InputIt last) { + assign(first, last, typename std::iterator_traits<InputIt>::iterator_category()); + } + + void assign(std::initializer_list<T> l) { + assign(l.begin(), l.end()); + } + + auto operator=(svector const& other) -> svector& { + if (&other == this) { + return *this; + } + + assign(other.begin(), other.end()); + return *this; + } + + auto operator=(svector&& other) noexcept -> svector& { + if (&other == this) { + // It doesn't seem to be required to do self-check, but let's do it anyways to be safe + return *this; + } + destroy(); + do_move_assign(std::move(other)); + return *this; + } + + auto operator=(std::initializer_list<T> l) -> svector& { + assign(l.begin(), l.end()); + return *this; + } + + void resize(size_t count) { + if (count > capacity()) { + reserve(count); + } + if (is_direct()) { + resize_after_reserve<direction::direct>(count); + } else { + resize_after_reserve<direction::indirect>(count); + } + } + + void resize(size_t count, T const& value) { + if (count > capacity()) { + reserve(count); + } + if (is_direct()) { + resize_after_reserve<direction::direct>(count, value); + } else { + resize_after_reserve<direction::indirect>(count, value); + } + } + + void reserve(size_t s) { + auto old_capacity = capacity(); + auto new_capacity = calculate_new_capacity(s, old_capacity); + if (new_capacity > old_capacity) { + realloc(new_capacity); + } + } + + [[nodiscard]] auto capacity() const -> size_t { + if (is_direct()) { + return capacity<direction::direct>(); + } + return capacity<direction::indirect>(); + } + + [[nodiscard]] auto size() const -> size_t { + if (is_direct()) { + return size<direction::direct>(); + } + return size<direction::indirect>(); + } + + [[nodiscard]] auto data() -> T* { + if (is_direct()) { + return direct_data(); + } + return indirect()->data(); + } + + [[nodiscard]] auto data() const -> T const* { + return const_cast<svector*>(this)->data(); // NOLINT(cppcoreguidelines-pro-type-const-cast) + } + + template <class... Args> + auto emplace_back(Args&&... args) -> T& { + size_t c; // NOLINT(cppcoreguidelines-init-variables) + size_t s; // NOLINT(cppcoreguidelines-init-variables) + bool is_dir = is_direct(); + if (is_dir) { + c = capacity<direction::direct>(); + s = size<direction::direct>(); + } else { + c = capacity<direction::indirect>(); + s = size<direction::indirect>(); + } + + if (s == c) { + auto new_capacity = calculate_new_capacity(s + 1, c); + realloc(new_capacity); + // reallocation happened, so we definitely are now in indirect mode + is_dir = false; + } + + T* ptr; // NOLINT(cppcoreguidelines-init-variables) + if (is_dir) { + ptr = data<direction::direct>() + s; + set_size<direction::direct>(s + 1); + } else { + ptr = data<direction::indirect>() + s; + set_size<direction::indirect>(s + 1); + } + return *new (static_cast<void*>(ptr)) T(std::forward<Args>(args)...); + } + + void push_back(T const& value) { + emplace_back(value); + } + + void push_back(T&& value) { + emplace_back(std::move(value)); + } + + [[nodiscard]] auto operator[](size_t idx) const -> T const& { + return *(data() + idx); + } + + [[nodiscard]] auto operator[](size_t idx) -> T& { + return *(data() + idx); + } + + auto at(size_t idx) -> T& { + if (is_direct()) { + return at<direction::direct>(idx); + } + return at<direction::indirect>(idx); + } + + auto at(size_t idx) const -> T const& { + return const_cast<svector*>(this)->at(idx); // NOLINT(cppcoreguidelines-pro-type-const-cast) + } + + [[nodiscard]] auto begin() const -> T const* { + return data(); + } + + [[nodiscard]] auto cbegin() const -> T const* { + return begin(); + } + + [[nodiscard]] auto begin() -> T* { + return data(); + } + + [[nodiscard]] auto end() -> T* { + if (is_direct()) { + return data<direction::direct>() + size<direction::direct>(); + } + return data<direction::indirect>() + size<direction::indirect>(); + } + + [[nodiscard]] auto end() const -> T const* { + return const_cast<svector*>(this)->end(); // NOLINT(cppcoreguidelines-pro-type-const-cast) + } + + [[nodiscard]] auto cend() const -> T const* { + return end(); + } + + [[nodiscard]] auto rbegin() -> reverse_iterator { + return reverse_iterator{end()}; + } + + [[nodiscard]] auto rbegin() const -> const_reverse_iterator { + return crbegin(); + } + + [[nodiscard]] auto crbegin() const -> const_reverse_iterator { + return const_reverse_iterator{end()}; + } + + [[nodiscard]] auto rend() -> reverse_iterator { + return reverse_iterator{begin()}; + } + + [[nodiscard]] auto rend() const -> const_reverse_iterator { + return crend(); + } + + [[nodiscard]] auto crend() const -> const_reverse_iterator { + return const_reverse_iterator{begin()}; + } + + [[nodiscard]] auto front() const -> T const& { + return *data(); + } + + [[nodiscard]] auto front() -> T& { + return *data(); + } + + [[nodiscard]] auto back() -> T& { + if (is_direct()) { + return *(data<direction::direct>() + size<direction::direct>() - 1); + } + return *(data<direction::indirect>() + size<direction::indirect>() - 1); + } + + [[nodiscard]] auto back() const -> T const& { + return const_cast<svector*>(this)->back(); // NOLINT(cppcoreguidelines-pro-type-const-cast) + } + + void clear() { + if constexpr (!std::is_trivially_destructible_v<T>) { + std::destroy(begin(), end()); + } + + if (is_direct()) { + set_size<direction::direct>(0); + } else { + set_size<direction::indirect>(0); + } + } + + [[nodiscard]] auto empty() const -> bool { + return 0U == size(); + } + + void pop_back() { + if (is_direct()) { + pop_back<direction::direct>(); + } else { + pop_back<direction::indirect>(); + } + } + + [[nodiscard]] static auto max_size() -> size_t { + return std::numeric_limits<std::ptrdiff_t>::max(); + } + + void swap(svector& other) { + // TODO we could try to do the minimum number of moves + std::swap(*this, other); + } + + void shrink_to_fit() { + // per the standard we wouldn't need to do anything here. But since we are so nice, + // let's do the shrink. + auto const c = capacity(); + auto const s = size(); + if (s >= c) { + return; + } + + auto new_capacity = calculate_new_capacity(s, N); + if (new_capacity == c) { + // nothing change! + return; + } + + realloc(new_capacity); + } + + template <class... Args> + auto emplace(const_iterator pos, Args&&... args) -> iterator { + auto* p = make_uninitialized_space(pos, 1); + return new (static_cast<void*>(p)) T(std::forward<Args>(args)...); + } + + auto insert(const_iterator pos, T const& value) -> iterator { + return emplace(pos, value); + } + + auto insert(const_iterator pos, T&& value) -> iterator { + return emplace(pos, std::move(value)); + } + + auto insert(const_iterator pos, size_t count, T const& value) -> iterator { + auto* p = make_uninitialized_space(pos, count); + std::uninitialized_fill_n(p, count, value); + return p; + } + + template <typename It> + auto insert(const_iterator pos, It first, It last, std::input_iterator_tag /*unused*/) { + if (!(first != last)) { + return const_cast<T*>(pos); // NOLINT(cppcoreguidelines-pro-type-const-cast) + } + + // just input_iterator_tag makes this very slow. Let's do the same as the STL. + if (pos == end()) { + auto s = size(); + while (first != last) { + emplace_back(*first); + ++first; + } + return begin() + s; + } + + auto tmp = svector(first, last); + return insert(pos, std::make_move_iterator(tmp.begin()), std::make_move_iterator(tmp.end())); + } + + template <typename It> + auto insert(const_iterator pos, It first, It last, std::forward_iterator_tag /*unused*/) -> iterator { + auto* p = make_uninitialized_space(pos, std::distance(first, last)); + std::uninitialized_copy(first, last, p); + return p; + } + + template <typename InputIt, typename = detail::enable_if_t<detail::is_input_iterator<InputIt>>> + auto insert(const_iterator pos, InputIt first, InputIt last) -> iterator { + return insert(pos, first, last, typename std::iterator_traits<InputIt>::iterator_category()); + } + + auto insert(const_iterator pos, std::initializer_list<T> l) -> iterator { + return insert(pos, l.begin(), l.end()); + } + + auto erase(const_iterator pos) -> iterator { + return erase(pos, pos + 1); + } + + auto erase(const_iterator first, const_iterator last) -> iterator { + if (is_direct()) { + return erase_checked_end<direction::direct>(first, last); + } + return erase_checked_end<direction::indirect>(first, last); + } +}; + +template <typename T, size_t NA, size_t NB> +[[nodiscard]] auto operator==(svector<T, NA> const& a, svector<T, NB> const& b) -> bool { + return std::equal(a.begin(), a.end(), b.begin(), b.end()); +} + +template <typename T, size_t NA, size_t NB> +[[nodiscard]] auto operator!=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool { + return !(a == b); +} + +template <typename T, size_t NA, size_t NB> +[[nodiscard]] auto operator<(svector<T, NA> const& a, svector<T, NB> const& b) -> bool { + return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); +} + +template <typename T, size_t NA, size_t NB> +[[nodiscard]] auto operator>=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool { + return !(a < b); +} + +template <typename T, size_t NA, size_t NB> +[[nodiscard]] auto operator>(svector<T, NA> const& a, svector<T, NB> const& b) -> bool { + return std::lexicographical_compare(b.begin(), b.end(), a.begin(), a.end()); +} + +template <typename T, size_t NA, size_t NB> +[[nodiscard]] auto operator<=(svector<T, NA> const& a, svector<T, NB> const& b) -> bool { + return !(a > b); +} + +} // namespace ANKERL_SVECTOR_NAMESPACE +} // namespace ankerl + +// NOLINTNEXTLINE(cert-dcl58-cpp) +namespace std { +inline namespace ANKERL_SVECTOR_NAMESPACE { + +template <class T, size_t N, class U> +constexpr auto erase(ankerl::svector<T, N>& sv, U const& value) -> typename ankerl::svector<T, N>::size_type { + auto* removed_begin = std::remove(sv.begin(), sv.end(), value); + auto num_removed = std::distance(removed_begin, sv.end()); + sv.erase(removed_begin, sv.end()); + return num_removed; +} + +template <class T, size_t N, class Pred> +constexpr auto erase_if(ankerl::svector<T, N>& sv, Pred pred) -> typename ankerl::svector<T, N>::size_type { + auto* removed_begin = std::remove_if(sv.begin(), sv.end(), pred); + auto num_removed = std::distance(removed_begin, sv.end()); + sv.erase(removed_begin, sv.end()); + return num_removed; +} + +} // namespace ANKERL_SVECTOR_NAMESPACE +} // namespace std + +#endif diff --git a/contrib/ankerl/unordered_dense.h b/contrib/ankerl/unordered_dense.h new file mode 100644 index 0000000..2aaacd6 --- /dev/null +++ b/contrib/ankerl/unordered_dense.h @@ -0,0 +1,2032 @@ +///////////////////////// ankerl::unordered_dense::{map, set} ///////////////////////// + +// A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion. +// Version 4.4.0 +// https://github.com/martinus/unordered_dense +// +// Licensed under the MIT License <http://opensource.org/licenses/MIT>. +// SPDX-License-Identifier: MIT +// Copyright (c) 2022-2023 Martin Leitner-Ankerl <martin.ankerl@gmail.com> +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ANKERL_UNORDERED_DENSE_H +#define ANKERL_UNORDERED_DENSE_H + +// see https://semver.org/spec/v2.0.0.html +#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes +#define ANKERL_UNORDERED_DENSE_VERSION_MINOR 4 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality +#define ANKERL_UNORDERED_DENSE_VERSION_PATCH 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes + +// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/ + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) +#define ANKERL_UNORDERED_DENSE_NAMESPACE \ + ANKERL_UNORDERED_DENSE_VERSION_CONCAT( \ + ANKERL_UNORDERED_DENSE_VERSION_MAJOR, ANKERL_UNORDERED_DENSE_VERSION_MINOR, ANKERL_UNORDERED_DENSE_VERSION_PATCH) + +#if defined(_MSVC_LANG) +# define ANKERL_UNORDERED_DENSE_CPP_VERSION _MSVC_LANG +#else +# define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus +#endif + +#if defined(__GNUC__) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__)) +#elif defined(_MSC_VER) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop)) +#endif + +// exceptions +#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND) +# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 1 // NOLINT(cppcoreguidelines-macro-usage) +#else +# define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 0 // NOLINT(cppcoreguidelines-macro-usage) +#endif +#ifdef _MSC_VER +# define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline) +#else +# define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline)) +#endif + +// defined in unordered_dense.cpp +#if !defined(ANKERL_UNORDERED_DENSE_EXPORT) +# define ANKERL_UNORDERED_DENSE_EXPORT +#endif + +#if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L +# error ankerl::unordered_dense requires C++17 or higher +#else +# include <array> // for array +# include <cstdint> // for uint64_t, uint32_t, uint8_t, UINT64_C +# include <cstring> // for size_t, memcpy, memset +# include <functional> // for equal_to, hash +# include <initializer_list> // for initializer_list +# include <iterator> // for pair, distance +# include <limits> // for numeric_limits +# include <memory> // for allocator, allocator_traits, shared_ptr +# include <optional> // for optional +# include <stdexcept> // for out_of_range +# include <string> // for basic_string +# include <string_view> // for basic_string_view, hash +# include <tuple> // for forward_as_tuple +# include <type_traits> // for enable_if_t, declval, conditional_t, ena... +# include <utility> // for forward, exchange, pair, as_const, piece... +# include <vector> // for vector +# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() == 0 +# include <cstdlib> // for abort +# endif + +# if defined(__has_include) +# if __has_include(<memory_resource>) +# define ANKERL_UNORDERED_DENSE_PMR std::pmr // NOLINT(cppcoreguidelines-macro-usage) +# include <memory_resource> // for polymorphic_allocator +# elif __has_include(<experimental/memory_resource>) +# define ANKERL_UNORDERED_DENSE_PMR std::experimental::pmr // NOLINT(cppcoreguidelines-macro-usage) +# include <experimental/memory_resource> // for polymorphic_allocator +# endif +# endif + +# if defined(_MSC_VER) && defined(_M_X64) +# include <intrin.h> +# pragma intrinsic(_umul128) +# endif + +# if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) +# define ANKERL_UNORDERED_DENSE_LIKELY(x) __builtin_expect(x, 1) // NOLINT(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) __builtin_expect(x, 0) // NOLINT(cppcoreguidelines-macro-usage) +# else +# define ANKERL_UNORDERED_DENSE_LIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_UNLIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage) +# endif + +namespace ankerl::unordered_dense { +inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE { + +namespace detail { + +# if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() + +// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other +// inlinings more difficult. Throws are also generally the slow path. +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_not_found() { + throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found"); +} +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_bucket_overflow() { + throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size"); +} +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() { + throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements"); +} + +# else + +[[noreturn]] inline void on_error_key_not_found() { + abort(); +} +[[noreturn]] inline void on_error_bucket_overflow() { + abort(); +} +[[noreturn]] inline void on_error_too_many_elements() { + abort(); +} + +# endif + +} // namespace detail + +// hash /////////////////////////////////////////////////////////////////////// + +// This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash +// No big-endian support (because different values on different machines don't matter), +// hardcodes seed and the secret, reformats the code, and clang-tidy fixes. +namespace detail::wyhash { + +inline void mum(uint64_t* a, uint64_t* b) { +# if defined(__SIZEOF_INT128__) + __uint128_t r = *a; + r *= *b; + *a = static_cast<uint64_t>(r); + *b = static_cast<uint64_t>(r >> 64U); +# elif defined(_MSC_VER) && defined(_M_X64) + *a = _umul128(*a, *b, b); +# else + uint64_t ha = *a >> 32U; + uint64_t hb = *b >> 32U; + uint64_t la = static_cast<uint32_t>(*a); + uint64_t lb = static_cast<uint32_t>(*b); + uint64_t hi{}; + uint64_t lo{}; + uint64_t rh = ha * hb; + uint64_t rm0 = ha * lb; + uint64_t rm1 = hb * la; + uint64_t rl = la * lb; + uint64_t t = rl + (rm0 << 32U); + auto c = static_cast<uint64_t>(t < rl); + lo = t + (rm1 << 32U); + c += static_cast<uint64_t>(lo < t); + hi = rh + (rm0 >> 32U) + (rm1 >> 32U) + c; + *a = lo; + *b = hi; +# endif +} + +// multiply and xor mix function, aka MUM +[[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t { + mum(&a, &b); + return a ^ b; +} + +// read functions. WARNING: we don't care about endianness, so results are different on big endian! +[[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t { + uint64_t v{}; + std::memcpy(&v, p, 8U); + return v; +} + +[[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t { + uint32_t v{}; + std::memcpy(&v, p, 4); + return v; +} + +// reads 1, 2, or 3 bytes +[[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t { + return (static_cast<uint64_t>(p[0]) << 16U) | (static_cast<uint64_t>(p[k >> 1U]) << 8U) | p[k - 1]; +} + +[[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t { + static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f), + UINT64_C(0xe7037ed1a0b428db), + UINT64_C(0x8ebc6af09c88c6e3), + UINT64_C(0x589965cc75374cc3)}; + + auto const* p = static_cast<uint8_t const*>(key); + uint64_t seed = secret[0]; + uint64_t a{}; + uint64_t b{}; + if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) { + if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) { + a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U)); + b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U)); + } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) { + a = r3(p, len); + b = 0; + } else { + a = 0; + b = 0; + } + } else { + size_t i = len; + if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) { + uint64_t see1 = seed; + uint64_t see2 = seed; + do { + seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); + see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1); + see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2); + p += 48; + i -= 48; + } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48)); + seed ^= see1 ^ see2; + } + while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) { + seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); + i -= 16; + p += 16; + } + a = r8(p + i - 16); + b = r8(p + i - 8); + } + + return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed)); +} + +[[nodiscard]] inline auto hash(uint64_t x) -> uint64_t { + return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15)); +} + +} // namespace detail::wyhash + +ANKERL_UNORDERED_DENSE_EXPORT template <typename T, typename Enable = void> +struct hash { + auto operator()(T const& obj) const noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) + -> uint64_t { + return std::hash<T>{}(obj); + } +}; + +template <typename CharT> +struct hash<std::basic_string<CharT>> { + using is_avalanching = void; + auto operator()(std::basic_string<CharT> const& str) const noexcept -> uint64_t { + return detail::wyhash::hash(str.data(), sizeof(CharT) * str.size()); + } +}; + +template <typename CharT> +struct hash<std::basic_string_view<CharT>> { + using is_avalanching = void; + auto operator()(std::basic_string_view<CharT> const& sv) const noexcept -> uint64_t { + return detail::wyhash::hash(sv.data(), sizeof(CharT) * sv.size()); + } +}; + +template <class T> +struct hash<T*> { + using is_avalanching = void; + auto operator()(T* ptr) const noexcept -> uint64_t { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr)); + } +}; + +template <class T> +struct hash<std::unique_ptr<T>> { + using is_avalanching = void; + auto operator()(std::unique_ptr<T> const& ptr) const noexcept -> uint64_t { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get())); + } +}; + +template <class T> +struct hash<std::shared_ptr<T>> { + using is_avalanching = void; + auto operator()(std::shared_ptr<T> const& ptr) const noexcept -> uint64_t { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get())); + } +}; + +template <typename Enum> +struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> { + using is_avalanching = void; + auto operator()(Enum e) const noexcept -> uint64_t { + using underlying = typename std::underlying_type_t<Enum>; + return detail::wyhash::hash(static_cast<underlying>(e)); + } +}; + +template <typename... Args> +struct tuple_hash_helper { + // Converts the value into 64bit. If it is an integral type, just cast it. Mixing is doing the rest. + // If it isn't an integral we need to hash it. + template <typename Arg> + [[nodiscard]] constexpr static auto to64(Arg const& arg) -> uint64_t { + if constexpr (std::is_integral_v<Arg> || std::is_enum_v<Arg>) { + return static_cast<uint64_t>(arg); + } else { + return hash<Arg>{}(arg); + } + } + + [[nodiscard]] static auto mix64(uint64_t state, uint64_t v) -> uint64_t { + return detail::wyhash::mix(state + v, uint64_t{0x9ddfea08eb382d69}); + } + + // Creates a buffer that holds all the data from each element of the tuple. If possible we memcpy the data directly. If + // not, we hash the object and use this for the array. Size of the array is known at compile time, and memcpy is optimized + // away, so filling the buffer is highly efficient. Finally, call wyhash with this buffer. + template <typename T, std::size_t... Idx> + [[nodiscard]] static auto calc_hash(T const& t, std::index_sequence<Idx...>) noexcept -> uint64_t { + auto h = uint64_t{}; + ((h = mix64(h, to64(std::get<Idx>(t)))), ...); + return h; + } +}; + +template <typename... Args> +struct hash<std::tuple<Args...>> : tuple_hash_helper<Args...> { + using is_avalanching = void; + auto operator()(std::tuple<Args...> const& t) const noexcept -> uint64_t { + return tuple_hash_helper<Args...>::calc_hash(t, std::index_sequence_for<Args...>{}); + } +}; + +template <typename A, typename B> +struct hash<std::pair<A, B>> : tuple_hash_helper<A, B> { + using is_avalanching = void; + auto operator()(std::pair<A, B> const& t) const noexcept -> uint64_t { + return tuple_hash_helper<A, B>::calc_hash(t, std::index_sequence_for<A, B>{}); + } +}; + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +# define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T) \ + template <> \ + struct hash<T> { \ + using is_avalanching = void; \ + auto operator()(T const& obj) const noexcept -> uint64_t { \ + return detail::wyhash::hash(static_cast<uint64_t>(obj)); \ + } \ + } + +# if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wuseless-cast" +# endif +// see https://en.cppreference.com/w/cpp/utility/hash +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char); +# if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L && defined(__cpp_char8_t) +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t); +# endif +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char32_t); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(wchar_t); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(short); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned short); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(int); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned int); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long long); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long); + +# if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic pop +# endif + +// bucket_type ////////////////////////////////////////////////////////// + +namespace bucket_type { + +struct standard { + static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint + static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint + + uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash + uint32_t m_value_idx; // index into the m_values vector. +}; + +ANKERL_UNORDERED_DENSE_PACK(struct big { + static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint + static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint + + uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash + size_t m_value_idx; // index into the m_values vector. +}); + +} // namespace bucket_type + +namespace detail { + +struct nonesuch {}; + +template <class Default, class AlwaysVoid, template <class...> class Op, class... Args> +struct detector { + using value_t = std::false_type; + using type = Default; +}; + +template <class Default, template <class...> class Op, class... Args> +struct detector<Default, std::void_t<Op<Args...>>, Op, Args...> { + using value_t = std::true_type; + using type = Op<Args...>; +}; + +template <template <class...> class Op, class... Args> +using is_detected = typename detail::detector<detail::nonesuch, void, Op, Args...>::value_t; + +template <template <class...> class Op, class... Args> +constexpr bool is_detected_v = is_detected<Op, Args...>::value; + +template <typename T> +using detect_avalanching = typename T::is_avalanching; + +template <typename T> +using detect_is_transparent = typename T::is_transparent; + +template <typename T> +using detect_iterator = typename T::iterator; + +template <typename T> +using detect_reserve = decltype(std::declval<T&>().reserve(size_t{})); + +// enable_if helpers + +template <typename Mapped> +constexpr bool is_map_v = !std::is_void_v<Mapped>; + +// clang-format off +template <typename Hash, typename KeyEqual> +constexpr bool is_transparent_v = is_detected_v<detect_is_transparent, Hash> && is_detected_v<detect_is_transparent, KeyEqual>; +// clang-format on + +template <typename From, typename To1, typename To2> +constexpr bool is_neither_convertible_v = !std::is_convertible_v<From, To1> && !std::is_convertible_v<From, To2>; + +template <typename T> +constexpr bool has_reserve = is_detected_v<detect_reserve, T>; + +// base type for map has mapped_type +template <class T> +struct base_table_type_map { + using mapped_type = T; +}; + +// base type for set doesn't have mapped_type +struct base_table_type_set {}; + +} // namespace detail + +// Very much like std::deque, but faster for indexing (in most cases). As of now this doesn't implement the full std::vector +// API, but merely what's necessary to work as an underlying container for ankerl::unordered_dense::{map, set}. +// It allocates blocks of equal size and puts them into the m_blocks vector. That means it can grow simply by adding a new +// block to the back of m_blocks, and doesn't double its size like an std::vector. The disadvantage is that memory is not +// linear and thus there is one more indirection necessary for indexing. +template <typename T, typename Allocator = std::allocator<T>, size_t MaxSegmentSizeBytes = 4096> +class segmented_vector { + template <bool IsConst> + class iter_t; + +public: + using allocator_type = Allocator; + using pointer = typename std::allocator_traits<allocator_type>::pointer; + using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer; + using difference_type = typename std::allocator_traits<allocator_type>::difference_type; + using value_type = T; + using size_type = std::size_t; + using reference = T&; + using const_reference = T const&; + using iterator = iter_t<false>; + using const_iterator = iter_t<true>; + +private: + using vec_alloc = typename std::allocator_traits<Allocator>::template rebind_alloc<pointer>; + std::vector<pointer, vec_alloc> m_blocks{}; + size_t m_size{}; + + // Calculates the maximum number for x in (s << x) <= max_val + static constexpr auto num_bits_closest(size_t max_val, size_t s) -> size_t { + auto f = size_t{0}; + while (s << (f + 1) <= max_val) { + ++f; + } + return f; + } + + using self_t = segmented_vector<T, Allocator, MaxSegmentSizeBytes>; + static constexpr auto num_bits = num_bits_closest(MaxSegmentSizeBytes, sizeof(T)); + static constexpr auto num_elements_in_block = 1U << num_bits; + static constexpr auto mask = num_elements_in_block - 1U; + + /** + * Iterator class doubles as const_iterator and iterator + */ + template <bool IsConst> + class iter_t { + using ptr_t = typename std::conditional_t<IsConst, segmented_vector::const_pointer const*, segmented_vector::pointer*>; + ptr_t m_data{}; + size_t m_idx{}; + + template <bool B> + friend class iter_t; + + public: + using difference_type = segmented_vector::difference_type; + using value_type = T; + using reference = typename std::conditional_t<IsConst, value_type const&, value_type&>; + using pointer = typename std::conditional_t<IsConst, segmented_vector::const_pointer, segmented_vector::pointer>; + using iterator_category = std::forward_iterator_tag; + + iter_t() noexcept = default; + + template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type> + // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions) + constexpr iter_t(iter_t<OtherIsConst> const& other) noexcept + : m_data(other.m_data) + , m_idx(other.m_idx) {} + + constexpr iter_t(ptr_t data, size_t idx) noexcept + : m_data(data) + , m_idx(idx) {} + + template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type> + constexpr auto operator=(iter_t<OtherIsConst> const& other) noexcept -> iter_t& { + m_data = other.m_data; + m_idx = other.m_idx; + return *this; + } + + constexpr auto operator++() noexcept -> iter_t& { + ++m_idx; + return *this; + } + + constexpr auto operator+(difference_type diff) noexcept -> iter_t { + return {m_data, static_cast<size_t>(static_cast<difference_type>(m_idx) + diff)}; + } + + template <bool OtherIsConst> + constexpr auto operator-(iter_t<OtherIsConst> const& other) noexcept -> difference_type { + return static_cast<difference_type>(m_idx) - static_cast<difference_type>(other.m_idx); + } + + constexpr auto operator*() const noexcept -> reference { + return m_data[m_idx >> num_bits][m_idx & mask]; + } + + constexpr auto operator->() const noexcept -> pointer { + return &m_data[m_idx >> num_bits][m_idx & mask]; + } + + template <bool O> + constexpr auto operator==(iter_t<O> const& o) const noexcept -> bool { + return m_idx == o.m_idx; + } + + template <bool O> + constexpr auto operator!=(iter_t<O> const& o) const noexcept -> bool { + return !(*this == o); + } + }; + + // slow path: need to allocate a new segment every once in a while + void increase_capacity() { + auto ba = Allocator(m_blocks.get_allocator()); + pointer block = std::allocator_traits<Allocator>::allocate(ba, num_elements_in_block); + m_blocks.push_back(block); + } + + // Moves everything from other + void append_everything_from(segmented_vector&& other) { + reserve(size() + other.size()); + for (auto&& o : other) { + emplace_back(std::move(o)); + } + } + + // Copies everything from other + void append_everything_from(segmented_vector const& other) { + reserve(size() + other.size()); + for (auto const& o : other) { + emplace_back(o); + } + } + + void dealloc() { + auto ba = Allocator(m_blocks.get_allocator()); + for (auto ptr : m_blocks) { + std::allocator_traits<Allocator>::deallocate(ba, ptr, num_elements_in_block); + } + } + + [[nodiscard]] static constexpr auto calc_num_blocks_for_capacity(size_t capacity) { + return (capacity + num_elements_in_block - 1U) / num_elements_in_block; + } + +public: + segmented_vector() = default; + + // NOLINTNEXTLINE(google-explicit-constructor,hicpp-explicit-conversions) + segmented_vector(Allocator alloc) + : m_blocks(vec_alloc(alloc)) {} + + segmented_vector(segmented_vector&& other, Allocator alloc) + : segmented_vector(alloc) { + *this = std::move(other); + } + + segmented_vector(segmented_vector const& other, Allocator alloc) + : m_blocks(vec_alloc(alloc)) { + append_everything_from(other); + } + + segmented_vector(segmented_vector&& other) noexcept + : segmented_vector(std::move(other), get_allocator()) {} + + segmented_vector(segmented_vector const& other) { + append_everything_from(other); + } + + auto operator=(segmented_vector const& other) -> segmented_vector& { + if (this == &other) { + return *this; + } + clear(); + append_everything_from(other); + return *this; + } + + auto operator=(segmented_vector&& other) noexcept -> segmented_vector& { + clear(); + dealloc(); + if (other.get_allocator() == get_allocator()) { + m_blocks = std::move(other.m_blocks); + m_size = std::exchange(other.m_size, {}); + } else { + // make sure to construct with other's allocator! + m_blocks = std::vector<pointer, vec_alloc>(vec_alloc(other.get_allocator())); + append_everything_from(std::move(other)); + } + return *this; + } + + ~segmented_vector() { + clear(); + dealloc(); + } + + [[nodiscard]] constexpr auto size() const -> size_t { + return m_size; + } + + [[nodiscard]] constexpr auto capacity() const -> size_t { + return m_blocks.size() * num_elements_in_block; + } + + // Indexing is highly performance critical + [[nodiscard]] constexpr auto operator[](size_t i) const noexcept -> T const& { + return m_blocks[i >> num_bits][i & mask]; + } + + [[nodiscard]] constexpr auto operator[](size_t i) noexcept -> T& { + return m_blocks[i >> num_bits][i & mask]; + } + + [[nodiscard]] constexpr auto begin() -> iterator { + return {m_blocks.data(), 0U}; + } + [[nodiscard]] constexpr auto begin() const -> const_iterator { + return {m_blocks.data(), 0U}; + } + [[nodiscard]] constexpr auto cbegin() const -> const_iterator { + return {m_blocks.data(), 0U}; + } + + [[nodiscard]] constexpr auto end() -> iterator { + return {m_blocks.data(), m_size}; + } + [[nodiscard]] constexpr auto end() const -> const_iterator { + return {m_blocks.data(), m_size}; + } + [[nodiscard]] constexpr auto cend() const -> const_iterator { + return {m_blocks.data(), m_size}; + } + + [[nodiscard]] constexpr auto back() -> reference { + return operator[](m_size - 1); + } + [[nodiscard]] constexpr auto back() const -> const_reference { + return operator[](m_size - 1); + } + + void pop_back() { + back().~T(); + --m_size; + } + + [[nodiscard]] auto empty() const { + return 0 == m_size; + } + + void reserve(size_t new_capacity) { + m_blocks.reserve(calc_num_blocks_for_capacity(new_capacity)); + while (new_capacity > capacity()) { + increase_capacity(); + } + } + + [[nodiscard]] auto get_allocator() const -> allocator_type { + return allocator_type{m_blocks.get_allocator()}; + } + + template <class... Args> + auto emplace_back(Args&&... args) -> reference { + if (m_size == capacity()) { + increase_capacity(); + } + auto* ptr = static_cast<void*>(&operator[](m_size)); + auto& ref = *new (ptr) T(std::forward<Args>(args)...); + ++m_size; + return ref; + } + + void clear() { + if constexpr (!std::is_trivially_destructible_v<T>) { + for (size_t i = 0, s = size(); i < s; ++i) { + operator[](i).~T(); + } + } + m_size = 0; + } + + void shrink_to_fit() { + auto ba = Allocator(m_blocks.get_allocator()); + auto num_blocks_required = calc_num_blocks_for_capacity(m_size); + while (m_blocks.size() > num_blocks_required) { + std::allocator_traits<Allocator>::deallocate(ba, m_blocks.back(), num_elements_in_block); + m_blocks.pop_back(); + } + m_blocks.shrink_to_fit(); + } +}; + +namespace detail { + +// This is it, the table. Doubles as map and set, and uses `void` for T when its used as a set. +template <class Key, + class T, // when void, treat it as a set. + class Hash, + class KeyEqual, + class AllocatorOrContainer, + class Bucket, + bool IsSegmented> +class table : public std::conditional_t<is_map_v<T>, base_table_type_map<T>, base_table_type_set> { + using underlying_value_type = typename std::conditional_t<is_map_v<T>, std::pair<Key, T>, Key>; + using underlying_container_type = std::conditional_t<IsSegmented, + segmented_vector<underlying_value_type, AllocatorOrContainer>, + std::vector<underlying_value_type, AllocatorOrContainer>>; + +public: + using value_container_type = std:: + conditional_t<is_detected_v<detect_iterator, AllocatorOrContainer>, AllocatorOrContainer, underlying_container_type>; + +private: + using bucket_alloc = + typename std::allocator_traits<typename value_container_type::allocator_type>::template rebind_alloc<Bucket>; + using bucket_alloc_traits = std::allocator_traits<bucket_alloc>; + + static constexpr uint8_t initial_shifts = 64 - 2; // 2^(64-m_shift) number of buckets + static constexpr float default_max_load_factor = 0.8F; + +public: + using key_type = Key; + using value_type = typename value_container_type::value_type; + using size_type = typename value_container_type::size_type; + using difference_type = typename value_container_type::difference_type; + using hasher = Hash; + using key_equal = KeyEqual; + using allocator_type = typename value_container_type::allocator_type; + using reference = typename value_container_type::reference; + using const_reference = typename value_container_type::const_reference; + using pointer = typename value_container_type::pointer; + using const_pointer = typename value_container_type::const_pointer; + using const_iterator = typename value_container_type::const_iterator; + using iterator = std::conditional_t<is_map_v<T>, typename value_container_type::iterator, const_iterator>; + using bucket_type = Bucket; + +private: + using value_idx_type = decltype(Bucket::m_value_idx); + using dist_and_fingerprint_type = decltype(Bucket::m_dist_and_fingerprint); + + static_assert(std::is_trivially_destructible_v<Bucket>, "assert there's no need to call destructor / std::destroy"); + static_assert(std::is_trivially_copyable_v<Bucket>, "assert we can just memset / memcpy"); + + value_container_type m_values{}; // Contains all the key-value pairs in one densely stored container. No holes. + using bucket_pointer = typename std::allocator_traits<bucket_alloc>::pointer; + bucket_pointer m_buckets{}; + size_t m_num_buckets = 0; + size_t m_max_bucket_capacity = 0; + float m_max_load_factor = default_max_load_factor; + Hash m_hash{}; + KeyEqual m_equal{}; + uint8_t m_shifts = initial_shifts; + + [[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type { + return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == m_num_buckets) + ? 0 + : static_cast<value_idx_type>(bucket_idx + 1U); + } + + // Helper to access bucket through pointer types + [[nodiscard]] static constexpr auto at(bucket_pointer bucket_ptr, size_t offset) -> Bucket& { + return *(bucket_ptr + static_cast<typename std::allocator_traits<bucket_alloc>::difference_type>(offset)); + } + + // use the dist_inc and dist_dec functions so that uint16_t types work without warning + [[nodiscard]] static constexpr auto dist_inc(dist_and_fingerprint_type x) -> dist_and_fingerprint_type { + return static_cast<dist_and_fingerprint_type>(x + Bucket::dist_inc); + } + + [[nodiscard]] static constexpr auto dist_dec(dist_and_fingerprint_type x) -> dist_and_fingerprint_type { + return static_cast<dist_and_fingerprint_type>(x - Bucket::dist_inc); + } + + // The goal of mixed_hash is to always produce a high quality 64bit hash. + template <typename K> + [[nodiscard]] constexpr auto mixed_hash(K const& key) const -> uint64_t { + if constexpr (is_detected_v<detect_avalanching, Hash>) { + // we know that the hash is good because is_avalanching. + if constexpr (sizeof(decltype(m_hash(key))) < sizeof(uint64_t)) { + // 32bit hash and is_avalanching => multiply with a constant to avalanche bits upwards + return m_hash(key) * UINT64_C(0x9ddfea08eb382d69); + } else { + // 64bit and is_avalanching => only use the hash itself. + return m_hash(key); + } + } else { + // not is_avalanching => apply wyhash + return wyhash::hash(m_hash(key)); + } + } + + [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type { + return Bucket::dist_inc | (static_cast<dist_and_fingerprint_type>(hash) & Bucket::fingerprint_mask); + } + + [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> value_idx_type { + return static_cast<value_idx_type>(hash >> m_shifts); + } + + [[nodiscard]] static constexpr auto get_key(value_type const& vt) -> key_type const& { + if constexpr (is_map_v<T>) { + return vt.first; + } else { + return vt; + } + } + + template <typename K> + [[nodiscard]] auto next_while_less(K const& key) const -> Bucket { + auto hash = mixed_hash(key); + auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); + auto bucket_idx = bucket_idx_from_hash(hash); + + while (dist_and_fingerprint < at(m_buckets, bucket_idx).m_dist_and_fingerprint) { + dist_and_fingerprint = dist_inc(dist_and_fingerprint); + bucket_idx = next(bucket_idx); + } + return {dist_and_fingerprint, bucket_idx}; + } + + void place_and_shift_up(Bucket bucket, value_idx_type place) { + while (0 != at(m_buckets, place).m_dist_and_fingerprint) { + bucket = std::exchange(at(m_buckets, place), bucket); + bucket.m_dist_and_fingerprint = dist_inc(bucket.m_dist_and_fingerprint); + place = next(place); + } + at(m_buckets, place) = bucket; + } + + [[nodiscard]] static constexpr auto calc_num_buckets(uint8_t shifts) -> size_t { + return (std::min)(max_bucket_count(), size_t{1} << (64U - shifts)); + } + + [[nodiscard]] constexpr auto calc_shifts_for_size(size_t s) const -> uint8_t { + auto shifts = initial_shifts; + while (shifts > 0 && static_cast<size_t>(static_cast<float>(calc_num_buckets(shifts)) * max_load_factor()) < s) { + --shifts; + } + return shifts; + } + + // assumes m_values has data, m_buckets=m_buckets_end=nullptr, m_shifts is INITIAL_SHIFTS + void copy_buckets(table const& other) { + // assumes m_values has already the correct data copied over. + if (empty()) { + // when empty, at least allocate an initial buckets and clear them. + allocate_buckets_from_shift(); + clear_buckets(); + } else { + m_shifts = other.m_shifts; + allocate_buckets_from_shift(); + std::memcpy(m_buckets, other.m_buckets, sizeof(Bucket) * bucket_count()); + } + } + + /** + * True when no element can be added any more without increasing the size + */ + [[nodiscard]] auto is_full() const -> bool { + return size() > m_max_bucket_capacity; + } + + void deallocate_buckets() { + auto ba = bucket_alloc(m_values.get_allocator()); + if (nullptr != m_buckets) { + bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count()); + m_buckets = nullptr; + } + m_num_buckets = 0; + m_max_bucket_capacity = 0; + } + + void allocate_buckets_from_shift() { + auto ba = bucket_alloc(m_values.get_allocator()); + m_num_buckets = calc_num_buckets(m_shifts); + m_buckets = bucket_alloc_traits::allocate(ba, m_num_buckets); + if (m_num_buckets == max_bucket_count()) { + // reached the maximum, make sure we can use each bucket + m_max_bucket_capacity = max_bucket_count(); + } else { + m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(m_num_buckets) * max_load_factor()); + } + } + + void clear_buckets() { + if (m_buckets != nullptr) { + std::memset(&*m_buckets, 0, sizeof(Bucket) * bucket_count()); + } + } + + void clear_and_fill_buckets_from_values() { + clear_buckets(); + for (value_idx_type value_idx = 0, end_idx = static_cast<value_idx_type>(m_values.size()); value_idx < end_idx; + ++value_idx) { + auto const& key = get_key(m_values[value_idx]); + auto [dist_and_fingerprint, bucket] = next_while_less(key); + + // we know for certain that key has not yet been inserted, so no need to check it. + place_and_shift_up({dist_and_fingerprint, value_idx}, bucket); + } + } + + void increase_size() { + if (m_max_bucket_capacity == max_bucket_count()) { + // remove the value again, we can't add it! + m_values.pop_back(); + on_error_bucket_overflow(); + } + --m_shifts; + deallocate_buckets(); + allocate_buckets_from_shift(); + clear_and_fill_buckets_from_values(); + } + + template <typename Op> + void do_erase(value_idx_type bucket_idx, Op handle_erased_value) { + auto const value_idx_to_remove = at(m_buckets, bucket_idx).m_value_idx; + + // shift down until either empty or an element with correct spot is found + auto next_bucket_idx = next(bucket_idx); + while (at(m_buckets, next_bucket_idx).m_dist_and_fingerprint >= Bucket::dist_inc * 2) { + at(m_buckets, bucket_idx) = {dist_dec(at(m_buckets, next_bucket_idx).m_dist_and_fingerprint), + at(m_buckets, next_bucket_idx).m_value_idx}; + bucket_idx = std::exchange(next_bucket_idx, next(next_bucket_idx)); + } + at(m_buckets, bucket_idx) = {}; + handle_erased_value(std::move(m_values[value_idx_to_remove])); + + // update m_values + if (value_idx_to_remove != m_values.size() - 1) { + // no luck, we'll have to replace the value with the last one and update the index accordingly + auto& val = m_values[value_idx_to_remove]; + val = std::move(m_values.back()); + + // update the values_idx of the moved entry. No need to play the info game, just look until we find the values_idx + auto mh = mixed_hash(get_key(val)); + bucket_idx = bucket_idx_from_hash(mh); + + auto const values_idx_back = static_cast<value_idx_type>(m_values.size() - 1); + while (values_idx_back != at(m_buckets, bucket_idx).m_value_idx) { + bucket_idx = next(bucket_idx); + } + at(m_buckets, bucket_idx).m_value_idx = value_idx_to_remove; + } + m_values.pop_back(); + } + + template <typename K, typename Op> + auto do_erase_key(K&& key, Op handle_erased_value) -> size_t { + if (empty()) { + return 0; + } + + auto [dist_and_fingerprint, bucket_idx] = next_while_less(key); + + while (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint && + !m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) { + dist_and_fingerprint = dist_inc(dist_and_fingerprint); + bucket_idx = next(bucket_idx); + } + + if (dist_and_fingerprint != at(m_buckets, bucket_idx).m_dist_and_fingerprint) { + return 0; + } + do_erase(bucket_idx, handle_erased_value); + return 1; + } + + template <class K, class M> + auto do_insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> { + auto it_isinserted = try_emplace(std::forward<K>(key), std::forward<M>(mapped)); + if (!it_isinserted.second) { + it_isinserted.first->second = std::forward<M>(mapped); + } + return it_isinserted; + } + + template <typename... Args> + auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint, value_idx_type bucket_idx, Args&&... args) + -> std::pair<iterator, bool> { + + // emplace the new value. If that throws an exception, no harm done; index is still in a valid state + m_values.emplace_back(std::forward<Args>(args)...); + + auto value_idx = static_cast<value_idx_type>(m_values.size() - 1); + if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) { + increase_size(); + } else { + place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); + } + + // place element and shift up until we find an empty spot + return {begin() + static_cast<difference_type>(value_idx), true}; + } + + template <typename K, typename... Args> + auto do_try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> { + auto hash = mixed_hash(key); + auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); + auto bucket_idx = bucket_idx_from_hash(hash); + + while (true) { + auto* bucket = &at(m_buckets, bucket_idx); + if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) { + if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) { + return {begin() + static_cast<difference_type>(bucket->m_value_idx), false}; + } + } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) { + return do_place_element(dist_and_fingerprint, + bucket_idx, + std::piecewise_construct, + std::forward_as_tuple(std::forward<K>(key)), + std::forward_as_tuple(std::forward<Args>(args)...)); + } + dist_and_fingerprint = dist_inc(dist_and_fingerprint); + bucket_idx = next(bucket_idx); + } + } + + template <typename K> + auto do_find(K const& key) -> iterator { + if (ANKERL_UNORDERED_DENSE_UNLIKELY(empty())) { + return end(); + } + + auto mh = mixed_hash(key); + auto dist_and_fingerprint = dist_and_fingerprint_from_hash(mh); + auto bucket_idx = bucket_idx_from_hash(mh); + auto* bucket = &at(m_buckets, bucket_idx); + + // unrolled loop. *Always* check a few directly, then enter the loop. This is faster. + if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) { + return begin() + static_cast<difference_type>(bucket->m_value_idx); + } + dist_and_fingerprint = dist_inc(dist_and_fingerprint); + bucket_idx = next(bucket_idx); + bucket = &at(m_buckets, bucket_idx); + + if (dist_and_fingerprint == bucket->m_dist_and_fingerprint && m_equal(key, get_key(m_values[bucket->m_value_idx]))) { + return begin() + static_cast<difference_type>(bucket->m_value_idx); + } + dist_and_fingerprint = dist_inc(dist_and_fingerprint); + bucket_idx = next(bucket_idx); + bucket = &at(m_buckets, bucket_idx); + + while (true) { + if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) { + if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) { + return begin() + static_cast<difference_type>(bucket->m_value_idx); + } + } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) { + return end(); + } + dist_and_fingerprint = dist_inc(dist_and_fingerprint); + bucket_idx = next(bucket_idx); + bucket = &at(m_buckets, bucket_idx); + } + } + + template <typename K> + auto do_find(K const& key) const -> const_iterator { + return const_cast<table*>(this)->do_find(key); // NOLINT(cppcoreguidelines-pro-type-const-cast) + } + + template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto do_at(K const& key) -> Q& { + if (auto it = find(key); ANKERL_UNORDERED_DENSE_LIKELY(end() != it)) { + return it->second; + } + on_error_key_not_found(); + } + + template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto do_at(K const& key) const -> Q const& { + return const_cast<table*>(this)->at(key); // NOLINT(cppcoreguidelines-pro-type-const-cast) + } + +public: + explicit table(size_t bucket_count, + Hash const& hash = Hash(), + KeyEqual const& equal = KeyEqual(), + allocator_type const& alloc_or_container = allocator_type()) + : m_values(alloc_or_container) + , m_hash(hash) + , m_equal(equal) { + if (0 != bucket_count) { + reserve(bucket_count); + } else { + allocate_buckets_from_shift(); + clear_buckets(); + } + } + + table() + : table(0) {} + + table(size_t bucket_count, allocator_type const& alloc) + : table(bucket_count, Hash(), KeyEqual(), alloc) {} + + table(size_t bucket_count, Hash const& hash, allocator_type const& alloc) + : table(bucket_count, hash, KeyEqual(), alloc) {} + + explicit table(allocator_type const& alloc) + : table(0, Hash(), KeyEqual(), alloc) {} + + template <class InputIt> + table(InputIt first, + InputIt last, + size_type bucket_count = 0, + Hash const& hash = Hash(), + KeyEqual const& equal = KeyEqual(), + allocator_type const& alloc = allocator_type()) + : table(bucket_count, hash, equal, alloc) { + insert(first, last); + } + + template <class InputIt> + table(InputIt first, InputIt last, size_type bucket_count, allocator_type const& alloc) + : table(first, last, bucket_count, Hash(), KeyEqual(), alloc) {} + + template <class InputIt> + table(InputIt first, InputIt last, size_type bucket_count, Hash const& hash, allocator_type const& alloc) + : table(first, last, bucket_count, hash, KeyEqual(), alloc) {} + + table(table const& other) + : table(other, other.m_values.get_allocator()) {} + + table(table const& other, allocator_type const& alloc) + : m_values(other.m_values, alloc) + , m_max_load_factor(other.m_max_load_factor) + , m_hash(other.m_hash) + , m_equal(other.m_equal) { + copy_buckets(other); + } + + table(table&& other) noexcept + : table(std::move(other), other.m_values.get_allocator()) {} + + table(table&& other, allocator_type const& alloc) noexcept + : m_values(alloc) { + *this = std::move(other); + } + + table(std::initializer_list<value_type> ilist, + size_t bucket_count = 0, + Hash const& hash = Hash(), + KeyEqual const& equal = KeyEqual(), + allocator_type const& alloc = allocator_type()) + : table(bucket_count, hash, equal, alloc) { + insert(ilist); + } + + table(std::initializer_list<value_type> ilist, size_type bucket_count, allocator_type const& alloc) + : table(ilist, bucket_count, Hash(), KeyEqual(), alloc) {} + + table(std::initializer_list<value_type> init, size_type bucket_count, Hash const& hash, allocator_type const& alloc) + : table(init, bucket_count, hash, KeyEqual(), alloc) {} + + ~table() { + if (nullptr != m_buckets) { + auto ba = bucket_alloc(m_values.get_allocator()); + bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count()); + } + } + + auto operator=(table const& other) -> table& { + if (&other != this) { + deallocate_buckets(); // deallocate before m_values is set (might have another allocator) + m_values = other.m_values; + m_max_load_factor = other.m_max_load_factor; + m_hash = other.m_hash; + m_equal = other.m_equal; + m_shifts = initial_shifts; + copy_buckets(other); + } + return *this; + } + + auto operator=(table&& other) noexcept(noexcept(std::is_nothrow_move_assignable_v<value_container_type> && + std::is_nothrow_move_assignable_v<Hash> && + std::is_nothrow_move_assignable_v<KeyEqual>)) -> table& { + if (&other != this) { + deallocate_buckets(); // deallocate before m_values is set (might have another allocator) + m_values = std::move(other.m_values); + other.m_values.clear(); + + // we can only reuse m_buckets when both maps have the same allocator! + if (get_allocator() == other.get_allocator()) { + m_buckets = std::exchange(other.m_buckets, nullptr); + m_num_buckets = std::exchange(other.m_num_buckets, 0); + m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0); + m_shifts = std::exchange(other.m_shifts, initial_shifts); + m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor); + m_hash = std::exchange(other.m_hash, {}); + m_equal = std::exchange(other.m_equal, {}); + other.allocate_buckets_from_shift(); + other.clear_buckets(); + } else { + // set max_load_factor *before* copying the other's buckets, so we have the same + // behavior + m_max_load_factor = other.m_max_load_factor; + + // copy_buckets sets m_buckets, m_num_buckets, m_max_bucket_capacity, m_shifts + copy_buckets(other); + // clear's the other's buckets so other is now already usable. + other.clear_buckets(); + m_hash = other.m_hash; + m_equal = other.m_equal; + } + // map "other" is now already usable, it's empty. + } + return *this; + } + + auto operator=(std::initializer_list<value_type> ilist) -> table& { + clear(); + insert(ilist); + return *this; + } + + auto get_allocator() const noexcept -> allocator_type { + return m_values.get_allocator(); + } + + // iterators ////////////////////////////////////////////////////////////// + + auto begin() noexcept -> iterator { + return m_values.begin(); + } + + auto begin() const noexcept -> const_iterator { + return m_values.begin(); + } + + auto cbegin() const noexcept -> const_iterator { + return m_values.cbegin(); + } + + auto end() noexcept -> iterator { + return m_values.end(); + } + + auto cend() const noexcept -> const_iterator { + return m_values.cend(); + } + + auto end() const noexcept -> const_iterator { + return m_values.end(); + } + + // capacity /////////////////////////////////////////////////////////////// + + [[nodiscard]] auto empty() const noexcept -> bool { + return m_values.empty(); + } + + [[nodiscard]] auto size() const noexcept -> size_t { + return m_values.size(); + } + + [[nodiscard]] static constexpr auto max_size() noexcept -> size_t { + if constexpr ((std::numeric_limits<value_idx_type>::max)() == (std::numeric_limits<size_t>::max)()) { + return size_t{1} << (sizeof(value_idx_type) * 8 - 1); + } else { + return size_t{1} << (sizeof(value_idx_type) * 8); + } + } + + // modifiers ////////////////////////////////////////////////////////////// + + void clear() { + m_values.clear(); + clear_buckets(); + } + + auto insert(value_type const& value) -> std::pair<iterator, bool> { + return emplace(value); + } + + auto insert(value_type&& value) -> std::pair<iterator, bool> { + return emplace(std::move(value)); + } + + template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true> + auto insert(P&& value) -> std::pair<iterator, bool> { + return emplace(std::forward<P>(value)); + } + + auto insert(const_iterator /*hint*/, value_type const& value) -> iterator { + return insert(value).first; + } + + auto insert(const_iterator /*hint*/, value_type&& value) -> iterator { + return insert(std::move(value)).first; + } + + template <class P, std::enable_if_t<std::is_constructible_v<value_type, P&&>, bool> = true> + auto insert(const_iterator /*hint*/, P&& value) -> iterator { + return insert(std::forward<P>(value)).first; + } + + template <class InputIt> + void insert(InputIt first, InputIt last) { + while (first != last) { + insert(*first); + ++first; + } + } + + void insert(std::initializer_list<value_type> ilist) { + insert(ilist.begin(), ilist.end()); + } + + // nonstandard API: *this is emptied. + // Also see "A Standard flat_map" https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p0429r9.pdf + auto extract() && -> value_container_type { + return std::move(m_values); + } + + // nonstandard API: + // Discards the internally held container and replaces it with the one passed. Erases non-unique elements. + auto replace(value_container_type&& container) { + if (ANKERL_UNORDERED_DENSE_UNLIKELY(container.size() > max_size())) { + on_error_too_many_elements(); + } + auto shifts = calc_shifts_for_size(container.size()); + if (0 == m_num_buckets || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) { + m_shifts = shifts; + deallocate_buckets(); + allocate_buckets_from_shift(); + } + clear_buckets(); + + m_values = std::move(container); + + // can't use clear_and_fill_buckets_from_values() because container elements might not be unique + auto value_idx = value_idx_type{}; + + // loop until we reach the end of the container. duplicated entries will be replaced with back(). + while (value_idx != static_cast<value_idx_type>(m_values.size())) { + auto const& key = get_key(m_values[value_idx]); + + auto hash = mixed_hash(key); + auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); + auto bucket_idx = bucket_idx_from_hash(hash); + + bool key_found = false; + while (true) { + auto const& bucket = at(m_buckets, bucket_idx); + if (dist_and_fingerprint > bucket.m_dist_and_fingerprint) { + break; + } + if (dist_and_fingerprint == bucket.m_dist_and_fingerprint && + m_equal(key, get_key(m_values[bucket.m_value_idx]))) { + key_found = true; + break; + } + dist_and_fingerprint = dist_inc(dist_and_fingerprint); + bucket_idx = next(bucket_idx); + } + + if (key_found) { + if (value_idx != static_cast<value_idx_type>(m_values.size() - 1)) { + m_values[value_idx] = std::move(m_values.back()); + } + m_values.pop_back(); + } else { + place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); + ++value_idx; + } + } + } + + template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto insert_or_assign(Key const& key, M&& mapped) -> std::pair<iterator, bool> { + return do_insert_or_assign(key, std::forward<M>(mapped)); + } + + template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto insert_or_assign(Key&& key, M&& mapped) -> std::pair<iterator, bool> { + return do_insert_or_assign(std::move(key), std::forward<M>(mapped)); + } + + template <typename K, + typename M, + typename Q = T, + typename H = Hash, + typename KE = KeyEqual, + std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true> + auto insert_or_assign(K&& key, M&& mapped) -> std::pair<iterator, bool> { + return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped)); + } + + template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto insert_or_assign(const_iterator /*hint*/, Key const& key, M&& mapped) -> iterator { + return do_insert_or_assign(key, std::forward<M>(mapped)).first; + } + + template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto insert_or_assign(const_iterator /*hint*/, Key&& key, M&& mapped) -> iterator { + return do_insert_or_assign(std::move(key), std::forward<M>(mapped)).first; + } + + template <typename K, + typename M, + typename Q = T, + typename H = Hash, + typename KE = KeyEqual, + std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true> + auto insert_or_assign(const_iterator /*hint*/, K&& key, M&& mapped) -> iterator { + return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped)).first; + } + + // Single arguments for unordered_set can be used without having to construct the value_type + template <class K, + typename Q = T, + typename H = Hash, + typename KE = KeyEqual, + std::enable_if_t<!is_map_v<Q> && is_transparent_v<H, KE>, bool> = true> + auto emplace(K&& key) -> std::pair<iterator, bool> { + auto hash = mixed_hash(key); + auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); + auto bucket_idx = bucket_idx_from_hash(hash); + + while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) { + if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint && + m_equal(key, m_values[at(m_buckets, bucket_idx).m_value_idx])) { + // found it, return without ever actually creating anything + return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false}; + } + dist_and_fingerprint = dist_inc(dist_and_fingerprint); + bucket_idx = next(bucket_idx); + } + + // value is new, insert element first, so when exception happens we are in a valid state + return do_place_element(dist_and_fingerprint, bucket_idx, std::forward<K>(key)); + } + + template <class... Args> + auto emplace(Args&&... args) -> std::pair<iterator, bool> { + // we have to instantiate the value_type to be able to access the key. + // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop. + auto& key = get_key(m_values.emplace_back(std::forward<Args>(args)...)); + auto hash = mixed_hash(key); + auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash); + auto bucket_idx = bucket_idx_from_hash(hash); + + while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) { + if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint && + m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) { + m_values.pop_back(); // value was already there, so get rid of it + return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false}; + } + dist_and_fingerprint = dist_inc(dist_and_fingerprint); + bucket_idx = next(bucket_idx); + } + + // value is new, place the bucket and shift up until we find an empty spot + auto value_idx = static_cast<value_idx_type>(m_values.size() - 1); + if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) { + // increase_size just rehashes all the data we have in m_values + increase_size(); + } else { + // place element and shift up until we find an empty spot + place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx); + } + return {begin() + static_cast<difference_type>(value_idx), true}; + } + + template <class... Args> + auto emplace_hint(const_iterator /*hint*/, Args&&... args) -> iterator { + return emplace(std::forward<Args>(args)...).first; + } + + template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto try_emplace(Key const& key, Args&&... args) -> std::pair<iterator, bool> { + return do_try_emplace(key, std::forward<Args>(args)...); + } + + template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto try_emplace(Key&& key, Args&&... args) -> std::pair<iterator, bool> { + return do_try_emplace(std::move(key), std::forward<Args>(args)...); + } + + template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto try_emplace(const_iterator /*hint*/, Key const& key, Args&&... args) -> iterator { + return do_try_emplace(key, std::forward<Args>(args)...).first; + } + + template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto try_emplace(const_iterator /*hint*/, Key&& key, Args&&... args) -> iterator { + return do_try_emplace(std::move(key), std::forward<Args>(args)...).first; + } + + template < + typename K, + typename... Args, + typename Q = T, + typename H = Hash, + typename KE = KeyEqual, + std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>, + bool> = true> + auto try_emplace(K&& key, Args&&... args) -> std::pair<iterator, bool> { + return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...); + } + + template < + typename K, + typename... Args, + typename Q = T, + typename H = Hash, + typename KE = KeyEqual, + std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> && is_neither_convertible_v<K&&, iterator, const_iterator>, + bool> = true> + auto try_emplace(const_iterator /*hint*/, K&& key, Args&&... args) -> iterator { + return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...).first; + } + + auto erase(iterator it) -> iterator { + auto hash = mixed_hash(get_key(*it)); + auto bucket_idx = bucket_idx_from_hash(hash); + + auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin()); + while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) { + bucket_idx = next(bucket_idx); + } + + do_erase(bucket_idx, [](value_type&& /*unused*/) { + }); + return begin() + static_cast<difference_type>(value_idx_to_remove); + } + + auto extract(iterator it) -> value_type { + auto hash = mixed_hash(get_key(*it)); + auto bucket_idx = bucket_idx_from_hash(hash); + + auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin()); + while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) { + bucket_idx = next(bucket_idx); + } + + auto tmp = std::optional<value_type>{}; + do_erase(bucket_idx, [&tmp](value_type&& val) { + tmp = std::move(val); + }); + return std::move(tmp).value(); + } + + template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto erase(const_iterator it) -> iterator { + return erase(begin() + (it - cbegin())); + } + + template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto extract(const_iterator it) -> value_type { + return extract(begin() + (it - cbegin())); + } + + auto erase(const_iterator first, const_iterator last) -> iterator { + auto const idx_first = first - cbegin(); + auto const idx_last = last - cbegin(); + auto const first_to_last = std::distance(first, last); + auto const last_to_end = std::distance(last, cend()); + + // remove elements from left to right which moves elements from the end back + auto const mid = idx_first + (std::min)(first_to_last, last_to_end); + auto idx = idx_first; + while (idx != mid) { + erase(begin() + idx); + ++idx; + } + + // all elements from the right are moved, now remove the last element until all done + idx = idx_last; + while (idx != mid) { + --idx; + erase(begin() + idx); + } + + return begin() + idx_first; + } + + auto erase(Key const& key) -> size_t { + return do_erase_key(key, [](value_type&& /*unused*/) { + }); + } + + auto extract(Key const& key) -> std::optional<value_type> { + auto tmp = std::optional<value_type>{}; + do_erase_key(key, [&tmp](value_type&& val) { + tmp = std::move(val); + }); + return tmp; + } + + template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true> + auto erase(K&& key) -> size_t { + return do_erase_key(std::forward<K>(key), [](value_type&& /*unused*/) { + }); + } + + template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true> + auto extract(K&& key) -> std::optional<value_type> { + auto tmp = std::optional<value_type>{}; + do_erase_key(std::forward<K>(key), [&tmp](value_type&& val) { + tmp = std::move(val); + }); + return tmp; + } + + void swap(table& other) noexcept(noexcept(std::is_nothrow_swappable_v<value_container_type> && + std::is_nothrow_swappable_v<Hash> && std::is_nothrow_swappable_v<KeyEqual>)) { + using std::swap; + swap(other, *this); + } + + // lookup ///////////////////////////////////////////////////////////////// + + template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto at(key_type const& key) -> Q& { + return do_at(key); + } + + template <typename K, + typename Q = T, + typename H = Hash, + typename KE = KeyEqual, + std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true> + auto at(K const& key) -> Q& { + return do_at(key); + } + + template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto at(key_type const& key) const -> Q const& { + return do_at(key); + } + + template <typename K, + typename Q = T, + typename H = Hash, + typename KE = KeyEqual, + std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true> + auto at(K const& key) const -> Q const& { + return do_at(key); + } + + template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto operator[](Key const& key) -> Q& { + return try_emplace(key).first->second; + } + + template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true> + auto operator[](Key&& key) -> Q& { + return try_emplace(std::move(key)).first->second; + } + + template <typename K, + typename Q = T, + typename H = Hash, + typename KE = KeyEqual, + std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true> + auto operator[](K&& key) -> Q& { + return try_emplace(std::forward<K>(key)).first->second; + } + + auto count(Key const& key) const -> size_t { + return find(key) == end() ? 0 : 1; + } + + template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true> + auto count(K const& key) const -> size_t { + return find(key) == end() ? 0 : 1; + } + + auto find(Key const& key) -> iterator { + return do_find(key); + } + + auto find(Key const& key) const -> const_iterator { + return do_find(key); + } + + template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true> + auto find(K const& key) -> iterator { + return do_find(key); + } + + template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true> + auto find(K const& key) const -> const_iterator { + return do_find(key); + } + + auto contains(Key const& key) const -> bool { + return find(key) != end(); + } + + template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true> + auto contains(K const& key) const -> bool { + return find(key) != end(); + } + + auto equal_range(Key const& key) -> std::pair<iterator, iterator> { + auto it = do_find(key); + return {it, it == end() ? end() : it + 1}; + } + + auto equal_range(const Key& key) const -> std::pair<const_iterator, const_iterator> { + auto it = do_find(key); + return {it, it == end() ? end() : it + 1}; + } + + template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true> + auto equal_range(K const& key) -> std::pair<iterator, iterator> { + auto it = do_find(key); + return {it, it == end() ? end() : it + 1}; + } + + template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true> + auto equal_range(K const& key) const -> std::pair<const_iterator, const_iterator> { + auto it = do_find(key); + return {it, it == end() ? end() : it + 1}; + } + + // bucket interface /////////////////////////////////////////////////////// + + auto bucket_count() const noexcept -> size_t { // NOLINT(modernize-use-nodiscard) + return m_num_buckets; + } + + static constexpr auto max_bucket_count() noexcept -> size_t { // NOLINT(modernize-use-nodiscard) + return max_size(); + } + + // hash policy //////////////////////////////////////////////////////////// + + [[nodiscard]] auto load_factor() const -> float { + return bucket_count() ? static_cast<float>(size()) / static_cast<float>(bucket_count()) : 0.0F; + } + + [[nodiscard]] auto max_load_factor() const -> float { + return m_max_load_factor; + } + + void max_load_factor(float ml) { + m_max_load_factor = ml; + if (m_num_buckets != max_bucket_count()) { + m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(bucket_count()) * max_load_factor()); + } + } + + void rehash(size_t count) { + count = (std::min)(count, max_size()); + auto shifts = calc_shifts_for_size((std::max)(count, size())); + if (shifts != m_shifts) { + m_shifts = shifts; + deallocate_buckets(); + m_values.shrink_to_fit(); + allocate_buckets_from_shift(); + clear_and_fill_buckets_from_values(); + } + } + + void reserve(size_t capa) { + capa = (std::min)(capa, max_size()); + if constexpr (has_reserve<value_container_type>) { + // std::deque doesn't have reserve(). Make sure we only call when available + m_values.reserve(capa); + } + auto shifts = calc_shifts_for_size((std::max)(capa, size())); + if (0 == m_num_buckets || shifts < m_shifts) { + m_shifts = shifts; + deallocate_buckets(); + allocate_buckets_from_shift(); + clear_and_fill_buckets_from_values(); + } + } + + // observers ////////////////////////////////////////////////////////////// + + auto hash_function() const -> hasher { + return m_hash; + } + + auto key_eq() const -> key_equal { + return m_equal; + } + + // nonstandard API: expose the underlying values container + [[nodiscard]] auto values() const noexcept -> value_container_type const& { + return m_values; + } + + // non-member functions /////////////////////////////////////////////////// + + friend auto operator==(table const& a, table const& b) -> bool { + if (&a == &b) { + return true; + } + if (a.size() != b.size()) { + return false; + } + for (auto const& b_entry : b) { + auto it = a.find(get_key(b_entry)); + if constexpr (is_map_v<T>) { + // map: check that key is here, then also check that value is the same + if (a.end() == it || !(b_entry.second == it->second)) { + return false; + } + } else { + // set: only check that the key is here + if (a.end() == it) { + return false; + } + } + } + return true; + } + + friend auto operator!=(table const& a, table const& b) -> bool { + return !(a == b); + } +}; + +} // namespace detail + +ANKERL_UNORDERED_DENSE_EXPORT template <class Key, + class T, + class Hash = hash<Key>, + class KeyEqual = std::equal_to<Key>, + class AllocatorOrContainer = std::allocator<std::pair<Key, T>>, + class Bucket = bucket_type::standard> +using map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>; + +ANKERL_UNORDERED_DENSE_EXPORT template <class Key, + class T, + class Hash = hash<Key>, + class KeyEqual = std::equal_to<Key>, + class AllocatorOrContainer = std::allocator<std::pair<Key, T>>, + class Bucket = bucket_type::standard> +using segmented_map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>; + +ANKERL_UNORDERED_DENSE_EXPORT template <class Key, + class Hash = hash<Key>, + class KeyEqual = std::equal_to<Key>, + class AllocatorOrContainer = std::allocator<Key>, + class Bucket = bucket_type::standard> +using set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, false>; + +ANKERL_UNORDERED_DENSE_EXPORT template <class Key, + class Hash = hash<Key>, + class KeyEqual = std::equal_to<Key>, + class AllocatorOrContainer = std::allocator<Key>, + class Bucket = bucket_type::standard> +using segmented_set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket, true>; + +# if defined(ANKERL_UNORDERED_DENSE_PMR) + +namespace pmr { + +ANKERL_UNORDERED_DENSE_EXPORT template <class Key, + class T, + class Hash = hash<Key>, + class KeyEqual = std::equal_to<Key>, + class Bucket = bucket_type::standard> +using map = + detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, false>; + +ANKERL_UNORDERED_DENSE_EXPORT template <class Key, + class T, + class Hash = hash<Key>, + class KeyEqual = std::equal_to<Key>, + class Bucket = bucket_type::standard> +using segmented_map = + detail::table<Key, T, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<std::pair<Key, T>>, Bucket, true>; + +ANKERL_UNORDERED_DENSE_EXPORT template <class Key, + class Hash = hash<Key>, + class KeyEqual = std::equal_to<Key>, + class Bucket = bucket_type::standard> +using set = detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, false>; + +ANKERL_UNORDERED_DENSE_EXPORT template <class Key, + class Hash = hash<Key>, + class KeyEqual = std::equal_to<Key>, + class Bucket = bucket_type::standard> +using segmented_set = + detail::table<Key, void, Hash, KeyEqual, ANKERL_UNORDERED_DENSE_PMR::polymorphic_allocator<Key>, Bucket, true>; + +} // namespace pmr + +# endif + +// deduction guides /////////////////////////////////////////////////////////// + +// deduction guides for alias templates are only possible since C++20 +// see https://en.cppreference.com/w/cpp/language/class_template_argument_deduction + +} // namespace ANKERL_UNORDERED_DENSE_NAMESPACE +} // namespace ankerl::unordered_dense + +// std extensions ///////////////////////////////////////////////////////////// + +namespace std { // NOLINT(cert-dcl58-cpp) + +ANKERL_UNORDERED_DENSE_EXPORT template <class Key, + class T, + class Hash, + class KeyEqual, + class AllocatorOrContainer, + class Bucket, + class Pred, + bool IsSegmented> +// NOLINTNEXTLINE(cert-dcl58-cpp) +auto erase_if(ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>& map, + Pred pred) -> size_t { + using map_t = ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket, IsSegmented>; + + // going back to front because erase() invalidates the end iterator + auto const old_size = map.size(); + auto idx = old_size; + while (idx) { + --idx; + auto it = map.begin() + static_cast<typename map_t::difference_type>(idx); + if (pred(*it)) { + map.erase(it); + } + } + + return old_size - map.size(); +} + +} // namespace std + +#endif +#endif |