From fddf6f3afba08835205f7f886108a6ce1016f23e Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 17 Nov 2025 14:50:02 +0300 Subject: [PATCH 01/34] sorting index tracks max series id --- pp/series_index/sorting_index.h | 19 +++++++---- pp/series_index/tests/sorting_index_tests.cpp | 32 +++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/pp/series_index/sorting_index.h b/pp/series_index/sorting_index.h index 6906e22ce..a5c704752 100644 --- a/pp/series_index/sorting_index.h +++ b/pp/series_index/sorting_index.h @@ -40,6 +40,7 @@ template class Vector, uint32_t kMaxIndexValue = st class SortingIndexBuilder { public: using Index = SortingIndex; + using IndexValueType = uint32_t; explicit SortingIndexBuilder(const Set& ls_id_set) : ls_id_set_(ls_id_set) {} @@ -57,10 +58,14 @@ class SortingIndexBuilder { return; } + const uint64_t ls_id = *ls_id_iterator; + max_id_ = std::max(max_id_, ls_id); + const uint64_t previous = get_previous(ls_id_iterator); const uint64_t next = get_next(ls_id_iterator); if (uint32_t value = (previous + next) / 2; value > previous) [[likely]] { - index_.index.emplace_back(value); + index_.index.resize(max_id_ + 1); + index_.index[ls_id] = value; } else { // If we can't insert item we don't need to rebuild index, because it's very expensive operation for CPU. // Index will be built on demand in sort method @@ -79,11 +84,13 @@ class SortingIndexBuilder { private: const Set& ls_id_set_; Index index_; + IndexValueType max_id_{0}; void rebuild() { - index_.index.resize(ls_id_set_.size()); + max_id_ = *std::max_element(ls_id_set_.begin(), ls_id_set_.end()); + index_.index.resize(max_id_ + 1); - const uint32_t step = kMaxIndexValue / (ls_id_set_.size() + 1); + const uint32_t step = kMaxIndexValue / (max_id_ + 2); uint32_t index_value = 0; for (auto ls_id : ls_id_set_) { index_value += step; @@ -93,15 +100,15 @@ class SortingIndexBuilder { PROMPP_ALWAYS_INLINE uint32_t get_previous(typename Set::const_iterator ls_id_iterator) const noexcept { if (ls_id_iterator != ls_id_set_.begin()) { - return index_.index[*--ls_id_iterator]; + return index_.index[*std::prev(ls_id_iterator)]; } return 0; } PROMPP_ALWAYS_INLINE uint32_t get_next(typename Set::const_iterator ls_id_iterator) const noexcept { - if (++ls_id_iterator != ls_id_set_.end()) { - return index_.index[*ls_id_iterator]; + if (const auto next_iterator = std::next(ls_id_iterator); next_iterator != ls_id_set_.end()) { + return index_.index[*next_iterator]; } return kMaxIndexValue; diff --git a/pp/series_index/tests/sorting_index_tests.cpp b/pp/series_index/tests/sorting_index_tests.cpp index 04be0d3fa..2c9892a63 100644 --- a/pp/series_index/tests/sorting_index_tests.cpp +++ b/pp/series_index/tests/sorting_index_tests.cpp @@ -101,6 +101,38 @@ TEST_F(SortingIndexFixture, BuildIndexInSort) { EXPECT_THAT(series_ids, testing::ElementsAre("a"_idx, "b"_idx, "c"_idx, "d"_idx)); } +TEST_F(SortingIndexFixture, BuildIndexOutOfOrderInSort) { + // Arrange + set_.emplace(2); + set_.emplace(3); + set_.emplace(0); + set_.emplace(1); + + std::array series_ids{"d"_idx, "c"_idx, "b"_idx, "a"_idx}; + + // Act + index_.sort(series_ids.begin(), series_ids.end()); + + // Assert + EXPECT_FALSE(index_.empty()); + EXPECT_THAT(series_ids, testing::ElementsAre("a"_idx, "b"_idx, "c"_idx, "d"_idx)); +} + +TEST_F(SortingIndexFixture, BuildIndexOutOfOrderWithSkipInSort) { + // Arrange + set_.emplace(3); + set_.emplace(0); + + std::array series_ids{"c"_idx, "b"_idx}; + + // Act + index_.sort(series_ids.begin(), series_ids.end()); + + // Assert + EXPECT_FALSE(index_.empty()); + EXPECT_THAT(series_ids, testing::ElementsAre("b"_idx, "c"_idx)); +} + TEST_F(SortingIndexFixture, IndexSnapshot) { // Arrange SortingIndexBuilder index{set_}; From eca36e788b1bdf5e476c755676b4cc6bf3e9b63d Mon Sep 17 00:00:00 2001 From: Gleb Shigin Date: Mon, 17 Nov 2025 17:17:35 +0300 Subject: [PATCH 02/34] after_items_load_impl now accepts range of ls_ids --- pp/bare_bones/snug_composite.h | 59 ++++++++++++++-------- pp/series_index/queryable_encoding_bimap.h | 9 ++-- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/pp/bare_bones/snug_composite.h b/pp/bare_bones/snug_composite.h index 1056b6515..07ea91b92 100644 --- a/pp/bare_bones/snug_composite.h +++ b/pp/bare_bones/snug_composite.h @@ -8,6 +8,8 @@ #include +#include + #include "bare_bones/allocator.h" #include "bare_bones/exception.h" #include "bare_bones/streams.h" @@ -24,6 +26,9 @@ namespace BareBones::SnugComposite { */ enum class SerializationMode : char { SNAPSHOT = 1, DELTA = 2 }; +template +concept ls_id_range = std::ranges::range && std::same_as, uint32_t>; + template concept is_shrinkable = requires(FilamentDataType& data_type) { { data_type.shrink_to(uint32_t()) }; @@ -39,10 +44,8 @@ concept has_rollback = requires(Derived derived, const Checkpoint& checkpoint) { { derived.rollback_impl(checkpoint) }; }; -template -concept has_after_items_load = requires(Derived derived) { - { derived.after_items_load_impl(uint32_t()) }; -}; +template +concept has_after_items_load = ls_id_range && requires(Derived derived, R&& range) { derived.after_items_load_impl(std::forward(range)); }; template class> class Filament, template class Vector> class GenericDecodingTable { @@ -381,9 +384,10 @@ class GenericDecodingTable { } // post processing - if constexpr (has_after_items_load) { - static_assert(noexcept(static_cast(this)->after_items_load_impl(original_size))); - static_cast(this)->after_items_load_impl(original_size); + if constexpr (has_after_items_load>) { + auto r = std::views::iota(original_size, items_.size()); + static_assert(noexcept(static_cast(this)->after_items_load_impl(r))); + static_cast(this)->after_items_load_impl(r); } } @@ -615,9 +619,12 @@ class ShrinkableEncodingBimap final : private GenericDecodingTable + PROMPP_ALWAYS_INLINE void after_items_load_impl(R&& loaded_ids) noexcept { + if constexpr (std::ranges::sized_range) { + set_.reserve(std::ranges::size(loaded_ids)); + } + for (const auto id : loaded_ids) { set_.emplace(typename Base::Proxy(id)); } } @@ -631,9 +638,12 @@ class EncodingBimap : public GenericDecodingTable set_{{}, 0, Base::hasher(), Base::equality_comparator()}; - PROMPP_ALWAYS_INLINE void after_items_load_impl(uint32_t first_loaded_id) noexcept { - set_.reserve(Base::items_.size()); - for (auto id = first_loaded_id; id != Base::items_.size(); ++id) { + template + PROMPP_ALWAYS_INLINE void after_items_load_impl(R&& loaded_ids) noexcept { + if constexpr (std::ranges::sized_range) { + set_.reserve(std::ranges::size(loaded_ids)); + } + for (const auto id : loaded_ids) { set_.emplace(typename Base::Proxy(id)); } } @@ -716,9 +726,12 @@ class ParallelEncodingBimap : public GenericDecodingTable set_; - PROMPP_ALWAYS_INLINE void after_items_load_impl(uint32_t first_loaded_id) noexcept { - set_.reserve(Base::items_.size()); - for (auto id = first_loaded_id; id != Base::items_.size(); ++id) { + template + PROMPP_ALWAYS_INLINE void after_items_load_impl(R&& loaded_ids) noexcept { + if constexpr (std::ranges::sized_range) { + set_.reserve(std::ranges::size(loaded_ids)); + } + for (const auto id : loaded_ids) { set_.emplace(typename Base::Proxy(id)); } } @@ -788,8 +801,9 @@ class OrderedEncodingBimap : public GenericDecodingTable + PROMPP_ALWAYS_INLINE void after_items_load_impl(R&& loaded_ids) noexcept { + for (const auto id : loaded_ids) { set_.emplace(typename Base::Proxy(id)); } } @@ -895,9 +909,12 @@ class EncodingBimapWithOrderedAccess : public GenericDecodingTable; Set set_; - PROMPP_ALWAYS_INLINE void after_items_load_impl(uint32_t first_loaded_id) noexcept { - set_.reserve(Base::items_.size()); - for (auto id = first_loaded_id; id != Base::items_.size(); ++id) { + template + PROMPP_ALWAYS_INLINE void after_items_load_impl(R&& loaded_ids) noexcept { + if constexpr (std::ranges::sized_range) { + set_.reserve(std::ranges::size(loaded_ids)); + } + for (const auto id : loaded_ids) { set_.emplace(typename Base::Proxy(id)); ordered_set_.emplace(typename Base::Proxy(id)); } diff --git a/pp/series_index/queryable_encoding_bimap.h b/pp/series_index/queryable_encoding_bimap.h index 1b265e320..5b368eccd 100644 --- a/pp/series_index/queryable_encoding_bimap.h +++ b/pp/series_index/queryable_encoding_bimap.h @@ -101,11 +101,14 @@ class QueryableEncodingBimap final : public BareBones::SnugComposite::GenericDec BareBones::Bitset added_series_; - PROMPP_ALWAYS_INLINE void after_items_load_impl(uint32_t first_loaded_id) noexcept { - ls_id_hash_set_.reserve(Base::items_.size()); + template + PROMPP_ALWAYS_INLINE void after_items_load_impl(R&& loaded_ids) noexcept { + if constexpr (std::ranges::sized_range) { + ls_id_hash_set_.reserve(std::ranges::size(loaded_ids)); + } const auto hasher = Base::hasher(); - for (auto ls_id = first_loaded_id; ls_id < Base::items_.size(); ++ls_id) { + for (const auto ls_id : loaded_ids) { auto label_set = this->operator[](ls_id); update_indexes(ls_id, label_set, phmap_hash(hasher(label_set))); } From c5237dd0293f76d7e5f6d4c6734e25f9838c17a3 Mon Sep 17 00:00:00 2001 From: Vladimir Pustovalov Date: Mon, 17 Nov 2025 18:14:12 +0300 Subject: [PATCH 03/34] used lazy_emplace_with_hash instead of find/emplace --- pp/series_index/benchmarks/BUILD | 10 +++ .../benchmarks/find_or_emplace_benchmark.cpp | 61 +++++++++++++++++++ pp/series_index/queryable_encoding_bimap.h | 20 +++--- .../tests/queryable_encoding_bimap_tests.cpp | 21 +++++++ 4 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 pp/series_index/benchmarks/find_or_emplace_benchmark.cpp diff --git a/pp/series_index/benchmarks/BUILD b/pp/series_index/benchmarks/BUILD index 2c63db31e..591f7ab2f 100644 --- a/pp/series_index/benchmarks/BUILD +++ b/pp/series_index/benchmarks/BUILD @@ -8,4 +8,14 @@ cc_binary( "//:series_index", "//:benchmark", ], +) + +cc_binary( + name = "find_or_emplace", + srcs = ["find_or_emplace_benchmark.cpp"], + malloc = "@jemalloc", + deps = [ + "//:series_index", + "//:benchmark", + ], ) \ No newline at end of file diff --git a/pp/series_index/benchmarks/find_or_emplace_benchmark.cpp b/pp/series_index/benchmarks/find_or_emplace_benchmark.cpp new file mode 100644 index 000000000..79c1fe3b7 --- /dev/null +++ b/pp/series_index/benchmarks/find_or_emplace_benchmark.cpp @@ -0,0 +1,61 @@ +#include + +#include "primitives/snug_composites.h" +#include "profiling/profiling.h" +#include "series_index/queryable_encoding_bimap.h" +#include "series_index/trie/cedarpp_tree.h" + +namespace { + +using QueryableEncodingBimap = + series_index::QueryableEncodingBimap; + +std::string_view get_lss_file() { + if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { + return context->operator[]("lss_file"); + } + + return {}; +} + +QueryableEncodingBimap& get_lss() { + static QueryableEncodingBimap lss; + if (lss.size() == 0) { + std::ifstream infile(get_lss_file().data(), std::ios_base::binary); + infile >> lss; + } + + return lss; +} + +void BenchmarkFindOrEmplaceWithEmplace(benchmark::State& state) { + ZoneScoped; + const auto& lss = get_lss(); + + for ([[maybe_unused]] auto _ : state) { + QueryableEncodingBimap lss2; + for (const auto& label_set : lss) { + lss2.find_or_emplace(label_set); + } + } +} + +void BenchmarkFindOrEmplaceWithFind(benchmark::State& state) { + ZoneScoped; + auto& lss = get_lss(); + + for ([[maybe_unused]] auto _ : state) { + for (const auto& label_set : lss) { + lss.find_or_emplace(label_set); + } + } +} + +double min_value(const std::vector& v) noexcept { + return *std::ranges::min_element(v); +} + +BENCHMARK(BenchmarkFindOrEmplaceWithEmplace)->ComputeStatistics("min", min_value); +BENCHMARK(BenchmarkFindOrEmplaceWithFind)->ComputeStatistics("min", min_value); + +} // namespace diff --git a/pp/series_index/queryable_encoding_bimap.h b/pp/series_index/queryable_encoding_bimap.h index 1b265e320..4346589a9 100644 --- a/pp/series_index/queryable_encoding_bimap.h +++ b/pp/series_index/queryable_encoding_bimap.h @@ -50,14 +50,14 @@ class QueryableEncodingBimap final : public BareBones::SnugComposite::GenericDec template PROMPP_ALWAYS_INLINE uint32_t find_or_emplace(const LabelSet& label_set, size_t hash) noexcept { hash = phmap_hash(hash); - if (auto it = ls_id_hash_set_.find(label_set, hash); it != ls_id_hash_set_.end()) { - mark_series_as_added(*it); - return *it; - } + const auto ls_id = *ls_id_hash_set_.lazy_emplace_with_hash(label_set, hash, [&](const auto& ctor) { + auto new_ls_id = Base::items_.size(); + ctor(typename Base::Proxy(new_ls_id)); + auto composite_label_set = Base::items_.emplace_back(Base::data_, label_set).composite(Base::data()); + update_indexes(new_ls_id, composite_label_set); + return new_ls_id; + }); - auto ls_id = Base::items_.size(); - auto composite_label_set = Base::items_.emplace_back(Base::data_, label_set).composite(Base::data()); - update_indexes(ls_id, composite_label_set, hash); mark_series_as_added(ls_id); return ls_id; } @@ -107,12 +107,12 @@ class QueryableEncodingBimap final : public BareBones::SnugComposite::GenericDec const auto hasher = Base::hasher(); for (auto ls_id = first_loaded_id; ls_id < Base::items_.size(); ++ls_id) { auto label_set = this->operator[](ls_id); - update_indexes(ls_id, label_set, phmap_hash(hasher(label_set))); + ls_id_hash_set_.emplace_with_hash(phmap_hash(hasher(label_set)), typename Base::Proxy(ls_id)); + update_indexes(ls_id, label_set); } } - void update_indexes(uint32_t ls_id, const LabelSet& label_set, size_t label_set_phmap_hash) { - ls_id_hash_set_.emplace_with_hash(label_set_phmap_hash, typename Base::Proxy(ls_id)); + void update_indexes(uint32_t ls_id, const LabelSet& label_set) { auto ls_id_set_iterator = ls_id_set_.emplace(ls_id).first; for (auto label = label_set.begin(); label != label_set.end(); ++label) { diff --git a/pp/series_index/tests/queryable_encoding_bimap_tests.cpp b/pp/series_index/tests/queryable_encoding_bimap_tests.cpp index 72f2e3910..c606e302c 100644 --- a/pp/series_index/tests/queryable_encoding_bimap_tests.cpp +++ b/pp/series_index/tests/queryable_encoding_bimap_tests.cpp @@ -119,6 +119,27 @@ TEST_F(QueryableEncodingBimapFixture, EmplaceDuplicatedLabelSet) { EXPECT_NE(ls_id1, ls_id2); } +TEST_F(QueryableEncodingBimapFixture, Load) { + // Arrange + const auto label_set1 = LabelViewSet{{"job", "cron"}, {"key", ""}, {"process", "php"}}; + const auto label_set2 = LabelViewSet{{"job", "cron"}, {"key", ""}, {"process", "php1"}}; + + const auto ls_id1 = lss_.find_or_emplace(label_set1); + const auto ls_id2 = lss_.find_or_emplace(label_set2); + + std::stringstream stream; + stream << lss_; + + Lss lss2; + + // Act + stream >> lss2; + + // Assert + EXPECT_EQ(ls_id1, lss2.find(label_set1)); + EXPECT_EQ(ls_id2, lss2.find(label_set2)); +} + class QueryableEncodingBimapCopierFixture : public QueryableEncodingBimapFixture { protected: BareBones::Vector dst_src_ids_mapping_; From 0594e0954321179967f8679228f00c54f44ac8ed Mon Sep 17 00:00:00 2001 From: Vladimir Pustovalov Date: Tue, 18 Nov 2025 13:12:45 +0300 Subject: [PATCH 04/34] removed ParallelEncodingBimap --- pp/bare_bones/snug_composite.h | 70 ------------------- pp/bare_bones/tests/snug_composite_tests.cpp | 1 - .../filling_snug_composites_from_stream.cpp | 7 -- pp/performance_tests/full_load_lss_test.cpp | 18 ----- pp/performance_tests/save_lss_to_wal_test.cpp | 2 +- pp/primitives/snug_composites.h | 9 --- 6 files changed, 1 insertion(+), 106 deletions(-) diff --git a/pp/bare_bones/snug_composite.h b/pp/bare_bones/snug_composite.h index 1056b6515..68707c8f4 100644 --- a/pp/bare_bones/snug_composite.h +++ b/pp/bare_bones/snug_composite.h @@ -708,76 +708,6 @@ class EncodingBimap : public GenericDecodingTable class> class Filament, template class Vector> -class ParallelEncodingBimap : public GenericDecodingTable, Filament, Vector> { - using Base = GenericDecodingTable; - - friend class GenericDecodingTable; - - phmap::parallel_flat_hash_set set_; - - PROMPP_ALWAYS_INLINE void after_items_load_impl(uint32_t first_loaded_id) noexcept { - set_.reserve(Base::items_.size()); - for (auto id = first_loaded_id; id != Base::items_.size(); ++id) { - set_.emplace(typename Base::Proxy(id)); - } - } - - public: - inline __attribute__((always_inline)) ParallelEncodingBimap() noexcept : set_({}, 0, Base::hasher(), Base::equality_comparator()) {} - - ParallelEncodingBimap(const ParallelEncodingBimap&) = delete; - ParallelEncodingBimap(ParallelEncodingBimap&&) = delete; - ParallelEncodingBimap& operator=(const ParallelEncodingBimap&) = delete; - ParallelEncodingBimap& operator=(ParallelEncodingBimap&&) = delete; - - template - inline __attribute__((always_inline)) uint32_t find_or_emplace(const Class& c) noexcept { - return *set_.lazy_emplace(c, [&](const auto& ctor) { - uint32_t id = Base::items_.size(); - Base::items_.emplace_back(Base::data_, c); - ctor(id); - }); - } - - template - inline __attribute__((always_inline)) uint32_t find_or_emplace(const Class& c, size_t hashval) noexcept { - return *set_.lazy_emplace_with_hash(c, phmap::phmap_mix()(hashval), [&](const auto& ctor) { - uint32_t id = Base::items_.size(); - Base::items_.emplace_back(Base::data_, c); - ctor(id); - }); - } - - template - inline __attribute__((always_inline)) std::optional find(const Class& c) const noexcept { - if (auto i = set_.find(c); i != set_.end()) { - return *i; - } - return {}; - } - - template - inline __attribute__((always_inline)) std::optional find(const Class& c, size_t hashval) const noexcept { - if (auto i = set_.find(c, phmap::phmap_mix()(hashval)); i != set_.end()) { - return *i; - } - return {}; - } - - inline __attribute__((always_inline)) void rollback_impl(const typename Base::checkpoint_type& s) noexcept - requires(!Base::kIsReadOnly) - { - assert(s.size() <= Base::items_.size()); - - for (uint32_t i = s.size(); i != Base::items_.size(); ++i) { - set_.erase(typename Base::Proxy(i)); - } - } - - // TODO wrap everything with read/write mutex, and test it! -}; - template