QC-1086 Replaying postprocessing with foreachobject trigger does not produce valid results

Framework/CMakeLists.txt

@@ -5,7 +5,7 @@ configure_file("include/QualityControl/Version.h.in"
                "${CMAKE_CURRENT_BINARY_DIR}/include/QualityControl/Version.h"
                @ONLY)
 
- # ---- Library for IL ----
+# ---- Library for IL ----
 add_library(O2QualityControlInfoLogger STATIC
   src/QcInfoLogger.cxx
 )
@@ -135,7 +135,8 @@ add_library(O2QualityControl
   src/RootFileStorage.cxx
   src/ReductorHelpers.cxx
   src/KafkaPoller.cxx
-  src/FlagHelpers.cxx)
+  src/FlagHelpers.cxx
+  src/ObjectMetadataHelpers.cxx)
 
 target_include_directories(
   O2QualityControl
@@ -263,7 +264,7 @@ endforeach()
 
 # ---- Tests ----
 
-add_executable(o2-qc-test-core 
+add_executable(o2-qc-test-core
                test/testActivity.cxx
                test/testActivityHelpers.cxx
                test/testAggregatorInterface.cxx
@@ -353,7 +354,7 @@ foreach(i RANGE ${count})
   get_filename_component(test_name ${test} NAME)
   string(REGEX REPLACE ".cxx" "" test_name ${test_name})
   string(REPLACE " " ";" arg "${arg}") # make list of string (arguments) out of
-                                       # one string
+  # one string
 
   add_executable(${test_name} ${test})
   set_property(TARGET ${test_name}

Framework/include/QualityControl/CcdbDatabase.h

@@ -76,9 +76,14 @@ class CcdbDatabase : public DatabaseInterface
                     const std::string& createdNotAfter = "", const std::string& createdNotBefore = "") override;
 
   // retrieval - MO - deprecated
-  std::shared_ptr<o2::quality_control::core::MonitorObject> retrieveMO(std::string objectPath, std::string objectName, long timestamp = Timestamp::Current, const core::Activity& activity = {}) override;
+  std::shared_ptr<o2::quality_control::core::MonitorObject> retrieveMO(std::string objectPath, std::string objectName,
+                                                                       long timestamp = Timestamp::Current,
+                                                                       const core::Activity& activity = {},
+                                                                       const std::map<std::string, std::string>& metadata = {}) override;
   // retrieval - QO - deprecated
-  std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = Timestamp::Current, const core::Activity& activity = {}) override;
+  std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = Timestamp::Current,
+                                                                       const core::Activity& activity = {},
+                                                                       const std::map<std::string, std::string>& metadata = {}) override;
 
   // retrieval - general
   std::string retrieveJson(std::string path, long timestamp, const std::map<std::string, std::string>& metadata) override;
@@ -91,10 +96,10 @@ class CcdbDatabase : public DatabaseInterface
   static long getCurrentTimestamp();
   static long getFutureTimestamp(int secondsInFuture);
   /**
-  * Return the listing of folder and/or objects in the subpath.
-  * @param subpath The folder we want to list the children of.
-  * @return The listing of folder and/or objects at the subpath.
-  */
+   * Return the listing of folder and/or objects in the subpath.
+   * @param subpath The folder we want to list the children of.
+   * @return The listing of folder and/or objects at the subpath.
+   */
   std::vector<std::string> getListing(const std::string& subpath = "");
 
   /**

Framework/include/QualityControl/DatabaseInterface.h

@@ -133,18 +133,24 @@ class DatabaseInterface
    * @param objectName Name of the object
    * @param timestamp Timestamp of the object in ms since epoch
    * @param activity Activity of the object
+   * @param metadata additional metadata to filter objects during retrieval
    * @deprecated
    */
-  virtual std::shared_ptr<o2::quality_control::core::MonitorObject> retrieveMO(std::string objectPath, std::string objectName, long timestamp = Timestamp::Current, const core::Activity& activity = {}) = 0;
+  virtual std::shared_ptr<o2::quality_control::core::MonitorObject> retrieveMO(std::string objectPath, std::string objectName,
+                                                                               long timestamp = Timestamp::Current, const core::Activity& activity = {},
+                                                                               const std::map<std::string, std::string>& metadata = {}) = 0;
   /**
    * \brief Look up a quality object and return it.
    * Look up a quality object and return it if found or nullptr if not.
    * @param qoPath Path of the object without the provenance prefix
    * @param timestamp Timestamp of the object in ms since epoch
    * @param activity Activity of the object
+   * @param metadata additional metadata to filter objects during retrieval
    * @deprecated
    */
-  virtual std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = Timestamp::Current, const core::Activity& activity = {}) = 0;
+  virtual std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string qoPath, long timestamp = Timestamp::Current,
+                                                                               const core::Activity& activity = {},
+                                                                               const std::map<std::string, std::string>& metadata = {}) = 0;
 
   /**
    * \brief Look up an object and return it.

Framework/include/QualityControl/DummyDatabase.h

@@ -35,10 +35,10 @@ class DummyDatabase : public DatabaseInterface
                 std::string const& detectorName, std::string const& taskName, long from = -1, long to = -1) override;
   // MonitorObject
   void storeMO(std::shared_ptr<const o2::quality_control::core::MonitorObject> q) override;
-  std::shared_ptr<o2::quality_control::core::MonitorObject> retrieveMO(std::string taskName, std::string objectName, long timestamp = -1, const core::Activity& activity = {}) override;
+  std::shared_ptr<o2::quality_control::core::MonitorObject> retrieveMO(std::string taskName, std::string objectName, long timestamp = -1, const core::Activity& activity = {}, const std::map<std::string, std::string>& metadata = {}) override;
   // QualityObject
   void storeQO(std::shared_ptr<const o2::quality_control::core::QualityObject> q) override;
-  std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string checkerName, long timestamp = -1, const core::Activity& activity = {}) override;
+  std::shared_ptr<o2::quality_control::core::QualityObject> retrieveQO(std::string checkerName, long timestamp = -1, const core::Activity& activity = {}, const std::map<std::string, std::string>& metadata = {}) override;
 
   // General
   void* retrieveAny(std::type_info const& tinfo, std::string const& path,

Framework/include/QualityControl/MonitorObject.h

@@ -18,6 +18,7 @@
 #define QC_CORE_MONITOROBJECT_H
 
 // std
+#include <optional>
 #include <string>
 #include <map>
 // ROOT
@@ -113,6 +114,8 @@ class MonitorObject : public TObject
   const std::map<std::string, std::string>& getMetadataMap() const;
   /// \brief Update the value of metadata or add it if it does not exist yet.
   void addOrUpdateMetadata(std::string key, std::string value);
+  /// \brief Get metadata value of given key, returns std::nullopt if none exists;
+  std::optional<std::string> getMetadata(const std::string& key);
 
   void Draw(Option_t* option) override;
   TObject* DrawClone(Option_t* option) const override;
@@ -146,7 +149,7 @@ class MonitorObject : public TObject
   void releaseObject();
   void cloneAndSetObject(const MonitorObject&);
 
-  ClassDefOverride(MonitorObject, 13);
+  ClassDefOverride(MonitorObject, 14);
 };
 
 } // namespace o2::quality_control::core

Framework/include/QualityControl/MonitorObjectCollection.h

@@ -40,15 +40,17 @@ class MonitorObjectCollection : public TObjArray, public mergers::MergeInterface
   void setTaskName(const std::string&);
   const std::string& getTaskName() const;
 
+  void addOrUpdateMetadata(const std::string& key, const std::string& value);
+
   MergeInterface* cloneMovingWindow() const override;
 
  private:
   std::string mDetector = "TST";
   std::string mTaskName = "Test";
 
-  ClassDefOverride(MonitorObjectCollection, 2);
+  ClassDefOverride(MonitorObjectCollection, 3);
 };
 
 } // namespace o2::quality_control::core
 
-#endif //QUALITYCONTROL_MONITOROBJECTCOLLECTION_H
+#endif // QUALITYCONTROL_MONITOROBJECTCOLLECTION_H

Framework/include/QualityControl/ObjectMetadataHelpers.h

@@ -0,0 +1,34 @@
+// Copyright 2025 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+///
+/// \file   ObjectMetadataHelpers.h
+/// \author Michal Tichak
+///
+
+#ifndef QUALITYCONTROL_OBJECTMETADATAHELPERS_H
+#define QUALITYCONTROL_OBJECTMETADATAHELPERS_H
+
+#include <optional>
+#include <string>
+
+namespace o2::quality_control::repository
+{
+/**
+ * \brief Parses metadata value stored under metadata_keys::cycle
+ * @param cycleStr string expecting unsigned number
+ * @return if parsing fails (eg. too big of a number, string wasn't a number) it returns nullopt
+ *
+ */
+std::optional<unsigned long> parseCycle(const std::string& cycleStr);
+} // namespace o2::quality_control::repository
+
+#endif

Framework/include/QualityControl/ObjectMetadataKeys.h

@@ -28,6 +28,7 @@ constexpr auto created = "Created";
 constexpr auto md5sum = "Content-MD5";
 constexpr auto objectType = "ObjectType";
 constexpr auto lastModified = "lastModified";
+
 // General QC framework
 constexpr auto qcVersion = "qc_version";
 constexpr auto qcDetectorCode = "qc_detector_name";
@@ -36,6 +37,8 @@ constexpr auto qcTaskClass = "qc_task_class";
 constexpr auto qcQuality = "qc_quality";
 constexpr auto qcCheckName = "qc_check_name";
 constexpr auto qcAdjustableEOV = "adjustableEOV"; // this is a keyword for the CCDB
+constexpr auto cycleNumber = "CycleNumber";
+
 // QC Activity
 constexpr auto runType = "RunType";
 constexpr auto runNumber = "RunNumber";

Framework/include/QualityControl/Quality.h

@@ -18,6 +18,7 @@
 #define QC_CORE_QUALITY_H
 
 #include <Rtypes.h>
+#include <optional>
 #include <string>
 #include <map>
 #include <vector>
@@ -105,6 +106,9 @@ class Quality
   /// \brief Get metadata
   /// \return the value corresponding to the key if it was found, default value otherwise
   std::string getMetadata(const std::string& key, const std::string& defaultValue) const;
+  /// \brief Get metadata
+  /// \return the value corresponding to the key if it was found, nulopt otherwise
+  std::optional<std::string> getMetadataOpt(const std::string&) const;
 
   /// \brief Associate the Quality with a new flag and an optional comment
   /// \return reference to *this
@@ -121,7 +125,7 @@ class Quality
   std::map<std::string, std::string> mUserMetadata;
   std::vector<std::pair<FlagType, std::string>> mFlags;
 
-  ClassDef(Quality, 2);
+  ClassDef(Quality, 3);
 };
 
 } // namespace o2::quality_control::core

Framework/include/QualityControl/QualityObject.h

@@ -14,6 +14,7 @@
 
 // std
 #include <map>
+#include <optional>
 #include <string>
 #include <vector>
 // ROOT
@@ -107,6 +108,9 @@ class QualityObject : public TObject
   /// \brief Get a metadata
   /// \return the value corresponding to the key if it was found, default value otherwise
   std::string getMetadata(std::string key, std::string defaultValue) const;
+  /// \brief Get a metadata
+  /// \return the value corresponding to the key if it was found, nullopt otherwise
+  std::optional<std::string> getMetadataOpt(const std::string& key) const;
 
   /// \brief Build the path to this object.
   /// Build the path to this object as it will appear in the GUI.
@@ -145,7 +149,7 @@ class QualityObject : public TObject
   std::vector<std::string> mMonitorObjectsNames;
   Activity mActivity;
 
-  ClassDefOverride(QualityObject, 6);
+  ClassDefOverride(QualityObject, 7);
 };
 
 using QualityObjectsType = std::vector<std::shared_ptr<QualityObject>>;

Framework/include/QualityControl/Triggers.h

@@ -21,6 +21,7 @@
 #include <functional>
 #include <iosfwd>
 #include <utility>
+#include <map>
 #include "QualityControl/Activity.h"
 
 namespace o2::quality_control::postprocessing
@@ -69,6 +70,7 @@ struct Trigger {
   core::Activity activity; // if tracking an object, it contains also its validity start and end
   uint64_t timestamp;      // if tracking an object, it is the validity start (validFrom)
   std::string config{};
+  std::map<std::string, std::string> metadata{}; // metadata to search in database
 };
 
 using TriggerFcn = std::function<Trigger()>;

Framework/src/Aggregator.cxx

@@ -16,8 +16,11 @@
 
 #include "QualityControl/Aggregator.h"
 #include "QualityControl/AggregatorSpec.h"
+#include "QualityControl/ObjectMetadataKeys.h"
+#include "QualityControl/QualityObject.h"
 #include "QualityControl/RootClassFactory.h"
 #include "QualityControl/AggregatorInterface.h"
+#include "QualityControl/ObjectMetadataHelpers.h"
 #include "QualityControl/UpdatePolicyType.h"
 #include "QualityControl/ActivityHelpers.h"
 #include "QualityControl/Activity.h"
@@ -106,6 +109,21 @@ QualityObjectsMapType Aggregator::filter(QualityObjectsMapType& qoMap)
   return result;
 }
 
+std::optional<unsigned long> getMaxCycle(const QualityObjectsMapType& qoMap)
+{
+  std::optional<unsigned long> max{};
+  for (const auto& [_, qo] : qoMap) {
+    auto cycle = qo->getMetadataOpt(repository::metadata_keys::cycleNumber);
+    if (cycle.has_value()) {
+      auto parsedCycle = repository::parseCycle(cycle.value());
+      if (parsedCycle) {
+        max = std::max(parsedCycle.value(), max.value_or(0));
+      }
+    }
+  }
+  return max;
+}
+
 QualityObjectsType Aggregator::aggregate(QualityObjectsMapType& qoMap, const Activity& defaultActivity)
 {
   auto filtered = filter(qoMap);
@@ -133,7 +151,8 @@ QualityObjectsType Aggregator::aggregate(QualityObjectsMapType& qoMap, const Act
     }
   }
 
-  auto results = mAggregatorInterface->aggregate(filtered);
+  const auto maxCycle = getMaxCycle(filtered);
+  const auto results = mAggregatorInterface->aggregate(filtered);
   QualityObjectsType qualityObjects;
   for (auto const& [qualityName, quality] : results) {
     qualityObjects.emplace_back(std::make_shared<QualityObject>(
@@ -142,6 +161,9 @@ QualityObjectsType Aggregator::aggregate(QualityObjectsMapType& qoMap, const Act
       mAggregatorConfig.detectorName,
       UpdatePolicyTypeUtils::ToString(mAggregatorConfig.policyType)));
     qualityObjects.back()->setActivity(resultActivity);
+    if (maxCycle.has_value()) {
+      qualityObjects.back()->addMetadata(repository::metadata_keys::cycleNumber, std::to_string(maxCycle.value()));
+    }
   }
   return qualityObjects;
 }

Framework/src/CcdbDatabase.cxx

@@ -283,6 +283,10 @@ TObject* CcdbDatabase::retrieveTObject(std::string path, std::map<std::string, s
   auto* object = ccdbApi->retrieveFromTFileAny<TObject>(path, metadata, timestamp, headers);
   if (object == nullptr) {
     ILOG(Warning, Support) << "We could NOT retrieve the object " << path << " with timestamp " << timestamp << "." << ENDM;
+    ILOG(Debug, Support) << "and with metadata:" << ENDM;
+    for (auto [metaKey, metaVal] : metadata) {
+      ILOG(Debug, Support) << metaKey << ", " << metaVal << ENDM;
+    }
     return nullptr;
   }
   ILOG(Debug, Support) << "Retrieved object " << path << " with timestamp " << timestamp << ENDM;
@@ -307,11 +311,14 @@ void* CcdbDatabase::retrieveAny(const type_info& tinfo, const string& path, cons
   return object;
 }
 
-std::shared_ptr<o2::quality_control::core::MonitorObject> CcdbDatabase::retrieveMO(std::string objectPath, std::string objectName, long timestamp, const core::Activity& activity)
+std::shared_ptr<o2::quality_control::core::MonitorObject> CcdbDatabase::retrieveMO(std::string objectPath, std::string objectName,
+                                                                                   long timestamp, const core::Activity& activity,
+                                                                                   const std::map<std::string, std::string>& metadataToRetrieve)
 {
   string fullPath = activity.mProvenance + "/" + objectPath + "/" + objectName;
   map<string, string> headers;
   map<string, string> metadata = activity_helpers::asDatabaseMetadata(activity, false);
+  metadata.insert(metadataToRetrieve.begin(), metadataToRetrieve.end());
   TObject* obj = retrieveTObject(fullPath, metadata, timestamp, &headers);
 
   // no object found
@@ -348,10 +355,13 @@ std::shared_ptr<o2::quality_control::core::MonitorObject> CcdbDatabase::retrieve
   return mo;
 }
 
-std::shared_ptr<o2::quality_control::core::QualityObject> CcdbDatabase::retrieveQO(std::string qoPath, long timestamp, const core::Activity& activity)
+std::shared_ptr<o2::quality_control::core::QualityObject> CcdbDatabase::retrieveQO(std::string qoPath, long timestamp,
+                                                                                   const core::Activity& activity,
+                                                                                   const std::map<std::string, std::string>& metadataToRetrieve)
 {
   map<string, string> headers;
   map<string, string> metadata = activity_helpers::asDatabaseMetadata(activity, false);
+  metadata.insert(metadataToRetrieve.begin(), metadataToRetrieve.end());
   auto fullPath = activity.mProvenance + "/" + qoPath;
   TObject* obj = retrieveTObject(fullPath, metadata, timestamp, &headers);
   if (obj == nullptr) {