From 8447af811b6ec372ec2ee840eb566c3c593f4539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Tich=C3=A1k?= Date: Mon, 14 Jul 2025 15:06:23 +0200 Subject: [PATCH 1/2] QC-1086 added cycles to MOs and QOs --- .../include/QualityControl/CcdbDatabase.h | 17 ++++--- .../QualityControl/DatabaseInterface.h | 8 ++- .../include/QualityControl/DummyDatabase.h | 4 +- .../include/QualityControl/MonitorObject.h | 5 +- .../QualityControl/MonitorObjectCollection.h | 6 ++- .../QualityControl/ObjectMetadataKeys.h | 3 ++ Framework/include/QualityControl/Triggers.h | 8 +-- Framework/src/CcdbDatabase.cxx | 14 ++++- Framework/src/Check.cxx | 22 +++++++- Framework/src/DummyDatabase.cxx | 4 +- Framework/src/MonitorObject.cxx | 10 ++++ Framework/src/MonitorObjectCollection.cxx | 51 +++++++++++++++++++ Framework/src/ReductorHelpers.cxx | 6 +-- Framework/src/SliceTrendingTask.cxx | 4 +- Framework/src/TaskRunner.cxx | 3 ++ Framework/src/Triggers.cxx | 4 ++ Framework/test/testCcdbDatabase.cxx | 42 +++++++++++++++ 17 files changed, 185 insertions(+), 26 deletions(-) diff --git a/Framework/include/QualityControl/CcdbDatabase.h b/Framework/include/QualityControl/CcdbDatabase.h index 3d43704718..94d876d1de 100644 --- a/Framework/include/QualityControl/CcdbDatabase.h +++ b/Framework/include/QualityControl/CcdbDatabase.h @@ -76,9 +76,14 @@ class CcdbDatabase : public DatabaseInterface const std::string& createdNotAfter = "", const std::string& createdNotBefore = "") override; // retrieval - MO - deprecated - std::shared_ptr retrieveMO(std::string objectPath, std::string objectName, long timestamp = Timestamp::Current, const core::Activity& activity = {}) override; + std::shared_ptr retrieveMO(std::string objectPath, std::string objectName, + long timestamp = Timestamp::Current, + const core::Activity& activity = {}, + const std::map& metadata = {}) override; // retrieval - QO - deprecated - std::shared_ptr retrieveQO(std::string qoPath, long timestamp = Timestamp::Current, const core::Activity& activity = {}) override; + std::shared_ptr retrieveQO(std::string qoPath, long timestamp = Timestamp::Current, + const core::Activity& activity = {}, + const std::map& metadata = {}) override; // retrieval - general std::string retrieveJson(std::string path, long timestamp, const std::map& metadata) override; @@ -91,10 +96,10 @@ class CcdbDatabase : public DatabaseInterface static long getCurrentTimestamp(); static long getFutureTimestamp(int secondsInFuture); /** - * Return the listing of folder and/or objects in the subpath. - * @param subpath The folder we want to list the children of. - * @return The listing of folder and/or objects at the subpath. - */ + * Return the listing of folder and/or objects in the subpath. + * @param subpath The folder we want to list the children of. + * @return The listing of folder and/or objects at the subpath. + */ std::vector getListing(const std::string& subpath = ""); /** diff --git a/Framework/include/QualityControl/DatabaseInterface.h b/Framework/include/QualityControl/DatabaseInterface.h index de81ad2042..1a331a53a3 100644 --- a/Framework/include/QualityControl/DatabaseInterface.h +++ b/Framework/include/QualityControl/DatabaseInterface.h @@ -135,7 +135,9 @@ class DatabaseInterface * @param activity Activity of the object * @deprecated */ - virtual std::shared_ptr retrieveMO(std::string objectPath, std::string objectName, long timestamp = Timestamp::Current, const core::Activity& activity = {}) = 0; + virtual std::shared_ptr retrieveMO(std::string objectPath, std::string objectName, + long timestamp = Timestamp::Current, const core::Activity& activity = {}, + const std::map& metadata = {}) = 0; /** * \brief Look up a quality object and return it. * Look up a quality object and return it if found or nullptr if not. @@ -144,7 +146,9 @@ class DatabaseInterface * @param activity Activity of the object * @deprecated */ - virtual std::shared_ptr retrieveQO(std::string qoPath, long timestamp = Timestamp::Current, const core::Activity& activity = {}) = 0; + virtual std::shared_ptr retrieveQO(std::string qoPath, long timestamp = Timestamp::Current, + const core::Activity& activity = {}, + const std::map& metadata = {}) = 0; /** * \brief Look up an object and return it. diff --git a/Framework/include/QualityControl/DummyDatabase.h b/Framework/include/QualityControl/DummyDatabase.h index 86ffd33f94..7f1aacd61f 100644 --- a/Framework/include/QualityControl/DummyDatabase.h +++ b/Framework/include/QualityControl/DummyDatabase.h @@ -35,10 +35,10 @@ class DummyDatabase : public DatabaseInterface std::string const& detectorName, std::string const& taskName, long from = -1, long to = -1) override; // MonitorObject void storeMO(std::shared_ptr q) override; - std::shared_ptr retrieveMO(std::string taskName, std::string objectName, long timestamp = -1, const core::Activity& activity = {}) override; + std::shared_ptr retrieveMO(std::string taskName, std::string objectName, long timestamp = -1, const core::Activity& activity = {}, const std::map& metadata = {}) override; // QualityObject void storeQO(std::shared_ptr q) override; - std::shared_ptr retrieveQO(std::string checkerName, long timestamp = -1, const core::Activity& activity = {}) override; + std::shared_ptr retrieveQO(std::string checkerName, long timestamp = -1, const core::Activity& activity = {}, const std::map& metadata = {}) override; // General void* retrieveAny(std::type_info const& tinfo, std::string const& path, diff --git a/Framework/include/QualityControl/MonitorObject.h b/Framework/include/QualityControl/MonitorObject.h index 0f045e41a3..f4bbc7e36d 100644 --- a/Framework/include/QualityControl/MonitorObject.h +++ b/Framework/include/QualityControl/MonitorObject.h @@ -18,6 +18,7 @@ #define QC_CORE_MONITOROBJECT_H // std +#include #include #include // ROOT @@ -113,6 +114,8 @@ class MonitorObject : public TObject const std::map& getMetadataMap() const; /// \brief Update the value of metadata or add it if it does not exist yet. void addOrUpdateMetadata(std::string key, std::string value); + /// \brief Get metadata value of given key, returns std::nullopt if none exists; + std::optional getMetadata(const std::string& key); void Draw(Option_t* option) override; TObject* DrawClone(Option_t* option) const override; @@ -146,7 +149,7 @@ class MonitorObject : public TObject void releaseObject(); void cloneAndSetObject(const MonitorObject&); - ClassDefOverride(MonitorObject, 13); + ClassDefOverride(MonitorObject, 14); }; } // namespace o2::quality_control::core diff --git a/Framework/include/QualityControl/MonitorObjectCollection.h b/Framework/include/QualityControl/MonitorObjectCollection.h index 7f1db6e4a4..75a5928b1b 100644 --- a/Framework/include/QualityControl/MonitorObjectCollection.h +++ b/Framework/include/QualityControl/MonitorObjectCollection.h @@ -40,15 +40,17 @@ class MonitorObjectCollection : public TObjArray, public mergers::MergeInterface void setTaskName(const std::string&); const std::string& getTaskName() const; + void addOrUpdateMetadata(std::string key, std::string value); + MergeInterface* cloneMovingWindow() const override; private: std::string mDetector = "TST"; std::string mTaskName = "Test"; - ClassDefOverride(MonitorObjectCollection, 2); + ClassDefOverride(MonitorObjectCollection, 3); }; } // namespace o2::quality_control::core -#endif //QUALITYCONTROL_MONITOROBJECTCOLLECTION_H +#endif // QUALITYCONTROL_MONITOROBJECTCOLLECTION_H diff --git a/Framework/include/QualityControl/ObjectMetadataKeys.h b/Framework/include/QualityControl/ObjectMetadataKeys.h index 0dac4cb8d6..4a8e564368 100644 --- a/Framework/include/QualityControl/ObjectMetadataKeys.h +++ b/Framework/include/QualityControl/ObjectMetadataKeys.h @@ -28,6 +28,7 @@ constexpr auto created = "Created"; constexpr auto md5sum = "Content-MD5"; constexpr auto objectType = "ObjectType"; constexpr auto lastModified = "lastModified"; + // General QC framework constexpr auto qcVersion = "qc_version"; constexpr auto qcDetectorCode = "qc_detector_name"; @@ -36,6 +37,8 @@ constexpr auto qcTaskClass = "qc_task_class"; constexpr auto qcQuality = "qc_quality"; constexpr auto qcCheckName = "qc_check_name"; constexpr auto qcAdjustableEOV = "adjustableEOV"; // this is a keyword for the CCDB +constexpr auto cycle = "CycleNumber"; + // QC Activity constexpr auto runType = "RunType"; constexpr auto runNumber = "RunNumber"; diff --git a/Framework/include/QualityControl/Triggers.h b/Framework/include/QualityControl/Triggers.h index 4a0fad49ec..d1efcd59a2 100644 --- a/Framework/include/QualityControl/Triggers.h +++ b/Framework/include/QualityControl/Triggers.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "QualityControl/Activity.h" namespace o2::quality_control::postprocessing @@ -48,12 +49,12 @@ struct Trigger { /// \brief Constructor. Timestamp is generated from the time of construction. Trigger(TriggerType triggerType, bool last = false, core::Activity activity = {}) - : triggerType(triggerType), last(last), activity(std::move(activity)), timestamp(msSinceEpoch()){}; + : triggerType(triggerType), last(last), activity(std::move(activity)), timestamp(msSinceEpoch()) {}; /// \brief Constructor. Trigger(TriggerType triggerType, bool last, core::Activity activity, uint64_t timestamp, std::string config = {}) - : triggerType(triggerType), last(last), activity(std::move(activity)), timestamp(timestamp), config(std::move(config)){}; + : triggerType(triggerType), last(last), activity(std::move(activity)), timestamp(timestamp), config(std::move(config)) {}; /// \brief Constructor. - Trigger(TriggerType triggerType, bool last, uint64_t timestamp) : triggerType(triggerType), last(last), activity(), timestamp(timestamp){}; + Trigger(TriggerType triggerType, bool last, uint64_t timestamp) : triggerType(triggerType), last(last), activity(), timestamp(timestamp) {}; operator bool() const { return triggerType != TriggerType::No && triggerType != TriggerType::INVALID; } friend std::ostream& operator<<(std::ostream& out, const Trigger& t); @@ -69,6 +70,7 @@ struct Trigger { core::Activity activity; // if tracking an object, it contains also its validity start and end uint64_t timestamp; // if tracking an object, it is the validity start (validFrom) std::string config{}; + std::map metadata{}; // metadata to search in database }; using TriggerFcn = std::function; diff --git a/Framework/src/CcdbDatabase.cxx b/Framework/src/CcdbDatabase.cxx index eaaf35265b..3fd6a0fd5c 100644 --- a/Framework/src/CcdbDatabase.cxx +++ b/Framework/src/CcdbDatabase.cxx @@ -283,6 +283,10 @@ TObject* CcdbDatabase::retrieveTObject(std::string path, std::mapretrieveFromTFileAny(path, metadata, timestamp, headers); if (object == nullptr) { ILOG(Warning, Support) << "We could NOT retrieve the object " << path << " with timestamp " << timestamp << "." << ENDM; + ILOG(Debug, Support) << "and with metadata:" << ENDM; + for (auto [metaKey, metaVal] : metadata) { + ILOG(Debug, Support) << metaKey << ", " << metaVal << ENDM; + } return nullptr; } ILOG(Debug, Support) << "Retrieved object " << path << " with timestamp " << timestamp << ENDM; @@ -307,11 +311,14 @@ void* CcdbDatabase::retrieveAny(const type_info& tinfo, const string& path, cons return object; } -std::shared_ptr CcdbDatabase::retrieveMO(std::string objectPath, std::string objectName, long timestamp, const core::Activity& activity) +std::shared_ptr CcdbDatabase::retrieveMO(std::string objectPath, std::string objectName, + long timestamp, const core::Activity& activity, + const std::map& metadataToRetrieve) { string fullPath = activity.mProvenance + "/" + objectPath + "/" + objectName; map headers; map metadata = activity_helpers::asDatabaseMetadata(activity, false); + metadata.insert(metadataToRetrieve.begin(), metadataToRetrieve.end()); TObject* obj = retrieveTObject(fullPath, metadata, timestamp, &headers); // no object found @@ -348,10 +355,13 @@ std::shared_ptr CcdbDatabase::retrieve return mo; } -std::shared_ptr CcdbDatabase::retrieveQO(std::string qoPath, long timestamp, const core::Activity& activity) +std::shared_ptr CcdbDatabase::retrieveQO(std::string qoPath, long timestamp, + const core::Activity& activity, + const std::map& metadataToRetrieve) { map headers; map metadata = activity_helpers::asDatabaseMetadata(activity, false); + metadata.insert(metadataToRetrieve.begin(), metadataToRetrieve.end()); auto fullPath = activity.mProvenance + "/" + qoPath; TObject* obj = retrieveTObject(fullPath, metadata, timestamp, &headers); if (obj == nullptr) { diff --git a/Framework/src/Check.cxx b/Framework/src/Check.cxx index 1e07661c0e..a1a6c7748d 100644 --- a/Framework/src/Check.cxx +++ b/Framework/src/Check.cxx @@ -13,6 +13,8 @@ #include #include +#include +#include #include #include // O2 @@ -24,6 +26,7 @@ #include "QualityControl/CommonSpec.h" #include "QualityControl/InputUtils.h" #include "QualityControl/MonitorObject.h" +#include "QualityControl/ObjectMetadataKeys.h" #include "QualityControl/RootClassFactory.h" #include "QualityControl/QcInfoLogger.h" #include "QualityControl/Quality.h" @@ -163,7 +166,20 @@ QualityObjectsType Check::check(std::map monitorObjectsNames; - std::ranges::copy(moMapToCheck | std::views::keys, std::back_inserter(monitorObjectsNames)); + unsigned long maxCycle{}; + for (const auto& [moName, mo] : moMapToCheck) { + monitorObjectsNames.emplace_back(moName); + if (const auto cycle = mo->getMetadata(repository::metadata_keys::cycle)) { + const auto& cycleStr = cycle.value(); + unsigned long cycleVal{}; + if (const auto fromCharsRed = std::from_chars(cycleStr.c_str(), cycleStr.c_str() + cycleStr.size(), cycleVal); fromCharsRed.ec == std::errc()) { + maxCycle = std::max(cycleVal, maxCycle); + } else { + ILOG(Warning, Support) << "metadata " << repository::metadata_keys::cycle << " with value " << cycleStr << " couldn't be parsed for a reason: " + << std::make_error_code(fromCharsRed.ec).message() << ENDM; + } + } + } // todo: take metadata from somewhere qualityObjects.emplace_back(std::make_shared( quality, @@ -172,7 +188,11 @@ QualityObjectsType Check::check(std::mapsetActivity(commonActivity); + if (maxCycle > 0) { + qualityObjects.back()->addMetadata(repository::metadata_keys::cycle, std::to_string(maxCycle)); + } beautify(moMapToCheck, quality); } diff --git a/Framework/src/DummyDatabase.cxx b/Framework/src/DummyDatabase.cxx index 33a4ecc9b4..34ab6c23f4 100644 --- a/Framework/src/DummyDatabase.cxx +++ b/Framework/src/DummyDatabase.cxx @@ -38,7 +38,7 @@ void DummyDatabase::storeMO(std::shared_ptr DummyDatabase::retrieveMO(std::string, std::string, long, const core::Activity& activity) +std::shared_ptr DummyDatabase::retrieveMO(std::string, std::string, long, const core::Activity& activity, const std::map&) { return {}; } @@ -47,7 +47,7 @@ void DummyDatabase::storeQO(std::shared_ptr DummyDatabase::retrieveQO(std::string, long, const core::Activity& activity) +std::shared_ptr DummyDatabase::retrieveQO(std::string, long, const core::Activity& activity, const std::map& metadata) { return {}; } diff --git a/Framework/src/MonitorObject.cxx b/Framework/src/MonitorObject.cxx index bde7ceb7db..23af08fa1d 100644 --- a/Framework/src/MonitorObject.cxx +++ b/Framework/src/MonitorObject.cxx @@ -20,6 +20,8 @@ #include "QualityControl/QcInfoLogger.h" #include +#include +#include using namespace std; @@ -153,6 +155,14 @@ void MonitorObject::addOrUpdateMetadata(std::string key, std::string value) } } +std::optional MonitorObject::getMetadata(const std::string& key) +{ + if (const auto foundIt = mUserMetadata.find(key); foundIt != std::end(mUserMetadata)) { + return foundIt->second; + } + return std::nullopt; +} + std::string MonitorObject::getPath() const { return RepoPathUtils::getMoPath(this); diff --git a/Framework/src/MonitorObjectCollection.cxx b/Framework/src/MonitorObjectCollection.cxx index abfc4572be..8be93b70d1 100644 --- a/Framework/src/MonitorObjectCollection.cxx +++ b/Framework/src/MonitorObjectCollection.cxx @@ -16,16 +16,56 @@ #include "QualityControl/MonitorObjectCollection.h" #include "QualityControl/MonitorObject.h" +#include "QualityControl/ObjectMetadataKeys.h" #include "QualityControl/QcInfoLogger.h" #include #include +#include +#include +#include using namespace o2::mergers; namespace o2::quality_control::core { +std::optional parseCycle(const std::string& cycleStr) +{ + unsigned long cycleVal{}; + if (auto parse_res = std::from_chars(cycleStr.c_str(), cycleStr.c_str() + cycleStr.size(), cycleVal); parse_res.ec != std::errc{}) { + ILOG(Warning, Support) << "failed to decypher " << repository::metadata_keys::cycle << " metadata with value " << cycleStr + << ", with std::errc " << std::make_error_code(parse_res.ec).message() << ENDM; + return std::nullopt; + } + return cycleVal; +} + +void mergeCycles(MonitorObject* targetMO, MonitorObject* otherMO) +{ + const auto otherCycle = otherMO->getMetadata(repository::metadata_keys::cycle); + const auto targetCycle = targetMO->getMetadata(repository::metadata_keys::cycle); + if (otherCycle.has_value() && targetCycle.has_value()) { + const auto targetCycleParsed = parseCycle(targetCycle.value()); + const auto otherCycleParsed = parseCycle(otherCycle.value()); + + if (targetCycleParsed && otherCycleParsed) { + targetMO->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(std::max(targetCycleParsed.value(), otherCycleParsed.value()))); + return; + } + + if (targetCycleParsed.value()) { + targetMO->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(targetCycleParsed.value())); + return; + } + + if (otherCycleParsed.value()) { + otherMO->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(otherCycleParsed.value())); + return; + } + } +} + void MonitorObjectCollection::merge(mergers::MergeInterface* const other) { auto otherCollection = dynamic_cast(other); // reinterpret_cast maybe? @@ -60,6 +100,8 @@ void MonitorObjectCollection::merge(mergers::MergeInterface* const other) continue; } + mergeCycles(targetMO, otherMO); + if (!reportedMismatchingRunNumbers && otherMO->getActivity().mId < targetMO->getActivity().mId) { ILOG(Error, Ops) << "The run number of the input object '" << otherMO->GetName() << "' (" << otherMO->getActivity().mId << ") " @@ -137,6 +179,15 @@ const std::string& MonitorObjectCollection::getTaskName() const return mTaskName; } +void MonitorObjectCollection::addOrUpdateMetadata(std::string key, std::string value) +{ + for (auto obj : *this) { + if (auto mo = dynamic_cast(obj)) { + mo->addOrUpdateMetadata(key, value); + } + } +} + std::string formatDuration(uint64_t durationMs) { auto remainder = durationMs; diff --git a/Framework/src/ReductorHelpers.cxx b/Framework/src/ReductorHelpers.cxx index 15db5e093b..5ed4b09715 100644 --- a/Framework/src/ReductorHelpers.cxx +++ b/Framework/src/ReductorHelpers.cxx @@ -34,7 +34,7 @@ bool updateReductorImpl(Reductor* r, const Trigger& t, const std::string& path, } if (type == "repository") { - auto mo = qcdb.retrieveMO(path, name, t.timestamp, t.activity); + auto mo = qcdb.retrieveMO(path, name, t.timestamp, t.activity, t.metadata); TObject* obj = mo ? mo->getObject() : nullptr; auto reductorTObject = dynamic_cast(r); if (obj && reductorTObject) { @@ -42,7 +42,7 @@ bool updateReductorImpl(Reductor* r, const Trigger& t, const std::string& path, return true; } } else if (type == "repository-quality") { - auto qo = qcdb.retrieveQO(path + "/" + name, t.timestamp, t.activity); + auto qo = qcdb.retrieveQO(path + "/" + name, t.timestamp, t.activity, t.metadata); auto reductorTObject = dynamic_cast(r); if (qo && reductorTObject) { reductorTObject->update(qo.get()); @@ -59,4 +59,4 @@ bool updateReductorImpl(Reductor* r, const Trigger& t, const std::string& path, return false; } -} // namespace o2::quality_control::postprocessing::reductor_helpers::implementation \ No newline at end of file +} // namespace o2::quality_control::postprocessing::reductor_helpers::implementation diff --git a/Framework/src/SliceTrendingTask.cxx b/Framework/src/SliceTrendingTask.cxx index 16c28d6968..f65cdc1a25 100644 --- a/Framework/src/SliceTrendingTask.cxx +++ b/Framework/src/SliceTrendingTask.cxx @@ -45,7 +45,7 @@ void SliceTrendingTask::configure(const boost::property_tree::ptree& config) mConfig = SliceTrendingTaskConfig(getID(), config); } -void SliceTrendingTask::initialize(Trigger, framework::ServiceRegistryRef services) +void SliceTrendingTask::initialize(Trigger t, framework::ServiceRegistryRef services) { // removing leftovers from any previous runs mTrend.reset(); @@ -151,7 +151,7 @@ void SliceTrendingTask::trendValues(const Trigger& t, mNumberPads[dataSource.name] = 0; mSources[dataSource.name]->clear(); if (dataSource.type == "repository") { - auto mo = qcdb.retrieveMO(dataSource.path, dataSource.name, t.timestamp, t.activity); + auto mo = qcdb.retrieveMO(dataSource.path, dataSource.name, t.timestamp, t.activity, t.metadata); TObject* obj = mo ? mo->getObject() : nullptr; mAxisDivision[dataSource.name] = dataSource.axisDivision; diff --git a/Framework/src/TaskRunner.cxx b/Framework/src/TaskRunner.cxx index 122e09de58..386ff58619 100644 --- a/Framework/src/TaskRunner.cxx +++ b/Framework/src/TaskRunner.cxx @@ -37,6 +37,7 @@ #include #include +#include "QualityControl/ObjectMetadataKeys.h" #include "QualityControl/QcInfoLogger.h" #include "QualityControl/TaskFactory.h" #include "QualityControl/runnerUtils.h" @@ -523,6 +524,8 @@ int TaskRunner::publish(DataAllocator& outputs) // getNonOwningArray creates a TObjArray containing the monitoring objects, but not // owning them. The array is created by new and must be cleaned up by the caller std::unique_ptr array(mObjectsManager->getNonOwningArray()); + // TODO: this will send object with cycle == 0. should I send here cycle + 1? (same for QOs) + array->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(mCycleNumber)); int objectsPublished = array->GetEntries(); outputs.snapshot( diff --git a/Framework/src/Triggers.cxx b/Framework/src/Triggers.cxx index 0a0dea29c4..bf5e036151 100644 --- a/Framework/src/Triggers.cxx +++ b/Framework/src/Triggers.cxx @@ -293,7 +293,11 @@ TriggerFcn ForEachObject(const std::string& databaseUrl, const std::string& data auto currentActivity = activity_helpers::asActivity(*currentObject, activity.mProvenance); bool last = currentObject + 1 == filteredObjects->end(); Trigger trigger(TriggerType::ForEachObject, last, currentActivity, currentObject->get(timestampSortKey)); + if (auto cycle = currentObject->get_optional(metadata_keys::cycle); cycle.has_value()) { + trigger.metadata.emplace(metadata_keys::cycle, cycle.value()); + } ++currentObject; + return trigger; } else { return { TriggerType::No, true, activity, Trigger::msSinceEpoch(), config }; diff --git a/Framework/test/testCcdbDatabase.cxx b/Framework/test/testCcdbDatabase.cxx index 8042084d1a..229765242c 100644 --- a/Framework/test/testCcdbDatabase.cxx +++ b/Framework/test/testCcdbDatabase.cxx @@ -15,6 +15,7 @@ /// \author Barthelemy von Haller /// +#include "QualityControl/Activity.h" #include "QualityControl/CcdbDatabase.h" #include "QualityControl/QcInfoLogger.h" #include "QualityControl/Version.h" @@ -24,6 +25,7 @@ #define BOOST_TEST_DYN_LINK #include +#include #include #include "QualityControl/RepoPathUtils.h" #include "QualityControl/ObjectMetadataKeys.h" @@ -129,6 +131,18 @@ BOOST_AUTO_TEST_CASE(ccdb_store) shared_ptr mo4 = make_shared(h4, f.taskName, "TestClass", "TST"); mo4->updateActivity(1234, "LHC66", "passName1", "qc_hello"); + TH1F* h5 = new TH1F("cycle", "asdf", 100, 0, 99); + shared_ptr mo5 = make_shared(h5, f.taskName, "TestClass", "TST"); + mo5->addMetadata(metadata_keys::cycle, "1"); + mo5->setValidity({ 10000, 20000 }); + mo5->updateActivity(1234, "LHC66", "passName1", "qc"); + + TH1F* h6 = new TH1F("cycle", "asdf", 100, 0, 99); + shared_ptr mo6 = make_shared(h6, f.taskName, "TestClass", "TST"); + mo6->addMetadata(metadata_keys::cycle, "2"); + mo6->setValidity({ 10000, 20000 }); + mo6->updateActivity(1234, "LHC66", "passName1", "qc"); + shared_ptr qo1 = make_shared(Quality::Bad, f.taskName + "/test-ccdb-check", "TST", "OnAll", vector{ string("input1"), string("input2") }); qo1->updateActivity(1234, "LHC66", "passName1", "qc"); shared_ptr qo2 = make_shared(Quality::Null, f.taskName + "/metadata", "TST", "OnAll", vector{ string("input1") }); @@ -141,6 +155,9 @@ BOOST_AUTO_TEST_CASE(ccdb_store) f.backend->storeMO(mo1); f.backend->storeMO(mo2); f.backend->storeMO(mo4); + f.backend->storeMO(mo5); + f.backend->storeMO(mo6); + f.backend->storeQO(qo1); f.backend->storeQO(qo2); f.backend->storeQO(qo4); @@ -210,6 +227,23 @@ BOOST_AUTO_TEST_CASE(ccdb_retrieve_inexisting_mo) BOOST_CHECK(mo == nullptr); } +BOOST_AUTO_TEST_CASE(ccdb_retrieve_mo_with_cycle, *utf::depends_on("ccdb_store")) +{ + test_fixture f; + std::shared_ptr mo{}; + mo = f.backend->retrieveMO(f.getMoFolder("cycle"), "cycle", + 15000, Activity{}, { { metadata_keys::cycle, "1" } }); + BOOST_REQUIRE(mo.get() != nullptr); + BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycle)); + BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycle) == "1"); + + mo = f.backend->retrieveMO(f.getMoFolder("cycle"), "cycle", + 15000, Activity{}, { { metadata_keys::cycle, "2" } }); + BOOST_REQUIRE(mo.get() != nullptr); + BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycle)); + BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycle) == "2"); +} + BOOST_AUTO_TEST_CASE(ccdb_retrieve_qo, *utf::depends_on("ccdb_store")) { test_fixture f; @@ -221,6 +255,14 @@ BOOST_AUTO_TEST_CASE(ccdb_retrieve_qo, *utf::depends_on("ccdb_store")) BOOST_CHECK_EQUAL(qo->getActivity().mPeriodName, "LHC66"); BOOST_CHECK_EQUAL(qo->getActivity().mPassName, "passName1"); BOOST_CHECK_EQUAL(qo->getActivity().mProvenance, "qc"); + + qo = f.backend->retrieveQO(RepoPathUtils::getQoPath("TST", f.taskName + "/metadata", "", {}, "", false), repository::CcdbDatabase::Timestamp::Current, {}, { { "my_meta", "is_good" } }); + BOOST_REQUIRE_NE(qo, nullptr); + BOOST_REQUIRE_NO_THROW(qo->getMetadata("my_meta")); + BOOST_CHECK_EQUAL(qo->getMetadata("my_meta"), "is_good"); + + qo = f.backend->retrieveQO(RepoPathUtils::getQoPath("TST", f.taskName + "/metadata", "", {}, "", false), repository::CcdbDatabase::Timestamp::Current, {}, { { "my_meta", "nonexistent" } }); + BOOST_REQUIRE_EQUAL(qo, nullptr); } BOOST_AUTO_TEST_CASE(ccdb_provenance, *utf::depends_on("ccdb_store")) From 22045a9ff74b99034753ab563d95ffedadb1a2fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Tich=C3=A1k?= Date: Thu, 17 Jul 2025 15:55:57 +0200 Subject: [PATCH 2/2] QC-1086 added cycle handling to Aggregators --- Framework/CMakeLists.txt | 9 +++-- .../QualityControl/DatabaseInterface.h | 2 + .../QualityControl/MonitorObjectCollection.h | 2 +- .../QualityControl/ObjectMetadataHelpers.h | 34 ++++++++++++++++ .../QualityControl/ObjectMetadataKeys.h | 2 +- Framework/include/QualityControl/Quality.h | 6 ++- .../include/QualityControl/QualityObject.h | 6 ++- Framework/include/QualityControl/Triggers.h | 6 +-- Framework/src/Aggregator.cxx | 24 +++++++++++- Framework/src/Check.cxx | 19 ++++----- Framework/src/MonitorObject.cxx | 1 - Framework/src/MonitorObjectCollection.cxx | 29 +++++--------- Framework/src/ObjectMetadataHelpers.cxx | 35 +++++++++++++++++ Framework/src/Quality.cxx | 11 +++++- Framework/src/QualityObject.cxx | 7 +++- Framework/src/ReductorHelpers.cxx | 1 + Framework/src/TaskRunner.cxx | 3 +- Framework/src/TriggerHelpers.cxx | 1 + Framework/src/Triggers.cxx | 4 +- Framework/test/testAggregatorRunner.cxx | 25 +++++++++++- Framework/test/testCcdbDatabase.cxx | 16 ++++---- .../test/testMonitorObjectCollection.cxx | 32 +++++++++++++++ Framework/test/testTriggers.cxx | 39 +++++++++++++++---- doc/Framework.md | 1 + doc/PostProcessing.md | 6 +-- 25 files changed, 251 insertions(+), 70 deletions(-) create mode 100644 Framework/include/QualityControl/ObjectMetadataHelpers.h create mode 100644 Framework/src/ObjectMetadataHelpers.cxx diff --git a/Framework/CMakeLists.txt b/Framework/CMakeLists.txt index 9b9961b5d6..8150becbff 100644 --- a/Framework/CMakeLists.txt +++ b/Framework/CMakeLists.txt @@ -5,7 +5,7 @@ configure_file("include/QualityControl/Version.h.in" "${CMAKE_CURRENT_BINARY_DIR}/include/QualityControl/Version.h" @ONLY) - # ---- Library for IL ---- +# ---- Library for IL ---- add_library(O2QualityControlInfoLogger STATIC src/QcInfoLogger.cxx ) @@ -135,7 +135,8 @@ add_library(O2QualityControl src/RootFileStorage.cxx src/ReductorHelpers.cxx src/KafkaPoller.cxx - src/FlagHelpers.cxx) + src/FlagHelpers.cxx + src/ObjectMetadataHelpers.cxx) target_include_directories( O2QualityControl @@ -263,7 +264,7 @@ endforeach() # ---- Tests ---- -add_executable(o2-qc-test-core +add_executable(o2-qc-test-core test/testActivity.cxx test/testActivityHelpers.cxx test/testAggregatorInterface.cxx @@ -353,7 +354,7 @@ foreach(i RANGE ${count}) get_filename_component(test_name ${test} NAME) string(REGEX REPLACE ".cxx" "" test_name ${test_name}) string(REPLACE " " ";" arg "${arg}") # make list of string (arguments) out of - # one string + # one string add_executable(${test_name} ${test}) set_property(TARGET ${test_name} diff --git a/Framework/include/QualityControl/DatabaseInterface.h b/Framework/include/QualityControl/DatabaseInterface.h index 1a331a53a3..3bc55b11ac 100644 --- a/Framework/include/QualityControl/DatabaseInterface.h +++ b/Framework/include/QualityControl/DatabaseInterface.h @@ -133,6 +133,7 @@ class DatabaseInterface * @param objectName Name of the object * @param timestamp Timestamp of the object in ms since epoch * @param activity Activity of the object + * @param metadata additional metadata to filter objects during retrieval * @deprecated */ virtual std::shared_ptr retrieveMO(std::string objectPath, std::string objectName, @@ -144,6 +145,7 @@ class DatabaseInterface * @param qoPath Path of the object without the provenance prefix * @param timestamp Timestamp of the object in ms since epoch * @param activity Activity of the object + * @param metadata additional metadata to filter objects during retrieval * @deprecated */ virtual std::shared_ptr retrieveQO(std::string qoPath, long timestamp = Timestamp::Current, diff --git a/Framework/include/QualityControl/MonitorObjectCollection.h b/Framework/include/QualityControl/MonitorObjectCollection.h index 75a5928b1b..2d41536d53 100644 --- a/Framework/include/QualityControl/MonitorObjectCollection.h +++ b/Framework/include/QualityControl/MonitorObjectCollection.h @@ -40,7 +40,7 @@ class MonitorObjectCollection : public TObjArray, public mergers::MergeInterface void setTaskName(const std::string&); const std::string& getTaskName() const; - void addOrUpdateMetadata(std::string key, std::string value); + void addOrUpdateMetadata(const std::string& key, const std::string& value); MergeInterface* cloneMovingWindow() const override; diff --git a/Framework/include/QualityControl/ObjectMetadataHelpers.h b/Framework/include/QualityControl/ObjectMetadataHelpers.h new file mode 100644 index 0000000000..0542ab3844 --- /dev/null +++ b/Framework/include/QualityControl/ObjectMetadataHelpers.h @@ -0,0 +1,34 @@ +// Copyright 2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file ObjectMetadataHelpers.h +/// \author Michal Tichak +/// + +#ifndef QUALITYCONTROL_OBJECTMETADATAHELPERS_H +#define QUALITYCONTROL_OBJECTMETADATAHELPERS_H + +#include +#include + +namespace o2::quality_control::repository +{ +/** + * \brief Parses metadata value stored under metadata_keys::cycle + * @param cycleStr string expecting unsigned number + * @return if parsing fails (eg. too big of a number, string wasn't a number) it returns nullopt + * + */ +std::optional parseCycle(const std::string& cycleStr); +} // namespace o2::quality_control::repository + +#endif diff --git a/Framework/include/QualityControl/ObjectMetadataKeys.h b/Framework/include/QualityControl/ObjectMetadataKeys.h index 4a8e564368..a111b6711b 100644 --- a/Framework/include/QualityControl/ObjectMetadataKeys.h +++ b/Framework/include/QualityControl/ObjectMetadataKeys.h @@ -37,7 +37,7 @@ constexpr auto qcTaskClass = "qc_task_class"; constexpr auto qcQuality = "qc_quality"; constexpr auto qcCheckName = "qc_check_name"; constexpr auto qcAdjustableEOV = "adjustableEOV"; // this is a keyword for the CCDB -constexpr auto cycle = "CycleNumber"; +constexpr auto cycleNumber = "CycleNumber"; // QC Activity constexpr auto runType = "RunType"; diff --git a/Framework/include/QualityControl/Quality.h b/Framework/include/QualityControl/Quality.h index cac8c85632..55e0a971f2 100644 --- a/Framework/include/QualityControl/Quality.h +++ b/Framework/include/QualityControl/Quality.h @@ -18,6 +18,7 @@ #define QC_CORE_QUALITY_H #include +#include #include #include #include @@ -105,6 +106,9 @@ class Quality /// \brief Get metadata /// \return the value corresponding to the key if it was found, default value otherwise std::string getMetadata(const std::string& key, const std::string& defaultValue) const; + /// \brief Get metadata + /// \return the value corresponding to the key if it was found, nulopt otherwise + std::optional getMetadataOpt(const std::string&) const; /// \brief Associate the Quality with a new flag and an optional comment /// \return reference to *this @@ -121,7 +125,7 @@ class Quality std::map mUserMetadata; std::vector> mFlags; - ClassDef(Quality, 2); + ClassDef(Quality, 3); }; } // namespace o2::quality_control::core diff --git a/Framework/include/QualityControl/QualityObject.h b/Framework/include/QualityControl/QualityObject.h index 6e7fb30a06..4790e796fb 100644 --- a/Framework/include/QualityControl/QualityObject.h +++ b/Framework/include/QualityControl/QualityObject.h @@ -14,6 +14,7 @@ // std #include +#include #include #include // ROOT @@ -107,6 +108,9 @@ class QualityObject : public TObject /// \brief Get a metadata /// \return the value corresponding to the key if it was found, default value otherwise std::string getMetadata(std::string key, std::string defaultValue) const; + /// \brief Get a metadata + /// \return the value corresponding to the key if it was found, nullopt otherwise + std::optional getMetadataOpt(const std::string& key) const; /// \brief Build the path to this object. /// Build the path to this object as it will appear in the GUI. @@ -145,7 +149,7 @@ class QualityObject : public TObject std::vector mMonitorObjectsNames; Activity mActivity; - ClassDefOverride(QualityObject, 6); + ClassDefOverride(QualityObject, 7); }; using QualityObjectsType = std::vector>; diff --git a/Framework/include/QualityControl/Triggers.h b/Framework/include/QualityControl/Triggers.h index d1efcd59a2..cedeb8f662 100644 --- a/Framework/include/QualityControl/Triggers.h +++ b/Framework/include/QualityControl/Triggers.h @@ -49,12 +49,12 @@ struct Trigger { /// \brief Constructor. Timestamp is generated from the time of construction. Trigger(TriggerType triggerType, bool last = false, core::Activity activity = {}) - : triggerType(triggerType), last(last), activity(std::move(activity)), timestamp(msSinceEpoch()) {}; + : triggerType(triggerType), last(last), activity(std::move(activity)), timestamp(msSinceEpoch()){}; /// \brief Constructor. Trigger(TriggerType triggerType, bool last, core::Activity activity, uint64_t timestamp, std::string config = {}) - : triggerType(triggerType), last(last), activity(std::move(activity)), timestamp(timestamp), config(std::move(config)) {}; + : triggerType(triggerType), last(last), activity(std::move(activity)), timestamp(timestamp), config(std::move(config)){}; /// \brief Constructor. - Trigger(TriggerType triggerType, bool last, uint64_t timestamp) : triggerType(triggerType), last(last), activity(), timestamp(timestamp) {}; + Trigger(TriggerType triggerType, bool last, uint64_t timestamp) : triggerType(triggerType), last(last), activity(), timestamp(timestamp){}; operator bool() const { return triggerType != TriggerType::No && triggerType != TriggerType::INVALID; } friend std::ostream& operator<<(std::ostream& out, const Trigger& t); diff --git a/Framework/src/Aggregator.cxx b/Framework/src/Aggregator.cxx index ac76398dfd..57d20cc7b4 100644 --- a/Framework/src/Aggregator.cxx +++ b/Framework/src/Aggregator.cxx @@ -16,8 +16,11 @@ #include "QualityControl/Aggregator.h" #include "QualityControl/AggregatorSpec.h" +#include "QualityControl/ObjectMetadataKeys.h" +#include "QualityControl/QualityObject.h" #include "QualityControl/RootClassFactory.h" #include "QualityControl/AggregatorInterface.h" +#include "QualityControl/ObjectMetadataHelpers.h" #include "QualityControl/UpdatePolicyType.h" #include "QualityControl/ActivityHelpers.h" #include "QualityControl/Activity.h" @@ -106,6 +109,21 @@ QualityObjectsMapType Aggregator::filter(QualityObjectsMapType& qoMap) return result; } +std::optional getMaxCycle(const QualityObjectsMapType& qoMap) +{ + std::optional max{}; + for (const auto& [_, qo] : qoMap) { + auto cycle = qo->getMetadataOpt(repository::metadata_keys::cycleNumber); + if (cycle.has_value()) { + auto parsedCycle = repository::parseCycle(cycle.value()); + if (parsedCycle) { + max = std::max(parsedCycle.value(), max.value_or(0)); + } + } + } + return max; +} + QualityObjectsType Aggregator::aggregate(QualityObjectsMapType& qoMap, const Activity& defaultActivity) { auto filtered = filter(qoMap); @@ -133,7 +151,8 @@ QualityObjectsType Aggregator::aggregate(QualityObjectsMapType& qoMap, const Act } } - auto results = mAggregatorInterface->aggregate(filtered); + const auto maxCycle = getMaxCycle(filtered); + const auto results = mAggregatorInterface->aggregate(filtered); QualityObjectsType qualityObjects; for (auto const& [qualityName, quality] : results) { qualityObjects.emplace_back(std::make_shared( @@ -142,6 +161,9 @@ QualityObjectsType Aggregator::aggregate(QualityObjectsMapType& qoMap, const Act mAggregatorConfig.detectorName, UpdatePolicyTypeUtils::ToString(mAggregatorConfig.policyType))); qualityObjects.back()->setActivity(resultActivity); + if (maxCycle.has_value()) { + qualityObjects.back()->addMetadata(repository::metadata_keys::cycleNumber, std::to_string(maxCycle.value())); + } } return qualityObjects; } diff --git a/Framework/src/Check.cxx b/Framework/src/Check.cxx index a1a6c7748d..9c0859cb6f 100644 --- a/Framework/src/Check.cxx +++ b/Framework/src/Check.cxx @@ -13,7 +13,6 @@ #include #include -#include #include #include #include @@ -31,7 +30,7 @@ #include "QualityControl/QcInfoLogger.h" #include "QualityControl/Quality.h" #include "QualityControl/HashDataDescription.h" -#include "QualityControl/runnerUtils.h" +#include "QualityControl/ObjectMetadataHelpers.h" #include @@ -166,17 +165,13 @@ QualityObjectsType Check::check(std::map monitorObjectsNames; - unsigned long maxCycle{}; + std::optional maxCycle{}; for (const auto& [moName, mo] : moMapToCheck) { monitorObjectsNames.emplace_back(moName); - if (const auto cycle = mo->getMetadata(repository::metadata_keys::cycle)) { + if (const auto cycle = mo->getMetadata(repository::metadata_keys::cycleNumber)) { const auto& cycleStr = cycle.value(); - unsigned long cycleVal{}; - if (const auto fromCharsRed = std::from_chars(cycleStr.c_str(), cycleStr.c_str() + cycleStr.size(), cycleVal); fromCharsRed.ec == std::errc()) { - maxCycle = std::max(cycleVal, maxCycle); - } else { - ILOG(Warning, Support) << "metadata " << repository::metadata_keys::cycle << " with value " << cycleStr << " couldn't be parsed for a reason: " - << std::make_error_code(fromCharsRed.ec).message() << ENDM; + if (const auto cycleVal = repository::parseCycle(cycleStr); cycleVal.has_value()) { + maxCycle = std::max(cycleVal.value(), maxCycle.value_or(0)); } } } @@ -190,8 +185,8 @@ QualityObjectsType Check::check(std::mapsetActivity(commonActivity); - if (maxCycle > 0) { - qualityObjects.back()->addMetadata(repository::metadata_keys::cycle, std::to_string(maxCycle)); + if (maxCycle.has_value()) { + qualityObjects.back()->addMetadata(repository::metadata_keys::cycleNumber, std::to_string(maxCycle.value())); } beautify(moMapToCheck, quality); } diff --git a/Framework/src/MonitorObject.cxx b/Framework/src/MonitorObject.cxx index 23af08fa1d..077b7f7807 100644 --- a/Framework/src/MonitorObject.cxx +++ b/Framework/src/MonitorObject.cxx @@ -19,7 +19,6 @@ #include "QualityControl/RepoPathUtils.h" #include "QualityControl/QcInfoLogger.h" -#include #include #include diff --git a/Framework/src/MonitorObjectCollection.cxx b/Framework/src/MonitorObjectCollection.cxx index 8be93b70d1..1f21fcd86a 100644 --- a/Framework/src/MonitorObjectCollection.cxx +++ b/Framework/src/MonitorObjectCollection.cxx @@ -18,49 +18,38 @@ #include "QualityControl/MonitorObject.h" #include "QualityControl/ObjectMetadataKeys.h" #include "QualityControl/QcInfoLogger.h" +#include "QualityControl/ObjectMetadataHelpers.h" #include #include #include #include -#include using namespace o2::mergers; namespace o2::quality_control::core { -std::optional parseCycle(const std::string& cycleStr) -{ - unsigned long cycleVal{}; - if (auto parse_res = std::from_chars(cycleStr.c_str(), cycleStr.c_str() + cycleStr.size(), cycleVal); parse_res.ec != std::errc{}) { - ILOG(Warning, Support) << "failed to decypher " << repository::metadata_keys::cycle << " metadata with value " << cycleStr - << ", with std::errc " << std::make_error_code(parse_res.ec).message() << ENDM; - return std::nullopt; - } - return cycleVal; -} - void mergeCycles(MonitorObject* targetMO, MonitorObject* otherMO) { - const auto otherCycle = otherMO->getMetadata(repository::metadata_keys::cycle); - const auto targetCycle = targetMO->getMetadata(repository::metadata_keys::cycle); + const auto otherCycle = otherMO->getMetadata(repository::metadata_keys::cycleNumber); + const auto targetCycle = targetMO->getMetadata(repository::metadata_keys::cycleNumber); if (otherCycle.has_value() && targetCycle.has_value()) { - const auto targetCycleParsed = parseCycle(targetCycle.value()); - const auto otherCycleParsed = parseCycle(otherCycle.value()); + const auto targetCycleParsed = repository::parseCycle(targetCycle.value()); + const auto otherCycleParsed = repository::parseCycle(otherCycle.value()); if (targetCycleParsed && otherCycleParsed) { - targetMO->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(std::max(targetCycleParsed.value(), otherCycleParsed.value()))); + targetMO->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, std::to_string(std::max(targetCycleParsed.value(), otherCycleParsed.value()))); return; } if (targetCycleParsed.value()) { - targetMO->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(targetCycleParsed.value())); + targetMO->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, std::to_string(targetCycleParsed.value())); return; } if (otherCycleParsed.value()) { - otherMO->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(otherCycleParsed.value())); + otherMO->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, std::to_string(otherCycleParsed.value())); return; } } @@ -179,7 +168,7 @@ const std::string& MonitorObjectCollection::getTaskName() const return mTaskName; } -void MonitorObjectCollection::addOrUpdateMetadata(std::string key, std::string value) +void MonitorObjectCollection::addOrUpdateMetadata(const std::string& key, const std::string& value) { for (auto obj : *this) { if (auto mo = dynamic_cast(obj)) { diff --git a/Framework/src/ObjectMetadataHelpers.cxx b/Framework/src/ObjectMetadataHelpers.cxx new file mode 100644 index 0000000000..76adf11de3 --- /dev/null +++ b/Framework/src/ObjectMetadataHelpers.cxx @@ -0,0 +1,35 @@ +// Copyright 2025 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file ObjectMetadataHelpers.cxx +/// \author Michal Tichak +/// + +#include +#include "QualityControl/ObjectMetadataHelpers.h" +#include "QualityControl/ObjectMetadataKeys.h" +#include "QualityControl/QcInfoLogger.h" + +namespace o2::quality_control::repository +{ +std::optional parseCycle(const std::string& cycleStr) +{ + unsigned long cycleVal{}; + if (auto parse_res = std::from_chars(cycleStr.c_str(), cycleStr.c_str() + cycleStr.size(), cycleVal); + parse_res.ec != std::errc{}) { + ILOG(Warning, Support) << "failed to decypher " << repository::metadata_keys::cycleNumber << " metadata with value " << cycleStr + << ", with std::errc " << std::make_error_code(parse_res.ec).message() << ENDM; + return std::nullopt; + } + return cycleVal; +} +} // namespace o2::quality_control::repository diff --git a/Framework/src/Quality.cxx b/Framework/src/Quality.cxx index a082e5d0d1..c37aca0c7e 100644 --- a/Framework/src/Quality.cxx +++ b/Framework/src/Quality.cxx @@ -15,8 +15,9 @@ /// #include "QualityControl/Quality.h" -#include #include +#include +#include #include #include #include @@ -93,6 +94,14 @@ std::string Quality::getMetadata(const std::string& key, const std::string& defa return mUserMetadata.count(key) > 0 ? mUserMetadata.at(key) : defaultValue; } +std::optional Quality::getMetadataOpt(const std::string& key) const +{ + if (auto found = mUserMetadata.find(key); found != mUserMetadata.end()) { + return found->second; + } + return std::nullopt; +} + Quality& Quality::addFlag(const FlagType& flag, std::string comment) { if (isWorseThan(Quality::Medium) && !flag.getBad()) { diff --git a/Framework/src/QualityObject.cxx b/Framework/src/QualityObject.cxx index 381dc3a6b7..758f24ef7e 100644 --- a/Framework/src/QualityObject.cxx +++ b/Framework/src/QualityObject.cxx @@ -69,7 +69,7 @@ std::string QualityObject::getName() const void QualityObject::updateQuality(Quality quality) { - //TODO: Update timestamp + // TODO: Update timestamp mQuality = std::move(quality); } Quality QualityObject::getQuality() const @@ -107,6 +107,11 @@ std::string QualityObject::getMetadata(std::string key, std::string defaultValue return mQuality.getMetadata(std::move(key), std::move(defaultValue)); } +std::optional QualityObject::getMetadataOpt(const std::string& key) const +{ + return mQuality.getMetadataOpt(key); +} + std::string QualityObject::getPath() const { std::string path; diff --git a/Framework/src/ReductorHelpers.cxx b/Framework/src/ReductorHelpers.cxx index 5ed4b09715..aa31f9acfe 100644 --- a/Framework/src/ReductorHelpers.cxx +++ b/Framework/src/ReductorHelpers.cxx @@ -16,6 +16,7 @@ #include "QualityControl/ReductorHelpers.h" +#include "QualityControl/QcInfoLogger.h" #include "QualityControl/Reductor.h" #include "QualityControl/ReductorTObject.h" #include "QualityControl/ReductorConditionAny.h" diff --git a/Framework/src/TaskRunner.cxx b/Framework/src/TaskRunner.cxx index 386ff58619..65979e8e28 100644 --- a/Framework/src/TaskRunner.cxx +++ b/Framework/src/TaskRunner.cxx @@ -524,8 +524,7 @@ int TaskRunner::publish(DataAllocator& outputs) // getNonOwningArray creates a TObjArray containing the monitoring objects, but not // owning them. The array is created by new and must be cleaned up by the caller std::unique_ptr array(mObjectsManager->getNonOwningArray()); - // TODO: this will send object with cycle == 0. should I send here cycle + 1? (same for QOs) - array->addOrUpdateMetadata(repository::metadata_keys::cycle, std::to_string(mCycleNumber)); + array->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, std::to_string(mCycleNumber)); int objectsPublished = array->GetEntries(); outputs.snapshot( diff --git a/Framework/src/TriggerHelpers.cxx b/Framework/src/TriggerHelpers.cxx index e56dd18462..2c263908c4 100644 --- a/Framework/src/TriggerHelpers.cxx +++ b/Framework/src/TriggerHelpers.cxx @@ -15,6 +15,7 @@ /// #include "QualityControl/TriggerHelpers.h" +#include "QualityControl/ObjectMetadataKeys.h" #include "QualityControl/PostProcessingConfig.h" #include "QualityControl/QcInfoLogger.h" #include diff --git a/Framework/src/Triggers.cxx b/Framework/src/Triggers.cxx index bf5e036151..39476d3744 100644 --- a/Framework/src/Triggers.cxx +++ b/Framework/src/Triggers.cxx @@ -293,8 +293,8 @@ TriggerFcn ForEachObject(const std::string& databaseUrl, const std::string& data auto currentActivity = activity_helpers::asActivity(*currentObject, activity.mProvenance); bool last = currentObject + 1 == filteredObjects->end(); Trigger trigger(TriggerType::ForEachObject, last, currentActivity, currentObject->get(timestampSortKey)); - if (auto cycle = currentObject->get_optional(metadata_keys::cycle); cycle.has_value()) { - trigger.metadata.emplace(metadata_keys::cycle, cycle.value()); + if (auto cycle = currentObject->get_optional(metadata_keys::cycleNumber); cycle.has_value()) { + trigger.metadata.emplace(metadata_keys::cycleNumber, cycle.value()); } ++currentObject; diff --git a/Framework/test/testAggregatorRunner.cxx b/Framework/test/testAggregatorRunner.cxx index 7f69cb4168..ef170a6d2d 100644 --- a/Framework/test/testAggregatorRunner.cxx +++ b/Framework/test/testAggregatorRunner.cxx @@ -20,7 +20,7 @@ #include "QualityControl/AggregatorRunnerConfig.h" #include "QualityControl/AggregatorConfig.h" #include "QualityControl/Aggregator.h" -#include "QualityControl/MonitorObject.h" +#include "QualityControl/ObjectMetadataKeys.h" #include "QualityControl/InfrastructureSpecReader.h" #include #include @@ -204,3 +204,26 @@ TEST_CASE("test_aggregator_activity_propagation") CHECK(result[0]->getActivity() == Activity{ 123, "PHYSICS", "LHC34b", "apass4", "qc", { 125, 175 }, "proton - mouton" }); CHECK(result[1]->getActivity() == Activity{ 123, "PHYSICS", "LHC34b", "apass4", "qc", { 125, 175 }, "proton - mouton" }); } + +TEST_CASE("test_aggregator_cycle") +{ + std::string configFilePath = std::string("json://") + getTestDataDirectory() + "testSharedConfig.json"; + auto [aggregatorRunnerConfig, aggregatorConfigs] = getAggregatorConfigs(configFilePath); + auto MyAggregatorBConfig = std::find_if(aggregatorConfigs.begin(), aggregatorConfigs.end(), [](const auto& cfg) { return cfg.name == "MyAggregatorB"; }); + REQUIRE(MyAggregatorBConfig != aggregatorConfigs.end()); + auto aggregator = make_shared(*MyAggregatorBConfig); + aggregator->init(); + + QualityObjectsMapType qoMap; + auto qo1 = make_shared(Quality::Good, "checkAll"); + qoMap["checkAll"] = qo1; + qo1->addMetadata(o2::quality_control::repository::metadata_keys::cycleNumber, "1"); + auto qo2 = make_shared(Quality::Bad, "dataSizeCheck2/skeletonTask/example"); + qo2->addMetadata(o2::quality_control::repository::metadata_keys::cycleNumber, "2"); + qoMap["dataSizeCheck2/skeletonTask/example"] = qo2; + auto results = aggregator->aggregate(qoMap); + for (auto r : results) { + REQUIRE_NOTHROW(r->getMetadata(o2::quality_control::repository::metadata_keys::cycleNumber)); + CHECK(r->getMetadata(o2::quality_control::repository::metadata_keys::cycleNumber) == "2"); + } +} diff --git a/Framework/test/testCcdbDatabase.cxx b/Framework/test/testCcdbDatabase.cxx index 229765242c..707e5ab301 100644 --- a/Framework/test/testCcdbDatabase.cxx +++ b/Framework/test/testCcdbDatabase.cxx @@ -133,13 +133,13 @@ BOOST_AUTO_TEST_CASE(ccdb_store) TH1F* h5 = new TH1F("cycle", "asdf", 100, 0, 99); shared_ptr mo5 = make_shared(h5, f.taskName, "TestClass", "TST"); - mo5->addMetadata(metadata_keys::cycle, "1"); + mo5->addMetadata(metadata_keys::cycleNumber, "1"); mo5->setValidity({ 10000, 20000 }); mo5->updateActivity(1234, "LHC66", "passName1", "qc"); TH1F* h6 = new TH1F("cycle", "asdf", 100, 0, 99); shared_ptr mo6 = make_shared(h6, f.taskName, "TestClass", "TST"); - mo6->addMetadata(metadata_keys::cycle, "2"); + mo6->addMetadata(metadata_keys::cycleNumber, "2"); mo6->setValidity({ 10000, 20000 }); mo6->updateActivity(1234, "LHC66", "passName1", "qc"); @@ -232,16 +232,16 @@ BOOST_AUTO_TEST_CASE(ccdb_retrieve_mo_with_cycle, *utf::depends_on("ccdb_store") test_fixture f; std::shared_ptr mo{}; mo = f.backend->retrieveMO(f.getMoFolder("cycle"), "cycle", - 15000, Activity{}, { { metadata_keys::cycle, "1" } }); + 15000, Activity{}, { { metadata_keys::cycleNumber, "1" } }); BOOST_REQUIRE(mo.get() != nullptr); - BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycle)); - BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycle) == "1"); + BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycleNumber)); + BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycleNumber) == "1"); mo = f.backend->retrieveMO(f.getMoFolder("cycle"), "cycle", - 15000, Activity{}, { { metadata_keys::cycle, "2" } }); + 15000, Activity{}, { { metadata_keys::cycleNumber, "2" } }); BOOST_REQUIRE(mo.get() != nullptr); - BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycle)); - BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycle) == "2"); + BOOST_REQUIRE_NO_THROW(mo->getMetadata(metadata_keys::cycleNumber)); + BOOST_REQUIRE(mo->getMetadata(metadata_keys::cycleNumber) == "2"); } BOOST_AUTO_TEST_CASE(ccdb_retrieve_qo, *utf::depends_on("ccdb_store")) diff --git a/Framework/test/testMonitorObjectCollection.cxx b/Framework/test/testMonitorObjectCollection.cxx index b097056eb5..0b77f2e39e 100644 --- a/Framework/test/testMonitorObjectCollection.cxx +++ b/Framework/test/testMonitorObjectCollection.cxx @@ -14,6 +14,7 @@ /// \author Piotr Konopka /// +#include "Framework/include/QualityControl/ObjectMetadataKeys.h" #include "QualityControl/MonitorObjectCollection.h" #include "QualityControl/MonitorObject.h" @@ -242,4 +243,35 @@ TEST_CASE("monitor_object_collection_clone_mw") delete mwMOC2; } +TEST_CASE("monitor_object_collection_merge_cycle") +{ + MonitorObjectCollection target; + MonitorObjectCollection other; + constexpr size_t bins = 10; + constexpr size_t min = 0; + constexpr size_t max = 10; + + target.SetOwner(true); + + TH1I* targetTH1I = new TH1I("histo 1d", "histo 1d", bins, min, max); + MonitorObject* targetMoTH1I = new MonitorObject(targetTH1I, "histo 1d", "class", "DET"); + targetMoTH1I->setIsOwner(true); + target.Add(targetMoTH1I); + targetMoTH1I->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, "1"); + + other.SetOwner(true); + + TH1I* otherTH1I = new TH1I("histo 1d", "histo 1d", bins, min, max); + MonitorObject* otherMoTH1I = new MonitorObject(otherTH1I, "histo 1d", "class", "DET"); + otherMoTH1I->setIsOwner(true); + other.Add(otherMoTH1I); + otherMoTH1I->addOrUpdateMetadata(repository::metadata_keys::cycleNumber, "2"); + + algorithm::merge(&target, &other); + + const auto mergedCycle = targetMoTH1I->getMetadata(repository::metadata_keys::cycleNumber); + REQUIRE(mergedCycle.has_value()); + REQUIRE(mergedCycle.value() == "2"); +} + } // namespace o2::quality_control::core diff --git a/Framework/test/testTriggers.cxx b/Framework/test/testTriggers.cxx index 80fee35f18..91e5410b88 100644 --- a/Framework/test/testTriggers.cxx +++ b/Framework/test/testTriggers.cxx @@ -14,6 +14,7 @@ /// \author Piotr Konopka /// +#include "QualityControl/ObjectMetadataKeys.h" #include "QualityControl/Triggers.h" #define BOOST_TEST_MODULE Triggers test @@ -161,14 +162,21 @@ BOOST_AUTO_TEST_CASE(test_trigger_for_each_object) mo->setActivity({ 101, "TECHNICAL", "FCC42x", "tpass1", "qc", { currentTimestamp + 1000, gInvalidValidityInterval.getMax() } }); repository->storeMO(mo); mo->setActivity({ 100, "TECHNICAL", "FCC42x", "tpass2", "qc", { currentTimestamp + 2000, gInvalidValidityInterval.getMax() } }); + mo->addOrUpdateMetadata(metadata_keys::cycleNumber, "42"); repository->storeMO(mo); { const Activity activityAllRunsPass1{ 0, "TECHNICAL", "FCC42x", "tpass1", "qc" }; auto forEachObjectTrigger = triggers::ForEachObject(CCDB_ENDPOINT, "qcdb", objectPath, activityAllRunsPass1); - BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::ForEachObject); - BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::ForEachObject); + auto trigger = forEachObjectTrigger(); + BOOST_CHECK_EQUAL(trigger, TriggerType::ForEachObject); + BOOST_CHECK_EQUAL(trigger.metadata.size(), 0); + + trigger = forEachObjectTrigger(); + BOOST_CHECK_EQUAL(trigger, TriggerType::ForEachObject); + BOOST_CHECK_EQUAL(trigger.metadata.size(), 0); + BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::No); } @@ -176,8 +184,15 @@ BOOST_AUTO_TEST_CASE(test_trigger_for_each_object) const Activity activityRun100AllPasses{ 100, "TECHNICAL", "FCC42x", "", "qc" }; auto forEachObjectTrigger = triggers::ForEachObject(CCDB_ENDPOINT, "qcdb", objectPath, activityRun100AllPasses); - BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::ForEachObject); - BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::ForEachObject); + auto trigger = forEachObjectTrigger(); + BOOST_CHECK_EQUAL(trigger, TriggerType::ForEachObject); + BOOST_CHECK_EQUAL(trigger.metadata.size(), 0); + + trigger = forEachObjectTrigger(); + BOOST_CHECK_EQUAL(trigger, TriggerType::ForEachObject); + BOOST_CHECK_EQUAL(trigger.metadata.size(), 1); + BOOST_CHECK_EQUAL(trigger.metadata.at(metadata_keys::cycleNumber), "42"); + BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::No); } @@ -185,9 +200,19 @@ BOOST_AUTO_TEST_CASE(test_trigger_for_each_object) const Activity activityAll{ 0, "NONE", "", "", "qc" }; auto forEachObjectTrigger = triggers::ForEachObject(CCDB_ENDPOINT, "qcdb", objectPath, activityAll); - BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::ForEachObject); - BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::ForEachObject); - BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::ForEachObject); + auto trigger = forEachObjectTrigger(); + BOOST_CHECK_EQUAL(trigger, TriggerType::ForEachObject); + BOOST_CHECK_EQUAL(trigger.metadata.size(), 0); + + trigger = forEachObjectTrigger(); + BOOST_CHECK_EQUAL(trigger, TriggerType::ForEachObject); + BOOST_CHECK_EQUAL(trigger.metadata.size(), 0); + + trigger = forEachObjectTrigger(); + BOOST_CHECK_EQUAL(trigger, TriggerType::ForEachObject); + BOOST_CHECK_EQUAL(trigger.metadata.size(), 1); + BOOST_CHECK_EQUAL(trigger.metadata.at(metadata_keys::cycleNumber), "42"); + BOOST_CHECK_EQUAL(forEachObjectTrigger(), TriggerType::No); } diff --git a/doc/Framework.md b/doc/Framework.md index 54ac81a388..e420e8498c 100644 --- a/doc/Framework.md +++ b/doc/Framework.md @@ -458,6 +458,7 @@ export O2_DPL_DEPLOYMENT_MODE=Grid && o2-qc --local-batch QC.root ... ## Monitor cycles The QC tasks monitor and process data continuously during a so-called "monitor cycle". At the end of such a cycle they publish the QC objects that will then continue their way in the QC data flow. +Since v1.178.0 cycles are stored in the metadata of QOs and MOs under the key `CycleNumber` inside the QCDB. A monitor cycle lasts typically between **1 and 5 minutes**, some reaching 10 minutes but never less than 1 minute for performance reasons. It is defined in the config file this way: diff --git a/doc/PostProcessing.md b/doc/PostProcessing.md index 2612afad64..137e48a999 100644 --- a/doc/PostProcessing.md +++ b/doc/PostProcessing.md @@ -154,7 +154,7 @@ Each of the three methods can be invoked by one or more triggers. Below are list * `"eof"` or `"endoffill"` - End Of Fill (not implemented yet) * `""` - Periodic - triggers when a specified period of time passes. For example: "5min", "0.001 seconds", "10sec", "2hours". * `"newobject:[qcdb/ccdb]:"` - New Object - triggers when an object in QCDB or CCDB is updated (applicable for synchronous processing). For example: `"newobject:qcdb:qc/TST/MO/QcTask/Example"` -* `"foreachobject:[qcdb/ccdb]:"` - For Each Object - triggers for each object in QCDB or CCDB which matches the activity indicated in the QC config file (applicable for asynchronous processing). +* `"foreachobject:[qcdb/ccdb]:"` - For Each Object - triggers for each object in QCDB or CCDB which matches the activity indicated in the QC config file (applicable for both synchronous and asynchronous processing). This trigger contains monitor cycle of required object in its metadata since v1.178.0 * `"foreachlatest:[qcdb/ccdb]:"` - For Each Latest - triggers for the latest object version in QCDB or CCDB for each matching activity (applicable for asynchronous processing). It sorts objects in ascending order by period, pass and run. @@ -1054,7 +1054,7 @@ For TrendingTask, it would be: ] ``` -### I want to run postprocessing on all already existing objects for a run +### I want to run postprocessing on all already existing objects for a run (usable in sync AND async) Use ForEachObject as the update trigger: @@ -1063,7 +1063,7 @@ Use ForEachObject as the update trigger: ``` Since objects are usually published in collections at the same time, you can use a path for one object to be triggered - for a collection of them (all objects produced by a QC Task). + for a collection of them (all objects produced by a QC Task). Use the Activity which matches the run, and (optionally) period and pass name: