diff --git a/Framework/include/QualityControl/CcdbDatabase.h b/Framework/include/QualityControl/CcdbDatabase.h index c47b7e37b8..3d43704718 100644 --- a/Framework/include/QualityControl/CcdbDatabase.h +++ b/Framework/include/QualityControl/CcdbDatabase.h @@ -67,7 +67,6 @@ class CcdbDatabase : public DatabaseInterface // storage void storeMO(std::shared_ptr q) override; void storeQO(std::shared_ptr q) override; - void storeQCFC(std::shared_ptr qcfc) override; void storeAny(const void* obj, std::type_info const& typeInfo, std::string const& path, std::map const& metadata, std::string const& detectorName, std::string const& taskName, long from = -1, long to = -1) override; @@ -80,9 +79,6 @@ class CcdbDatabase : public DatabaseInterface std::shared_ptr retrieveMO(std::string objectPath, std::string objectName, long timestamp = Timestamp::Current, const core::Activity& activity = {}) override; // retrieval - QO - deprecated std::shared_ptr retrieveQO(std::string qoPath, long timestamp = Timestamp::Current, const core::Activity& activity = {}) override; - std::shared_ptr retrieveQCFC(const std::string& name, const std::string& detector, int runNumber = 0, - const std::string& passName = "", const std::string& periodName = "", - const std::string& provenance = "", long timestamp = -1) override; // retrieval - general std::string retrieveJson(std::string path, long timestamp, const std::map& metadata) override; diff --git a/Framework/include/QualityControl/DatabaseInterface.h b/Framework/include/QualityControl/DatabaseInterface.h index 066f459390..de81ad2042 100644 --- a/Framework/include/QualityControl/DatabaseInterface.h +++ b/Framework/include/QualityControl/DatabaseInterface.h @@ -27,7 +27,6 @@ #include "QualityControl/QualityObject.h" #include "QualityControl/MonitorObject.h" #include "QualityControl/Activity.h" -#include namespace o2::quality_control::repository { @@ -127,11 +126,6 @@ class DatabaseInterface */ virtual void storeQO(std::shared_ptr qo) = 0; - /** - * Stores the serialized QualityControlFlagCollection in the database. - * @param qcfc The QualityControlFlagCollection to serialize and store. It should contain correct time validity inside the object. - */ - virtual void storeQCFC(std::shared_ptr qcfc) = 0; /** * \brief Look up a monitor object and return it. * Look up a monitor object and return it if found or nullptr if not. @@ -151,13 +145,7 @@ class DatabaseInterface * @deprecated */ virtual std::shared_ptr retrieveQO(std::string qoPath, long timestamp = Timestamp::Current, const core::Activity& activity = {}) = 0; - /** - * \brief Look up a QualityControlFlagCollection object and return it. - * Look up a QualityControlFlagCollection and return it if found or nullptr if not. - */ - virtual std::shared_ptr retrieveQCFC(const std::string& name, const std::string& detector, int runNumber = 0, - const std::string& passName = "", const std::string& periodName = "", - const std::string& provenance = "", long timestamp = Timestamp::Current) = 0; + /** * \brief Look up an object and return it. * Look up an object and return it if found or nullptr if not. It is a raw pointer because we might need it to build a MO. diff --git a/Framework/include/QualityControl/DummyDatabase.h b/Framework/include/QualityControl/DummyDatabase.h index d7fb530996..86ffd33f94 100644 --- a/Framework/include/QualityControl/DummyDatabase.h +++ b/Framework/include/QualityControl/DummyDatabase.h @@ -39,11 +39,7 @@ class DummyDatabase : public DatabaseInterface // QualityObject void storeQO(std::shared_ptr q) override; std::shared_ptr retrieveQO(std::string checkerName, long timestamp = -1, const core::Activity& activity = {}) override; - // QCFC - void storeQCFC(std::shared_ptr qcfc) override; - std::shared_ptr retrieveQCFC(const std::string& name, const std::string& detector, int runNumber = 0, - const std::string& passName = "", const std::string& periodName = "", - const std::string& provenance = "", long timestamp = -1) override; + // General void* retrieveAny(std::type_info const& tinfo, std::string const& path, std::map const& metadata, long timestamp = -1, diff --git a/Framework/include/QualityControl/ObjectMetadataKeys.h b/Framework/include/QualityControl/ObjectMetadataKeys.h index 4fc4a01c4e..cc3e322469 100644 --- a/Framework/include/QualityControl/ObjectMetadataKeys.h +++ b/Framework/include/QualityControl/ObjectMetadataKeys.h @@ -35,7 +35,6 @@ constexpr auto qcTaskName = "qc_task_name"; constexpr auto qcTaskClass = "qc_task_class"; constexpr auto qcQuality = "qc_quality"; constexpr auto qcCheckName = "qc_check_name"; -constexpr auto qcQCFCName = "qc_qcfc_name"; constexpr auto qcAdjustableEOV = "adjustableEOV"; // this is a keyword for the CCDB // QC Activity constexpr auto runType = "RunType"; diff --git a/Framework/include/QualityControl/RepoPathUtils.h b/Framework/include/QualityControl/RepoPathUtils.h index bc3445bd8c..3fd69ade58 100644 --- a/Framework/include/QualityControl/RepoPathUtils.h +++ b/Framework/include/QualityControl/RepoPathUtils.h @@ -22,7 +22,6 @@ #include "QualityControl/MonitorObject.h" #include "QualityControl/QualityObject.h" #include -#include namespace o2::quality_control::core { @@ -107,25 +106,6 @@ class RepoPathUtils includeProvenance); } - /** - * Compute and return the path to the FlagCollection. - * Current algorithm does //QCFC/ - * @param detectorCode - * @param qcfcName - * @param provenance - * @return the path to the QCFCollection - */ - static std::string getQcfcPath(const std::string& detectorCode, - const std::string& qcfcName, - const std::string& provenance = "qc"); - /** - * Compute and return the path to the QCFCollection. - * Current algorithm does //QCFC/ - * @param qcfc - * @return the path to the QCFCollection - */ - static std::string getQcfcPath(const QualityControlFlagCollection* qcfc); - static constexpr auto allowedProvenancesMessage = R"(Allowed provenances are "qc" (real data processed synchronously), "qc_async" (real data processed asynchronously) and "qc_mc" (simulated data).)"; static bool isProvenanceAllowed(const std::string& provenance); diff --git a/Framework/src/CcdbDatabase.cxx b/Framework/src/CcdbDatabase.cxx index 46bbe0636b..eaaf35265b 100644 --- a/Framework/src/CcdbDatabase.cxx +++ b/Framework/src/CcdbDatabase.cxx @@ -270,33 +270,6 @@ void CcdbDatabase::storeQO(std::shared_ptr qcfc) -{ - // metadata - map metadata; - metadata[metadata_keys::runNumber] = std::to_string(qcfc->getRunNumber()); - metadata[metadata_keys::periodName] = qcfc->getPeriodName(); - metadata[metadata_keys::passName] = qcfc->getPassName(); - // QC metadata (prefix qc_) - addFrameworkMetadata(metadata, qcfc->getDetector(), qcfc->IsA()->GetName()); - metadata[metadata_keys::qcQCFCName] = qcfc->getName(); - - // other attributes - string path = RepoPathUtils::getQcfcPath(qcfc.get()); - auto from = qcfc->getStart(); - auto to = qcfc->getEnd(); - if (from > to) { - ILOG(Error, Support) << "QCFC '" + qcfc->getName() + "' cannot be stored in CCDB, because it has invalid validity range (" + std::to_string(from) + ", " + std::to_string(to) + ")." << ENDM; - } - std::stringstream buffer; - qcfc->streamTo(buffer); - ILOG(Debug, Support) << "Storing QualityControlFlagCollection at " << path << " (" << qcfc->getName() << ")" << ENDM; - auto result = ccdbApi->storeAsBinaryFile(buffer.str().c_str(), buffer.str().size(), qcfc->getName(), qcfc->IsA()->GetName(), path, metadata, from, to); - if (result != 0) { - ILOG(Error, Support) << "QCFC '" + qcfc->getName() + "' could not be stored in CCDB, error: " + std::to_string(result) << ENDM; - } -} - TObject* CcdbDatabase::retrieveTObject(std::string path, std::map const& metadata, long timestamp, std::map* headers) { if (timestamp == Timestamp::Latest) { @@ -396,62 +369,6 @@ std::shared_ptr CcdbDatabase::retrieve return qo; } -std::shared_ptr CcdbDatabase::retrieveQCFC(const std::string& qcfcName, const std::string& detector, int runNumber, const string& passName, const string& periodName, const std::string& provenance, long timestamp) -{ - map headers; - map metadata; - if (runNumber != 0) { - metadata[metadata_keys::runNumber] = std::to_string(runNumber); - } - if (!passName.empty()) { - metadata[metadata_keys::passName] = passName; - } - if (!periodName.empty()) { - metadata[metadata_keys::periodName] = periodName; - } - const auto qcfcPath = RepoPathUtils::getQcfcPath(detector, qcfcName, provenance); - const std::string localFileDir = "/tmp"; - const std::string localFileName = "qcfc_" + qcfcName + std::to_string(time_point_cast(system_clock::now()).time_since_epoch().count()); - const std::string localFilePath = localFileDir + std::filesystem::path::preferred_separator + localFileName; - if (localFilePath.find("..") != std::string::npos || localFilePath.find('~') != std::string::npos) { - ILOG(Error, Support) << "The path '" << localFilePath << "' looks hacky, will not download any files there." << ENDM; - return nullptr; - } - - auto resultMetadata = ccdbApi->retrieveHeaders(qcfcPath, metadata, timestamp); - if (resultMetadata.empty()) { - ILOG(Error, Support) << "Could not extract headers of QCFC at '" << qcfcPath << "' with the metadata: " << ENDM; // TODO - ILOG(Error, Support) << " - RunNumber : " << metadata[metadata_keys::runNumber] << ENDM; - ILOG(Error, Support) << " - PassName : " << metadata[metadata_keys::passName] << ENDM; - ILOG(Error, Support) << " - PeriodName : " << metadata[metadata_keys::periodName] << ENDM; - return nullptr; - } - - auto success = ccdbApi->retrieveBlob(qcfcPath, localFileDir, metadata, timestamp, false, localFileName); - if (!success) { - ILOG(Error, Support) << "Could not retrieve the QCFC at '" << qcfcPath << "' with the metadata: " << ENDM; // TODO - ILOG(Error, Support) << " - RunNumber : " << metadata[metadata_keys::runNumber] << ENDM; - ILOG(Error, Support) << " - PassName : " << metadata[metadata_keys::passName] << ENDM; - ILOG(Error, Support) << " - PeriodName : " << metadata[metadata_keys::periodName] << ENDM; - return nullptr; - } - - std::ifstream localFile(localFilePath); - if (!localFile.is_open()) { - ILOG(Error, Support) << "Could not open a file at '" << localFilePath << "'" << ENDM; - std::filesystem::remove(localFilePath); - return nullptr; - } - - QualityControlFlagCollection::RangeInterval validity{ std::stoull(resultMetadata[metadata_keys::validFrom]), std::stoull(resultMetadata[metadata_keys::validUntil]) }; - auto qcfc = std::make_shared(qcfcName, detector, validity, runNumber, periodName, passName, provenance); - qcfc->streamFrom(localFile); - localFile.close(); - std::filesystem::remove(localFilePath); - - return qcfc; -} - std::string CcdbDatabase::retrieveJson(std::string path, long timestamp, const std::map& metadata) { map headers; diff --git a/Framework/src/DummyDatabase.cxx b/Framework/src/DummyDatabase.cxx index 2ca2343c8c..33a4ecc9b4 100644 --- a/Framework/src/DummyDatabase.cxx +++ b/Framework/src/DummyDatabase.cxx @@ -47,10 +47,6 @@ void DummyDatabase::storeQO(std::shared_ptr qcfc) -{ -} - std::shared_ptr DummyDatabase::retrieveQO(std::string, long, const core::Activity& activity) { return {}; @@ -87,10 +83,6 @@ void* DummyDatabase::retrieveAny(const std::type_info&, const std::string&, cons { return nullptr; } -std::shared_ptr DummyDatabase::retrieveQCFC(const std::string& name, const std::string& detector, int runNumber, const std::string& passName, const std::string& periodName, const std::string& provenance, long timestamp) -{ - return nullptr; -} void DummyDatabase::setMaxObjectSize(size_t maxObjectSize) { diff --git a/Framework/src/RepoPathUtils.cxx b/Framework/src/RepoPathUtils.cxx index bd12721d1a..8400832e7f 100644 --- a/Framework/src/RepoPathUtils.cxx +++ b/Framework/src/RepoPathUtils.cxx @@ -15,18 +15,6 @@ namespace o2::quality_control::core { -std::string RepoPathUtils::getQcfcPath(const std::string& detectorCode, - const std::string& qcfcName, - const std::string& provenance) -{ - return provenance + "/" + detectorCode + "/QCFC/" + qcfcName; -} - -std::string RepoPathUtils::getQcfcPath(const QualityControlFlagCollection* qcfc) -{ - return getQcfcPath(qcfc->getDetector(), qcfc->getName(), qcfc->getProvenance()); -} - bool RepoPathUtils::isProvenanceAllowed(const std::string& provenance) { return provenance == "qc" || provenance == "qc_async" || provenance == "qc_mc"; diff --git a/Framework/test/testCcdbDatabase.cxx b/Framework/test/testCcdbDatabase.cxx index 02a957ed9c..8042084d1a 100644 --- a/Framework/test/testCcdbDatabase.cxx +++ b/Framework/test/testCcdbDatabase.cxx @@ -417,30 +417,5 @@ BOOST_AUTO_TEST_CASE(ccdb_store_retrieve_latest) BOOST_CHECK_EQUAL(h1Back->GetEntries(), 20000); } -BOOST_AUTO_TEST_CASE(ccdb_qcfc) -{ - test_fixture f; - const std::string pid = std::to_string(getpid()); - const std::string qcfcName = "Test_pid" + pid; // TODO we can use a 'Test' directory once https://github.com/AliceO2Group/AliceO2/pull/8195 is merged - - std::shared_ptr qcfc1{ new QualityControlFlagCollection{ qcfcName, "TST", { 45, 500000 }, 42, "LHC42x", "spass", "qc" } }; - qcfc1->insert({ 50, 77, FlagTypeFactory::Invalid(), "a comment", "a source" }); - qcfc1->insert({ 51, 77, FlagTypeFactory::Invalid() }); - qcfc1->insert({ 1234, 3434, FlagTypeFactory::BadPID() }); - qcfc1->insert({ 50, 77, FlagTypeFactory::BadPID() }); - qcfc1->insert({ 43434, 63421, FlagTypeFactory::Good() }); - - f.backend->storeQCFC(qcfc1); - - auto qcfc2 = f.backend->retrieveQCFC(qcfc1->getName(), qcfc1->getDetector(), qcfc1->getRunNumber(), - qcfc1->getPassName(), qcfc1->getPeriodName(), qcfc1->getProvenance(), 400000); - BOOST_REQUIRE(qcfc2 != nullptr); - - BOOST_REQUIRE_EQUAL(qcfc1->size(), qcfc2->size()); - for (auto it1 = qcfc1->begin(), it2 = qcfc2->begin(); it1 != qcfc1->end() && it2 != qcfc2->end(); ++it1, ++it2) { - BOOST_CHECK_EQUAL(*it1, *it2); - } -} - } // namespace } // namespace o2::quality_control::core diff --git a/Modules/Common/CMakeLists.txt b/Modules/Common/CMakeLists.txt index d46b3c91cc..ddd93c9122 100644 --- a/Modules/Common/CMakeLists.txt +++ b/Modules/Common/CMakeLists.txt @@ -12,8 +12,6 @@ add_library(O2QcCommon) target_sources(O2QcCommon PRIVATE src/IncreasingEntries.cxx src/WorstOfAllAggregator.cxx - src/FlagCollectionTask.cxx - src/FlagCollectionTaskConfig.cxx src/QualityTask.cxx src/QualityTaskConfig.cxx src/BigScreenCanvas.cxx @@ -60,7 +58,6 @@ install(TARGETS O2QcCommon add_root_dictionary(O2QcCommon HEADERS include/Common/NonEmpty.h include/Common/WorstOfAllAggregator.h - include/Common/FlagCollectionTask.h include/Common/QualityTask.h include/Common/BigScreen.h include/Common/CcdbInspectorTask.h @@ -87,9 +84,6 @@ add_root_dictionary(O2QcCommon include/Common/LHCClockPhaseReductor.h LINKDEF include/Common/LinkDef.h) -install(FILES etc/flagcollection-example.json - DESTINATION Modules/Common) - install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/Common DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/QualityControl") diff --git a/Modules/Common/etc/flagcollection-example.json b/Modules/Common/etc/flagcollection-example.json deleted file mode 100644 index bf2e69b292..0000000000 --- a/Modules/Common/etc/flagcollection-example.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "NONE", - "periodName": "LHC42x", "": "Period name / Production tag - e.g. LHC22c, LHC22c1b_test", - "passName": "spass", "": "Pass type - e.g. spass, cpass1", - "provenance": "qc", "": "Provenance - qc or qc_mc depending whether it is normal data or monte carlo data" - }, - "monitoring": { - "url": "infologger:///debug?qc" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - } - }, - "postprocessing": { - "FlagCollectionQcCheck": { - "active": "true", - "className": "o2::quality_control_modules::common::FlagCollectionTask", - "moduleName": "QcCommon", - "detectorName": "TST", - "initTrigger": [], - "updateTrigger": [], - "stopTrigger": [], - "QOs": [ - "QcCheck" - ] - } - } - } -} diff --git a/Modules/Common/include/Common/FlagCollectionTask.h b/Modules/Common/include/Common/FlagCollectionTask.h deleted file mode 100644 index d91e18b714..0000000000 --- a/Modules/Common/include/Common/FlagCollectionTask.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// -/// \file FlagCollectionTask.h -/// \author Piotr Konopka -/// - -#ifndef QUALITYCONTROL_QCFCOLLECTIONTASK_H -#define QUALITYCONTROL_QCFCOLLECTIONTASK_H - -#include "QualityControl/PostProcessingInterface.h" -#include "Common/FlagCollectionTaskConfig.h" -#include - -namespace o2::quality_control::repository -{ -class DatabaseInterface; -} - -namespace o2::quality_control_modules::common -{ - -/// \brief Example Quality Control Postprocessing Task -/// \author My Name -class FlagCollectionTask final : public quality_control::postprocessing::PostProcessingInterface -{ - public: - /// \brief Constructor - FlagCollectionTask() = default; - /// \brief Destructor - ~FlagCollectionTask() override; - - void configure(const boost::property_tree::ptree& config) override; - void initialize(quality_control::postprocessing::Trigger, framework::ServiceRegistryRef) override; - void update(quality_control::postprocessing::Trigger, framework::ServiceRegistryRef) override; - void finalize(quality_control::postprocessing::Trigger, framework::ServiceRegistryRef) override; - - private: - std::unique_ptr - transformQualities(quality_control::repository::DatabaseInterface& qcdb, const uint64_t timestampLimitStart, const uint64_t timestampLimitEnd); - - private: - FlagCollectionTaskConfig mConfig; - uint64_t mLastTimestampLimitStart = 0; -}; - -} // namespace o2::quality_control_modules::common - -#endif // QUALITYCONTROL_QCFCOLLECTIONTASK_H \ No newline at end of file diff --git a/Modules/Common/include/Common/FlagCollectionTaskConfig.h b/Modules/Common/include/Common/FlagCollectionTaskConfig.h deleted file mode 100644 index 0bbaff493f..0000000000 --- a/Modules/Common/include/Common/FlagCollectionTaskConfig.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// -/// \file FlagCollectionTaskConfig.h -/// \author Piotr Konopka -/// - -#ifndef QUALITYCONTROL_TIMERANGESTASKCONFIG -#define QUALITYCONTROL_TIMERANGESTASKCONFIG - -#include -#include -#include "QualityControl/PostProcessingConfig.h" - -namespace o2::quality_control_modules::common -{ - -// todo pretty print -/// \brief FlagCollectionTaskConfig configuration structure -struct FlagCollectionTaskConfig : quality_control::postprocessing::PostProcessingConfig { - FlagCollectionTaskConfig() = default; - FlagCollectionTaskConfig(std::string name, const boost::property_tree::ptree& config); - ~FlagCollectionTaskConfig() = default; - - // todo: almost all here duplicates the parent class, remove - std::string name; - std::string detector; - int runNumber; - std::string passName; - std::string periodName; - std::string provenance; - std::vector qualityObjects; -}; - -} // namespace o2::quality_control_modules::common - -#endif // QUALITYCONTROL_TIMERANGESTASKCONFIG diff --git a/Modules/Common/include/Common/LinkDef.h b/Modules/Common/include/Common/LinkDef.h index 679880fae9..e889a847ff 100644 --- a/Modules/Common/include/Common/LinkDef.h +++ b/Modules/Common/include/Common/LinkDef.h @@ -14,7 +14,6 @@ #pragma link C++ class o2::quality_control_modules::common::THnSparse5Reductor + ; #pragma link C++ class o2::quality_control_modules::common::QualityReductor + ; #pragma link C++ class o2::quality_control_modules::common::EverIncreasingGraph + ; -#pragma link C++ class o2::quality_control_modules::common::FlagCollectionTask + ; #pragma link C++ class o2::quality_control_modules::common::QualityTask + ; #pragma link C++ class o2::quality_control_modules::common::BigScreen + ; #pragma link C++ class o2::quality_control_modules::common::CcdbInspectorTask + ; diff --git a/Modules/Common/src/FlagCollectionTask.cxx b/Modules/Common/src/FlagCollectionTask.cxx deleted file mode 100644 index cc1d6fe779..0000000000 --- a/Modules/Common/src/FlagCollectionTask.cxx +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// -/// \file PostProcessingInterface.cxx -/// \author Piotr Konopka -/// - -#include "Common/FlagCollectionTask.h" -#include "QualityControl/QcInfoLogger.h" -#include "QualityControl/DatabaseInterface.h" -#include "QualityControl/CcdbDatabase.h" -#include "QualityControl/RepoPathUtils.h" -#include "QualityControl/QualitiesToFlagCollectionConverter.h" - -#include - -using namespace o2::quality_control::postprocessing; -using namespace o2::quality_control::core; -using namespace o2::quality_control; - -namespace o2::quality_control_modules::common -{ - -FlagCollectionTask::~FlagCollectionTask() -{ -} - -void FlagCollectionTask::configure(const boost::property_tree::ptree& config) -{ - mConfig = FlagCollectionTaskConfig(getID(), config); -} - -void FlagCollectionTask::initialize(Trigger t, framework::ServiceRegistryRef) -{ - mLastTimestampLimitStart = t.timestamp; -} - -std::unique_ptr FlagCollectionTask::transformQualities(repository::DatabaseInterface& qcdb, const uint64_t timestampLimitStart, const uint64_t timestampLimitEnd) -{ - // ------ HELPERS ------ - std::function(const std::string& /*QO*/)> fetchAvailableTimestamps; - try { - fetchAvailableTimestamps = [&qcdbAsCcdb = dynamic_cast(qcdb), &detector = mConfig.detector](const std::string& qo) { - std::string path = RepoPathUtils::getQoPath(detector, qo); - return qcdbAsCcdb.getTimestampsForObject(path); - }; - } catch (std::bad_cast& ex) { - ILOG(Error) << "Could not cast the database interface to CcdbDatabase, this task supports only the CCDB backend" << ENDM - } - - auto makeQCFC = [&]() { - return std::make_unique(mConfig.name, mConfig.detector, - QualityControlFlagCollection::RangeInterval{ timestampLimitStart, timestampLimitEnd }, - mConfig.runNumber, mConfig.periodName, mConfig.passName, mConfig.provenance); - }; - - // ------ IMPLEMENTATION ------ - // stats - size_t totalQOsIncluded = 0; - size_t totalWorseThanGoodQOs = 0; - - auto mainFlagCollection = makeQCFC(); - for (const auto& qoName : mConfig.qualityObjects) { - - std::string qoPath = RepoPathUtils::getQoPath(mConfig.detector, qoName); - QualitiesToFlagCollectionConverter converter(makeQCFC(), qoPath); - - auto availableTimestamps = fetchAvailableTimestamps(qoName); - auto firstMatchingTimestamp = std::upper_bound(availableTimestamps.begin(), availableTimestamps.end(), timestampLimitStart); - - if (firstMatchingTimestamp == availableTimestamps.end()) { - ILOG(Warning) << "No object under the path '" << qoPath << "' available after timestamp '" << timestampLimitStart << "'" << ENDM; - continue; - } - - // if available, we move one timestamp back, because 'validUntil' might cover our period. - if (firstMatchingTimestamp != availableTimestamps.begin()) { - - auto currentObjTimestamp = firstMatchingTimestamp - 1; - auto qo = qcdb.retrieveQO(qoPath, *currentObjTimestamp); - if (qo == nullptr) { - throw std::runtime_error("Could not retrieve a QO for timestamp '" + std::to_string(*currentObjTimestamp) + "'"); - } - converter(*qo); - } - - // the main loop over QOs - for (auto currentStartTime = firstMatchingTimestamp; - currentStartTime != availableTimestamps.end() && *currentStartTime < timestampLimitEnd; - currentStartTime++) { - - auto newQO = qcdb.retrieveQO(qoPath, *currentStartTime); - if (newQO == nullptr) { - throw std::runtime_error("Could not retrieve a QO for timestamp '" + std::to_string(*currentStartTime) + "'"); - } - - converter(*newQO); - } - - totalQOsIncluded += converter.getQOsIncluded(); - totalWorseThanGoodQOs += converter.getWorseThanGoodQOs(); - mainFlagCollection->merge(*converter.getResult()); - } - - ILOG(Info, Support) << "Total number of QOs included in FlagCollection: " << totalQOsIncluded << ENDM; - ILOG(Info, Support) << "Total number of worse than good QOs: " << totalWorseThanGoodQOs << ENDM; - ILOG(Info, Support) << "Number of Flags: " << mainFlagCollection->size() << ENDM; - // TODO: now we print it, but it should be stored in the QCDB after we have QC-547. - ILOG(Info) << *mainFlagCollection << ENDM; - - return mainFlagCollection; -} - -void FlagCollectionTask::update(Trigger, framework::ServiceRegistryRef) -{ - throw std::runtime_error("Only two timestamps should be given to the task"); -} - -void FlagCollectionTask::finalize(Trigger t, framework::ServiceRegistryRef services) -{ - auto timestampLimitEnd = t.timestamp; - - auto& qcdb = services.get(); - auto flagCollection = transformQualities(qcdb, mLastTimestampLimitStart, timestampLimitEnd); - qcdb.storeQCFC(std::shared_ptr(flagCollection.release())); - - mLastTimestampLimitStart = timestampLimitEnd; -} - -} // namespace o2::quality_control_modules::common diff --git a/Modules/Common/src/FlagCollectionTaskConfig.cxx b/Modules/Common/src/FlagCollectionTaskConfig.cxx deleted file mode 100644 index 59c8f4b347..0000000000 --- a/Modules/Common/src/FlagCollectionTaskConfig.cxx +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// -/// \file FlagCollectionTaskConfig.cxx -/// \author Piotr Konopka -/// - -#include "Common/FlagCollectionTaskConfig.h" -#include - -namespace o2::quality_control_modules::common -{ - -FlagCollectionTaskConfig::FlagCollectionTaskConfig(std::string id, const boost::property_tree::ptree& config) - : PostProcessingConfig(id, config), name(id) -{ - detector = config.get("qc.postprocessing." + id + ".detectorName"); - for (const auto& qoPath : config.get_child("qc.postprocessing." + id + ".QOs")) { - qualityObjects.push_back(qoPath.second.data()); - } - runNumber = config.get("qc.config.Activity.number", 0); - passName = config.get("qc.config.Activity.passName", ""); - periodName = config.get("qc.config.Activity.periodName", ""); - provenance = config.get("qc.config.Activity.provenance", "qc"); -} - -} // namespace o2::quality_control_modules::common diff --git a/Modules/EMCAL/src/SupermoduleProjectorTask.cxx b/Modules/EMCAL/src/SupermoduleProjectorTask.cxx index 8656d75cdf..cfa786581a 100644 --- a/Modules/EMCAL/src/SupermoduleProjectorTask.cxx +++ b/Modules/EMCAL/src/SupermoduleProjectorTask.cxx @@ -12,6 +12,7 @@ #include "QualityControl/QcInfoLogger.h" #include "EMCAL/SupermoduleProjectorTask.h" #include "QualityControl/DatabaseInterface.h" +#include #include diff --git a/README.md b/README.md index 1565a828ef..5e4dc6faa1 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,6 @@ For a general overview of our (O2) software, organization and processes, please * [The TrendingTask class](doc/PostProcessing.md#the-trendingtask-class) * [The SliceTrendingTask class](doc/PostProcessing.md#the-slicetrendingtask-class) * [The QualityTask class](doc/PostProcessing.md#the-qualitytask-class) - * [The FlagCollectionTask class](doc/PostProcessing.md#the-flagcollectiontask-class) * [The BigScreen class](doc/PostProcessing.md#the-bigscreen-class) * [More examples](doc/PostProcessing.md#more-examples) * [Advanced topics](doc/Advanced.md) diff --git a/doc/PostProcessing.md b/doc/PostProcessing.md index 4d9d4d1842..0f9129e34f 100644 --- a/doc/PostProcessing.md +++ b/doc/PostProcessing.md @@ -15,7 +15,6 @@ * [The ReferenceComparatorTask class](#the-referencecomparatortask-class) * [The CcdbInspectorTask class](#the-ccdbinspectortask-class) * [The QualityTask class](#the-qualitytask-class) - * [The FlagCollectionTask class](#the-flagcollectiontask-class) * [The BigScreen class](#the-bigscreen-class) * [More examples](#more-examples) @@ -869,50 +868,6 @@ Here is a complete example of `QualityTask` configuration: } ``` -### The FlagCollectionTask class - -This task allows to transform a set of QualityObjects stored QCDB across certain timespan (usually for the duration of a data acquisition run) into a QualityControlFlagCollection. -It is meant to be run after for each detector/subsystem separately and when all QualityObjects for a run are generated. -After generating timestamps, final data tags can be computed as the next step. -The data formats for tagging data quality are described [here](https://github.com/AliceO2Group/AliceO2/tree/dev/DataFormats/QualityControl/README.md). - -The task should be run asynchronously to data-taking and should be given the start and end of a time range to process. -For example: - -```bash -o2-qc-run-postprocessing --config json://${QUALITYCONTROL_ROOT}/Modules/Common/etc/flagcollection-example.json \ - --name FlagCollectionQcCheck --timestamps 1612707603626 1613999652000 -``` - -The task is configured as follows: - -```json -{ - "qc": { - "config": { - "": "The usual global configuration variables" - }, - "postprocessing": { - "FlagCollectionQcCheck": { - "active": "true", - "className": "o2::quality_control_modules::common::FlagCollectionTask", - "moduleName": "QcCommon", - "detectorName": "TST", "": "One task should concatenate Qualities from detector, defined here.", - "initTrigger": [], "": "The triggers can be left empty,", - "updateTrigger": [], "": "because we run the task with a defined set of timestamps.", - "stopTrigger": [], - "": "The list of Quality Object to process.", - "QOs": [ - "QcCheck" - ] - } - } - } -} -``` - -QualityControlFlagCollections are meant to be used as a base to derive Data Tags for analysis (WIP). - ### The BigScreen class This task allows to display the aggregated quality flags of each system (detectors + global QC) in a single canvas, arranged as a matrix of colored boxes.