From e93c43e68aa425429fc0afe1c3d9491a5da4ccd9 Mon Sep 17 00:00:00 2001 From: Anton Alkin Date: Mon, 9 Nov 2026 08:59:38 +0100 Subject: [PATCH 1/3] DPL Analysis: fix parent paths replacements The AOD reader replaces every occurrence of the aod-parent-base-path-replacement pattern in a parent file path, while metadata discovery replaced only the first one, so the two could open different files. Use the same replacement in both. An empty pattern is now ignored, as it is in the reader. --- Framework/AnalysisSupport/src/Plugin.cxx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx index 412f00a9eb37e..cebe968f799e9 100644 --- a/Framework/AnalysisSupport/src/Plugin.cxx +++ b/Framework/AnalysisSupport/src/Plugin.cxx @@ -242,9 +242,9 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin { } auto from = parentFileReplacement.substr(0, pos); auto to = parentFileReplacement.substr(pos + 1); - pos = parentFilename.find(from); - if (pos != std::string::npos) { - parentFilename.replace(pos, from.length(), to); + // Same as TString::ReplaceAll + for (size_t at = 0; !from.empty() && (at = parentFilename.find(from, at)) != std::string::npos; at += to.size()) { + parentFilename.replace(at, from.size(), to); } } From b5ed7eb822c592882d1ce6f491423db2c3e432c0 Mon Sep 17 00:00:00 2001 From: Anton Alkin Date: Mon, 9 Nov 2026 10:49:12 +0100 Subject: [PATCH 2/3] Make AOD metadata discovery aware of the origin level mapping With --aod-origin-replace and --aod-origin-level-mapping, every device received the metadata of level 0, even when different files are read. Each device now sees the metadata of the level its origin maps to. - Parse and validate aod-origin-level-mapping once during discovery, into the aod-origin-level-map LabeledArray which the AOD reader now consumes. - Read each mapped level through AODInputFile, opening every file in the chain of parents only once, and forward the result as the aod-metadata-origins LabeledArray and aod-metadata-origin--tables. - Overlay the row of the origin of the process on the plain aod-metadata-* options at the end of discovery, erasing the keys which are missing at that level. - Add LabeledArray::getRow by label, and tests for the above. Without a mapping the plain options are unchanged. --- .../src/AODJAlienReaderHelpers.cxx | 26 +- Framework/AnalysisSupport/src/Plugin.cxx | 231 +++++++++++++----- Framework/Core/CMakeLists.txt | 1 + Framework/Core/include/Framework/Array2D.h | 7 + .../include/Framework/ConfigParamDiscovery.h | 13 + Framework/Core/src/ConfigParamDiscovery.cxx | 89 +++++++ Framework/Core/src/Plugin.cxx | 13 +- .../Core/test/test_ConfigParamDiscovery.cxx | 145 +++++++++++ .../Core/test/test_DeviceSpecHelpers.cxx | 5 +- Framework/Core/test/test_Variants.cxx | 4 + 10 files changed, 450 insertions(+), 84 deletions(-) create mode 100644 Framework/Core/test/test_ConfigParamDiscovery.cxx diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx index 4f0e438651cc2..078bb95b81157 100644 --- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx +++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx @@ -11,12 +11,10 @@ #include "AODJAlienReaderHelpers.h" #include -#include #include #include #include #include -#include #include #include #include @@ -27,6 +25,7 @@ #include "Framework/AlgorithmSpec.h" #include "Framework/ArrowContext.h" #include "Framework/ConfigParamRegistry.h" +#include "Framework/Array2D.h" #include "Framework/ControlService.h" #include "Framework/CallbackService.h" #include "Framework/EndOfStreamContext.h" @@ -156,24 +155,11 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const parentAccessLevel = ctx.options().get("aod-parent-access-level"); } std::vector> originLevelMapping; - if (ctx.options().isSet("aod-origin-level-mapping")) { - auto originLevelMappingStr = ctx.options().get("aod-origin-level-mapping"); - for (auto pairRange : originLevelMappingStr | std::views::split(',')) { - std::string_view pair{pairRange.begin(), pairRange.end()}; - auto colonPos = pair.find(':'); - if (colonPos == std::string_view::npos) { - LOGP(fatal, "Badly formatted aod-origin-level-mapping entry: \"{}\"", pair); - continue; - } - std::string key(pair.substr(0, colonPos)); - std::string_view valueStr = pair.substr(colonPos + 1); - int value{}; - auto [ptr, ec] = std::from_chars(valueStr.data(), valueStr.data() + valueStr.size(), value); - if (ec == std::errc{}) { - originLevelMapping.emplace_back(std::move(key), value); - } else { - LOGP(fatal, "Unable to parse level in aod-origin-level-mapping entry: \"{}\"", pair); - } + if (ctx.options().hasOption("aod-origin-level-map")) { + auto originLevelMap = ctx.options().get>("aod-origin-level-map"); + auto origins = originLevelMap.getLabelsRows(); + for (auto i = 0U; i < originLevelMap.rows(); ++i) { + originLevelMapping.emplace_back(origins[i], originLevelMap.get(i, 0U)); } } auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel, originLevelMapping](ConfigParamRegistry const& options, diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx index cebe968f799e9..b0a5b0b8001cf 100644 --- a/Framework/AnalysisSupport/src/Plugin.cxx +++ b/Framework/AnalysisSupport/src/Plugin.cxx @@ -21,9 +21,13 @@ #include #include #include +#include #include #include +#include #include +#include +#include #include O2_DECLARE_DYNAMIC_LOG(analysis_support); @@ -146,25 +150,27 @@ std::vector getListOfTables(AODInputFile& f) return r; } -auto readMetadata(AODInputFile& currentFile) -> std::vector +// Key / value pairs of the metadata, empty if there is none +auto readMetadata(AODInputFile& currentFile) -> std::map { - // Get the metadata, if any - auto m = currentFile.metaData(); - std::vector results; - - // Serialise metadata into a ; separated string with : separating key and value + auto metadata = currentFile.metaData(); bool first = true; - for (auto& [name, value] : m) { + for (auto const& [name, value] : metadata) { if (first) { LOGP(info, "Metadata for file \"{}\":", currentFile.name()); first = false; } LOGP(info, "- {}: {}", name, value); - std::string key = "aod-metadata-" + name; - char const* valueCopy = strdup(value.c_str()); - results.push_back(ConfigParamSpec{key, VariantType::String, valueCopy, {"Metadata in AOD"}}); } + return metadata; +} +auto metadataToSpecs(std::map const& metadata) -> std::vector +{ + std::vector results; + for (auto const& [name, value] : metadata) { + results.push_back(ConfigParamSpec{"aod-metadata-" + name, VariantType::String, value, {"Metadata in AOD"}}); + } return results; } @@ -176,6 +182,55 @@ std::shared_ptr openForDiscovery(std::string const& filename) return RootAODInputFile::open(filename, noTables); } +struct FileAtLevel { + std::shared_ptr file; // nullptr if there is no file at this level + std::string name; +}; + +// Open the first parent file of `child` +FileAtLevel openParent(FileAtLevel const& child, ConfigParamRegistry& registry) +{ + auto parentFiles = child.file->parentFiles(); + if (parentFiles.empty()) { + LOGP(info, "No parent file found for \"{}\"", child.name); + return {nullptr, child.name}; + } + std::string parentFilename = parentFiles.begin()->second; + // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be + // a workflow option (because the metadata itself is potentially changing the topology). + if (registry.isSet("aod-parent-base-path-replacement")) { + auto parentFileReplacement = registry.get("aod-parent-base-path-replacement"); + auto pos = parentFileReplacement.find(';'); + if (pos == std::string::npos) { + throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str())); + } + auto from = parentFileReplacement.substr(0, pos); + auto to = parentFileReplacement.substr(pos + 1); + // Same as TString::ReplaceAll + for (size_t at = 0; !from.empty() && (at = parentFilename.find(from, at)) != std::string::npos; at += to.size()) { + parentFilename.replace(at, from.size(), to); + } + } + + if (parentFilename.starts_with("alien://") && !gGrid) { + TGrid::Connect("alien://"); + } + + auto parentFile = openForDiscovery(parentFilename); + if (parentFile.get() == nullptr) { + LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename); + } + return {std::move(parentFile), parentFilename}; +} + +// What was found at a given parent level +struct LevelMetadata { + bool found = false; // the file at this level exists + std::string source; + std::map metadata; + std::vector tables; +}; + struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin { ConfigDiscovery* create() override { @@ -204,73 +259,125 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin { if (currentFile.get() == nullptr) { throw std::runtime_error(fmt::format("Couldn't open file \"{}\"!", filename)); } - std::vector results = readMetadata(*currentFile); - const bool metaDataEmpty = results.empty(); - auto tables = getListOfTables(*currentFile); - if (tables.empty() == false) { - results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}}); - } - // Found metadata already in the main file. - if (!metaDataEmpty) { - results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}}); - return results; + // The files opened so far, by level. Ends with an empty file if the chain of parents is exhausted. + std::vector chain{{currentFile, filename}}; + // Each level is read only once, even if several origins map to it + std::map levels; + auto readLevel = [&](int level) -> LevelMetadata const& { + if (auto it = levels.find(level); it != levels.end()) { + return it->second; + } + while (chain.back().file && chain.size() <= static_cast(level)) { + chain.push_back(openParent(chain.back(), registry)); + } + LevelMetadata data; + auto const& atLevel = chain[std::min(static_cast(level), chain.size() - 1)]; + if (atLevel.file) { + data.found = true; + data.source = atLevel.name; + data.metadata = readMetadata(*atLevel.file); + data.tables = getListOfTables(*atLevel.file); + } + return levels.emplace(level, std::move(data)).first->second; + }; + int parentAccessLevel = 0; + if (registry.isSet("aod-parent-access-level")) { + parentAccessLevel = registry.get("aod-parent-access-level"); } - if (!registry.isSet("aod-parent-access-level") || registry.get("aod-parent-access-level") == 0) { + // The plain aod-metadata-* options, from the file itself or from the closest parent which has them + LevelMetadata const* plain = nullptr; + auto const& top = readLevel(0); + std::vector results = metadataToSpecs(top.metadata); + if (!top.tables.empty()) { + results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, top.tables, {"Tables in first AOD"}}); + } + if (!top.metadata.empty()) { + // Found metadata already in the main file. + results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}}); + plain = ⊤ + } else if (parentAccessLevel == 0) { LOGP(info, "No metadata found in file \"{}\" and parent level 0 prevents further lookup.", filename); results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}}); + } else { + // Lets try in the parent files, up to the allowed parent level. + LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename); + for (int level = 1; level <= parentAccessLevel && plain == nullptr; ++level) { + auto const& parent = readLevel(level); + if (!parent.found) { + break; + } + results = metadataToSpecs(parent.metadata); + if (parent.metadata.empty()) { + LOGP(info, "No metadata found in parent file \"{}\" at level {}", parent.source, level); + continue; + } + if (!parent.tables.empty()) { + results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, parent.tables, {"Tables in first AOD"}}); + } + results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}}); + plain = &parent; + } + if (plain == nullptr) { + LOGP(info, "No metadata found in file \"{}\" nor in its parents", filename); + results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}}); + } + } + + if (!registry.hasOption("aod-origin-level-map")) { return results; } - // Lets try in the parent files, up to the allowed parent level. - auto childFile = currentFile; - for (int level = 1; level <= registry.get("aod-parent-access-level"); ++level) { - auto parentFiles = childFile->parentFiles(); - if (parentFiles.empty()) { - break; - } - LOGP(info, "No metadata found in file \"{}\", checking in its parents.", childFile->name()); - std::string parentFilename = parentFiles.begin()->second; - // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be - // a workflow option (because the metadata itself is potentially changing the topology). - if (registry.isSet("aod-parent-base-path-replacement")) { - auto parentFileReplacement = registry.get("aod-parent-base-path-replacement"); - auto pos = parentFileReplacement.find(';'); - if (pos == std::string::npos) { - throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str())); - } - auto from = parentFileReplacement.substr(0, pos); - auto to = parentFileReplacement.substr(pos + 1); - // Same as TString::ReplaceAll - for (size_t at = 0; !from.empty() && (at = parentFilename.find(from, at)) != std::string::npos; at += to.size()) { - parentFilename.replace(at, from.size(), to); + // The metadata of the level each origin reads, one row per origin. The columns are + // all the keys found, including the plain ones, so that an empty cell always means + // that the key is missing for that origin. + auto levelMap = registry.get>("aod-origin-level-map"); + auto origins = levelMap.getLabelsRows(); + std::vector keys; + auto addKeys = [&keys](std::map const& metadata) { + for (auto const& [key, value] : metadata) { + if (std::ranges::find(keys, key) == keys.end()) { + keys.push_back(key); } } - - if (parentFilename.starts_with("alien://") && !gGrid) { - TGrid::Connect("alien://"); + }; + if (plain != nullptr) { + addKeys(plain->metadata); + } + for (auto i = 0U; i < levelMap.rows(); ++i) { + auto level = levelMap.get(i, 0U); + if (level > parentAccessLevel) { + LOGP(fatal, "Origin {} is mapped to parent level {}, but aod-parent-access-level is {}", origins[i], level, parentAccessLevel); } + addKeys(readLevel(level).metadata); + } + auto sourceColumn = keys.size(); + keys.emplace_back("source"); + auto disableColumn = keys.size(); + keys.emplace_back("disable"); - auto parentFile = openForDiscovery(parentFilename); - if (parentFile.get() == nullptr) { - LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename); + std::vector values(origins.size() * keys.size()); + for (auto i = 0U; i < origins.size(); ++i) { + auto const& data = readLevel(levelMap.get(i, 0U)); + auto* row = values.data() + i * keys.size(); + for (auto const& [key, value] : data.metadata) { + row[std::ranges::find(keys, key) - keys.begin()] = value; } - results = readMetadata(*parentFile); - // Found metadata already in the main file. - if (!results.empty()) { - auto tables = getListOfTables(*parentFile); - if (tables.empty() == false) { - results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}}); - } - results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}}); - return results; + row[sourceColumn] = data.source; + if (data.metadata.empty()) { + LOGP(info, "Metadata info missing for origin {}", origins[i]); + row[disableColumn] = "1"; + } + } + LabeledArray originMetadata{values.data(), static_cast(origins.size()), static_cast(keys.size()), origins, keys}; + results.push_back(ConfigParamSpec{"aod-metadata-origins", VariantType::LabeledArrayString, originMetadata, {"Metadata in AOD for each origin: one row per origin, one column per key"}}); + for (auto i = 0U; i < origins.size(); ++i) { + auto const& tables = readLevel(levelMap.get(i, 0U)).tables; + if (!tables.empty()) { + results.push_back(ConfigParamSpec{"aod-metadata-origin-" + origins[i] + "-tables", VariantType::ArrayString, tables, {"Tables in the AOD read for this origin"}}); } - LOGP(info, "No metadata found in parent file \"{}\" at level {}", parentFilename, level); - childFile = parentFile; } - LOGP(info, "No metadata found in file \"{}\" nor in its parents", filename); - results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}}); return results; }}; } diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt index 437dc5ee6478c..862669136624d 100644 --- a/Framework/Core/CMakeLists.txt +++ b/Framework/Core/CMakeLists.txt @@ -215,6 +215,7 @@ add_executable(o2-test-framework-core test/test_ComputingQuotaEvaluator.cxx test/test_Concepts.cxx test/test_ControlServiceHelpers.cxx + test/test_ConfigParamDiscovery.cxx test/test_ConfigParamStore.cxx test/test_ConfigParamRegistry.cxx test/test_DataDescriptorMatcher.cxx diff --git a/Framework/Core/include/Framework/Array2D.h b/Framework/Core/include/Framework/Array2D.h index de877797105c9..4b21ac2ba5b32 100644 --- a/Framework/Core/include/Framework/Array2D.h +++ b/Framework/Core/include/Framework/Array2D.h @@ -241,6 +241,13 @@ class LabeledArray : public LabelMap return values[y]; } + // nullptr if there is no row with this label + T* getRow(const char* y) const + { + auto it = rowmap.find(y); + return it == rowmap.end() ? nullptr : values[it->second]; + } + T* operator[](uint32_t y) const { return values[y]; diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h index 3d13d3662ceb1..c64c4c8255cb9 100644 --- a/Framework/Core/include/Framework/ConfigParamDiscovery.h +++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h @@ -13,6 +13,9 @@ #define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_ #include "Framework/ConfigParamRegistry.h" +#include "Framework/Array2D.h" + +#include namespace o2::framework { @@ -20,6 +23,16 @@ namespace o2::framework // options. struct ConfigParamDiscovery { static std::vector discover(ConfigParamRegistry&, int, char**); + + /// Parse the value of aod-origin-level-mapping, "ORIGIN:LEVEL[,ORIGIN2:LEVEL2,...]", + /// into a LabeledArray with one row per origin and a single "level" column. + /// Throws std::runtime_error if the mapping is malformed. + static LabeledArray parseOriginLevelMapping(std::string_view mapping); + + /// Replace the plain aod-metadata-* options with the ones in the row of + /// aod-metadata-origins which corresponds to the origin this process reads + /// (aod-origin-replace, or AOD). Does nothing if there is no such row. + static void applyOriginMetadata(ConfigParamRegistry& registry); }; struct ConfigDiscovery { diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx index fc8d6f2600bb4..76448ea8222db 100644 --- a/Framework/Core/src/ConfigParamDiscovery.cxx +++ b/Framework/Core/src/ConfigParamDiscovery.cxx @@ -16,6 +16,12 @@ #include #include #include +#include + +#include +#include +#include +#include O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery); @@ -77,7 +83,90 @@ std::vector ConfigParamDiscovery::discover(ConfigParamRegistry& } registry.loadExtra(extras); } + applyOriginMetadata(registry); return result; } +LabeledArray ConfigParamDiscovery::parseOriginLevelMapping(std::string_view mapping) +{ + std::vector origins; + std::vector levels; + for (auto pairRange : mapping | std::views::split(',')) { + std::string_view pair{pairRange.begin(), pairRange.end()}; + auto colonPos = pair.find(':'); + if (colonPos == std::string_view::npos) { + throw std::runtime_error(fmt::format("Badly formatted aod-origin-level-mapping entry: \"{}\"", pair)); + } + std::string origin{pair.substr(0, colonPos)}; + if (origin.empty() || origin.size() > 4) { + throw std::runtime_error(fmt::format("Origin in aod-origin-level-mapping entry must have 1 to 4 characters: \"{}\"", pair)); + } + if (std::ranges::find(origins, origin) != origins.end()) { + throw std::runtime_error(fmt::format("Duplicate origin in aod-origin-level-mapping: \"{}\"", origin)); + } + auto levelStr = pair.substr(colonPos + 1); + int level{}; + auto [ptr, ec] = std::from_chars(levelStr.data(), levelStr.data() + levelStr.size(), level); + if (levelStr.empty() || ec != std::errc{} || ptr != levelStr.data() + levelStr.size() || level < 0) { + throw std::runtime_error(fmt::format("Unable to parse level in aod-origin-level-mapping entry: \"{}\"", pair)); + } + origins.push_back(std::move(origin)); + levels.push_back(level); + } + if (origins.empty()) { + throw std::runtime_error("Empty aod-origin-level-mapping"); + } + return LabeledArray{levels.data(), static_cast(levels.size()), 1, origins, {"level"}}; +} + +void ConfigParamDiscovery::applyOriginMetadata(ConfigParamRegistry& registry) +{ + if (!registry.hasOption("aod-metadata-origins")) { + return; + } + std::string origin = "AOD"; + if (registry.hasOption("aod-origin-replace")) { + auto replace = registry.get("aod-origin-replace"); + if (!replace.empty()) { + origin = replace; + } + } + auto metadata = registry.get>("aod-metadata-origins"); + auto* row = metadata.getRow(origin.c_str()); + if (row == nullptr) { + return; + } + O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery); + auto const& keys = metadata.getLabelsCols(); + auto disable = metadata.colmap.find("disable"); + if (disable != metadata.colmap.end() && row[disable->second] == "1") { + O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "origin metadata", "No metadata for origin %{public}s", origin.c_str()); + for (auto const& key : keys) { + registry.erase(("aod-metadata-" + key).c_str()); + } + registry.erase("aod-metadata-tables"); + registry.override("aod-metadata-disable", std::string{"1"}); + return; + } + O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "origin metadata", "Using metadata for origin %{public}s", origin.c_str()); + for (auto j = 0U; j < metadata.cols(); ++j) { + if (keys[j] == "disable") { + continue; + } + auto key = "aod-metadata-" + keys[j]; + if (row[j].empty()) { + registry.erase(key.c_str()); + } else { + registry.override(key.c_str(), row[j]); + } + } + registry.erase("aod-metadata-disable"); + auto tablesKey = "aod-metadata-origin-" + origin + "-tables"; + if (registry.hasOption(tablesKey.c_str())) { + registry.override("aod-metadata-tables", registry.get>(tablesKey.c_str())); + } else { + registry.erase("aod-metadata-tables"); + } +} + } // namespace o2::framework diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx index 0222b76fc4246..875c2131439e3 100644 --- a/Framework/Core/src/Plugin.cxx +++ b/Framework/Core/src/Plugin.cxx @@ -130,10 +130,14 @@ struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin { std::string value = argv[i + 1]; O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine", "Found %{public}s with value %{public}s.", key.c_str(), value.c_str()); - if (key == "aod-metadata-tables") { + if (key == "aod-metadata-tables" || (key.starts_with("aod-metadata-origin-") && key.ends_with("-tables"))) { std::stringstream is(value); auto arrayValue = VariantJSONHelpers::read(is); results.push_back(ConfigParamSpec{key, VariantType::ArrayString, arrayValue, {"Metadata in command line"}}); + } else if (key == "aod-metadata-origins") { + std::stringstream is(value); + auto arrayValue = VariantJSONHelpers::read(is); + results.push_back(ConfigParamSpec{key, VariantType::LabeledArrayString, arrayValue, {"Metadata in command line"}}); } else { results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}}); } @@ -176,6 +180,13 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin { } if (key == "aod-origin-level-mapping") { results.push_back(ConfigParamSpec{"aod-origin-level-mapping", VariantType::String, value, {"Map origin to parent level for AOD reading. Syntax: ORIGIN:LEVEL[,ORIGIN2:LEVEL2,...]. E.g. \"DYN:1\"."}}); + LabeledArray levelMap; + try { + levelMap = ConfigParamDiscovery::parseOriginLevelMapping(value); + } catch (std::runtime_error const& e) { + LOGP(fatal, "{}", e.what()); + } + results.push_back(ConfigParamSpec{"aod-origin-level-map", VariantType::LabeledArrayInt, levelMap, {"Parsed aod-origin-level-mapping: one row per origin, with its parent level"}}); } } if (injectOption) { diff --git a/Framework/Core/test/test_ConfigParamDiscovery.cxx b/Framework/Core/test/test_ConfigParamDiscovery.cxx new file mode 100644 index 0000000000000..60677284fad78 --- /dev/null +++ b/Framework/Core/test/test_ConfigParamDiscovery.cxx @@ -0,0 +1,145 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include "Framework/ConfigParamDiscovery.h" +#include "Framework/ConfigParamRegistry.h" +#include "Framework/ConfigParamStore.h" +#include "Framework/Array2D.h" + +#include + +using namespace o2::framework; + +TEST_CASE("TestParseOriginLevelMapping") +{ + auto map = ConfigParamDiscovery::parseOriginLevelMapping("EMB:0,AOD:1"); + REQUIRE(map.getLabelsRows() == std::vector{"EMB", "AOD"}); + REQUIRE(map.getLabelsCols() == std::vector{"level"}); + REQUIRE(map.rows() == 2); + REQUIRE(map.cols() == 1); + REQUIRE(map.get(0U, 0U) == 0); + REQUIRE(map.get(1U, 0U) == 1); + REQUIRE(map.getRow("AOD")[0] == 1); + REQUIRE(map.getRow("DYN") == nullptr); + + auto single = ConfigParamDiscovery::parseOriginLevelMapping("DYN:12"); + REQUIRE(single.getLabelsRows() == std::vector{"DYN"}); + REQUIRE(single.get(0U, 0U) == 12); + + for (auto bad : {"", "EMB", "EMB:", ":1", "EMB:x", "EMB:1x", "EMB:-1", "TOOLONG:1", "EMB:0,EMB:1", "EMB:0,"}) { + INFO("mapping: \"" << bad << "\""); + REQUIRE_THROWS_AS(ConfigParamDiscovery::parseOriginLevelMapping(bad), std::runtime_error); + } +} + +namespace +{ +// The plain options as discovered from level 0 (EMB), plus the metadata of each origin: +// AOD reads level 1, XYZ reads a level without metadata. +std::vector makeSpecs(std::string const& replace, bool withOrigins) +{ + static std::vector plainTables{"O2bc", "O2track"}; + static std::vector embTables{"O2bc", "O2track"}; + static std::vector aodTables{"O2bc"}; + static std::string values[3][6] = { + {"MC", "LHC24a2", "x", "", "/path/L0.root", ""}, + {"RAW", "LHC23zzm", "", "y", "/path/L1.root", ""}, + {"", "", "", "", "/path/L2.root", "1"}}; + static LabeledArray origins{&values[0][0], 3, 6, + {"EMB", "AOD", "XYZ"}, + {"DataType", "LPMProductionTag", "OnlyL0", "OnlyL1", "source", "disable"}}; + std::vector specs{ + ConfigParamSpec{"aod-origin-replace", VariantType::String, replace, {"origin"}}, + ConfigParamSpec{"aod-metadata-DataType", VariantType::String, "MC", {"metadata"}}, + ConfigParamSpec{"aod-metadata-LPMProductionTag", VariantType::String, "LHC24a2", {"metadata"}}, + ConfigParamSpec{"aod-metadata-OnlyL0", VariantType::String, "x", {"metadata"}}, + ConfigParamSpec{"aod-metadata-source", VariantType::String, "/path/L0.root", {"metadata"}}, + ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, plainTables, {"metadata"}}, + }; + if (withOrigins) { + specs.push_back(ConfigParamSpec{"aod-metadata-origins", VariantType::LabeledArrayString, origins, {"metadata per origin"}}); + specs.push_back(ConfigParamSpec{"aod-metadata-origin-EMB-tables", VariantType::ArrayString, embTables, {"tables"}}); + specs.push_back(ConfigParamSpec{"aod-metadata-origin-AOD-tables", VariantType::ArrayString, aodTables, {"tables"}}); + } + return specs; +} + +std::unique_ptr makeRegistry(std::vector const& specs) +{ + auto store = std::make_unique(specs, std::vector>{}); + store->preload(); + store->activate(); + auto registry = std::make_unique(std::move(store)); + ConfigParamDiscovery::applyOriginMetadata(*registry); + return registry; +} + +void requireLevel0(ConfigParamRegistry const& registry) +{ + REQUIRE(registry.get("aod-metadata-DataType") == "MC"); + REQUIRE(registry.get("aod-metadata-LPMProductionTag") == "LHC24a2"); + REQUIRE(registry.get("aod-metadata-OnlyL0") == "x"); + REQUIRE(registry.hasOption("aod-metadata-OnlyL1") == false); + REQUIRE(registry.get("aod-metadata-source") == "/path/L0.root"); + REQUIRE(registry.get>("aod-metadata-tables") == std::vector{"O2bc", "O2track"}); + REQUIRE(registry.hasOption("aod-metadata-disable") == false); +} +} // namespace + +TEST_CASE("TestApplyOriginMetadata") +{ + SECTION("replaced origin reading level 0") + { + auto specs = makeSpecs("EMB", true); + auto registry = makeRegistry(specs); + requireLevel0(*registry); + } + + SECTION("AOD reading level 1") + { + auto specs = makeSpecs("", true); + auto registry = makeRegistry(specs); + REQUIRE(registry->get("aod-metadata-DataType") == "RAW"); + REQUIRE(registry->get("aod-metadata-LPMProductionTag") == "LHC23zzm"); + // a key which exists only at level 0 is removed + REQUIRE(registry->hasOption("aod-metadata-OnlyL0") == false); + REQUIRE(registry->get("aod-metadata-OnlyL1") == "y"); + REQUIRE(registry->get("aod-metadata-source") == "/path/L1.root"); + REQUIRE(registry->get>("aod-metadata-tables") == std::vector{"O2bc"}); + REQUIRE(registry->hasOption("aod-metadata-disable") == false); + } + + SECTION("origin without a row") + { + auto specs = makeSpecs("FOO", true); + auto registry = makeRegistry(specs); + requireLevel0(*registry); + } + + SECTION("no metadata per origin") + { + auto specs = makeSpecs("", false); + auto registry = makeRegistry(specs); + requireLevel0(*registry); + } + + SECTION("origin reading a level without metadata") + { + auto specs = makeSpecs("XYZ", true); + auto registry = makeRegistry(specs); + for (auto key : {"aod-metadata-DataType", "aod-metadata-LPMProductionTag", "aod-metadata-OnlyL0", "aod-metadata-OnlyL1", "aod-metadata-source", "aod-metadata-tables"}) { + INFO("key: " << key); + REQUIRE(registry->hasOption(key) == false); + } + REQUIRE(registry->get("aod-metadata-disable") == "1"); + } +} diff --git a/Framework/Core/test/test_DeviceSpecHelpers.cxx b/Framework/Core/test/test_DeviceSpecHelpers.cxx index d689e7a2765c4..bf163f919c795 100644 --- a/Framework/Core/test/test_DeviceSpecHelpers.cxx +++ b/Framework/Core/test/test_DeviceSpecHelpers.cxx @@ -102,12 +102,15 @@ void check(const std::vector& arguments, TEST_CASE("test_prepareArguments") { int lm[1][2] = {{1, 2}}; + std::string om[2][3] = {{"MC", "/path/L0.root", ""}, {"", "/path/L1.root", "1"}}; std::vector workflowOptions{ {"foo", VariantType::String, "bar", {"charlie foxtrot"}}, {"depth", VariantType::Int, 1, {"height"}}, {"labeled", VariantType::LabeledArrayInt, LabeledArray{&lm[0][0], 1, 2, {"r"}, {"c1", "c2"}}, {"a labeled array"}}, + {"aod-metadata-origins", VariantType::LabeledArrayString, LabeledArray{&om[0][0], 2, 3, {"EMB", "AOD"}, {"DataType", "source", "disable"}}, {"metadata per origin"}}, }; std::string labeledDefault = R"({"labels_rows":["r"],"labels_cols":["c1","c2"],"values":[[1,2]]})"; + std::string originsDefault = R"({"labels_rows":["EMB","AOD"],"labels_cols":["DataType","source","disable"],"values":[["MC","/path/L0.root",""],["","/path/L1.root","1"]]})"; std::string labeledOther = R"({"labels_rows":["r"],"labels_cols":["c1","c2"],"values":[[3,4]]})"; auto algorithm = [](ProcessingContext& /*ctx*/) {}; @@ -183,7 +186,7 @@ TEST_CASE("test_prepareArguments") check({"--depth", "2", "--processor0", "--mode silly"}, workflowOptions, deviceSpecs, matrix); // a LabeledArray workflow option is forwarded as a single JSON argument, defaulted or set - matrix["processor0"] = {{"--depth", "1"}, {"--labeled", labeledDefault}}; + matrix["processor0"] = {{"--depth", "1"}, {"--labeled", labeledDefault}, {"--aod-metadata-origins", originsDefault}}; matrix["processor1"] = matrix["processor0"]; check({}, workflowOptions, deviceSpecs, matrix); matrix["processor0"] = {{"--depth", "1"}, {"--labeled", labeledOther}}; diff --git a/Framework/Core/test/test_Variants.cxx b/Framework/Core/test/test_Variants.cxx index 77f6a7d2918e3..bbdb0fff45e52 100644 --- a/Framework/Core/test/test_Variants.cxx +++ b/Framework/Core/test/test_Variants.cxx @@ -387,7 +387,11 @@ TEST_CASE("LabeledArrayTest") REQUIRE(las.get(i, xl[j].c_str()) == las.get(i, j)); REQUIRE(las.get(yl[i].c_str(), j) == las.get(i, j)); } + REQUIRE(laf.getRow(yl[i].c_str()) == laf.getRow(i)); + REQUIRE(las.getRow(yl[i].c_str()) == las.getRow(i)); } + REQUIRE(laf.getRow("r4") == nullptr); + REQUIRE(las.getRow("r4") == nullptr); } TEST_CASE("VariantTreeConversionsTest") From 2d0ffd27a29b94fc378b830e39690fce4521c1f4 Mon Sep 17 00:00:00 2001 From: Anton Alkin Date: Mon, 9 Nov 2026 10:58:37 +0100 Subject: [PATCH 3/3] format --- Framework/Core/test/test_ConfigParamDiscovery.cxx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Framework/Core/test/test_ConfigParamDiscovery.cxx b/Framework/Core/test/test_ConfigParamDiscovery.cxx index 60677284fad78..2bdf5b1e38a7f 100644 --- a/Framework/Core/test/test_ConfigParamDiscovery.cxx +++ b/Framework/Core/test/test_ConfigParamDiscovery.cxx @@ -54,9 +54,7 @@ std::vector makeSpecs(std::string const& replace, bool withOrig {"MC", "LHC24a2", "x", "", "/path/L0.root", ""}, {"RAW", "LHC23zzm", "", "y", "/path/L1.root", ""}, {"", "", "", "", "/path/L2.root", "1"}}; - static LabeledArray origins{&values[0][0], 3, 6, - {"EMB", "AOD", "XYZ"}, - {"DataType", "LPMProductionTag", "OnlyL0", "OnlyL1", "source", "disable"}}; + static LabeledArray origins{&values[0][0], 3, 6, {"EMB", "AOD", "XYZ"}, {"DataType", "LPMProductionTag", "OnlyL0", "OnlyL1", "source", "disable"}}; std::vector specs{ ConfigParamSpec{"aod-origin-replace", VariantType::String, replace, {"origin"}}, ConfigParamSpec{"aod-metadata-DataType", VariantType::String, "MC", {"metadata"}},