diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx index 4f0e438651cc2..078bb95b81157 100644 --- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx +++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx @@ -11,12 +11,10 @@ #include "AODJAlienReaderHelpers.h" #include -#include #include #include #include #include -#include #include #include #include @@ -27,6 +25,7 @@ #include "Framework/AlgorithmSpec.h" #include "Framework/ArrowContext.h" #include "Framework/ConfigParamRegistry.h" +#include "Framework/Array2D.h" #include "Framework/ControlService.h" #include "Framework/CallbackService.h" #include "Framework/EndOfStreamContext.h" @@ -156,24 +155,11 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const parentAccessLevel = ctx.options().get("aod-parent-access-level"); } std::vector> originLevelMapping; - if (ctx.options().isSet("aod-origin-level-mapping")) { - auto originLevelMappingStr = ctx.options().get("aod-origin-level-mapping"); - for (auto pairRange : originLevelMappingStr | std::views::split(',')) { - std::string_view pair{pairRange.begin(), pairRange.end()}; - auto colonPos = pair.find(':'); - if (colonPos == std::string_view::npos) { - LOGP(fatal, "Badly formatted aod-origin-level-mapping entry: \"{}\"", pair); - continue; - } - std::string key(pair.substr(0, colonPos)); - std::string_view valueStr = pair.substr(colonPos + 1); - int value{}; - auto [ptr, ec] = std::from_chars(valueStr.data(), valueStr.data() + valueStr.size(), value); - if (ec == std::errc{}) { - originLevelMapping.emplace_back(std::move(key), value); - } else { - LOGP(fatal, "Unable to parse level in aod-origin-level-mapping entry: \"{}\"", pair); - } + if (ctx.options().hasOption("aod-origin-level-map")) { + auto originLevelMap = ctx.options().get>("aod-origin-level-map"); + auto origins = originLevelMap.getLabelsRows(); + for (auto i = 0U; i < originLevelMap.rows(); ++i) { + originLevelMapping.emplace_back(origins[i], originLevelMap.get(i, 0U)); } } auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel, originLevelMapping](ConfigParamRegistry const& options, diff --git a/Framework/AnalysisSupport/src/Plugin.cxx b/Framework/AnalysisSupport/src/Plugin.cxx index 412f00a9eb37e..b0a5b0b8001cf 100644 --- a/Framework/AnalysisSupport/src/Plugin.cxx +++ b/Framework/AnalysisSupport/src/Plugin.cxx @@ -21,9 +21,13 @@ #include #include #include +#include #include #include +#include #include +#include +#include #include O2_DECLARE_DYNAMIC_LOG(analysis_support); @@ -146,25 +150,27 @@ std::vector getListOfTables(AODInputFile& f) return r; } -auto readMetadata(AODInputFile& currentFile) -> std::vector +// Key / value pairs of the metadata, empty if there is none +auto readMetadata(AODInputFile& currentFile) -> std::map { - // Get the metadata, if any - auto m = currentFile.metaData(); - std::vector results; - - // Serialise metadata into a ; separated string with : separating key and value + auto metadata = currentFile.metaData(); bool first = true; - for (auto& [name, value] : m) { + for (auto const& [name, value] : metadata) { if (first) { LOGP(info, "Metadata for file \"{}\":", currentFile.name()); first = false; } LOGP(info, "- {}: {}", name, value); - std::string key = "aod-metadata-" + name; - char const* valueCopy = strdup(value.c_str()); - results.push_back(ConfigParamSpec{key, VariantType::String, valueCopy, {"Metadata in AOD"}}); } + return metadata; +} +auto metadataToSpecs(std::map const& metadata) -> std::vector +{ + std::vector results; + for (auto const& [name, value] : metadata) { + results.push_back(ConfigParamSpec{"aod-metadata-" + name, VariantType::String, value, {"Metadata in AOD"}}); + } return results; } @@ -176,6 +182,55 @@ std::shared_ptr openForDiscovery(std::string const& filename) return RootAODInputFile::open(filename, noTables); } +struct FileAtLevel { + std::shared_ptr file; // nullptr if there is no file at this level + std::string name; +}; + +// Open the first parent file of `child` +FileAtLevel openParent(FileAtLevel const& child, ConfigParamRegistry& registry) +{ + auto parentFiles = child.file->parentFiles(); + if (parentFiles.empty()) { + LOGP(info, "No parent file found for \"{}\"", child.name); + return {nullptr, child.name}; + } + std::string parentFilename = parentFiles.begin()->second; + // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be + // a workflow option (because the metadata itself is potentially changing the topology). + if (registry.isSet("aod-parent-base-path-replacement")) { + auto parentFileReplacement = registry.get("aod-parent-base-path-replacement"); + auto pos = parentFileReplacement.find(';'); + if (pos == std::string::npos) { + throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str())); + } + auto from = parentFileReplacement.substr(0, pos); + auto to = parentFileReplacement.substr(pos + 1); + // Same as TString::ReplaceAll + for (size_t at = 0; !from.empty() && (at = parentFilename.find(from, at)) != std::string::npos; at += to.size()) { + parentFilename.replace(at, from.size(), to); + } + } + + if (parentFilename.starts_with("alien://") && !gGrid) { + TGrid::Connect("alien://"); + } + + auto parentFile = openForDiscovery(parentFilename); + if (parentFile.get() == nullptr) { + LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename); + } + return {std::move(parentFile), parentFilename}; +} + +// What was found at a given parent level +struct LevelMetadata { + bool found = false; // the file at this level exists + std::string source; + std::map metadata; + std::vector tables; +}; + struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin { ConfigDiscovery* create() override { @@ -204,73 +259,125 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin { if (currentFile.get() == nullptr) { throw std::runtime_error(fmt::format("Couldn't open file \"{}\"!", filename)); } - std::vector results = readMetadata(*currentFile); - const bool metaDataEmpty = results.empty(); - auto tables = getListOfTables(*currentFile); - if (tables.empty() == false) { - results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}}); - } - // Found metadata already in the main file. - if (!metaDataEmpty) { - results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}}); - return results; + // The files opened so far, by level. Ends with an empty file if the chain of parents is exhausted. + std::vector chain{{currentFile, filename}}; + // Each level is read only once, even if several origins map to it + std::map levels; + auto readLevel = [&](int level) -> LevelMetadata const& { + if (auto it = levels.find(level); it != levels.end()) { + return it->second; + } + while (chain.back().file && chain.size() <= static_cast(level)) { + chain.push_back(openParent(chain.back(), registry)); + } + LevelMetadata data; + auto const& atLevel = chain[std::min(static_cast(level), chain.size() - 1)]; + if (atLevel.file) { + data.found = true; + data.source = atLevel.name; + data.metadata = readMetadata(*atLevel.file); + data.tables = getListOfTables(*atLevel.file); + } + return levels.emplace(level, std::move(data)).first->second; + }; + int parentAccessLevel = 0; + if (registry.isSet("aod-parent-access-level")) { + parentAccessLevel = registry.get("aod-parent-access-level"); } - if (!registry.isSet("aod-parent-access-level") || registry.get("aod-parent-access-level") == 0) { + // The plain aod-metadata-* options, from the file itself or from the closest parent which has them + LevelMetadata const* plain = nullptr; + auto const& top = readLevel(0); + std::vector results = metadataToSpecs(top.metadata); + if (!top.tables.empty()) { + results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, top.tables, {"Tables in first AOD"}}); + } + if (!top.metadata.empty()) { + // Found metadata already in the main file. + results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}}); + plain = ⊤ + } else if (parentAccessLevel == 0) { LOGP(info, "No metadata found in file \"{}\" and parent level 0 prevents further lookup.", filename); results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}}); + } else { + // Lets try in the parent files, up to the allowed parent level. + LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename); + for (int level = 1; level <= parentAccessLevel && plain == nullptr; ++level) { + auto const& parent = readLevel(level); + if (!parent.found) { + break; + } + results = metadataToSpecs(parent.metadata); + if (parent.metadata.empty()) { + LOGP(info, "No metadata found in parent file \"{}\" at level {}", parent.source, level); + continue; + } + if (!parent.tables.empty()) { + results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, parent.tables, {"Tables in first AOD"}}); + } + results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}}); + plain = &parent; + } + if (plain == nullptr) { + LOGP(info, "No metadata found in file \"{}\" nor in its parents", filename); + results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}}); + } + } + + if (!registry.hasOption("aod-origin-level-map")) { return results; } - // Lets try in the parent files, up to the allowed parent level. - auto childFile = currentFile; - for (int level = 1; level <= registry.get("aod-parent-access-level"); ++level) { - auto parentFiles = childFile->parentFiles(); - if (parentFiles.empty()) { - break; - } - LOGP(info, "No metadata found in file \"{}\", checking in its parents.", childFile->name()); - std::string parentFilename = parentFiles.begin()->second; - // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be - // a workflow option (because the metadata itself is potentially changing the topology). - if (registry.isSet("aod-parent-base-path-replacement")) { - auto parentFileReplacement = registry.get("aod-parent-base-path-replacement"); - auto pos = parentFileReplacement.find(';'); - if (pos == std::string::npos) { - throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str())); - } - auto from = parentFileReplacement.substr(0, pos); - auto to = parentFileReplacement.substr(pos + 1); - pos = parentFilename.find(from); - if (pos != std::string::npos) { - parentFilename.replace(pos, from.length(), to); + // The metadata of the level each origin reads, one row per origin. The columns are + // all the keys found, including the plain ones, so that an empty cell always means + // that the key is missing for that origin. + auto levelMap = registry.get>("aod-origin-level-map"); + auto origins = levelMap.getLabelsRows(); + std::vector keys; + auto addKeys = [&keys](std::map const& metadata) { + for (auto const& [key, value] : metadata) { + if (std::ranges::find(keys, key) == keys.end()) { + keys.push_back(key); } } - - if (parentFilename.starts_with("alien://") && !gGrid) { - TGrid::Connect("alien://"); + }; + if (plain != nullptr) { + addKeys(plain->metadata); + } + for (auto i = 0U; i < levelMap.rows(); ++i) { + auto level = levelMap.get(i, 0U); + if (level > parentAccessLevel) { + LOGP(fatal, "Origin {} is mapped to parent level {}, but aod-parent-access-level is {}", origins[i], level, parentAccessLevel); } + addKeys(readLevel(level).metadata); + } + auto sourceColumn = keys.size(); + keys.emplace_back("source"); + auto disableColumn = keys.size(); + keys.emplace_back("disable"); - auto parentFile = openForDiscovery(parentFilename); - if (parentFile.get() == nullptr) { - LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename); + std::vector values(origins.size() * keys.size()); + for (auto i = 0U; i < origins.size(); ++i) { + auto const& data = readLevel(levelMap.get(i, 0U)); + auto* row = values.data() + i * keys.size(); + for (auto const& [key, value] : data.metadata) { + row[std::ranges::find(keys, key) - keys.begin()] = value; } - results = readMetadata(*parentFile); - // Found metadata already in the main file. - if (!results.empty()) { - auto tables = getListOfTables(*parentFile); - if (tables.empty() == false) { - results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}}); - } - results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}}); - return results; + row[sourceColumn] = data.source; + if (data.metadata.empty()) { + LOGP(info, "Metadata info missing for origin {}", origins[i]); + row[disableColumn] = "1"; + } + } + LabeledArray originMetadata{values.data(), static_cast(origins.size()), static_cast(keys.size()), origins, keys}; + results.push_back(ConfigParamSpec{"aod-metadata-origins", VariantType::LabeledArrayString, originMetadata, {"Metadata in AOD for each origin: one row per origin, one column per key"}}); + for (auto i = 0U; i < origins.size(); ++i) { + auto const& tables = readLevel(levelMap.get(i, 0U)).tables; + if (!tables.empty()) { + results.push_back(ConfigParamSpec{"aod-metadata-origin-" + origins[i] + "-tables", VariantType::ArrayString, tables, {"Tables in the AOD read for this origin"}}); } - LOGP(info, "No metadata found in parent file \"{}\" at level {}", parentFilename, level); - childFile = parentFile; } - LOGP(info, "No metadata found in file \"{}\" nor in its parents", filename); - results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}}); return results; }}; } diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt index 437dc5ee6478c..862669136624d 100644 --- a/Framework/Core/CMakeLists.txt +++ b/Framework/Core/CMakeLists.txt @@ -215,6 +215,7 @@ add_executable(o2-test-framework-core test/test_ComputingQuotaEvaluator.cxx test/test_Concepts.cxx test/test_ControlServiceHelpers.cxx + test/test_ConfigParamDiscovery.cxx test/test_ConfigParamStore.cxx test/test_ConfigParamRegistry.cxx test/test_DataDescriptorMatcher.cxx diff --git a/Framework/Core/include/Framework/Array2D.h b/Framework/Core/include/Framework/Array2D.h index de877797105c9..4b21ac2ba5b32 100644 --- a/Framework/Core/include/Framework/Array2D.h +++ b/Framework/Core/include/Framework/Array2D.h @@ -241,6 +241,13 @@ class LabeledArray : public LabelMap return values[y]; } + // nullptr if there is no row with this label + T* getRow(const char* y) const + { + auto it = rowmap.find(y); + return it == rowmap.end() ? nullptr : values[it->second]; + } + T* operator[](uint32_t y) const { return values[y]; diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h index 3d13d3662ceb1..c64c4c8255cb9 100644 --- a/Framework/Core/include/Framework/ConfigParamDiscovery.h +++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h @@ -13,6 +13,9 @@ #define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_ #include "Framework/ConfigParamRegistry.h" +#include "Framework/Array2D.h" + +#include namespace o2::framework { @@ -20,6 +23,16 @@ namespace o2::framework // options. struct ConfigParamDiscovery { static std::vector discover(ConfigParamRegistry&, int, char**); + + /// Parse the value of aod-origin-level-mapping, "ORIGIN:LEVEL[,ORIGIN2:LEVEL2,...]", + /// into a LabeledArray with one row per origin and a single "level" column. + /// Throws std::runtime_error if the mapping is malformed. + static LabeledArray parseOriginLevelMapping(std::string_view mapping); + + /// Replace the plain aod-metadata-* options with the ones in the row of + /// aod-metadata-origins which corresponds to the origin this process reads + /// (aod-origin-replace, or AOD). Does nothing if there is no such row. + static void applyOriginMetadata(ConfigParamRegistry& registry); }; struct ConfigDiscovery { diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx index fc8d6f2600bb4..76448ea8222db 100644 --- a/Framework/Core/src/ConfigParamDiscovery.cxx +++ b/Framework/Core/src/ConfigParamDiscovery.cxx @@ -16,6 +16,12 @@ #include #include #include +#include + +#include +#include +#include +#include O2_DECLARE_DYNAMIC_LOG(config_parameter_discovery); @@ -77,7 +83,90 @@ std::vector ConfigParamDiscovery::discover(ConfigParamRegistry& } registry.loadExtra(extras); } + applyOriginMetadata(registry); return result; } +LabeledArray ConfigParamDiscovery::parseOriginLevelMapping(std::string_view mapping) +{ + std::vector origins; + std::vector levels; + for (auto pairRange : mapping | std::views::split(',')) { + std::string_view pair{pairRange.begin(), pairRange.end()}; + auto colonPos = pair.find(':'); + if (colonPos == std::string_view::npos) { + throw std::runtime_error(fmt::format("Badly formatted aod-origin-level-mapping entry: \"{}\"", pair)); + } + std::string origin{pair.substr(0, colonPos)}; + if (origin.empty() || origin.size() > 4) { + throw std::runtime_error(fmt::format("Origin in aod-origin-level-mapping entry must have 1 to 4 characters: \"{}\"", pair)); + } + if (std::ranges::find(origins, origin) != origins.end()) { + throw std::runtime_error(fmt::format("Duplicate origin in aod-origin-level-mapping: \"{}\"", origin)); + } + auto levelStr = pair.substr(colonPos + 1); + int level{}; + auto [ptr, ec] = std::from_chars(levelStr.data(), levelStr.data() + levelStr.size(), level); + if (levelStr.empty() || ec != std::errc{} || ptr != levelStr.data() + levelStr.size() || level < 0) { + throw std::runtime_error(fmt::format("Unable to parse level in aod-origin-level-mapping entry: \"{}\"", pair)); + } + origins.push_back(std::move(origin)); + levels.push_back(level); + } + if (origins.empty()) { + throw std::runtime_error("Empty aod-origin-level-mapping"); + } + return LabeledArray{levels.data(), static_cast(levels.size()), 1, origins, {"level"}}; +} + +void ConfigParamDiscovery::applyOriginMetadata(ConfigParamRegistry& registry) +{ + if (!registry.hasOption("aod-metadata-origins")) { + return; + } + std::string origin = "AOD"; + if (registry.hasOption("aod-origin-replace")) { + auto replace = registry.get("aod-origin-replace"); + if (!replace.empty()) { + origin = replace; + } + } + auto metadata = registry.get>("aod-metadata-origins"); + auto* row = metadata.getRow(origin.c_str()); + if (row == nullptr) { + return; + } + O2_SIGNPOST_ID_GENERATE(sid, config_parameter_discovery); + auto const& keys = metadata.getLabelsCols(); + auto disable = metadata.colmap.find("disable"); + if (disable != metadata.colmap.end() && row[disable->second] == "1") { + O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "origin metadata", "No metadata for origin %{public}s", origin.c_str()); + for (auto const& key : keys) { + registry.erase(("aod-metadata-" + key).c_str()); + } + registry.erase("aod-metadata-tables"); + registry.override("aod-metadata-disable", std::string{"1"}); + return; + } + O2_SIGNPOST_EVENT_EMIT(config_parameter_discovery, sid, "origin metadata", "Using metadata for origin %{public}s", origin.c_str()); + for (auto j = 0U; j < metadata.cols(); ++j) { + if (keys[j] == "disable") { + continue; + } + auto key = "aod-metadata-" + keys[j]; + if (row[j].empty()) { + registry.erase(key.c_str()); + } else { + registry.override(key.c_str(), row[j]); + } + } + registry.erase("aod-metadata-disable"); + auto tablesKey = "aod-metadata-origin-" + origin + "-tables"; + if (registry.hasOption(tablesKey.c_str())) { + registry.override("aod-metadata-tables", registry.get>(tablesKey.c_str())); + } else { + registry.erase("aod-metadata-tables"); + } +} + } // namespace o2::framework diff --git a/Framework/Core/src/Plugin.cxx b/Framework/Core/src/Plugin.cxx index 0222b76fc4246..875c2131439e3 100644 --- a/Framework/Core/src/Plugin.cxx +++ b/Framework/Core/src/Plugin.cxx @@ -130,10 +130,14 @@ struct DiscoverMetadataInCommandLine : o2::framework::ConfigDiscoveryPlugin { std::string value = argv[i + 1]; O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverMetadataInCommandLine", "Found %{public}s with value %{public}s.", key.c_str(), value.c_str()); - if (key == "aod-metadata-tables") { + if (key == "aod-metadata-tables" || (key.starts_with("aod-metadata-origin-") && key.ends_with("-tables"))) { std::stringstream is(value); auto arrayValue = VariantJSONHelpers::read(is); results.push_back(ConfigParamSpec{key, VariantType::ArrayString, arrayValue, {"Metadata in command line"}}); + } else if (key == "aod-metadata-origins") { + std::stringstream is(value); + auto arrayValue = VariantJSONHelpers::read(is); + results.push_back(ConfigParamSpec{key, VariantType::LabeledArrayString, arrayValue, {"Metadata in command line"}}); } else { results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in command line"}}); } @@ -176,6 +180,13 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin { } if (key == "aod-origin-level-mapping") { results.push_back(ConfigParamSpec{"aod-origin-level-mapping", VariantType::String, value, {"Map origin to parent level for AOD reading. Syntax: ORIGIN:LEVEL[,ORIGIN2:LEVEL2,...]. E.g. \"DYN:1\"."}}); + LabeledArray levelMap; + try { + levelMap = ConfigParamDiscovery::parseOriginLevelMapping(value); + } catch (std::runtime_error const& e) { + LOGP(fatal, "{}", e.what()); + } + results.push_back(ConfigParamSpec{"aod-origin-level-map", VariantType::LabeledArrayInt, levelMap, {"Parsed aod-origin-level-mapping: one row per origin, with its parent level"}}); } } if (injectOption) { diff --git a/Framework/Core/test/test_ConfigParamDiscovery.cxx b/Framework/Core/test/test_ConfigParamDiscovery.cxx new file mode 100644 index 0000000000000..2bdf5b1e38a7f --- /dev/null +++ b/Framework/Core/test/test_ConfigParamDiscovery.cxx @@ -0,0 +1,143 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include +#include "Framework/ConfigParamDiscovery.h" +#include "Framework/ConfigParamRegistry.h" +#include "Framework/ConfigParamStore.h" +#include "Framework/Array2D.h" + +#include + +using namespace o2::framework; + +TEST_CASE("TestParseOriginLevelMapping") +{ + auto map = ConfigParamDiscovery::parseOriginLevelMapping("EMB:0,AOD:1"); + REQUIRE(map.getLabelsRows() == std::vector{"EMB", "AOD"}); + REQUIRE(map.getLabelsCols() == std::vector{"level"}); + REQUIRE(map.rows() == 2); + REQUIRE(map.cols() == 1); + REQUIRE(map.get(0U, 0U) == 0); + REQUIRE(map.get(1U, 0U) == 1); + REQUIRE(map.getRow("AOD")[0] == 1); + REQUIRE(map.getRow("DYN") == nullptr); + + auto single = ConfigParamDiscovery::parseOriginLevelMapping("DYN:12"); + REQUIRE(single.getLabelsRows() == std::vector{"DYN"}); + REQUIRE(single.get(0U, 0U) == 12); + + for (auto bad : {"", "EMB", "EMB:", ":1", "EMB:x", "EMB:1x", "EMB:-1", "TOOLONG:1", "EMB:0,EMB:1", "EMB:0,"}) { + INFO("mapping: \"" << bad << "\""); + REQUIRE_THROWS_AS(ConfigParamDiscovery::parseOriginLevelMapping(bad), std::runtime_error); + } +} + +namespace +{ +// The plain options as discovered from level 0 (EMB), plus the metadata of each origin: +// AOD reads level 1, XYZ reads a level without metadata. +std::vector makeSpecs(std::string const& replace, bool withOrigins) +{ + static std::vector plainTables{"O2bc", "O2track"}; + static std::vector embTables{"O2bc", "O2track"}; + static std::vector aodTables{"O2bc"}; + static std::string values[3][6] = { + {"MC", "LHC24a2", "x", "", "/path/L0.root", ""}, + {"RAW", "LHC23zzm", "", "y", "/path/L1.root", ""}, + {"", "", "", "", "/path/L2.root", "1"}}; + static LabeledArray origins{&values[0][0], 3, 6, {"EMB", "AOD", "XYZ"}, {"DataType", "LPMProductionTag", "OnlyL0", "OnlyL1", "source", "disable"}}; + std::vector specs{ + ConfigParamSpec{"aod-origin-replace", VariantType::String, replace, {"origin"}}, + ConfigParamSpec{"aod-metadata-DataType", VariantType::String, "MC", {"metadata"}}, + ConfigParamSpec{"aod-metadata-LPMProductionTag", VariantType::String, "LHC24a2", {"metadata"}}, + ConfigParamSpec{"aod-metadata-OnlyL0", VariantType::String, "x", {"metadata"}}, + ConfigParamSpec{"aod-metadata-source", VariantType::String, "/path/L0.root", {"metadata"}}, + ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, plainTables, {"metadata"}}, + }; + if (withOrigins) { + specs.push_back(ConfigParamSpec{"aod-metadata-origins", VariantType::LabeledArrayString, origins, {"metadata per origin"}}); + specs.push_back(ConfigParamSpec{"aod-metadata-origin-EMB-tables", VariantType::ArrayString, embTables, {"tables"}}); + specs.push_back(ConfigParamSpec{"aod-metadata-origin-AOD-tables", VariantType::ArrayString, aodTables, {"tables"}}); + } + return specs; +} + +std::unique_ptr makeRegistry(std::vector const& specs) +{ + auto store = std::make_unique(specs, std::vector>{}); + store->preload(); + store->activate(); + auto registry = std::make_unique(std::move(store)); + ConfigParamDiscovery::applyOriginMetadata(*registry); + return registry; +} + +void requireLevel0(ConfigParamRegistry const& registry) +{ + REQUIRE(registry.get("aod-metadata-DataType") == "MC"); + REQUIRE(registry.get("aod-metadata-LPMProductionTag") == "LHC24a2"); + REQUIRE(registry.get("aod-metadata-OnlyL0") == "x"); + REQUIRE(registry.hasOption("aod-metadata-OnlyL1") == false); + REQUIRE(registry.get("aod-metadata-source") == "/path/L0.root"); + REQUIRE(registry.get>("aod-metadata-tables") == std::vector{"O2bc", "O2track"}); + REQUIRE(registry.hasOption("aod-metadata-disable") == false); +} +} // namespace + +TEST_CASE("TestApplyOriginMetadata") +{ + SECTION("replaced origin reading level 0") + { + auto specs = makeSpecs("EMB", true); + auto registry = makeRegistry(specs); + requireLevel0(*registry); + } + + SECTION("AOD reading level 1") + { + auto specs = makeSpecs("", true); + auto registry = makeRegistry(specs); + REQUIRE(registry->get("aod-metadata-DataType") == "RAW"); + REQUIRE(registry->get("aod-metadata-LPMProductionTag") == "LHC23zzm"); + // a key which exists only at level 0 is removed + REQUIRE(registry->hasOption("aod-metadata-OnlyL0") == false); + REQUIRE(registry->get("aod-metadata-OnlyL1") == "y"); + REQUIRE(registry->get("aod-metadata-source") == "/path/L1.root"); + REQUIRE(registry->get>("aod-metadata-tables") == std::vector{"O2bc"}); + REQUIRE(registry->hasOption("aod-metadata-disable") == false); + } + + SECTION("origin without a row") + { + auto specs = makeSpecs("FOO", true); + auto registry = makeRegistry(specs); + requireLevel0(*registry); + } + + SECTION("no metadata per origin") + { + auto specs = makeSpecs("", false); + auto registry = makeRegistry(specs); + requireLevel0(*registry); + } + + SECTION("origin reading a level without metadata") + { + auto specs = makeSpecs("XYZ", true); + auto registry = makeRegistry(specs); + for (auto key : {"aod-metadata-DataType", "aod-metadata-LPMProductionTag", "aod-metadata-OnlyL0", "aod-metadata-OnlyL1", "aod-metadata-source", "aod-metadata-tables"}) { + INFO("key: " << key); + REQUIRE(registry->hasOption(key) == false); + } + REQUIRE(registry->get("aod-metadata-disable") == "1"); + } +} diff --git a/Framework/Core/test/test_DeviceSpecHelpers.cxx b/Framework/Core/test/test_DeviceSpecHelpers.cxx index d689e7a2765c4..bf163f919c795 100644 --- a/Framework/Core/test/test_DeviceSpecHelpers.cxx +++ b/Framework/Core/test/test_DeviceSpecHelpers.cxx @@ -102,12 +102,15 @@ void check(const std::vector& arguments, TEST_CASE("test_prepareArguments") { int lm[1][2] = {{1, 2}}; + std::string om[2][3] = {{"MC", "/path/L0.root", ""}, {"", "/path/L1.root", "1"}}; std::vector workflowOptions{ {"foo", VariantType::String, "bar", {"charlie foxtrot"}}, {"depth", VariantType::Int, 1, {"height"}}, {"labeled", VariantType::LabeledArrayInt, LabeledArray{&lm[0][0], 1, 2, {"r"}, {"c1", "c2"}}, {"a labeled array"}}, + {"aod-metadata-origins", VariantType::LabeledArrayString, LabeledArray{&om[0][0], 2, 3, {"EMB", "AOD"}, {"DataType", "source", "disable"}}, {"metadata per origin"}}, }; std::string labeledDefault = R"({"labels_rows":["r"],"labels_cols":["c1","c2"],"values":[[1,2]]})"; + std::string originsDefault = R"({"labels_rows":["EMB","AOD"],"labels_cols":["DataType","source","disable"],"values":[["MC","/path/L0.root",""],["","/path/L1.root","1"]]})"; std::string labeledOther = R"({"labels_rows":["r"],"labels_cols":["c1","c2"],"values":[[3,4]]})"; auto algorithm = [](ProcessingContext& /*ctx*/) {}; @@ -183,7 +186,7 @@ TEST_CASE("test_prepareArguments") check({"--depth", "2", "--processor0", "--mode silly"}, workflowOptions, deviceSpecs, matrix); // a LabeledArray workflow option is forwarded as a single JSON argument, defaulted or set - matrix["processor0"] = {{"--depth", "1"}, {"--labeled", labeledDefault}}; + matrix["processor0"] = {{"--depth", "1"}, {"--labeled", labeledDefault}, {"--aod-metadata-origins", originsDefault}}; matrix["processor1"] = matrix["processor0"]; check({}, workflowOptions, deviceSpecs, matrix); matrix["processor0"] = {{"--depth", "1"}, {"--labeled", labeledOther}}; diff --git a/Framework/Core/test/test_Variants.cxx b/Framework/Core/test/test_Variants.cxx index 77f6a7d2918e3..bbdb0fff45e52 100644 --- a/Framework/Core/test/test_Variants.cxx +++ b/Framework/Core/test/test_Variants.cxx @@ -387,7 +387,11 @@ TEST_CASE("LabeledArrayTest") REQUIRE(las.get(i, xl[j].c_str()) == las.get(i, j)); REQUIRE(las.get(yl[i].c_str(), j) == las.get(i, j)); } + REQUIRE(laf.getRow(yl[i].c_str()) == laf.getRow(i)); + REQUIRE(las.getRow(yl[i].c_str()) == las.getRow(i)); } + REQUIRE(laf.getRow("r4") == nullptr); + REQUIRE(las.getRow("r4") == nullptr); } TEST_CASE("VariantTreeConversionsTest")