Skip to content

DPL Analysis: refactor origin level mapping (reworked) #16086

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aalkin wants to merge 3 commits into
base: dev
Choose a base branch
from
+448 −84
Open

DPL Analysis: refactor origin level mapping (reworked) #16086

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e93c43e
DPL Analysis: fix parent paths replacements
aalkin Nov 9, 2026
b5ed7eb
Make AOD metadata discovery aware of the origin level mapping
aalkin Nov 9, 2026
2d0ffd2
format
aalkin Nov 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 6 additions & 20 deletions Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,10 @@

#include "AODJAlienReaderHelpers.h"
#include <algorithm>
#include <charconv>
#include <cctype>
#include <cstdlib>
#include <exception>
#include <memory>
#include <ranges>
#include <string>
#include <string_view>
#include <vector>
Expand All @@ -27,6 +25,7 @@
#include "Framework/AlgorithmSpec.h"
#include "Framework/ArrowContext.h"
#include "Framework/ConfigParamRegistry.h"
#include "Framework/Array2D.h"
#include "Framework/ControlService.h"
#include "Framework/CallbackService.h"
#include "Framework/EndOfStreamContext.h"
Expand Down Expand Up @@ -156,24 +155,11 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
parentAccessLevel = ctx.options().get<int>("aod-parent-access-level");
}
std::vector<std::pair<std::string, int>> originLevelMapping;
if (ctx.options().isSet("aod-origin-level-mapping")) {
auto originLevelMappingStr = ctx.options().get<std::string>("aod-origin-level-mapping");
for (auto pairRange : originLevelMappingStr | std::views::split(',')) {
std::string_view pair{pairRange.begin(), pairRange.end()};
auto colonPos = pair.find(':');
if (colonPos == std::string_view::npos) {
LOGP(fatal, "Badly formatted aod-origin-level-mapping entry: \"{}\"", pair);
continue;
}
std::string key(pair.substr(0, colonPos));
std::string_view valueStr = pair.substr(colonPos + 1);
int value{};
auto [ptr, ec] = std::from_chars(valueStr.data(), valueStr.data() + valueStr.size(), value);
if (ec == std::errc{}) {
originLevelMapping.emplace_back(std::move(key), value);
} else {
LOGP(fatal, "Unable to parse level in aod-origin-level-mapping entry: \"{}\"", pair);
}
if (ctx.options().hasOption("aod-origin-level-map")) {
auto originLevelMap = ctx.options().get<LabeledArray<int>>("aod-origin-level-map");
auto origins = originLevelMap.getLabelsRows();
for (auto i = 0U; i < originLevelMap.rows(); ++i) {
originLevelMapping.emplace_back(origins[i], originLevelMap.get(i, 0U));
}
}
auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement, parentAccessLevel, originLevelMapping](ConfigParamRegistry const& options,
Expand Down
231 changes: 169 additions & 62 deletions Framework/AnalysisSupport/src/Plugin.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,13 @@
#include <TDataType.h>
#include <TGrid.h>
#include <fmt/format.h>
#include <algorithm>
#include <cstring>
#include <fstream>
#include <map>
#include <memory>
#include <string>
#include <vector>
#include <unistd.h>

O2_DECLARE_DYNAMIC_LOG(analysis_support);
Expand Down Expand Up @@ -146,25 +150,27 @@ std::vector<std::string> getListOfTables(AODInputFile& f)
return r;
}

auto readMetadata(AODInputFile& currentFile) -> std::vector<ConfigParamSpec>
// Key / value pairs of the metadata, empty if there is none
auto readMetadata(AODInputFile& currentFile) -> std::map<std::string, std::string>
{
// Get the metadata, if any
auto m = currentFile.metaData();
std::vector<ConfigParamSpec> results;

// Serialise metadata into a ; separated string with : separating key and value
auto metadata = currentFile.metaData();
bool first = true;
for (auto& [name, value] : m) {
for (auto const& [name, value] : metadata) {
if (first) {
LOGP(info, "Metadata for file \"{}\":", currentFile.name());
first = false;
}
LOGP(info, "- {}: {}", name, value);
std::string key = "aod-metadata-" + name;
char const* valueCopy = strdup(value.c_str());
results.push_back(ConfigParamSpec{key, VariantType::String, valueCopy, {"Metadata in AOD"}});
}
return metadata;
}

auto metadataToSpecs(std::map<std::string, std::string> const& metadata) -> std::vector<ConfigParamSpec>
{
std::vector<ConfigParamSpec> results;
for (auto const& [name, value] : metadata) {
results.push_back(ConfigParamSpec{"aod-metadata-" + name, VariantType::String, value, {"Metadata in AOD"}});
}
return results;
}

Expand All @@ -176,6 +182,55 @@ std::shared_ptr<AODInputFile> openForDiscovery(std::string const& filename)
return RootAODInputFile::open(filename, noTables);
}

struct FileAtLevel {
std::shared_ptr<AODInputFile> file; // nullptr if there is no file at this level
std::string name;
};

// Open the first parent file of `child`
FileAtLevel openParent(FileAtLevel const& child, ConfigParamRegistry& registry)
{
auto parentFiles = child.file->parentFiles();
if (parentFiles.empty()) {
LOGP(info, "No parent file found for \"{}\"", child.name);
return {nullptr, child.name};
}
std::string parentFilename = parentFiles.begin()->second;
// Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
// a workflow option (because the metadata itself is potentially changing the topology).
if (registry.isSet("aod-parent-base-path-replacement")) {
auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
auto pos = parentFileReplacement.find(';');
if (pos == std::string::npos) {
throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
}
auto from = parentFileReplacement.substr(0, pos);
auto to = parentFileReplacement.substr(pos + 1);
// Same as TString::ReplaceAll
for (size_t at = 0; !from.empty() && (at = parentFilename.find(from, at)) != std::string::npos; at += to.size()) {
parentFilename.replace(at, from.size(), to);
}
}

if (parentFilename.starts_with("alien://") && !gGrid) {
TGrid::Connect("alien://");
}

auto parentFile = openForDiscovery(parentFilename);
if (parentFile.get() == nullptr) {
LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
}
return {std::move(parentFile), parentFilename};
}

// What was found at a given parent level
struct LevelMetadata {
bool found = false; // the file at this level exists
std::string source;
std::map<std::string, std::string> metadata;
std::vector<std::string> tables;
};

struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
ConfigDiscovery* create() override
{
Expand Down Expand Up @@ -204,73 +259,125 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
if (currentFile.get() == nullptr) {
throw std::runtime_error(fmt::format("Couldn't open file \"{}\"!", filename));
}
std::vector<ConfigParamSpec> results = readMetadata(*currentFile);
const bool metaDataEmpty = results.empty();
auto tables = getListOfTables(*currentFile);
if (tables.empty() == false) {
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
}

// Found metadata already in the main file.
if (!metaDataEmpty) {
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
return results;
// The files opened so far, by level. Ends with an empty file if the chain of parents is exhausted.
std::vector<FileAtLevel> chain{{currentFile, filename}};
// Each level is read only once, even if several origins map to it
std::map<int, LevelMetadata> levels;
auto readLevel = [&](int level) -> LevelMetadata const& {
if (auto it = levels.find(level); it != levels.end()) {
return it->second;
}
while (chain.back().file && chain.size() <= static_cast<size_t>(level)) {
chain.push_back(openParent(chain.back(), registry));
}
LevelMetadata data;
auto const& atLevel = chain[std::min(static_cast<size_t>(level), chain.size() - 1)];
if (atLevel.file) {
data.found = true;
data.source = atLevel.name;
data.metadata = readMetadata(*atLevel.file);
data.tables = getListOfTables(*atLevel.file);
}
return levels.emplace(level, std::move(data)).first->second;
};
int parentAccessLevel = 0;
if (registry.isSet("aod-parent-access-level")) {
parentAccessLevel = registry.get<int>("aod-parent-access-level");
}

if (!registry.isSet("aod-parent-access-level") || registry.get<int>("aod-parent-access-level") == 0) {
// The plain aod-metadata-* options, from the file itself or from the closest parent which has them
LevelMetadata const* plain = nullptr;
auto const& top = readLevel(0);
std::vector<ConfigParamSpec> results = metadataToSpecs(top.metadata);
if (!top.tables.empty()) {
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, top.tables, {"Tables in first AOD"}});
}
if (!top.metadata.empty()) {
// Found metadata already in the main file.
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
plain = &top;
} else if (parentAccessLevel == 0) {
LOGP(info, "No metadata found in file \"{}\" and parent level 0 prevents further lookup.", filename);
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
} else {
// Lets try in the parent files, up to the allowed parent level.
LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename);
for (int level = 1; level <= parentAccessLevel && plain == nullptr; ++level) {
auto const& parent = readLevel(level);
if (!parent.found) {
break;
}
results = metadataToSpecs(parent.metadata);
if (parent.metadata.empty()) {
LOGP(info, "No metadata found in parent file \"{}\" at level {}", parent.source, level);
continue;
}
if (!parent.tables.empty()) {
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, parent.tables, {"Tables in first AOD"}});
}
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
plain = &parent;
}
if (plain == nullptr) {
LOGP(info, "No metadata found in file \"{}\" nor in its parents", filename);
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
}
}

if (!registry.hasOption("aod-origin-level-map")) {
return results;
}

// Lets try in the parent files, up to the allowed parent level.
auto childFile = currentFile;
for (int level = 1; level <= registry.get<int>("aod-parent-access-level"); ++level) {
auto parentFiles = childFile->parentFiles();
if (parentFiles.empty()) {
break;
}
LOGP(info, "No metadata found in file \"{}\", checking in its parents.", childFile->name());
std::string parentFilename = parentFiles.begin()->second;
// Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
// a workflow option (because the metadata itself is potentially changing the topology).
if (registry.isSet("aod-parent-base-path-replacement")) {
auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
auto pos = parentFileReplacement.find(';');
if (pos == std::string::npos) {
throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
}
auto from = parentFileReplacement.substr(0, pos);
auto to = parentFileReplacement.substr(pos + 1);
pos = parentFilename.find(from);
if (pos != std::string::npos) {
parentFilename.replace(pos, from.length(), to);
// The metadata of the level each origin reads, one row per origin. The columns are
// all the keys found, including the plain ones, so that an empty cell always means
// that the key is missing for that origin.
auto levelMap = registry.get<LabeledArray<int>>("aod-origin-level-map");
auto origins = levelMap.getLabelsRows();
std::vector<std::string> keys;
auto addKeys = [&keys](std::map<std::string, std::string> const& metadata) {
for (auto const& [key, value] : metadata) {
if (std::ranges::find(keys, key) == keys.end()) {
keys.push_back(key);
}
}

if (parentFilename.starts_with("alien://") && !gGrid) {
TGrid::Connect("alien://");
};
if (plain != nullptr) {
addKeys(plain->metadata);
}
for (auto i = 0U; i < levelMap.rows(); ++i) {
auto level = levelMap.get(i, 0U);
if (level > parentAccessLevel) {
LOGP(fatal, "Origin {} is mapped to parent level {}, but aod-parent-access-level is {}", origins[i], level, parentAccessLevel);
}
addKeys(readLevel(level).metadata);
}
auto sourceColumn = keys.size();
keys.emplace_back("source");
auto disableColumn = keys.size();
keys.emplace_back("disable");

auto parentFile = openForDiscovery(parentFilename);
if (parentFile.get() == nullptr) {
LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
std::vector<std::string> values(origins.size() * keys.size());
for (auto i = 0U; i < origins.size(); ++i) {
auto const& data = readLevel(levelMap.get(i, 0U));
auto* row = values.data() + i * keys.size();
for (auto const& [key, value] : data.metadata) {
row[std::ranges::find(keys, key) - keys.begin()] = value;
}
results = readMetadata(*parentFile);
// Found metadata already in the main file.
if (!results.empty()) {
auto tables = getListOfTables(*parentFile);
if (tables.empty() == false) {
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
}
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
return results;
row[sourceColumn] = data.source;
if (data.metadata.empty()) {
LOGP(info, "Metadata info missing for origin {}", origins[i]);
row[disableColumn] = "1";
}
}
LabeledArray<std::string> originMetadata{values.data(), static_cast<uint32_t>(origins.size()), static_cast<uint32_t>(keys.size()), origins, keys};
results.push_back(ConfigParamSpec{"aod-metadata-origins", VariantType::LabeledArrayString, originMetadata, {"Metadata in AOD for each origin: one row per origin, one column per key"}});
for (auto i = 0U; i < origins.size(); ++i) {
auto const& tables = readLevel(levelMap.get(i, 0U)).tables;
if (!tables.empty()) {
results.push_back(ConfigParamSpec{"aod-metadata-origin-" + origins[i] + "-tables", VariantType::ArrayString, tables, {"Tables in the AOD read for this origin"}});
}
LOGP(info, "No metadata found in parent file \"{}\" at level {}", parentFilename, level);
childFile = parentFile;
}
LOGP(info, "No metadata found in file \"{}\" nor in its parents", filename);
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
return results;
}};
}
Expand Down
1 change: 1 addition & 0 deletions Framework/Core/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ add_executable(o2-test-framework-core
test/test_ComputingQuotaEvaluator.cxx
test/test_Concepts.cxx
test/test_ControlServiceHelpers.cxx
test/test_ConfigParamDiscovery.cxx
test/test_ConfigParamStore.cxx
test/test_ConfigParamRegistry.cxx
test/test_DataDescriptorMatcher.cxx
Expand Down
7 changes: 7 additions & 0 deletions Framework/Core/include/Framework/Array2D.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,13 @@ class LabeledArray : public LabelMap
return values[y];
}

// nullptr if there is no row with this label
T* getRow(const char* y) const
{
auto it = rowmap.find(y);
return it == rowmap.end() ? nullptr : values[it->second];
}

T* operator[](uint32_t y) const
{
return values[y];
Expand Down
13 changes: 13 additions & 0 deletions Framework/Core/include/Framework/ConfigParamDiscovery.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,26 @@
#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_

#include "Framework/ConfigParamRegistry.h"
#include "Framework/Array2D.h"

#include <string_view>

namespace o2::framework
{
// A plugin which can be used to inject extra configuration
// options.
struct ConfigParamDiscovery {
static std::vector<ConfigParamSpec> discover(ConfigParamRegistry&, int, char**);

/// Parse the value of aod-origin-level-mapping, "ORIGIN:LEVEL[,ORIGIN2:LEVEL2,...]",
/// into a LabeledArray with one row per origin and a single "level" column.
/// Throws std::runtime_error if the mapping is malformed.
static LabeledArray<int> parseOriginLevelMapping(std::string_view mapping);

/// Replace the plain aod-metadata-* options with the ones in the row of
/// aod-metadata-origins which corresponds to the origin this process reads
/// (aod-origin-replace, or AOD). Does nothing if there is no such row.
static void applyOriginMetadata(ConfigParamRegistry& registry);
};

struct ConfigDiscovery {
Expand Down
Loading
Loading