Skip to content

Commit 77d49c5

Browse files
committed
WIP QC-1251 possible solution for postprocessing without accessing DB
This proposes "late tasks", which can take as input merged MOs and any QOs within the same workflow. See JIRA for more deliberations about why it's probably the best approach. Definitely not finished, I'm just saving the progress and making it available for early review.
1 parent 986040f commit 77d49c5

File tree

3 files changed

+264
-0
lines changed

3 files changed

+264
-0
lines changed
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{
2+
"qc": {
3+
"config": {
4+
"database": {
5+
"implementation": "CCDB",
6+
"host": "ccdb-test.cern.ch:8080"
7+
},
8+
"Activity": {
9+
"number": "42",
10+
"type": "NONE",
11+
"provenance": "qc_async"
12+
},
13+
"monitoring": {
14+
"url": "infologger:///debug?qc"
15+
},
16+
"consul": {
17+
"url": ""
18+
},
19+
"conditionDB": {
20+
"url": "ccdb-test.cern.ch:8080"
21+
},
22+
"infologger": {
23+
"filterDiscardDebug": "false",
24+
"filterDiscardLevel": "12"
25+
},
26+
"bookkeeping": {
27+
"url": ""
28+
}
29+
},
30+
"tasks": {},
31+
"checks": {},
32+
"lateTasks": {
33+
"QcTaskLate": { "": "this is a way to configure an asynchronous late task",
34+
"active": "true",
35+
"className": "o2::quality_control::core::GeneralPurposeLateTask",
36+
"moduleName": "QualityControl",
37+
"detectorName": "TST",
38+
"policy": "OnAll", "": "just like Checks and Aggregators, late tasks support update policies",
39+
"dataSources": [{ "" : "provides the late task with an MO",
40+
"type": "repository",
41+
"detectorName": "TST",
42+
"taskName": "QcTask",
43+
"MOs": ["example", "example2"], "": "framework should attempt to retrieve the two objects corresponding to the same validity",
44+
"restrict": {
45+
"number": "", "": "takes any run number",
46+
"periodName": "LHC23k", "": "objects should belong to period LHC23k",
47+
"passName": "derived", "": "objects should belongs to the same pass name as the globally defined one in this file"
48+
}
49+
}],
50+
"TODO": "how to describe expected plots? i guess this should be task-specific"
51+
}
52+
}
53+
},
54+
"dataSamplingPolicies": []
55+
}

Framework/basic-latetask.json

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
{
2+
"qc": {
3+
"config": {
4+
"database": {
5+
"implementation": "CCDB",
6+
"host": "ccdb-test.cern.ch:8080"
7+
},
8+
"Activity": {
9+
"number": "42",
10+
"type": "NONE"
11+
},
12+
"monitoring": {
13+
"url": "infologger:///debug?qc"
14+
},
15+
"consul": {
16+
"url": ""
17+
},
18+
"conditionDB": {
19+
"url": "ccdb-test.cern.ch:8080"
20+
},
21+
"infologger": {
22+
"filterDiscardDebug": "false",
23+
"filterDiscardLevel": "12"
24+
},
25+
"bookkeeping": {
26+
"url": ""
27+
}
28+
},
29+
"tasks": {
30+
"QcTask": {
31+
"active": "true",
32+
"className": "o2::quality_control_modules::skeleton::SkeletonTask",
33+
"moduleName": "QcSkeleton",
34+
"detectorName": "TST",
35+
"cycleDurations": [
36+
{"cycleDurationSeconds": 10, "validitySeconds": 35},
37+
{"cycleDurationSeconds": 12, "validitySeconds": 1}
38+
],
39+
"dataSource": {
40+
"type": "dataSamplingPolicy",
41+
"name": "tst-raw"
42+
}
43+
}
44+
},
45+
"checks": {
46+
"QcCheck": {
47+
"active": "true",
48+
"className": "o2::quality_control_modules::skeleton::SkeletonCheck",
49+
"moduleName": "QcSkeleton",
50+
"policy": "OnAny",
51+
"detectorName": "TST",
52+
"dataSource": [{
53+
"type": "Task",
54+
"name": "QcTask",
55+
"MOs": ["example"]
56+
}]
57+
}
58+
},
59+
"lateTasks": {
60+
"QcTaskLate": {
61+
"active": "true",
62+
"className": "o2::quality_control::core::GeneralPurposeLateTask",
63+
"moduleName": "QualityControl",
64+
"detectorName": "TST",
65+
"policy": "OnAll", "": "just like Checks and Aggregators, late tasks support update policies",
66+
"dataSources": [{ "" : "provides the late task with an MO",
67+
"type": "Task",
68+
"name": "QcTask",
69+
"MOs": ["example"]
70+
}, {
71+
"type": "TaskReduced", "": "provides the late task with a bunch of statistics derived from declared object(s)",
72+
"name": "QcTask",
73+
"MOs": ["example"],
74+
"reducerName": "o2::quality_control_modules::common::TH1Reducer",
75+
"moduleName": "QcCommon"
76+
}],
77+
"TODO": "how to describe expected plots? i guess this should be task-specific"
78+
}
79+
}
80+
},
81+
"dataSamplingPolicies": [
82+
{
83+
"id": "tst-raw",
84+
"active": "true",
85+
"machines": [],
86+
"query": "data:TST/RAWDATA/0",
87+
"samplingConditions": [
88+
{
89+
"condition": "random",
90+
"fraction": "0.1",
91+
"seed": "1234"
92+
}
93+
]
94+
}
95+
]
96+
}
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
///
13+
/// \file LateTaskInterface.h
14+
/// \author Piotr Konopka
15+
///
16+
17+
#ifndef LATETASKINTERFACE_H
18+
#define LATETASKINTERFACE_H
19+
20+
#include <memory>
21+
// O2
22+
#include <Framework/InitContext.h>
23+
#include <Framework/ProcessingContext.h>
24+
// QC
25+
#include "QualityControl/Activity.h"
26+
#include "QualityControl/ObjectsManager.h"
27+
#include "QualityControl/UserCodeInterface.h"
28+
#include "QualityControl/MonitorObject.h"
29+
#include "QualityControl/QualityObject.h"
30+
31+
namespace o2::monitoring
32+
{
33+
class Monitoring;
34+
}
35+
36+
//namespace o2::globaltracking
37+
//{
38+
//struct DataRequest;
39+
//}
40+
41+
namespace o2::framework
42+
{
43+
struct ConcreteDataMatcher;
44+
}
45+
46+
namespace o2::quality_control::core
47+
{
48+
49+
/// \brief Skeleton of a late QC task.
50+
///
51+
/// Purely abstract class defining the skeleton and the common interface of a late QC task.
52+
/// It is therefore the parent class of any late QC task.
53+
/// It is responsible for the instantiation, modification and destruction of the TObjects that are published.
54+
///
55+
/// Late tasks can process any output of a Task, Check or Aggregator and produce new MonitorObjects.
56+
/// In a multinode setup, they always run on remote (QC) nodes, so they can access merged MonitorObjects and any
57+
/// QualityObjects. Thus, it is not possible to run late Tasks on FLPs or EPNs.
58+
/// In async QC, late Tasks can be used in combination with a QCDB reader (not implemented yet) to perform
59+
/// any trends or correlations on series of objects which are available only in the QCDB.
60+
///
61+
/// TODO: one could even consider allowing to feed late tasks with output of Reductors.
62+
/// It could be an opportunity to refactor them as well (and rename them to Reducers, which sounds more like English).
63+
/// TODO: to allow for more structured configuration, i see no other choice than providing the user with the late task config tree.
64+
/// CustomParameters do not support tree-like structures. one could consider extending it, but i'm not sure if we can be fully backwards compatible.
65+
/// TODO: think how to allow to produce new plots after each `process()` in sync, while producing just one at the end for async.
66+
/// \author Piotr Konopka
67+
class LateTaskInterface : public UserCodeInterface
68+
{
69+
public:
70+
/// \brief Constructor
71+
/// Can't be used when dynamically loading the class with ROOT.
72+
/// @param objectsManager
73+
explicit LateTaskInterface(ObjectsManager* objectsManager);
74+
75+
/// \brief Default constructor
76+
LateTaskInterface() = default;
77+
78+
/// \brief Destructor
79+
virtual ~LateTaskInterface() noexcept = default;
80+
/// Copy constructor
81+
LateTaskInterface(const LateTaskInterface& other) = default;
82+
/// Move constructor
83+
LateTaskInterface(LateTaskInterface&& other) noexcept = default;
84+
/// Copy assignment operator
85+
LateTaskInterface& operator=(const LateTaskInterface& other) = default;
86+
/// Move assignment operator
87+
LateTaskInterface& operator=(LateTaskInterface&& other) /* noexcept */ = default; // error with gcc if noexcept
88+
89+
// Definition of the methods for the template method pattern
90+
virtual void initialize(o2::framework::InitContext& ctx) = 0;
91+
virtual void startOfActivity(const Activity& activity) = 0;
92+
virtual void process(std::map<std::string, std::shared_ptr<const core::MonitorObject>>& moMap, std::map<std::string, std::shared_ptr<const core::QualityObject>>& qoMap) = 0;
93+
virtual void endOfActivity(const Activity& activity) = 0;
94+
virtual void reset() = 0;
95+
96+
/// \brief Called each time mCustomParameters is updated.
97+
virtual void configure() override;
98+
99+
// Setters and getters
100+
void setObjectsManager(std::shared_ptr<ObjectsManager> objectsManager);
101+
void setMonitoring(const std::shared_ptr<o2::monitoring::Monitoring>& mMonitoring);
102+
103+
protected:
104+
std::shared_ptr<ObjectsManager> getObjectsManager();
105+
std::shared_ptr<o2::monitoring::Monitoring> mMonitoring;
106+
107+
private:
108+
std::shared_ptr<ObjectsManager> mObjectsManager;
109+
};
110+
111+
} // namespace o2::quality_control::core
112+
113+
#endif // LATETASKINTERFACE_H

0 commit comments

Comments
 (0)