Make TRD optional; Anchoring improvements

* respect ALIEN_JDL_WORKFLOWDETECTORS in anchoring python script
* take out TRD from workflow when not readout
* bugfix for fetching options from anchoring JSON (was lacking '--')

Author: sawenzel

MC/bin/o2dpg_dpl_config_tools.py

@@ -309,7 +309,11 @@ def dpl_option_from_config(config, dpl_workflow, key, section = "filtered", defa
       """
       if "Executables" in config:
         # new standard
-        return config["Executables"].get(dpl_workflow,{}).get(section,{}).get(key, default_value)
+        value = config["Executables"].get(dpl_workflow,{}).get(section,{}).get(key, None)
+        if value == None:
+           print (f"Could not lookup key/option {key} from {dpl_workflow}")
+           value = default_value
+        return value
       else:
         # backward compatible versions
         dpl_workflow_key = dpl_workflow + '-options'

MC/bin/o2dpg_qc_finalization_workflow.py

@@ -95,8 +95,9 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif
   add_QC_finalization('emcBCQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json')
   #add_QC_finalization('tpcTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-tracking-direct.json')
   add_QC_finalization('tpcStandardQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-standard-direct.json')
-  add_QC_finalization('trdDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-standalone-task.json')
-  add_QC_finalization('trdTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-tracking-task.json')
+  if isActive('TRD'):
+     add_QC_finalization('trdDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-standalone-task.json')
+     add_QC_finalization('trdTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-tracking-task.json')
   add_QC_finalization('vertexQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/vertexing-qc-direct-mc.json')
   add_QC_finalization('ITSTPCmatchQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json')
   add_QC_finalization('TOFMatchQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json')

MC/bin/o2dpg_sim_workflow.py

@@ -261,7 +261,7 @@ def load_external_config(configfile):
 # these are all detectors that should be assumed active
 readout_detectors = args.readoutDets
 # here are all detectors that have been set in an anchored script
-activeDetectors = dpl_option_from_config(anchorConfig, 'o2-ctf-reader-workflow', key='onlyDet', default_value='all')
+activeDetectors = dpl_option_from_config(anchorConfig, 'o2-ctf-reader-workflow', key='--onlyDet', default_value='all')
 if activeDetectors == 'all':
     # if "all" here, there was in fact nothing in the anchored script, set to what is passed to this script (which it either also "all" or a subset)
     activeDetectors = readout_detectors
@@ -1122,7 +1122,8 @@ def putConfigValues(listOfMainKeys=[], localCF = {}):
                          + ' --onlyDet TRD --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' \
                          + putConfigValues(localCF={"TRDSimParams.digithreads" : NWORKERS_TF, "DigiParams.seed" : str(TFSEED)}) + " --forceSelectedDets"
    TRDDigitask['cmd'] += ('',' --disable-mc')[args.no_mc_labels]
-   workflow['stages'].append(TRDDigitask)
+   if isActive("TRD"):
+      workflow['stages'].append(TRDDigitask)
 
    # these are digitizers which are single threaded
    def createRestDigiTask(name, det='ALLSMALLER'):
@@ -1361,12 +1362,13 @@ def getDigiTaskName(det):
                                                 getDPL_global_options(), 
                                                 putConfigValues(),
                                                 ('',' --disable-mc')[args.no_mc_labels]])
-   workflow['stages'].append(TRDTRACKINGtask)
+   if isActive("TRD"):
+      workflow['stages'].append(TRDTRACKINGtask)
 
    #<--------- TRD global tracking 
    # FIXME This is so far a workaround to avoud a race condition for trdcalibratedtracklets.root
    TRDTRACKINGtask2 = createTask(name='trdreco2_'+str(tf), needs=[TRDTRACKINGtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu='1', mem='2000')
-   trd_track_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-trd-global-tracking', 'track-sources', default_value='TPC,ITS-TPC'))
+   trd_track_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-trd-global-tracking', '--track-sources', default_value='TPC,ITS-TPC'))
    TRDTRACKINGtask2['cmd'] = task_finalizer([
       '${O2_ROOT}/bin/o2-trd-global-tracking',
       getDPL_global_options(bigshm=True), 
@@ -1381,7 +1383,8 @@ def getDigiTaskName(det):
       '--track-sources ' + trd_track_sources,
       tpc_corr_scaling_options, 
       tpc_corr_options_mc])
-   workflow['stages'].append(TRDTRACKINGtask2)
+   if isActive("TRD"):
+      workflow['stages'].append(TRDTRACKINGtask2)
 
    #<--------- TOF reco task
    TOFRECOtask = createTask(name='tofmatch_'+str(tf), needs=[ITSTPCMATCHtask['name'], getDigiTaskName("TOF")], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1500')
@@ -1396,8 +1399,11 @@ def getDigiTaskName(det):
       workflow['stages'].append(TOFRECOtask)
 
    #<--------- TOF-TPC(-ITS) global track matcher workflow
-   toftpcmatchneeds = [TOFRECOtask['name'], TPCRECOtask['name'], ITSTPCMATCHtask['name'], TRDTRACKINGtask2['name']]
-   toftracksrcdefault = dpl_option_from_config(anchorConfig, 'o2-tof-matcher-workflow', 'track-sources', default_value='TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD')
+   toftpcmatchneeds = [TOFRECOtask['name'],
+                       TPCRECOtask['name'],
+                       ITSTPCMATCHtask['name'],
+                       TRDTRACKINGtask2['name'] if isActive("TRD") else None]
+   toftracksrcdefault = dpl_option_from_config(anchorConfig, 'o2-tof-matcher-workflow', '--track-sources', default_value='TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD')
    tofusefit = option_if_available('o2-tof-matcher-workflow', '--use-fit', envfile=async_envfile)
    TOFTPCMATCHERtask = createTask(name='toftpcmatch_'+str(tf), needs=toftpcmatchneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000')
    tofmatcher_cmd_parts = [
@@ -1591,9 +1597,9 @@ def getDigiTaskName(det):
    hmpmatchneeds = [HMPRECOtask['name'], 
                     ITSTPCMATCHtask['name'], 
                     TOFTPCMATCHERtask['name'] if isActive("TOF") else None, 
-                    TRDTRACKINGtask2['name']]
+                    TRDTRACKINGtask2['name'] if isActive("TRD") else None]
    hmpmatchneeds = [ n for n in hmpmatchneeds if n != None ]
-   hmp_match_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-hmpid-matcher-workflow', 'track-sources', default_value='ITS-TPC,ITS-TPC-TRD,TPC-TRD'))
+   hmp_match_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-hmpid-matcher-workflow', '--track-sources', default_value='ITS-TPC,ITS-TPC-TRD,TPC-TRD'))
    HMPMATCHtask = createTask(name='hmpmatch_'+str(tf), needs=hmpmatchneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000')
    HMPMATCHtask['cmd'] = task_finalizer(
       ['${O2_ROOT}/bin/o2-hmpid-matcher-workflow',
@@ -1606,17 +1612,17 @@ def getDigiTaskName(det):
    #<---------- primary vertex finding
    pvfinder_sources = dpl_option_from_config(anchorConfig,
                                              'o2-primary-vertexing-workflow',
-                                             'vertexing-sources',
+                                             '--vertexing-sources',
                                              default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,FDD,HMP,FV0,TRD,MCH,CTP')
    pvfinder_sources = cleanDetectorInputList(pvfinder_sources)
 
    pvfinder_matching_sources = dpl_option_from_config(anchorConfig,
                                                       'o2-primary-vertexing-workflow',
-                                                      'vertex-track-matching-sources',
+                                                      '--vertex-track-matching-sources',
                                                       default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,FDD,HMP,FV0,TRD,MCH,CTP')
    pvfinder_matching_sources = cleanDetectorInputList(pvfinder_matching_sources)
 
-   pvfinderneeds = [TRDTRACKINGtask2['name'],
+   pvfinderneeds = [TRDTRACKINGtask2['name'] if isActive("TRD") else None,
                     FT0RECOtask['name'] if isActive("FT0") else None,
                     FV0RECOtask['name'] if isActive("FV0") else None,
                     EMCRECOtask['name'] if isActive("EMC") else None,
@@ -1658,7 +1664,7 @@ def getDigiTaskName(det):
 
    svfinder_sources = dpl_option_from_config(anchorConfig,
                           'o2-primary-vertexing-workflow',
-                          'vertex-track-matching-sources',
+                          '--vertex-track-matching-sources',
                           default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,ZDC,FDD,HMP,FV0,TRD,MCH,CTP')
    svfinder_sources = cleanDetectorInputList(svfinder_sources)
    SVFINDERtask = createTask(name='svfinder_'+str(tf), needs=[PVFINDERtask['name'], FT0FV0EMCCTPDIGItask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=svfinder_cpu, mem='5000')   
@@ -1679,7 +1685,7 @@ def getDigiTaskName(det):
    #<------------- AOD producer
    # TODO This needs further refinement, sources and dependencies should be constructed dynamically
    aod_info_souces_default = 'ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,ZDC,FDD,HMP,FV0,TRD,MCH,CTP'
-   aodinfosources = dpl_option_from_config(anchorConfig, 'o2-aod-producer-workflow', 'info-sources', default_value=aod_info_souces_default)
+   aodinfosources = dpl_option_from_config(anchorConfig, 'o2-aod-producer-workflow', '--info-sources', default_value=aod_info_souces_default)
    aodinfosources = cleanDetectorInputList(aodinfosources)
    aodneeds = [PVFINDERtask['name'], SVFINDERtask['name']]
 
@@ -1736,17 +1742,17 @@ def getDigiTaskName(det):
       #<------------- TPC residuals extraction
       scdcalib_vertex_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig,
                                                        'o2-tpc-scdcalib-interpolation-workflow',
-                                                       'vtx-sources',
+                                                       '--vtx-sources',
                                                        default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,FDD,HMP,FV0,TRD,MCH,CTP'))
 
       scdcalib_track_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig,
                                                       'o2-tpc-scdcalib-interpolation-workflow',
-                                                      'tracking-sources',
+                                                      '--tracking-sources',
                                                       default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,FDD,HMP,FV0,TRD,MCH,CTP'))
 
       scdcalib_track_extraction = cleanDetectorInputList(dpl_option_from_config(anchorConfig,
                                                          'o2-tpc-scdcalib-interpolation-workflow',
-                                                         'tracking-sources-map-extraction',
+                                                         '--tracking-sources-map-extraction',
                                                          default_value='ITS-TPC'))
 
       SCDCALIBtask = createTask(name='scdcalib_'+str(tf), needs=[PVFINDERtask['name']], tf=tf, cwd=timeframeworkdir, lab=["CALIB"], mem='4000')
@@ -1765,11 +1771,11 @@ def getDigiTaskName(det):
       #<------------- TPC residuals aggregator
       scdaggreg_secperslot = dpl_option_from_config(anchorConfig,
                                                     'o2-calibration-residual-aggregator',
-                                                    'sec-per-slot',
+                                                    '--sec-per-slot',
                                                     default_value='600')
       scdaggreg_outputtype = dpl_option_from_config(anchorConfig,
                                                     'o2-calibration-residual-aggregator',
-                                                    'output-type',
+                                                    '--output-type',
                                                     default_value='trackParams,unbinnedResid')
 
       SCDAGGREGtask = createTask(name='scdaggreg_'+str(tf), needs=[SCDCALIBtask['name']], tf=tf, cwd=timeframeworkdir, lab=["CALIB"], mem='1500')
@@ -1849,12 +1855,13 @@ def remove_json_prefix(path):
 
      ### TRD
      # TODO: check if the readerCommand also reperforms tracklet construction (which already done in digitization)
-     addQCPerTF(taskName='trdDigitsQC',
+     if isActive('TRD'):
+         addQCPerTF(taskName='trdDigitsQC',
                 needs=[TRDDigitask['name']],
                 readerCommand='o2-trd-trap-sim --disable-root-output true',
                 configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/trd-standalone-task.json')
 
-     addQCPerTF(taskName='trdTrackingQC',
+         addQCPerTF(taskName='trdTrackingQC',
                 needs=[TRDTRACKINGtask2['name']],
                 readerCommand='o2-global-track-cluster-reader --track-types "ITS-TPC-TRD,TPC-TRD" --cluster-types none',
                 configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/trd-tracking-task.json')

MC/bin/o2dpg_sim_workflow_anchored.py

@@ -122,6 +122,14 @@ def retrieve_Aggregated_RunInfos(run_number):
     detList = o2.detectors.DetID.getNames(runInfo.grpECS.getDetsReadOut())
     assert (run_number == runInfo.runNumber)
     assert (run_number == runInfo.grpECS.getRun())
+
+    print (f"Detector list from RunInfo/GRPECS is {detList}")
+    # potential overwrite in detector list if special env variable ALIEN_JDL_WORKFLOWDETECTORS is set
+    detlist_overwrite = os.getenv("ALIEN_JDL_WORKFLOWDETECTORS")
+    if detlist_overwrite:
+       detList = detlist_overwrite
+       print (f"Detector list is overwritten to {detList} via JDL")
+
     return {"SOR" : runInfo.sor,
             "EOR" : runInfo.eor,
             "FirstOrbit" : runInfo.orbitSOR,
@@ -218,6 +226,12 @@ def retrieve_params_fromGRPECS_and_OrbitReset(ccdbreader, run_number, run_start,
     detList = o2.detectors.DetID.getNames(grp["mDetsReadout"]['v'])
     print ("Detector list is ", detList)
 
+    # potential reduction in detector list if special env variable ALIEN_JDL_WORKFLOWDETECTORS is set
+    detlist_overwrite = os.getenv("ALIEN_JDL_WORKFLOWDETECTORS")
+    if detlist_overwrite:
+       detList = detlist_overwrite
+       print ("Detector list is overwritten to ", detList)
+
     # orbitReset.get(run_number)
     return {"FirstOrbit" : orbitFirst, "LastOrbit" : orbitLast, "OrbitsPerTF" : int(grp["mNHBFPerTF"]), "detList" : detList}