From 6c618872e7b304a372b6da9c7bd2c9f421f56421 Mon Sep 17 00:00:00 2001 From: swenzel Date: Wed, 5 Feb 2025 11:36:21 +0100 Subject: [PATCH] Developments for 2-tag software processing a) make workflow construction more independent from actual O2 commit: Allow to detect if options are present in DPL executables and react accordingly. This can allow to write more resilit workflow construction code. b) Improvements to swap in and out software environments in the anchoring script --- MC/bin/o2dpg_sim_config.py | 42 ++++++++++++++++++++++++++++++++++++ MC/bin/o2dpg_sim_workflow.py | 7 ++++-- MC/run/ANCHOR/anchorMC.sh | 29 +++++++++++++++++++++---- 3 files changed, 72 insertions(+), 6 deletions(-) diff --git a/MC/bin/o2dpg_sim_config.py b/MC/bin/o2dpg_sim_config.py index d1a18f4fb..8ff128663 100755 --- a/MC/bin/o2dpg_sim_config.py +++ b/MC/bin/o2dpg_sim_config.py @@ -1,3 +1,7 @@ +from functools import lru_cache +import subprocess +import re + def create_sim_config(args): # creates a generic simulation config # based on arguments args (run number, energy, ...) originally passed @@ -138,3 +142,41 @@ def constructConfigKeyArg(config): arg = arg + mainkey + '.' + subkey + '=' + config[mainkey][subkey] + ';' arg = arg + '"' return arg + +# some functions to determine dpl option availability on the fly +def parse_dpl_help_output(executable): + """Parses the --help full output of an executable to extract available options.""" + try: + output = subprocess.check_output([executable, "--help", "full"], text=True) + except subprocess.CalledProcessError: + return {}, {} + + option_pattern = re.compile(r"(\-\-[\w\-]+)") + sections = {} + inverse_lookup = {} + current_section = "global" + + for line in output.split("\n"): + section_match = re.match(r"^([A-Za-z\s]+):$", line.strip()) + if section_match: + current_section = section_match.group(1).strip() + sections[current_section] = [] + continue + + option_match = option_pattern.findall(line) + if option_match: + for option in option_match: + sections.setdefault(current_section, []).append(option) + inverse_lookup.setdefault(option, []).append(current_section) + + return sections, inverse_lookup + +@lru_cache(maxsize=10) +def get_dpl_options_for_executable(executable): + """Returns available options and inverse lookup for a given executable, caching the result.""" + return parse_dpl_help_output(executable) + +def option_if_available(executable, option): + """Checks if an option is available for a given executable and returns it as a string. Otherwise empty string""" + _, inverse_lookup = get_dpl_options_for_executable(executable) + return ' ' + option if option in inverse_lookup else '' diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 60391438b..cea030685 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -27,6 +27,8 @@ import itertools import math import requests, re +from functools import lru_cache + pandas_available = True try: import pandas as pd @@ -37,7 +39,7 @@ from o2dpg_workflow_utils import createTask, createGlobalInitTask, dump_workflow, adjust_RECO_environment, isActive, activate_detector, deactivate_detector, compute_n_workers, merge_dicts from o2dpg_qc_finalization_workflow import include_all_QC_finalization -from o2dpg_sim_config import create_sim_config, create_geant_config, constructConfigKeyArg +from o2dpg_sim_config import create_sim_config, create_geant_config, constructConfigKeyArg, option_if_available parser = argparse.ArgumentParser(description='Create an ALICE (Run3) MC simulation workflow') @@ -1154,7 +1156,8 @@ def getDigiTaskName(det): TPCRECOtask['cmd'] = '${O2_ROOT}/bin/o2-tpc-reco-workflow ' + getDPL_global_options(bigshm=True) + ' --input-type clusters --output-type tracks,send-clusters-per-sector ' \ + putConfigValuesNew(["GPU_global","TPCGasParam", "TPCCorrMap", "GPU_rec_tpc", "trackTuneParams"], {"GPU_proc.ompThreads":NWORKERS_TF} | tpcLocalCFreco) + ('',' --disable-mc')[args.no_mc_labels] \ + tpc_corr_scaling_options + tpc_corr_options_mc \ - + ' --tpc-mc-time-gain' + + option_if_available('o2-tpc-reco-workflow', '--tpc-mc-time-gain') + workflow['stages'].append(TPCRECOtask) # END TPC reco diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index a7beb3907..8a33a9124 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -145,10 +145,23 @@ NWORKERS=${NWORKERS:-8} # set a default seed if not given SEED=${ALIEN_PROC_ID:-${SEED:-1}} -#<----- START OF part that should run under a clean alternative software environment if this was given ------ -( +ONCVMFS=0 +[[ "${BASEDIR}" == /cvmfs/* ]] && ONCVMFS=1 +if [ ! "${MODULEPATH}" ]; then + export MODULEPATH=${BASEDIR}/../Modules/modulefiles + if [ "${ONCVMFS}" == "1" ]; then + PLATFORM=$(echo "${BASEDIR}" | sed -E 's|.*/([^/]+)/Packages|\1|') + export MODULEPATH=${MODULEPATH}:${BASEDIR}/../../etc/toolchain/modulefiles/${PLATFORM} + fi + echo "Determined Modulepath to be ${MODULEPATH}" +fi +#<----- START OF part that should run under a clean alternative software environment if this was given ------ if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then + if [ "${LOADEDMODULES}" ]; then + module save initial_modules.list # we stash the current modules environment + module purge + fi echo_info "Using tag ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG} to setup anchored MC" /cvmfs/alice.cern.ch/bin/alienv printenv "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" &> async_environment.env source async_environment.env @@ -201,8 +214,6 @@ if [[ "${RECO_RC}" != "0" ]] ; then exit ${RECO_RC} fi -) -#<----- END OF part that should run under a clean alternative software environment if this was given ------ ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG_KEEP echo_info "Setting back ALIEN_JDL_LPMPRODUCTIONTAG to $ALIEN_JDL_LPMPRODUCTIONTAG" @@ -217,6 +228,16 @@ if [[ "${ASYNC_WF_RC}" != "0" || `grep "o2-ctf-reader-workflow-options" config-j exit 1 fi +# get rid of the temporary software environment +if [ "${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG}" ]; then + module purge + # restore the initial software environment + echo "Restoring initial environment" + module --no-pager restore initial_modules.list + module saverm initial_modules.list +fi +#<----- END OF part that should run under a clean alternative software environment if this was given ------ + # -- CREATE THE MC JOB DESCRIPTION ANCHORED TO RUN -- MODULES="--skipModules ZDC"