diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 3006ea06..df9409d9 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -21,6 +21,7 @@ jobs: - name: Test shell: bash -l {0} run: | + conda install -c conda-forge -y aiida-workgraph=0.5.2 sudo apt-get install -y $(cat apt.txt) conda env create -n preprocessing -f source/envs/preprocessing.yaml -y conda env create -n processing -f source/envs/processing.yaml -y @@ -44,6 +45,7 @@ jobs: - name: Test shell: bash -l {0} run: | + conda install -c conda-forge -y jobflow=0.1.19 sudo apt-get install -y $(cat apt.txt) conda env create -n preprocessing -f source/envs/preprocessing.yaml -y conda env create -n processing -f source/envs/processing.yaml -y @@ -66,9 +68,34 @@ jobs: - name: Test shell: bash -l {0} run: | + conda install -c conda-forge -y pyiron_base=0.11.11 sudo apt-get install -y $(cat apt.txt) conda env create -n preprocessing -f source/envs/preprocessing.yaml -y conda env create -n processing -f source/envs/processing.yaml -y conda env create -n postprocessing -f source/envs/postprocessing.yaml -y echo -e 'from python_workflow_definition.pyiron_base import load_workflow_json\n\n\nif __name__ == "__main__":\n delayed_object_lst = load_workflow_json(file_name="workflow.json")\n print(delayed_object_lst[-1].pull())' > test_with_pyiron.py python test_with_pyiron.py + + cwl: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Conda config + run: echo -e "channels:\n - conda-forge\n" > .condarc + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: "3.12" + miniforge-version: latest + condarc-file: .condarc + environment-file: environment.yml + - name: Test + shell: bash -l {0} + run: | + conda install -c conda-forge -y cwltool=3.1.20250110105449 + sudo apt-get install -y $(cat apt.txt) + conda env create -n preprocessing -f source/envs/preprocessing.yaml -y + conda env create -n processing -f source/envs/processing.yaml -y + conda env create -n postprocessing -f source/envs/postprocessing.yaml -y + echo -e 'import os\nimport json\nfrom python_workflow_definition.cwl import write_workflow\n\n\nif __name__ == "__main__":\n with open("workflow.json") as f:\n content = json.load(f)\n node_lst = []\n for n in content["nodes"]:\n if "name" in n and n["name"] == "source_directory":\n n["value"] = os.path.abspath(n["value"])\n node_lst.append(n)\n content["nodes"] = node_lst\n with open("workflow.json", "w") as f:\n json.dump(content, f)\n write_workflow(file_name="workflow.json")' > test_with_cwl.py + python test_with_cwl.py + cwltool --preserve-environment=CONDA_EXE workflow.cwl workflow.yml diff --git a/environment.yml b/environment.yml index dd5d92c3..275c6c5e 100644 --- a/environment.yml +++ b/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge dependencies: - python =3.12 -- python-workflow-definition =0.0.1 +- python-workflow-definition =0.1.0 - conda_subprocess =0.0.6 diff --git a/source/envs/processing.yaml b/source/envs/processing.yaml index dd928d5b..ae989e47 100644 --- a/source/envs/processing.yaml +++ b/source/envs/processing.yaml @@ -1,5 +1,5 @@ channels: - conda-forge - - defaults dependencies: - fenics=2019.1.0=py39hf3d152e_26 + - sympy=1.13.3 diff --git a/workflow.json b/workflow.json index 911beb1f..b9135dce 100644 --- a/workflow.json +++ b/workflow.json @@ -1,24 +1,33 @@ { + "version": "0.1.0", "nodes": [ - {"id": 0, "function": "workflow.generate_mesh"}, - {"id": 1, "function": "workflow.convert_to_xdmf"}, - {"id": 2, "function": "workflow.poisson"}, - {"id": 3, "function": "workflow.plot_over_line"}, - {"id": 4, "function": "workflow.substitute_macros"}, - {"id": 5, "function": "workflow.compile_paper"}, - {"id": 6, "value": 2.0} + {"id": 0, "type": "function", "value": "workflow.generate_mesh"}, + {"id": 1, "type": "function", "value": "workflow.convert_to_xdmf"}, + {"id": 2, "type": "function", "value": "workflow.poisson"}, + {"id": 3, "type": "function", "value": "workflow.plot_over_line"}, + {"id": 4, "type": "function", "value": "workflow.substitute_macros"}, + {"id": 5, "type": "function", "value": "workflow.compile_paper"}, + {"id": 6, "type": "input", "value": 2.0, "name": "domain_size"}, + {"id": 7, "type": "input", "value": "source", "name": "source_directory"}, + {"id": 8, "type": "output", "name": "result"} ], "edges": [ {"target": 0, "targetPort": "domain_size", "source": 6, "sourcePort": null}, + {"target": 0, "targetPort": "source_directory", "source": 7, "sourcePort": null}, {"target": 1, "targetPort": "gmsh_output_file", "source": 0, "sourcePort": null}, {"target": 2, "targetPort": "meshio_output_xdmf", "source": 1, "sourcePort": "xdmf_file"}, {"target": 2, "targetPort": "meshio_output_h5", "source": 1, "sourcePort": "h5_file"}, + {"target": 2, "targetPort": "source_directory", "source": 7, "sourcePort": null}, {"target": 3, "targetPort": "poisson_output_pvd_file", "source": 2, "sourcePort": "pvd_file"}, {"target": 3, "targetPort": "poisson_output_vtu_file", "source": 2, "sourcePort": "vtu_file"}, + {"target": 3, "targetPort": "source_directory", "source": 7, "sourcePort": null}, {"target": 4, "targetPort": "pvbatch_output_file", "source": 3, "sourcePort": null}, {"target": 4, "targetPort": "ndofs", "source": 2, "sourcePort": "numdofs"}, {"target": 4, "targetPort": "domain_size", "source": 6, "sourcePort": null}, + {"target": 4, "targetPort": "source_directory", "source": 7, "sourcePort": null}, {"target": 5, "targetPort": "macros_tex", "source": 4, "sourcePort": null}, - {"target": 5, "targetPort": "plot_file", "source": 3, "sourcePort": null} + {"target": 5, "targetPort": "plot_file", "source": 3, "sourcePort": null}, + {"target": 5, "targetPort": "source_directory", "source": 7, "sourcePort": null}, + {"target": 8, "targetPort": null, "source": 5, "sourcePort": null} ] } \ No newline at end of file diff --git a/workflow.py b/workflow.py index 3893a38e..d8f795d5 100644 --- a/workflow.py +++ b/workflow.py @@ -2,15 +2,13 @@ from conda_subprocess import check_output import shutil -source_directory = os.path.join(os.path.dirname(os.path.abspath(__file__)), "source") - -def generate_mesh(domain_size: float = 2.0) -> str: +def generate_mesh(domain_size: float, source_directory: str) -> str: stage_name = "preprocessing" gmsh_output_file_name = "square.msh" source_file_name ="unit_square.geo" os.makedirs(stage_name, exist_ok=True) - _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name, source_directory=source_directory) _ = check_output( [ "gmsh", "-2", "-setnumber", "domain_size", str(domain_size), @@ -40,13 +38,13 @@ def convert_to_xdmf(gmsh_output_file : str) -> dict: } -def poisson(meshio_output_xdmf: str, meshio_output_h5: str) -> dict: +def poisson(meshio_output_xdmf: str, meshio_output_h5: str, source_directory: str) -> dict: stage_name = "processing" poisson_output_pvd_file_name = "poisson.pvd" poisson_output_numdofs_file_name = "numdofs.txt" source_file_name = "poisson.py" os.makedirs(stage_name, exist_ok=True) - _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name, source_directory=source_directory) _copy_file(stage_name=stage_name, source_file=meshio_output_xdmf) _copy_file(stage_name=stage_name, source_file=meshio_output_h5) _ = check_output( @@ -65,12 +63,12 @@ def poisson(meshio_output_xdmf: str, meshio_output_h5: str) -> dict: } -def plot_over_line(poisson_output_pvd_file: str, poisson_output_vtu_file: str) -> str: +def plot_over_line(poisson_output_pvd_file: str, poisson_output_vtu_file: str, source_directory: str) -> str: stage_name = "postprocessing" pvbatch_output_file_name = "plotoverline.csv" source_file_name = "postprocessing.py" os.makedirs(stage_name, exist_ok=True) - _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name, source_directory=source_directory) _copy_file(stage_name=stage_name, source_file=poisson_output_pvd_file) _copy_file(stage_name=stage_name, source_file=poisson_output_vtu_file) _ = check_output( @@ -82,14 +80,14 @@ def plot_over_line(poisson_output_pvd_file: str, poisson_output_vtu_file: str) - return os.path.abspath(os.path.join("postprocessing", pvbatch_output_file_name)) -def substitute_macros(pvbatch_output_file: str, ndofs: int, domain_size: float = 2.0) -> str: +def substitute_macros(pvbatch_output_file: str, ndofs: int, domain_size: float, source_directory: str) -> str: stage_name = "postprocessing" source_file_name = "prepare_paper_macros.py" template_file_name = "macros.tex.template" macros_output_file_name = "macros.tex" os.makedirs(stage_name, exist_ok=True) - _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) - _copy_file_from_source(stage_name=stage_name, source_file_name=template_file_name) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name, source_directory=source_directory) + _copy_file_from_source(stage_name=stage_name, source_file_name=template_file_name, source_directory=source_directory) _copy_file(stage_name=stage_name, source_file=pvbatch_output_file) _ = check_output( [ @@ -104,12 +102,12 @@ def substitute_macros(pvbatch_output_file: str, ndofs: int, domain_size: float = return os.path.abspath(os.path.join(stage_name, macros_output_file_name)) -def compile_paper(macros_tex: str, plot_file: str) -> str: +def compile_paper(macros_tex: str, plot_file: str, source_directory: str) -> str: stage_name = "postprocessing" paper_output = "paper.pdf" source_file_name = "paper.tex" os.makedirs(stage_name, exist_ok=True) - _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name) + _copy_file_from_source(stage_name=stage_name, source_file_name=source_file_name, source_directory=source_directory) _copy_file(stage_name=stage_name, source_file=macros_tex) _copy_file(stage_name=stage_name, source_file=plot_file) _ = check_output( @@ -126,12 +124,12 @@ def _poisson_collect_output(numdofs_file: str) -> int: return int(f.read()) -def _copy_file(stage_name, source_file): +def _copy_file(stage_name: str, source_file: str): input_file = os.path.join(os.path.abspath(stage_name), os.path.basename(source_file)) if input_file != source_file: shutil.copyfile(source_file, input_file) -def _copy_file_from_source(stage_name, source_file_name): +def _copy_file_from_source(stage_name: str, source_file_name: str, source_directory: str): source_file = os.path.join(source_directory, source_file_name) shutil.copyfile(source_file, os.path.join(stage_name, source_file_name)) \ No newline at end of file