Merge remote-tracking branch 'upstream/main' into aiida-pythonjob

Author: GeigerJ2

.github/workflows/executorlib.yml

@@ -0,0 +1,29 @@
+name: executorlib
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        auto-update-conda: true
+        python-version: "3.12"
+        environment-file: environment.yml
+        auto-activate-base: false
+    - name: Tests
+      shell: bash -l {0}
+      run: |
+        pip install -e adis_tools
+        pip install -e python_workflow_definition
+        conda install -c conda-forge jupyter papermill
+        export ESPRESSO_PSEUDO=$(pwd)/espresso/pseudo
+        papermill universal_simple_to_executorlib.ipynb universal_simple_to_executorlib_out.ipynb -k "python3"
+        papermill universal_qe_to_executorlib.ipynb universal_qe_to_executorlib_out.ipynb -k "python3"

.github/workflows/python.yml

@@ -0,0 +1,29 @@
+name: python
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        auto-update-conda: true
+        python-version: "3.12"
+        environment-file: environment.yml
+        auto-activate-base: false
+    - name: Tests
+      shell: bash -l {0}
+      run: |
+        pip install -e adis_tools
+        pip install -e python_workflow_definition
+        conda install -c conda-forge jupyter papermill
+        export ESPRESSO_PSEUDO=$(pwd)/espresso/pseudo
+        papermill universal_simple_to_python.ipynb universal_simple_to_python_out.ipynb -k "python3"
+        papermill universal_qe_to_python.ipynb universal_qe_to_python_out.ipynb -k "python3"

README.md

@@ -62,6 +62,8 @@ The corresponding Jupyter notebooks demonstrate this functionality:
 |--------------------------------------------------------------------------------|------------------------------------------------------------------------------|
 | [universal_simple_to_jobflow.ipynb](universal_simple_to_jobflow.ipynb)         | Execute workflow defined in the Python Workflow Definition with jobflow.     |
 | [universal_simple_to_pyiron_base.ipynb](universal_simple_to_pyiron_base.ipynb) | Execute workflow defined in the Python Workflow Definition with pyrion_base. |
+| [universal_simple_to_python.ipynb](universal_simple_to_python.ipynb)           | Execute workflow defined in the Python Workflow Definition with Python.      |
+| [universal_simple_to_executorlib.ipynb](universal_simple_to_executorlib.ipynb) | Execute workflow defined in the Python Workflow Definition with executorlib. |
 | [jobflow_to_pyiron_base_simple.ipynb](jobflow_to_pyiron_base_simple.ipynb)     | Define Workflow with jobflow and execute it with pyiron_base.                |
 | [pyiron_base_to_jobflow_simple.ipynb](pyiron_base_to_jobflow_simple.ipynb)     | Define Workflow with pyiron_base and execute it with jobflow.                |
 
@@ -76,5 +78,7 @@ initial structure is relaxed, afterwards it is strained and the total energy is
 |------------------------------------------------------------------------|------------------------------------------------------------------------------|
 | [universal_qe_to_jobflow.ipynb](universal_qe_to_jobflow.ipynb)         | Execute workflow defined in the Python Workflow Definition with jobflow.     |
 | [universal_qe_to_pyiron_base.ipynb](universal_qe_to_pyiron_base.ipynb) | Execute workflow defined in the Python Workflow Definition with pyrion_base. |
+| [universal_qe_to_python.ipynb](universal_qe_to_python.ipynb)           | Execute workflow defined in the Python Workflow Definition with Python.      |
+| [universal_qe_to_executorlib.ipynb](universal_qe_to_executorlib.ipynb) | Execute workflow defined in the Python Workflow Definition with executorlib. |
 | [jobflow_to_pyiron_base_qe.ipynb](jobflow_to_pyiron_base_qe.ipynb)     | Define Workflow with jobflow and execute it with pyiron_base.                |
 | [pyiron_base_to_jobflow_qe.ipynb](pyiron_base_to_jobflow_qe.ipynb)     | Define Workflow with pyiron_base and execute it with jobflow.                |

environment.yml

@@ -2,6 +2,7 @@ channels:
 - conda-forge
 dependencies:
 - python=3.12
+- aiida-workgraph=0.4.10
 - pyiron_base=0.11.7
 - qe=7.2
 - qe-tools=2.0.0

jobflow_to_pyiron_base_simple.ipynb

@@ -41,16 +41,6 @@
    "outputs": [],
    "execution_count": 2
   },
-  {
-   "id": "78ca455d-d0b8-4814-81fa-6039f6adb4c4",
-   "cell_type": "code",
-   "source": "from python_workflow_definition.jobflow import load_workflow_json",
-   "metadata": {
-    "trusted": true
-   },
-   "outputs": [],
-   "execution_count": 3
-  },
   {
    "id": "14d87342-706e-4120-99e6-b5363f724601",
    "cell_type": "code",

pyiron_base_to_jobflow_qe.ipynb

@@ -1 +1,198 @@
-{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.8"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"28d69730-d8cb-4174-ae3f-aa70da8a8108","cell_type":"code","source":"import numpy as np","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"cacaa0a8-27c8-44de-9e37-69cd3d13408b","cell_type":"code","source":"from jobflow.managers.local import run_locally","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"78ca455d-d0b8-4814-81fa-6039f6adb4c4","cell_type":"code","source":"from python_workflow_definition.jobflow import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"14d87342-706e-4120-99e6-b5363f724601","cell_type":"code","source":"from pyiron_base import Project, job","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"a2ed2608-9e1b-4a81-81cb-5079573ea2d1","cell_type":"code","source":"from python_workflow_definition.pyiron_base import write_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":5},{"id":"444347b0-d5ba-4903-b8d4-a9d7fc35d268","cell_type":"code","source":"from python_workflow_definition.jobflow import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":6},{"id":"b75c2530-9b89-4185-838f-a17e517fa68f","cell_type":"code","source":"from simple_workflow import (\n    add_x_and_y as _add_x_and_y, \n    add_x_and_y_and_z as _add_x_and_y_and_z,\n)","metadata":{"trusted":true},"outputs":[],"execution_count":7},{"id":"208ddfb8-dfda-4227-aa1f-3dc29e34ea82","cell_type":"code","source":"add_x_and_y = job(_add_x_and_y, output_key_lst=[\"x\", \"y\", \"z\"])\nadd_x_and_y_and_z = job(_add_x_and_y_and_z)","metadata":{"trusted":true},"outputs":[],"execution_count":8},{"id":"13debfcd-362d-4fd0-a91f-5ab33632fceb","cell_type":"code","source":"pr = Project(\"test\")\npr.remove_jobs(recursive=True, silently=True)","metadata":{"trusted":true},"outputs":[{"output_type":"display_data","data":{"text/plain":"0it [00:00, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1bcb58b981944c688403d1eaca7b3598"}},"metadata":{}}],"execution_count":9},{"id":"1f1cb12f-4001-478d-8ea0-b369f4f2981a","cell_type":"code","source":"obj = add_x_and_y(x=1, y=2, pyiron_project=pr)","metadata":{"trusted":true},"outputs":[],"execution_count":10},{"id":"4140f07b-af54-41ef-be22-df6b5b53bf95","cell_type":"code","source":"w = add_x_and_y_and_z(x=obj.output.x, y=obj.output.y, z=obj.output.z, pyiron_project=pr)","metadata":{"trusted":true},"outputs":[],"execution_count":11},{"id":"52646121-335f-48e6-bd5e-a1d69d00e8bc","cell_type":"code","source":"write_workflow_json(delayed_object=w, file_name=\"workflow_pyiron.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":12},{"id":"72c8e6e6-8e60-4a4f-81f3-968b4b5f36ee","cell_type":"code","source":"!cat workflow_pyiron.json","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"{\"nodes\": {\"0\": \"simple_workflow.add_x_and_y_and_z\", \"1\": \"simple_workflow.add_x_and_y\", \"2\": 1, \"3\": 2}, \"edges\": [{\"target\": 0, \"targetHandle\": \"x\", \"source\": 1, \"sourceHandle\": \"x\"}, {\"target\": 1, \"targetHandle\": \"x\", \"source\": 2, \"sourceHandle\": null}, {\"target\": 1, \"targetHandle\": \"y\", \"source\": 3, \"sourceHandle\": null}, {\"target\": 0, \"targetHandle\": \"y\", \"source\": 1, \"sourceHandle\": \"y\"}, {\"target\": 0, \"targetHandle\": \"z\", \"source\": 1, \"sourceHandle\": \"z\"}]}"}],"execution_count":13},{"id":"32fcd4b2-4f0a-442d-b098-827672823796","cell_type":"code","source":"flow = load_workflow_json(file_name=\"workflow_pyiron.json\")","metadata":{"trusted":true},"outputs":[],"execution_count":14},{"id":"a80b59bd-fe30-49c6-92ca-35ef2d77a6fb","cell_type":"code","source":"result = run_locally(flow)\nresult","metadata":{"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":"2025-03-13 06:29:36,854 INFO Started executing jobs locally\n2025-03-13 06:29:37,166 INFO Starting job - add_x_and_y (75f3eb14-326b-46bb-848d-ef501b2d43b4)\n2025-03-13 06:29:37,169 INFO Finished job - add_x_and_y (75f3eb14-326b-46bb-848d-ef501b2d43b4)\n2025-03-13 06:29:37,170 INFO Starting job - add_x_and_y_and_z (609c1659-ec60-4211-8996-670770e4b36c)\n2025-03-13 06:29:37,172 INFO Finished job - add_x_and_y_and_z (609c1659-ec60-4211-8996-670770e4b36c)\n2025-03-13 06:29:37,172 INFO Finished executing jobs locally\n"},{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"{'75f3eb14-326b-46bb-848d-ef501b2d43b4': {1: Response(output={'x': 1, 'y': 2, 'z': 3}, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False, job_dir=PosixPath('/home/jovyan'))},\n '609c1659-ec60-4211-8996-670770e4b36c': {1: Response(output=6, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False, job_dir=PosixPath('/home/jovyan'))}}"},"metadata":{}}],"execution_count":15},{"id":"6a8d1a17-3698-4873-8937-616e9e7dc7ca","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}
+{
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.8"
+  }
+ },
+ "nbformat_minor": 5,
+ "nbformat": 4,
+ "cells": [
+  {
+   "id": "cacaa0a8-27c8-44de-9e37-69cd3d13408b",
+   "cell_type": "code",
+   "source": "from jobflow.managers.local import run_locally",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 2
+  },
+  {
+   "id": "14d87342-706e-4120-99e6-b5363f724601",
+   "cell_type": "code",
+   "source": "from pyiron_base import Project, job",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 4
+  },
+  {
+   "id": "a2ed2608-9e1b-4a81-81cb-5079573ea2d1",
+   "cell_type": "code",
+   "source": "from python_workflow_definition.pyiron_base import write_workflow_json",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 5
+  },
+  {
+   "id": "444347b0-d5ba-4903-b8d4-a9d7fc35d268",
+   "cell_type": "code",
+   "source": "from python_workflow_definition.jobflow import load_workflow_json",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 6
+  },
+  {
+   "id": "b75c2530-9b89-4185-838f-a17e517fa68f",
+   "cell_type": "code",
+   "source": "from simple_workflow import (\n    add_x_and_y as _add_x_and_y, \n    add_x_and_y_and_z as _add_x_and_y_and_z,\n)",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 7
+  },
+  {
+   "id": "208ddfb8-dfda-4227-aa1f-3dc29e34ea82",
+   "cell_type": "code",
+   "source": "add_x_and_y = job(_add_x_and_y, output_key_lst=[\"x\", \"y\", \"z\"])\nadd_x_and_y_and_z = job(_add_x_and_y_and_z)",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 8
+  },
+  {
+   "id": "13debfcd-362d-4fd0-a91f-5ab33632fceb",
+   "cell_type": "code",
+   "source": "pr = Project(\"test\")\npr.remove_jobs(recursive=True, silently=True)",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "output_type": "display_data",
+     "data": {
+      "text/plain": "0it [00:00, ?it/s]",
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "1bcb58b981944c688403d1eaca7b3598"
+      }
+     },
+     "metadata": {}
+    }
+   ],
+   "execution_count": 9
+  },
+  {
+   "id": "1f1cb12f-4001-478d-8ea0-b369f4f2981a",
+   "cell_type": "code",
+   "source": "obj = add_x_and_y(x=1, y=2, pyiron_project=pr)",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 10
+  },
+  {
+   "id": "4140f07b-af54-41ef-be22-df6b5b53bf95",
+   "cell_type": "code",
+   "source": "w = add_x_and_y_and_z(x=obj.output.x, y=obj.output.y, z=obj.output.z, pyiron_project=pr)",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 11
+  },
+  {
+   "id": "52646121-335f-48e6-bd5e-a1d69d00e8bc",
+   "cell_type": "code",
+   "source": "write_workflow_json(delayed_object=w, file_name=\"workflow_pyiron.json\")",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 12
+  },
+  {
+   "id": "72c8e6e6-8e60-4a4f-81f3-968b4b5f36ee",
+   "cell_type": "code",
+   "source": "!cat workflow_pyiron.json",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": "{\"nodes\": {\"0\": \"simple_workflow.add_x_and_y_and_z\", \"1\": \"simple_workflow.add_x_and_y\", \"2\": 1, \"3\": 2}, \"edges\": [{\"target\": 0, \"targetHandle\": \"x\", \"source\": 1, \"sourceHandle\": \"x\"}, {\"target\": 1, \"targetHandle\": \"x\", \"source\": 2, \"sourceHandle\": null}, {\"target\": 1, \"targetHandle\": \"y\", \"source\": 3, \"sourceHandle\": null}, {\"target\": 0, \"targetHandle\": \"y\", \"source\": 1, \"sourceHandle\": \"y\"}, {\"target\": 0, \"targetHandle\": \"z\", \"source\": 1, \"sourceHandle\": \"z\"}]}"
+    }
+   ],
+   "execution_count": 13
+  },
+  {
+   "id": "32fcd4b2-4f0a-442d-b098-827672823796",
+   "cell_type": "code",
+   "source": "flow = load_workflow_json(file_name=\"workflow_pyiron.json\")",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": 14
+  },
+  {
+   "id": "a80b59bd-fe30-49c6-92ca-35ef2d77a6fb",
+   "cell_type": "code",
+   "source": "result = run_locally(flow)\nresult",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": "2025-03-13 06:29:36,854 INFO Started executing jobs locally\n2025-03-13 06:29:37,166 INFO Starting job - add_x_and_y (75f3eb14-326b-46bb-848d-ef501b2d43b4)\n2025-03-13 06:29:37,169 INFO Finished job - add_x_and_y (75f3eb14-326b-46bb-848d-ef501b2d43b4)\n2025-03-13 06:29:37,170 INFO Starting job - add_x_and_y_and_z (609c1659-ec60-4211-8996-670770e4b36c)\n2025-03-13 06:29:37,172 INFO Finished job - add_x_and_y_and_z (609c1659-ec60-4211-8996-670770e4b36c)\n2025-03-13 06:29:37,172 INFO Finished executing jobs locally\n"
+    },
+    {
+     "execution_count": 15,
+     "output_type": "execute_result",
+     "data": {
+      "text/plain": "{'75f3eb14-326b-46bb-848d-ef501b2d43b4': {1: Response(output={'x': 1, 'y': 2, 'z': 3}, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False, job_dir=PosixPath('/home/jovyan'))},\n '609c1659-ec60-4211-8996-670770e4b36c': {1: Response(output=6, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False, job_dir=PosixPath('/home/jovyan'))}}"
+     },
+     "metadata": {}
+    }
+   ],
+   "execution_count": 15
+  },
+  {
+   "id": "6a8d1a17-3698-4873-8937-616e9e7dc7ca",
+   "cell_type": "code",
+   "source": "",
+   "metadata": {
+    "trusted": true
+   },
+   "outputs": [],
+   "execution_count": null
+  }
+ ]
+}

pyiron_base_to_jobflow_simple.ipynb

@@ -0,0 +1,57 @@
+import json
+from importlib import import_module
+from inspect import isfunction
+
+
+from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles
+from python_workflow_definition.purepython import resort_total_lst, group_edges
+
+
+def get_item(obj, key):
+    return obj[key]
+
+
+def _get_value(result_dict, nodes_new_dict, link_dict, exe):
+    source, source_handle = link_dict["source"], link_dict["sourceHandle"]
+    if source in result_dict.keys():
+        result = result_dict[source]
+    elif source in nodes_new_dict.keys():
+        result = nodes_new_dict[source]
+    else:
+        raise KeyError()
+    if source_handle is None:
+        return result
+    else:
+        return exe.submit(fn=get_item, obj=result, key=source_handle)
+
+
+def load_workflow_json(file_name, exe):
+    with open(file_name, "r") as f:
+        content = json.load(f)
+
+    edges_new_lst = content["edges"]
+    nodes_new_dict = {}
+    for k, v in content["nodes"].items():
+        if isinstance(v, str) and "." in v:
+            p, m = v.rsplit('.', 1)
+            mod = import_module(p)
+            nodes_new_dict[int(k)] = getattr(mod, m)
+        else:
+            nodes_new_dict[int(k)] = v
+
+    total_lst = group_edges(edges_new_lst)
+    total_new_lst = resort_total_lst(total_lst=total_lst, nodes_dict=nodes_new_dict)
+
+    result_dict = {}
+    last_key = None
+    for lst in total_new_lst:
+        node = nodes_new_dict[lst[0]]
+        if isfunction(node):
+            kwargs = {
+                k: _get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v, exe=exe)
+                for k, v in lst[1].items()
+            }
+            result_dict[lst[0]] = exe.submit(node, **kwargs)
+            last_key = lst[0]
+
+    return result_dict[last_key]