Test Executorlib (#17)

* Clean up pure python

* Add executorlib interface

* Add notebooks

* fix syntax

* get_item()

* update notebooks

Author: jan-janssen

.github/workflows/executorlib.yml

@@ -0,0 +1,29 @@
+name: executorlib
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        auto-update-conda: true
+        python-version: "3.12"
+        environment-file: environment.yml
+        auto-activate-base: false
+    - name: Tests
+      shell: bash -l {0}
+      run: |
+        pip install -e adis_tools
+        pip install -e python_workflow_definition
+        conda install -c conda-forge jupyter papermill
+        export ESPRESSO_PSEUDO=$(pwd)/espresso/pseudo
+        papermill universal_simple_to_executorlib.ipynb universal_simple_to_executorlib_out.ipynb -k "python3"
+        papermill universal_qe_to_executorlib.ipynb universal_qe_to_executorlib_out.ipynb -k "python3"

python_workflow_definition/src/python_workflow_definition/executorlib.py

@@ -0,0 +1,57 @@
+import json
+from importlib import import_module
+from inspect import isfunction
+
+
+from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles
+from python_workflow_definition.purepython import resort_total_lst, group_edges
+
+
+def get_item(obj, key):
+    return obj[key]
+
+
+def _get_value(result_dict, nodes_new_dict, link_dict, exe):
+    source, source_handle = link_dict["source"], link_dict["sourceHandle"]
+    if source in result_dict.keys():
+        result = result_dict[source]
+    elif source in nodes_new_dict.keys():
+        result = nodes_new_dict[source]
+    else:
+        raise KeyError()
+    if source_handle is None:
+        return result
+    else:
+        return exe.submit(fn=get_item, obj=result, key=source_handle)
+
+
+def load_workflow_json(file_name, exe):
+    with open(file_name, "r") as f:
+        content = json.load(f)
+
+    edges_new_lst = content["edges"]
+    nodes_new_dict = {}
+    for k, v in content["nodes"].items():
+        if isinstance(v, str) and "." in v:
+            p, m = v.rsplit('.', 1)
+            mod = import_module(p)
+            nodes_new_dict[int(k)] = getattr(mod, m)
+        else:
+            nodes_new_dict[int(k)] = v
+
+    total_lst = group_edges(edges_new_lst)
+    total_new_lst = resort_total_lst(total_lst=total_lst, nodes_dict=nodes_new_dict)
+
+    result_dict = {}
+    last_key = None
+    for lst in total_new_lst:
+        node = nodes_new_dict[lst[0]]
+        if isfunction(node):
+            kwargs = {
+                k: _get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v, exe=exe)
+                for k, v in lst[1].items()
+            }
+            result_dict[lst[0]] = exe.submit(node, **kwargs)
+            last_key = lst[0]
+
+    return result_dict[last_key]

python_workflow_definition/src/python_workflow_definition/purepython.py

@@ -3,7 +3,7 @@
 from inspect import isfunction
 
 
-from python_workflow_definition.shared import get_dict, get_list, get_kwargs
+from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles
 
 
 def resort_total_lst(total_lst, nodes_dict):
@@ -35,20 +35,7 @@ def group_edges(edges_lst):
     return total_lst
 
 
-def get_source_handles(edges_lst):
-    source_handle_dict = {}
-    for ed in edges_lst:
-        if ed['source'] not in source_handle_dict.keys():
-            source_handle_dict[ed['source']] = [ed['sourceHandle']]
-        else:
-            source_handle_dict[ed['source']].append(ed['sourceHandle'])
-    return {
-        k: list(range(len(v))) if len(v) > 1 and all([el is None for el in v]) else v
-        for k, v in source_handle_dict.items()
-    }
-
-
-def get_value(result_dict, nodes_new_dict, link_dict):
+def _get_value(result_dict, nodes_new_dict, link_dict):
     source, source_handle = link_dict["source"], link_dict["sourceHandle"]
     if source in result_dict.keys():
         result = result_dict[source]
@@ -85,7 +72,7 @@ def load_workflow_json(file_name):
         node = nodes_new_dict[lst[0]]
         if isfunction(node):
             kwargs = {
-                k: get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v)
+                k: _get_value(result_dict=result_dict, nodes_new_dict=nodes_new_dict, link_dict=v)
                 for k, v in lst[1].items()
             }
             result_dict[lst[0]] = node(**kwargs)

universal_qe_to_executorlib.ipynb

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"ae0b5775-fb9d-4150-be79-d48dfb0b2cf4","cell_type":"code","source":"from executorlib import SingleNodeExecutor","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"b0f920a2-c646-485b-af0f-48251e061163","cell_type":"code","source":"from python_workflow_definition.executorlib import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"46559120-d58a-492f-b734-db27424b5659","cell_type":"code","source":"with SingleNodeExecutor(max_workers=1) as exe:\n    result = load_workflow_json(file_name=\"workflow_qe.json\", exe=exe).result()","metadata":{"trusted":true},"outputs":[{"name":"stderr","output_type":"stream","text":"[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:01655] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:01891] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:01962] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:02029] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:02093] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n[jupyter-pyiron-dev-pyth-flow-definition-4gvxle22:02159] mca_base_component_repository_open: unable to open mca_btl_openib: librdmacm.so.1: cannot open shared object file: No such file or directory (ignored)\nNote: The following floating-point exceptions are signalling: IEEE_INVALID_FLAG\n"}],"execution_count":3},{"id":"b37f476d-b7f8-4400-8070-83f8f2eb8bf9","cell_type":"code","source":"result","metadata":{"trusted":true},"outputs":[],"execution_count":4},{"id":"336dfbea-7e5c-4950-a0b3-c027a2a8e935","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}

universal_simple_to_executorlib.ipynb

@@ -0,0 +1 @@
+{"metadata":{"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.8","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"ae0b5775-fb9d-4150-be79-d48dfb0b2cf4","cell_type":"code","source":"from executorlib import SingleNodeExecutor","metadata":{"trusted":true},"outputs":[],"execution_count":1},{"id":"b0f920a2-c646-485b-af0f-48251e061163","cell_type":"code","source":"from python_workflow_definition.executorlib import load_workflow_json","metadata":{"trusted":true},"outputs":[],"execution_count":2},{"id":"46559120-d58a-492f-b734-db27424b5659","cell_type":"code","source":"with SingleNodeExecutor(max_workers=1) as exe:\n    result = load_workflow_json(file_name=\"workflow_simple.json\", exe=exe).result()","metadata":{"trusted":true},"outputs":[],"execution_count":3},{"id":"b37f476d-b7f8-4400-8070-83f8f2eb8bf9","cell_type":"code","source":"result","metadata":{"trusted":true},"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"6"},"metadata":{}}],"execution_count":4},{"id":"336dfbea-7e5c-4950-a0b3-c027a2a8e935","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}