Convert nodes to list of dictionaries (#53)

* Convert nodes to list of dictionaries

* update workflows

* Update readme

* separate values and functions

* update readme

Author: jan-janssen

README.md

@@ -34,14 +34,14 @@ Python module. The connection of the Python functions are stored in the [workflo
 JSON file:
 ```
 {
-  "nodes": {
-    "0": "simple_workflow.add_x_and_y_and_z",
-    "1": "simple_workflow.add_x_and_y",
-    "2": "simple_workflow.add_x_and_y",
-    "3": "simple_workflow.add_x_and_y",
-    "4": 1,
-    "5": 2
-  },
+  "nodes": [
+    {"id": 0, "function": "simple_workflow.add_x_and_y_and_z"},
+    {"id": 1, "function": "simple_workflow.add_x_and_y"},
+    {"id": 2, "function": "simple_workflow.add_x_and_y"},
+    {"id": 3, "function": "simple_workflow.add_x_and_y"},
+    {"id": 4, "value": 1},
+    {"id": 5, "value": 2}
+  ],
   "edges": [
     {"target": 0, "targetPort": "x", "source": 1, "sourcePort": "x"},
     {"target": 1, "targetPort": "x", "source": 4, "sourcePort": null},

book/simple.md

@@ -19,14 +19,14 @@ Python module. The connection of the Python functions are stored in the [workflo
 JSON file:
 ```
 {
-  "nodes": {
-    "0": "simple_workflow.add_x_and_y_and_z",
-    "1": "simple_workflow.add_x_and_y",
-    "2": "simple_workflow.add_x_and_y",
-    "3": "simple_workflow.add_x_and_y",
-    "4": 1,
-    "5": 2
-  },
+  "nodes": [
+    {"id": 0, "function": "simple_workflow.add_x_and_y_and_z"},
+    {"id": 1, "function": "simple_workflow.add_x_and_y"},
+    {"id": 2, "function": "simple_workflow.add_x_and_y"},
+    {"id": 3, "function": "simple_workflow.add_x_and_y"},
+    {"id": 4, "value": 1},
+    {"id": 5, "value": 2}
+  ],
   "edges": [
     {"target": 0, "targetPort": "x", "source": 1, "sourcePort": "x"},
     {"target": 1, "targetPort": "x", "source": 4, "sourcePort": null},

python_workflow_definition/src/python_workflow_definition/aiida.py

@@ -7,6 +7,8 @@
 from aiida_workgraph import WorkGraph, task
 from aiida_workgraph.socket import TaskSocketNamespace
 
+from python_workflow_definition.shared import convert_nodes_list_to_dict
+
 
 def load_workflow_json(file_name):
     with open(file_name) as f:
@@ -15,7 +17,7 @@ def load_workflow_json(file_name):
     wg = WorkGraph()
     task_name_mapping = {}
 
-    for id, identifier in data["nodes"].items():
+    for id, identifier in convert_nodes_list_to_dict(nodes_list=data["nodes"]).items():
         if isinstance(identifier, str) and "." in identifier:
             p, m = identifier.rsplit(".", 1)
             mod = import_module(p)
@@ -66,7 +68,7 @@ def load_workflow_json(file_name):
 
 
 def write_workflow_json(wg, file_name):
-    data = {"nodes": {}, "edges": []}
+    data = {"nodes": [], "edges": []}
     node_name_mapping = {}
     data_node_name_mapping = {}
     i = 0
@@ -76,7 +78,7 @@ def write_workflow_json(wg, file_name):
 
         callable_name = executor["callable_name"]
         callable_name = f"{executor['module_path']}.{callable_name}"
-        data["nodes"][str(i)] = callable_name
+        data["nodes"].append({"id": i, "function": callable_name})
         i += 1
 
     for link in wg.links:
@@ -105,7 +107,7 @@ def write_workflow_json(wg, file_name):
                         raw_value.pop("node_type", None)
                     else:
                         raw_value = input.value.value
-                    data["nodes"][str(i)] = raw_value
+                    data["nodes"].append({"id": i, "value": raw_value})
                     input_node_name = i
                     data_node_name_mapping[input.value.uuid] = input_node_name
                     i += 1

python_workflow_definition/src/python_workflow_definition/executorlib.py

@@ -3,7 +3,7 @@
 from inspect import isfunction
 
 
-from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles
+from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles, convert_nodes_list_to_dict
 from python_workflow_definition.purepython import resort_total_lst, group_edges
 
 
@@ -31,7 +31,8 @@ def load_workflow_json(file_name, exe):
 
     edges_new_lst = content["edges"]
     nodes_new_dict = {}
-    for k, v in content["nodes"].items():
+
+    for k, v in convert_nodes_list_to_dict(nodes_list=content["nodes"]).items():
         if isinstance(v, str) and "." in v:
             p, m = v.rsplit('.', 1)
             mod = import_module(p)

python_workflow_definition/src/python_workflow_definition/jobflow.py

@@ -5,7 +5,7 @@
 import numpy as np
 from jobflow import job, Flow
 
-from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles
+from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles, convert_nodes_list_to_dict
 
 
 def _get_function_dict(flow):
@@ -173,7 +173,7 @@ def load_workflow_json(file_name):
             )
 
     nodes_new_dict = {}
-    for k, v in content["nodes"].items():
+    for k, v in convert_nodes_list_to_dict(nodes_list=content["nodes"]).items():
         if isinstance(v, str) and "." in v:
             p, m = v.rsplit('.', 1)
             mod = import_module(p)
@@ -204,14 +204,14 @@ def write_workflow_json(flow, file_name="workflow.json"):
         nodes_dict=nodes_dict,
     )
 
-    nodes_store_dict = {}
+    nodes_store_lst = []
     for k, v in nodes_dict.items():
         if isfunction(v):
-            nodes_store_dict[k] = v.__module__ + "." + v.__name__
+            nodes_store_lst.append({"id": k, "function": v.__module__ + "." + v.__name__})
         elif isinstance(v, np.ndarray):
-            nodes_store_dict[k] = v.tolist()
+            nodes_store_lst.append({"id": k, "value": v.tolist()})
         else:
-            nodes_store_dict[k] = v
+            nodes_store_lst.append({"id": k, "value": v})
 
     with open(file_name, "w") as f:
-        json.dump({"nodes": nodes_store_dict, "edges": edges_lst}, f)
+        json.dump({"nodes": nodes_store_lst, "edges": edges_lst}, f)

python_workflow_definition/src/python_workflow_definition/purepython.py

@@ -3,7 +3,7 @@
 from inspect import isfunction
 
 
-from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles
+from python_workflow_definition.shared import get_dict, get_list, get_kwargs, get_source_handles, convert_nodes_list_to_dict
 
 
 def resort_total_lst(total_lst, nodes_dict):
@@ -55,7 +55,7 @@ def load_workflow_json(file_name):
 
     edges_new_lst = content["edges"]
     nodes_new_dict = {}
-    for k, v in content["nodes"].items():
+    for k, v in convert_nodes_list_to_dict(nodes_list=content["nodes"]).items():
         if isinstance(v, str) and "." in v:
             p, m = v.rsplit('.', 1)
             mod = import_module(p)

python_workflow_definition/src/python_workflow_definition/pyiron_base.py

@@ -6,7 +6,7 @@
 from pyiron_base import job
 from pyiron_base.project.delayed import DelayedObject
 
-from python_workflow_definition.shared import get_kwargs, get_source_handles
+from python_workflow_definition.shared import get_kwargs, get_source_handles, convert_nodes_list_to_dict
 
 
 def _resort_total_lst(total_lst, nodes_dict):
@@ -181,7 +181,7 @@ def load_workflow_json(project, file_name):
 
     edges_new_lst = content["edges"]
     nodes_new_dict = {}
-    for k, v in content["nodes"].items():
+    for k, v in convert_nodes_list_to_dict(nodes_list=content["nodes"]).items():
         if isinstance(v, str) and "." in v:
             p, m = v.rsplit('.', 1)
             if p == "python_workflow_definition.shared":
@@ -211,17 +211,17 @@ def write_workflow_json(delayed_object, file_name="workflow.json"):
     nodes_new_dict = _get_nodes(connection_dict=connection_dict, delayed_object_updated_dict=delayed_object_updated_dict)
     edges_new_lst = _get_edges_dict(edges_lst=edges_lst, nodes_dict=nodes_dict, connection_dict=connection_dict, lookup_dict=lookup_dict)
 
-    nodes_store_dict = {}
+    nodes_store_lst = []
     for k, v in nodes_new_dict.items():
         if isfunction(v):
             mod = v.__module__
             if mod == "python_workflow_definition.pyiron_base":
                 mod = "python_workflow_definition.shared"
-            nodes_store_dict[k] = mod + "." + v.__name__
+            nodes_store_lst.append({"id": k, "function": mod + "." + v.__name__})
         elif isinstance(v, np.ndarray):
-            nodes_store_dict[k] = v.tolist()
+            nodes_store_lst.append({"id": k, "value": v.tolist()})
         else:
-            nodes_store_dict[k] = v
+            nodes_store_lst.append({"id": k, "value": v})
 
     with open(file_name, "w") as f:
-        json.dump({"nodes": nodes_store_dict, "edges": edges_new_lst}, f)
+        json.dump({"nodes": nodes_store_lst, "edges": edges_new_lst}, f)

python_workflow_definition/src/python_workflow_definition/shared.py

@@ -22,4 +22,11 @@ def get_source_handles(edges_lst):
     return {
         k: list(range(len(v))) if len(v) > 1 and all([el is None for el in v]) else v
         for k, v in source_handle_dict.items()
-    }
+    }
+
+
+def convert_nodes_list_to_dict(nodes_list):
+    return {
+        str(el["id"]): el["value"] if "value" in el else el["function"]
+        for el in sorted(nodes_list, key=lambda d: d["id"])
+    }

workflow_nfdi.json

@@ -1,13 +1,13 @@
 {
-  "nodes": {
-    "0": "nfdi_ing_workflow.generate_mesh",
-    "1": "nfdi_ing_workflow.convert_to_xdmf",
-    "2": "nfdi_ing_workflow.poisson",
-    "3": "nfdi_ing_workflow.plot_over_line",
-    "4": "nfdi_ing_workflow.substitute_macros",
-    "5": "nfdi_ing_workflow.compile_paper",
-    "6": 2.0
-  },
+  "nodes": [
+    {"id": 0, "function": "nfdi_ing_workflow.generate_mesh"},
+    {"id": 1, "function": "nfdi_ing_workflow.convert_to_xdmf"},
+    {"id": 2, "function": "nfdi_ing_workflow.poisson"},
+    {"id": 3, "function": "nfdi_ing_workflow.plot_over_line"},
+    {"id": 4, "function": "nfdi_ing_workflow.substitute_macros"},
+    {"id": 5, "function": "nfdi_ing_workflow.compile_paper"},
+    {"id": 6, "value": 2.0}
+  ],
   "edges": [
     {"target": 0, "targetPort": "domain_size", "source": 6, "sourcePort": null},
     {"target": 1, "targetPort": "gmsh_output_file", "source": 0, "sourcePort": null},

workflow_qe.json

@@ -1,38 +1,38 @@
 {
-  "nodes": {
-    "0": "quantum_espresso_workflow.get_bulk_structure",
-    "1": "quantum_espresso_workflow.calculate_qe",
-    "2": "quantum_espresso_workflow.generate_structures",
-    "3": "quantum_espresso_workflow.calculate_qe",
-    "4": "quantum_espresso_workflow.calculate_qe",
-    "5": "quantum_espresso_workflow.calculate_qe",
-    "6": "quantum_espresso_workflow.calculate_qe",
-    "7": "quantum_espresso_workflow.calculate_qe",
-    "8": "quantum_espresso_workflow.plot_energy_volume_curve",
-    "9": "Al",
-    "10": 4.05,
-    "11": true,
-    "12": "mini",
-    "13": "python_workflow_definition.shared.get_dict",
-    "14": {"Al": "Al.pbe-n-kjpaw_psl.1.0.0.UPF"},
-    "15": [3, 3, 3],
-    "16": "vc-relax",
-    "17": 0.02,
-    "18": [0.9, 0.95, 1.0, 1.05, 1.1],
-    "19": "strain_0",
-    "20": "python_workflow_definition.shared.get_dict",
-    "21": "scf",
-    "22": "strain_1",
-    "23": "python_workflow_definition.shared.get_dict",
-    "24": "strain_2",
-    "25": "python_workflow_definition.shared.get_dict",
-    "26": "strain_3",
-    "27": "python_workflow_definition.shared.get_dict",
-    "28": "strain_4",
-    "29": "python_workflow_definition.shared.get_dict",
-    "30": "python_workflow_definition.shared.get_list",
-    "31": "python_workflow_definition.shared.get_list"
-  },
+  "nodes": [
+    {"id": 0, "function": "quantum_espresso_workflow.get_bulk_structure"},
+    {"id": 1, "function": "quantum_espresso_workflow.calculate_qe"},
+    {"id": 2, "function": "quantum_espresso_workflow.generate_structures"},
+    {"id": 3, "function": "quantum_espresso_workflow.calculate_qe"},
+    {"id": 4, "function": "quantum_espresso_workflow.calculate_qe"},
+    {"id": 5, "function": "quantum_espresso_workflow.calculate_qe"},
+    {"id": 6, "function": "quantum_espresso_workflow.calculate_qe"},
+    {"id": 7, "function": "quantum_espresso_workflow.calculate_qe"},
+    {"id": 8, "function": "quantum_espresso_workflow.plot_energy_volume_curve"},
+    {"id": 9, "value": "Al"},
+    {"id": 10, "value": 4.05},
+    {"id": 11, "value": true},
+    {"id": 12, "value": "mini"},
+    {"id": 13, "function": "python_workflow_definition.shared.get_dict"},
+    {"id": 14, "value": {"Al": "Al.pbe-n-kjpaw_psl.1.0.0.UPF"}},
+    {"id": 15, "value": [3, 3, 3]},
+    {"id": 16, "value": "vc-relax"},
+    {"id": 17, "value": 0.02},
+    {"id": 18, "value": [0.9, 0.95, 1.0, 1.05, 1.1]},
+    {"id": 19, "value": "strain_0"},
+    {"id": 20, "function": "python_workflow_definition.shared.get_dict"},
+    {"id": 21, "value": "scf"},
+    {"id": 22, "value": "strain_1"},
+    {"id": 23, "function": "python_workflow_definition.shared.get_dict"},
+    {"id": 24, "value": "strain_2"},
+    {"id": 25, "function": "python_workflow_definition.shared.get_dict"},
+    {"id": 26, "value": "strain_3"},
+    {"id": 27, "function": "python_workflow_definition.shared.get_dict"},
+    {"id": 28, "value": "strain_4"},
+    {"id": 29, "function": "python_workflow_definition.shared.get_dict"},
+    {"id": 30, "function": "python_workflow_definition.shared.get_list"},
+    {"id": 31, "function": "python_workflow_definition.shared.get_list"}
+  ],
   "edges": [
     {"target": 0, "targetPort": "element", "source": 9, "sourcePort": null},
     {"target": 0, "targetPort": "a", "source": 10, "sourcePort": null},