From 54335eece63334d51b552ec7e246729bebd7c86b Mon Sep 17 00:00:00 2001 From: Jan-Oliver Joswig Date: Fri, 28 Aug 2020 18:01:15 +0200 Subject: [PATCH 1/2] MIGRATION: Python 3 --- .gitignore | 1 + debacl/level_set_tree.py | 49 +++++++++++++++++++-------------------- debacl/test/test_lst.py | 20 ++++++++-------- debacl/test/test_utils.py | 8 +++---- debacl/utils.py | 4 ++-- 5 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index f964f54..b752544 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ MANIFEST *.pyc *.lst examples/.ipynb_checkpoints +.vscode/ diff --git a/debacl/level_set_tree.py b/debacl/level_set_tree.py index 99cbcba..8ff0ff8 100644 --- a/debacl/level_set_tree.py +++ b/debacl/level_set_tree.py @@ -271,7 +271,7 @@ def plot(self, form='mass', horizontal_spacing='uniform', color_nodes=[], split_coords = {} ## Find the root connected components and corresponding plot intervals - ix_root = _np.array([k for k, v in self.nodes.iteritems() + ix_root = _np.array([k for k, v in self.nodes.items() if v.parent is None]) n_root = len(ix_root) census = _np.array([len(self.nodes[x].members) for x in ix_root], @@ -331,15 +331,15 @@ def plot(self, form='mass', horizontal_spacing='uniform', color_nodes=[], elif form == 'density': ax.set_ylabel("density level") - ymin = min([v.start_level for v in self.nodes.itervalues()]) - ymax = max([v.end_level for v in self.nodes.itervalues()]) + ymin = min([v.start_level for v in self.nodes.values()]) + ymax = max([v.end_level for v in self.nodes.values()]) yrange = ymax - ymin ax.set_ylim(ymin - gap * yrange, ymax + 0.05 * yrange) elif form == 'mass': ax.set_ylabel("mass level") - ymin = min([v.start_mass for v in self.nodes.itervalues()]) - ymax = max([v.end_mass for v in self.nodes.itervalues()]) + ymin = min([v.start_mass for v in self.nodes.values()]) + ymax = max([v.end_mass for v in self.nodes.values()]) yrange = ymax - ymin ax.set_ylim(ymin - gap * yrange, ymax + 0.05 * yrange) @@ -604,16 +604,16 @@ def _merge_by_size(self, threshold): tree.prune_threshold = threshold ## remove small root branches - small_roots = [k for k, v in tree.nodes.iteritems() + small_roots = [k for k, v in tree.nodes.items() if v.parent is None and len(v.members) <= threshold] for root in small_roots: root_tree = tree._make_subtree(root) - for ix in root_tree.nodes.iterkeys(): + for ix in root_tree.nodes.keys(): del tree.nodes[ix] ## main pruning - parents = [k for k, v in tree.nodes.iteritems() + parents = [k for k, v in tree.nodes.items() if len(v.children) >= 1] parents = _np.sort(parents)[::-1] @@ -624,7 +624,7 @@ def _merge_by_size(self, threshold): kid_size = {k: len(tree.nodes[k].members) for k in parent.children} # count children larger than 'threshold' - n_bigkid = sum(_np.array(kid_size.values()) >= threshold) + n_bigkid = sum(_np.array(list(kid_size.values())) >= threshold) if n_bigkid == 0: # update parent's end level and end mass @@ -641,7 +641,7 @@ def _merge_by_size(self, threshold): elif n_bigkid == 1: pass # identify the big kid - ix_bigkid = [k for k, v in kid_size.iteritems() + ix_bigkid = [k for k, v in kid_size.items() if v >= threshold][0] bigkid = tree.nodes[ix_bigkid] @@ -787,7 +787,7 @@ def _upper_set_cluster(self, threshold, form='mass'): else: upper_level_set = _np.where(_np.array(self.density) > threshold)[0] - active_nodes = [k for k, v in self.nodes.iteritems() + active_nodes = [k for k, v in self.nodes.items() if (v.start_level <= threshold and v.end_level > threshold)] @@ -835,7 +835,7 @@ def _first_K_level_cluster(self, k): """ cut = self._find_K_cut(k) - nodes = [e for e, v in self.nodes.iteritems() + nodes = [e for e, v in self.nodes.items() if v.start_level <= cut and v.end_level > cut] points = [] @@ -896,25 +896,25 @@ def _find_K_cut(self, k): """ ## Find the lowest level to cut at that has k or more clusters - starts = [v.start_level for v in self.nodes.itervalues()] - ends = [v.end_level for v in self.nodes.itervalues()] + starts = [v.start_level for v in self.nodes.values()] + ends = [v.end_level for v in self.nodes.values()] crits = _np.unique(starts + ends) nclust = {} for c in crits: - nclust[c] = len([e for e, v in self.nodes.iteritems() + nclust[c] = len([e for e, v in self.nodes.items() if v.start_level <= c and v.end_level > c]) width = _np.max(nclust.values()) if k in nclust.values(): - cut = _np.min([e for e, v in nclust.iteritems() if v == k]) + cut = _np.min([e for e, v in nclust.items() if v == k]) else: if width < k: - cut = _np.min([e for e, v in nclust.iteritems() if v == width]) + cut = _np.min([e for e, v in nclust.items() if v == width]) else: - ktemp = _np.min([v for v in nclust.itervalues() if v > k]) - cut = _np.min([e for e, v in nclust.iteritems() if v == ktemp]) + ktemp = _np.min([v for v in nclust.values() if v > k]) + cut = _np.min([e for e, v in nclust.items() if v == ktemp]) return cut @@ -1361,7 +1361,7 @@ def construct_tree_from_graph(adjacency_list, density, prune_threshold=None, cc0 = _nx.connected_components(G) for i, c in enumerate(cc0): # c is only the vertex list, not the subgraph - T._subgraphs[i] = G.subgraph(c) + T._subgraphs[i] = _nx.Graph(G.subgraph(c)) # unfreeze T.nodes[i] = ConnectedComponent( i, parent=None, children=[], start_level=0., end_level=None, start_mass=0., end_mass=None, members=c) @@ -1380,15 +1380,14 @@ def construct_tree_from_graph(adjacency_list, density, prune_threshold=None, ## compute the mass after the current bg set is removed old_vcount = sum([x.number_of_nodes() - for x in T._subgraphs.itervalues()]) + for x in T._subgraphs.values()]) current_mass = 1. - ((old_vcount - len(bg)) / n) # loop through active components, i.e. subgraphs deactivate_keys = [] # subgraphs to deactivate at the iter end activate_subgraphs = {} # new subgraphs to add at the end of the iter - for (k, H) in T._subgraphs.iteritems(): - + for k, H in T._subgraphs.items(): ## remove nodes at the current level H.remove_nodes_from(bg) @@ -1415,7 +1414,7 @@ def construct_tree_from_graph(adjacency_list, density, prune_threshold=None, for c in cc: new_key = max(T.nodes.keys()) + 1 T.nodes[k].children.append(new_key) - activate_subgraphs[new_key] = H.subgraph(c) + activate_subgraphs[new_key] = _nx.Graph(H.subgraph(c)) T.nodes[new_key] = ConnectedComponent( new_key, parent=k, children=[], start_level=level, @@ -1461,6 +1460,6 @@ def load_tree(filename): >>> tree2 = debacl.load_tree('my_tree') """ with open(filename, 'rb') as f: - T = _pickle.load(f) + T = _pickle.load(f, encoding="latin1") return T diff --git a/debacl/test/test_lst.py b/debacl/test/test_lst.py index 63a21bd..94ec16c 100644 --- a/debacl/test/test_lst.py +++ b/debacl/test/test_lst.py @@ -134,8 +134,8 @@ def _check_tree_correctness(self, tree): ans_end_levels = [0.104, 0.189, 1.001, 0.345, 0.741, 0.508, 0.381, 0.862, 0.804, 1.349, 1.004] - self.assertItemsEqual(start_levels, ans_start_levels) - self.assertItemsEqual(end_levels, ans_end_levels) + self.assertCountEqual(start_levels, ans_start_levels) + self.assertCountEqual(end_levels, ans_end_levels) ## Masses start_masses = [round(node.start_mass, 3) @@ -148,8 +148,8 @@ def _check_tree_correctness(self, tree): ans_end_masses = [0.018, 0.079, 0.947, 0.293, 0.667, 0.473, 0.359, 0.816, 0.734, 1.0, 0.949] - self.assertItemsEqual(start_masses, ans_start_masses) - self.assertItemsEqual(end_masses, ans_end_masses) + self.assertCountEqual(start_masses, ans_start_masses) + self.assertCountEqual(end_masses, ans_end_masses) ## Sizes and parents sizes = [len(node.members) for node in tree.nodes.values()] @@ -158,8 +158,8 @@ def _check_tree_correctness(self, tree): ans_sizes = [1000, 767, 215, 238, 472, 76, 23, 231, 12, 103, 48] ans_parents = [None, 0, 0, 1, 1, 3, 3, 4, 4, 7, 7] - self.assertItemsEqual(sizes, ans_sizes) - self.assertItemsEqual(parents, ans_parents) + self.assertCountEqual(sizes, ans_sizes) + self.assertCountEqual(parents, ans_parents) def test_construct_from_graph(self): """ @@ -262,7 +262,7 @@ def test_old_trees(self): self.assertEqual(len(partition[:, 0]), len(np.unique(partition[:, 0]))) self.assertEqual(np.min(partition[:, 0]), 0) - self.assertItemsEqual(np.unique(partition[:, 1]), + self.assertCountEqual(np.unique(partition[:, 1]), tree.nodes.keys()) # save and load @@ -270,7 +270,7 @@ def test_old_trees(self): tree.save(t.name) tree2 = dcl.load_tree(t.name) - self.assertItemsEqual( + self.assertCountEqual( [x.start_level for x in tree.nodes.values()], [y.start_level for y in tree2.nodes.values()]) @@ -481,7 +481,7 @@ def test_clusters(self): leaves = [idx for idx, node in self.tree.nodes.items() if len(node.children) == 0] - self.assertItemsEqual(np.unique(leaf_labels[:, 1]), leaves) + self.assertCountEqual(np.unique(leaf_labels[:, 1]), leaves) ## Check that background filling works correctly. full_labels = self.tree.get_clusters(method='leaf', @@ -519,5 +519,5 @@ def test_branch_partition(self): self.assertEqual(np.min(partition[:, 0]), 0) # Labels should match tree nodes exactly. - self.assertItemsEqual(np.unique(partition[:, 1]), + self.assertCountEqual(np.unique(partition[:, 1]), self.tree.nodes.keys()) diff --git a/debacl/test/test_utils.py b/debacl/test/test_utils.py index 28826f3..777e171 100644 --- a/debacl/test/test_utils.py +++ b/debacl/test/test_utils.py @@ -77,7 +77,7 @@ def test_knn_graph(self): assert_array_equal(radii, ans_radii) for neighbors, ans_neighbors in zip(knn, ans_graph): - self.assertItemsEqual(neighbors, ans_neighbors) + self.assertCountEqual(neighbors, ans_neighbors) def test_epsilon_graph(self): """ @@ -95,7 +95,7 @@ def test_epsilon_graph(self): enn = utl.epsilon_graph(self.X, epsilon) for neighbors, ans_neighbors in zip(enn, ans_graph): - self.assertItemsEqual(neighbors, ans_neighbors) + self.assertCountEqual(neighbors, ans_neighbors) class TestDensityGrids(unittest.TestCase): @@ -179,7 +179,7 @@ def test_mass_grid(self): ## Test uniform input. levels = utl.define_density_mass_grid(self.uniform_density) - self.assertItemsEqual(levels, [1.]) + self.assertCountEqual(levels, [1.]) def _check_level_grid_answer(self, density, levels): """ @@ -226,7 +226,7 @@ def test_level_grid(self): ## Uniform input should have a single value. levels = utl.define_density_level_grid(self.uniform_density) - self.assertItemsEqual(levels, [1.]) + self.assertCountEqual(levels, [1.]) class TestClusterReindexing(unittest.TestCase): diff --git a/debacl/utils.py b/debacl/utils.py index d69f506..3464533 100644 --- a/debacl/utils.py +++ b/debacl/utils.py @@ -484,6 +484,6 @@ def reindex_cluster_labels(labels): unique_labels = _np.unique(labels[:, 1]) label_map = {v: k for k, v in enumerate(unique_labels)} - new_labels = map(lambda x: label_map[x], labels[:, 1]) - new_labels = _np.vstack((labels[:, 0], new_labels)).T + new_labels = list(map(lambda x: label_map[x], labels[:, 1])) + new_labels = _np.vstack([labels[:, 0], new_labels]).T return new_labels From d7dd773c7d5c89c7c4d0678ca8ea10a0217c75ed Mon Sep 17 00:00:00 2001 From: Jan-Oliver Joswig Date: Tue, 1 Sep 2020 16:13:32 +0200 Subject: [PATCH 2/2] FIX: dict.items() + dict.items() --- debacl/level_set_tree.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debacl/level_set_tree.py b/debacl/level_set_tree.py index 8ff0ff8..529f2a5 100644 --- a/debacl/level_set_tree.py +++ b/debacl/level_set_tree.py @@ -1039,8 +1039,8 @@ def _construct_branch_map(self, ix, interval, form, horizontal_spacing, segmap += branch_segmap splitmap += branch_splitmap - splits = dict(splits.items() + branch_splits.items()) - segments = dict(segments.items() + branch_segs.items()) + splits = dict(list(splits.items()) + list(branch_splits.items())) + segments = dict(list(segments.items()) + list(branch_segs.items())) ## find the middle of the children's x-position and make vertical # segment ix @@ -1183,8 +1183,8 @@ def _construct_mass_map(self, ix, start_pile, interval, segmap += branch_segmap splitmap += branch_splitmap - splits = dict(splits.items() + branch_splits.items()) - segments = dict(segments.items() + branch_segs.items()) + splits = dict(list(splits.items()) + list(branch_splits.items())) + segments = dict(list(segments.items()) + list(branch_segs.items())) ## find the middle of the children's x-position and make vertical ## segment ix