fix circular dependency detection + removal

flake.nix

@@ -19,6 +19,7 @@
         packages = flake-utils.lib.flattenTree {
           mach-nix = mach-nix-default.mach-nix;
           "with" = mach-nix-default.pythonWith;
+          pythonWith = mach-nix-default.pythonWith;
           shellWith = mach-nix-default.shellWith;
           dockerImageWith = mach-nix-default.dockerImageWith;
         };

mach_nix/deptree.py

@@ -1,79 +1,79 @@
-from operator import itemgetter
-from typing import Iterable, List
+from typing import Iterable, Dict
 
+from networkx import DiGraph, NetworkXNoCycle
 from tree_format import format_tree
 
 from mach_nix.data.nixpkgs import NixpkgsIndex
 from mach_nix.resolver import ResolvedPkg
 
 
-class Node:
-    def __init__(self, pkg: ResolvedPkg, name, parent: 'Node' = None):
-        self.pkg = pkg
-        self.name = name
-        self.children = []
-        self.parent = parent
-        if parent:
-            self.parent.children.append(self)
-
-    def all_parents(self) -> List['Node']:
-        if self.parent is None:
-            return []
-        return [self.parent] + self.parent.all_parents()
+def mark_removed_circular_dep(pkgs: Dict[str, ResolvedPkg], G: DiGraph, node, removed_node):
+    pkgs[node].removed_circular_deps.add(removed_node)
+    for pred in G.predecessors(node):
+        mark_removed_circular_dep(pkgs, G, pred, removed_node)
 
 
-def make_name(pkg: ResolvedPkg, nixpkgs: NixpkgsIndex):
-    pi = pkg.provider_info
-    extras = f"[{' '.join(pkg.extras_selected)}]" if pkg.extras_selected else ''
-    name = f"{pkg.name}{extras} - {pkg.ver} - {pi.provider.name}"
-    if pi.provider == 'wheel':
-        name += f" - {'-'.join(pi.wheel_fname.split('-')[-3:])[:-4]}"
-    if pi.provider == 'nixpkgs':
-        name += f" (attrs: {' '.join(c.nix_key for c in nixpkgs.get_all_candidates(pkg.name))})"
-    return name
-
-
-def build_tree(pkgs: dict, root: Node, nixpkgs: NixpkgsIndex) -> List[str]:
-    """
-    Recursively adds children to given root node.
-    Removes cycles from original graph while processing.
-    Returns list of warnings.
-    """
-    root_pkg = pkgs[root.pkg.name]
-    warnings = []
-    for name in sorted(root_pkg.build_inputs + root_pkg.prop_build_inputs):
-        child_pkg: ResolvedPkg = pkgs[name]
-        # detect circles
-        if child_pkg in [node.pkg for node in root.all_parents()]:
-            warnings.append(
-                f"WARNING: Circular dependency detected and removed:"
-                f" {root.pkg.name}:{root.pkg.ver} -> {child_pkg.name}:{child_pkg.ver}")
-            root.pkg.build_inputs = [bi for bi in root.pkg.build_inputs if bi != child_pkg.name]
-            root.pkg.prop_build_inputs = [bi for bi in root.pkg.prop_build_inputs if bi != child_pkg.name]
-            root.pkg.removed_circular_deps.append(child_pkg.name)
-            continue
-        child_node = Node(child_pkg, make_name(child_pkg, nixpkgs), root)
-        warnings += build_tree(pkgs, child_node, nixpkgs)
-    return warnings
-
-
-def tree_to_dict(root_node: Node):
-    return root_node.name, [tree_to_dict(child) for child in root_node.children]
-
-
-def print_tree(root_node):
-    d = tree_to_dict(root_node)
-    print(format_tree(
-        d, format_node=itemgetter(0), get_children=itemgetter(1)))
+def remove_dependecy(pkgs: Dict[str, ResolvedPkg], G: DiGraph, node_from, node_to):
+    if node_to in pkgs[node_from].build_inputs:
+        raise Exception(
+            f"Fata error: cycle detected in setup requirements\n"
+            f"Cannot fix automatically.\n{[node_from, node_to]}")
+    G.remove_edge(node_from, node_to)
+    pkgs[node_from].prop_build_inputs.remove(node_to)
+    print(
+        f"WARNING: Circular dependency detected and removed:"
+        f" {node_from}:{ pkgs[node_from].ver} -> {node_to}:{ pkgs[node_to].ver}")
 
 
 def remove_circles_and_print(pkgs: Iterable[ResolvedPkg], nixpkgs: NixpkgsIndex):
+    import networkx as nx
     print("\n### Resolved Dependencies ###\n")
     indexed_pkgs = {p.name: p for p in sorted(pkgs, key=lambda p: p.name)}
-    roots: Iterable[ResolvedPkg] = (p for p in pkgs if p.is_root)
+    roots: Iterable[ResolvedPkg] = sorted([p for p in pkgs if p.is_root], key=lambda p: p.name)
+
+    edges = set()
+    for p in pkgs:
+        for child in p.build_inputs + p.prop_build_inputs:
+            edges.add((p.name, child))
+    G = nx.DiGraph(sorted(list(edges)))
+
+    cycle_count = 0
+    removed_edges = []
     for root in roots:
-        root_node = Node(root, make_name(root, nixpkgs))
-        warnings = build_tree(indexed_pkgs, root_node, nixpkgs)
-        print_tree(root_node)
-        if warnings:
-            print(''.join(warnings) + '\n')
+        try:
+            while True:
+                cycle = nx.find_cycle(G, root.name)
+                cycle_count += 1
+                remove_dependecy(indexed_pkgs, G, cycle[-1][0], cycle[-1][1])
+                removed_edges.append((cycle[-1][0], cycle[-1][1]))
+        except NetworkXNoCycle:
+            continue
+    for node, removed_node in removed_edges:
+        mark_removed_circular_dep(indexed_pkgs, G, node, removed_node)
+
+    class Limiter:
+        visited = set()
+
+        def name(self, node_name):
+            if node_name in self.visited:
+                if indexed_pkgs[node_name].build_inputs + indexed_pkgs[node_name].prop_build_inputs == []:
+                    return node_name
+                return f"{node_name} -> ..."
+            return node_name
+
+        def get_children(self, node_name):
+            if node_name in self.visited:
+                return []
+            self.visited.add(node_name)
+            return list(set(indexed_pkgs[node_name].build_inputs + indexed_pkgs[node_name].prop_build_inputs))
+
+    for root in roots:
+        limiter = Limiter()
+        print(format_tree(
+            root.name,
+            format_node=limiter.name,
+            get_children=limiter.get_children)
+        )
+    print(f"Total number of python modules: {len(indexed_pkgs)}")
+    print(f"Removed circular dependencies: {cycle_count}\n")
+

mach_nix/nix/python-deps.nix

@@ -25,6 +25,7 @@ rec {
     doCheck = false;
   };
 
+  networkx = python.pkgs.networkx;
   packaging = python.pkgs.packaging;
   setuptools = python.pkgs.setuptools;
   toml = python.pkgs.toml;

mach_nix/resolver/__init__.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
-from typing import List, Iterable, Optional
+from typing import List, Iterable, Set
 
 from packaging.version import Version
 
@@ -17,7 +17,8 @@ class ResolvedPkg:
     is_root: bool
     provider_info: ProviderInfo
     extras_selected: List[str]
-    removed_circular_deps: List[str] = field(default_factory=list)
+    # contains direct or indirect children wich have been diconnected due to circular deps
+    removed_circular_deps: Set[str] = field(default_factory=set)
 
 
 class Resolver(ABC):