fix circular dependency detection + removal
This commit is contained in:
parent
0d0d2003f9
commit
5b77d1c846
4 changed files with 69 additions and 66 deletions
|
@ -19,6 +19,7 @@
|
||||||
packages = flake-utils.lib.flattenTree {
|
packages = flake-utils.lib.flattenTree {
|
||||||
mach-nix = mach-nix-default.mach-nix;
|
mach-nix = mach-nix-default.mach-nix;
|
||||||
"with" = mach-nix-default.pythonWith;
|
"with" = mach-nix-default.pythonWith;
|
||||||
|
pythonWith = mach-nix-default.pythonWith;
|
||||||
shellWith = mach-nix-default.shellWith;
|
shellWith = mach-nix-default.shellWith;
|
||||||
dockerImageWith = mach-nix-default.dockerImageWith;
|
dockerImageWith = mach-nix-default.dockerImageWith;
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,79 +1,79 @@
|
||||||
from operator import itemgetter
|
from typing import Iterable, Dict
|
||||||
from typing import Iterable, List
|
|
||||||
|
|
||||||
|
from networkx import DiGraph, NetworkXNoCycle
|
||||||
from tree_format import format_tree
|
from tree_format import format_tree
|
||||||
|
|
||||||
from mach_nix.data.nixpkgs import NixpkgsIndex
|
from mach_nix.data.nixpkgs import NixpkgsIndex
|
||||||
from mach_nix.resolver import ResolvedPkg
|
from mach_nix.resolver import ResolvedPkg
|
||||||
|
|
||||||
|
|
||||||
class Node:
|
def mark_removed_circular_dep(pkgs: Dict[str, ResolvedPkg], G: DiGraph, node, removed_node):
|
||||||
def __init__(self, pkg: ResolvedPkg, name, parent: 'Node' = None):
|
pkgs[node].removed_circular_deps.add(removed_node)
|
||||||
self.pkg = pkg
|
for pred in G.predecessors(node):
|
||||||
self.name = name
|
mark_removed_circular_dep(pkgs, G, pred, removed_node)
|
||||||
self.children = []
|
|
||||||
self.parent = parent
|
|
||||||
if parent:
|
|
||||||
self.parent.children.append(self)
|
|
||||||
|
|
||||||
def all_parents(self) -> List['Node']:
|
|
||||||
if self.parent is None:
|
|
||||||
return []
|
|
||||||
return [self.parent] + self.parent.all_parents()
|
|
||||||
|
|
||||||
|
|
||||||
def make_name(pkg: ResolvedPkg, nixpkgs: NixpkgsIndex):
|
def remove_dependecy(pkgs: Dict[str, ResolvedPkg], G: DiGraph, node_from, node_to):
|
||||||
pi = pkg.provider_info
|
if node_to in pkgs[node_from].build_inputs:
|
||||||
extras = f"[{' '.join(pkg.extras_selected)}]" if pkg.extras_selected else ''
|
raise Exception(
|
||||||
name = f"{pkg.name}{extras} - {pkg.ver} - {pi.provider.name}"
|
f"Fata error: cycle detected in setup requirements\n"
|
||||||
if pi.provider == 'wheel':
|
f"Cannot fix automatically.\n{[node_from, node_to]}")
|
||||||
name += f" - {'-'.join(pi.wheel_fname.split('-')[-3:])[:-4]}"
|
G.remove_edge(node_from, node_to)
|
||||||
if pi.provider == 'nixpkgs':
|
pkgs[node_from].prop_build_inputs.remove(node_to)
|
||||||
name += f" (attrs: {' '.join(c.nix_key for c in nixpkgs.get_all_candidates(pkg.name))})"
|
print(
|
||||||
return name
|
f"WARNING: Circular dependency detected and removed:"
|
||||||
|
f" {node_from}:{ pkgs[node_from].ver} -> {node_to}:{ pkgs[node_to].ver}")
|
||||||
|
|
||||||
def build_tree(pkgs: dict, root: Node, nixpkgs: NixpkgsIndex) -> List[str]:
|
|
||||||
"""
|
|
||||||
Recursively adds children to given root node.
|
|
||||||
Removes cycles from original graph while processing.
|
|
||||||
Returns list of warnings.
|
|
||||||
"""
|
|
||||||
root_pkg = pkgs[root.pkg.name]
|
|
||||||
warnings = []
|
|
||||||
for name in sorted(root_pkg.build_inputs + root_pkg.prop_build_inputs):
|
|
||||||
child_pkg: ResolvedPkg = pkgs[name]
|
|
||||||
# detect circles
|
|
||||||
if child_pkg in [node.pkg for node in root.all_parents()]:
|
|
||||||
warnings.append(
|
|
||||||
f"WARNING: Circular dependency detected and removed:"
|
|
||||||
f" {root.pkg.name}:{root.pkg.ver} -> {child_pkg.name}:{child_pkg.ver}")
|
|
||||||
root.pkg.build_inputs = [bi for bi in root.pkg.build_inputs if bi != child_pkg.name]
|
|
||||||
root.pkg.prop_build_inputs = [bi for bi in root.pkg.prop_build_inputs if bi != child_pkg.name]
|
|
||||||
root.pkg.removed_circular_deps.append(child_pkg.name)
|
|
||||||
continue
|
|
||||||
child_node = Node(child_pkg, make_name(child_pkg, nixpkgs), root)
|
|
||||||
warnings += build_tree(pkgs, child_node, nixpkgs)
|
|
||||||
return warnings
|
|
||||||
|
|
||||||
|
|
||||||
def tree_to_dict(root_node: Node):
|
|
||||||
return root_node.name, [tree_to_dict(child) for child in root_node.children]
|
|
||||||
|
|
||||||
|
|
||||||
def print_tree(root_node):
|
|
||||||
d = tree_to_dict(root_node)
|
|
||||||
print(format_tree(
|
|
||||||
d, format_node=itemgetter(0), get_children=itemgetter(1)))
|
|
||||||
|
|
||||||
|
|
||||||
def remove_circles_and_print(pkgs: Iterable[ResolvedPkg], nixpkgs: NixpkgsIndex):
|
def remove_circles_and_print(pkgs: Iterable[ResolvedPkg], nixpkgs: NixpkgsIndex):
|
||||||
|
import networkx as nx
|
||||||
print("\n### Resolved Dependencies ###\n")
|
print("\n### Resolved Dependencies ###\n")
|
||||||
indexed_pkgs = {p.name: p for p in sorted(pkgs, key=lambda p: p.name)}
|
indexed_pkgs = {p.name: p for p in sorted(pkgs, key=lambda p: p.name)}
|
||||||
roots: Iterable[ResolvedPkg] = (p for p in pkgs if p.is_root)
|
roots: Iterable[ResolvedPkg] = sorted([p for p in pkgs if p.is_root], key=lambda p: p.name)
|
||||||
|
|
||||||
|
edges = set()
|
||||||
|
for p in pkgs:
|
||||||
|
for child in p.build_inputs + p.prop_build_inputs:
|
||||||
|
edges.add((p.name, child))
|
||||||
|
G = nx.DiGraph(sorted(list(edges)))
|
||||||
|
|
||||||
|
cycle_count = 0
|
||||||
|
removed_edges = []
|
||||||
for root in roots:
|
for root in roots:
|
||||||
root_node = Node(root, make_name(root, nixpkgs))
|
try:
|
||||||
warnings = build_tree(indexed_pkgs, root_node, nixpkgs)
|
while True:
|
||||||
print_tree(root_node)
|
cycle = nx.find_cycle(G, root.name)
|
||||||
if warnings:
|
cycle_count += 1
|
||||||
print(''.join(warnings) + '\n')
|
remove_dependecy(indexed_pkgs, G, cycle[-1][0], cycle[-1][1])
|
||||||
|
removed_edges.append((cycle[-1][0], cycle[-1][1]))
|
||||||
|
except NetworkXNoCycle:
|
||||||
|
continue
|
||||||
|
for node, removed_node in removed_edges:
|
||||||
|
mark_removed_circular_dep(indexed_pkgs, G, node, removed_node)
|
||||||
|
|
||||||
|
class Limiter:
|
||||||
|
visited = set()
|
||||||
|
|
||||||
|
def name(self, node_name):
|
||||||
|
if node_name in self.visited:
|
||||||
|
if indexed_pkgs[node_name].build_inputs + indexed_pkgs[node_name].prop_build_inputs == []:
|
||||||
|
return node_name
|
||||||
|
return f"{node_name} -> ..."
|
||||||
|
return node_name
|
||||||
|
|
||||||
|
def get_children(self, node_name):
|
||||||
|
if node_name in self.visited:
|
||||||
|
return []
|
||||||
|
self.visited.add(node_name)
|
||||||
|
return list(set(indexed_pkgs[node_name].build_inputs + indexed_pkgs[node_name].prop_build_inputs))
|
||||||
|
|
||||||
|
for root in roots:
|
||||||
|
limiter = Limiter()
|
||||||
|
print(format_tree(
|
||||||
|
root.name,
|
||||||
|
format_node=limiter.name,
|
||||||
|
get_children=limiter.get_children)
|
||||||
|
)
|
||||||
|
print(f"Total number of python modules: {len(indexed_pkgs)}")
|
||||||
|
print(f"Removed circular dependencies: {cycle_count}\n")
|
||||||
|
|
||||||
|
|
|
@ -25,6 +25,7 @@ rec {
|
||||||
doCheck = false;
|
doCheck = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
networkx = python.pkgs.networkx;
|
||||||
packaging = python.pkgs.packaging;
|
packaging = python.pkgs.packaging;
|
||||||
setuptools = python.pkgs.setuptools;
|
setuptools = python.pkgs.setuptools;
|
||||||
toml = python.pkgs.toml;
|
toml = python.pkgs.toml;
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import List, Iterable, Optional
|
from typing import List, Iterable, Set
|
||||||
|
|
||||||
from packaging.version import Version
|
from packaging.version import Version
|
||||||
|
|
||||||
|
@ -17,7 +17,8 @@ class ResolvedPkg:
|
||||||
is_root: bool
|
is_root: bool
|
||||||
provider_info: ProviderInfo
|
provider_info: ProviderInfo
|
||||||
extras_selected: List[str]
|
extras_selected: List[str]
|
||||||
removed_circular_deps: List[str] = field(default_factory=list)
|
# contains direct or indirect children wich have been diconnected due to circular deps
|
||||||
|
removed_circular_deps: Set[str] = field(default_factory=set)
|
||||||
|
|
||||||
|
|
||||||
class Resolver(ABC):
|
class Resolver(ABC):
|
||||||
|
|
Loading…
Reference in a new issue