From 2d2f50cb2ac4cd9ddad41ad503055d8021ad34f0 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Fri, 15 Oct 2021 15:46:46 -0600 Subject: [PATCH 1/3] Upgrade resolvelib to 0.4.0. --- mach_nix/data/providers.py | 62 +++++++++++++----------- mach_nix/nix/python-deps.nix | 6 +-- mach_nix/resolver/resolvelib_resolver.py | 4 +- setup.py | 2 +- 4 files changed, 41 insertions(+), 33 deletions(-) diff --git a/mach_nix/data/providers.py b/mach_nix/data/providers.py index 26094db..71f7a81 100644 --- a/mach_nix/data/providers.py +++ b/mach_nix/data/providers.py @@ -98,16 +98,21 @@ class DependencyProviderBase(ABC): self.platform = platform self.system = system - @cached() - def find_matches(self, req) -> List[Candidate]: - all = list(self.all_candidates_sorted(req.key, req.extras, req.build)) - matching_versions = set(filter_versions([c.ver for c in all], req)) + @cached(keyfunc=lambda args: (args[0], tuple(args[1]))) + def find_matches(self, reqs) -> List[Candidate]: + extras = tuple({extra for req in reqs for extra in req.extras}) + builds = tuple({req.build for req in reqs if req.build is not None}) + all = list(self.all_candidates_sorted(reqs[0].key, extras, builds)) + matching_versions = [c.ver for c in all] + for req in reqs: + matching_versions = filter_versions(matching_versions, req) + matching_versions = set(matching_versions) matching_candidates = [c for c in all if c.ver in matching_versions] return matching_candidates - def all_candidates_sorted(self, name, extras=None, build=None) -> Iterable[Candidate]: - candidates = list(self.all_candidates(name, extras, build)) - candidates.sort(key=lambda c: c.ver) + def all_candidates_sorted(self, name, extras, builds) -> Iterable[Candidate]: + candidates = list(self.all_candidates(name, extras, builds)) + candidates.sort(key=lambda c: c.ver, reverse=True) return candidates @property @@ -128,7 +133,7 @@ class DependencyProviderBase(ABC): pass @abstractmethod - def all_candidates(self, name, extras=None, build=None) -> Iterable[Candidate]: + def all_candidates(self, name, extras, builds) -> Iterable[Candidate]: pass @@ -203,18 +208,18 @@ class CombinedDependencyProvider(DependencyProviderBase): exit(1) @cached() - def all_candidates_sorted(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]: + def all_candidates_sorted(self, pkg_name, extras, builds) -> Iterable[Candidate]: # use dict as ordered set candidates = [] # order by reversed preference expected - for provider in reversed(tuple(self.allowed_providers_for_pkg(pkg_name).values())): - candidates += list(provider.all_candidates_sorted(pkg_name, extras, build)) + for provider in tuple(self.allowed_providers_for_pkg(pkg_name).values()): + candidates += list(provider.all_candidates_sorted(pkg_name, extras, builds)) if not candidates: - self.print_error_no_versions_available(pkg_name, extras, build) + self.print_error_no_versions_available(pkg_name, extras, builds) return tuple(candidates) - def all_candidates(self, name, extras=None, build=None) -> Iterable[Candidate]: - return self.all_candidates_sorted(name, extras, build) + def all_candidates(self, name, extras, builds) -> Iterable[Candidate]: + return self.all_candidates_sorted(name, extras, builds) class NixpkgsDependencyProvider(DependencyProviderBase): @@ -240,15 +245,15 @@ class NixpkgsDependencyProvider(DependencyProviderBase): for provider in (self.sdist_provider, self.wheel_provider): candidates = [ candidate - for candidate in provider.all_candidates(c.name) + for candidate in provider.all_candidates(c.name, None, None) if candidate.ver == c.ver ] if len(candidates) > 0: return provider.get_pkg_reqs(candidates[0]) return None, None - def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]: - if build: + def all_candidates(self, pkg_name, extras, builds) -> Iterable[Candidate]: + if builds: return [] name = self.unify_key(pkg_name) if not self.nixpkgs.exists(name): @@ -305,8 +310,8 @@ class WheelDependencyProvider(DependencyProviderBase): else: raise Exception(f"Unsupported Platform {platform.system()}") - def all_candidates(self, pkg_name, extras=None, build=None) -> List[Candidate]: - if build: + def all_candidates(self, pkg_name, extras, builds) -> List[Candidate]: + if builds: return [] return [Candidate( w.name, @@ -471,8 +476,8 @@ class SdistDependencyProvider(DependencyProviderBase): requirements['install_requires'] += self._get_reqs_for_extras(pkg, c.selected_extras) return requirements['install_requires'], requirements['setup_requires'] - def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]: - if build: + def all_candidates(self, pkg_name, extras, builds) -> Iterable[Candidate]: + if builds: return [] return [Candidate( pkg_name, @@ -573,16 +578,19 @@ class CondaDependencyProvider(DependencyProviderBase): @cached() def all_candidates_sorted(self, name, extras, build) -> Iterable[Candidate]: candidates = self.all_candidates(name, extras, build) - candidates.sort(key=lambda c: (c.ver, c.provider_info.data['build_number'])) + candidates.sort( + key=lambda c: (c.ver, c.provider_info.data["build_number"]), + reverse=True, + ) return candidates - def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]: + def all_candidates(self, pkg_name, extras, builds) -> Iterable[Candidate]: pkg_name = normalize_name(pkg_name) if pkg_name not in self.pkgs: return [] candidates = [] - for p in self.compatible_builds(pkg_name, build): + for p in self.compatible_builds(pkg_name, builds): if 'sha256' not in p: print( f"Ignoring conda package {p['name']}:{p['version']} from provider {self.channel} \n" @@ -625,11 +633,11 @@ class CondaDependencyProvider(DependencyProviderBase): return True @cached() - def compatible_builds(self, pkg_name, build_pattern) -> list: - if build_pattern: + def compatible_builds(self, pkg_name, build_patterns) -> list: + if build_patterns: # fnmatch caches the regexes it builds. def build_matches(build): - return fnmatch.fnmatch(build, build_pattern) + return all((fnmatch.fnmatch(build, build_pattern) for build_pattern in build_patterns)) else: def build_matches(build): return True diff --git a/mach_nix/nix/python-deps.nix b/mach_nix/nix/python-deps.nix index 6ca6bc6..2433efc 100644 --- a/mach_nix/nix/python-deps.nix +++ b/mach_nix/nix/python-deps.nix @@ -9,10 +9,10 @@ rec { doCheck = false; }; resolvelib = python.pkgs.buildPythonPackage { - name = "resolvelib-0.3.0"; + name = "resolvelib-0.4.0"; src = fetchurl { - url = "https://files.pythonhosted.org/packages/e1/84/5c20d9bed18041343eeb537cc2b76aa17c18102ecf5873c12cd78a04cc69/resolvelib-0.3.0.tar.gz"; - sha256 = "9781c2038be2ba3377d075dd3aa8f5f0f7b508b6f59779b1414bea08ed402f1e"; + url = "https://files.pythonhosted.org/packages/20/64/5a93bc4f169f84c45107e69292a5a81d63fa50b6a23431005a68c6ae888c/resolvelib-0.4.0.tar.gz"; + sha256 = "93e42cf712534cf8dccb146c0c20521a5ac344ef8ec6e7742a49b22a24335662"; }; doCheck = false; }; diff --git a/mach_nix/resolver/resolvelib_resolver.py b/mach_nix/resolver/resolvelib_resolver.py index 1db84de..cf5aad4 100644 --- a/mach_nix/resolver/resolvelib_resolver.py +++ b/mach_nix/resolver/resolvelib_resolver.py @@ -28,8 +28,8 @@ class Provider(resolvelib.providers.AbstractProvider): def get_preference(self, resolution, candidates, information): return len(candidates) - def find_matches(self, req): - return self.provider.find_matches(req) + def find_matches(self, reqs): + return self.provider.find_matches(reqs) def is_satisfied_by(self, requirement, candidate: Candidate): res = bool(len(list(filter_versions([candidate.ver], requirement)))) diff --git a/setup.py b/setup.py index a051707..2e0b7c8 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( 'distlib == 0.3.3', 'networkx', 'packaging >= 19.0', - 'resolvelib == 0.3.0', + 'resolvelib == 0.4.0', 'toml', 'tree-format == 0.1.2', ], From 3ecc3bfbf104452a2d73803e5a1f21ba3d66eec1 Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 9 Feb 2022 12:55:25 -0700 Subject: [PATCH 2/3] Upgrade resolvelib to 0.6.0. --- mach_nix/nix/python-deps.nix | 6 +++--- mach_nix/resolver/resolvelib_resolver.py | 12 ++++++++---- setup.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/mach_nix/nix/python-deps.nix b/mach_nix/nix/python-deps.nix index 2433efc..6cd9102 100644 --- a/mach_nix/nix/python-deps.nix +++ b/mach_nix/nix/python-deps.nix @@ -9,10 +9,10 @@ rec { doCheck = false; }; resolvelib = python.pkgs.buildPythonPackage { - name = "resolvelib-0.4.0"; + name = "resolvelib-0.6.0"; src = fetchurl { - url = "https://files.pythonhosted.org/packages/20/64/5a93bc4f169f84c45107e69292a5a81d63fa50b6a23431005a68c6ae888c/resolvelib-0.4.0.tar.gz"; - sha256 = "93e42cf712534cf8dccb146c0c20521a5ac344ef8ec6e7742a49b22a24335662"; + url = "https://files.pythonhosted.org/packages/ab/38/6564745f294e7182b1d2fcb04acde9c74f376f63e0fd0be6788af176fcc8/resolvelib-0.6.0.tar.gz"; + sha256 = "9da653f664be0fba1a1ee9b339f0046a84d084e5c1bcab0469eab941a63f5117"; }; doCheck = false; }; diff --git a/mach_nix/resolver/resolvelib_resolver.py b/mach_nix/resolver/resolvelib_resolver.py index cf5aad4..509442d 100644 --- a/mach_nix/resolver/resolvelib_resolver.py +++ b/mach_nix/resolver/resolvelib_resolver.py @@ -22,14 +22,18 @@ class Provider(resolvelib.providers.AbstractProvider): def get_base_requirement(self, candidate): return Requirement("{}=={}".format(candidate.name, candidate.ver)) - def identify(self, dependency): - return dependency.name + def identify(self, requirement_or_candidate): + return requirement_or_candidate.name def get_preference(self, resolution, candidates, information): return len(candidates) - def find_matches(self, reqs): - return self.provider.find_matches(reqs) + def find_matches(self, identifier, requirements, incompatibilities): + return [ + candidate + for candidate in self.provider.find_matches(list(requirements[identifier])) + if candidate not in incompatibilities.get(identifier, ()) + ] def is_satisfied_by(self, requirement, candidate: Candidate): res = bool(len(list(filter_versions([candidate.ver], requirement)))) diff --git a/setup.py b/setup.py index 2e0b7c8..1aaf937 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( 'distlib == 0.3.3', 'networkx', 'packaging >= 19.0', - 'resolvelib == 0.4.0', + 'resolvelib == 0.6.0', 'toml', 'tree-format == 0.1.2', ], From 8c0a86b93b5d98bc9a717aa14b1f34be4d5b0c3e Mon Sep 17 00:00:00 2001 From: Tom Prince Date: Wed, 9 Feb 2022 13:09:18 -0700 Subject: [PATCH 3/3] Upgrade resolvelib to 0.8.1. --- mach_nix/nix/python-deps.nix | 6 +++--- mach_nix/resolver/resolvelib_resolver.py | 9 +++++++-- setup.py | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mach_nix/nix/python-deps.nix b/mach_nix/nix/python-deps.nix index 6cd9102..7a620d9 100644 --- a/mach_nix/nix/python-deps.nix +++ b/mach_nix/nix/python-deps.nix @@ -9,10 +9,10 @@ rec { doCheck = false; }; resolvelib = python.pkgs.buildPythonPackage { - name = "resolvelib-0.6.0"; + name = "resolvelib-0.8.1"; src = fetchurl { - url = "https://files.pythonhosted.org/packages/ab/38/6564745f294e7182b1d2fcb04acde9c74f376f63e0fd0be6788af176fcc8/resolvelib-0.6.0.tar.gz"; - sha256 = "9da653f664be0fba1a1ee9b339f0046a84d084e5c1bcab0469eab941a63f5117"; + url = "https://files.pythonhosted.org/packages/ac/20/9541749d77aebf66dd92e2b803f38a50e3a5c76e7876f45eb2b37e758d82/resolvelib-0.8.1.tar.gz"; + sha256 = "c6ea56732e9fb6fca1b2acc2ccc68a0b6b8c566d8f3e78e0443310ede61dbd37"; }; doCheck = false; }; diff --git a/mach_nix/resolver/resolvelib_resolver.py b/mach_nix/resolver/resolvelib_resolver.py index 509442d..7dbeaab 100644 --- a/mach_nix/resolver/resolvelib_resolver.py +++ b/mach_nix/resolver/resolvelib_resolver.py @@ -25,8 +25,13 @@ class Provider(resolvelib.providers.AbstractProvider): def identify(self, requirement_or_candidate): return requirement_or_candidate.name - def get_preference(self, resolution, candidates, information): - return len(candidates) + def get_preference( + self, identifier, resolutions, candidates, information, backtrack_causes + ): + # This logic could be improved, for example to prefer the cause of backtracking. + # See https://github.com/pypa/pip/blob/main/src/pip/_internal/resolution/resolvelib/provider.py + # for the implementation in pip. + return sum(1 for _ in candidates[identifier]) def find_matches(self, identifier, requirements, incompatibilities): return [ diff --git a/setup.py b/setup.py index 1aaf937..a7e3e70 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( 'distlib == 0.3.3', 'networkx', 'packaging >= 19.0', - 'resolvelib == 0.6.0', + 'resolvelib == 0.8.1', 'toml', 'tree-format == 0.1.2', ],