From 17aa8ceedddb82e7d52ade2e02f89a339baad473 Mon Sep 17 00:00:00 2001 From: DavHau Date: Fri, 7 Aug 2020 14:35:50 +0700 Subject: [PATCH] respect requires_python for wheels, support macos, add caching --- debug/debug.py | 4 + interpreter.nix | 2 +- mach_nix/cache.py | 21 +++ mach_nix/data/providers.py | 194 ++++++++++++---------- mach_nix/generators/overides_generator.py | 13 +- mach_nix/nix/PYPI_DEPS_DB_COMMIT | 2 +- mach_nix/nix/PYPI_DEPS_DB_SHA256 | 2 +- mach_nix/nix/python-deps.nix | 9 + mach_nix/requirements.py | 3 +- mach_nix/resolver/resolvelib_resolver.py | 2 + mach_nix/versions.py | 7 + mach_nix/visualize.py | 46 +++++ shell.nix | 2 +- 13 files changed, 214 insertions(+), 93 deletions(-) create mode 100644 mach_nix/cache.py create mode 100644 mach_nix/visualize.py diff --git a/debug/debug.py b/debug/debug.py index 565b0da..6b236b3 100644 --- a/debug/debug.py +++ b/debug/debug.py @@ -3,6 +3,7 @@ import os import subprocess as sp import tempfile from os.path import realpath, dirname +from time import time import toml @@ -43,4 +44,7 @@ with open(pwd + "/reqs.txt") as f: os.environ['requirements'] = f.read() # generates and writes nix expression into ./debug/expr.nix +start = time() main() +dur = round(time() - start, 1) +print(f"resolving took: {dur}s") diff --git a/interpreter.nix b/interpreter.nix index 188b0b2..72048c4 100644 --- a/interpreter.nix +++ b/interpreter.nix @@ -1,4 +1,4 @@ # python interpreter for dev environment import ./mach_nix/nix/python.nix { - pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix).stable { config = {}; }; + pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; }; } diff --git a/mach_nix/cache.py b/mach_nix/cache.py new file mode 100644 index 0000000..d178452 --- /dev/null +++ b/mach_nix/cache.py @@ -0,0 +1,21 @@ +from inspect import isgenerator + +cache = {} + + +def cached(keyfunc=None): + def cached_deco(func): + def cache_wrapper(*args, **kwargs): + args_save = keyf(args) + key = (func, args_save, tuple(kwargs.items())) + if key not in cache: + result = func(*args, **kwargs) + if isgenerator(result): + result = tuple(result) + cache[key] = result + return cache[key] + return cache_wrapper + + keyf = (lambda x: x) if keyfunc is None else keyfunc + return cached_deco + diff --git a/mach_nix/data/providers.py b/mach_nix/data/providers.py index 38fd4a7..a2a638c 100644 --- a/mach_nix/data/providers.py +++ b/mach_nix/data/providers.py @@ -1,24 +1,25 @@ import json +import platform import re import sys from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import List, Tuple, Iterable, Set +from typing import List, Tuple, Iterable import distlib.markers from packaging.version import Version, parse from .nixpkgs import NixpkgsDirectory from mach_nix.requirements import filter_reqs_by_eval_marker, Requirement, parse_reqs, context -from mach_nix.versions import PyVer, ver_sort_key +from mach_nix.versions import PyVer, ver_sort_key, filter_versions from .bucket_dict import LazyBucketDict +from ..cache import cached @dataclass class ProviderInfo: provider: str # following args are only required in case of wheel - wheel_pyver: str = None wheel_fname: str = None @@ -69,6 +70,7 @@ class DependencyProviderBase(ABC): self.context_wheel['extra'] = None self.py_ver_digits = py_ver.digits() + @cached() def available_versions(self, pkg_name: str) -> Iterable[Version]: """ returns available versions for given package name in reversed preference @@ -98,6 +100,7 @@ class DependencyProviderBase(ABC): pass @abstractmethod + @cached() def get_pkg_reqs(self, pkg_name, pkg_version, extras=None) -> Tuple[List[Requirement], List[Requirement]]: """ Get all requirements of a candidate for the current platform and the specified extras @@ -178,6 +181,7 @@ class CombinedDependencyProvider(DependencyProviderBase): print(error_text, file=sys.stderr) exit(1) + @cached() def available_versions(self, pkg_name: str) -> Iterable[Version]: # use dict as ordered set available_versions = [] @@ -202,11 +206,12 @@ class NixpkgsDependencyProvider(DependencyProviderBase): # ) # TODO: implement extras by looking them up via the equivalent wheel - def __init__(self, - nixpkgs: NixpkgsDirectory, - wheel_provider: 'WheelDependencyProvider', - sdist_provider: 'SdistDependencyProvider', - *args, **kwargs): + def __init__( + self, + nixpkgs: NixpkgsDirectory, + wheel_provider: 'WheelDependencyProvider', + sdist_provider: 'SdistDependencyProvider', + *args, **kwargs): super(NixpkgsDependencyProvider, self).__init__(*args, **kwargs) self.nixpkgs = nixpkgs self.wheel_provider = wheel_provider @@ -234,106 +239,126 @@ class NixpkgsDependencyProvider(DependencyProviderBase): return [] +@dataclass +class WheelRelease: + fn_pyver: str # the python version indicated by the filename + name: str + ver: str + fn: str + requires_dist: list + provided_extras: list + requires_python: str # the python version of the wheel metadata + + def __hash__(self): + return hash(self.fn) + + class WheelDependencyProvider(DependencyProviderBase): name = 'wheel' - def __init__(self, data_dir: str, *args, **kwargs): super(WheelDependencyProvider, self).__init__(*args, **kwargs) self.data = LazyBucketDict(data_dir) major, minor = self.py_ver_digits self.py_ver_re = re.compile(rf"^(py|cp)?{major}\.?{minor}?$") - self.preferred_wheels = ( - 'manylinux2014_x86_64.whl', - 'manylinux2010_x86_64.whl', - 'manylinux1_x86_64.whl', - 'none-any.whl' - ) + m = self.py_ver_digits[-1] + if platform.system() == "Linux": + self.preferred_wheels = ( + re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-manylinux2014_x86_64"), + re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-manylinux2010_x86_64"), + re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-manylinux1_x86_64"), + re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-linux_x86_64"), + re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-any"), + ) + elif platform.system() == "Darwin": + self.preferred_wheels = ( + re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_universal"), + re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_x86_64"), + re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_intel"), + re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_(fat64|fat32)"), + re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-any"),) + else: + raise Exception(f"Unsupported Platform {platform.system()}") def _available_versions(self, pkg_name: str) -> Iterable[Version]: - name = self.unify_key(pkg_name) - result = [] - for pyver in self._get_pyvers_for_pkg(name): - vers = self.data[name][pyver] - for ver, fnames in vers.items(): - for fn, deps in fnames.items(): - if self._wheel_ok(fn): - result.append(parse(ver)) - break - return result - - def choose_wheel(self, pkg_name, pkg_version: Version) -> Tuple[str, str]: - name = self.unify_key(pkg_name) - ver = str(pkg_version) - ok_fnames = {} - for pyver in self._get_pyvers_for_pkg(name): - if not ver in self.data[name][pyver]: - continue - fnames = self.data[name][pyver][ver] - ok_fnames = {fn: pyver for fn in fnames if - self._wheel_ok(fn) and any(fn.endswith(end) for end in self.preferred_wheels)} - if not ok_fnames: - raise Exception(f"No wheel available for {name}:{ver}") - if len(ok_fnames) > 1: - fn = self._select_preferred_wheel(ok_fnames) - else: - fn = list(ok_fnames.keys())[0] - pyver = ok_fnames[fn] - return pyver, fn - - def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo: - wheel_pyver, wheel_fname = self.choose_wheel(pkg_name, pkg_version) - return ProviderInfo(provider=self.name, wheel_pyver=wheel_pyver, wheel_fname=wheel_fname) - - def get_pkg_reqs_raw(self, pkg_name, pkg_version: Version): - name = self.unify_key(pkg_name) - if pkg_version not in self._available_versions(pkg_name): - raise PackageNotFound(pkg_name, pkg_version, self.name) - ver = str(pkg_version) - pyver, fn = self.choose_wheel(pkg_name, pkg_version) - deps = self.data[name][pyver][ver][fn] - if isinstance(deps, str): - key_ver, key_fn = deps.split('@') - versions = self.data[name][pyver] - deps = versions[key_ver][key_fn] - return deps['requires_dist'] if 'requires_dist' in deps else [] + return (parse(wheel.ver) for wheel in self._suitable_wheels(pkg_name)) def get_pkg_reqs(self, pkg_name, pkg_version: Version, extras=None) -> Tuple[List[Requirement], List[Requirement]]: """ Get requirements for package """ - reqs_raw = self.get_pkg_reqs_raw(pkg_name, pkg_version) + reqs_raw = self._choose_wheel(pkg_name, pkg_version).requires_dist + if reqs_raw is None: + reqs_raw = [] # handle extras by evaluationg markers install_reqs = list(filter_reqs_by_eval_marker(parse_reqs(reqs_raw), self.context_wheel, extras)) return install_reqs, [] - def _get_pyvers_for_pkg(self, name) -> Iterable: - name = self.unify_key(name) + def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo: + wheel = self._choose_wheel(pkg_name, pkg_version) + return ProviderInfo(provider=self.name, wheel_fname=wheel.fn) + + def _all_releases(self, pkg_name): + name = self.unify_key(pkg_name) if name not in self.data: return [] - ok_pyvers = (pyver for pyver in self.data[name].keys() if self._pyver_ok(pyver)) - return ok_pyvers + for fn_pyver, vers in self.data[name].items(): + for ver, fnames in vers.items(): + for fn, deps in fnames.items(): + if isinstance(deps, str): + key_ver, key_fn = deps.split('@') + versions = self.data[name][fn_pyver] + deps = versions[key_ver][key_fn] + assert isinstance(deps, dict) + yield WheelRelease( + fn_pyver, + name, + ver, + fn, + deps['requires_dist'] if 'requires_dist' in deps else None, + deps['requires_extras'] if 'requires_extras' in deps else None, + deps['requires_python'] if 'requires_python' in deps else None, + ) - def _select_preferred_wheel(self, filenames: Iterable[str]): - for key in self.preferred_wheels: - for fn in filenames: - if fn.endswith(key): - return fn - raise Exception("No wheel matches expected format") + def _apply_filters(self, filters: List[callable], objects: Iterable): + """ + Applies multiple filters to objects. First filter in the list is applied first + """ + assert len(filters) > 0 + if len(filters) == 1: + return filter(filters[0], objects) + return filter(filters[-1], self._apply_filters(filters[:-1], objects)) - def _pyver_ok(self, ver: str): - ver = ver.strip() - major, minor = self.py_ver_digits - if re.fullmatch(self.py_ver_re, ver) \ - or ver == f"py{major}"\ - or ver == "py2.py3": - return True - return False + @cached() + def _choose_wheel(self, pkg_name, pkg_version: Version) -> WheelRelease: + return self._select_preferred_wheel(self._suitable_wheels(pkg_name, pkg_version)) - def _wheel_ok(self, fn): - if fn.endswith('any.whl'): - return True - elif "manylinux" in fn: + def _suitable_wheels(self, pkg_name: str, ver: Version = None) -> Iterable[WheelRelease]: + wheels = self._all_releases(pkg_name) + if ver is not None: + wheels = filter(lambda w: parse(w.ver) == ver, wheels) + return self._apply_filters( + [ + self._wheel_type_ok, + self._python_requires_ok, + ], + wheels) + + def _select_preferred_wheel(self, wheels: Iterable[WheelRelease]): + wheels = list(wheels) + for pattern in self.preferred_wheels: + for wheel in wheels: + if re.search(pattern, wheel.fn): + return wheel + raise Exception("No wheel type found that is compatible to the current system") + + def _wheel_type_ok(self, wheel: WheelRelease): + return any(re.search(pattern, wheel.fn) for pattern in self.preferred_wheels) + + def _python_requires_ok(self, wheel: WheelRelease): + if not wheel.requires_python: return True + ver = parse('.'.join(self.py_ver_digits)) + return bool(filter_versions([ver], list(parse_reqs(f"python{wheel.requires_python}"))[0].specs)) class SdistDependencyProvider(DependencyProviderBase): @@ -342,6 +367,7 @@ class SdistDependencyProvider(DependencyProviderBase): self.data = LazyBucketDict(data_dir) super(SdistDependencyProvider, self).__init__(*args, **kwargs) + @cached() def _get_candidates(self, name) -> dict: """ returns all candidates for the give name which are available for the current python version diff --git a/mach_nix/generators/overides_generator.py b/mach_nix/generators/overides_generator.py index d627f06..fa3d6e9 100644 --- a/mach_nix/generators/overides_generator.py +++ b/mach_nix/generators/overides_generator.py @@ -14,10 +14,15 @@ def unindent(text: str, remove: int): class OverridesGenerator(ExpressionGenerator): - def __init__(self, py_ver, nixpkgs: NixpkgsDirectory, pypi_fetcher_commit, - pypi_fetcher_sha256, disable_checks, - *args, - **kwargs): + def __init__( + self, + py_ver, + nixpkgs: NixpkgsDirectory, + pypi_fetcher_commit, + pypi_fetcher_sha256, + disable_checks, + *args, + **kwargs): self.nixpkgs = nixpkgs self.disable_checks = disable_checks self.pypi_fetcher_commit = pypi_fetcher_commit diff --git a/mach_nix/nix/PYPI_DEPS_DB_COMMIT b/mach_nix/nix/PYPI_DEPS_DB_COMMIT index 39eb8f3..2017f40 100644 --- a/mach_nix/nix/PYPI_DEPS_DB_COMMIT +++ b/mach_nix/nix/PYPI_DEPS_DB_COMMIT @@ -1 +1 @@ -d8936568152dff29bd51c8bf492609a1f686fc7f \ No newline at end of file +2a2501aab1fe4eb50763652c4f74c5327f976f85 \ No newline at end of file diff --git a/mach_nix/nix/PYPI_DEPS_DB_SHA256 b/mach_nix/nix/PYPI_DEPS_DB_SHA256 index a14d5b2..4c60710 100644 --- a/mach_nix/nix/PYPI_DEPS_DB_SHA256 +++ b/mach_nix/nix/PYPI_DEPS_DB_SHA256 @@ -1 +1 @@ -1f9kazr6ny6v68d55qbdcccl34s97370lpd6w43rizpjdznv04hv \ No newline at end of file +03c64w1x4q2li57cn6zsb2shjk2fc4f5ksmsz3695dmvckp0lj14 \ No newline at end of file diff --git a/mach_nix/nix/python-deps.nix b/mach_nix/nix/python-deps.nix index c7228dc..ba29e7d 100644 --- a/mach_nix/nix/python-deps.nix +++ b/mach_nix/nix/python-deps.nix @@ -24,6 +24,15 @@ rec { }; doCheck = false; }; + tree-format = python.pkgs.buildPythonPackage { + name = "tree-format-0.1.2"; + src = fetchurl { + url = "https://files.pythonhosted.org/packages/0d/91/8d860c75c3e70e6bbec7b898b5f753bf5da404be9296e245034360759645/tree-format-0.1.2.tar.gz"; + sha256 = "a538523aa78ae7a4b10003b04f3e1b37708e0e089d99c9d3b9e1c71384c9a7f9"; + }; + doCheck = false; + }; + packaging = python.pkgs.packaging; setuptools = python.pkgs.setuptools; } diff --git a/mach_nix/requirements.py b/mach_nix/requirements.py index b85f149..7ebc92a 100644 --- a/mach_nix/requirements.py +++ b/mach_nix/requirements.py @@ -6,6 +6,7 @@ from distlib.markers import DEFAULT_CONTEXT from packaging.version import parse, _Version from pkg_resources._vendor.packaging.specifiers import SpecifierSet +from mach_nix.cache import cached from mach_nix.versions import PyVer @@ -58,7 +59,7 @@ def filter_reqs_by_eval_marker(reqs: Iterable[Requirement], context: dict, selec yield req - +@cached(lambda args: tuple(args[0]) if isinstance(args[0], list) else args[0]) def parse_reqs(strs): reqs = list(pkg_resources.parse_requirements(strs)) for req in reqs: diff --git a/mach_nix/resolver/resolvelib_resolver.py b/mach_nix/resolver/resolvelib_resolver.py index 8088712..b946576 100644 --- a/mach_nix/resolver/resolvelib_resolver.py +++ b/mach_nix/resolver/resolvelib_resolver.py @@ -9,6 +9,7 @@ from mach_nix.data.nixpkgs import NixpkgsDirectory from mach_nix.requirements import Requirement from mach_nix.resolver import Resolver, ResolvedPkg from mach_nix.versions import filter_versions +from mach_nix.visualize import print_deps @dataclass @@ -80,4 +81,5 @@ class ResolvelibResolver(Resolver): provider_info=provider_info, extras_selected=list(result.mapping[name].extras) )) + print_deps(nix_py_pkgs) return nix_py_pkgs diff --git a/mach_nix/versions.py b/mach_nix/versions.py index 79dd198..8132eeb 100644 --- a/mach_nix/versions.py +++ b/mach_nix/versions.py @@ -5,6 +5,8 @@ from typing import Iterable, Tuple, List from packaging.version import Version, parse, LegacyVersion +from mach_nix.cache import cached + class PyVer(Version): def nix(self): @@ -69,9 +71,14 @@ def best_version(versions: Iterable[Version]) -> Version: return best +@cached(keyfunc=lambda args: tuple(args[0]) + tuple(args[1])) def filter_versions( versions: Iterable[Version], specs: Iterable[Tuple[str, Version]]) -> List[Version]: + """ + Reduces a given list of versions to contain only versions + which are allowed according to the given specifiers + """ for op, ver in specs: ver = parse(ver) if op == '==': diff --git a/mach_nix/visualize.py b/mach_nix/visualize.py new file mode 100644 index 0000000..c314e8e --- /dev/null +++ b/mach_nix/visualize.py @@ -0,0 +1,46 @@ +from operator import itemgetter +from typing import Iterable +from tree_format import format_tree + +from mach_nix.resolver import ResolvedPkg + + +class Node: + def __init__(self, pkg: ResolvedPkg, parent: 'Node' = None): + self.pkg = pkg + pi = pkg.provider_info + self.name = f"{pkg.name} - {pkg.ver} - {pi.provider}" + if pi.provider == 'wheel': + self.name += f" - {'-'.join(pi.wheel_fname.split('-')[-3:])[:-4]}" + self.children = [] + self.parent = parent + if parent: + self.parent.children.append(self) + + +def build_tree(pkgs: dict, root: Node): + root_pkg = pkgs[root.pkg.name] + for name in sorted(root_pkg.build_inputs + root_pkg.prop_build_inputs): + child_pkg: ResolvedPkg = pkgs[name] + child_node = Node(child_pkg, root) + build_tree(pkgs, child_node) + + +def tree_to_dict(root_node: Node): + return root_node.name, [tree_to_dict(child) for child in root_node.children] + + +def print_tree(root_node): + d = tree_to_dict(root_node) + print(format_tree( + d, format_node=itemgetter(0), get_children=itemgetter(1))) + + +def print_deps(pkgs: Iterable[ResolvedPkg]): + print("\n### Resolved Dependencies ###\n") + indexed_pkgs = {p.name: p for p in sorted(pkgs, key=lambda p: p.name)} + roots: Iterable[ResolvedPkg] = (p for p in pkgs if p.is_root) + for root in roots: + root_node = Node(root) + build_tree(indexed_pkgs, root_node) + print_tree(root_node) diff --git a/shell.nix b/shell.nix index 590b227..84dba77 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,4 @@ -with import (import ./mach_nix/nix/nixpkgs-src.nix).stable { config = {}; }; +with import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; }; mkShell { buildInputs = [ (import ./mach_nix/nix/python.nix { inherit pkgs; })