respect requires_python for wheels, support macos, add caching

debug/debug.py

@@ -3,6 +3,7 @@ import os
 import subprocess as sp
 import tempfile
 from os.path import realpath, dirname
+from time import time
 
 import toml
 
@@ -43,4 +44,7 @@ with open(pwd + "/reqs.txt") as f:
     os.environ['requirements'] = f.read()
 
 # generates and writes nix expression into ./debug/expr.nix
+start = time()
 main()
+dur = round(time() - start, 1)
+print(f"resolving took: {dur}s")

interpreter.nix

@@ -1,4 +1,4 @@
 # python interpreter for dev environment
 import ./mach_nix/nix/python.nix {
-  pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix).stable { config = {}; };
+  pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; };
 }

mach_nix/cache.py

@@ -0,0 +1,21 @@
+from inspect import isgenerator
+
+cache = {}
+
+
+def cached(keyfunc=None):
+    def cached_deco(func):
+        def cache_wrapper(*args, **kwargs):
+            args_save = keyf(args)
+            key = (func, args_save, tuple(kwargs.items()))
+            if key not in cache:
+                result = func(*args, **kwargs)
+                if isgenerator(result):
+                    result = tuple(result)
+                cache[key] = result
+            return cache[key]
+        return cache_wrapper
+
+    keyf = (lambda x: x) if keyfunc is None else keyfunc
+    return cached_deco
+

mach_nix/data/providers.py

@@ -1,24 +1,25 @@
 import json
+import platform
 import re
 import sys
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import List, Tuple, Iterable, Set
+from typing import List, Tuple, Iterable
 
 import distlib.markers
 from packaging.version import Version, parse
 
 from .nixpkgs import NixpkgsDirectory
 from mach_nix.requirements import filter_reqs_by_eval_marker, Requirement, parse_reqs, context
-from mach_nix.versions import PyVer, ver_sort_key
+from mach_nix.versions import PyVer, ver_sort_key, filter_versions
 from .bucket_dict import LazyBucketDict
+from ..cache import cached
 
 
 @dataclass
 class ProviderInfo:
     provider: str
     # following args are only required in case of wheel
-    wheel_pyver: str = None
     wheel_fname: str = None
 
 
@@ -69,6 +70,7 @@ class DependencyProviderBase(ABC):
         self.context_wheel['extra'] = None
         self.py_ver_digits = py_ver.digits()
 
+    @cached()
     def available_versions(self, pkg_name: str) -> Iterable[Version]:
         """
         returns available versions for given package name in reversed preference
@@ -98,6 +100,7 @@ class DependencyProviderBase(ABC):
         pass
 
     @abstractmethod
+    @cached()
     def get_pkg_reqs(self, pkg_name, pkg_version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
         """
         Get all requirements of a candidate for the current platform and the specified extras
@@ -178,6 +181,7 @@ class CombinedDependencyProvider(DependencyProviderBase):
         print(error_text, file=sys.stderr)
         exit(1)
 
+    @cached()
     def available_versions(self, pkg_name: str) -> Iterable[Version]:
         # use dict as ordered set
         available_versions = []
@@ -202,11 +206,12 @@ class NixpkgsDependencyProvider(DependencyProviderBase):
     # )
 
     # TODO: implement extras by looking them up via the equivalent wheel
-    def __init__(self,
-                 nixpkgs: NixpkgsDirectory,
-                 wheel_provider: 'WheelDependencyProvider',
-                 sdist_provider: 'SdistDependencyProvider',
-                 *args, **kwargs):
+    def __init__(
+            self,
+            nixpkgs: NixpkgsDirectory,
+            wheel_provider: 'WheelDependencyProvider',
+            sdist_provider: 'SdistDependencyProvider',
+            *args, **kwargs):
         super(NixpkgsDependencyProvider, self).__init__(*args, **kwargs)
         self.nixpkgs = nixpkgs
         self.wheel_provider = wheel_provider
@@ -234,106 +239,126 @@ class NixpkgsDependencyProvider(DependencyProviderBase):
         return []
 
 
+@dataclass
+class WheelRelease:
+    fn_pyver: str  # the python version indicated by the filename
+    name: str
+    ver: str
+    fn: str
+    requires_dist: list
+    provided_extras: list
+    requires_python: str  # the python version of the wheel metadata
+
+    def __hash__(self):
+        return hash(self.fn)
+
+
 class WheelDependencyProvider(DependencyProviderBase):
     name = 'wheel'
-
     def __init__(self, data_dir: str, *args, **kwargs):
         super(WheelDependencyProvider, self).__init__(*args, **kwargs)
         self.data = LazyBucketDict(data_dir)
         major, minor = self.py_ver_digits
         self.py_ver_re = re.compile(rf"^(py|cp)?{major}\.?{minor}?$")
-        self.preferred_wheels = (
-            'manylinux2014_x86_64.whl',
-            'manylinux2010_x86_64.whl',
-            'manylinux1_x86_64.whl',
-            'none-any.whl'
-        )
+        m = self.py_ver_digits[-1]
+        if platform.system() == "Linux":
+            self.preferred_wheels = (
+                re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-manylinux2014_x86_64"),
+                re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-manylinux2010_x86_64"),
+                re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-manylinux1_x86_64"),
+                re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-linux_x86_64"),
+                re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-any"),
+            )
+        elif platform.system() == "Darwin":
+            self.preferred_wheels = (
+                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_universal"),
+                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_x86_64"),
+                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_intel"),
+                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_(fat64|fat32)"),
+                re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-any"),)
+        else:
+            raise Exception(f"Unsupported Platform {platform.system()}")
 
     def _available_versions(self, pkg_name: str) -> Iterable[Version]:
-        name = self.unify_key(pkg_name)
-        result = []
-        for pyver in self._get_pyvers_for_pkg(name):
-            vers = self.data[name][pyver]
-            for ver, fnames in vers.items():
-                for fn, deps in fnames.items():
-                    if self._wheel_ok(fn):
-                        result.append(parse(ver))
-                        break
-        return result
-
-    def choose_wheel(self, pkg_name, pkg_version: Version) -> Tuple[str, str]:
-        name = self.unify_key(pkg_name)
-        ver = str(pkg_version)
-        ok_fnames = {}
-        for pyver in self._get_pyvers_for_pkg(name):
-            if not ver in self.data[name][pyver]:
-                continue
-            fnames = self.data[name][pyver][ver]
-            ok_fnames = {fn: pyver for fn in fnames if
-                         self._wheel_ok(fn) and any(fn.endswith(end) for end in self.preferred_wheels)}
-        if not ok_fnames:
-            raise Exception(f"No wheel available for {name}:{ver}")
-        if len(ok_fnames) > 1:
-            fn = self._select_preferred_wheel(ok_fnames)
-        else:
-            fn = list(ok_fnames.keys())[0]
-        pyver = ok_fnames[fn]
-        return pyver, fn
-
-    def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo:
-        wheel_pyver, wheel_fname = self.choose_wheel(pkg_name, pkg_version)
-        return ProviderInfo(provider=self.name, wheel_pyver=wheel_pyver, wheel_fname=wheel_fname)
-
-    def get_pkg_reqs_raw(self, pkg_name, pkg_version: Version):
-        name = self.unify_key(pkg_name)
-        if pkg_version not in self._available_versions(pkg_name):
-            raise PackageNotFound(pkg_name, pkg_version, self.name)
-        ver = str(pkg_version)
-        pyver, fn = self.choose_wheel(pkg_name, pkg_version)
-        deps = self.data[name][pyver][ver][fn]
-        if isinstance(deps, str):
-            key_ver, key_fn = deps.split('@')
-            versions = self.data[name][pyver]
-            deps = versions[key_ver][key_fn]
-        return deps['requires_dist'] if 'requires_dist' in deps else []
+        return (parse(wheel.ver) for wheel in self._suitable_wheels(pkg_name))
 
     def get_pkg_reqs(self, pkg_name, pkg_version: Version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
         """
         Get requirements for package
         """
-        reqs_raw = self.get_pkg_reqs_raw(pkg_name, pkg_version)
+        reqs_raw = self._choose_wheel(pkg_name, pkg_version).requires_dist
+        if reqs_raw is None:
+            reqs_raw = []
         # handle extras by evaluationg markers
         install_reqs = list(filter_reqs_by_eval_marker(parse_reqs(reqs_raw), self.context_wheel, extras))
         return install_reqs, []
 
-    def _get_pyvers_for_pkg(self, name) -> Iterable:
-        name = self.unify_key(name)
+    def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo:
+        wheel = self._choose_wheel(pkg_name, pkg_version)
+        return ProviderInfo(provider=self.name, wheel_fname=wheel.fn)
+
+    def _all_releases(self, pkg_name):
+        name = self.unify_key(pkg_name)
         if name not in self.data:
             return []
-        ok_pyvers = (pyver for pyver in self.data[name].keys() if self._pyver_ok(pyver))
-        return ok_pyvers
+        for fn_pyver, vers in self.data[name].items():
+            for ver, fnames in vers.items():
+                for fn, deps in fnames.items():
+                    if isinstance(deps, str):
+                        key_ver, key_fn = deps.split('@')
+                        versions = self.data[name][fn_pyver]
+                        deps = versions[key_ver][key_fn]
+                    assert isinstance(deps, dict)
+                    yield WheelRelease(
+                        fn_pyver,
+                        name,
+                        ver,
+                        fn,
+                        deps['requires_dist'] if 'requires_dist' in deps else None,
+                        deps['requires_extras'] if 'requires_extras' in deps else None,
+                        deps['requires_python'] if 'requires_python' in deps else None,
+                    )
 
-    def _select_preferred_wheel(self, filenames: Iterable[str]):
-        for key in self.preferred_wheels:
-            for fn in filenames:
-                if fn.endswith(key):
-                    return fn
-        raise Exception("No wheel matches expected format")
+    def _apply_filters(self, filters: List[callable], objects: Iterable):
+        """
+        Applies multiple filters to objects. First filter in the list is applied first
+        """
+        assert len(filters) > 0
+        if len(filters) == 1:
+            return filter(filters[0], objects)
+        return filter(filters[-1], self._apply_filters(filters[:-1], objects))
 
-    def _pyver_ok(self, ver: str):
-        ver = ver.strip()
-        major, minor = self.py_ver_digits
-        if re.fullmatch(self.py_ver_re, ver) \
-                or ver == f"py{major}"\
-                or ver == "py2.py3":
-            return True
-        return False
+    @cached()
+    def _choose_wheel(self, pkg_name, pkg_version: Version) -> WheelRelease:
+        return self._select_preferred_wheel(self._suitable_wheels(pkg_name, pkg_version))
 
-    def _wheel_ok(self, fn):
-        if fn.endswith('any.whl'):
-            return True
-        elif "manylinux" in fn:
+    def _suitable_wheels(self, pkg_name: str, ver: Version = None) -> Iterable[WheelRelease]:
+        wheels = self._all_releases(pkg_name)
+        if ver is not None:
+            wheels = filter(lambda w: parse(w.ver) == ver, wheels)
+        return self._apply_filters(
+            [
+                self._wheel_type_ok,
+                self._python_requires_ok,
+            ],
+            wheels)
+
+    def _select_preferred_wheel(self, wheels: Iterable[WheelRelease]):
+        wheels = list(wheels)
+        for pattern in self.preferred_wheels:
+            for wheel in wheels:
+                if re.search(pattern, wheel.fn):
+                    return wheel
+        raise Exception("No wheel type found that is compatible to the current system")
+
+    def _wheel_type_ok(self, wheel: WheelRelease):
+        return any(re.search(pattern, wheel.fn) for pattern in self.preferred_wheels)
+
+    def _python_requires_ok(self, wheel: WheelRelease):
+        if not wheel.requires_python:
             return True
+        ver = parse('.'.join(self.py_ver_digits))
+        return bool(filter_versions([ver], list(parse_reqs(f"python{wheel.requires_python}"))[0].specs))
 
 
 class SdistDependencyProvider(DependencyProviderBase):
@@ -342,6 +367,7 @@ class SdistDependencyProvider(DependencyProviderBase):
         self.data = LazyBucketDict(data_dir)
         super(SdistDependencyProvider, self).__init__(*args, **kwargs)
 
+    @cached()
     def _get_candidates(self, name) -> dict:
         """
         returns all candidates for the give name which are available for the current python version

mach_nix/generators/overides_generator.py

@@ -14,10 +14,15 @@ def unindent(text: str, remove: int):
 
 class OverridesGenerator(ExpressionGenerator):
 
-    def __init__(self, py_ver, nixpkgs: NixpkgsDirectory, pypi_fetcher_commit,
-                 pypi_fetcher_sha256, disable_checks,
-                 *args,
-                 **kwargs):
+    def __init__(
+            self,
+            py_ver,
+            nixpkgs: NixpkgsDirectory,
+            pypi_fetcher_commit,
+            pypi_fetcher_sha256,
+            disable_checks,
+            *args,
+            **kwargs):
         self.nixpkgs = nixpkgs
         self.disable_checks = disable_checks
         self.pypi_fetcher_commit = pypi_fetcher_commit

mach_nix/nix/PYPI_DEPS_DB_COMMIT

@@ -1 +1 @@
-d8936568152dff29bd51c8bf492609a1f686fc7f
+2a2501aab1fe4eb50763652c4f74c5327f976f85

mach_nix/nix/PYPI_DEPS_DB_SHA256

@@ -1 +1 @@
-1f9kazr6ny6v68d55qbdcccl34s97370lpd6w43rizpjdznv04hv
+03c64w1x4q2li57cn6zsb2shjk2fc4f5ksmsz3695dmvckp0lj14

mach_nix/nix/python-deps.nix

@@ -24,6 +24,15 @@ rec {
     };
     doCheck = false;
   };
+  tree-format = python.pkgs.buildPythonPackage {
+    name = "tree-format-0.1.2";
+    src = fetchurl {
+      url = "https://files.pythonhosted.org/packages/0d/91/8d860c75c3e70e6bbec7b898b5f753bf5da404be9296e245034360759645/tree-format-0.1.2.tar.gz";
+      sha256 = "a538523aa78ae7a4b10003b04f3e1b37708e0e089d99c9d3b9e1c71384c9a7f9";
+    };
+    doCheck = false;
+  };
+
   packaging = python.pkgs.packaging;
   setuptools = python.pkgs.setuptools;
 }

mach_nix/requirements.py

@@ -6,6 +6,7 @@ from distlib.markers import DEFAULT_CONTEXT
 from packaging.version import parse, _Version
 from pkg_resources._vendor.packaging.specifiers import SpecifierSet
 
+from mach_nix.cache import cached
 from mach_nix.versions import PyVer
 
 
@@ -58,7 +59,7 @@ def filter_reqs_by_eval_marker(reqs: Iterable[Requirement], context: dict, selec
                 yield req
 
 
-
+@cached(lambda args: tuple(args[0]) if isinstance(args[0], list) else args[0])
 def parse_reqs(strs):
     reqs = list(pkg_resources.parse_requirements(strs))
     for req in reqs:

mach_nix/resolver/resolvelib_resolver.py

@@ -9,6 +9,7 @@ from mach_nix.data.nixpkgs import NixpkgsDirectory
 from mach_nix.requirements import Requirement
 from mach_nix.resolver import Resolver, ResolvedPkg
 from mach_nix.versions import filter_versions
+from mach_nix.visualize import print_deps
 
 
 @dataclass
@@ -80,4 +81,5 @@ class ResolvelibResolver(Resolver):
                 provider_info=provider_info,
                 extras_selected=list(result.mapping[name].extras)
             ))
+        print_deps(nix_py_pkgs)
         return nix_py_pkgs

mach_nix/versions.py

@@ -5,6 +5,8 @@ from typing import Iterable, Tuple, List
 
 from packaging.version import Version, parse, LegacyVersion
 
+from mach_nix.cache import cached
+
 
 class PyVer(Version):
     def nix(self):
@@ -69,9 +71,14 @@ def best_version(versions: Iterable[Version]) -> Version:
     return best
 
 
+@cached(keyfunc=lambda args: tuple(args[0]) + tuple(args[1]))
 def filter_versions(
         versions: Iterable[Version],
         specs: Iterable[Tuple[str, Version]]) -> List[Version]:
+    """
+    Reduces a given list of versions to contain only versions
+    which are allowed according to the given specifiers
+    """
     for op, ver in specs:
         ver = parse(ver)
         if op == '==':

mach_nix/visualize.py

@@ -0,0 +1,46 @@
+from operator import itemgetter
+from typing import Iterable
+from tree_format import format_tree
+
+from mach_nix.resolver import ResolvedPkg
+
+
+class Node:
+    def __init__(self, pkg: ResolvedPkg, parent: 'Node' = None):
+        self.pkg = pkg
+        pi = pkg.provider_info
+        self.name = f"{pkg.name} - {pkg.ver} - {pi.provider}"
+        if pi.provider == 'wheel':
+            self.name += f" - {'-'.join(pi.wheel_fname.split('-')[-3:])[:-4]}"
+        self.children = []
+        self.parent = parent
+        if parent:
+            self.parent.children.append(self)
+
+
+def build_tree(pkgs: dict, root: Node):
+    root_pkg = pkgs[root.pkg.name]
+    for name in sorted(root_pkg.build_inputs + root_pkg.prop_build_inputs):
+        child_pkg: ResolvedPkg = pkgs[name]
+        child_node = Node(child_pkg, root)
+        build_tree(pkgs, child_node)
+
+
+def tree_to_dict(root_node: Node):
+    return root_node.name, [tree_to_dict(child) for child in root_node.children]
+
+
+def print_tree(root_node):
+    d = tree_to_dict(root_node)
+    print(format_tree(
+        d, format_node=itemgetter(0), get_children=itemgetter(1)))
+
+
+def print_deps(pkgs: Iterable[ResolvedPkg]):
+    print("\n### Resolved Dependencies ###\n")
+    indexed_pkgs = {p.name: p for p in sorted(pkgs, key=lambda p: p.name)}
+    roots: Iterable[ResolvedPkg] = (p for p in pkgs if p.is_root)
+    for root in roots:
+        root_node = Node(root)
+        build_tree(indexed_pkgs, root_node)
+        print_tree(root_node)

shell.nix

@@ -1,4 +1,4 @@
-with import (import ./mach_nix/nix/nixpkgs-src.nix).stable { config = {}; };
+with import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; };
 mkShell {
   buildInputs = [
     (import ./mach_nix/nix/python.nix { inherit pkgs; })