respect requires_python for wheels, support macos, add caching
This commit is contained in:
parent
bfa6943e3b
commit
17aa8ceedd
13 changed files with 214 additions and 93 deletions
|
@ -3,6 +3,7 @@ import os
|
|||
import subprocess as sp
|
||||
import tempfile
|
||||
from os.path import realpath, dirname
|
||||
from time import time
|
||||
|
||||
import toml
|
||||
|
||||
|
@ -43,4 +44,7 @@ with open(pwd + "/reqs.txt") as f:
|
|||
os.environ['requirements'] = f.read()
|
||||
|
||||
# generates and writes nix expression into ./debug/expr.nix
|
||||
start = time()
|
||||
main()
|
||||
dur = round(time() - start, 1)
|
||||
print(f"resolving took: {dur}s")
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
# python interpreter for dev environment
|
||||
import ./mach_nix/nix/python.nix {
|
||||
pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix).stable { config = {}; };
|
||||
pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; };
|
||||
}
|
||||
|
|
21
mach_nix/cache.py
Normal file
21
mach_nix/cache.py
Normal file
|
@ -0,0 +1,21 @@
|
|||
from inspect import isgenerator
|
||||
|
||||
cache = {}
|
||||
|
||||
|
||||
def cached(keyfunc=None):
|
||||
def cached_deco(func):
|
||||
def cache_wrapper(*args, **kwargs):
|
||||
args_save = keyf(args)
|
||||
key = (func, args_save, tuple(kwargs.items()))
|
||||
if key not in cache:
|
||||
result = func(*args, **kwargs)
|
||||
if isgenerator(result):
|
||||
result = tuple(result)
|
||||
cache[key] = result
|
||||
return cache[key]
|
||||
return cache_wrapper
|
||||
|
||||
keyf = (lambda x: x) if keyfunc is None else keyfunc
|
||||
return cached_deco
|
||||
|
|
@ -1,24 +1,25 @@
|
|||
import json
|
||||
import platform
|
||||
import re
|
||||
import sys
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Tuple, Iterable, Set
|
||||
from typing import List, Tuple, Iterable
|
||||
|
||||
import distlib.markers
|
||||
from packaging.version import Version, parse
|
||||
|
||||
from .nixpkgs import NixpkgsDirectory
|
||||
from mach_nix.requirements import filter_reqs_by_eval_marker, Requirement, parse_reqs, context
|
||||
from mach_nix.versions import PyVer, ver_sort_key
|
||||
from mach_nix.versions import PyVer, ver_sort_key, filter_versions
|
||||
from .bucket_dict import LazyBucketDict
|
||||
from ..cache import cached
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProviderInfo:
|
||||
provider: str
|
||||
# following args are only required in case of wheel
|
||||
wheel_pyver: str = None
|
||||
wheel_fname: str = None
|
||||
|
||||
|
||||
|
@ -69,6 +70,7 @@ class DependencyProviderBase(ABC):
|
|||
self.context_wheel['extra'] = None
|
||||
self.py_ver_digits = py_ver.digits()
|
||||
|
||||
@cached()
|
||||
def available_versions(self, pkg_name: str) -> Iterable[Version]:
|
||||
"""
|
||||
returns available versions for given package name in reversed preference
|
||||
|
@ -98,6 +100,7 @@ class DependencyProviderBase(ABC):
|
|||
pass
|
||||
|
||||
@abstractmethod
|
||||
@cached()
|
||||
def get_pkg_reqs(self, pkg_name, pkg_version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
|
||||
"""
|
||||
Get all requirements of a candidate for the current platform and the specified extras
|
||||
|
@ -178,6 +181,7 @@ class CombinedDependencyProvider(DependencyProviderBase):
|
|||
print(error_text, file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
@cached()
|
||||
def available_versions(self, pkg_name: str) -> Iterable[Version]:
|
||||
# use dict as ordered set
|
||||
available_versions = []
|
||||
|
@ -202,11 +206,12 @@ class NixpkgsDependencyProvider(DependencyProviderBase):
|
|||
# )
|
||||
|
||||
# TODO: implement extras by looking them up via the equivalent wheel
|
||||
def __init__(self,
|
||||
nixpkgs: NixpkgsDirectory,
|
||||
wheel_provider: 'WheelDependencyProvider',
|
||||
sdist_provider: 'SdistDependencyProvider',
|
||||
*args, **kwargs):
|
||||
def __init__(
|
||||
self,
|
||||
nixpkgs: NixpkgsDirectory,
|
||||
wheel_provider: 'WheelDependencyProvider',
|
||||
sdist_provider: 'SdistDependencyProvider',
|
||||
*args, **kwargs):
|
||||
super(NixpkgsDependencyProvider, self).__init__(*args, **kwargs)
|
||||
self.nixpkgs = nixpkgs
|
||||
self.wheel_provider = wheel_provider
|
||||
|
@ -234,106 +239,126 @@ class NixpkgsDependencyProvider(DependencyProviderBase):
|
|||
return []
|
||||
|
||||
|
||||
@dataclass
|
||||
class WheelRelease:
|
||||
fn_pyver: str # the python version indicated by the filename
|
||||
name: str
|
||||
ver: str
|
||||
fn: str
|
||||
requires_dist: list
|
||||
provided_extras: list
|
||||
requires_python: str # the python version of the wheel metadata
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.fn)
|
||||
|
||||
|
||||
class WheelDependencyProvider(DependencyProviderBase):
|
||||
name = 'wheel'
|
||||
|
||||
def __init__(self, data_dir: str, *args, **kwargs):
|
||||
super(WheelDependencyProvider, self).__init__(*args, **kwargs)
|
||||
self.data = LazyBucketDict(data_dir)
|
||||
major, minor = self.py_ver_digits
|
||||
self.py_ver_re = re.compile(rf"^(py|cp)?{major}\.?{minor}?$")
|
||||
self.preferred_wheels = (
|
||||
'manylinux2014_x86_64.whl',
|
||||
'manylinux2010_x86_64.whl',
|
||||
'manylinux1_x86_64.whl',
|
||||
'none-any.whl'
|
||||
)
|
||||
m = self.py_ver_digits[-1]
|
||||
if platform.system() == "Linux":
|
||||
self.preferred_wheels = (
|
||||
re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-manylinux2014_x86_64"),
|
||||
re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-manylinux2010_x86_64"),
|
||||
re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-manylinux1_x86_64"),
|
||||
re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-linux_x86_64"),
|
||||
re.compile(rf"(py3|cp3)[{m}]?-(cp3{m}m|abi3|none)-any"),
|
||||
)
|
||||
elif platform.system() == "Darwin":
|
||||
self.preferred_wheels = (
|
||||
re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_universal"),
|
||||
re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_x86_64"),
|
||||
re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_intel"),
|
||||
re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_(fat64|fat32)"),
|
||||
re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-any"),)
|
||||
else:
|
||||
raise Exception(f"Unsupported Platform {platform.system()}")
|
||||
|
||||
def _available_versions(self, pkg_name: str) -> Iterable[Version]:
|
||||
name = self.unify_key(pkg_name)
|
||||
result = []
|
||||
for pyver in self._get_pyvers_for_pkg(name):
|
||||
vers = self.data[name][pyver]
|
||||
for ver, fnames in vers.items():
|
||||
for fn, deps in fnames.items():
|
||||
if self._wheel_ok(fn):
|
||||
result.append(parse(ver))
|
||||
break
|
||||
return result
|
||||
|
||||
def choose_wheel(self, pkg_name, pkg_version: Version) -> Tuple[str, str]:
|
||||
name = self.unify_key(pkg_name)
|
||||
ver = str(pkg_version)
|
||||
ok_fnames = {}
|
||||
for pyver in self._get_pyvers_for_pkg(name):
|
||||
if not ver in self.data[name][pyver]:
|
||||
continue
|
||||
fnames = self.data[name][pyver][ver]
|
||||
ok_fnames = {fn: pyver for fn in fnames if
|
||||
self._wheel_ok(fn) and any(fn.endswith(end) for end in self.preferred_wheels)}
|
||||
if not ok_fnames:
|
||||
raise Exception(f"No wheel available for {name}:{ver}")
|
||||
if len(ok_fnames) > 1:
|
||||
fn = self._select_preferred_wheel(ok_fnames)
|
||||
else:
|
||||
fn = list(ok_fnames.keys())[0]
|
||||
pyver = ok_fnames[fn]
|
||||
return pyver, fn
|
||||
|
||||
def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo:
|
||||
wheel_pyver, wheel_fname = self.choose_wheel(pkg_name, pkg_version)
|
||||
return ProviderInfo(provider=self.name, wheel_pyver=wheel_pyver, wheel_fname=wheel_fname)
|
||||
|
||||
def get_pkg_reqs_raw(self, pkg_name, pkg_version: Version):
|
||||
name = self.unify_key(pkg_name)
|
||||
if pkg_version not in self._available_versions(pkg_name):
|
||||
raise PackageNotFound(pkg_name, pkg_version, self.name)
|
||||
ver = str(pkg_version)
|
||||
pyver, fn = self.choose_wheel(pkg_name, pkg_version)
|
||||
deps = self.data[name][pyver][ver][fn]
|
||||
if isinstance(deps, str):
|
||||
key_ver, key_fn = deps.split('@')
|
||||
versions = self.data[name][pyver]
|
||||
deps = versions[key_ver][key_fn]
|
||||
return deps['requires_dist'] if 'requires_dist' in deps else []
|
||||
return (parse(wheel.ver) for wheel in self._suitable_wheels(pkg_name))
|
||||
|
||||
def get_pkg_reqs(self, pkg_name, pkg_version: Version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
|
||||
"""
|
||||
Get requirements for package
|
||||
"""
|
||||
reqs_raw = self.get_pkg_reqs_raw(pkg_name, pkg_version)
|
||||
reqs_raw = self._choose_wheel(pkg_name, pkg_version).requires_dist
|
||||
if reqs_raw is None:
|
||||
reqs_raw = []
|
||||
# handle extras by evaluationg markers
|
||||
install_reqs = list(filter_reqs_by_eval_marker(parse_reqs(reqs_raw), self.context_wheel, extras))
|
||||
return install_reqs, []
|
||||
|
||||
def _get_pyvers_for_pkg(self, name) -> Iterable:
|
||||
name = self.unify_key(name)
|
||||
def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo:
|
||||
wheel = self._choose_wheel(pkg_name, pkg_version)
|
||||
return ProviderInfo(provider=self.name, wheel_fname=wheel.fn)
|
||||
|
||||
def _all_releases(self, pkg_name):
|
||||
name = self.unify_key(pkg_name)
|
||||
if name not in self.data:
|
||||
return []
|
||||
ok_pyvers = (pyver for pyver in self.data[name].keys() if self._pyver_ok(pyver))
|
||||
return ok_pyvers
|
||||
for fn_pyver, vers in self.data[name].items():
|
||||
for ver, fnames in vers.items():
|
||||
for fn, deps in fnames.items():
|
||||
if isinstance(deps, str):
|
||||
key_ver, key_fn = deps.split('@')
|
||||
versions = self.data[name][fn_pyver]
|
||||
deps = versions[key_ver][key_fn]
|
||||
assert isinstance(deps, dict)
|
||||
yield WheelRelease(
|
||||
fn_pyver,
|
||||
name,
|
||||
ver,
|
||||
fn,
|
||||
deps['requires_dist'] if 'requires_dist' in deps else None,
|
||||
deps['requires_extras'] if 'requires_extras' in deps else None,
|
||||
deps['requires_python'] if 'requires_python' in deps else None,
|
||||
)
|
||||
|
||||
def _select_preferred_wheel(self, filenames: Iterable[str]):
|
||||
for key in self.preferred_wheels:
|
||||
for fn in filenames:
|
||||
if fn.endswith(key):
|
||||
return fn
|
||||
raise Exception("No wheel matches expected format")
|
||||
def _apply_filters(self, filters: List[callable], objects: Iterable):
|
||||
"""
|
||||
Applies multiple filters to objects. First filter in the list is applied first
|
||||
"""
|
||||
assert len(filters) > 0
|
||||
if len(filters) == 1:
|
||||
return filter(filters[0], objects)
|
||||
return filter(filters[-1], self._apply_filters(filters[:-1], objects))
|
||||
|
||||
def _pyver_ok(self, ver: str):
|
||||
ver = ver.strip()
|
||||
major, minor = self.py_ver_digits
|
||||
if re.fullmatch(self.py_ver_re, ver) \
|
||||
or ver == f"py{major}"\
|
||||
or ver == "py2.py3":
|
||||
return True
|
||||
return False
|
||||
@cached()
|
||||
def _choose_wheel(self, pkg_name, pkg_version: Version) -> WheelRelease:
|
||||
return self._select_preferred_wheel(self._suitable_wheels(pkg_name, pkg_version))
|
||||
|
||||
def _wheel_ok(self, fn):
|
||||
if fn.endswith('any.whl'):
|
||||
return True
|
||||
elif "manylinux" in fn:
|
||||
def _suitable_wheels(self, pkg_name: str, ver: Version = None) -> Iterable[WheelRelease]:
|
||||
wheels = self._all_releases(pkg_name)
|
||||
if ver is not None:
|
||||
wheels = filter(lambda w: parse(w.ver) == ver, wheels)
|
||||
return self._apply_filters(
|
||||
[
|
||||
self._wheel_type_ok,
|
||||
self._python_requires_ok,
|
||||
],
|
||||
wheels)
|
||||
|
||||
def _select_preferred_wheel(self, wheels: Iterable[WheelRelease]):
|
||||
wheels = list(wheels)
|
||||
for pattern in self.preferred_wheels:
|
||||
for wheel in wheels:
|
||||
if re.search(pattern, wheel.fn):
|
||||
return wheel
|
||||
raise Exception("No wheel type found that is compatible to the current system")
|
||||
|
||||
def _wheel_type_ok(self, wheel: WheelRelease):
|
||||
return any(re.search(pattern, wheel.fn) for pattern in self.preferred_wheels)
|
||||
|
||||
def _python_requires_ok(self, wheel: WheelRelease):
|
||||
if not wheel.requires_python:
|
||||
return True
|
||||
ver = parse('.'.join(self.py_ver_digits))
|
||||
return bool(filter_versions([ver], list(parse_reqs(f"python{wheel.requires_python}"))[0].specs))
|
||||
|
||||
|
||||
class SdistDependencyProvider(DependencyProviderBase):
|
||||
|
@ -342,6 +367,7 @@ class SdistDependencyProvider(DependencyProviderBase):
|
|||
self.data = LazyBucketDict(data_dir)
|
||||
super(SdistDependencyProvider, self).__init__(*args, **kwargs)
|
||||
|
||||
@cached()
|
||||
def _get_candidates(self, name) -> dict:
|
||||
"""
|
||||
returns all candidates for the give name which are available for the current python version
|
||||
|
|
|
@ -14,10 +14,15 @@ def unindent(text: str, remove: int):
|
|||
|
||||
class OverridesGenerator(ExpressionGenerator):
|
||||
|
||||
def __init__(self, py_ver, nixpkgs: NixpkgsDirectory, pypi_fetcher_commit,
|
||||
pypi_fetcher_sha256, disable_checks,
|
||||
*args,
|
||||
**kwargs):
|
||||
def __init__(
|
||||
self,
|
||||
py_ver,
|
||||
nixpkgs: NixpkgsDirectory,
|
||||
pypi_fetcher_commit,
|
||||
pypi_fetcher_sha256,
|
||||
disable_checks,
|
||||
*args,
|
||||
**kwargs):
|
||||
self.nixpkgs = nixpkgs
|
||||
self.disable_checks = disable_checks
|
||||
self.pypi_fetcher_commit = pypi_fetcher_commit
|
||||
|
|
|
@ -1 +1 @@
|
|||
d8936568152dff29bd51c8bf492609a1f686fc7f
|
||||
2a2501aab1fe4eb50763652c4f74c5327f976f85
|
|
@ -1 +1 @@
|
|||
1f9kazr6ny6v68d55qbdcccl34s97370lpd6w43rizpjdznv04hv
|
||||
03c64w1x4q2li57cn6zsb2shjk2fc4f5ksmsz3695dmvckp0lj14
|
|
@ -24,6 +24,15 @@ rec {
|
|||
};
|
||||
doCheck = false;
|
||||
};
|
||||
tree-format = python.pkgs.buildPythonPackage {
|
||||
name = "tree-format-0.1.2";
|
||||
src = fetchurl {
|
||||
url = "https://files.pythonhosted.org/packages/0d/91/8d860c75c3e70e6bbec7b898b5f753bf5da404be9296e245034360759645/tree-format-0.1.2.tar.gz";
|
||||
sha256 = "a538523aa78ae7a4b10003b04f3e1b37708e0e089d99c9d3b9e1c71384c9a7f9";
|
||||
};
|
||||
doCheck = false;
|
||||
};
|
||||
|
||||
packaging = python.pkgs.packaging;
|
||||
setuptools = python.pkgs.setuptools;
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ from distlib.markers import DEFAULT_CONTEXT
|
|||
from packaging.version import parse, _Version
|
||||
from pkg_resources._vendor.packaging.specifiers import SpecifierSet
|
||||
|
||||
from mach_nix.cache import cached
|
||||
from mach_nix.versions import PyVer
|
||||
|
||||
|
||||
|
@ -58,7 +59,7 @@ def filter_reqs_by_eval_marker(reqs: Iterable[Requirement], context: dict, selec
|
|||
yield req
|
||||
|
||||
|
||||
|
||||
@cached(lambda args: tuple(args[0]) if isinstance(args[0], list) else args[0])
|
||||
def parse_reqs(strs):
|
||||
reqs = list(pkg_resources.parse_requirements(strs))
|
||||
for req in reqs:
|
||||
|
|
|
@ -9,6 +9,7 @@ from mach_nix.data.nixpkgs import NixpkgsDirectory
|
|||
from mach_nix.requirements import Requirement
|
||||
from mach_nix.resolver import Resolver, ResolvedPkg
|
||||
from mach_nix.versions import filter_versions
|
||||
from mach_nix.visualize import print_deps
|
||||
|
||||
|
||||
@dataclass
|
||||
|
@ -80,4 +81,5 @@ class ResolvelibResolver(Resolver):
|
|||
provider_info=provider_info,
|
||||
extras_selected=list(result.mapping[name].extras)
|
||||
))
|
||||
print_deps(nix_py_pkgs)
|
||||
return nix_py_pkgs
|
||||
|
|
|
@ -5,6 +5,8 @@ from typing import Iterable, Tuple, List
|
|||
|
||||
from packaging.version import Version, parse, LegacyVersion
|
||||
|
||||
from mach_nix.cache import cached
|
||||
|
||||
|
||||
class PyVer(Version):
|
||||
def nix(self):
|
||||
|
@ -69,9 +71,14 @@ def best_version(versions: Iterable[Version]) -> Version:
|
|||
return best
|
||||
|
||||
|
||||
@cached(keyfunc=lambda args: tuple(args[0]) + tuple(args[1]))
|
||||
def filter_versions(
|
||||
versions: Iterable[Version],
|
||||
specs: Iterable[Tuple[str, Version]]) -> List[Version]:
|
||||
"""
|
||||
Reduces a given list of versions to contain only versions
|
||||
which are allowed according to the given specifiers
|
||||
"""
|
||||
for op, ver in specs:
|
||||
ver = parse(ver)
|
||||
if op == '==':
|
||||
|
|
46
mach_nix/visualize.py
Normal file
46
mach_nix/visualize.py
Normal file
|
@ -0,0 +1,46 @@
|
|||
from operator import itemgetter
|
||||
from typing import Iterable
|
||||
from tree_format import format_tree
|
||||
|
||||
from mach_nix.resolver import ResolvedPkg
|
||||
|
||||
|
||||
class Node:
|
||||
def __init__(self, pkg: ResolvedPkg, parent: 'Node' = None):
|
||||
self.pkg = pkg
|
||||
pi = pkg.provider_info
|
||||
self.name = f"{pkg.name} - {pkg.ver} - {pi.provider}"
|
||||
if pi.provider == 'wheel':
|
||||
self.name += f" - {'-'.join(pi.wheel_fname.split('-')[-3:])[:-4]}"
|
||||
self.children = []
|
||||
self.parent = parent
|
||||
if parent:
|
||||
self.parent.children.append(self)
|
||||
|
||||
|
||||
def build_tree(pkgs: dict, root: Node):
|
||||
root_pkg = pkgs[root.pkg.name]
|
||||
for name in sorted(root_pkg.build_inputs + root_pkg.prop_build_inputs):
|
||||
child_pkg: ResolvedPkg = pkgs[name]
|
||||
child_node = Node(child_pkg, root)
|
||||
build_tree(pkgs, child_node)
|
||||
|
||||
|
||||
def tree_to_dict(root_node: Node):
|
||||
return root_node.name, [tree_to_dict(child) for child in root_node.children]
|
||||
|
||||
|
||||
def print_tree(root_node):
|
||||
d = tree_to_dict(root_node)
|
||||
print(format_tree(
|
||||
d, format_node=itemgetter(0), get_children=itemgetter(1)))
|
||||
|
||||
|
||||
def print_deps(pkgs: Iterable[ResolvedPkg]):
|
||||
print("\n### Resolved Dependencies ###\n")
|
||||
indexed_pkgs = {p.name: p for p in sorted(pkgs, key=lambda p: p.name)}
|
||||
roots: Iterable[ResolvedPkg] = (p for p in pkgs if p.is_root)
|
||||
for root in roots:
|
||||
root_node = Node(root)
|
||||
build_tree(indexed_pkgs, root_node)
|
||||
print_tree(root_node)
|
|
@ -1,4 +1,4 @@
|
|||
with import (import ./mach_nix/nix/nixpkgs-src.nix).stable { config = {}; };
|
||||
with import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; };
|
||||
mkShell {
|
||||
buildInputs = [
|
||||
(import ./mach_nix/nix/python.nix { inherit pkgs; })
|
||||
|
|
Loading…
Reference in a new issue