WIP: conda

mach_nix/data/nixpkgs.py

@@ -3,10 +3,8 @@ from collections import UserDict
 from dataclasses import dataclass
 from typing import List
 
-from packaging.version import Version
-
 from mach_nix.cache import cached
-from mach_nix.versions import parse_ver
+from mach_nix.versions import parse_ver, Version
 
 
 @dataclass
@@ -57,9 +55,9 @@ class NixpkgsIndex(UserDict):
 
     @staticmethod
     def is_same_ver(ver1, ver2, ver_idx):
-        if any(not ver.release or len(ver.release) <= ver_idx for ver in (ver1, ver2)):
+        if any(len(ver.version) <= ver_idx for ver in (ver1, ver2)):
             return False
-        return ver1.release[ver_idx] == ver2.release[ver_idx]
+        return ver1.version[ver_idx] == ver2.version[ver_idx]
 
     @cached(lambda args: (args[1], args[2]))
     def find_best_nixpkgs_candidate(self, name, ver):

mach_nix/data/providers.py

@@ -3,21 +3,28 @@ import platform
 import re
 import sys
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from os.path import abspath, dirname
 from typing import List, Tuple, Iterable
 
 import distlib.markers
-from packaging.version import Version, parse, LegacyVersion
 from pkg_resources import RequirementParseError
 
 from .nixpkgs import NixpkgsIndex
 from mach_nix.requirements import filter_reqs_by_eval_marker, Requirement, parse_reqs, context
-from mach_nix.versions import PyVer, ver_sort_key, filter_versions
+from mach_nix.versions import PyVer, ver_sort_key, filter_versions, parse_ver, Version
 from .bucket_dict import LazyBucketDict
 from ..cache import cached
 
 
+@dataclass
+class Candidate:
+    name: str
+    ver: Version
+    extras: tuple
+    build: str = None
+
+
 @dataclass
 class ProviderInfo:
     provider: 'DependencyProviderBase'
@@ -25,7 +32,7 @@ class ProviderInfo:
     wheel_fname: str = None
 
 
-def unify_key(key: str) -> str:
+def normalize_name(key: str) -> str:
     return key.replace('_', '-').lower()
 
 
@@ -46,14 +53,14 @@ class ProviderSettings:
 
     def _parse_provider_list(self, str_or_list) -> Tuple[str]:
         if isinstance(str_or_list, str):
-            return tuple(unify_key(p.strip()) for p in str_or_list.strip().split(','))
+            return tuple(normalize_name(p.strip()) for p in str_or_list.strip().split(','))
         elif isinstance(str_or_list, list):
-            return tuple(unify_key(k) for k in str_or_list)
+            return tuple(normalize_name(k) for k in str_or_list)
         else:
             raise Exception("Provider specifiers must be lists or comma separated strings")
 
     def provider_names_for_pkg(self, pkg_name):
-        name = unify_key(pkg_name)
+        name = normalize_name(pkg_name)
         if name in self.pkg_providers:
             return self.pkg_providers[name]
         else:
@@ -71,17 +78,18 @@ class DependencyProviderBase(ABC):
         self.context_wheel = self.context.copy()
         self.context_wheel['extra'] = None
         self.py_ver = py_ver
-        self.py_ver_parsed = parse(py_ver.python_full_version())
+        self.py_ver_parsed = parse_ver(py_ver.python_full_version())
         self.py_ver_digits = py_ver.digits()
         self.platform = platform
         self.system = system
 
     @cached()
-    def available_versions(self, pkg_name: str) -> Iterable[Version]:
-        """
-        returns available versions for given package name in reversed preference
-        """
-        return sorted(self._available_versions(pkg_name), key=ver_sort_key)
+    def find_matches(self, req) -> List[Candidate]:
+        all = {c.ver: c for c in self.all_candidates(req.key, req.extras, req.build)}
+        mathing_versions = set(filter_versions(all.keys(), req.specs))
+        matching_candidates = [c for c in all.values() if c.ver in mathing_versions]
+        matching_candidates.sort(key=lambda c: (ver_sort_key(c.ver)))
+        return matching_candidates
 
     @property
     @abstractmethod
@@ -97,12 +105,12 @@ class DependencyProviderBase(ABC):
     def unify_key(self, key: str) -> str:
         return key.replace('_', '-').lower()
 
-    def get_provider_info(self, pkg_name, pkg_version):
+    def get_provider_info(self, pkg_name, pkg_version, extras=None, build=None):
         return ProviderInfo(provider=self)
 
     @abstractmethod
     @cached()
-    def get_pkg_reqs(self, pkg_name, pkg_version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
+    def get_pkg_reqs(self, candidate: Candidate) -> Tuple[List[Requirement], List[Requirement]]:
         """
         Get all requirements of a candidate for the current platform and the specified extras
         returns two lists: install_requires, setup_requires
@@ -110,11 +118,11 @@ class DependencyProviderBase(ABC):
         pass
 
     @abstractmethod
-    def _available_versions(self, pkg_name: str) -> Iterable[Version]:
+    def all_candidates(self, name, extras=None, build=None) -> Iterable[Candidate]:
         pass
 
     @abstractmethod
-    def deviated_version(self, pkg_name, normalized_version: Version):
+    def deviated_version(self, pkg_name, normalized_version: Version, build):
         # returns version like originally specified by package maintainer without normalization
         pass
 
@@ -154,23 +162,24 @@ class CombinedDependencyProvider(DependencyProviderBase):
         selected_providers = ((name, p) for name, p in self._all_providers.items() if name in provider_keys)
         return dict(sorted(selected_providers, key=lambda x: provider_keys.index(x[0])))
 
-    def get_provider(self, pkg_name, pkg_version) -> DependencyProviderBase:
+    def get_provider(self, pkg_name, pkg_version, extras, build) -> DependencyProviderBase:
         for type, provider in self.allowed_providers_for_pkg(pkg_name).items():
-            if pkg_version in provider.available_versions(pkg_name):
+            if pkg_version in (c.ver for c in provider.all_candidates(pkg_name, extras, build)):
                 return provider
 
-    def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo:
-        return self.get_provider(pkg_name, pkg_version).get_provider_info(pkg_name, pkg_version)
+    def get_provider_info(self, pkg_name, pkg_version, extras=None, build=None) -> ProviderInfo:
+        return self.get_provider(pkg_name, pkg_version, extras, build)\
+            .get_provider_info(pkg_name, pkg_version, extras, build)
 
-    def get_pkg_reqs(self, pkg_name, pkg_version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
-        for provider in self.allowed_providers_for_pkg(pkg_name).values():
-            if pkg_version in provider.available_versions(pkg_name):
-                return provider.get_pkg_reqs(pkg_name, pkg_version, extras=extras)
+    def get_pkg_reqs(self, c: Candidate) -> Tuple[List[Requirement], List[Requirement]]:
+        for provider in self.allowed_providers_for_pkg(c.name).values():
+            if c in provider.all_candidates(c.name, c.extras, c.build):
+                return provider.get_pkg_reqs(c)
 
     def list_all_providers_for_pkg(self, pkg_name):
         result = []
         for p_name, provider in self._all_providers.items():
-            if provider.available_versions(pkg_name):
+            if provider.all_candidates(pkg_name):
                 result.append(p_name)
         return result
 
@@ -194,22 +203,19 @@ class CombinedDependencyProvider(DependencyProviderBase):
         exit(1)
 
     @cached()
-    def available_versions(self, pkg_name: str) -> Iterable[Version]:
+    def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]:
         # use dict as ordered set
-        available_versions = []
+        candidates = []
         # order by reversed preference expected
         for provider in reversed(tuple(self.allowed_providers_for_pkg(pkg_name).values())):
-            for ver in provider.available_versions(pkg_name):
-                available_versions.append(ver)
-        if available_versions:
-            return tuple(available_versions)
+            for c in provider.all_candidates(pkg_name, extras, build):
+                candidates.append(c)
+        if candidates:
+            return tuple(candidates)
         self.print_error_no_versions_available(pkg_name)
 
-    def _available_versions(self, pkg_name: str) -> Iterable[Version]:
-        return self.available_versions(pkg_name)
-
-    def deviated_version(self, pkg_name, pkg_version: Version):
-        self.get_provider(pkg_name, pkg_version).deviated_version(pkg_name, pkg_version)
+    def deviated_version(self, pkg_name, pkg_version: Version, build):
+        self.get_provider(pkg_name, pkg_version, tuple(), build).deviated_version(pkg_name, pkg_version)
 
 
 class NixpkgsDependencyProvider(DependencyProviderBase):
@@ -227,25 +233,26 @@ class NixpkgsDependencyProvider(DependencyProviderBase):
         self.wheel_provider = wheel_provider
         self.sdist_provider = sdist_provider
 
-    def get_pkg_reqs(self, pkg_name, pkg_version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
-        name = self.unify_key(pkg_name)
-        if not self.nixpkgs.exists(name, pkg_version):
-            raise Exception(f"Cannot find {name}:{pkg_version} in nixpkgs")
+    def get_pkg_reqs(self, c: Candidate) -> Tuple[List[Requirement], List[Requirement]]:
+        if not self.nixpkgs.exists(c.name, c.ver):
+            raise Exception(f"Cannot find {c.name}:{c.ver} in nixpkgs")
         install_reqs, setup_reqs = [], []
         for provider in (self.sdist_provider, self.wheel_provider):
             try:
-                install_reqs, setup_reqs = provider.get_pkg_reqs(pkg_name, pkg_version, extras=extras)
+                install_reqs, setup_reqs = provider.get_pkg_reqs(c)
             except PackageNotFound:
                 pass
         return install_reqs, setup_reqs
 
-    def _available_versions(self, pkg_name: str) -> Iterable[Version]:
+    def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]:
+        if build:
+            return []
         name = self.unify_key(pkg_name)
-        if self.nixpkgs.exists(name):
-            return [p.ver for p in self.nixpkgs.get_all_candidates(name)]
-        return []
+        if not self.nixpkgs.exists(name):
+            return []
+        return [Candidate(pkg_name, p.ver, extras) for p in self.nixpkgs.get_all_candidates(name)]
 
-    def deviated_version(self, pkg_name, normalized_version: Version):
+    def deviated_version(self, pkg_name, normalized_version: Version, build):
         # not necessary for nixpkgs provider since source doesn't need to be fetched
         return str(normalized_version)
 
@@ -290,25 +297,27 @@ class WheelDependencyProvider(DependencyProviderBase):
         else:
             raise Exception(f"Unsupported Platform {platform.system()}")
 
-    def _available_versions(self, pkg_name: str) -> Iterable[Version]:
-        return (parse(wheel.ver) for wheel in self._suitable_wheels(pkg_name))
+    def all_candidates(self, pkg_name, extras=None, build=None) -> List[Candidate]:
+        if build:
+            return []
+        return [Candidate(w.name, parse_ver(w.ver), extras) for w in self._suitable_wheels(pkg_name)]
 
-    def get_pkg_reqs(self, pkg_name, pkg_version: Version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
+    def get_pkg_reqs(self, c: Candidate) -> Tuple[List[Requirement], List[Requirement]]:
         """
         Get requirements for package
         """
-        reqs_raw = self._choose_wheel(pkg_name, pkg_version).requires_dist
+        reqs_raw = self._choose_wheel(c.name, c.ver).requires_dist
         if reqs_raw is None:
             reqs_raw = []
         # handle extras by evaluationg markers
-        install_reqs = list(filter_reqs_by_eval_marker(parse_reqs(reqs_raw), self.context_wheel, extras))
+        install_reqs = list(filter_reqs_by_eval_marker(parse_reqs(reqs_raw), self.context_wheel, c.extras))
         return install_reqs, []
 
-    def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo:
+    def get_provider_info(self, pkg_name, pkg_version, extras=None, build=None) -> ProviderInfo:
         wheel = self._choose_wheel(pkg_name, pkg_version)
         return ProviderInfo(provider=self, wheel_fname=wheel.fn)
 
-    def deviated_version(self, pkg_name, pkg_version: Version):
+    def deviated_version(self, pkg_name, pkg_version: Version, build):
         return self._choose_wheel(pkg_name, pkg_version).ver
 
     def _all_releases(self, pkg_name):
@@ -352,7 +361,7 @@ class WheelDependencyProvider(DependencyProviderBase):
     def _suitable_wheels(self, pkg_name: str, ver: Version = None) -> Iterable[WheelRelease]:
         wheels = self._all_releases(pkg_name)
         if ver is not None:
-            wheels = filter(lambda w: parse(w.ver) == ver, wheels)
+            wheels = filter(lambda w: parse_ver(w.ver) == ver, wheels)
         return self._apply_filters(
             [
                 self._wheel_type_ok,
@@ -374,7 +383,7 @@ class WheelDependencyProvider(DependencyProviderBase):
     def _python_requires_ok(self, wheel: WheelRelease):
         if not wheel.requires_python:
             return True
-        ver = parse('.'.join(self.py_ver_digits))
+        ver = parse_ver('.'.join(self.py_ver_digits))
         try:
             parsed_py_requires = list(parse_reqs(f"python{wheel.requires_python}"))
             return bool(filter_versions([ver], parsed_py_requires[0].specs))
@@ -407,20 +416,21 @@ class SdistDependencyProvider(DependencyProviderBase):
             # in case pyver is a string, it is a reference to another pyver which we need to resolve
             if self.py_ver_digits in pyvers:
                 pyver = pyvers[self.py_ver_digits]
+                parsed_ver = parse_ver(ver)
                 if isinstance(pyver, str):
-                    candidates[parse(ver)] = pyvers[pyver]
+                    candidates[parsed_ver] = pyvers[pyver]
                 else:
-                    candidates[parse(ver)] = pyvers[self.py_ver_digits]
+                    candidates[parsed_ver] = pyvers[self.py_ver_digits]
         return candidates
 
-    def deviated_version(self, pkg_name, normalized_version: Version):
-        for raw_ver in self.data[unify_key(pkg_name)].keys():
-            if parse(raw_ver) == normalized_version:
+    def deviated_version(self, pkg_name, normalized_version: Version, build):
+        for raw_ver in self.data[normalize_name(pkg_name)].keys():
+            if parse_ver(raw_ver) == normalized_version:
                 return raw_ver
         raise Exception(
             f"Something went wrong while trying to find the deviated version for {pkg_name}:{normalized_version}")
 
-    def get_reqs_for_extras(self, pkg_name, pkg_ver, extras):
+    def get_reqs_for_extras(self, pkg_name, pkg_ver: Version, extras):
         name = self.unify_key(pkg_name)
         pkg = self._get_candidates(name)[pkg_ver]
         extras = set(extras)
@@ -436,13 +446,13 @@ class SdistDependencyProvider(DependencyProviderBase):
                     requirements += list(filter_reqs_by_eval_marker(parse_reqs(reqs_str), self.context))
         return requirements
 
-    def get_pkg_reqs(self, pkg_name, pkg_version: Version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
+    def get_pkg_reqs(self, c: Candidate) -> Tuple[List[Requirement], List[Requirement]]:
         """
         Get requirements for package
         """
-        if pkg_version not in self._get_candidates(pkg_name):
-            raise PackageNotFound(pkg_name, pkg_version, self.name)
-        pkg = self._get_candidates(pkg_name)[pkg_version]
+        if c.ver not in self._get_candidates(c.name):
+            raise PackageNotFound(c.name, c.ver, self.name)
+        pkg = self._get_candidates(c.name)[c.ver]
         requirements = dict(
             setup_requires=[],
             install_requires=[]
@@ -454,15 +464,17 @@ class SdistDependencyProvider(DependencyProviderBase):
                 reqs_raw = pkg[t]
                 reqs = parse_reqs(reqs_raw)
                 requirements[t] = list(filter_reqs_by_eval_marker(reqs, self.context))
-        if not extras:
+        if not c.extras:
             extras = []
         # even if no extras are selected we need to collect reqs for extras,
         # because some extras consist of only a marker which needs to be evaluated
-        requirements['install_requires'] += self.get_reqs_for_extras(pkg_name, pkg_version, extras)
+        requirements['install_requires'] += self.get_reqs_for_extras(c.name, c.ver, c.extras)
         return requirements['install_requires'], requirements['setup_requires']
 
-    def _available_versions(self, pkg_name: str) -> Iterable[Version]:
-        return [ver for ver in self._get_candidates(pkg_name).keys()]
+    def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Candidate]:
+        if build:
+            return []
+        return [Candidate(pkg_name, ver, extras) for ver, pkg in self._get_candidates(pkg_name).items()]
 
 
 class CondaDependencyProvider(DependencyProviderBase):
@@ -470,8 +482,8 @@ class CondaDependencyProvider(DependencyProviderBase):
     ignored_pkgs = (
         "ld_impl_linux-64",
         #"libffi",
-        "libgcc-ng",
-        "libstdcxx-ng",
+        ##"libgcc-ng",
+        ##"libstdcxx-ng",
         "ncurses",
         #"openssl",
         "readline",
@@ -493,7 +505,7 @@ class CondaDependencyProvider(DependencyProviderBase):
             with open(file) as f:
                 content = json.load(f)
             for fname, p in content['packages'].items():
-                name = p['name']
+                name = p['name'].replace('_', '-').lower()
                 ver = p['version']
                 build = p['build']
                 if name not in self.pkgs:
@@ -510,23 +522,29 @@ class CondaDependencyProvider(DependencyProviderBase):
     def name(self):
         return "conda"
 
-    def get_pkg_reqs(self, pkg_name, pkg_version: Version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
-        candidate = self.choose_candidate(pkg_name, pkg_version)
-        depends = list(filter(lambda d: d.split()[0] not in self.ignored_pkgs, candidate['depends']))
-        print(f"candidate {pkg_name}:{pkg_version} depends on {list(parse_reqs(depends))}")
+    def get_pkg_reqs(self, c: Candidate) -> Tuple[List[Requirement], List[Requirement]]:
+        candidate = self.choose_candidate(c.name, c.ver)
+        depends = list(filter(
+            lambda d: d.split()[0] not in self.ignored_pkgs and not d.startswith('_'),
+            candidate['depends']))
+        # print(f"candidate {c.name}:{c.ver} depends on {depends}")
         return list(parse_reqs(depends)), []
 
     @cached()
-    def _available_versions(self, pkg_name: str) -> Iterable[Version]:
-        versions = []
+    def all_candidates(self, pkg_name, extras=None, build=None) -> Iterable[Version]:
+        pkg_name = normalize_name(pkg_name)
+        candidates = []
         for ver in self.pkgs[pkg_name].keys():
-            versions += [parse(b['version']) for b in self.compatible_builds(pkg_name, parse(ver))]
-        return versions
+            candidates += [
+                Candidate(b['name'], parse_ver(b['version']), extras, b['build'])
+                for b in self.compatible_builds(pkg_name, parse_ver(ver), build)
+            ]
+        return candidates
 
-    def deviated_version(self, pkg_name, normalized_version: Version):
+    def deviated_version(self, pkg_name, normalized_version: Version, build):
         for builds in self.pkgs[pkg_name].values():
             for p in builds.values():
-                if parse(p['version']) == normalized_version:
+                if parse_ver(p['version']) == normalized_version:
                     return p['version']
         raise Exception(f"Cannot find deviated version for {pkg_name}:{normalized_version}")
 
@@ -539,9 +557,14 @@ class CondaDependencyProvider(DependencyProviderBase):
         return True
 
     @cached()
-    def compatible_builds(self, pkg_name, pkg_version):
+    def compatible_builds(self, pkg_name, pkg_version: Version, build=None) -> list:
+        deviated_ver = self.deviated_version(pkg_name, pkg_version, build)
+        if build:
+            if build not in self.pkgs[pkg_name][deviated_ver]:
+                return []
+            return [self.pkgs[pkg_name][deviated_ver][build]]
         compatible = []
-        for build in self.pkgs[pkg_name][self.deviated_version(pkg_name, pkg_version)].values():
+        for build in self.pkgs[pkg_name][deviated_ver].values():
             # continue if python incompatible
             if not self.python_ok(build):
                 continue
@@ -550,12 +573,12 @@ class CondaDependencyProvider(DependencyProviderBase):
         return compatible
 
     def choose_candidate(self, pkg_name, pkg_version: Version):
-        assert(isinstance(pkg_version, Version) or isinstance(pkg_version, LegacyVersion))
+        pkg_name = normalize_name(pkg_name)
         candidate = self.compatible_builds(pkg_name, pkg_version)[0]
-        print(f"chosen candidate {pkg_name}{candidate['version']} for {pkg_version}")
+        # print(f"chosen candidate {pkg_name}{candidate['version']} for {pkg_version}")
         return candidate
 
-    def pkg_url_src(self, pkg_name, pkg_version):
+    def pkg_url_src(self, pkg_name, pkg_version: Version):
         pkg = self.choose_candidate(pkg_name, pkg_version)
         url = f"https://anaconda.org/anaconda/{pkg['name']}/{pkg['version']}/download/{pkg['subdir']}/{pkg['fname']}"
         return url, pkg['sha256']

mach_nix/generators/overides_generator.py

@@ -264,7 +264,7 @@ class OverridesGenerator(ExpressionGenerator):
                 if self.nixpkgs.exists(pkg.name):
                     out += self._gen_overrideAttrs(
                         pkg.name,
-                        pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver),
+                        pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver, pkg.build),
                         pkg.removed_circular_deps,
                         nix_name,
                         'sdist',
@@ -273,7 +273,7 @@ class OverridesGenerator(ExpressionGenerator):
                 else:
                     out += self._gen_buildPythonPackage(
                         pkg.name,
-                        pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver),
+                        pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver, pkg.build),
                         pkg.removed_circular_deps,
                         nix_name,
                         build_inputs_str,
@@ -282,7 +282,7 @@ class OverridesGenerator(ExpressionGenerator):
             elif isinstance(pkg.provider_info.provider, WheelDependencyProvider):
                 out += self._gen_wheel_buildPythonPackage(
                     pkg.name,
-                    pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver),
+                    pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver, pkg.build),
                     pkg.removed_circular_deps,
                     self._get_ref_name(pkg.name, pkg.ver),
                     prop_build_inputs_str,
@@ -291,7 +291,7 @@ class OverridesGenerator(ExpressionGenerator):
             elif isinstance(pkg.provider_info.provider, CondaDependencyProvider):
                 out += self._gen_conda_buildPythonPackage(
                     pkg.name,
-                    pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver),
+                    pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver, pkg.build),
                     pkg.removed_circular_deps,
                     self._get_ref_name(pkg.name, pkg.ver),
                     prop_build_inputs_str,

mach_nix/nix/auto_patchelf_hook.nix

@@ -8,6 +8,7 @@ let
     '';
   };
   autoPatchelfHook = makeSetupHook { name = "auto-patchelf-hook-machnix"; }
-    auto_patchelf_script;
+    #auto_patchelf_script;
+    ./auto-patchelf.sh;
 in
 autoPatchelfHook

mach_nix/nix/mk-python-derivation-overlay.nix

@@ -29,10 +29,12 @@ pkgs: python: with pkgs.lib;
     buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ./mk-python-derivation.nix {
       inherit namePrefix;     # We want Python libraries to be named like e.g. "python3.6-${name}"
       inherit toPythonModule; # Libraries provide modules
+      autoPatchelfHook = callPackage ./auto_patchelf_hook.nix {};
     }));
 
     buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ./mk-python-derivation.nix {
       namePrefix = "";        # Python applications should not have any prefix
       toPythonModule = x: x;  # Application does not provide modules.
+      autoPatchelfHook = callPackage ./auto_patchelf_hook.nix {};
     }));
   }

mach_nix/nix/mk-python-derivation.nix

@@ -37,13 +37,14 @@
 
 # conda pkgs
 , glibc
+, gcc-unwrapped
 
 # conda python
 , binutils
-, libffi
+#, libffi
 , libgcc
 , ncurses
-, openssl
+#, openssl
 , readline
 , sqlite
 , tk
@@ -124,7 +125,7 @@ if disabled
 then throw "${name} not supported for interpreter ${python.executable}"
 else
 
-let supportedFormats = [ "condabin" "egg" "flit" "pyproject" "setuptools" "wheel" ]; in
+let supportedFormats = [ "condabin" "egg" "flit" "other" "pyproject" "setuptools" "wheel" ]; in
 if ! lib.elem format supportedFormats then
   throw ''wrong format "${format}" for buildPythonPackage. Must be one of: [${toString supportedFormats}]''
 else
@@ -164,8 +165,8 @@ let
         [ autoPatchelfHook alsaLib cups libGL ]
         ++ (with xorg; [ libSM libICE libX11 libXau libXdamage libXi libXrender libXrandr libXcomposite libXcursor libXtst libXScrnSaver])
         # dependencies of conds python interpreter dstribution
-        ++ [ binutils libffi libgcc ncurses openssl readline sqlite tk xz zlib ]
-    ) ++ lib.optionals (!(builtins.elem format [ "condabin" "other"]) || dontUsePipInstall) [
+        ++ [ binutils glibc gcc-unwrapped.lib libgcc ncurses readline sqlite tk xz zlib ]
+    ) ++ lib.optionals (!(builtins.elem format [ "condabin" "other" ]) || dontUsePipInstall) [
       pipInstallHook
     ] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
       # This is a test, however, it should be ran independent of the checkPhase and checkInputs
@@ -226,6 +227,7 @@ let
         mkdir -p $out/lib/$pyDir/site-packages/
         cp -r ./site-packages/* $out/lib/$pyDir/site-packages/
       else
+        rm env-vars
         cp -r . $out
       fi
     '';
@@ -235,6 +237,9 @@ let
       fi
       rm -rf $out/ssl
     '';
+    preFixup = ''
+      echo $nativeBuildInputs
+    '';
   }
 
   ));

mach_nix/nix/python-deps.nix

@@ -9,7 +9,8 @@ rec {
     };
     doCheck = false;
     propagatedBuildInputs = with python.pkgs; [ pycosat requests ruamel_yaml ];
-    patchPhase = ''
+    patches = [ ./conda_hashable_versionOrder.patch ];
+    postPatch = ''
       echo '
       def get_version(dunder_file):
         return "${version}"

mach_nix/requirements.py

@@ -24,26 +24,30 @@ def context(py_ver: PyVer, platform: str, system: str):
 
 
 class Requirement(pkg_resources.Requirement):
-    def __init__(self, line):
+    def __init__(self, line, build=None):
+        self.build = build
         super(Requirement, self).__init__(line)
         self.name = self.name.lower().replace('_', '-')
-        self.specs = list(self.norm_specs(self.specs))
+        #self.specs = list(self.norm_specs(self.specs))
         self.specifier = SpecifierSet(','.join(f"{op}{ver}" for op, ver in self.specs))
 
-    @staticmethod
-    def norm_specs(specs):
-        # PEP 440: Compatible Release
-        for spec in specs:
-            if spec[0] == "~=":
-                ver = spec[1]
-                yield ('>=', ver)
-                ver = parse(parse(ver).base_version)
-                ver_as_dict = ver._version._asdict()
-                ver_as_dict['release'] = ver_as_dict['release'][:-1] + ('*',)
-                ver._version = _Version(**ver_as_dict)
-                yield ('==', str(ver))
-            else:
-                yield spec
+    def __hash__(self):
+        return hash((super().__hash__(), self.build))
+
+    # @staticmethod
+    # def norm_specs(specs):
+    #     # PEP 440: Compatible Release
+    #     for spec in specs:
+    #         if spec[0] == "~=":
+    #             ver = spec[1]
+    #             yield ('>=', ver)
+    #             ver = parse(parse(ver).base_version)
+    #             ver_as_dict = ver._version._asdict()
+    #             ver_as_dict['release'] = ver_as_dict['release'][:-1] + ('*',)
+    #             ver._version = _Version(**ver_as_dict)
+    #             yield ('==', str(ver))
+    #         else:
+    #             yield spec
 
 
 def filter_reqs_by_eval_marker(reqs: Iterable[Requirement], context: dict, selected_extras=None):
@@ -62,9 +66,43 @@ def filter_reqs_by_eval_marker(reqs: Iterable[Requirement], context: dict, selec
                 yield req
 
 
+# @cached(lambda args: tuple(args[0]) if isinstance(args[0], list) else args[0])
+# def parse_reqs(strs):
+#     if isinstance(strs, str):
+#         strs = [strs]
+#     strs = list(map(
+#         lambda s: s.replace(' ', '==') if not any(op in s for op in ('==', '!=', '<=', '>=', '<', '>', '~=')) else s,
+#         strs
+#     ))
+#     reqs = list(pkg_resources.parse_requirements(strs))
+#     for req in reqs:
+#         r = Requirement(str(req))
+#         yield r
+
+
 @cached(lambda args: tuple(args[0]) if isinstance(args[0], list) else args[0])
 def parse_reqs(strs):
-    reqs = list(pkg_resources.parse_requirements(strs))
-    for req in reqs:
-        r = Requirement(str(req))
-        yield r
+    lines = iter(pkg_resources.yield_lines(strs))
+    for line in lines:
+        if ' #' in line:
+            line = line[:line.find(' #')]
+        if line.endswith('\\'):
+            line = line[:-2].strip()
+            try:
+                line += next(lines)
+            except StopIteration:
+                return
+
+        # handle conda requirements
+        build = None
+        if not any(op in line for op in ('==', '!=', '<=', '>=', '<', '>', '~=')):
+            # conda spec with build like "tensorflow-base 2.0.0 gpu_py36h0ec5d1f_0"
+            splitted = line.split(' ')
+            if len(splitted) == 3:
+                name, ver, build = splitted
+                line = f"{name}=={ver}"
+            # transform conda specifiers without operator like "requests 2.24.*"
+            else:
+                line = line.replace(' ', '==')
+
+        yield Requirement(line, build)

mach_nix/resolver/__init__.py

@@ -2,10 +2,9 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from typing import List, Iterable, Set
 
-from packaging.version import Version
-
 from mach_nix.data.providers import ProviderInfo
 from mach_nix.requirements import Requirement
+from mach_nix.versions import Version
 
 
 @dataclass
@@ -19,6 +18,7 @@ class ResolvedPkg:
     extras_selected: List[str]
     # contains direct or indirect children wich have been diconnected due to circular deps
     removed_circular_deps: Set[str] = field(default_factory=set)
+    build: str = None
 
 
 class Resolver(ABC):

mach_nix/resolver/resolvelib_resolver.py

@@ -2,28 +2,20 @@ from dataclasses import dataclass
 from typing import Iterable, List
 
 import resolvelib
-from packaging.version import Version
 
-from mach_nix.data.providers import DependencyProviderBase
 from mach_nix.data.nixpkgs import NixpkgsIndex
+from mach_nix.data.providers import DependencyProviderBase, Candidate
+from mach_nix.deptree import remove_circles_and_print
 from mach_nix.requirements import Requirement
 from mach_nix.resolver import Resolver, ResolvedPkg
-from mach_nix.versions import filter_versions
-from mach_nix.deptree import remove_circles_and_print
-
-
-@dataclass
-class Candidate:
-    name: str
-    ver: Version
-    extras: tuple
+from mach_nix.versions import filter_versions, Version
 
 
 # Implement logic so the resolver understands the requirement format.
 class Provider:
     def __init__(self, nixpkgs: NixpkgsIndex, deps_db: DependencyProviderBase):
         self.nixpkgs = nixpkgs
-        self.deps_db = deps_db
+        self.provider = deps_db
 
     def get_extras_for(self, dependency):
         return tuple(sorted(dependency.extras))
@@ -38,17 +30,18 @@ class Provider:
         return len(candidates)
 
     def find_matches(self, req):
-        all = self.deps_db.available_versions(req.key)
-        matching_versions = filter_versions(all, req.specs)
-        return [Candidate(name=req.name, ver=ver, extras=req.extras) for ver in matching_versions]
+        return self.provider.find_matches(req)
 
-    def is_satisfied_by(self, requirement, candidate):
+    def is_satisfied_by(self, requirement, candidate: Candidate):
+        res = None
         if not set(requirement.extras).issubset(set(candidate.extras)):
-            return False
-        return bool(len(list(filter_versions([candidate.ver], requirement.specs))))
+            res = False
+        res = bool(len(list(filter_versions([candidate.ver], requirement.specs))))
+        #print(f"{res} {requirement} satisfied by {candidate}")
+        return res
 
     def get_dependencies(self, candidate):
-        install_requires, setup_requires = self.deps_db.get_pkg_reqs(candidate.name, candidate.ver, candidate.extras)
+        install_requires, setup_requires = self.provider.get_pkg_reqs(candidate)
         deps = install_requires + setup_requires
         return deps
 
@@ -66,8 +59,7 @@ class ResolvelibResolver(Resolver):
             if name is None:
                 continue
             ver = result.mapping[name].ver
-            install_requires, setup_requires = self.deps_provider.get_pkg_reqs(
-                name, ver, extras=result.mapping[name].extras)
+            install_requires, setup_requires = self.deps_provider.get_pkg_reqs(result.mapping[name])
             provider_info = self.deps_provider.get_provider_info(name, ver)
             prop_build_inputs = list({req.key for req in install_requires})
             build_inputs = list({req.key for req in setup_requires})
@@ -79,7 +71,8 @@ class ResolvelibResolver(Resolver):
                 prop_build_inputs=prop_build_inputs,
                 is_root=is_root,
                 provider_info=provider_info,
-                extras_selected=list(result.mapping[name].extras)
+                extras_selected=list(result.mapping[name].extras),
+                build=result.mapping[name].build
             ))
         remove_circles_and_print(nix_py_pkgs, self.nixpkgs)
         return nix_py_pkgs

mach_nix/versions.py

@@ -3,80 +3,99 @@ import sys
 import traceback
 from typing import Iterable, Tuple, List
 
-from packaging.version import Version, LegacyVersion
+from conda.common.compat import with_metaclass
+from packaging.version import LegacyVersion
+from conda.models.version import ver_eval, VersionOrder, SingleStrArgCachingType
 
 from mach_nix.cache import cached
 import packaging.version
 
 
+Version = VersionOrder
+
+
 class PyVer(Version):
+    def __init__(self, vstr):
+        self.pypa_ver = packaging.version.Version(vstr)
+        super(PyVer, self).__init__(vstr)
     def nix(self):
         res = 'python'
-        res += str(self.release[0])
-        if len(self.release) >= 2:
-            res += str(self.release[1])
+        res += str(self.pypa_ver.release[0])
+        if len(self.pypa_ver.release) >= 2:
+            res += str(self.pypa_ver.release[1])
         return res
 
     def digits(self):
-        return ''.join(filter(lambda c: c.isdigit(), str(self)))[:2]
+        return ''.join(filter(lambda c: c.isdigit(), str(self.pypa_ver)))[:2]
 
     def python_version(self):
-        return f"{self.release[0]}.{self.release[1]}"
+        return f"{self.pypa_ver.release[0]}.{self.pypa_ver.release[1]}"
 
     def python_full_version(self):
         try:
-            return f"{self.release[0]}.{self.release[1]}.{self.release[2]}"
+            return f"{self.pypa_ver.release[0]}.{self.pypa_ver.release[1]}.{self.pypa_ver.release[2]}"
         except IndexError:
             traceback.print_exc()
             print("Error: please specify full python version including bugfix version (like 3.7.5)", file=sys.stderr)
             exit(1)
 
 
-def parse_ver(ver_str):
-    return packaging.version.parse(ver_str)
+def parse_ver(ver_str) -> Version:
+    #return packaging.version.parse(ver_str)
+    return Version(ver_str)
 
 
 def ver_better_than_other(v: Version, o: Version) -> bool:
-    # print(inspect.getfile(v.__class__))
-    instability = {v: 0, o: 0}
-    if v >= o:
-        for ver in [v, o]:
-            if ver.dev:
-                instability[ver] += 2
-            if ver.pre:
-                instability[ver] += 1
-        if instability[v] <= instability[o]:
-            return True
-    return False
-
-
-def ver_lt_other(v: Version, o: Version) -> bool:
-    return not (ver_better_than_other(v, o) or v == o)
+    return ver_eval(v, f">{o}")
+    # instability = {v: 0, o: 0}
+    # if v >= o:
+    #     for ver in [v, o]:
+    #         if ver.dev:
+    #             instability[ver] += 2
+    #         if ver.pre:
+    #             instability[ver] += 1
+    #     if instability[v] <= instability[o]:
+    #         return True
+    # return False
 
 
 def ver_sort_key(ver: Version):
     """
     For sorting versions by preference in reversed order. (last elem == highest preference)
     """
-    if isinstance(ver, LegacyVersion):
-        return 0, 0, 0, ver
-    is_dev = int(ver.is_devrelease)
-    is_pre = int(ver.is_prerelease)
-    return 1, not is_dev, not is_pre, ver
+    is_dev = 0
+    is_rc = 0
+    for component in ver.version:
+        if len(component) > 1:
+            for elem in component:
+                if not isinstance(elem, str):
+                    continue
+                if 'dev' in elem.lower():
+                    is_dev = 1
+                    break
+                if 'rc' in elem.lower():
+                    is_rc = 1
+                    break
+    # if isinstance(ver, LegacyVersion):
+    #     return 0, 0, 0, ver
+    # is_dev = int(ver.is_devrelease)
+    # is_pre = int(ver.is_prerelease)
+    return 1, not is_dev, not is_rc, ver
 
 
 def best_version(versions: Iterable[Version]) -> Version:
-    best = None
-    for ver in versions:
-        if best is None:
-            best = ver
-            continue
-        if ver_better_than_other(ver, best):
-            best = ver
-    return best
+    return sorted(versions)[-1]
+    # best = None
+    # for ver in versions:
+    #     if best is None:
+    #         best = ver
+    #         continue
+    #     if ver_better_than_other(ver, best):
+    #         best = ver
+    # return best
 
 
-@cached(keyfunc=lambda args: tuple(args[0]) + tuple(args[1]))
+#@cached(keyfunc=lambda args: tuple(args[0]) + tuple(args[1]))
 def filter_versions(
         versions: Iterable[Version],
         specs: Iterable[Tuple[str, str]]) -> List[Version]:
@@ -86,16 +105,22 @@ def filter_versions(
     """
     versions = list(versions)
     for op, ver in specs:
-        ver = parse_ver(ver)
         if op == '==':
-            versions_str = (str(ver) for ver in versions)
-            versions_str_filtered = list(ver_str for ver_str in fnmatch.filter(versions_str, str(ver)))
-            versions = [ver for ver in versions if str(ver) in versions_str_filtered]
-        elif op == '!=':
-            versions_str = (str(ver) for ver in versions)
-            bad_versions_str = set(fnmatch.filter(versions_str, str(ver)))
-            versions = list(filter(lambda v: str(v) not in bad_versions_str, versions))
-        else:
-            versions = list(filter(lambda v: eval(f'v {op} ver', dict(v=v, ver=ver)), versions))
+            if str(ver) == "*":
+                return versions
+            elif '*' in str(ver):
+                op = '='
+        ver = parse_ver(ver)
+        versions = list(filter(lambda v: ver_eval(v, f"{op}{ver}"), versions))
+        # if op == '==':
+        #     versions_str = (str(ver) for ver in versions)
+        #     versions_str_filtered = list(ver_str for ver_str in fnmatch.filter(versions_str, str(ver)))
+        #     versions = [ver for ver in versions if str(ver) in versions_str_filtered]
+        # elif op == '!=':
+        #     versions_str = (str(ver) for ver in versions)
+        #     bad_versions_str = set(fnmatch.filter(versions_str, str(ver)))
+        #     versions = list(filter(lambda v: str(v) not in bad_versions_str, versions))
+        # else:
+        #     versions = list(filter(lambda v: eval(f'v {op} ver', dict(v=v, ver=ver)), versions))
     return list(versions)