From a20691048e87cbaafe33469cb096158f1783f7be Mon Sep 17 00:00:00 2001 From: DavHau Date: Sun, 9 Aug 2020 20:24:12 +0700 Subject: [PATCH] version 2.2.0 (#60) # 2.2.0 (09 Aug 2020) Improved success rate, MacOS support, bugfixes, optimizations ### Features - Improved selection of wheel releases. MacOS is now supported and architectures besides x86_64 should be handled correctly. - Whenever mach-nix resolves dependencies, a visualization of the resulting dependency tree is printed on the terminal. - The dependency DB is now accessed through a caching layer which reduces the resolver's CPU time significantly for larger environments. - The python platform context is now generated from the nix build environment variable `system`. This should decrease the chance of impurities during dependency resolution. ### Fixes - The requires_python attribute of wheels was not respected. This lead to failing builds especially for older python versions. Now `requires_python` is part of the dependency graph and affects resolution. - Detecting the correct package name for python packages in nixpkgs often failed since the attribute names don't follow a fixed schema. This lead to a handful of different errors in different situations. Now the package names are extracted from the pypi `url` inside the `src` attribute which is much more reliable. For packages which are not fetched from pypi, the `pname` attribute is used as fallback. - Fixed bug which lead to the error `attribute 'sdist' missing` if a package from the nixpkgs provider was used which doesn't publish it's source on pypi. (For example `tensorflow`) ### Other Changes - Mach-nix now uses a revision of the nixpkgs-unstable branch instead of nixos-20.03 as base fo the tool and the nixpkgs provider. - Updated revision of the dependency DB --- Changelog.md | 18 ++++++++++ Readme.md | 6 ++-- debug/nixpkgs-json.nix | 6 +++- debug/overrides_pre.nix.example | 19 +++++++++++ default.nix | 33 +++++++++--------- examples.md | 14 ++++---- mach_nix/VERSION | 2 +- mach_nix/data/nixpkgs.py | 3 ++ mach_nix/data/providers.py | 41 ++++++++++++----------- mach_nix/generate.py | 2 +- mach_nix/generators/overides_generator.py | 10 ++++-- mach_nix/nix/NIXPKGS_COMMIT | 2 +- mach_nix/nix/NIXPKGS_SHA256 | 2 +- mach_nix/nix/PYPI_DEPS_DB_COMMIT | 2 +- mach_nix/nix/PYPI_DEPS_DB_SHA256 | 2 +- mach_nix/nix/mach.nix | 6 +++- mach_nix/nix/nixpkgs-json.nix | 4 +-- mach_nix/requirements.py | 7 ++-- 18 files changed, 117 insertions(+), 62 deletions(-) create mode 100644 debug/overrides_pre.nix.example diff --git a/Changelog.md b/Changelog.md index f3f64d2..6a3aef7 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,21 @@ +# 2.2.0 (09 Aug 2020) +Improved success rate, MacOS support, bugfixes, optimizations + +### Features + - Improved selection of wheel releases. MacOS is now supported and architectures besides x86_64 should be handled correctly. + - Whenever mach-nix resolves dependencies, a visualization of the resulting dependency tree is printed on the terminal. + - The dependency DB is now accessed through a caching layer which reduces the resolver's CPU time significantly for larger environments. + - The python platform context is now generated from the nix build environment variable `system`. This should decrease the chance of impurities during dependency resolution. + +### Fixes + - The requires_python attribute of wheels was not respected. This lead to failing builds especially for older python versions. Now `requires_python` is part of the dependency graph and affects resolution. + - Detecting the correct package name for python packages in nixpkgs often failed since the attribute names don't follow a fixed schema. This lead to a handful of different errors in different situations. Now the package names are extracted from the pypi `url` inside the `src` attribute which is much more reliable. For packages which are not fetched from pypi, the `pname` attribute is used as fallback. + - Fixed bug which lead to the error `attribute 'sdist' missing` if a package from the nixpkgs provider was used which doesn't publish it's source on pypi. (For example `tensorflow`) + +### Other Changes + - Mach-nix now uses a revision of the nixpkgs-unstable branch instead of nixos-20.03 as base fo the tool and the nixpkgs provider. + - Updated revision of the dependency DB + # 2.1.1 (30 Jul 2020) Fix broken wheel packages ### Fixes: diff --git a/Readme.md b/Readme.md index 8e9c8b2..d4d821b 100644 --- a/Readme.md +++ b/Readme.md @@ -55,11 +55,11 @@ Table of Contents You can either install mach-nix via pip or by using nix in case you already have the nix package manager installed. #### Installing via pip ```shell -pip install git+git://github.com/DavHau/mach-nix@2.1.1 +pip install git+git://github.com/DavHau/mach-nix@2.2.0 ``` #### Installing via nix ```shell -nix-env -if https://github.com/DavHau/mach-nix/tarball/2.1.1 -A mach-nix +nix-env -if https://github.com/DavHau/mach-nix/tarball/2.2.0 -A mach-nix ``` --- @@ -91,7 +91,7 @@ You can call mach-nix directly from a nix expression let mach-nix = import (builtins.fetchGit { url = "https://github.com/DavHau/mach-nix/"; - ref = "2.1.1"; + ref = "2.2.0"; }); in mach-nix.mkPython { diff --git a/debug/nixpkgs-json.nix b/debug/nixpkgs-json.nix index 339248e..70152fc 100644 --- a/debug/nixpkgs-json.nix +++ b/debug/nixpkgs-json.nix @@ -10,4 +10,8 @@ let python = (import target_nixpkgs_src { config = {}; }).python37; in with import target_nixpkgs_src { config = {}; overlays = []; }; -import ../mach_nix/nix/nixpkgs-json.nix { inherit pkgs python; } \ No newline at end of file +import ../mach_nix/nix/nixpkgs-json.nix rec { + inherit pkgs python; + overrides = [(import ./overrides_pre.nix)]; + mergeOverrides = lib.foldr lib.composeExtensions (self: super: { }); +} \ No newline at end of file diff --git a/debug/overrides_pre.nix.example b/debug/overrides_pre.nix.example new file mode 100644 index 0000000..9109b13 --- /dev/null +++ b/debug/overrides_pre.nix.example @@ -0,0 +1,19 @@ +pySelf: pySuper: { + pulsar = pySuper.buildPythonPackage { + src = fetchGit { + url = "https://github.com/DavHau/pulsar"; + ref = "dev"; + rev = "4b01fa19cc28ac912a6ffb18ef64d46bae77f800"; + }; + pname = "pulsar"; + version = "2.0.2"; # not really + doCheck = false; + doInstallCheck = false; + }; + asyncssh = pySuper.asyncssh.overridePythonAttrs (oldAttrs: { + patches = []; + }); + pytest-dependency = pySuper.pytest-dependency.overridePythonAttrs (oldAttrs: { + patches = []; + }); +} \ No newline at end of file diff --git a/default.nix b/default.nix index fa90632..a846bd9 100644 --- a/default.nix +++ b/default.nix @@ -2,18 +2,7 @@ let pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; overlays = []; }; python = import ./mach_nix/nix/python.nix { inherit pkgs; }; python_deps = (builtins.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = pkgs.fetchurl; })); - mergeOverrides = with pkgs.lib; overrides: - if length overrides == 0 - then a: b: {} # return dummy overrides - else - if length overrides == 1 - then elemAt overrides 0 - else - let - last = head ( reverseList overrides ); - rest = reverseList (tail ( reverseList overrides )); - in - composeExtensions (mergeOverrides rest) last; + mergeOverrides = with pkgs.lib; foldr composeExtensions (self: super: { }); autoPatchelfHook = import ./mach_nix/nix/auto_patchelf_hook.nix {inherit (pkgs) fetchurl makeSetupHook writeText;}; in rec { @@ -37,9 +26,15 @@ rec { # Returns `overrides` and `select_pkgs` which satisfy your requirements machNix = args: - let result = import "${machNixFile args}/share/mach_nix_file.nix"; + let + result = import "${machNixFile args}/share/mach_nix_file.nix"; + manylinux = + if pkgs.stdenv.hostPlatform.system == "x86_64-darwin" then + [] + else + pkgs.pythonManylinuxPackages.manylinux1; in { - overrides = result.overrides pkgs.pythonManylinuxPackages.manylinux1 autoPatchelfHook; + overrides = result.overrides manylinux autoPatchelfHook; select_pkgs = result.select_pkgs; }; @@ -57,11 +52,11 @@ rec { disable_checks ? true, # Disable tests wherever possible to decrease build time. overrides_pre ? [], # list of pythonOverrides to apply before the machnix overrides overrides_post ? [], # list of pythonOverrides to apply after the machnix overrides - pkgs ? nixpkgs, # pass custom nixpkgs. Only used for manylinux wheel dependencies + pkgs ? nixpkgs, # pass custom nixpkgs. providers ? {}, # define provider preferences pypi_deps_db_commit ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_COMMIT, # python dependency DB version pypi_deps_db_sha256 ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_SHA256, - python ? pkgs.python3, # select custom python to base overrides on. Should be from nixpkgs >= 20.03 + python ? pkgs.python3, # select custom python to base overrides onto. Should be from nixpkgs >= 20.03 _provider_defaults ? with builtins; fromTOML (readFile ./mach_nix/provider_defaults.toml), ... }: @@ -69,6 +64,7 @@ rec { py = python.override { packageOverrides = mergeOverrides overrides_pre; }; result = machNix { inherit requirements disable_checks providers pypi_deps_db_commit pypi_deps_db_sha256 _provider_defaults; + overrides = overrides_pre; python = py; }; py_final = python.override { packageOverrides = mergeOverrides ( @@ -88,17 +84,18 @@ rec { disable_checks ? true, # Disable tests wherever possible to decrease build time. overrides_pre ? [], # list of pythonOverrides to apply before the machnix overrides overrides_post ? [], # list of pythonOverrides to apply after the machnix overrides - pkgs ? nixpkgs, # pass custom nixpkgs. Only used for manylinux wheel dependencies + pkgs ? nixpkgs, # pass custom nixpkgs. providers ? {}, # define provider preferences pypi_deps_db_commit ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_COMMIT, # python dependency DB version pypi_deps_db_sha256 ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_SHA256, - python ? pkgs.python3, # select custom python to base overrides on. Should be from nixpkgs >= 20.03 + python ? pkgs.python3, # select custom python to base overrides onto. Should be from nixpkgs >= 20.03 _provider_defaults ? with builtins; fromTOML (readFile ./mach_nix/provider_defaults.toml) }: let py = python.override { packageOverrides = mergeOverrides overrides_pre; }; result = machNix { inherit requirements disable_checks providers pypi_deps_db_commit pypi_deps_db_sha256 _provider_defaults; + overrides = overrides_pre; python = py; }; py_final = python.override { packageOverrides = mergeOverrides ( diff --git a/examples.md b/examples.md index b230172..b490ff7 100644 --- a/examples.md +++ b/examples.md @@ -24,7 +24,7 @@ build a python environment from a list of requirements let mach-nix = import (builtins.fetchGit { url = "https://github.com/DavHau/mach-nix/"; - ref = "2.1.1"; + ref = "2.2.0"; }); in mach-nix.mkPython { requirements = builtins.readFile ./requirements.txt; @@ -37,7 +37,7 @@ Build a python package from its source code and a list of requirements let mach-nix = import (builtins.fetchGit { url = "https://github.com/DavHau/mach-nix/"; - ref = "2.1.1"; + ref = "2.2.0"; }); in mach-nix.buildPythonPackage { pname = "my-package"; @@ -53,7 +53,7 @@ Build a python package from its source code and a list of requirements let mach-nix = import (builtins.fetchGit { url = "https://github.com/DavHau/mach-nix/"; - ref = "2.1.1"; + ref = "2.2.0"; }); in mach-nix.buildPythonPackage rec { pname = "projectname"; @@ -77,7 +77,7 @@ I have a complex set of requirements including tensorflow. I'd like to have tens let mach-nix = import (builtins.fetchGit { url = "https://github.com/DavHau/mach-nix/"; - ref = "2.1.1"; + ref = "2.2.0"; }); in mach-nix.mkPython { @@ -100,7 +100,7 @@ I'd like to install a more recent version of tensorflow which is not available f let mach-nix = import (builtins.fetchGit { url = "https://github.com/DavHau/mach-nix/"; - ref = "2.1.1"; + ref = "2.2.0"; }); in mach-nix.mkPython { @@ -129,7 +129,7 @@ I'd like to use a recent version of Pytorch from wheel, but I'd like to build th let mach-nix = import (builtins.fetchGit { url = "https://github.com/DavHau/mach-nix/"; - ref = "2.1.1"; + ref = "2.2.0"; }); overlays = []; # some very useful overlays in mach-nix.mkPython rec { @@ -164,7 +164,7 @@ I have a complex requirements.txt which includes `imagecodecs`. It is available let mach-nix = import (builtins.fetchGit { url = "https://github.com/DavHau/mach-nix/"; - ref = "2.1.1"; + ref = "2.2.0"; }); in mach-nix.mkPython rec { diff --git a/mach_nix/VERSION b/mach_nix/VERSION index 50aea0e..e3a4f19 100644 --- a/mach_nix/VERSION +++ b/mach_nix/VERSION @@ -1 +1 @@ -2.1.0 \ No newline at end of file +2.2.0 \ No newline at end of file diff --git a/mach_nix/data/nixpkgs.py b/mach_nix/data/nixpkgs.py index b999ea2..104ea53 100644 --- a/mach_nix/data/nixpkgs.py +++ b/mach_nix/data/nixpkgs.py @@ -5,6 +5,8 @@ from typing import List from packaging.version import Version, parse +from mach_nix.cache import cached + @dataclass class NixpkgsPyPkg: @@ -48,6 +50,7 @@ class NixpkgsIndex(UserDict): return False return ver1.release[ver_idx] == ver2.release[ver_idx] + @cached(lambda args: (args[1], args[2])) def find_best_nixpkgs_candidate(self, name, ver): """ In case a python package has more than one candidate in nixpkgs diff --git a/mach_nix/data/providers.py b/mach_nix/data/providers.py index 3ef86d6..9c86fbc 100644 --- a/mach_nix/data/providers.py +++ b/mach_nix/data/providers.py @@ -8,6 +8,7 @@ from typing import List, Tuple, Iterable import distlib.markers from packaging.version import Version, parse +from pkg_resources import RequirementParseError from .nixpkgs import NixpkgsIndex from mach_nix.requirements import filter_reqs_by_eval_marker, Requirement, parse_reqs, context @@ -65,7 +66,7 @@ class PackageNotFound(Exception): class DependencyProviderBase(ABC): def __init__(self, py_ver: PyVer, platform, system, *args, **kwargs): - self.context = context(py_ver) + self.context = context(py_ver, platform, system) self.context_wheel = self.context.copy() self.context_wheel['extra'] = None self.py_ver_digits = py_ver.digits() @@ -202,11 +203,6 @@ class CombinedDependencyProvider(DependencyProviderBase): class NixpkgsDependencyProvider(DependencyProviderBase): name = 'nixpkgs' - # mapping from pypi name to nix key - # _aliases = dict( - # torch='pytorch', - # tensorboard='tensorflow-tensorboard' - # ) # TODO: implement extras by looking them up via the equivalent wheel def __init__( @@ -261,22 +257,24 @@ class WheelDependencyProvider(DependencyProviderBase): def __init__(self, data_dir: str, *args, **kwargs): super(WheelDependencyProvider, self).__init__(*args, **kwargs) self.data = LazyBucketDict(data_dir) - m = self.py_ver_digits[-1] + maj = self.py_ver_digits[0] # major version + min = self.py_ver_digits[1] # minor version if self.system == "linux": + cp_abi = f"cp{maj}{min}mu" if int(maj) == 2 else f"cp{maj}{min}m" self.preferred_wheels = ( - re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-manylinux2014_{self.platform}"), - re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-manylinux2010_{self.platform}"), - re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-manylinux1_{self.platform}"), - re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-linux_{self.platform}"), - re.compile(rf"(py3|cp3){m}?-(cp3{m}m|abi3|none)-any"), + re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-manylinux2014_{self.platform}"), + re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-manylinux2010_{self.platform}"), + re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-manylinux1_{self.platform}"), + re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-linux_{self.platform}"), + re.compile(rf"(py{maj}|cp{maj}){min}?-({cp_abi}|abi3|none)-any"), ) elif self.system == "darwin": self.preferred_wheels = ( - re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_universal"), - re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_x86_64"), - re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_intel"), - re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-macosx_\d*_\d*_(fat64|fat32)"), - re.compile(rf"(py3|cp3){m}?-(cp3{m}|abi3|none)-any"),) + re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-macosx_\d*_\d*_universal"), + re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-macosx_\d*_\d*_x86_64"), + re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-macosx_\d*_\d*_intel"), + re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-macosx_\d*_\d*_(fat64|fat32)"), + re.compile(rf"(py{maj}|cp{maj}){min}?-(cp{maj}{min}|abi3|none)-any"),) else: raise Exception(f"Unsupported Platform {platform.system()}") @@ -317,7 +315,7 @@ class WheelDependencyProvider(DependencyProviderBase): fn, deps['requires_dist'] if 'requires_dist' in deps else None, deps['requires_extras'] if 'requires_extras' in deps else None, - deps['requires_python'] if 'requires_python' in deps else None, + deps['requires_python'].strip(',') if 'requires_python' in deps else None, ) def _apply_filters(self, filters: List[callable], objects: Iterable): @@ -362,7 +360,12 @@ class WheelDependencyProvider(DependencyProviderBase): if not wheel.requires_python: return True ver = parse('.'.join(self.py_ver_digits)) - return bool(filter_versions([ver], list(parse_reqs(f"python{wheel.requires_python}"))[0].specs)) + try: + parsed_py_requires = list(parse_reqs(f"python{wheel.requires_python}")) + return bool(filter_versions([ver], parsed_py_requires[0].specs)) + except RequirementParseError: + print(f"WARNING: `requires_python` attribute of wheel {wheel.name}:{wheel.ver} could not be parsed") + return False class SdistDependencyProvider(DependencyProviderBase): diff --git a/mach_nix/generate.py b/mach_nix/generate.py index 721c9ca..afdbfe8 100644 --- a/mach_nix/generate.py +++ b/mach_nix/generate.py @@ -48,7 +48,7 @@ def main(): disable_checks, ResolvelibResolver(nixpkgs, deps_provider), ) - reqs = filter_reqs_by_eval_marker(parse_reqs(requirements), context(py_ver)) + reqs = filter_reqs_by_eval_marker(parse_reqs(requirements), context(py_ver, platform, system)) expr = generator.generate(reqs) with open(out_file, 'w') as f: f.write(expr) diff --git a/mach_nix/generators/overides_generator.py b/mach_nix/generators/overides_generator.py index 3b2b7db..7e408bd 100644 --- a/mach_nix/generators/overides_generator.py +++ b/mach_nix/generators/overides_generator.py @@ -68,11 +68,13 @@ class OverridesGenerator(ExpressionGenerator): f"{b}" for b in sorted(prop_build_inputs_local | prop_build_inputs_nixpkgs)) return prop_build_inputs_str - def _gen_overrideAttrs(self, name, ver, nix_name, build_inputs_str, prop_build_inputs_str): + def _gen_overrideAttrs(self, name, ver, nix_name, build_inputs_str, prop_build_inputs_str, keep_src=False): out = f""" {nix_name} = python-super.{nix_name}.overridePythonAttrs ( oldAttrs: {{ pname = "{name}"; - version = "{ver}"; + version = "{ver}";""" + if not keep_src: + out += f""" src = fetchPypi "{name}" "{ver}";""" if build_inputs_str: out += f""" @@ -183,7 +185,9 @@ class OverridesGenerator(ExpressionGenerator): out += self._unify_nixpkgs_keys(pkg.name) elif pkg.provider_info.provider == NixpkgsDependencyProvider.name: nix_name = self.nixpkgs.find_best_nixpkgs_candidate(pkg.name, pkg.ver) - out += self._gen_overrideAttrs(pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str) + out += self._gen_overrideAttrs( + pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str, + keep_src=True) out += self._unify_nixpkgs_keys(pkg.name, main_key=nix_name) end_overlay_section = f""" }}; diff --git a/mach_nix/nix/NIXPKGS_COMMIT b/mach_nix/nix/NIXPKGS_COMMIT index 26a50f0..8d67be2 100644 --- a/mach_nix/nix/NIXPKGS_COMMIT +++ b/mach_nix/nix/NIXPKGS_COMMIT @@ -1 +1 @@ -f8248ab6d9e69ea9c07950d73d48807ec595e923 \ No newline at end of file +1365b9ac700aabe8dc959e9897bc1376e416c553 \ No newline at end of file diff --git a/mach_nix/nix/NIXPKGS_SHA256 b/mach_nix/nix/NIXPKGS_SHA256 index 0423e18..64662a2 100644 --- a/mach_nix/nix/NIXPKGS_SHA256 +++ b/mach_nix/nix/NIXPKGS_SHA256 @@ -1 +1 @@ -009i9j6mbq6i481088jllblgdnci105b2q4mscprdawg3knlyahk \ No newline at end of file +180bq107z7qc2h4dxs3fxmrag3l8p9lvgjf3530qknp2h3ji8vqh \ No newline at end of file diff --git a/mach_nix/nix/PYPI_DEPS_DB_COMMIT b/mach_nix/nix/PYPI_DEPS_DB_COMMIT index 2017f40..4b73a58 100644 --- a/mach_nix/nix/PYPI_DEPS_DB_COMMIT +++ b/mach_nix/nix/PYPI_DEPS_DB_COMMIT @@ -1 +1 @@ -2a2501aab1fe4eb50763652c4f74c5327f976f85 \ No newline at end of file +521ca251792514f1a8170dd89b11edd158d7bfbe \ No newline at end of file diff --git a/mach_nix/nix/PYPI_DEPS_DB_SHA256 b/mach_nix/nix/PYPI_DEPS_DB_SHA256 index 4c60710..27fee5a 100644 --- a/mach_nix/nix/PYPI_DEPS_DB_SHA256 +++ b/mach_nix/nix/PYPI_DEPS_DB_SHA256 @@ -1 +1 @@ -03c64w1x4q2li57cn6zsb2shjk2fc4f5ksmsz3695dmvckp0lj14 \ No newline at end of file +01hi8r7pzvpimw282zsnk1giysznarx8a067dnh8m9h7wz506lgc \ No newline at end of file diff --git a/mach_nix/nix/mach.nix b/mach_nix/nix/mach.nix index 9dd800c..b09964f 100644 --- a/mach_nix/nix/mach.nix +++ b/mach_nix/nix/mach.nix @@ -2,6 +2,7 @@ requirements, # content from a requirements.txt file python, # python from nixpkgs as base for overlay disable_checks ? true, # disable tests wherever possible + overrides ? [], providers ? {}, # re-order to change provider priority or remove providers pypi_deps_db_commit ? builtins.readFile ./PYPI_DEPS_DB_COMMIT, # python dependency DB version # Hash obtained using `nix-prefetch-url --unpack https://github.com/DavHau/pypi-deps-db/tarball/` @@ -10,7 +11,10 @@ }: let pkgs = import (import ./nixpkgs-src.nix) { config = {}; overlays = []; }; - nixpkgs_json = import ./nixpkgs-json.nix { inherit pkgs python; }; + nixpkgs_json = import ./nixpkgs-json.nix { + inherit overrides pkgs python; + mergeOverrides = with pkgs.lib; foldr composeExtensions (self: super: { }); + }; builder_python = pkgs.python37.withPackages(ps: (pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python37; fetchurl = pkgs.fetchurl; })) ); diff --git a/mach_nix/nix/nixpkgs-json.nix b/mach_nix/nix/nixpkgs-json.nix index fce7199..75e9ebe 100644 --- a/mach_nix/nix/nixpkgs-json.nix +++ b/mach_nix/nix/nixpkgs-json.nix @@ -1,4 +1,4 @@ -{pkgs, python}: +{pkgs, python, mergeOverrides, overrides}: let fetchPypiPnamePassthruOverride = pySelf: PySuper: { fetchPypi = let @@ -22,7 +22,7 @@ let }); }; - py = python.override { packageOverrides = fetchPypiPnamePassthruOverride; }; + py = python.override { packageOverrides = mergeOverrides ( overrides ++ [ fetchPypiPnamePassthruOverride ] ); }; in with pkgs; diff --git a/mach_nix/requirements.py b/mach_nix/requirements.py index 7ebc92a..bb16e72 100644 --- a/mach_nix/requirements.py +++ b/mach_nix/requirements.py @@ -10,10 +10,13 @@ from mach_nix.cache import cached from mach_nix.versions import PyVer -def context(py_ver: PyVer): +def context(py_ver: PyVer, platform: str, system: str): context = DEFAULT_CONTEXT.copy() context.update(dict( - platform_version='', # remove highly impure platform_version + platform_version='', # remove impure platform_version + platform_release='', # remove impure kernel verison + platform_system=system[0].upper() + system[1:], # eg. Linux or Darwin + platform_machine=platform, # eg. x86_64 python_version=py_ver.python_version(), python_full_version=py_ver.python_full_version() ))