diff --git a/Changelog.md b/Changelog.md index 00575c6..f2e0170 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,34 @@ +# 3.3.0 (22 May 2021) +bugfixes, improvements + +### Changes + - The flakes cmdline api has been changed. New usage: + ``` + nix (build|shell) mach-nix#gen.(python|docker).package1.package2... + ``` + (Despite this change being backward incompatible, I did not bump the major version since everything flakes related should be considered experimental anyways) + +### Improvements + - Mach-nix (used via flakes) will now throw an error if the selected nixpkgs version is newer than the dependency DB since this can cause conflicts in the resulting environment. + - When used via flakes, it was impossible to select the python version because the import function is not used anymore. Now `python` can be passed to `mkPython` alternatively. + - For the flakes cmdline api, collisions are now ignored by default + - The simplified override interface did not deal well with non-existent values. + - Now the `.add` directive automatically assumes an empty list/set/string when the attribute to be extended doesn't exist. + - Now the `.mod` directive will pass `null` to the given function if the attribute to modify doesn't exist instead. + +### Fixes + - Generating an environment with a package named `overrides` failed due to a variable name collision in the resulting nix expression. + - When used via flakes, the pypiData was downloaded twice, because the legacy code path for fetching was still used instead of the flakes input. + - `nix flake show mach-nix` failed because it required IFD for foreign platforms. + - For environments generated via `mach-nix env ...` the `python` command referred to the wrong interpreter. + - When checking wheels for compatibility, the minor version for python was not respected which could lead to invalid environments. + - Some python modules in nixpkgs propagate unnecessary dependencies which could lead to collisions in the final environment. Now mach-nix recursively removes all python dependencies which are not strictly required. + +### Package Fixes + - cryptography: remove rust related hook when version < 3.4 + + + # 3.2.0 (11 Mar 2021) bugfixes, ignoreCollisions diff --git a/default.nix b/default.nix index 9a94277..ba2dda1 100644 --- a/default.nix +++ b/default.nix @@ -110,7 +110,6 @@ rec { # expose dot interface for flakes cmdline "with" = pythonWith; pythonWith = (withDot (__mkPython "'.pythonWith'")).pythonWith; - shellWith = (withDot (__mkPython "'.shellWith'")).shellWith; dockerImageWith = (withDot (__mkPython "'.dockerImageWith'")).dockerImageWith; # expose mach-nix' nixpkgs diff --git a/flake.nix b/flake.nix index d9ac192..b6e8fb9 100644 --- a/flake.nix +++ b/flake.nix @@ -19,7 +19,7 @@ true else false; - usageGen = "usage: nix (build|develop) mach-nix#gen.(python|shell|docker).package1.package2..."; + usageGen = "usage: nix (build|shell) mach-nix#gen.(python|docker).package1.package2..."; in (flake-utils.lib.eachDefaultSystem (system: let @@ -49,11 +49,9 @@ src = throw usageGen; passthru = { python = mach-nix-default.pythonWith; - shell = mach-nix-default.shellWith; docker = mach-nix-default.dockerImageWith; inherit (mach-nix-default) pythonWith - shellWith dockerImageWith; }; }; diff --git a/interpreter.nix b/interpreter.nix index ac24b5a..64acaef 100644 --- a/interpreter.nix +++ b/interpreter.nix @@ -1,7 +1,7 @@ # python interpreter for dev environment let pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; }; - python = pkgs.python37; + python = pkgs.python38; deps = (pkgs.lib.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = pkgs.fetchurl; })); in python.withPackages (ps: deps ++ [ diff --git a/mach_nix/data/providers.py b/mach_nix/data/providers.py index d200e33..094a430 100644 --- a/mach_nix/data/providers.py +++ b/mach_nix/data/providers.py @@ -199,10 +199,10 @@ class CombinedDependencyProvider(DependencyProviderBase): f"Consider adding them via 'providers='" else: error_text += \ - f"\nIf the package's initial release date predates the release date of mach-nix, " \ - f"either upgrade mach-nix itself or manually specify 'pypi_deps_db_commit' and\n" \ - f"'pypi_deps_db_sha256 for a newer commit of https://github.com/DavHau/pypi-deps-db/commits/master\n" \ - f"If it still doesn't work, there was probably a problem while crawling pypi.\n" \ + f"\nThe required package might just not yet be part of the dependency DB currently used.\n" \ + f"The DB can be updated by specifying 'pypiDataRev' when importing mach-nix.\n" \ + f"For examples see: https://github.com/DavHau/mach-nix/blob/master/examples.md\n" \ + f"If it still doesn't work, there might have bene an error while building the DB.\n" \ f"Please open an issue at: https://github.com/DavHau/mach-nix/issues/new\n" print(error_text, file=sys.stderr) exit(1) diff --git a/mach_nix/fixes.nix b/mach_nix/fixes.nix index 0141271..d613143 100644 --- a/mach_nix/fixes.nix +++ b/mach_nix/fixes.nix @@ -20,28 +20,36 @@ let result = parse('${ver1}') ${op} parse('${ver2}') print(json.dumps(result)) ''; + + # prevent the normal fitler from being used + filter = raise "Error: only use filterSafe (doesn't crash on null input)"; + + # filter function that doesn't crash on 'objects = null' + # This is important because null will be returned whenever the attribute to modify doesn't exist + filterSafe = func: objects: if isNull objects then [] else builtins.filter func objects; + in ### Put Fixes here rec { -### FORMAT ############################################################################ -# # -# package-to-fix = { # -# name-of-the-fix = { # -# # optionally limit the fix to a condtion # -# _cond = {prov, ver, ... }: some boolean expression; # -# # -# # define overrides # -# key-to-override = ...; # replace # -# key-to-override.add = ...; # append # -# key-to-override.mod = oldVal: ...; # modify # -# key-to-override.mod = pySelf: oldAttrs: oldVal: ...; # modify (more args) # -# }; # -# }; # -# # -######################################################################################### +### FORMAT ################################################################################## +# # +# package-to-fix = { # +# name-of-the-fix = { # +# # optionally limit the fix to a condtion # +# _cond = {prov, ver, ... }: some boolean expression; # +# # +# # define overrides # +# key-to-override = ...; # replace # +# key-to-override.add = ...; # append (list/attrs/string) # +# key-to-override.mod = oldVal: ...; # modify # +# key-to-override.mod = pySelf: oldAttrs: oldVal: ...; # modify (accessing all pkgs) # +# }; # +# }; # +# # +############################################################################################### ### _cond #################################### # possible arguments: # @@ -57,7 +65,7 @@ rec { cryptography.no-rust-build = { _cond = { prov, ver, ... }: prov == "sdist" && comp_ver ver "<" "3.4"; - nativeBuildInputs.mod = old: filter (inp: (inp.name or "") != "cargo-setup-hook.sh") old; + nativeBuildInputs.mod = old: filterSafe (inp: (inp.name or "") != "cargo-setup-hook.sh") old; }; # remove if merged: https://github.com/NixOS/nixpkgs/pull/114384 @@ -83,7 +91,7 @@ rec { pip.remove-reproducible-patch = { _cond = { prov, ver, ... }: prov == "sdist" && comp_ver ver "<" "20.0"; - patches.mod = oldPatches: filter (patch: ! hasSuffix "reproducible.patch" patch) oldPatches; + patches.mod = oldPatches: filterSafe (patch: ! hasSuffix "reproducible.patch" patch) oldPatches; }; pyqt5 = { @@ -91,7 +99,7 @@ rec { # fix mach-nix induced problem: mach-nix removes all previous python inputs from propagatedBuildInputs _cond = {prov, ... }: prov == "nixpkgs"; propagatedBuildInputs.mod = pySelf: oldAttrs: oldVal: - (filter (p: p.pname != "pyqt5-sip") oldVal) ++ [ pySelf.sip pySelf.dbus-python ]; + (filterSafe (p: p.pname != "pyqt5-sip") oldVal) ++ [ pySelf.sip pySelf.dbus-python ]; }; fix-wheel-inputs = { _cond = {prov, ... }: prov == "wheel"; @@ -106,7 +114,7 @@ rec { # https://github.com/rpy2/rpy2/commit/39e1cb6fca0d4107f1078727d8670c422e3c6f7f prov == "sdist" && comp_ver ver ">=" "3.2.6"; - patches.mod = oldPatches: filter (p: ! hasSuffix "pandas-1.x.patch" p) oldPatches; + patches.mod = oldPatches: filterSafe (p: ! hasSuffix "pandas-1.x.patch" p) oldPatches; }; tensorflow.rm-tensorboard = { diff --git a/mach_nix/flake.lock b/mach_nix/flake.lock index 8d8d9ec..f9b13cb 100644 --- a/mach_nix/flake.lock +++ b/mach_nix/flake.lock @@ -34,11 +34,11 @@ "pypi-deps-db": { "flake": false, "locked": { - "lastModified": 1620853232, - "narHash": "sha256-SeePgys1mlGUrN/YhCRoo8nApRx5H+KKjyKM1zQ+yAE=", + "lastModified": 1621672288, + "narHash": "sha256-hUOZnSr1ZaTCOUlC8k946RyRjZsfN2w/9jzic0UKj58=", "owner": "DavHau", "repo": "pypi-deps-db", - "rev": "15175225c8295e08e5b95b2965892820a64e9746", + "rev": "19dc04a5c7412431a58a07ceecfe7b3ed3c164a5", "type": "github" }, "original": { diff --git a/mach_nix/generate.py b/mach_nix/generate.py index 1e2a1f5..24d1e3a 100644 --- a/mach_nix/generate.py +++ b/mach_nix/generate.py @@ -90,8 +90,12 @@ def handle_resolution_impossible(exc: ResolutionImpossible, reqs_str, providers_ f"Python: {py_ver_str}\n" f"Cause: {exc.__context__}\n" f"The requirements which caused the error:" - f"{causes_str}\n", - file=sys.stderr) + f"{causes_str}\n" + f"\nThe given requirements might contain package versions which are not yet part of the dependency DB\n" + f"currently used. The DB can be updated by specifying 'pypiDataRev' when importing mach-nix.\n" + f"For examples see: https://github.com/DavHau/mach-nix/blob/master/examples.md\n", + file=sys.stderr + ) def main(): diff --git a/mach_nix/nix/compileOverrides.nix b/mach_nix/nix/compileOverrides.nix index c752ba6..f0c33f7 100644 --- a/mach_nix/nix/compileOverrides.nix +++ b/mach_nix/nix/compileOverrides.nix @@ -44,8 +44,8 @@ let nixpkgs_json = import ./nixpkgs-json.nix { inherit overrides pkgs python; }; - builder_python = pkgs.pkgsBuildHost.python37.withPackages(ps: - (pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python37; fetchurl = pkgs.fetchurl; })) + builder_python = pkgs.pkgsBuildHost.python38.withPackages(ps: + (pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python38; fetchurl = pkgs.fetchurl; })) ); src = ./../../.; diff --git a/mach_nix/nix/deps-db-and-fetcher.nix b/mach_nix/nix/deps-db-and-fetcher.nix index 8bb668b..ff1b212 100644 --- a/mach_nix/nix/deps-db-and-fetcher.nix +++ b/mach_nix/nix/deps-db-and-fetcher.nix @@ -21,18 +21,11 @@ let inherit pkgs; }; in -let - pypi_deps_db_src = pkgs.buildEnv { - name = "pypi-deps-db-src"; - paths = [ deps_db_src ]; - }; -in -{ inherit - pypi_deps_db_src +{ + pypi_deps_db_src = deps_db_src; + inherit pypi_fetcher_src - pypi_fetcher_commit pypi_fetcher_sha256 - pypi_fetcher; } diff --git a/mach_nix/nix/lib.nix b/mach_nix/nix/lib.nix index 503ae46..d3c2535 100644 --- a/mach_nix/nix/lib.nix +++ b/mach_nix/nix/lib.nix @@ -242,9 +242,9 @@ rec { }; combine = pname: key: val1: val2: - if isList val2 then val1 ++ val2 - else if isAttrs val2 then val1 // val2 - else if isString val2 then val1 + val2 + if isList val2 then (if ! isNull val1 then val1 else []) ++ val2 + else if isAttrs val2 then (if ! isNull val1 then val1 else {}) // val2 + else if isString val2 then (if ! isNull val1 then val1 else "") + val2 else throw "_.${pname}.${key}.add only accepts list or attrs or string."; fixes_to_overrides = fixes: @@ -258,14 +258,14 @@ rec { mapAttrs (key: val: trace "\napplying fix '${fix}' (${key}) for ${pkg}:${oa.version}\n" ( if isAttrs val && hasAttr "add" val then - combine pkg key oa."${key}" val.add + combine pkg key (oa."${key}" or null) val.add else if isAttrs val && hasAttr "mod" val && isFunction val.mod then - let result = val.mod oa."${key}"; in + let result = val.mod (oa."${key}" or null); in # if the mod function wants more argument, call with more arguments (alternative style) if ! isFunction result then result else - val.mod pySelf oa oa."${key}" + val.mod pySelf oa (oa."${key}" or null) else val ) diff --git a/mach_nix/nix/python.nix b/mach_nix/nix/python.nix index ba249a5..90aa41f 100644 --- a/mach_nix/nix/python.nix +++ b/mach_nix/nix/python.nix @@ -4,7 +4,7 @@ }: with pkgs; let - python = python37; + python = python38; python_deps = (lib.attrValues (import ./python-deps.nix { inherit python; fetchurl = fetchurl; })); in python.withPackages ( ps: python_deps ) diff --git a/mach_nix/nix/withDot.nix b/mach_nix/nix/withDot.nix index 23d7906..ed56d7b 100644 --- a/mach_nix/nix/withDot.nix +++ b/mach_nix/nix/withDot.nix @@ -23,6 +23,5 @@ let in { "pythonWith" = gen "" []; - "shellWith" = gen "env" []; "dockerImageWith" = gen "dockerImage" []; } diff --git a/shell.nix b/shell.nix index 42a69f9..f572ae4 100644 --- a/shell.nix +++ b/shell.nix @@ -4,7 +4,7 @@ }: with pkgs; let - python = python37; + python = python38; machnixDeps = (lib.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = fetchurl; })); in mkShell { diff --git a/tests/test_flakes.nix b/tests/test_flakes.nix index 55f1dab..ca51f06 100644 --- a/tests/test_flakes.nix +++ b/tests/test_flakes.nix @@ -5,7 +5,6 @@ with builtins; let pyEnv = (builtins.getFlake (toString ../.)).packages.x86_64-linux.gen.python.requests; - pyEnvShell = (builtins.getFlake (toString ../.)).packages.x86_64-linux.gen.shell.requests; pyEnvDockerImage = (builtins.getFlake (toString ../.)).packages.x86_64-linux.gen.docker.requests; in (map (p: @@ -15,7 +14,7 @@ in throw "Error" ) [pyEnv pyEnvDockerImage]) ++ [ - (if ! pyEnvShell ? _passthru.python.pkgs.requests then + (if ! pyEnv ? _passthru.python.pkgs.requests then throw "Error with shell" else mach-nix.nixpkgs.hello)