yogsototh/mach-nix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into conda-beta

Browse source
This commit is contained in:
DavHau 2021-05-23 16:22:02 +07:00
commit c455ed8c56
15 changed files with 88 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
Changelog.md
View file

@ -1,3 +1,34 @@
# 3.3.0 (22 May 2021)
bugfixes, improvements
### Changes
- The flakes cmdline api has been changed. New usage:
```
nix (build|shell) mach-nix#gen.(python|docker).package1.package2...
```
(Despite this change being backward incompatible, I did not bump the major version since everything flakes related should be considered experimental anyways)
### Improvements
- Mach-nix (used via flakes) will now throw an error if the selected nixpkgs version is newer than the dependency DB since this can cause conflicts in the resulting environment.
- When used via flakes, it was impossible to select the python version because the import function is not used anymore. Now `python` can be passed to `mkPython` alternatively.
- For the flakes cmdline api, collisions are now ignored by default
- The simplified override interface did not deal well with non-existent values.
- Now the `.add` directive automatically assumes an empty list/set/string when the attribute to be extended doesn't exist.
- Now the `.mod` directive will pass `null` to the given function if the attribute to modify doesn't exist instead.
### Fixes
- Generating an environment with a package named `overrides` failed due to a variable name collision in the resulting nix expression.
- When used via flakes, the pypiData was downloaded twice, because the legacy code path for fetching was still used instead of the flakes input.
- `nix flake show mach-nix` failed because it required IFD for foreign platforms.
- For environments generated via `mach-nix env ...` the `python` command referred to the wrong interpreter.
- When checking wheels for compatibility, the minor version for python was not respected which could lead to invalid environments.
- Some python modules in nixpkgs propagate unnecessary dependencies which could lead to collisions in the final environment. Now mach-nix recursively removes all python dependencies which are not strictly required.
### Package Fixes
- cryptography: remove rust related hook when version < 3.4
# 3.2.0 (11 Mar 2021)
bugfixes, ignoreCollisions

1
default.nix
View file

@ -110,7 +110,6 @@ rec {
# expose dot interface for flakes cmdline
"with" = pythonWith;
pythonWith = (withDot (__mkPython "'.pythonWith'")).pythonWith;
shellWith = (withDot (__mkPython "'.shellWith'")).shellWith;
dockerImageWith = (withDot (__mkPython "'.dockerImageWith'")).dockerImageWith;
# expose mach-nix' nixpkgs

4
flake.nix
View file

@ -19,7 +19,7 @@
true
else
false;
usageGen = "usage: nix (build|develop) mach-nix#gen.(python|shell|docker).package1.package2...";
usageGen = "usage: nix (build|shell) mach-nix#gen.(python|docker).package1.package2...";
in
(flake-utils.lib.eachDefaultSystem (system:
let
@ -49,11 +49,9 @@
src = throw usageGen;
passthru = {
python = mach-nix-default.pythonWith;
shell = mach-nix-default.shellWith;
docker = mach-nix-default.dockerImageWith;
inherit (mach-nix-default)
pythonWith
shellWith
dockerImageWith;
};
};

2
interpreter.nix
View file

@ -1,7 +1,7 @@
# python interpreter for dev environment
let
pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; };
python = pkgs.python37;
python = pkgs.python38;
deps = (pkgs.lib.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = pkgs.fetchurl; }));
in
python.withPackages (ps: deps ++ [

8
mach_nix/data/providers.py
View file

@ -199,10 +199,10 @@ class CombinedDependencyProvider(DependencyProviderBase):
f"Consider adding them via 'providers='"
else:
error_text += \
f"\nIf the package's initial release date predates the release date of mach-nix, " \
f"either upgrade mach-nix itself or manually specify 'pypi_deps_db_commit' and\n" \
f"'pypi_deps_db_sha256 for a newer commit of https://github.com/DavHau/pypi-deps-db/commits/master\n" \
f"If it still doesn't work, there was probably a problem while crawling pypi.\n" \
f"\nThe required package might just not yet be part of the dependency DB currently used.\n" \
f"The DB can be updated by specifying 'pypiDataRev' when importing mach-nix.\n" \
f"For examples see: https://github.com/DavHau/mach-nix/blob/master/examples.md\n" \
f"If it still doesn't work, there might have bene an error while building the DB.\n" \
f"Please open an issue at: https://github.com/DavHau/mach-nix/issues/new\n"
print(error_text, file=sys.stderr)
exit(1)

48
mach_nix/fixes.nix
View file

@ -20,28 +20,36 @@ let
result = parse('${ver1}') ${op} parse('${ver2}')
print(json.dumps(result))
'';
# prevent the normal fitler from being used
filter = raise "Error: only use filterSafe (doesn't crash on null input)";
# filter function that doesn't crash on 'objects = null'
# This is important because null will be returned whenever the attribute to modify doesn't exist
filterSafe = func: objects: if isNull objects then [] else builtins.filter func objects;
in
### Put Fixes here
rec {
### FORMAT ############################################################################
# #
# package-to-fix = { #
# name-of-the-fix = { #
# # optionally limit the fix to a condtion #
# _cond = {prov, ver, ... }: some boolean expression; #
# #
# # define overrides #
# key-to-override = ...; # replace #
# key-to-override.add = ...; # append #
# key-to-override.mod = oldVal: ...; # modify #
# key-to-override.mod = pySelf: oldAttrs: oldVal: ...; # modify (more args) #
# }; #
# }; #
# #
#########################################################################################
### FORMAT ##################################################################################
# #
# package-to-fix = { #
# name-of-the-fix = { #
# # optionally limit the fix to a condtion #
# _cond = {prov, ver, ... }: some boolean expression; #
# #
# # define overrides #
# key-to-override = ...; # replace #
# key-to-override.add = ...; # append (list/attrs/string) #
# key-to-override.mod = oldVal: ...; # modify #
# key-to-override.mod = pySelf: oldAttrs: oldVal: ...; # modify (accessing all pkgs) #
# }; #
# }; #
# #
###############################################################################################
### _cond ####################################
# possible arguments: #
@ -57,7 +65,7 @@ rec {
cryptography.no-rust-build = {
_cond = { prov, ver, ... }: prov == "sdist" && comp_ver ver "<" "3.4";
nativeBuildInputs.mod = old: filter (inp: (inp.name or "") != "cargo-setup-hook.sh") old;
nativeBuildInputs.mod = old: filterSafe (inp: (inp.name or "") != "cargo-setup-hook.sh") old;
};
# remove if merged: https://github.com/NixOS/nixpkgs/pull/114384
@ -83,7 +91,7 @@ rec {
pip.remove-reproducible-patch = {
_cond = { prov, ver, ... }: prov == "sdist" && comp_ver ver "<" "20.0";
patches.mod = oldPatches: filter (patch: ! hasSuffix "reproducible.patch" patch) oldPatches;
patches.mod = oldPatches: filterSafe (patch: ! hasSuffix "reproducible.patch" patch) oldPatches;
};
pyqt5 = {
@ -91,7 +99,7 @@ rec {
# fix mach-nix induced problem: mach-nix removes all previous python inputs from propagatedBuildInputs
_cond = {prov, ... }: prov == "nixpkgs";
propagatedBuildInputs.mod = pySelf: oldAttrs: oldVal:
(filter (p: p.pname != "pyqt5-sip") oldVal) ++ [ pySelf.sip pySelf.dbus-python ];
(filterSafe (p: p.pname != "pyqt5-sip") oldVal) ++ [ pySelf.sip pySelf.dbus-python ];
};
fix-wheel-inputs = {
_cond = {prov, ... }: prov == "wheel";
@ -106,7 +114,7 @@ rec {
# https://github.com/rpy2/rpy2/commit/39e1cb6fca0d4107f1078727d8670c422e3c6f7f
prov == "sdist"
&& comp_ver ver ">=" "3.2.6";
patches.mod = oldPatches: filter (p: ! hasSuffix "pandas-1.x.patch" p) oldPatches;
patches.mod = oldPatches: filterSafe (p: ! hasSuffix "pandas-1.x.patch" p) oldPatches;
};
tensorflow.rm-tensorboard = {

6
mach_nix/flake.lock
View file

@ -34,11 +34,11 @@
"pypi-deps-db": {
"flake": false,
"locked": {
"lastModified": 1620853232,
"narHash": "sha256-SeePgys1mlGUrN/YhCRoo8nApRx5H+KKjyKM1zQ+yAE=",
"lastModified": 1621672288,
"narHash": "sha256-hUOZnSr1ZaTCOUlC8k946RyRjZsfN2w/9jzic0UKj58=",
"owner": "DavHau",
"repo": "pypi-deps-db",
"rev": "15175225c8295e08e5b95b2965892820a64e9746",
"rev": "19dc04a5c7412431a58a07ceecfe7b3ed3c164a5",
"type": "github"
},
"original": {

8
mach_nix/generate.py
View file

@ -90,8 +90,12 @@ def handle_resolution_impossible(exc: ResolutionImpossible, reqs_str, providers_
f"Python: {py_ver_str}\n"
f"Cause: {exc.__context__}\n"
f"The requirements which caused the error:"
f"{causes_str}\n",
file=sys.stderr)
f"{causes_str}\n"
f"\nThe given requirements might contain package versions which are not yet part of the dependency DB\n"
f"currently used. The DB can be updated by specifying 'pypiDataRev' when importing mach-nix.\n"
f"For examples see: https://github.com/DavHau/mach-nix/blob/master/examples.md\n",
file=sys.stderr
)
def main():

4
mach_nix/nix/compileOverrides.nix
View file

@ -44,8 +44,8 @@ let
nixpkgs_json = import ./nixpkgs-json.nix {
inherit overrides pkgs python;
};
builder_python = pkgs.pkgsBuildHost.python37.withPackages(ps:
(pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python37; fetchurl = pkgs.fetchurl; }))
builder_python = pkgs.pkgsBuildHost.python38.withPackages(ps:
(pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python38; fetchurl = pkgs.fetchurl; }))
);
src = ./../../.;

13
mach_nix/nix/deps-db-and-fetcher.nix
View file

@ -21,18 +21,11 @@ let
inherit pkgs;
};
in
let
pypi_deps_db_src = pkgs.buildEnv {
name = "pypi-deps-db-src";
paths = [ deps_db_src ];
};
in
{ inherit
pypi_deps_db_src
{
pypi_deps_db_src = deps_db_src;
inherit
pypi_fetcher_src
pypi_fetcher_commit
pypi_fetcher_sha256
pypi_fetcher;
}

12
mach_nix/nix/lib.nix
View file

@ -242,9 +242,9 @@ rec {
};
combine = pname: key: val1: val2:
if isList val2 then val1 ++ val2
else if isAttrs val2 then val1 // val2
else if isString val2 then val1 + val2
if isList val2 then (if ! isNull val1 then val1 else []) ++ val2
else if isAttrs val2 then (if ! isNull val1 then val1 else {}) // val2
else if isString val2 then (if ! isNull val1 then val1 else "") + val2
else throw "_.${pname}.${key}.add only accepts list or attrs or string.";
fixes_to_overrides = fixes:
@ -258,14 +258,14 @@ rec {
mapAttrs (key: val:
trace "\napplying fix '${fix}' (${key}) for ${pkg}:${oa.version}\n" (
if isAttrs val && hasAttr "add" val then
combine pkg key oa."${key}" val.add
combine pkg key (oa."${key}" or null) val.add
else if isAttrs val && hasAttr "mod" val && isFunction val.mod then
let result = val.mod oa."${key}"; in
let result = val.mod (oa."${key}" or null); in
# if the mod function wants more argument, call with more arguments (alternative style)
if ! isFunction result then
result
else
val.mod pySelf oa oa."${key}"
val.mod pySelf oa (oa."${key}" or null)
else
val
)

2
mach_nix/nix/python.nix
View file

@ -4,7 +4,7 @@
}:
with pkgs;
let
python = python37;
python = python38;
python_deps = (lib.attrValues (import ./python-deps.nix { inherit python; fetchurl = fetchurl; }));
in
python.withPackages ( ps: python_deps )

1
mach_nix/nix/withDot.nix
View file

@ -23,6 +23,5 @@ let
in
{
"pythonWith" = gen "" [];
"shellWith" = gen "env" [];
"dockerImageWith" = gen "dockerImage" [];
}

2
shell.nix
View file

@ -4,7 +4,7 @@
}:
with pkgs;
let
python = python37;
python = python38;
machnixDeps = (lib.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = fetchurl; }));
in
mkShell {

3
tests/test_flakes.nix
View file

@ -5,7 +5,6 @@
with builtins;
let
pyEnv = (builtins.getFlake (toString ../.)).packages.x86_64-linux.gen.python.requests;
pyEnvShell = (builtins.getFlake (toString ../.)).packages.x86_64-linux.gen.shell.requests;
pyEnvDockerImage = (builtins.getFlake (toString ../.)).packages.x86_64-linux.gen.docker.requests;
in
(map (p:
@ -15,7 +14,7 @@ in
throw "Error"
) [pyEnv pyEnvDockerImage])
++ [
(if ! pyEnvShell ? _passthru.python.pkgs.requests then
(if ! pyEnv ? _passthru.python.pkgs.requests then
throw "Error with shell"
else
mach-nix.nixpkgs.hello)