Release 3.0.0 (#159)
* fix non-python packages passed via packagesExtra * refactor examples and readme for 3.0.0 * add changelog for 3.0.0 * bump version to 3.0.0 and update nixpkgs and pypi-deps-db dependencies
This commit is contained in:
DavHau
GitHub
parent
72157591ff
commit
2f9b022d91
9 changed files with 159 additions and 44 deletions
76
Changelog.md
76
Changelog.md
|
|
@ -1,3 +1,79 @@
|
|||
# 3.0.0 (14 Oct 2020)
|
||||
flakes pypi gateway, R support, new output formats, more packages for python 3.5/3.6, improved providers nixpkgs/wheel
|
||||
|
||||
### IMPORTANT NOTICE
|
||||
The UI has been reworked. It is backward compatible with a few exceptions. Most importantly, when importing mach-nix, an attribute set must be passed. It can be empty. Example:
|
||||
```nix
|
||||
let
|
||||
mach-nix = import (builtins.fetchGit {
|
||||
url = "https://github.com/DavHau/mach-nix/";
|
||||
ref = "refs/tags/3.0.0";
|
||||
}) {
|
||||
# optionally bring your own nixpkgs
|
||||
# pkgs = import <nixpkgs> {};
|
||||
|
||||
# or specify the python version
|
||||
# python = "python38";
|
||||
};
|
||||
in
|
||||
...
|
||||
```
|
||||
|
||||
### Features
|
||||
- Flakes gateway to pypi. Get a shell with arbitrary python packages. Example:
|
||||
```shell
|
||||
nix develop github:davhau/mach-nix#shellWith.requests.tensorflow.aiohttp
|
||||
```
|
||||
- New output formats:
|
||||
* **mkDockerImage** -> produces layered docker image containing a python environment
|
||||
* **mkNixpkgs** -> returns nixpkgs which is conform to the given requirements
|
||||
* **mkOverlay** -> returns an overlay function to make nixpkgs conform to the given requirements
|
||||
* **mkPythonOverrides** -> produces pythonOverrides to make python conform to the given requirements.
|
||||
|
||||
- New functions **fetchPypiSdist** and **fetchPypiWheel**. Example:
|
||||
```
|
||||
mach-nix.buildPythonPackge {
|
||||
src = fetchPypiSdist "requests" "2.24.0"
|
||||
};
|
||||
```
|
||||
|
||||
- When using the mach-nix cmdline tool, the nixpkgs channel can now be picked via:
|
||||
```
|
||||
mach-nix env ./env -r requirements.txt --nixpkgs nixos-20.09
|
||||
```
|
||||
|
||||
- R support (experimental): R packages can be passed via `packagesExtra`. Mach-nix will setup rpy2 accordingly. See [usage example](https://github.com/DavHau/mach-nix/blob/master/examples.md#r-and-python).
|
||||
|
||||
- Non-python packages can be passed via `packagesExtra` to include them into the environment.
|
||||
|
||||
|
||||
### Improvements
|
||||
- rework the logic for inheriting dependencies from nixpkgs
|
||||
- fixes.nix: allow alternative mod function signature with more arguments:
|
||||
`key-to-override.mod = pySelf: oldAttrs: oldVal: ...;`
|
||||
- allow derivations passed as `src` argument to buildPythonPackage
|
||||
- stop inheriting attribute names from nixpkgs, instead use normalized package names
|
||||
- rework the public API of mach-nix (largely downwards compatible)
|
||||
- add example on how to build aarch64 image containing a mach-nix env
|
||||
- tests are now enabled/disabled via global override which is more reliable
|
||||
- raise error if python version of any package in packagesExtra doesn't match to one of the environment
|
||||
|
||||
|
||||
### Fixes
|
||||
- nixpkgs packages with identical versions swallowed
|
||||
- pname/version null in buildPythonPackage
|
||||
- update dependency extractor to use "LANG=C.utf8" (increases available packages for python 3.5 and 3.6)
|
||||
- wheel provider picked wheels incompatible to python version
|
||||
- unwanted python buildInput inheritance when overriding nixpkgs
|
||||
- properly parse setup/install_requires if they are strings instead of lists
|
||||
|
||||
### Package Fixes
|
||||
- rpy2: sdist: remove conflicting patch for versions newer than 3.2.6
|
||||
- pytorch from nixpkgs was not detected as `torch`
|
||||
- pyqt5: fix for providers nixpkgs and wheel
|
||||
- httpx: remove patches
|
||||
|
||||
|
||||
# 2.4.1 (21 Sep 2020)
|
||||
bugfixes
|
||||
|
||||
|
|
|
|||
35
Readme.md
35
Readme.md
|
|
@ -61,18 +61,17 @@ Table of Contents
|
|||
You can either install mach-nix via pip or by using nix in case you already have the nix package manager installed.
|
||||
#### Installing via pip
|
||||
```shell
|
||||
pip install git+git://github.com/DavHau/mach-nix@2.4.1
|
||||
pip install git+git://github.com/DavHau/mach-nix@3.0.0
|
||||
```
|
||||
#### Installing via nix
|
||||
```shell
|
||||
nix-env -if https://github.com/DavHau/mach-nix/tarball/2.4.1 -A mach-nix
|
||||
nix-env -if https://github.com/DavHau/mach-nix/tarball/3.0.0 -A mach-nix
|
||||
```
|
||||
|
||||
---
|
||||
### Build a virtualenv-style python environment from a requirements.txt
|
||||
```bash
|
||||
mach-nix env ./env -r requirements.txt
|
||||
# mach-nix env -r ./requirements.txt -p 3.7 --nixpkgs '{ "rev": "c59ea8b8a0e7f927e7291c14ea6cd1bd3a16ff38", "sha256": "1ak7jqx94fjhc68xh1lh35kh3w3ndbadprrb762qgvcfb8351x8v", }' "./env"
|
||||
```
|
||||
This will generate the python environment into `./env`. To activate it, execute:
|
||||
```bash
|
||||
|
|
@ -99,7 +98,7 @@ You can call mach-nix directly from a nix expression
|
|||
let
|
||||
mach-nix = import (builtins.fetchGit {
|
||||
url = "https://github.com/DavHau/mach-nix/";
|
||||
ref = "refs/tags/2.4.1";
|
||||
ref = "refs/tags/3.0.0";
|
||||
});
|
||||
in
|
||||
mach-nix.mkPython {
|
||||
|
|
@ -113,31 +112,35 @@ mach-nix.mkPython {
|
|||
find more examples under [./examples.md](/examples.md)
|
||||
|
||||
### Advanced
|
||||
Mach-nix can be fine tuned with additional arguments by importing it via `builtins.fetchGit`. Examples can be found in [./examples.md](/examples.md). There are 4 different methods which can be invoked:
|
||||
Mach-nix can be fine tuned with additional arguments. Examples can be found in [./examples.md](/examples.md).
|
||||
|
||||
Functions for building python environments:
|
||||
1. **mkPython** - builds a python environment for a given `requirements.txt`.
|
||||
1. **mkPythonShell** - returns the python environment suitable for nix-shell.
|
||||
1. **mkPythonShell** - builds a python environment suitable for nix-shell.
|
||||
1. **mkDockerImage** - builds a layered docker image containing a python environment.
|
||||
1. **mkNixpkgs** - returns nixpkgs which is conform to the given requirements.
|
||||
1. **mkOverlay** - returns an overlay function to make nixpkgs conform to the given requirements .
|
||||
1. **mkPythonOverrides** - produces pythonOverrides to make python conform to the given requirements.
|
||||
|
||||
Functions for building python packages or applications:
|
||||
1. **buildPythonPackage** - build a single python package from a source code while automatically detecting requirements.
|
||||
1. **buildPythonApplication** - same as **buildPythonPackage**, but package will not be importable by other python packages.
|
||||
|
||||
**buildPythonPackage** and **buildPythonApplication** accept the same arguments like their equally named partners in nixpkgs, plus the arguments of **mkPython**. If name/version/requirements arguments are omitted, mach-nix attempts to detect them automatically. See [./examples.md](/examples.md).
|
||||
|
||||
**mkPython** and **mkPythonShell** take exactly the following arguments:
|
||||
**mkPython** and all other **mk...** functions take exactly the following arguments:
|
||||
|
||||
#### Required Arguments:
|
||||
- **requirements** (string): Text content of a typical `requirements.txt`.
|
||||
|
||||
#### Optional Arguments:
|
||||
- **disable_checks** (bool): Disable tests wherever possible to decrease build time and failures due to nix incompatible tests
|
||||
- **extra_pkgs** (list) Include packages which are not available from pypi. Can contain tarball-URLs, paths, or `mach-nix.buildPythonPackage` calls.
|
||||
- **overrides_pre** (list): list of pythonOverrides to apply before the machnix overrides. Use this to include additional packages which can then be selected inside the `requirements`
|
||||
- **overrides_post** (list): list of pythonOverrides to apply after the machnix overrides. Use this to fixup packages.
|
||||
- **pkgs** (set): pass custom nixpkgs (20.03 or higher is required for wheel support). Base it on `mach-nix.nixpkgs.path` to avoid incompatibilities.
|
||||
- **providers** (set): define provider preferences (see examples below)
|
||||
- **pypi_deps_db_commit** (string): commit hash of a specific version of the dependency graph ([pypi-deps-db](https://github.com/DavHau/pypi-deps-db)).
|
||||
- **pypi_deps_db_sha256** (string): sha256 hash obtained via `nix-prefetch-url --unpack https://github.com/DavHau/pypi-deps-db/tarball/<pypi_deps_db_commit>`
|
||||
- **python** (set): select custom python version. To prevent compatibility issues, only take python packages from the nixpkgs version used by mach-nix. For example: `mach-nix.nixpkgs.python36`
|
||||
- **_provider_defaults** (set): builtin provider defaults. Disable them by passing {}
|
||||
- **packagesExtra** (list) Add extra packages. Can contain tarball-URLs or paths of python source code, packages built via `mach-nix.buildPythonPackage`, or R Packages.
|
||||
- **_** (set): use underscore argument to easily modify arbitrary attributes of packages. For example to add built inputs use `_.{package}.builtInputs.add = [...]`. Or to remove patches use `_.{package}.patches = [...]`.
|
||||
- **overridesPre** (list): (advanced) list of pythonOverrides to apply before the machnix overrides. Use this to include additional packages which can then be selected inside the `requirements`
|
||||
- **overridesPost** (list): (advanced) list of pythonOverrides to apply after the machnix overrides. Use this to fixup packages.
|
||||
- **tests** (bool): Whether to enable tests (default: false)
|
||||
- **_providerDefaults** (set): builtin provider defaults. Disable them by passing {}
|
||||
|
||||
#### Configure Providers
|
||||
**Providers** allow you to configure the origin for your packages on a granular basis.
|
||||
|
|
|
|||
61
examples.md
61
examples.md
|
|
@ -35,15 +35,22 @@ This page contains basic and advanced examples for using mach-nix inside a nix e
|
|||
<!--te-->
|
||||
|
||||
### Import mach-nix
|
||||
(every mach-nix expression should begin like this)
|
||||
every mach-nix expression should begin like this:
|
||||
```nix
|
||||
let
|
||||
mach-nix = import (builtins.fetchGit {
|
||||
url = "https://github.com/DavHau/mach-nix/";
|
||||
ref = "refs/tags/2.4.1";
|
||||
ref = "refs/tags/3.0.0";
|
||||
}) {
|
||||
# optionally bring your own nixpkgs
|
||||
# pkgs = import <nixpkgs> {};
|
||||
|
||||
# optionally specify the python version
|
||||
# python = "python38";
|
||||
|
||||
# optionally update pypi data revision from https://github.com/DavHau/pypi-deps-db
|
||||
# pypiDataRev = "some_revision";
|
||||
# pypiDataSha256 = "some_sha256";
|
||||
};
|
||||
in
|
||||
...
|
||||
|
|
@ -57,20 +64,26 @@ mach-nix.mkPython { # replace with mkPythonShell if shell is wanted
|
|||
}
|
||||
```
|
||||
|
||||
#### Include packages from arbitrary sources.
|
||||
`extra_pkgs` accepts python packages built via `mach-nix.buildPythonPackage`. Alternatively, paths or URLs can be passed which are then automatically wrapped in a `mach-nix.buildPythonPackage` call.
|
||||
#### Include extra packages.
|
||||
`packagesExtra` accepts:
|
||||
- python packages built via `mach-nix.buildPythonPackage`.
|
||||
- R Packages from `nixpkgs.rPackages` (see R example further down)
|
||||
- python package source trees as paths or derivations
|
||||
- URLs pointing to a tarball archives containing a python source tree
|
||||
```nix
|
||||
...
|
||||
mach-nix.mkPython {
|
||||
requirements = builtins.readFile ./requirements.txt;
|
||||
extra_pkgs = [
|
||||
packagesExtra = [
|
||||
"https://github.com/psf/requests/tarball/2a7832b5b06d" # from tarball url
|
||||
./some/local/project # from local path
|
||||
mach-nix.buildPythonPackage { ... }; # from package
|
||||
];
|
||||
}
|
||||
```
|
||||
Alternatively, if requirements are not needed, extra_pkgs can be passed directly to mkPython
|
||||
Alternatively, if requirements are not needed, packagesExtra can be passed directly to mkPython
|
||||
```nix
|
||||
...
|
||||
mach-nix.mkPython [
|
||||
"https://github.com/psf/requests/tarball/2a7832b5b06d" # from tarball url
|
||||
./some/local/project # from local path
|
||||
|
|
@ -79,20 +92,23 @@ mach-nix.mkPython [
|
|||
```
|
||||
|
||||
### buildPythonPackage / buildPythonApplication
|
||||
These functions can be used to manually build individual python modules or applications. Those can either be used directly, or fed as `extra_pkgs` of `mkPython`.
|
||||
These functions can be used to manually build individual python modules or applications. Those can either be used directly, or fed as `packagesExtra` of `mkPython`.
|
||||
Whenever `requirements` are not explicitly specified, they will be extracted automatically from the packages setup.py/setup.cfg. The same goes for the `name` and `version`.
|
||||
#### Build python package from its source code
|
||||
```nix
|
||||
...
|
||||
mach-nix.buildPythonPackage /python-project-path
|
||||
```
|
||||
|
||||
#### buildPythonPackage from GitHub
|
||||
```nix
|
||||
...
|
||||
mach-nix.buildPythonPackage "https://github.com/psf/requests/tarball/2a7832b5b06d"
|
||||
```
|
||||
|
||||
#### buildPythonPackage from GitHub with extras
|
||||
```nix
|
||||
...
|
||||
mach-nix.buildPythonPackage {
|
||||
src = "https://github.com/psf/requests/tarball/2a7832b5b06d";
|
||||
extras = "socks";
|
||||
|
|
@ -100,16 +116,18 @@ mach-nix.buildPythonPackage {
|
|||
```
|
||||
|
||||
#### buildPythonPackage from GitHub and add requirements
|
||||
Use `add_requirements` in case the auto detected requirements are incomplete
|
||||
Use `requirementsExtra` in case the auto detected requirements are incomplete
|
||||
```nix
|
||||
...
|
||||
mach-nix.buildPythonPackage {
|
||||
src = "https://github.com/psf/requests/tarball/2a7832b5b06d";
|
||||
add_requirements = "pytest";
|
||||
requirementsExtra = "pytest";
|
||||
}
|
||||
```
|
||||
|
||||
#### buildPythonPackage from GitHub (reproducible source)
|
||||
```nix
|
||||
...
|
||||
mach-nix.buildPythonPackage {
|
||||
src = builtins.fetchGit{
|
||||
url = "https://github.com/user/projectname";
|
||||
|
|
@ -122,6 +140,7 @@ mach-nix.buildPythonPackage {
|
|||
#### buildPythonPackage from GitHub (manual requirements)
|
||||
Use this if automatic requirements extraction doesn't work at all.
|
||||
```nix
|
||||
...
|
||||
mach-nix.buildPythonPackage {
|
||||
src = "https://github.com/psf/requests/tarball/2a7832b5b06d";
|
||||
requirements = ''
|
||||
|
|
@ -152,6 +171,7 @@ mach-nix.mkPython {
|
|||
### Example: add missing build inputs
|
||||
For example the package web2ldap depends on another python package `ldap0` which fails to build because of missing dependencies.
|
||||
```nix
|
||||
...
|
||||
with mach-nix.nixpkgs;
|
||||
mach-nix.mkPython {
|
||||
|
||||
|
|
@ -168,6 +188,7 @@ mach-nix.mkPython {
|
|||
`imagecodecs` is available via wheel, but if one wants to build it from source, dependencies will be missing since there is no nixpkgs candidate available.
|
||||
poetry2nix luckily maintains overrides for this package. They can be included into the mach-nix build like this.
|
||||
```nix
|
||||
...
|
||||
mach-nix.mkPython rec {
|
||||
|
||||
requirements = ''
|
||||
|
|
@ -181,7 +202,7 @@ mach-nix.mkPython rec {
|
|||
|
||||
# Import overrides from poetry2nix
|
||||
# Caution! Use poetry2nix overrides only in `overrides_post`, not `overrides_pre`.
|
||||
overrides_post = [
|
||||
overridesPost = [
|
||||
(
|
||||
import (builtins.fetchurl {
|
||||
url = "https://raw.githubusercontent.com/nix-community/poetry2nix/1cfaa4084d651d73af137866622e3d0699851008/overrides.nix";
|
||||
|
|
@ -197,6 +218,7 @@ mach-nix.mkPython rec {
|
|||
Tensorflow from pypi does not provide any hardware optimization support. To get a SSE/AVX/FMA enabled version, set the provider for tensorflow to `nixpkgs`.
|
||||
|
||||
```nix
|
||||
...
|
||||
mach-nix.mkPython {
|
||||
|
||||
requirements = ''
|
||||
|
|
@ -213,6 +235,7 @@ This only works if the restrictions in `requirements.txt` allow for the tensorfl
|
|||
### Tensorflow via wheel (newer versions, quicker builds)
|
||||
Install recent tensorflow via wheel
|
||||
```nix
|
||||
...
|
||||
mach-nix.mkPython {
|
||||
|
||||
requirements = ''
|
||||
|
|
@ -228,6 +251,7 @@ mach-nix.mkPython {
|
|||
### Recent PyTorch with nixpkgs dependencies, and custom python
|
||||
Recent pytorch version, Build dependencies from source
|
||||
```nix
|
||||
...
|
||||
mach-nix.mkPython rec {
|
||||
|
||||
requirements = ''
|
||||
|
|
@ -241,9 +265,6 @@ mach-nix.mkPython rec {
|
|||
# allow wheels only for torch
|
||||
torch = "wheel";
|
||||
};
|
||||
|
||||
# Select custom python version (Must be taken from pkgs with the overlay applied)
|
||||
python = mach-nix.nixpkgs.python36;
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -251,11 +272,10 @@ mach-nix.mkPython rec {
|
|||
|
||||
### Starting point for a geospatial environment
|
||||
```nix
|
||||
...
|
||||
let
|
||||
pyEnv = mach-nix.mkPython rec {
|
||||
|
||||
python = "python37";
|
||||
|
||||
requirements = ''
|
||||
jupyterlab
|
||||
geopandas
|
||||
|
|
@ -280,10 +300,11 @@ mkShell rec {
|
|||
```
|
||||
|
||||
## Docker
|
||||
For every python environment a docker image is available via the `dockerImage` attribute of the `mkPython` result
|
||||
Docker images can be built by using `mkDockerImage` instead of `mkPython`. It accepts the same arguments.
|
||||
### JupyterLab Docker Image
|
||||
Assuming the following expression under `./jupyter-docker.nix`:
|
||||
```nix
|
||||
...
|
||||
let
|
||||
image = mach-nix.mkDockerImage {
|
||||
requirements = ''
|
||||
|
|
@ -314,16 +335,17 @@ docker run --rm -it -p 8888:8888 -v $HOME:/mnt jupyterlab
|
|||
|
||||
## R and Python
|
||||
The following is an example for a Python environment mixed with R packages.
|
||||
R packages can be added via `extra_pkgs`.
|
||||
If mach-nix finds R packages inside `extra_pkgs`, it will automatically include `rpy2` and add the selected R packages to its buildInputs.
|
||||
R packages can be added via `packagesExtra`.
|
||||
If mach-nix finds R packages inside `packagesExtra`, it will automatically include `rpy2` and add the selected R packages to its buildInputs.
|
||||
To get a list of available R packages, execute: `echo "builtins.attrNames(import <nixpkgs> {}).rPackages" | nix repl`
|
||||
|
||||
```nix
|
||||
...
|
||||
mach-nix.mkPython {
|
||||
requirements = ''
|
||||
# some python requirements
|
||||
'';
|
||||
extra_pkgs = with mach-nix.rPackages; [
|
||||
packagesExtra = with mach-nix.rPackages; [
|
||||
data_table
|
||||
];
|
||||
}
|
||||
|
|
@ -340,6 +362,7 @@ For the SD-image, create a configuration.nix file which adds the mach-nix tool a
|
|||
let
|
||||
machNix = import (builtins.fetchGit {
|
||||
url = "https://github.com/DavHau/mach-nix/";
|
||||
ref = "refs/tags/put_version_here";
|
||||
}) { inherit pkgs; };
|
||||
|
||||
defaultPythonEnv = machNix.mkPython {
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.4.1
|
||||
3.0.0
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"url": "https://github.com/nixos/nixpkgs",
|
||||
"rev": "502845c3e31ef3de0e424f3fcb09217df2ce6df6",
|
||||
"date": "2020-09-26T09:12:52+02:00",
|
||||
"sha256": "0fcqpsy6y7dgn0y0wgpa56gsg0b0p8avlpjrd79fp4mp9bl18nda",
|
||||
"rev": "41c0f49681009ceb57a65b7cd7d4e6d605df712c",
|
||||
"date": "2020-10-07T08:10:38+02:00",
|
||||
"sha256": "0ahi76lb38fcnbzl40k53yjr5vcc75kqg0ddcw6bbc6ckz0z27kg",
|
||||
"fetchSubmodules": false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"url": "https://github.com/davhau/pypi-deps-db",
|
||||
"rev": "d92b7eb48762f4f6d912003c3277bea1c4d7eb9a",
|
||||
"date": "2020-10-08T05:03:16+00:00",
|
||||
"sha256": "1ig2sg26fn7cvln9y83adiv0145lv4dzicmx589jrm6lpphd58wg",
|
||||
"rev": "d6ca2a67a76f98f168d5c6414f1fbc311b464bdf",
|
||||
"date": "2020-10-13T16:58:13+00:00",
|
||||
"sha256": "1c0zq1hprqnk9g0ymn60c38748k0fqxwv7py7h5bgy7ja7ihsg6p",
|
||||
"fetchSubmodules": false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,9 +116,8 @@ let
|
|||
py_final_with_pkgs = py_final.withPackages (ps: selectPkgs ps);
|
||||
final_env = pkgs.buildEnv {
|
||||
name = "mach-nix-python-env";
|
||||
paths = [
|
||||
paths = extra_pkgs_other ++ [
|
||||
py_final_with_pkgs
|
||||
extra_pkgs_other
|
||||
];
|
||||
};
|
||||
in let
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ map (file: import file { inherit mach-nix; }) [
|
|||
./test_flakes.nix
|
||||
./test_jupyterlab_nixpkgs.nix
|
||||
./test_lazy_usage.nix
|
||||
./test_non_python_extra_pkgs.nix
|
||||
./test_overrides_selectPkgs.nix
|
||||
./test_passthru_select_pypi_pname.nix
|
||||
./test_py38_cp38_wheel.nix
|
||||
|
|
|
|||
13
tests/test_non_python_extra_pkgs.nix
Normal file
13
tests/test_non_python_extra_pkgs.nix
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
mach-nix ? import ../. {},
|
||||
...
|
||||
}:
|
||||
with builtins;
|
||||
mach-nix.mkPython {
|
||||
requirements = ''
|
||||
requests
|
||||
'';
|
||||
packagesExtra = [
|
||||
mach-nix.nixpkgs.hello
|
||||
];
|
||||
}
|
||||
Loading…
Reference in a new issue