mach-nix/mach_nix/nix/compileOverrides.nix

79 lines
2.8 KiB
Nix
Raw Permalink Normal View History

with builtins;
{
requirements, # content from a requirements.txt file
python, # python from nixpkgs as base for overlay
2020-10-07 18:50:34 +00:00
pkgs,
2020-11-17 11:29:44 +00:00
condaChannelsExtra ? {},
2020-10-13 08:46:06 +00:00
tests ? false, # disable tests wherever possible
version 2.2.0 (#60) # 2.2.0 (09 Aug 2020) Improved success rate, MacOS support, bugfixes, optimizations ### Features - Improved selection of wheel releases. MacOS is now supported and architectures besides x86_64 should be handled correctly. - Whenever mach-nix resolves dependencies, a visualization of the resulting dependency tree is printed on the terminal. - The dependency DB is now accessed through a caching layer which reduces the resolver's CPU time significantly for larger environments. - The python platform context is now generated from the nix build environment variable `system`. This should decrease the chance of impurities during dependency resolution. ### Fixes - The requires_python attribute of wheels was not respected. This lead to failing builds especially for older python versions. Now `requires_python` is part of the dependency graph and affects resolution. - Detecting the correct package name for python packages in nixpkgs often failed since the attribute names don't follow a fixed schema. This lead to a handful of different errors in different situations. Now the package names are extracted from the pypi `url` inside the `src` attribute which is much more reliable. For packages which are not fetched from pypi, the `pname` attribute is used as fallback. - Fixed bug which lead to the error `attribute 'sdist' missing` if a package from the nixpkgs provider was used which doesn't publish it's source on pypi. (For example `tensorflow`) ### Other Changes - Mach-nix now uses a revision of the nixpkgs-unstable branch instead of nixos-20.03 as base fo the tool and the nixpkgs provider. - Updated revision of the dependency DB
2020-08-09 13:24:12 +00:00
overrides ? [],
providers ? {}, # re-order to change provider priority or remove providers
pypiData,
condaDataRev ? (builtins.fromJSON (builtins.readFile ./CONDA_CHANNELS.json)).rev,
condaDataSha256 ? (builtins.fromJSON (builtins.readFile ./CONDA_CHANNELS.json)).indexSha256,
cudaVersion ? pkgs.cudatoolkit.version, # max allowed cuda version for conda packages
2021-06-09 06:11:17 +00:00
_providerDefaults ? (import ./lib.nix { inherit (pkgs) lib; inherit pkgs; }).makeProviderDefaults requirements
}:
with pkgs.lib;
let
2020-11-17 11:29:44 +00:00
l = import ./lib.nix { inherit (pkgs) lib; inherit pkgs; };
2020-11-18 18:09:56 +00:00
processedReqs = l.preProcessRequirements requirements;
_requirements = processedReqs.requirements;
__providerDefaults = l.parseProviders _providerDefaults;
_providers =
let
2020-11-18 18:09:56 +00:00
extraChannelProviders =(map (n: "conda/" + n) (attrNames condaChannelsExtra));
extraProviders =
extraChannelProviders
++ filter (p: ! extraChannelProviders ? p) processedReqs.providers;
defaults = recursiveUpdate __providerDefaults {
_default =
2020-11-18 18:09:56 +00:00
__providerDefaults._default
++ (filter (p: ! elem p __providerDefaults._default) extraProviders);
};
in
(l.parseProviders (defaults // providers));
2020-11-17 11:29:44 +00:00
version 2.2.0 (#60) # 2.2.0 (09 Aug 2020) Improved success rate, MacOS support, bugfixes, optimizations ### Features - Improved selection of wheel releases. MacOS is now supported and architectures besides x86_64 should be handled correctly. - Whenever mach-nix resolves dependencies, a visualization of the resulting dependency tree is printed on the terminal. - The dependency DB is now accessed through a caching layer which reduces the resolver's CPU time significantly for larger environments. - The python platform context is now generated from the nix build environment variable `system`. This should decrease the chance of impurities during dependency resolution. ### Fixes - The requires_python attribute of wheels was not respected. This lead to failing builds especially for older python versions. Now `requires_python` is part of the dependency graph and affects resolution. - Detecting the correct package name for python packages in nixpkgs often failed since the attribute names don't follow a fixed schema. This lead to a handful of different errors in different situations. Now the package names are extracted from the pypi `url` inside the `src` attribute which is much more reliable. For packages which are not fetched from pypi, the `pname` attribute is used as fallback. - Fixed bug which lead to the error `attribute 'sdist' missing` if a package from the nixpkgs provider was used which doesn't publish it's source on pypi. (For example `tensorflow`) ### Other Changes - Mach-nix now uses a revision of the nixpkgs-unstable branch instead of nixos-20.03 as base fo the tool and the nixpkgs provider. - Updated revision of the dependency DB
2020-08-09 13:24:12 +00:00
nixpkgs_json = import ./nixpkgs-json.nix {
inherit overrides pkgs python;
};
2022-02-03 16:02:45 +00:00
builder_python = pkgs.pkgsBuildHost.python39.withPackages(ps:
(pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python39; fetchurl = pkgs.fetchurl; }))
);
2020-11-17 11:29:44 +00:00
src = ./../../.;
2020-11-17 11:29:44 +00:00
2020-10-13 08:46:06 +00:00
db_and_fetcher = import ./deps-db-and-fetcher.nix {
inherit pkgs;
deps_db_src = pypiData;
2020-10-13 08:46:06 +00:00
};
2020-11-17 11:29:44 +00:00
providers_json_file = pkgs.writeText "providers" (builtins.toJSON _providers);
mach_nix_file = pkgs.runCommand "mach_nix_file"
{ buildInputs = [ src builder_python db_and_fetcher.pypi_deps_db_src];
2020-11-18 18:09:56 +00:00
inherit nixpkgs_json;
inherit (db_and_fetcher) pypi_deps_db_src pypi_fetcher_commit pypi_fetcher_sha256;
2020-11-17 11:29:44 +00:00
conda_channels_json = (import ./conda-channels.nix {
inherit condaChannelsExtra condaDataRev condaDataSha256 pkgs;
2020-11-17 11:29:44 +00:00
providers = _providers;
}).condaChannelsJson;
2020-10-13 08:46:06 +00:00
disable_checks = ! tests;
2020-11-17 11:29:44 +00:00
providers = providers_json_file;
py_ver_str = python.version;
2020-11-18 18:09:56 +00:00
requirements = _requirements;
}
''
mkdir -p $out/share
export out_file=$out/share/mach_nix_file.nix
export PYTHONPATH=${src}
export MACHNIX_CUDA_VERSION=${cudaVersion}
${builder_python}/bin/python ${src}/mach_nix/generate.py
'';
in
# single file derivation containing $out/share/mach_nix_file.nix
mach_nix_file