From b230865660581061e266155d814b1385a8df61b6 Mon Sep 17 00:00:00 2001 From: DavHau Date: Mon, 24 Aug 2020 18:20:29 +0700 Subject: [PATCH] automatically extract requirements in buildPythonPackage --- default.nix | 38 +++++++++++++++++-- .../src => lib}/extractor/default.nix | 11 ++++++ .../src => lib}/extractor/distutils.patch | 0 .../src => lib}/extractor/extractor.nix | 0 .../src => lib}/extractor/fast-extractor.nix | 0 .../src => lib}/extractor/setuptools.patch | 0 pypi-crawlers/nix/crawler/configuration.nix | 4 +- pypi-crawlers/src/crawl_sdist_deps.py | 4 +- 8 files changed, 51 insertions(+), 6 deletions(-) rename {pypi-crawlers/src => lib}/extractor/default.nix (92%) rename {pypi-crawlers/src => lib}/extractor/distutils.patch (100%) rename {pypi-crawlers/src => lib}/extractor/extractor.nix (100%) rename {pypi-crawlers/src => lib}/extractor/fast-extractor.nix (100%) rename {pypi-crawlers/src => lib}/extractor/setuptools.patch (100%) diff --git a/default.nix b/default.nix index 70bc43b..986d960 100644 --- a/default.nix +++ b/default.nix @@ -48,8 +48,10 @@ rec { buildPythonApplication = _buildPython "buildPythonApplication"; _buildPython = func: args@{ - requirements, # content from a requirements.txt file + requirements ? null, # content from a requirements.txt file disable_checks ? true, # Disable tests wherever possible to decrease build time. + extras ? [], + doCheck ? ! disable_checks, overrides_pre ? [], # list of pythonOverrides to apply before the machnix overrides overrides_post ? [], # list of pythonOverrides to apply after the machnix overrides pkgs ? nixpkgs, # pass custom nixpkgs. @@ -61,22 +63,50 @@ rec { ... }: let + # Extract dependencies automatically if requirements is unset + reqs = + with builtins; + if requirements == null then + if builtins.hasAttr "format" args && args.format != "setuptools" then + throw "Automatic dependency extraction is only available for 'setuptools' format." + " Please specify requirements=" + else + let + file = (builtins.readFile "${(import ./lib/extractor).extract_from_src { + py = python; + src = args.src; + }}/python.json"); + data = fromJSON file; + setup_requires = if hasAttr "setup_requires" data then data.setup_requires else []; + install_requires = if hasAttr "install_requires" data then data.install_requires else []; + extras_require = + if hasAttr "install_requires" data then + nixpkgs.lib.flatten (map (extra: data.extras_require."${extra}") extras) + else []; + concat = s1: s2: s1 + "\n" + s2; + all_reqs = builtins.foldl' concat "" (setup_requires ++ install_requires ++ extras_require); + in + trace "${all_reqs}\n${if doCheck then "check: true" else "check: false"}" all_reqs + else + requirements; py = python.override { packageOverrides = mergeOverrides overrides_pre; }; result = machNix { - inherit requirements disable_checks providers pypi_deps_db_commit pypi_deps_db_sha256 _provider_defaults; + inherit disable_checks providers pypi_deps_db_commit pypi_deps_db_sha256 _provider_defaults; overrides = overrides_pre; python = py; + requirements = reqs; }; py_final = python.override { packageOverrides = mergeOverrides ( overrides_pre ++ [ result.overrides ] ++ overrides_post );}; pass_args = removeAttrs args (builtins.attrNames ({ - inherit requirements disable_checks overrides_pre overrides_post pkgs providers - pypi_deps_db_commit pypi_deps_db_sha256 python _provider_defaults; + inherit disable_checks overrides_pre overrides_post pkgs providers + requirements pypi_deps_db_commit pypi_deps_db_sha256 python _provider_defaults; })); in py_final.pkgs."${func}" ( pass_args // { propagatedBuildInputs = result.select_pkgs py_final.pkgs; + inherit doCheck; }); diff --git a/pypi-crawlers/src/extractor/default.nix b/lib/extractor/default.nix similarity index 92% rename from pypi-crawlers/src/extractor/default.nix rename to lib/extractor/default.nix index cdfe1f3..98f2f6b 100644 --- a/pypi-crawlers/src/extractor/default.nix +++ b/lib/extractor/default.nix @@ -102,6 +102,11 @@ let echo "python38" out_file=$out/python38.json ${py38}/bin/python -c "${setuptools_shim}" install &> $out/python38.log || true ''; + script_single = py: '' + mkdir $out + echo "extracting dependencies" + out_file=$out/python.json ${py}/bin/python -c "${setuptools_shim}" install &> $out/python.log || true + ''; base_derivation = with pkgs; { buildInputs = [ unzip pkg-config pipenv ]; phases = ["unpackPhase" "installPhase"]; @@ -117,6 +122,12 @@ rec { all = { inherit py27 py35 py36 py37 py38; }; inherit machnix_source; example = extractor {pkg = "requests"; version = "2.22.0";}; + extract_from_src = {py, src}: + stdenv.mkDerivation ( base_derivation // { + inherit src; + name = "package-requirements"; + installPhase = script_single (mkPy py); + }); extractor = {pkg, version}: stdenv.mkDerivation ({ name = "${pkg}-${version}-requirements"; diff --git a/pypi-crawlers/src/extractor/distutils.patch b/lib/extractor/distutils.patch similarity index 100% rename from pypi-crawlers/src/extractor/distutils.patch rename to lib/extractor/distutils.patch diff --git a/pypi-crawlers/src/extractor/extractor.nix b/lib/extractor/extractor.nix similarity index 100% rename from pypi-crawlers/src/extractor/extractor.nix rename to lib/extractor/extractor.nix diff --git a/pypi-crawlers/src/extractor/fast-extractor.nix b/lib/extractor/fast-extractor.nix similarity index 100% rename from pypi-crawlers/src/extractor/fast-extractor.nix rename to lib/extractor/fast-extractor.nix diff --git a/pypi-crawlers/src/extractor/setuptools.patch b/lib/extractor/setuptools.patch similarity index 100% rename from pypi-crawlers/src/extractor/setuptools.patch rename to lib/extractor/setuptools.patch diff --git a/pypi-crawlers/nix/crawler/configuration.nix b/pypi-crawlers/nix/crawler/configuration.nix index 8f712eb..b51617e 100644 --- a/pypi-crawlers/nix/crawler/configuration.nix +++ b/pypi-crawlers/nix/crawler/configuration.nix @@ -5,7 +5,8 @@ let src = "${../../src}"; nixpkgs_src = (import ../../nix/nixpkgs-src.nix).stable; db_host = "10.147.19.69"; - extractor = import ../../src/extractor; + extractor_dir = "${../../../lib/extractor}"; + extractor = import extractor_dir; branch = "master"; enable = true; serviceConfig = { @@ -134,6 +135,7 @@ in EMAIL = "hsngrmpf+pypidepscrawler@gmail.com"; CLEANUP = "y"; pypi_fetcher = "/home/${user}/nix-pypi-fetcher"; + inherit extractor_dir; }; in { diff --git a/pypi-crawlers/src/crawl_sdist_deps.py b/pypi-crawlers/src/crawl_sdist_deps.py index 5ba369f..012a9a5 100644 --- a/pypi-crawlers/src/crawl_sdist_deps.py +++ b/pypi-crawlers/src/crawl_sdist_deps.py @@ -35,7 +35,9 @@ class JobResult: def extractor_cmd(pkg_name, pkg_ver, out='./result', url=None, sha256=None, substitutes=True, store=None) -> List[str]: - extractor_dir = os.path.dirname(os.path.abspath(__file__)) + '/extractor/' + extractor_dir = os.environ.get("extractor_dir") + if not extractor_dir: + raise Exception("Set env variable 'extractor_dir'") base_args = [ "--arg", "pkg", f'"{pkg_name}"', "--arg", "version", f'"{pkg_ver}"',