mach-nix/mach_nix/generate.py

95 lines
3.2 KiB
Python
Raw Normal View History

import json
2020-04-22 09:28:58 +00:00
import os
import sys
from os.path import dirname
from pprint import pformat
from typing import List
2020-04-22 09:28:58 +00:00
from resolvelib import ResolutionImpossible
from resolvelib.resolvers import RequirementInformation
import mach_nix
2020-08-08 09:59:55 +00:00
from mach_nix.data.nixpkgs import NixpkgsIndex
from mach_nix.data.providers import CombinedDependencyProvider, ProviderSettings
from mach_nix.generators.overides_generator import OverridesGenerator
from mach_nix.requirements import parse_reqs, filter_reqs_by_eval_marker, context
2020-04-22 09:28:58 +00:00
from mach_nix.resolver.resolvelib_resolver import ResolvelibResolver
from mach_nix.versions import PyVer
def load_env(name, *args, **kwargs):
var = os.environ.get(name, *args, **kwargs)
if var is None:
print(f'Error: env variable "{name}" must not be empty', file=sys.stderr)
exit(1)
return var.strip()
def main():
providers_json = load_env('providers')
2020-04-22 09:28:58 +00:00
disable_checks = load_env('disable_checks')
nixpkgs_json = load_env('nixpkgs_json')
out_file = load_env('out_file')
provider_settings = ProviderSettings(providers_json)
2020-04-22 09:28:58 +00:00
py_ver_str = load_env('py_ver_str')
pypi_deps_db_src = load_env('pypi_deps_db_src')
2020-04-22 09:28:58 +00:00
pypi_fetcher_commit = load_env('pypi_fetcher_commit')
pypi_fetcher_sha256 = load_env('pypi_fetcher_sha256')
2020-04-22 09:28:58 +00:00
requirements = load_env('requirements')
2020-08-07 17:29:13 +00:00
platform, system = load_env('system').split('-')
2020-04-22 09:28:58 +00:00
py_ver = PyVer(py_ver_str)
2020-08-08 09:59:55 +00:00
nixpkgs = NixpkgsIndex(nixpkgs_json)
deps_provider = CombinedDependencyProvider(
nixpkgs=nixpkgs,
provider_settings=provider_settings,
pypi_deps_db_src=pypi_deps_db_src,
2020-08-07 17:29:13 +00:00
py_ver=py_ver,
platform=platform,
system=system
)
generator = OverridesGenerator(
2020-04-22 09:28:58 +00:00
py_ver,
nixpkgs,
pypi_fetcher_commit,
pypi_fetcher_sha256,
2020-04-22 09:28:58 +00:00
disable_checks,
ResolvelibResolver(nixpkgs, deps_provider),
2020-04-22 09:28:58 +00:00
)
version 2.2.0 (#60) # 2.2.0 (09 Aug 2020) Improved success rate, MacOS support, bugfixes, optimizations ### Features - Improved selection of wheel releases. MacOS is now supported and architectures besides x86_64 should be handled correctly. - Whenever mach-nix resolves dependencies, a visualization of the resulting dependency tree is printed on the terminal. - The dependency DB is now accessed through a caching layer which reduces the resolver's CPU time significantly for larger environments. - The python platform context is now generated from the nix build environment variable `system`. This should decrease the chance of impurities during dependency resolution. ### Fixes - The requires_python attribute of wheels was not respected. This lead to failing builds especially for older python versions. Now `requires_python` is part of the dependency graph and affects resolution. - Detecting the correct package name for python packages in nixpkgs often failed since the attribute names don't follow a fixed schema. This lead to a handful of different errors in different situations. Now the package names are extracted from the pypi `url` inside the `src` attribute which is much more reliable. For packages which are not fetched from pypi, the `pname` attribute is used as fallback. - Fixed bug which lead to the error `attribute 'sdist' missing` if a package from the nixpkgs provider was used which doesn't publish it's source on pypi. (For example `tensorflow`) ### Other Changes - Mach-nix now uses a revision of the nixpkgs-unstable branch instead of nixos-20.03 as base fo the tool and the nixpkgs provider. - Updated revision of the dependency DB
2020-08-09 13:24:12 +00:00
reqs = filter_reqs_by_eval_marker(parse_reqs(requirements), context(py_ver, platform, system))
try:
expr = generator.generate(reqs)
except ResolutionImpossible as e:
handle_resolution_impossible(e, requirements, providers_json, py_ver_str)
exit(1)
else:
with open(out_file, 'w') as f:
f.write(expr)
def handle_resolution_impossible(exc: ResolutionImpossible, reqs_str, providers_json, py_ver_str):
causes: List[RequirementInformation] = exc.causes
causes_str = ''
for ri in causes:
causes_str += f"\n {ri.requirement}"
if ri.parent:
causes_str += \
f" - parent: {ri.parent.name}{ri.parent.extras if ri.parent.extras else ''}:{ri.parent.ver}"
nl = '\n'
print(
f"\nSome requirements could not be resolved.\n"
f"Top level requirements: \n {' '.join(l for l in reqs_str.splitlines())}\n"
f"Providers:\n {f'{nl} '.join(pformat(json.loads(providers_json)).splitlines())}\n"
f"Mach-nix version: {open(dirname(mach_nix.__file__) + '/VERSION').read().strip()}\n"
f"Python: {py_ver_str}\n"
f"Cause: {exc.__context__}\n"
f"The requirements which caused the error:"
f"{causes_str}\n",
file=sys.stderr)
2020-04-22 09:28:58 +00:00
if __name__ == "__main__":
main()