yogsototh/mach-nix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WIP: conda

Browse source
This commit is contained in:
DavHau 2020-10-25 12:09:08 +07:00
parent ddddcaa774
commit c302ceaafd
9 changed files with 455 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
mach_nix/data/nixpkgs.py
View file

@ -3,9 +3,10 @@ from collections import UserDict
from dataclasses import dataclass
from typing import List
from packaging.version import Version, parse
from packaging.version import Version
from mach_nix.cache import cached
from mach_nix.versions import parse_ver
@dataclass
@ -48,7 +49,7 @@ class NixpkgsIndex(UserDict):
def get_all_candidates(self, name) -> List[NixpkgsPyPkg]:
result = []
for ver, nix_keys in self.data[name].items():
result += [NixpkgsPyPkg(nix_key, parse(ver)) for nix_key in nix_keys]
result += [NixpkgsPyPkg(nix_key, parse_ver(ver)) for nix_key in nix_keys]
return result
def get_highest_ver(self, pkgs: List[NixpkgsPyPkg]):

118
mach_nix/data/providers.py
View file

@ -4,10 +4,11 @@ import re
import sys
from abc import ABC, abstractmethod
from dataclasses import dataclass
from os.path import abspath, dirname
from typing import List, Tuple, Iterable
import distlib.markers
from packaging.version import Version, parse
from packaging.version import Version, parse, LegacyVersion
from pkg_resources import RequirementParseError
from .nixpkgs import NixpkgsIndex
@ -69,6 +70,8 @@ class DependencyProviderBase(ABC):
self.context = context(py_ver, platform, system)
self.context_wheel = self.context.copy()
self.context_wheel['extra'] = None
self.py_ver = py_ver
self.py_ver_parsed = parse(py_ver.python_full_version())
self.py_ver_digits = py_ver.digits()
self.platform = platform
self.system = system
@ -94,19 +97,15 @@ class DependencyProviderBase(ABC):
def unify_key(self, key: str) -> str:
return key.replace('_', '-').lower()
@abstractmethod
def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo:
"""
returns info about a candidate by it's provider.
This is later needed to identify the origin of a package and how to retrieve it
"""
pass
def get_provider_info(self, pkg_name, pkg_version):
return ProviderInfo(provider=self)
@abstractmethod
@cached()
def get_pkg_reqs(self, pkg_name, pkg_version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
"""
Get all requirements of a candidate for the current platform and the specified extras
returns two lists: install_requires, setup_requires
"""
pass
@ -134,11 +133,13 @@ class CombinedDependencyProvider(DependencyProviderBase):
self.provider_settings = provider_settings
wheel = WheelDependencyProvider(f"{pypi_deps_db_src}/wheel", *args, **kwargs)
sdist = SdistDependencyProvider(f"{pypi_deps_db_src}/sdist", *args, **kwargs)
conda = CondaDependencyProvider(f"{dirname(abspath(__file__))}", *args, **kwargs)
nixpkgs = NixpkgsDependencyProvider(nixpkgs, wheel, sdist, *args, **kwargs)
self._all_providers = {
f"{wheel.name}": wheel,
f"{sdist.name}": sdist,
f"{nixpkgs.name}": nixpkgs,
f"{conda.name}": conda,
}
providers_used = set(provider_settings.default_providers)
for p_list in provider_settings.pkg_providers.values():
@ -226,9 +227,6 @@ class NixpkgsDependencyProvider(DependencyProviderBase):
self.wheel_provider = wheel_provider
self.sdist_provider = sdist_provider
def get_provider_info(self, pkg_name, pkg_version) -> ProviderInfo:
return ProviderInfo(self)
def get_pkg_reqs(self, pkg_name, pkg_version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
name = self.unify_key(pkg_name)
if not self.nixpkgs.exists(name, pkg_version):
@ -422,9 +420,6 @@ class SdistDependencyProvider(DependencyProviderBase):
raise Exception(
f"Something went wrong while trying to find the deviated version for {pkg_name}:{normalized_version}")
def get_provider_info(self, pkg_name, pkg_version):
return ProviderInfo(provider=self)
def get_reqs_for_extras(self, pkg_name, pkg_ver, extras):
name = self.unify_key(pkg_name)
pkg = self._get_candidates(name)[pkg_ver]
@ -469,3 +464,98 @@ class SdistDependencyProvider(DependencyProviderBase):
def _available_versions(self, pkg_name: str) -> Iterable[Version]:
return [ver for ver in self._get_candidates(pkg_name).keys()]
class CondaDependencyProvider(DependencyProviderBase):
ignored_pkgs = (
"ld_impl_linux-64",
#"libffi",
"libgcc-ng",
"libstdcxx-ng",
"ncurses",
#"openssl",
"readline",
"sqlite",
"tk",
"tzdata",
"xz",
"zlib",
"python",
)
def __init__(self, repodata_file, py_ver: PyVer, platform, system, *args, **kwargs):
files = (
f"{repodata_file}/repodata.json",
f"{repodata_file}/noarch.json"
)
self.pkgs = {}
for file in files:
with open(file) as f:
content = json.load(f)
for fname, p in content['packages'].items():
name = p['name']
ver = p['version']
build = p['build']
if name not in self.pkgs:
self.pkgs[name] = {}
if ver not in self.pkgs[name]:
self.pkgs[name][ver] = {}
if build in self.pkgs[name][ver]:
print("WARNING: colliding package")
self.pkgs[name][ver][build] = p
self.pkgs[name][ver][build]['fname'] = fname
super().__init__(py_ver, platform, system, *args, **kwargs)
@property
def name(self):
return "conda"
def get_pkg_reqs(self, pkg_name, pkg_version: Version, extras=None) -> Tuple[List[Requirement], List[Requirement]]:
candidate = self.choose_candidate(pkg_name, pkg_version)
depends = list(filter(lambda d: d.split()[0] not in self.ignored_pkgs, candidate['depends']))
print(f"candidate {pkg_name}:{pkg_version} depends on {list(parse_reqs(depends))}")
return list(parse_reqs(depends)), []
@cached()
def _available_versions(self, pkg_name: str) -> Iterable[Version]:
versions = []
for ver in self.pkgs[pkg_name].keys():
versions += [parse(b['version']) for b in self.compatible_builds(pkg_name, parse(ver))]
return versions
def deviated_version(self, pkg_name, normalized_version: Version):
for builds in self.pkgs[pkg_name].values():
for p in builds.values():
if parse(p['version']) == normalized_version:
return p['version']
raise Exception(f"Cannot find deviated version for {pkg_name}:{normalized_version}")
def python_ok(self, build):
for dep in build['depends']:
if dep.startswith("python "):
req = next(iter(parse_reqs([dep])))
if not filter_versions([self.py_ver_parsed], req.specs):
return False
return True
@cached()
def compatible_builds(self, pkg_name, pkg_version):
compatible = []
for build in self.pkgs[pkg_name][self.deviated_version(pkg_name, pkg_version)].values():
# continue if python incompatible
if not self.python_ok(build):
continue
# python is compatible
compatible.append(build)
return compatible
def choose_candidate(self, pkg_name, pkg_version: Version):
assert(isinstance(pkg_version, Version) or isinstance(pkg_version, LegacyVersion))
candidate = self.compatible_builds(pkg_name, pkg_version)[0]
print(f"chosen candidate {pkg_name}{candidate['version']} for {pkg_version}")
return candidate
def pkg_url_src(self, pkg_name, pkg_version):
pkg = self.choose_candidate(pkg_name, pkg_version)
url = f"https://anaconda.org/anaconda/{pkg['name']}/{pkg['version']}/download/{pkg['subdir']}/{pkg['fname']}"
return url, pkg['sha256']

40
mach_nix/generators/overides_generator.py
View file

@ -1,7 +1,8 @@
import json
from typing import Dict, List
from mach_nix.data.providers import WheelDependencyProvider, SdistDependencyProvider, NixpkgsDependencyProvider
from mach_nix.data.providers import WheelDependencyProvider, SdistDependencyProvider, NixpkgsDependencyProvider, \
CondaDependencyProvider
from mach_nix.data.nixpkgs import NixpkgsIndex
from mach_nix.generators import ExpressionGenerator
from mach_nix.resolver import ResolvedPkg
@ -160,7 +161,7 @@ class OverridesGenerator(ExpressionGenerator):
});\n"""
return unindent(out, 8)
def _gen_builPythonPackage(self, name, ver, circular_deps, nix_name, build_inputs_str, prop_build_inputs_str):
def _gen_buildPythonPackage(self, name, ver, circular_deps, nix_name, build_inputs_str, prop_build_inputs_str):
out = f"""
"{name}" = python-self.buildPythonPackage {{
pname = "{name}";
@ -206,6 +207,28 @@ class OverridesGenerator(ExpressionGenerator):
};\n"""
return unindent(out, 8)
def _gen_conda_buildPythonPackage(
self, name, ver, circular_deps, nix_name, prop_build_inputs_str, src_url, src_sha256):
out = f"""
"{name}" = python-self.buildPythonPackage {{
pname = "{name}";
version = "{ver}";
src = builtins.fetchurl {{
url = "{src_url}";
sha256 = "{src_sha256}";
}};
format = "condabin";
passthru = (get_passthru python-super "{name}" "{nix_name}") // {{ provider = "sdist"; }};"""
if circular_deps:
out += f"""
pipInstallFlags = "--no-dependencies";"""
if prop_build_inputs_str.strip():
out += f"""
propagatedBuildInputs = with python-self; [ {prop_build_inputs_str} ];"""
out += """
};\n"""
return unindent(out, 8)
def _gen_overrides(self, pkgs: Dict[str, ResolvedPkg], overrides_keys):
pkg_names_str = "".join(
(f"ps.\"{name}\"\n{' ' * 14}"
@ -247,7 +270,7 @@ class OverridesGenerator(ExpressionGenerator):
build_inputs_str,
prop_build_inputs_str)
else:
out += self._gen_builPythonPackage(
out += self._gen_buildPythonPackage(
pkg.name,
pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver),
pkg.removed_circular_deps,
@ -263,6 +286,15 @@ class OverridesGenerator(ExpressionGenerator):
self._get_ref_name(pkg.name, pkg.ver),
prop_build_inputs_str,
pkg.provider_info.wheel_fname)
# CONDA
elif isinstance(pkg.provider_info.provider, CondaDependencyProvider):
out += self._gen_conda_buildPythonPackage(
pkg.name,
pkg.provider_info.provider.deviated_version(pkg.name, pkg.ver),
pkg.removed_circular_deps,
self._get_ref_name(pkg.name, pkg.ver),
prop_build_inputs_str,
*pkg.provider_info.provider.pkg_url_src(pkg.name, pkg.ver))
# NIXPKGS
elif isinstance(pkg.provider_info.provider, NixpkgsDependencyProvider):
nix_name = self.nixpkgs.find_best_nixpkgs_candidate(pkg.name, pkg.ver)
@ -275,6 +307,8 @@ class OverridesGenerator(ExpressionGenerator):
build_inputs_str,
prop_build_inputs_str,
keep_src=True)
else:
raise Exception("unknown provider")
end_overlay_section = f"""
}}; in self);
"""

38
mach_nix/nix/mk-python-derivation-overlay.nix Normal file
View file

@ -0,0 +1,38 @@
pkgs: python: with pkgs.lib;
let
namePrefix = python.libPrefix + "-";
makeOverridablePythonPackage = f: origArgs:
let
ff = f origArgs;
overrideWith = newArgs: origArgs // (if isFunction newArgs then newArgs origArgs else newArgs);
in
if builtins.isAttrs ff then (ff // {
overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
})
else if builtins.isFunction ff then {
overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
__functor = self: ff;
}
else ff;
toPythonModule = drv:
drv.overrideAttrs( oldAttrs: {
# Use passthru in order to prevent rebuilds when possible.
passthru = (oldAttrs.passthru or {})// {
pythonModule = python;
pythonPath = [ ]; # Deprecated, for compatibility.
requiredPythonModules = python.pkgs.requiredPythonModules drv.propagatedBuildInputs;
};
});
callPackage = pkgs.newScope python.pkgs;
in
pySelf: pySuper: {
buildPythonPackage = makeOverridablePythonPackage ( makeOverridable (callPackage ./mk-python-derivation.nix {
inherit namePrefix; # We want Python libraries to be named like e.g. "python3.6-${name}"
inherit toPythonModule; # Libraries provide modules
}));
buildPythonApplication = makeOverridablePythonPackage ( makeOverridable (callPackage ./mk-python-derivation.nix {
namePrefix = ""; # Python applications should not have any prefix
toPythonModule = x: x; # Application does not provide modules.
}));
}

245
mach_nix/nix/mk-python-derivation.nix Normal file
View file

@ -0,0 +1,245 @@
# Generic builder.
{ lib
, config
, python
, wrapPython
, unzip
, ensureNewerSourcesForZipFilesHook
# Whether the derivation provides a Python module or not.
, toPythonModule
, namePrefix
, update-python-libraries
, setuptools
, flitBuildHook
, pipBuildHook
, pipInstallHook
, pythonCatchConflictsHook
, pythonImportsCheckHook
, pythonNamespacesHook
, pythonRecompileBytecodeHook
, pythonRemoveBinBytecodeHook
, pythonRemoveTestsDirHook
, setuptoolsBuildHook
, setuptoolsCheckHook
, wheelUnpackHook
, eggUnpackHook
, eggBuildHook
, eggInstallHook
, autoPatchelfHook
, alsaLib
, cups
, gnutar
, libGL
, libselinux
, xorg
# conda pkgs
, glibc
# conda python
, binutils
, libffi
, libgcc
, ncurses
, openssl
, readline
, sqlite
, tk
, xz
, zlib
, tzdata
}:
{ name ? "${attrs.pname}-${attrs.version}"
# Build-time dependencies for the package
, nativeBuildInputs ? []
# Run-time dependencies for the package
, buildInputs ? []
# Dependencies needed for running the checkPhase.
# These are added to buildInputs when doCheck = true.
, checkInputs ? []
# propagate build dependencies so in case we have A -> B -> C,
# C can import package A propagated by B
, propagatedBuildInputs ? []
# DEPRECATED: use propagatedBuildInputs
, pythonPath ? []
# Enabled to detect some (native)BuildInputs mistakes
, strictDeps ? true
# used to disable derivation, useful for specific python versions
, disabled ? false
# Raise an error if two packages are installed with the same name
, catchConflicts ? true
# Additional arguments to pass to the makeWrapper function, which wraps
# generated binaries.
, makeWrapperArgs ? []
# Skip wrapping of python programs altogether
, dontWrapPythonPrograms ? false
# Don't use Pip to install a wheel
# Note this is actually a variable for the pipInstallPhase in pip's setupHook.
# It's included here to prevent an infinite recursion.
, dontUsePipInstall ? false
# Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs
, permitUserSite ? false
# Remove bytecode from bin folder.
# When a Python script has the extension `.py`, bytecode is generated
# Typically, executables in bin have no extension, so no bytecode is generated.
# However, some packages do provide executables with extensions, and thus bytecode is generated.
, removeBinBytecode ? true
# Several package formats are supported.
# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
# "wheel" : Install from a pre-compiled wheel.
# "flit" : Install a flit package. This builds a wheel.
# "pyproject": Install a package using a ``pyproject.toml`` file (PEP517). This builds a wheel.
# "egg": Install a package from an egg.
# "other" : Provide your own buildPhase and installPhase.
, format ? "setuptools"
, meta ? {}
, passthru ? {}
, doCheck ? config.doCheckByDefault or false
, ... } @ attrs:
# Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
if disabled
then throw "${name} not supported for interpreter ${python.executable}"
else
let supportedFormats = [ "condabin" "egg" "flit" "pyproject" "setuptools" "wheel" ]; in
if ! lib.elem format supportedFormats then
throw ''wrong format "${format}" for buildPythonPackage. Must be one of: [${toString supportedFormats}]''
else
let
inherit (python) stdenv;
self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [
"disabled" "checkPhase" "checkInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "format"
]) // {
name = namePrefix + name;
nativeBuildInputs = [
python
wrapPython
ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)?
pythonRecompileBytecodeHook # Remove when solved https://github.com/NixOS/nixpkgs/issues/81441
pythonRemoveTestsDirHook
] ++ lib.optionals catchConflicts [
setuptools pythonCatchConflictsHook
] ++ lib.optionals removeBinBytecode [
pythonRemoveBinBytecodeHook
] ++ lib.optionals (lib.hasSuffix "zip" (attrs.src.name or "")) [
unzip
] ++ lib.optionals (format == "setuptools") [
setuptoolsBuildHook
] ++ lib.optionals (format == "flit") [
flitBuildHook
] ++ lib.optionals (format == "pyproject") [
pipBuildHook
] ++ lib.optionals (format == "wheel") [
wheelUnpackHook
] ++ lib.optionals (format == "egg") [
eggUnpackHook eggBuildHook eggInstallHook
] ++ lib.optionals (format == "condabin") (
[ autoPatchelfHook alsaLib cups libGL ]
++ (with xorg; [ libSM libICE libX11 libXau libXdamage libXi libXrender libXrandr libXcomposite libXcursor libXtst libXScrnSaver])
# dependencies of conds python interpreter dstribution
++ [ binutils libffi libgcc ncurses openssl readline sqlite tk xz zlib ]
) ++ lib.optionals (!(builtins.elem format [ "condabin" "other"]) || dontUsePipInstall) [
pipInstallHook
] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
# This is a test, however, it should be ran independent of the checkPhase and checkInputs
pythonImportsCheckHook
] ++ lib.optionals (python.pythonAtLeast "3.3") [
# Optionally enforce PEP420 for python3
pythonNamespacesHook
] ++ nativeBuildInputs;
buildInputs = buildInputs ++ pythonPath;
propagatedBuildInputs = propagatedBuildInputs ++ [ python ];
inherit strictDeps;
LANG = "${if python.stdenv.isDarwin then "en_US" else "C"}.UTF-8";
# Python packages don't have a checkPhase, only an installCheckPhase
doCheck = false;
doInstallCheck = attrs.doCheck or true;
installCheckInputs = [
] ++ lib.optionals (format == "setuptools") [
# Longer-term we should get rid of this and require
# users of this function to set the `installCheckPhase` or
# pass in a hook that sets it.
setuptoolsCheckHook
] ++ checkInputs;
postFixup = lib.optionalString (!dontWrapPythonPrograms) ''
wrapPythonPrograms
'' + attrs.postFixup or '''';
# Python packages built through cross-compilation are always for the host platform.
disallowedReferences = lib.optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ python.pythonForBuild ];
# For now, revert recompilation of bytecode.
dontUsePythonRecompileBytecode = true;
meta = {
# default to python's platforms
platforms = python.meta.platforms;
isBuildPythonPackage = python.meta.platforms;
} // meta;
} // lib.optionalAttrs (attrs?checkPhase) {
# If given use the specified checkPhase, otherwise use the setup hook.
# Longer-term we should get rid of `checkPhase` and use `installCheckPhase`.
installCheckPhase = attrs.checkPhase;
} // lib.optionalAttrs (format == "condabin") {
# If given use the specified checkPhase, otherwise use the setup hook.
# Longer-term we should get rid of `checkPhase` and use `installCheckPhase`.
unpackPhase = ''
${gnutar}/bin/tar --exclude='info' -xf $src
'';
buildPhase = ''
pyDir=$(echo ${lib.removeSuffix "-" namePrefix})
if [ -e ./site-packages ]; then
mkdir -p $out/lib/$pyDir/site-packages/
cp -r ./site-packages/* $out/lib/$pyDir/site-packages/
else
cp -r . $out
fi
'';
installPhase = ''
if [ -e $out/bin ]; then
find $out/bin -type f -exec sed -i "s|/opt/anaconda1anaconda2anaconda3|$out|g" {} \;
fi
rm -rf $out/ssl
'';
}
));
passthru.updateScript = let
filename = builtins.head (lib.splitString ":" self.meta.position);
in attrs.passthru.updateScript or [ update-python-libraries filename ];
in lib.extendDerivation true passthru self

2
mach_nix/nix/mkPython.nix
View file

@ -1,4 +1,5 @@
{ pkgs, pypiDataRev, pypiDataSha256, ... }:
with builtins;
with pkgs.lib;
let
@ -105,6 +106,7 @@ let
all_overrides = l.mergeOverrides (
overridesPre ++ overrides_pre_extra
++ extra_pkgs_py_overrides
++ [ (import ./mk-python-derivation-overlay.nix pkgs python_pkg) ]
++ [ result.overrides ]
++ (l.fixes_to_overrides _fixes)
++ overrides_post_extra ++ overridesPost

17
mach_nix/nix/python-deps.nix
View file

@ -1,5 +1,22 @@
{python, fetchurl, ...}:
rec {
conda = python.pkgs.buildPythonPackage rec {
pname = "conda";
version = "4.9.0";
src = builtins.fetchurl {
url = "https://github.com/conda/conda/archive/4.9.0.tar.gz";
sha256 = "1flvvkc2i6yd6h9n22i6ic9nkwvyqs381qgwfkvz0pvxg4fw7a8q";
};
doCheck = false;
propagatedBuildInputs = with python.pkgs; [ pycosat requests ruamel_yaml ];
patchPhase = ''
echo '
def get_version(dunder_file):
return "${version}"
' >> conda/_vendor/auxlib/packaging.py
cat conda/_vendor/auxlib/packaging.py
'';
};
distlib = python.pkgs.buildPythonPackage {
name = "distlib-0.3.0";
src = fetchurl {

3
mach_nix/provider_defaults.toml
View file

@ -1,4 +1,5 @@
_default = "wheel,sdist,nixpkgs"
#_default = "wheel,sdist,nixpkgs"
_default = "conda"
gdal = "nixpkgs"
pip = "nixpkgs,sdist"

9
mach_nix/versions.py
View file

@ -3,9 +3,10 @@ import sys
import traceback
from typing import Iterable, Tuple, List
from packaging.version import Version, parse, LegacyVersion
from packaging.version import Version, LegacyVersion
from mach_nix.cache import cached
import packaging.version
class PyVer(Version):
@ -31,6 +32,10 @@ class PyVer(Version):
exit(1)
def parse_ver(ver_str):
return packaging.version.parse(ver_str)
def ver_better_than_other(v: Version, o: Version) -> bool:
# print(inspect.getfile(v.__class__))
instability = {v: 0, o: 0}
@ -81,7 +86,7 @@ def filter_versions(
"""
versions = list(versions)
for op, ver in specs:
ver = parse(ver)
ver = parse_ver(ver)
if op == '==':
versions_str = (str(ver) for ver in versions)
versions_str_filtered = list(ver_str for ver_str in fnmatch.filter(versions_str, str(ver)))