2020-10-08 14:29:14 +00:00
|
|
|
let nixpkgsSrc = import ./mach_nix/nix/nixpkgs-src.nix; in
|
2020-10-03 19:11:50 +00:00
|
|
|
{
|
2020-10-08 14:29:14 +00:00
|
|
|
pkgs ? import nixpkgsSrc { config = {}; overlays = []; },
|
2020-10-03 19:11:50 +00:00
|
|
|
...
|
|
|
|
}:
|
2020-09-13 06:53:23 +00:00
|
|
|
with builtins;
|
2020-04-22 09:28:58 +00:00
|
|
|
let
|
|
|
|
python = import ./mach_nix/nix/python.nix { inherit pkgs; };
|
2020-05-19 09:46:48 +00:00
|
|
|
python_deps = (builtins.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = pkgs.fetchurl; }));
|
2020-10-11 07:41:09 +00:00
|
|
|
mergeOverrides = with pkgs.lib; foldl composeExtensions (self: super: { });
|
2020-05-19 09:46:48 +00:00
|
|
|
autoPatchelfHook = import ./mach_nix/nix/auto_patchelf_hook.nix {inherit (pkgs) fetchurl makeSetupHook writeText;};
|
2020-10-07 16:36:08 +00:00
|
|
|
pypiFetcher = (import ./mach_nix/nix/deps-db-and-fetcher.nix { inherit pkgs; }).pypi_fetcher;
|
2020-10-08 16:58:08 +00:00
|
|
|
withDot = mkPython: import ./mach_nix/nix/withDot.nix { inherit mkPython pypiFetcher; };
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-08-25 08:21:49 +00:00
|
|
|
concat_reqs = reqs_list:
|
|
|
|
let
|
|
|
|
concat = s1: s2: s1 + "\n" + s2;
|
|
|
|
in
|
|
|
|
builtins.foldl' concat "" reqs_list;
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-09-20 07:35:50 +00:00
|
|
|
extract = python: src: fail_msg:
|
|
|
|
let
|
|
|
|
file_path = "${(import ./lib/extractor).extract_from_src {
|
|
|
|
py = python;
|
|
|
|
src = src;
|
|
|
|
}}/python.json";
|
|
|
|
in
|
|
|
|
if pathExists file_path then fromJSON (readFile file_path) else throw fail_msg;
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-09-20 09:32:50 +00:00
|
|
|
extract_requirements = python: src: name: extras:
|
2020-09-20 07:35:50 +00:00
|
|
|
with pkgs.lib;
|
2020-08-25 03:11:08 +00:00
|
|
|
let
|
2020-09-20 07:35:50 +00:00
|
|
|
data = extract python src ''
|
|
|
|
Automatic requirements extraction failed for ${name}.
|
|
|
|
Please manually specify 'requirements' '';
|
2020-08-25 03:11:08 +00:00
|
|
|
setup_requires = if hasAttr "setup_requires" data then data.setup_requires else [];
|
|
|
|
install_requires = if hasAttr "install_requires" data then data.install_requires else [];
|
|
|
|
extras_require =
|
2020-09-15 17:37:16 +00:00
|
|
|
if hasAttr "extras_require" data then
|
2020-08-25 03:11:08 +00:00
|
|
|
pkgs.lib.flatten (map (extra: data.extras_require."${extra}") extras)
|
|
|
|
else [];
|
2020-08-25 08:21:49 +00:00
|
|
|
all_reqs = concat_reqs (setup_requires ++ install_requires ++ extras_require);
|
2020-09-20 07:35:50 +00:00
|
|
|
msg = "\n automatically detected requirements of ${name} ${version}:${all_reqs}\n\n";
|
2020-08-25 03:11:08 +00:00
|
|
|
in
|
2020-09-20 07:35:50 +00:00
|
|
|
trace msg all_reqs;
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-10-03 09:53:21 +00:00
|
|
|
extract_meta = python: src: attr: for_attr:
|
2020-09-20 07:35:50 +00:00
|
|
|
with pkgs.lib;
|
|
|
|
let
|
|
|
|
error_msg = ''
|
2020-10-03 09:53:21 +00:00
|
|
|
Automatic extraction of '${for_attr}' from python package source ${src} failed.
|
|
|
|
Please manually specify '${for_attr}' '';
|
2020-09-20 07:35:50 +00:00
|
|
|
data = extract python src error_msg;
|
|
|
|
result = if hasAttr attr data then data."${attr}" else throw error_msg;
|
2020-10-03 09:53:21 +00:00
|
|
|
msg = "\n automatically detected ${for_attr}: '${result}'";
|
2020-09-20 07:35:50 +00:00
|
|
|
in
|
|
|
|
trace msg result;
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-10-11 17:17:54 +00:00
|
|
|
is_src = input: ! input ? passthru;
|
|
|
|
|
2020-08-25 03:11:08 +00:00
|
|
|
is_http_url = url:
|
|
|
|
with builtins;
|
|
|
|
if (substring 0 8 url) == "https://" || (substring 0 7 url) == "http://" then true else false;
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-08-25 03:11:08 +00:00
|
|
|
get_src = src:
|
|
|
|
with builtins;
|
|
|
|
if isString src && is_http_url src then (fetchTarball src) else src;
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-10-03 14:59:47 +00:00
|
|
|
get_py_ver = python: with pkgs.lib; {
|
|
|
|
major = elemAt (splitString "." python.version) 0;
|
|
|
|
minor = elemAt (splitString "." python.version) 1;
|
|
|
|
};
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-09-20 09:32:50 +00:00
|
|
|
combine = pname: key: val1: val2:
|
2020-09-13 06:53:23 +00:00
|
|
|
if isList val2 then val1 ++ val2
|
|
|
|
else if isAttrs val2 then val1 // val2
|
|
|
|
else if isString val2 then val1 + val2
|
2020-09-20 09:32:50 +00:00
|
|
|
else throw "_.${pname}.${key}.add only accepts list or attrs or string.";
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-09-13 06:53:23 +00:00
|
|
|
meets_cond = oa: condition:
|
|
|
|
let
|
2020-09-20 08:30:20 +00:00
|
|
|
provider = if hasAttr "provider" oa.passthru then oa.passthru.provider else "nixpkgs";
|
2020-09-13 06:53:23 +00:00
|
|
|
in
|
2020-09-20 08:30:20 +00:00
|
|
|
condition { prov = provider; ver = oa.version; pyver = oa.pythonModule.version; };
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-09-11 17:54:26 +00:00
|
|
|
simple_overrides = args: with pkgs.lib;
|
2020-09-20 09:32:50 +00:00
|
|
|
flatten ( mapAttrsToList (pkg: keys: pySelf: pySuper: {
|
|
|
|
"${pkg}" = pySuper."${pkg}".overrideAttrs (oa:
|
|
|
|
mapAttrs (key: val:
|
2020-09-11 17:54:26 +00:00
|
|
|
if isAttrs val && hasAttr "add" val then
|
2020-09-20 09:32:50 +00:00
|
|
|
combine pkg key oa."${key}" val.add
|
2020-09-11 17:54:26 +00:00
|
|
|
else if isAttrs val && hasAttr "mod" val && isFunction val.mod then
|
2020-09-22 10:48:35 +00:00
|
|
|
let result = val.mod oa."${key}"; in
|
|
|
|
# if the mod function wants more argument, call with more arguments (alternative style)
|
|
|
|
if ! isFunction result then
|
|
|
|
result
|
|
|
|
else
|
|
|
|
val.mod pySelf oa oa."${key}"
|
2020-09-20 09:32:50 +00:00
|
|
|
else
|
|
|
|
val
|
2020-09-11 17:54:26 +00:00
|
|
|
) keys
|
2020-09-20 09:32:50 +00:00
|
|
|
);
|
|
|
|
}) args);
|
2020-10-11 10:47:12 +00:00
|
|
|
|
2020-09-13 06:53:23 +00:00
|
|
|
fixes_to_overrides = fixes: with pkgs.lib;
|
|
|
|
flatten (flatten (
|
|
|
|
mapAttrsToList (pkg: p_fixes:
|
|
|
|
mapAttrsToList (fix: keys: pySelf: pySuper:
|
|
|
|
let cond = if hasAttr "_cond" keys then keys._cond else ({prov, ver, pyver}: true); in
|
|
|
|
if ! hasAttr "${pkg}" pySuper then {} else
|
|
|
|
{
|
|
|
|
"${pkg}" = pySuper."${pkg}".overrideAttrs (oa:
|
|
|
|
mapAttrs (key: val:
|
2020-09-22 10:48:35 +00:00
|
|
|
trace "\napplying fix '${fix}' (${key}) for ${pkg}:${oa.version}\n" (
|
2020-09-13 06:53:23 +00:00
|
|
|
if isAttrs val && hasAttr "add" val then
|
2020-09-20 09:32:50 +00:00
|
|
|
combine pkg key oa."${key}" val.add
|
2020-09-13 06:53:23 +00:00
|
|
|
else if isAttrs val && hasAttr "mod" val && isFunction val.mod then
|
2020-09-22 10:48:35 +00:00
|
|
|
let result = val.mod oa."${key}"; in
|
|
|
|
# if the mod function wants more argument, call with more arguments (alternative style)
|
|
|
|
if ! isFunction result then
|
|
|
|
result
|
|
|
|
else
|
|
|
|
val.mod pySelf oa oa."${key}"
|
2020-09-13 06:53:23 +00:00
|
|
|
else
|
|
|
|
val
|
|
|
|
)
|
2020-09-20 08:30:20 +00:00
|
|
|
) (filterAttrs (k: v: k != "_cond" && meets_cond oa cond) keys)
|
2020-09-13 06:53:23 +00:00
|
|
|
);
|
|
|
|
}
|
|
|
|
) p_fixes
|
|
|
|
) fixes
|
|
|
|
));
|
2020-10-03 09:55:28 +00:00
|
|
|
|
2020-10-11 10:47:12 +00:00
|
|
|
# call this to generate a nix expression which contains the mach-nix overrides
|
|
|
|
compileExpression = args: import ./mach_nix/nix/mach.nix args;
|
2020-05-19 09:46:48 +00:00
|
|
|
|
|
|
|
# Returns `overrides` and `select_pkgs` which satisfy your requirements
|
2020-10-11 10:47:12 +00:00
|
|
|
compileOverrides = args:
|
2020-08-09 13:24:12 +00:00
|
|
|
let
|
2020-10-11 10:47:12 +00:00
|
|
|
result = import "${compileExpression args}/share/mach_nix_file.nix" { pkgs = args.pkgs; };
|
2020-08-09 13:24:12 +00:00
|
|
|
manylinux =
|
2020-09-11 17:54:26 +00:00
|
|
|
if args.pkgs.stdenv.hostPlatform.system == "x86_64-darwin" then
|
2020-08-09 13:24:12 +00:00
|
|
|
[]
|
|
|
|
else
|
2020-09-11 17:54:26 +00:00
|
|
|
args.pkgs.pythonManylinuxPackages.manylinux1;
|
2020-07-03 11:03:10 +00:00
|
|
|
in {
|
2020-08-09 13:24:12 +00:00
|
|
|
overrides = result.overrides manylinux autoPatchelfHook;
|
2020-07-03 11:03:10 +00:00
|
|
|
select_pkgs = result.select_pkgs;
|
|
|
|
};
|
2020-10-11 10:47:40 +00:00
|
|
|
|
|
|
|
__buildPython = with builtins; func: args:
|
|
|
|
if args ? pkgs then
|
|
|
|
throw "${func} does not accept 'pkgs' anymore. 'pkgs' need to be specified when importing mach-nix"
|
|
|
|
else if args ? extra_pkgs then
|
|
|
|
throw "'extra_pkgs' cannot be passed to ${func}. Please pass it to a mkPython call."
|
|
|
|
else if isString args || isPath args || pkgs.lib.isDerivation args then
|
|
|
|
_buildPython func { src = args; }
|
|
|
|
else
|
|
|
|
_buildPython func args;
|
2020-07-03 11:03:10 +00:00
|
|
|
|
|
|
|
_buildPython = func: args@{
|
2020-08-25 03:38:30 +00:00
|
|
|
add_requirements ? "", # add additional requirements to the packge
|
2020-08-24 11:20:29 +00:00
|
|
|
requirements ? null, # content from a requirements.txt file
|
2020-07-03 11:03:10 +00:00
|
|
|
disable_checks ? true, # Disable tests wherever possible to decrease build time.
|
2020-08-24 11:20:29 +00:00
|
|
|
extras ? [],
|
|
|
|
doCheck ? ! disable_checks,
|
2020-07-03 11:03:10 +00:00
|
|
|
overrides_pre ? [], # list of pythonOverrides to apply before the machnix overrides
|
|
|
|
overrides_post ? [], # list of pythonOverrides to apply after the machnix overrides
|
2020-09-16 04:13:09 +00:00
|
|
|
passthru ? {},
|
2020-07-03 11:03:10 +00:00
|
|
|
providers ? {}, # define provider preferences
|
|
|
|
pypi_deps_db_commit ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_COMMIT, # python dependency DB version
|
|
|
|
pypi_deps_db_sha256 ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_SHA256,
|
2020-10-08 14:29:14 +00:00
|
|
|
python ? "python3", # select custom python to base overrides onto. Should be from nixpkgs >= 20.03
|
2020-07-03 11:03:10 +00:00
|
|
|
_provider_defaults ? with builtins; fromTOML (readFile ./mach_nix/provider_defaults.toml),
|
2020-09-11 17:54:26 +00:00
|
|
|
_ ? {}, # simplified overrides
|
2020-09-21 05:14:48 +00:00
|
|
|
_fixes ? import ./mach_nix/fixes.nix {pkgs = pkgs;},
|
2020-07-03 11:03:10 +00:00
|
|
|
...
|
|
|
|
}:
|
2020-08-25 03:11:08 +00:00
|
|
|
with builtins;
|
2020-07-03 11:03:10 +00:00
|
|
|
let
|
2020-10-08 16:42:48 +00:00
|
|
|
python_arg = if isString python then python else throw '''python' must be a string. Example: "python38"'';
|
2020-09-16 04:14:28 +00:00
|
|
|
in
|
|
|
|
let
|
2020-10-09 15:37:06 +00:00
|
|
|
python_pkg = pkgs."${python_arg}";
|
2020-08-25 03:11:08 +00:00
|
|
|
src = get_src pass_args.src;
|
|
|
|
# Extract dependencies automatically if 'requirements' is unset
|
2020-09-20 07:35:50 +00:00
|
|
|
pname =
|
2020-09-28 04:09:00 +00:00
|
|
|
if hasAttr "pname" args then args.pname
|
2020-10-09 15:37:06 +00:00
|
|
|
else extract_meta python_pkg src "name" "pname";
|
2020-09-20 07:35:50 +00:00
|
|
|
version =
|
2020-09-28 04:09:00 +00:00
|
|
|
if hasAttr "version" args then args.version
|
2020-10-09 15:37:06 +00:00
|
|
|
else extract_meta python_pkg src "version" "version";
|
|
|
|
meta_reqs = extract_requirements python_pkg src "${pname}:${version}" extras;
|
2020-08-24 11:20:29 +00:00
|
|
|
reqs =
|
2020-08-25 08:21:49 +00:00
|
|
|
(if requirements == null then
|
2020-08-24 11:20:29 +00:00
|
|
|
if builtins.hasAttr "format" args && args.format != "setuptools" then
|
|
|
|
throw "Automatic dependency extraction is only available for 'setuptools' format."
|
2020-08-25 03:11:08 +00:00
|
|
|
" Please specify 'requirements' if setuptools is not used."
|
2020-08-24 11:20:29 +00:00
|
|
|
else
|
2020-09-20 07:35:50 +00:00
|
|
|
meta_reqs
|
2020-08-24 11:20:29 +00:00
|
|
|
else
|
2020-09-11 17:54:26 +00:00
|
|
|
requirements)
|
|
|
|
+ "\n" + add_requirements;
|
2020-10-09 15:37:06 +00:00
|
|
|
py = python_pkg.override { packageOverrides = mergeOverrides overrides_pre; };
|
2020-10-11 10:47:12 +00:00
|
|
|
result = compileOverrides {
|
2020-09-11 17:54:26 +00:00
|
|
|
inherit disable_checks pkgs providers pypi_deps_db_commit pypi_deps_db_sha256 _provider_defaults;
|
2020-08-09 13:24:12 +00:00
|
|
|
overrides = overrides_pre;
|
2020-07-03 11:03:10 +00:00
|
|
|
python = py;
|
2020-08-25 08:21:49 +00:00
|
|
|
requirements = reqs;
|
2020-07-03 11:03:10 +00:00
|
|
|
};
|
2020-10-09 15:37:06 +00:00
|
|
|
py_final = python_pkg.override { packageOverrides = mergeOverrides (
|
2020-09-21 05:14:48 +00:00
|
|
|
overrides_pre ++ [ result.overrides ] ++ (fixes_to_overrides _fixes) ++ overrides_post ++ (simple_overrides _)
|
2020-08-17 11:16:56 +00:00
|
|
|
);};
|
|
|
|
pass_args = removeAttrs args (builtins.attrNames ({
|
2020-08-25 03:38:30 +00:00
|
|
|
inherit add_requirements disable_checks overrides_pre overrides_post pkgs providers
|
2020-09-11 17:54:26 +00:00
|
|
|
requirements pypi_deps_db_commit pypi_deps_db_sha256 python _provider_defaults _ ;
|
2020-08-17 11:16:56 +00:00
|
|
|
}));
|
2020-07-03 11:03:10 +00:00
|
|
|
in
|
|
|
|
py_final.pkgs."${func}" ( pass_args // {
|
2020-09-20 09:59:17 +00:00
|
|
|
propagatedBuildInputs =
|
|
|
|
(result.select_pkgs py_final.pkgs)
|
|
|
|
++ (if hasAttr "propagatedBuildInputs" args then args.propagatedBuildInputs else []);
|
2020-08-25 03:11:08 +00:00
|
|
|
src = src;
|
|
|
|
inherit doCheck pname version;
|
2020-09-16 04:13:09 +00:00
|
|
|
passthru = passthru // {
|
2020-08-25 08:21:49 +00:00
|
|
|
requirements = reqs;
|
2020-09-11 17:54:26 +00:00
|
|
|
inherit overrides_pre overrides_post _;
|
2020-08-25 08:21:49 +00:00
|
|
|
};
|
2020-07-03 11:03:10 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
|
2020-05-19 09:46:48 +00:00
|
|
|
# (High level API) generates a python environment with minimal user effort
|
2020-10-11 10:47:12 +00:00
|
|
|
mkPythonBase = caller: args:
|
2020-10-11 04:22:43 +00:00
|
|
|
if args ? pkgs then
|
2020-10-11 10:47:12 +00:00
|
|
|
throw "${caller} does not accept 'pkgs' anymore. 'pkgs' need to be specified when importing mach-nix"
|
2020-10-11 04:22:43 +00:00
|
|
|
else if builtins.isList args then
|
2020-10-11 10:47:12 +00:00
|
|
|
_mkPythonBase { extra_pkgs = args; }
|
2020-10-11 04:22:43 +00:00
|
|
|
else
|
2020-10-11 10:47:12 +00:00
|
|
|
_mkPythonBase args;
|
2020-08-25 08:21:49 +00:00
|
|
|
|
2020-10-11 10:47:12 +00:00
|
|
|
_mkPythonBase =
|
2020-05-19 09:46:48 +00:00
|
|
|
{
|
2020-08-25 08:21:49 +00:00
|
|
|
requirements ? "", # content from a requirements.txt file
|
2020-05-19 09:46:48 +00:00
|
|
|
disable_checks ? true, # Disable tests wherever possible to decrease build time.
|
2020-08-25 08:21:49 +00:00
|
|
|
extra_pkgs ? [],
|
2020-05-19 09:46:48 +00:00
|
|
|
overrides_pre ? [], # list of pythonOverrides to apply before the machnix overrides
|
|
|
|
overrides_post ? [], # list of pythonOverrides to apply after the machnix overrides
|
|
|
|
providers ? {}, # define provider preferences
|
|
|
|
pypi_deps_db_commit ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_COMMIT, # python dependency DB version
|
|
|
|
pypi_deps_db_sha256 ? builtins.readFile ./mach_nix/nix/PYPI_DEPS_DB_SHA256,
|
2020-10-08 14:29:14 +00:00
|
|
|
python ? "python3", # select custom python to base overrides onto. Should be from nixpkgs >= 20.03
|
2020-09-13 06:53:23 +00:00
|
|
|
_ ? {}, # simplified overrides
|
2020-08-26 08:24:50 +00:00
|
|
|
_provider_defaults ? with builtins; fromTOML (readFile ./mach_nix/provider_defaults.toml),
|
2020-09-13 06:53:23 +00:00
|
|
|
_fixes ? import ./mach_nix/fixes.nix {pkgs = pkgs;}
|
2020-05-19 09:46:48 +00:00
|
|
|
}:
|
2020-08-25 08:21:49 +00:00
|
|
|
with builtins;
|
2020-09-11 17:54:26 +00:00
|
|
|
with pkgs.lib;
|
2020-05-19 09:46:48 +00:00
|
|
|
let
|
2020-10-08 16:42:48 +00:00
|
|
|
python_arg = if isString python then python else throw '''python' must be a string. Example: "python38"'';
|
2020-09-16 04:14:28 +00:00
|
|
|
in
|
|
|
|
let
|
2020-10-09 15:37:06 +00:00
|
|
|
python_pkg = pkgs."${python_arg}";
|
|
|
|
pyver = get_py_ver python_pkg;
|
2020-10-11 17:17:54 +00:00
|
|
|
# and separate pkgs into groups
|
|
|
|
extra_pkgs_python = map (p:
|
2020-10-03 14:59:47 +00:00
|
|
|
# check if element is a package built via mach-nix
|
2020-10-11 17:17:54 +00:00
|
|
|
if p ? pythomModule && ! p ? passthru._ then
|
|
|
|
throw ''
|
|
|
|
python packages from nixpkgs cannot be passed via `extra_pkgs`.
|
|
|
|
Instead, add the package's name to your `requirements` and set `providers.{package} = "nixpkgs"`
|
|
|
|
''
|
|
|
|
else if p ? passthru._ then
|
2020-10-03 14:59:47 +00:00
|
|
|
let
|
|
|
|
pkg_pyver = get_py_ver p.pythonModule;
|
|
|
|
in
|
|
|
|
if pkg_pyver != pyver then
|
|
|
|
throw ''
|
|
|
|
${p.pname} from 'extra_pkgs' is built with python ${p.pythonModule.version},
|
|
|
|
but the environment is based on python ${pyver.major}.${pyver.minor}.
|
|
|
|
Please build ${p.pname} with 'python = "python${pyver.major}${pyver.minor}"'.
|
|
|
|
''
|
|
|
|
else
|
|
|
|
p
|
2020-10-11 17:17:54 +00:00
|
|
|
# translate sources to python packages
|
2020-10-03 09:55:28 +00:00
|
|
|
else
|
2020-09-11 17:54:26 +00:00
|
|
|
_buildPython "buildPythonPackage" {
|
|
|
|
src = p;
|
|
|
|
inherit disable_checks pkgs providers pypi_deps_db_commit pypi_deps_db_sha256 python _provider_defaults;
|
|
|
|
}
|
2020-10-11 17:17:54 +00:00
|
|
|
) (filter (p: is_src p || p ? pythonModule) extra_pkgs);
|
|
|
|
extra_pkgs_r = filter (p: p ? rCommand) extra_pkgs;
|
|
|
|
extra_pkgs_other = filter (p: ! (p ? rCommand || p ? pythonModule || is_src p)) extra_pkgs;
|
|
|
|
|
|
|
|
# gather requirements of exra pkgs
|
|
|
|
extra_pkgs_py_reqs =
|
2020-08-25 08:21:49 +00:00
|
|
|
map (p:
|
2020-09-11 17:54:26 +00:00
|
|
|
if hasAttr "requirements" p then p.requirements
|
2020-08-25 08:21:49 +00:00
|
|
|
else throw "Packages passed via 'extra_pkgs' must be built via mach-nix.buildPythonPackage"
|
2020-10-11 17:17:54 +00:00
|
|
|
) extra_pkgs_python;
|
2020-10-11 17:59:09 +00:00
|
|
|
extra_pkgs_r_reqs = if extra_pkgs_r == [] then "" else ''
|
|
|
|
rpy2
|
|
|
|
ipython
|
|
|
|
jinja2
|
|
|
|
pytz
|
|
|
|
pandas
|
|
|
|
numpy
|
|
|
|
cffi
|
|
|
|
tzlocal
|
|
|
|
simplegeneric
|
|
|
|
'';
|
2020-10-11 17:17:54 +00:00
|
|
|
|
|
|
|
# gather overrides necessary by extra_pkgs
|
|
|
|
extra_pkgs_python_attrs = foldl' (a: b: a // b) {} (map (p: { "${p.pname}" = p; }) extra_pkgs_python);
|
|
|
|
extra_pkgs_py_overrides = [ (pySelf: pySuper: extra_pkgs_python_attrs) ];
|
|
|
|
extra_pkgs_r_overrides = simple_overrides {
|
|
|
|
rpy2.buildInputs.add = extra_pkgs_r;
|
|
|
|
};
|
2020-09-11 17:54:26 +00:00
|
|
|
overrides_simple_extra = flatten (
|
|
|
|
(map simple_overrides (
|
2020-10-11 17:17:54 +00:00
|
|
|
map (p: if hasAttr "_" p then p._ else {}) extra_pkgs_python
|
2020-09-11 17:54:26 +00:00
|
|
|
))
|
|
|
|
);
|
2020-10-11 17:17:54 +00:00
|
|
|
overrides_pre_extra = flatten (map (p: p.passthru.overrides_pre) extra_pkgs_python);
|
|
|
|
overrides_post_extra = flatten (map (p: p.passthru.overrides_post) extra_pkgs_python);
|
|
|
|
|
2020-10-09 15:37:06 +00:00
|
|
|
py = python_pkg.override { packageOverrides = mergeOverrides overrides_pre; };
|
2020-10-11 10:47:12 +00:00
|
|
|
result = compileOverrides {
|
2020-09-11 17:54:26 +00:00
|
|
|
inherit disable_checks pkgs providers pypi_deps_db_commit pypi_deps_db_sha256 _provider_defaults;
|
2020-10-11 17:17:54 +00:00
|
|
|
overrides = overrides_pre ++ overrides_pre_extra ++ extra_pkgs_py_overrides;
|
2020-05-19 09:46:48 +00:00
|
|
|
python = py;
|
2020-10-11 17:17:54 +00:00
|
|
|
requirements = concat_reqs ([requirements] ++ extra_pkgs_py_reqs ++ [extra_pkgs_r_reqs]);
|
2020-05-19 09:46:48 +00:00
|
|
|
};
|
2020-09-28 04:09:25 +00:00
|
|
|
all_overrides = mergeOverrides (
|
2020-09-11 17:54:26 +00:00
|
|
|
overrides_pre ++ overrides_pre_extra
|
2020-10-11 17:17:54 +00:00
|
|
|
++ extra_pkgs_py_overrides
|
2020-09-11 17:54:26 +00:00
|
|
|
++ [ result.overrides ]
|
2020-09-13 06:53:23 +00:00
|
|
|
++ (fixes_to_overrides _fixes)
|
2020-09-11 17:54:26 +00:00
|
|
|
++ overrides_post_extra ++ overrides_post
|
2020-10-11 17:17:54 +00:00
|
|
|
++ extra_pkgs_r_overrides
|
2020-09-11 17:54:26 +00:00
|
|
|
++ overrides_simple_extra ++ (simple_overrides _)
|
2020-09-28 04:09:25 +00:00
|
|
|
);
|
2020-10-09 15:37:06 +00:00
|
|
|
py_final = python_pkg.override { packageOverrides = all_overrides;};
|
2020-09-28 05:23:09 +00:00
|
|
|
select_pkgs = ps:
|
2020-09-20 09:32:50 +00:00
|
|
|
(result.select_pkgs ps)
|
2020-10-11 17:17:54 +00:00
|
|
|
++ (map (name: ps."${name}") (attrNames extra_pkgs_python_attrs));
|
2020-09-28 05:23:09 +00:00
|
|
|
py_final_with_pkgs = py_final.withPackages (ps: select_pkgs ps);
|
2020-10-11 17:17:54 +00:00
|
|
|
final_env = pkgs.buildEnv {
|
|
|
|
name = "mach-nix-python-env";
|
|
|
|
paths = [
|
|
|
|
py_final_with_pkgs
|
|
|
|
extra_pkgs_other
|
|
|
|
];
|
|
|
|
};
|
2020-10-08 16:42:48 +00:00
|
|
|
in let
|
2020-10-11 17:17:54 +00:00
|
|
|
self = final_env.overrideAttrs (oa: {
|
2020-09-29 10:10:46 +00:00
|
|
|
passthru = oa.passthru // rec {
|
|
|
|
selectPkgs = select_pkgs;
|
|
|
|
pythonOverrides = all_overrides;
|
2020-10-11 09:11:19 +00:00
|
|
|
python = py_final;
|
2020-09-29 10:10:46 +00:00
|
|
|
overlay = self: super:
|
|
|
|
let
|
2020-10-03 14:59:47 +00:00
|
|
|
py_attr_name = "python${pyver.major}${pyver.minor}";
|
2020-09-29 10:10:46 +00:00
|
|
|
in
|
|
|
|
{
|
|
|
|
"${py_attr_name}" = super."${py_attr_name}".override {
|
|
|
|
packageOverrides = pythonOverrides;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
nixpkgs = import pkgs.path { config = pkgs.config; overlays = pkgs.overlays ++ [ overlay ]; };
|
2020-10-08 16:42:48 +00:00
|
|
|
dockerImage = makeOverridable
|
|
|
|
(args: pkgs.dockerTools.buildLayeredImage args)
|
|
|
|
{
|
|
|
|
name = "mach-nix-python";
|
|
|
|
tag = "latest";
|
|
|
|
contents = [
|
|
|
|
pkgs.busybox
|
|
|
|
self
|
|
|
|
];
|
|
|
|
config = {
|
|
|
|
Cmd = [ "${self}/bin/python" ];
|
|
|
|
Env = {
|
|
|
|
python = "${self}";
|
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
2020-09-28 04:09:25 +00:00
|
|
|
};
|
2020-10-08 16:42:48 +00:00
|
|
|
});
|
|
|
|
in self;
|
2020-10-11 10:47:12 +00:00
|
|
|
in
|
|
|
|
rec {
|
|
|
|
# the mach-nix cmdline tool derivation
|
|
|
|
mach-nix = python.pkgs.buildPythonPackage rec {
|
|
|
|
pname = "mach-nix";
|
|
|
|
version = builtins.readFile ./mach_nix/VERSION;
|
|
|
|
name = "${pname}-${version}";
|
|
|
|
src = ./.;
|
|
|
|
propagatedBuildInputs = python_deps;
|
|
|
|
doCheck = false;
|
|
|
|
};
|
|
|
|
|
|
|
|
# the main functions
|
|
|
|
mkPython = args: mkPythonBase "mkPython" args;
|
2020-10-11 17:17:54 +00:00
|
|
|
mkPythonShell = args: pkgs.mkShell { buildInputs = (mkPythonBase "mkPythonShell" args) ;};
|
2020-10-11 10:47:12 +00:00
|
|
|
mkDockerImage = args: (mkPythonBase "mkDockerImage" args).dockerImage;
|
|
|
|
mkOverlay = args: (mkPythonBase "mkOverlay" args).overlay;
|
|
|
|
mkNixpkgs = args: (mkPythonBase "mkNixpkgs" args).nixpkgs;
|
|
|
|
mkPythonOverrides = args: (mkPythonBase "mkPythonOverrides" args).pythonOverrides;
|
|
|
|
|
|
|
|
# equivalent to buildPythonPackage of nixpkgs
|
|
|
|
buildPythonPackage = __buildPython "buildPythonPackage";
|
|
|
|
|
|
|
|
# equivalent to buildPythonApplication of nixpkgs
|
|
|
|
buildPythonApplication = __buildPython "buildPythonApplication";
|
|
|
|
|
|
|
|
# provide pypi fetcher to user
|
|
|
|
fetchPypiSdist = pypiFetcher.fetchPypiSdist;
|
|
|
|
fetchPypiWheel = pypiFetcher.fetchPypiWheel;
|
|
|
|
|
|
|
|
# expose dot interface for flakes cmdline
|
|
|
|
"with" = (withDot (mkPythonBase "'.with'"))."with";
|
|
|
|
shellWith = (withDot (mkPythonBase "'.shellWith'")).shellWith;
|
2020-10-11 10:47:40 +00:00
|
|
|
dockerImageWith = (withDot (mkPythonBase "'.dockerImageWith'")).dockerImageWith;
|
2020-10-11 10:47:12 +00:00
|
|
|
|
|
|
|
# expose mach-nix' nixpkgs
|
|
|
|
# those are equivalent to the pkgs passed by the user
|
|
|
|
nixpkgs = pkgs;
|
|
|
|
|
2020-10-11 17:17:54 +00:00
|
|
|
# expose R packages
|
|
|
|
rPackages = pkgs.rPackages;
|
|
|
|
|
2020-10-11 10:47:12 +00:00
|
|
|
# this might beuseful for someone
|
|
|
|
inherit mergeOverrides;
|
2020-04-24 10:35:42 +00:00
|
|
|
}
|